[ { "path": ".devcontainer/Dockerfile", "content": "FROM python:3.12-bookworm\n\n# Install Node.js 20.x\nRUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \\\n && apt-get install -y nodejs \\\n && rm -rf /var/lib/apt/lists/*\n\n# Install global npm packages\nRUN npm install -g husky vite\n\n# Create and activate Python virtual environment\nRUN python -m venv /opt/venv\nENV PATH=\"/opt/venv/bin:$PATH\"\n\nWORKDIR /workspace" }, { "path": ".devcontainer/devc-welcome.md", "content": "# Welcome to DocsGPT Devcontainer\n\nWelcome to the DocsGPT development environment! This guide will help you get started quickly.\n\n## Starting Services\n\nTo run DocsGPT, you need to start three main services: Flask (backend), Celery (task queue), and Vite (frontend). Here are the commands to start each service within the devcontainer:\n\n### Vite (Frontend)\n\n```bash\ncd frontend\nnpm run dev -- --host\n```\n\n### Flask (Backend)\n\n```bash\nflask --app application/app.py run --host=0.0.0.0 --port=7091\n```\n\n### Celery (Task Queue)\n\n```bash\ncelery -A application.app.celery worker -l INFO\n```\n\n## Github Codespaces Instructions\n\n### 1. Make Ports Public:\n\nGo to the \"Ports\" panel in Codespaces (usually located at the bottom of the VS Code window).\n\nFor both port 5173 and 7091, right-click on the port and select \"Make Public\".\n\n![CleanShot 2025-02-12 at 09 46 14@2x](https://github.com/user-attachments/assets/00a34b16-a7ef-47af-9648-87a7e3008475)\n\n\n ### 2. Update VITE_API_HOST:\n\nAfter making port 7091 public, copy the public URL provided by Codespaces for port 7091.\n\nOpen the file frontend/.env.development.\n\nFind the line VITE_API_HOST=http://localhost:7091.\n\nReplace http://localhost:7091 with the public URL you copied from Codespaces.\n\n![CleanShot 2025-02-12 at 09 46 56@2x](https://github.com/user-attachments/assets/c472242f-1079-4cd8-bc0b-2d78db22b94c)\n" }, { "path": ".devcontainer/devcontainer.json", "content": "{\n\t\"name\": \"DocsGPT Dev Container\",\n\t\"dockerComposeFile\": [\"docker-compose-dev.yaml\", \"docker-compose.override.yaml\"],\n\t\"service\": \"dev\",\n\t\"workspaceFolder\": \"/workspace\",\n\t\"postCreateCommand\": \".devcontainer/post-create-command.sh\",\n\t\"forwardPorts\": [7091, 5173, 6379, 27017],\n\t\"customizations\": {\n\t \"vscode\": {\n\t\t\"extensions\": [\n\t\t \"ms-python.python\",\n\t\t \"ms-toolsai.jupyter\",\n\t\t \"esbenp.prettier-vscode\",\n\t\t \"dbaeumer.vscode-eslint\"\n\t\t]\n\t },\n\t \"codespaces\": {\n\t\t\t\"openFiles\": [\n\t\t\t\".devcontainer/devc-welcome.md\",\n\t\t\t\"CONTRIBUTING.md\"\n\t\t\t]\n\t\t}\n\t}\n }" }, { "path": ".devcontainer/docker-compose-dev.yaml", "content": "services:\n\n redis:\n image: redis:6-alpine\n ports:\n - 6379:6379\n\n mongo:\n image: mongo:6\n ports:\n - 27017:27017\n volumes:\n - mongodb_data_container:/data/db\n\n\n\nvolumes:\n mongodb_data_container:" }, { "path": ".devcontainer/docker-compose.override.yaml", "content": "version: '3.8'\n\nservices:\n dev:\n build:\n context: .\n dockerfile: Dockerfile\n volumes:\n - ../:/workspace:cached\n command: sleep infinity\n depends_on:\n redis:\n condition: service_healthy\n mongo:\n condition: service_healthy\n environment:\n - CELERY_BROKER_URL=redis://redis:6379/0\n - CELERY_RESULT_BACKEND=redis://redis:6379/1\n - MONGO_URI=mongodb://mongo:27017/docsgpt\n - CACHE_REDIS_URL=redis://redis:6379/2\n networks:\n - default\n\n redis:\n healthcheck:\n test: [\"CMD\", \"redis-cli\", \"ping\"]\n interval: 5s\n timeout: 30s\n retries: 5\n\n mongo:\n healthcheck:\n test: [\"CMD\", \"mongosh\", \"--eval\", \"db.adminCommand('ping')\"]\n interval: 5s\n timeout: 30s\n retries: 5\n\nnetworks:\n default:\n name: docsgpt-dev-network" }, { "path": ".devcontainer/post-create-command.sh", "content": "#!/bin/bash\n\nset -e # Exit immediately if a command exits with a non-zero status\n\nif [ ! -f frontend/.env.development ]; then\n cp -n .env-template frontend/.env.development || true # Assuming .env-template is in the root\nfi\n\n# Determine VITE_API_HOST based on environment\nif [ -n \"$CODESPACES\" ]; then\n # Running in Codespaces\n CODESPACE_NAME=$(echo \"$CODESPACES\" | cut -d'-' -f1) # Extract codespace name\n PUBLIC_API_HOST=\"https://${CODESPACE_NAME}-7091.${GITHUB_CODESPACES_PORT_FORWARDING_DOMAIN}\"\n echo \"Setting VITE_API_HOST for Codespaces: $PUBLIC_API_HOST in frontend/.env.development\"\n sed -i \"s|VITE_API_HOST=.*|VITE_API_HOST=$PUBLIC_API_HOST|\" frontend/.env.development\nelse\n # Not running in Codespaces (local devcontainer)\n DEFAULT_API_HOST=\"http://localhost:7091\"\n echo \"Setting VITE_API_HOST for local dev: $DEFAULT_API_HOST in frontend/.env.development\"\n sed -i \"s|VITE_API_HOST=.*|VITE_API_HOST=$DEFAULT_API_HOST|\" frontend/.env.development\nfi\n\n\nmkdir -p model\nif [ ! -d model/all-mpnet-base-v2 ]; then\n wget -q https://d3dg1063dc54p9.cloudfront.net/models/embeddings/mpnet-base-v2.zip -O model/mpnet-base-v2.zip\n unzip -q model/mpnet-base-v2.zip -d model\n rm model/mpnet-base-v2.zip\nfi\npip install -r application/requirements.txt\ncd frontend\nnpm install --include=dev" }, { "path": ".env-template", "content": "API_KEY=\nLLM_NAME=docsgpt\nVITE_API_STREAMING=true\nINTERNAL_KEY=\n\n# Remote Embeddings (Optional - for using a remote embeddings API instead of local SentenceTransformer)\n# When set, the app will use the remote API and won't load SentenceTransformer (saves RAM)\nEMBEDDINGS_BASE_URL=\nEMBEDDINGS_KEY=\n\n#For Azure (you can delete it if you don't use Azure)\nOPENAI_API_BASE=\nOPENAI_API_VERSION=\nAZURE_DEPLOYMENT_NAME=\nAZURE_EMBEDDINGS_DEPLOYMENT_NAME=\n\n#Azure AD Application (client) ID\nMICROSOFT_CLIENT_ID=your-azure-ad-client-id\n#Azure AD Application client secret\nMICROSOFT_CLIENT_SECRET=your-azure-ad-client-secret\n#Azure AD Tenant ID (or 'common' for multi-tenant)\nMICROSOFT_TENANT_ID=your-azure-ad-tenant-id\n#If you are using a Microsoft Entra ID tenant,\n#configure the AUTHORITY variable as\n#\"https://login.microsoftonline.com/TENANT_GUID\"\n#or \"https://login.microsoftonline.com/contoso.onmicrosoft.com\".\n#Alternatively, use \"https://login.microsoftonline.com/common\" for multi-tenant app.\nMICROSOFT_AUTHORITY=https://{tenantId}.ciamlogin.com/{tenantId}\n" }, { "path": ".gitattributes", "content": "# Auto detect text files and perform LF normalization\n* text=auto\n" }, { "path": ".github/FUNDING.yml", "content": "# These are supported funding model platforms\n\ngithub: arc53\n" }, { "path": ".github/ISSUE_TEMPLATE/bug_report.yml", "content": "name: \"🐛 Bug Report\"\ndescription: \"Submit a bug report to help us improve\"\ntitle: \"🐛 Bug Report: \"\nlabels: [\"type: bug\"]\nbody:\n - type: markdown\n attributes:\n value: We value your time and your efforts to submit this bug report is appreciated. 🙏\n\n - type: textarea\n id: description\n validations:\n required: true\n attributes:\n label: \"📜 Description\"\n description: \"A clear and concise description of what the bug is.\"\n placeholder: \"It bugs out when ...\"\n\n - type: textarea\n id: steps-to-reproduce\n validations:\n required: true\n attributes:\n label: \"👟 Reproduction steps\"\n description: \"How do you trigger this bug? Please walk us through it step by step.\"\n placeholder: \"1. Go to '...'\n 2. Click on '....'\n 3. Scroll down to '....'\n 4. See error\"\n\n - type: textarea\n id: expected-behavior\n validations:\n required: true\n attributes:\n label: \"👍 Expected behavior\"\n description: \"What did you think should happen?\"\n placeholder: \"It should ...\"\n\n - type: textarea\n id: actual-behavior\n validations:\n required: true\n attributes:\n label: \"👎 Actual Behavior with Screenshots\"\n description: \"What did actually happen? Add screenshots, if applicable.\"\n placeholder: \"It actually ...\"\n\n - type: dropdown\n id: operating-system\n attributes:\n label: \"💻 Operating system\"\n description: \"What OS is your app running on?\"\n options:\n - Linux\n - MacOS\n - Windows\n - Something else\n validations:\n required: true\n\n - type: dropdown\n id: browsers\n attributes:\n label: What browsers are you seeing the problem on?\n multiple: true\n options:\n - Firefox\n - Chrome\n - Safari\n - Microsoft Edge\n - Something else\n\n - type: dropdown\n id: dev-environment\n validations:\n required: true\n attributes:\n label: \"🤖 What development environment are you experiencing this bug on?\"\n options:\n - Docker\n - Local dev server\n\n - type: textarea\n id: env-vars\n validations:\n required: false\n attributes:\n label: \"🔒 Did you set the correct environment variables in the right path? List the environment variable names (not values please!)\"\n description: \"Please refer to the [Project setup instructions](https://github.com/arc53/DocsGPT#quickstart) if you are unsure.\"\n placeholder: \"It actually ...\"\n\n - type: textarea\n id: additional-context\n validations:\n required: false\n attributes:\n label: \"📃 Provide any additional context for the Bug.\"\n description: \"Add any other context about the problem here.\"\n placeholder: \"It actually ...\"\n\n - type: textarea\n id: logs\n validations:\n required: false\n attributes:\n label: 📖 Relevant log output\n description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks.\n render: shell\n\n - type: checkboxes\n id: no-duplicate-issues\n attributes:\n label: \"👀 Have you spent some time to check if this bug has been raised before?\"\n options:\n - label: \"I checked and didn't find similar issue\"\n required: true\n\n - type: dropdown\n id: willing-to-submit-pr\n attributes:\n label: 🔗 Are you willing to submit PR?\n description: This is absolutely not required, but we are happy to guide you in the contribution process.\n options: # Added options key\n - \"Yes, I am willing to submit a PR!\"\n - \"No\"\n validations:\n required: false\n\n\n - type: checkboxes\n id: terms\n attributes:\n label: 🧑‍⚖️ Code of Conduct\n description: By submitting this issue, you agree to follow our [Code of Conduct](https://github.com/arc53/DocsGPT/blob/main/CODE_OF_CONDUCT.md)\n options:\n - label: I agree to follow this project's Code of Conduct\n required: true\n" }, { "path": ".github/ISSUE_TEMPLATE/feature_request.yml", "content": "name: 🚀 Feature\ndescription: \"Submit a proposal for a new feature\"\ntitle: \"🚀 Feature: \"\nlabels: [feature]\nbody:\n - type: markdown\n attributes:\n value: We value your time and your efforts to submit this bug report is appreciated. 🙏\n - type: textarea\n id: feature-description\n validations:\n required: true\n attributes:\n label: \"🔖 Feature description\"\n description: \"A clear and concise description of what the feature is.\"\n placeholder: \"You should add ...\"\n - type: textarea\n id: pitch\n validations:\n required: true\n attributes:\n label: \"🎤 Why is this feature needed ?\"\n description: \"Please explain why this feature should be implemented and how it would be used. Add examples, if applicable.\"\n placeholder: \"In my use-case, ...\"\n - type: textarea\n id: solution\n validations:\n required: true\n attributes:\n label: \"✌️ How do you aim to achieve this?\"\n description: \"A clear and concise description of what you want to happen.\"\n placeholder: \"I want this feature to, ...\"\n - type: textarea\n id: alternative\n validations:\n required: false\n attributes:\n label: \"🔄️ Additional Information\"\n description: \"A clear and concise description of any alternative solutions or additional solutions you've considered.\"\n placeholder: \"I tried, ...\"\n - type: checkboxes\n id: no-duplicate-issues\n attributes:\n label: \"👀 Have you spent some time to check if this feature request has been raised before?\"\n options:\n - label: \"I checked and didn't find similar issue\"\n required: true\n - type: dropdown\n id: willing-to-submit-pr\n attributes:\n label: Are you willing to submit PR?\n description: This is absolutely not required, but we are happy to guide you in the contribution process.\n options:\n - \"Yes I am willing to submit a PR!\"\n" }, { "path": ".github/PULL_REQUEST_TEMPLATE.md", "content": "- **What kind of change does this PR introduce?** (Bug fix, feature, docs update, ...)\n\n- **Why was this change needed?** (You can also link to an open issue here)\n\n- **Other information**:" }, { "path": ".github/dependabot.yml", "content": "# To get started with Dependabot version updates, you'll need to specify which\n# package ecosystems to update and where the package manifests are located.\n# Please see the documentation for all configuration options:\n# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file\n\nversion: 2\nupdates:\n - package-ecosystem: \"pip\" # See documentation for possible values\n directory: \"/application\" # Location of package manifests\n schedule:\n interval: \"daily\"\n - package-ecosystem: \"npm\" # See documentation for possible values\n directory: \"/frontend\" # Location of package manifests\n schedule:\n interval: \"daily\"\n - package-ecosystem: \"npm\"\n directory: \"/extensions/react-widget\"\n schedule:\n interval: \"daily\"\n - package-ecosystem: \"github-actions\"\n directory: \"/\"\n schedule:\n interval: \"daily\"" }, { "path": ".github/holopin.yml", "content": "organization: docsgpt\ndefaultSticker: cm1ulwkkl180570cl82rtzympu\nstickers:\n - id: cm1ulwkkl180570cl82rtzympu\n alias: contributor2024\n - id: cm1ureg8o130450cl8c1po6mil\n alias: api\n - id: cm1urhmag148240cl8yvqxkthx\n alias: lpc\n - id: cm1urlcpq622090cl2tvu4w71y\n alias: lexeu\n" }, { "path": ".github/labeler.yml", "content": "repo:\n- changed-files:\n - any-glob-to-any-file: '*'\n\ngithub:\n- changed-files:\n - any-glob-to-any-file: '.github/**/*'\n\napplication:\n- changed-files:\n - any-glob-to-any-file: 'application/**/*'\n\ndocs:\n- changed-files:\n - any-glob-to-any-file: 'docs/**/*'\n\nextensions:\n- changed-files:\n - any-glob-to-any-file: 'extensions/**/*'\n\nfrontend:\n- changed-files:\n - any-glob-to-any-file: 'frontend/**/*'\n\nscripts:\n- changed-files:\n - any-glob-to-any-file: 'scripts/**/*'\n\ntests:\n- changed-files:\n - any-glob-to-any-file: 'tests/**/*'\n" }, { "path": ".github/styles/DocsGPT/Spelling.yml", "content": "extends: spelling\nlevel: warning\nmessage: \"Did you really mean '%s'?\"\nignore:\n - \"**/node_modules/**\"\n - \"**/dist/**\"\n - \"**/build/**\"\n - \"**/coverage/**\"\n - \"**/public/**\"\n - \"**/static/**\"\nvocab: DocsGPT\n" }, { "path": ".github/styles/config/vocabularies/DocsGPT/accept.txt", "content": "Ollama\nQdrant\nMilvus\nChatwoot\nNextra\nVSCode\nnpm\nLLMs\nAPIs\nGroq\nSGLang\nLMDeploy\nOAuth\nVite\nLLM\nJSONPath\nUIs\nconfigs\nuncomment\nqdrant\nvectorstore\ndocsgpt\nllm\nGPUs\nkubectl\nLightsail\nenqueues\nchatbot\nVSCode's\nShareability\nfeedbacks\nautomations\nPremade\nSignup\nRepo\nrepo\nenv\nURl\nagentic\nllama_cpp\nparsable\nSDKs\nboolean\nbool\nhardcode\nEOL\n" }, { "path": ".github/workflows/bandit.yaml", "content": "name: Bandit Security Scan\n\non:\n push:\n branches:\n - main\n pull_request:\n types: [opened, synchronize, reopened]\n\njobs:\n bandit_scan:\n if: ${{ github.repository == 'arc53/DocsGPT' }}\n runs-on: ubuntu-latest\n permissions:\n security-events: write\n actions: read\n contents: read\n\n steps:\n - name: Checkout code\n uses: actions/checkout@v4\n\n - name: Set up Python\n uses: actions/setup-python@v5\n with:\n python-version: '3.12'\n - name: Install dependencies\n run: |\n python -m pip install --upgrade pip\n pip install bandit # Bandit is needed for this action\n if [ -f application/requirements.txt ]; then pip install -r application/requirements.txt; fi\n\n - name: Run Bandit scan\n uses: PyCQA/bandit-action@v1\n with:\n severity: medium\n confidence: medium\n targets: application/\n env:\n GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}" }, { "path": ".github/workflows/ci.yml", "content": "name: Build and push DocsGPT Docker image\n\non:\n release:\n types: [published]\n\njobs:\n build:\n if: github.repository == 'arc53/DocsGPT'\n strategy:\n matrix:\n include:\n - platform: linux/amd64\n runner: ubuntu-latest\n suffix: amd64\n - platform: linux/arm64\n runner: ubuntu-24.04-arm\n suffix: arm64\n runs-on: ${{ matrix.runner }}\n permissions:\n contents: read\n packages: write\n steps:\n - uses: actions/checkout@v4\n\n - name: Set up QEMU # Only needed for emulation, not for native arm64 builds\n if: matrix.platform == 'linux/arm64'\n uses: docker/setup-qemu-action@v3\n\n - name: Set up Docker Buildx\n uses: docker/setup-buildx-action@v3\n with:\n driver: docker-container\n install: true\n\n - name: Login to DockerHub\n uses: docker/login-action@v3\n with:\n username: ${{ secrets.DOCKER_USERNAME }}\n password: ${{ secrets.DOCKER_PASSWORD }}\n\n - name: Login to ghcr.io\n uses: docker/login-action@v3\n with:\n registry: ghcr.io\n username: ${{ github.repository_owner }}\n password: ${{ secrets.GITHUB_TOKEN }}\n\n - name: Build and push platform-specific images\n uses: docker/build-push-action@v6\n with:\n file: './application/Dockerfile'\n platforms: ${{ matrix.platform }}\n context: ./application\n push: true\n tags: |\n ${{ secrets.DOCKER_USERNAME }}/docsgpt:${{ github.event.release.tag_name }}-${{ matrix.suffix }}\n ghcr.io/${{ github.repository_owner }}/docsgpt:${{ github.event.release.tag_name }}-${{ matrix.suffix }}\n provenance: false\n sbom: false\n cache-from: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/docsgpt:latest\n cache-to: type=inline\n\n manifest:\n if: github.repository == 'arc53/DocsGPT'\n needs: build\n runs-on: ubuntu-latest\n permissions:\n packages: write\n steps:\n - name: Set up Docker Buildx\n uses: docker/setup-buildx-action@v3\n with:\n driver: docker-container\n install: true\n\n - name: Login to DockerHub\n uses: docker/login-action@v3\n with:\n username: ${{ secrets.DOCKER_USERNAME }}\n password: ${{ secrets.DOCKER_PASSWORD }}\n\n - name: Login to ghcr.io\n uses: docker/login-action@v3\n with:\n registry: ghcr.io\n username: ${{ github.repository_owner }}\n password: ${{ secrets.GITHUB_TOKEN }}\n\n - name: Create and push manifest for DockerHub\n run: |\n set -e\n docker manifest create ${{ secrets.DOCKER_USERNAME }}/docsgpt:${{ github.event.release.tag_name }} \\\n --amend ${{ secrets.DOCKER_USERNAME }}/docsgpt:${{ github.event.release.tag_name }}-amd64 \\\n --amend ${{ secrets.DOCKER_USERNAME }}/docsgpt:${{ github.event.release.tag_name }}-arm64\n docker manifest push ${{ secrets.DOCKER_USERNAME }}/docsgpt:${{ github.event.release.tag_name }}\n docker manifest create ${{ secrets.DOCKER_USERNAME }}/docsgpt:latest \\\n --amend ${{ secrets.DOCKER_USERNAME }}/docsgpt:${{ github.event.release.tag_name }}-amd64 \\\n --amend ${{ secrets.DOCKER_USERNAME }}/docsgpt:${{ github.event.release.tag_name }}-arm64\n docker manifest push ${{ secrets.DOCKER_USERNAME }}/docsgpt:latest\n\n - name: Create and push manifest for ghcr.io\n run: |\n set -e\n docker manifest create ghcr.io/${{ github.repository_owner }}/docsgpt:${{ github.event.release.tag_name }} \\\n --amend ghcr.io/${{ github.repository_owner }}/docsgpt:${{ github.event.release.tag_name }}-amd64 \\\n --amend ghcr.io/${{ github.repository_owner }}/docsgpt:${{ github.event.release.tag_name }}-arm64\n docker manifest push ghcr.io/${{ github.repository_owner }}/docsgpt:${{ github.event.release.tag_name }}\n docker manifest create ghcr.io/${{ github.repository_owner }}/docsgpt:latest \\\n --amend ghcr.io/${{ github.repository_owner }}/docsgpt:${{ github.event.release.tag_name }}-amd64 \\\n --amend ghcr.io/${{ github.repository_owner }}/docsgpt:${{ github.event.release.tag_name }}-arm64\n docker manifest push ghcr.io/${{ github.repository_owner }}/docsgpt:latest" }, { "path": ".github/workflows/cife.yml", "content": "name: Build and push DocsGPT-FE Docker image\n\non:\n release:\n types: [published]\n\njobs:\n build:\n if: github.repository == 'arc53/DocsGPT'\n strategy:\n matrix:\n include:\n - platform: linux/amd64\n runner: ubuntu-latest\n suffix: amd64\n - platform: linux/arm64\n runner: ubuntu-24.04-arm\n suffix: arm64\n runs-on: ${{ matrix.runner }}\n permissions:\n contents: read\n packages: write\n steps:\n - uses: actions/checkout@v4\n\n - name: Set up QEMU # Only needed for emulation, not for native arm64 builds\n if: matrix.platform == 'linux/arm64'\n uses: docker/setup-qemu-action@v3\n\n - name: Set up Docker Buildx\n uses: docker/setup-buildx-action@v3\n with:\n driver: docker-container\n install: true\n\n - name: Login to DockerHub\n uses: docker/login-action@v3\n with:\n username: ${{ secrets.DOCKER_USERNAME }}\n password: ${{ secrets.DOCKER_PASSWORD }}\n\n - name: Login to ghcr.io\n uses: docker/login-action@v3\n with:\n registry: ghcr.io\n username: ${{ github.repository_owner }}\n password: ${{ secrets.GITHUB_TOKEN }}\n\n - name: Build and push platform-specific images\n uses: docker/build-push-action@v6\n with:\n file: './frontend/Dockerfile'\n platforms: ${{ matrix.platform }}\n context: ./frontend\n push: true\n tags: |\n ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:${{ github.event.release.tag_name }}-${{ matrix.suffix }}\n ghcr.io/${{ github.repository_owner }}/docsgpt-fe:${{ github.event.release.tag_name }}-${{ matrix.suffix }}\n provenance: false\n sbom: false\n cache-from: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:latest\n cache-to: type=inline\n\n manifest:\n if: github.repository == 'arc53/DocsGPT'\n needs: build\n runs-on: ubuntu-latest\n permissions:\n packages: write\n steps:\n - name: Set up Docker Buildx\n uses: docker/setup-buildx-action@v3\n with:\n driver: docker-container\n install: true\n\n - name: Login to DockerHub\n uses: docker/login-action@v3\n with:\n username: ${{ secrets.DOCKER_USERNAME }}\n password: ${{ secrets.DOCKER_PASSWORD }}\n\n - name: Login to ghcr.io\n uses: docker/login-action@v3\n with:\n registry: ghcr.io\n username: ${{ github.repository_owner }}\n password: ${{ secrets.GITHUB_TOKEN }}\n\n - name: Create and push manifest for DockerHub\n run: |\n set -e\n docker manifest create ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:${{ github.event.release.tag_name }} \\\n --amend ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:${{ github.event.release.tag_name }}-amd64 \\\n --amend ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:${{ github.event.release.tag_name }}-arm64\n docker manifest push ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:${{ github.event.release.tag_name }}\n docker manifest create ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:latest \\\n --amend ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:${{ github.event.release.tag_name }}-amd64 \\\n --amend ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:${{ github.event.release.tag_name }}-arm64\n docker manifest push ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:latest\n\n - name: Create and push manifest for ghcr.io\n run: |\n set -e\n docker manifest create ghcr.io/${{ github.repository_owner }}/docsgpt-fe:${{ github.event.release.tag_name }} \\\n --amend ghcr.io/${{ github.repository_owner }}/docsgpt-fe:${{ github.event.release.tag_name }}-amd64 \\\n --amend ghcr.io/${{ github.repository_owner }}/docsgpt-fe:${{ github.event.release.tag_name }}-arm64\n docker manifest push ghcr.io/${{ github.repository_owner }}/docsgpt-fe:${{ github.event.release.tag_name }}\n docker manifest create ghcr.io/${{ github.repository_owner }}/docsgpt-fe:latest \\\n --amend ghcr.io/${{ github.repository_owner }}/docsgpt-fe:${{ github.event.release.tag_name }}-amd64 \\\n --amend ghcr.io/${{ github.repository_owner }}/docsgpt-fe:${{ github.event.release.tag_name }}-arm64\n docker manifest push ghcr.io/${{ github.repository_owner }}/docsgpt-fe:latest" }, { "path": ".github/workflows/docker-develop-build.yml", "content": "name: Build and push multi-arch DocsGPT Docker image\n\non:\n workflow_dispatch:\n push:\n branches:\n - main\n\njobs:\n build:\n if: github.repository == 'arc53/DocsGPT'\n strategy:\n matrix:\n include:\n - platform: linux/amd64\n runner: ubuntu-latest\n suffix: amd64\n - platform: linux/arm64\n runner: ubuntu-24.04-arm\n suffix: arm64\n runs-on: ${{ matrix.runner }}\n permissions:\n contents: read\n packages: write\n steps:\n - uses: actions/checkout@v4\n\n - name: Set up Docker Buildx\n uses: docker/setup-buildx-action@v3\n with:\n driver: docker-container\n install: true\n\n - name: Login to DockerHub\n uses: docker/login-action@v3\n with:\n username: ${{ secrets.DOCKER_USERNAME }}\n password: ${{ secrets.DOCKER_PASSWORD }}\n \n - name: Login to ghcr.io\n uses: docker/login-action@v3\n with:\n registry: ghcr.io\n username: ${{ github.repository_owner }}\n password: ${{ secrets.GITHUB_TOKEN }}\n\n - name: Build and push platform-specific images\n uses: docker/build-push-action@v6\n with:\n file: './application/Dockerfile'\n platforms: ${{ matrix.platform }}\n context: ./application\n push: true\n tags: |\n ${{ secrets.DOCKER_USERNAME }}/docsgpt:develop-${{ matrix.suffix }}\n ghcr.io/${{ github.repository_owner }}/docsgpt:develop-${{ matrix.suffix }}\n provenance: false\n sbom: false\n cache-from: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/docsgpt:develop\n cache-to: type=inline\n\n manifest:\n if: github.repository == 'arc53/DocsGPT'\n needs: build\n runs-on: ubuntu-latest\n permissions:\n packages: write\n steps:\n - name: Set up Docker Buildx\n uses: docker/setup-buildx-action@v3\n with:\n driver: docker-container\n install: true\n\n - name: Login to DockerHub\n uses: docker/login-action@v3\n with:\n username: ${{ secrets.DOCKER_USERNAME }}\n password: ${{ secrets.DOCKER_PASSWORD }}\n \n - name: Login to ghcr.io\n uses: docker/login-action@v3\n with:\n registry: ghcr.io\n username: ${{ github.repository_owner }}\n password: ${{ secrets.GITHUB_TOKEN }}\n \n - name: Create and push manifest for DockerHub\n run: |\n docker manifest create ${{ secrets.DOCKER_USERNAME }}/docsgpt:develop \\\n --amend ${{ secrets.DOCKER_USERNAME }}/docsgpt:develop-amd64 \\\n --amend ${{ secrets.DOCKER_USERNAME }}/docsgpt:develop-arm64\n docker manifest push ${{ secrets.DOCKER_USERNAME }}/docsgpt:develop\n\n - name: Create and push manifest for ghcr.io\n run: |\n docker manifest create ghcr.io/${{ github.repository_owner }}/docsgpt:develop \\\n --amend ghcr.io/${{ github.repository_owner }}/docsgpt:develop-amd64 \\\n --amend ghcr.io/${{ github.repository_owner }}/docsgpt:develop-arm64\n docker manifest push ghcr.io/${{ github.repository_owner }}/docsgpt:develop" }, { "path": ".github/workflows/docker-develop-fe-build.yml", "content": "name: Build and push DocsGPT FE Docker image for development\n\non:\n workflow_dispatch:\n push:\n branches:\n - main\n\njobs:\n build:\n if: github.repository == 'arc53/DocsGPT'\n strategy:\n matrix:\n include:\n - platform: linux/amd64\n runner: ubuntu-latest\n suffix: amd64\n - platform: linux/arm64\n runner: ubuntu-24.04-arm\n suffix: arm64\n runs-on: ${{ matrix.runner }}\n permissions:\n contents: read\n packages: write\n steps:\n - uses: actions/checkout@v4\n\n - name: Set up QEMU # Only needed for emulation, not for native arm64 builds\n if: matrix.platform == 'linux/arm64'\n uses: docker/setup-qemu-action@v3\n\n - name: Set up Docker Buildx\n uses: docker/setup-buildx-action@v3\n with:\n driver: docker-container\n install: true\n\n - name: Login to DockerHub\n uses: docker/login-action@v3\n with:\n username: ${{ secrets.DOCKER_USERNAME }}\n password: ${{ secrets.DOCKER_PASSWORD }}\n\n - name: Login to ghcr.io\n uses: docker/login-action@v3\n with:\n registry: ghcr.io\n username: ${{ github.repository_owner }}\n password: ${{ secrets.GITHUB_TOKEN }}\n\n - name: Build and push platform-specific images\n uses: docker/build-push-action@v6\n with:\n file: './frontend/Dockerfile'\n platforms: ${{ matrix.platform }}\n context: ./frontend\n push: true\n tags: |\n ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:develop-${{ matrix.suffix }}\n ghcr.io/${{ github.repository_owner }}/docsgpt-fe:develop-${{ matrix.suffix }}\n provenance: false\n sbom: false\n cache-from: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:develop\n cache-to: type=inline\n\n manifest:\n if: github.repository == 'arc53/DocsGPT'\n needs: build\n runs-on: ubuntu-latest\n permissions:\n packages: write\n steps:\n - name: Set up Docker Buildx\n uses: docker/setup-buildx-action@v3\n with:\n driver: docker-container\n install: true\n\n - name: Login to DockerHub\n uses: docker/login-action@v3\n with:\n username: ${{ secrets.DOCKER_USERNAME }}\n password: ${{ secrets.DOCKER_PASSWORD }}\n\n - name: Login to ghcr.io\n uses: docker/login-action@v3\n with:\n registry: ghcr.io\n username: ${{ github.repository_owner }}\n password: ${{ secrets.GITHUB_TOKEN }}\n\n - name: Create and push manifest for DockerHub\n run: |\n docker manifest create ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:develop \\\n --amend ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:develop-amd64 \\\n --amend ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:develop-arm64\n docker manifest push ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:develop\n\n - name: Create and push manifest for ghcr.io\n run: |\n docker manifest create ghcr.io/${{ github.repository_owner }}/docsgpt-fe:develop \\\n --amend ghcr.io/${{ github.repository_owner }}/docsgpt-fe:develop-amd64 \\\n --amend ghcr.io/${{ github.repository_owner }}/docsgpt-fe:develop-arm64\n docker manifest push ghcr.io/${{ github.repository_owner }}/docsgpt-fe:develop" }, { "path": ".github/workflows/labeler.yml", "content": "# https://github.com/actions/labeler\nname: Pull Request Labeler\non:\n - pull_request_target\njobs:\n triage:\n if: github.repository == 'arc53/DocsGPT'\n permissions:\n contents: read\n pull-requests: write\n runs-on: ubuntu-latest\n steps:\n - uses: actions/labeler@v5\n with:\n repo-token: \"${{ secrets.GITHUB_TOKEN }}\"\n sync-labels: true\n" }, { "path": ".github/workflows/lint.yml", "content": "name: Python linting\n\non:\n push:\n branches:\n - '*'\n pull_request:\n types: [ opened, synchronize ]\n\npermissions:\n contents: read\n\njobs:\n ruff:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v4\n\n - name: Lint with Ruff\n uses: chartboost/ruff-action@v1\n" }, { "path": ".github/workflows/pytest.yml", "content": "name: Run python tests with pytest\non: [push, pull_request]\n\npermissions:\n contents: read\n\njobs:\n pytest_and_coverage:\n name: Run tests and count coverage\n runs-on: ubuntu-latest\n strategy:\n matrix:\n python-version: [\"3.12\"]\n steps:\n - uses: actions/checkout@v4\n - name: Set up Python ${{ matrix.python-version }}\n uses: actions/setup-python@v5\n with:\n python-version: ${{ matrix.python-version }}\n - name: Install dependencies\n run: |\n python -m pip install --upgrade pip\n cd application\n if [ -f requirements.txt ]; then pip install -r requirements.txt; fi\n cd ../tests\n if [ -f requirements.txt ]; then pip install -r requirements.txt; fi\n - name: Test with pytest and generate coverage report\n run: |\n python -m pytest --cov=application --cov-report=xml --cov-report=term-missing\n - name: Upload coverage reports to Codecov\n if: github.event_name == 'pull_request' && matrix.python-version == '3.12'\n uses: codecov/codecov-action@v5\n env:\n CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}\n" }, { "path": ".github/workflows/sync_fork.yaml", "content": "name: Upstream Sync\n\npermissions:\n contents: write\n\non:\n schedule:\n - cron: \"0 0 * * *\" # every hour\n workflow_dispatch:\n\njobs:\n sync_latest_from_upstream:\n name: Sync latest commits from upstream repo\n runs-on: ubuntu-latest\n if: ${{ github.event.repository.fork }}\n\n steps:\n # Step 1: run a standard checkout action\n - name: Checkout target repo\n uses: actions/checkout@v4\n\n # Step 2: run the sync action\n - name: Sync upstream changes\n id: sync\n uses: aormsby/Fork-Sync-With-Upstream-action@v3.4\n with:\n # set your upstream repo and branch\n upstream_sync_repo: arc53/DocsGPT\n upstream_sync_branch: main\n target_sync_branch: main\n target_repo_token: ${{ secrets.GITHUB_TOKEN }} # automatically generated, no need to set\n\n # Set test_mode true to run tests instead of the true action!!\n test_mode: false\n\n - name: Sync check\n if: failure()\n run: |\n echo \"::error::由于权限不足，导致同步失败（这是预期的行为），请前往仓库首页手动执行[Sync fork]。\"\n echo \"::error::Due to insufficient permissions, synchronization failed (as expected). Please go to the repository homepage and manually perform [Sync fork].\"\n exit 1" }, { "path": ".github/workflows/vale.yml", "content": "name: Vale Documentation Linter\n\non:\n pull_request:\n paths:\n - 'docs/**/*.md'\n - 'docs/**/*.mdx'\n - '**/*.md'\n - '.vale.ini'\n - '.github/styles/**'\n\npermissions:\n contents: read\n pull-requests: write\n\njobs:\n vale:\n runs-on: ubuntu-latest\n steps:\n - name: Checkout code\n uses: actions/checkout@v4\n\n - name: Vale linter\n uses: errata-ai/vale-action@v2\n with:\n files: docs\n fail_on_error: false\n version: 3.0.5\n env:\n GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n" }, { "path": ".gitignore", "content": "# Byte-compiled / optimized / DLL files\n__pycache__/\n*.py[cod]\n*$py.class\nexperiments/\n\nexperiments\n# C extensions\n*.so\n*.next\n# Distribution / packaging\n.Python\nbuild/\ndevelop-eggs/\ndownloads/\neggs/\n.eggs/\nlib/\nlib64/\nparts/\nsdist/\nvar/\nwheels/\npip-wheel-metadata/\nshare/python-wheels/\n*.egg-info/\n.installed.cfg\n*.egg\nMANIFEST\n\n# PyInstaller\n# Usually these files are written by a python script from a template\n# before PyInstaller builds the exe, so as to inject date/other infos into it.\n*.manifest\n*.spec\n\n# Installer logs\npip-log.txt\npip-delete-this-directory.txt\n\n# Unit test / coverage reports\nhtmlcov/\n.tox/\n.nox/\n.coverage\n.coverage.*\n.cache\nnosetests.xml\ncoverage.xml\n*.cover\n*.py,cover\n.hypothesis/\n.pytest_cache/\n\n# Translations\n*.mo\n*.pot\n\n# Django stuff:\n*.log\nlocal_settings.py\ndb.sqlite3\ndb.sqlite3-journal\n\n# Flask stuff:\ninstance/\n.webassets-cache\n\n# Scrapy stuff:\n.scrapy\n\n# Sphinx documentation\ndocs/_build/\ndocs/public/_pagefind/\n\n# PyBuilder\ntarget/\n\n# Jupyter Notebook\n.ipynb_checkpoints\n**/*.ipynb\n\n# IPython\nprofile_default/\nipython_config.py\n\n# pyenv\n.python-version\n\n# pipenv\n# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.\n# However, in case of collaboration, if having platform-specific dependencies or dependencies\n# having no cross-platform support, pipenv may install dependencies that don't work, or not\n# install all needed dependencies.\n#Pipfile.lock\n\n# PEP 582; used by e.g. github.com/David-OConnor/pyflow\n__pypackages__/\n\n# Celery stuff\ncelerybeat-schedule\ncelerybeat.pid\n\n# SageMath parsed files\n*.sage.py\n\n# Environments\n.env\n.venv\nenv/\nvenv/\nENV/\nenv.bak/\nvenv.bak/\n.flaskenv\n# Spyder project settings\n.spyderproject\n.spyproject\n.jwt_secret_key\n\n# Rope project settings\n.ropeproject\n\n# mkdocs documentation\n/site\n\n# mypy\n.mypy_cache/\n.dmypy.json\ndmypy.json\n\n# Pyre type checker\n.pyre/\n\n#pycharm\n.idea/\n\n# macOS\n.DS_Store\n\n#frontend\n# Logs\nfrontend/logs\nfrontend/*.log\nfrontend/npm-debug.log*\nfrontend/yarn-debug.log*\nfrontend/yarn-error.log*\nfrontend/pnpm-debug.log*\nfrontend/lerna-debug.log*\n\n# Keep frontend utility helpers tracked (overrides global lib/ ignore)\n!frontend/src/lib/\n!frontend/src/lib/**\n\nfrontend/node_modules\nfrontend/dist\nfrontend/dist-ssr\nfrontend/*.local\n\n# Editor directories and files\nfrontend/.vscode/*\nfrontend/!.vscode/extensions.json\nfrontend/.idea\nfrontend/.DS_Store\nfrontend/*.suo\nfrontend/*.ntvs*\nfrontend/*.njsproj\nfrontend/*.sln\nfrontend/*.sw?\n\napplication/vectors/\n\n**/inputs\n\n**/indexes\n\n**/temp\n\n**/yarn.lock\n\nnode_modules/\n.vscode/settings.json\n/models/\nmodel/\n" }, { "path": ".ruff.toml", "content": "# Allow lines to be as long as 120 characters.\nline-length = 120\n\n[lint.per-file-ignores]\n# Integration tests use sys.path.insert() before imports for standalone execution\n\"tests/integration/*.py\" = [\"E402\"]" }, { "path": ".vale.ini", "content": "MinAlertLevel = warning\nStylesPath = .github/styles\n\n[*.{md,mdx}]\nBasedOnStyles = DocsGPT\n" }, { "path": ".vscode/launch.json", "content": "{\n \"version\": \"0.2.0\",\n \"configurations\": [\n {\n \"name\": \"Frontend Debug (npm)\",\n \"type\": \"node-terminal\",\n \"request\": \"launch\",\n \"command\": \"npm run dev\",\n \"cwd\": \"${workspaceFolder}/frontend\"\n },\n {\n \"name\": \"Flask Debugger\",\n \"type\": \"debugpy\",\n \"request\": \"launch\",\n \"module\": \"flask\",\n \"env\": {\n \"FLASK_APP\": \"application/app.py\",\n \"PYTHONPATH\": \"${workspaceFolder}\",\n \"FLASK_ENV\": \"development\",\n \"FLASK_DEBUG\": \"1\",\n \"FLASK_RUN_PORT\": \"7091\",\n \"FLASK_RUN_HOST\": \"0.0.0.0\"\n\n },\n \"args\": [\n \"run\",\n \"--no-debugger\"\n ],\n \"cwd\": \"${workspaceFolder}\",\n },\n {\n \"name\": \"Celery Debugger\",\n \"type\": \"debugpy\",\n \"request\": \"launch\",\n \"module\": \"celery\",\n \"env\": {\n \"PYTHONPATH\": \"${workspaceFolder}\",\n },\n \"args\": [\n \"-A\",\n \"application.app.celery\",\n \"worker\",\n \"-l\",\n \"INFO\",\n \"--pool=solo\"\n ],\n \"cwd\": \"${workspaceFolder}\"\n },\n {\n \"name\": \"Dev Containers (Mongo + Redis)\",\n \"type\": \"node-terminal\",\n \"request\": \"launch\",\n \"command\": \"docker compose -f deployment/docker-compose-dev.yaml up --build\",\n \"cwd\": \"${workspaceFolder}\"\n }\n ],\n \"compounds\": [\n {\n \"name\": \"DocsGPT: Full Stack\",\n \"configurations\": [\n \"Frontend Debug (npm)\",\n \"Flask Debugger\",\n \"Celery Debugger\"\n ],\n \"presentation\": {\n \"group\": \"DocsGPT\",\n \"order\": 1\n }\n }\n ]\n}" }, { "path": "AGENTS.md", "content": "# AGENTS.md\n\n- Read `CONTRIBUTING.md` before making non-trivial changes.\n- For day-to-day development and feature work, follow the development-environment workflow rather than defaulting to `setup.sh` / `setup.ps1`.\n- Avoid using the setup scripts during normal feature work unless the user explicitly asks for them. Users configure `.env` usually.\n- Try to follow red/green TDD\n\n### Check existing dev prerequisites first\n\nFor feature work, do **not** assume the environment needs to be recreated.\n\n- Check whether the user already has a Python virtual environment such as `venv/` or `.venv/`.\n- Check whether MongoDB is already running.\n- Check whether Redis is already running.\n- Reuse what is already working. Do not stop or recreate MongoDB, Redis, or the Python environment unless the task is environment setup or troubleshooting.\n\n## Normal local development commands\n\nUse these commands once the dev prerequisites above are satisfied.\n\n### Backend\n\n```bash\nsource .venv/bin/activate # macOS/Linux\nuv pip install -r application/requirements.txt # or: pip install -r application/requirements.txt\n```\n\nRun the Flask API (if needed):\n\n```bash\nflask --app application/app.py run --host=0.0.0.0 --port=7091\n```\n\nRun the Celery worker in a separate terminal (if needed):\n\n```bash\ncelery -A application.app.celery worker -l INFO\n```\n\nOn macOS, prefer the solo pool for Celery:\n\n```bash\npython -m celery -A application.app.celery worker -l INFO --pool=solo\n```\n\n### Frontend\n\nInstall dependencies only when needed, then run the dev server:\n\n```bash\ncd frontend\nnpm install --include=dev\nnpm run dev\n```\n\n### Docs site\n\n```bash\ncd docs\nnpm install\n```\n\n### Python / backend changes validation\n\n```bash\nruff check .\npython -m pytest\n```\n\n### Frontend changes\n\n```bash\ncd frontend && npm run lint\ncd frontend && npm run build\n```\n\n### Documentation changes\n\n```bash\ncd docs && npm run build\n```\n\nIf Vale is installed locally and you edited prose, also run:\n\n```bash\nvale .\n```\n\n## Repository map\n\n- `application/`: Flask backend, API routes, agent logic, retrieval, parsing, security, storage, Celery worker, and WSGI entrypoints.\n- `tests/`: backend unit/integration tests and test-only Python dependencies.\n- `frontend/`: Vite + React + TypeScript application.\n- `frontend/src/`: main UI code, including `components`, `conversation`, `hooks`, `locale`, `settings`, `upload`, and Redux store wiring in `store.ts`.\n- `docs/`: separate documentation site built with Next.js/Nextra.\n- `extensions/`: integrations and widgets such as Chatwoot, Chrome, Discord, React widget, Slack bot, and web widget.\n- `deployment/`: Docker Compose variants and Kubernetes manifests.\n\n## Coding rules\n\n### Backend\n\n- Follow PEP 8 and keep Python line length at or under 120 characters.\n- Use type hints for function arguments and return values.\n- Add Google-style docstrings to new or substantially changed functions and classes.\n- Add or update tests under `tests/` for backend behavior changes.\n- Keep changes narrow in `api`, `auth`, `security`, `parser`, `retriever`, and `storage` areas.\n\n### Backend Abstractions\n\n- LLM providers implement a common interface in `application/llm/` (add new providers by extending the base class).\n- Vector stores are abstracted in `application/vectorstore/`.\n- Parsers live in `application/parser/` and handle different document formats in the ingestion stage.\n- Agents and tools are in `application/agents/` and `application/agents/tools/`.\n- Celery setup/config lives in `application/celery_init.py` and `application/celeryconfig.py`.\n- Settings and env vars are managed via Pydantic in `application/core/settings.py`.\n\n### Frontend\n\n- Follow the existing ESLint + Prettier setup.\n- Prefer small, reusable functional components and hooks.\n- If shared state must be added, use Redux rather than introducing a new global state library.\n- Avoid broad UI refactors unless the task explicitly asks for them.\n- Do not re-create components if we already have some in the app.\n\n## PR readiness\n\nBefore opening a PR:\n\n- run the relevant validation commands above\n- confirm backend changes still work end-to-end after ingesting sample data when applicable\n- clearly summarize user-visible behavior changes\n- mention any config, dependency, or deployment implications\n- Ask your user to attach a screenshot or a video to it" }, { "path": "CODE_OF_CONDUCT.md", "content": "# Contributor Covenant Code of Conduct\n\n## Our Pledge\n\nWe as members, contributors and leaders pledge to make participation in our\ncommunity, a harassment-free experience for everyone, regardless of age, body\nsize, visible or invisible disability, ethnicity, sex characteristics, gender\nidentity and expression, level of experience, education, socio-economic status,\nnationality, personal appearance, race, religion or sexual identity\nand orientation.\n\nWe pledge to act and interact in ways that contribute to an open, welcoming,\ndiverse, inclusive and a healthy community.\n\n## Our Standards\n\nExamples of behavior that contribute to a positive environment for our\ncommunity include:\n\n## Demonstrating empathy and kindness towards other people\n1. Being respectful and open to differing opinions, viewpoints, and experiences\n2. Giving and gracefully accepting constructive feedback\n3. Taking accountability and offering apologies to those who have been impacted by our errors,\n while also gaining insights from the situation\n4. Focusing on what is best not just for us as individuals but for the\n community as a whole\n\nExamples of unacceptable behavior include:\n\n1. The use of sexualized language or imagery, and sexual attention or\n advances of any kind\n2. Trolling, insulting or derogatory comments, and personal or political attacks\n3. Public or private harassment\n4. Publishing other's private information, such as a physical or email\n address, without their explicit permission\n5. Other conduct which could reasonably be considered inappropriate in a\n professional setting\n\n## Enforcement Responsibilities\n\nCommunity leaders are responsible for clarifying and enforcing our standards of\nacceptable behavior and will take appropriate and fair corrective action in\nresponse to any behavior that they deem inappropriate, threatening, offensive\nor harmful.\n\nCommunity leaders have the right and responsibility to remove, edit, or reject\ncomments, commits, code, wiki edits, issues, and other contributions that are\nnot aligned to this Code of Conduct and will communicate reasons for moderation\ndecisions when appropriate.\n\n## Scope\n\nThis Code of Conduct applies within all community spaces and also applies when\nan individual is officially representing the community in public spaces.\nExamples of representing our community include using an official e-mail address,\nposting via an official social media account or acting as an appointed\nrepresentative at an online or offline event.\n\n## Enforcement\n\nInstances of abusive, harassing, or otherwise unacceptable behavior may be\nreported to the community leaders responsible for enforcement at\ncontact@arc53.com.\nAll complaints will be reviewed and investigated promptly and fairly.\n\nAll community leaders are obligated to be respectful towards the privacy and security of the\nreporter of any incident.\n\n## Enforcement Guidelines\n\nCommunity leaders will follow these Community Impact Guidelines in determining\nthe consequences for any action that they deem in violation of this Code of Conduct:\n\n### 1. Correction\n* **Community Impact**: Use of inappropriate language or other behavior deemed\nunprofessional or unwelcome in the community space.\n\n* **Consequence**: A private, written warning from community leaders, providing\nclarity around the nature of the violation and an explanation of why the\nbehavior was inappropriate. A public apology may be requested.\n\n### 2. Warning\n* **Community Impact**: A violation through a single incident or series\nof actions.\n\n* **Consequence**: A warning with consequences for continued behavior. No\ninteraction with the people involved, including unsolicited interaction with\nthose enforcing the Code of Conduct, for a specified period of time. This\nincludes avoiding interactions in community spaces as well as external channels\nlike social media. Violating these terms may lead to a temporary or\npermanent ban.\n\n### 3. Temporary Ban\n* **Community Impact**: A serious violation of community standards, including\nsustained inappropriate behavior.\n\n* **Consequence**: A temporary ban from any sort of interaction or public\ncommunication with the community for a specified period of time. No public or\nprivate interaction with the people involved, including unsolicited interaction\nwith those enforcing the Code of Conduct, is allowed during this period.\nViolating these terms may lead to a permanent ban.\n\n### 4. Permanent Ban\n* **Community Impact**: Demonstrating a pattern of violation of community\nstandards, including sustained inappropriate behavior,harassment of an\nindividual or aggression towards or disparagement of classes of individuals.\n\n* **Consequence**: A permanent ban from any sort of public interaction within\nthe community.\n\n## Attribution\n\nThis Code of Conduct is adapted from the [Contributor Covenant][homepage],\nversion 2.0, available at\nhttps://www.contributor-covenant.org/version/2/0/code_of_conduct.html.\n\nCommunity Impact Guidelines were inspired by [Mozilla's code of conduct\nenforcement ladder](https://github.com/mozilla/diversity).\n\n[homepage]: https://www.contributor-covenant.org\n\nFor answers to common questions about this code of conduct, see the FAQ at\nhttps://www.contributor-covenant.org/faq. Translations are available at\nhttps://www.contributor-covenant.org/translations.\n" }, { "path": "CONTRIBUTING.md", "content": "# Welcome to DocsGPT Contributing Guidelines\n\nThank you for choosing to contribute to DocsGPT! We are all very grateful! \n\n# We accept different types of contributions\n\n📣 **Discussions** - Engage in conversations, start new topics, or help answer questions.\n\n🐞 **Issues** - This is where we keep track of tasks. It could be bugs, fixes or suggestions for new features.\n\n🛠️ **Pull requests** - Suggest changes to our repository, either by working on existing issues or adding new features.\n\n📚 **Wiki** - This is where our documentation resides.\n\n\n## 🐞 Issues and Pull requests\n\n- We value contributions in the form of discussions or suggestions. We recommend taking a look at existing issues and our [roadmap](https://github.com/orgs/arc53/projects/2).\n\n\n- If you're interested in contributing code, here are some important things to know:\n\n- We have a frontend built on React (Vite) and a backend in Python.\n\n> **Required for every PR:** Please attach screenshots or a short screen\n> recording that shows the working version of your changes. This makes the\n> requirement visible to reviewers and helps them quickly verify what you are\n> submitting.\n\n \nBefore creating issues, please check out how the latest version of our app looks and works by launching it via [Quickstart](https://github.com/arc53/DocsGPT#quickstart) the version on our live demo is slightly modified with login. Your issues should relate to the version you can launch via [Quickstart](https://github.com/arc53/DocsGPT#quickstart).\n\n### 👨‍💻 If you're interested in contributing code, here are some important things to know:\n\nFor instructions on setting up a development environment, please refer to our [Development Deployment Guide](https://docs.docsgpt.cloud/Deploying/Development-Environment).\n\nTech Stack Overview:\n\n- 🌐 Frontend: Built with React (Vite) ⚛️,\n\n- 🖥 Backend: Developed in Python 🐍\n\n### 🌐 Frontend Contributions (⚛️ React, Vite)\n\n* The updated Figma design can be found [here](https://www.figma.com/file/OXLtrl1EAy885to6S69554/DocsGPT?node-id=0%3A1&t=hjWVuxRg9yi5YkJ9-1). Please try to follow the guidelines.\n* **Coding Style:** We follow a strict coding style enforced by ESLint and Prettier. Please ensure your code adheres to the configuration provided in our repository's `fronetend/.eslintrc.js` file. We recommend configuring your editor with ESLint and Prettier to help with this.\n* **Component Structure:** Strive for small, reusable components. Favor functional components and hooks over class components where possible.\n* **State Management** If you need to add stores, please use Redux.\n\n### 🖥 Backend Contributions (🐍 Python)\n\n- Review our issues and contribute to [`/application`](https://github.com/arc53/DocsGPT/tree/main/application) \n- All new code should be covered with unit tests ([pytest](https://github.com/pytest-dev/pytest)). Please find tests under [`/tests`](https://github.com/arc53/DocsGPT/tree/main/tests) folder.\n- Before submitting your Pull Request, ensure it can be queried after ingesting some test data.\n- **Coding Style:** We adhere to the [PEP 8](https://www.python.org/dev/peps/pep-0008/) style guide for Python code. We use `ruff` as our linter and code formatter. Please ensure your code is formatted correctly and passes `ruff` checks before submitting.\n- **Type Hinting:** Please use type hints for all function arguments and return values. This improves code readability and helps catch errors early. Example:\n\n ```python\n def my_function(name: str, count: int) -> list[str]:\n ...\n ```\n- **Docstrings:** All functions and classes should have docstrings explaining their purpose, parameters, and return values. We prefer the [Google style docstrings](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html). Example:\n\n ```python\n def my_function(name: str, count: int) -> list[str]:\n \"\"\"Does something with a name and a count.\n\n Args:\n name: The name to use.\n count: The number of times to do it.\n\n Returns:\n A list of strings.\n \"\"\"\n ...\n ```\n \n### Testing\n\nTo run unit tests from the root of the repository, execute:\n```\npython -m pytest\n```\n\n## Workflow 📈\n\nHere's a step-by-step guide on how to contribute to DocsGPT:\n\n1. **Fork the Repository:**\n - Click the \"Fork\" button at the top-right of this repository to create your fork.\n\n2. **Clone the Forked Repository:**\n - Clone the repository using:\n ``` shell\n git clone https://github.com//DocsGPT.git\n ```\n\n3. **Keep your Fork in Sync:**\n - Before you make any changes, make sure that your fork is in sync to avoid merge conflicts using:\n ```shell\n git remote add upstream https://github.com/arc53/DocsGPT.git\n git pull upstream main\n ```\n\n4. **Create and Switch to a New Branch:**\n - Create a new branch for your contribution using:\n ```shell\n git checkout -b your-branch-name\n ```\n\n5. **Make Changes:**\n - Make the required changes in your branch.\n\n6. **Add Changes to the Staging Area:**\n - Add your changes to the staging area using:\n ```shell\n git add .\n ```\n\n7. **Commit Your Changes:**\n - Commit your changes with a descriptive commit message using:\n ```shell\n git commit -m \"Your descriptive commit message\"\n ```\n\n8. **Push Your Changes to the Remote Repository:**\n - Push your branch with changes to your fork on GitHub using:\n ```shell\n git push origin your-branch-name\n ```\n\n9. **Submit a Pull Request (PR):**\n - Create a Pull Request from your branch to the main repository. Make sure to include a detailed description of your changes, reference any related issues, and attach screenshots or a screen recording showing the working version.\n\n10. **Collaborate:**\n - Be responsive to comments and feedback on your PR.\n - Make necessary updates as suggested.\n - Once your PR is approved, it will be merged into the main repository.\n\n11. **Testing:**\n - Before submitting a Pull Request, ensure your code passes all unit tests.\n - To run unit tests from the root of the repository, execute:\n ```shell\n python -m pytest\n ```\n\n*Note: You should run the unit test only after making the changes to the backend code.*\n\n12. **Questions and Collaboration:**\n - Feel free to join our Discord. We're very friendly and welcoming to new contributors, so don't hesitate to reach out.\n\nThank you for considering contributing to DocsGPT! 🙏\n\n## Questions/collaboration\nFeel free to join our [Discord](https://discord.gg/vN7YFfdMpj). We're very friendly and welcoming to new contributors, so don't hesitate to reach out.\n# Thank you so much for considering to contributing DocsGPT!🙏\n" }, { "path": "HACKTOBERFEST.md", "content": "# **🎉 Join the Hacktoberfest with DocsGPT and win a Free T-shirt for a meaningful PR! 🎉**\n\nWelcome, contributors! We're excited to announce that DocsGPT is participating in Hacktoberfest. Get involved by submitting meaningful pull requests.\n\nAll Meaningful contributors with accepted PRs that were created for issues with the `hacktoberfest` label (set by our maintainer team: dartpain, siiddhantt, pabik, ManishMadan2882) will receive a cool T-shirt! 🤩.\n\"hacktoberfest-mocks-preview\"\n\nFill in [this form](https://forms.gle/Npaba4n9Epfyx56S8\n) after your PR was merged please \n\nIf you are in doubt don't hesitate to ping us on discord, ping me - Alex (dartpain).\n\n## 📜 Here's How to Contribute:\n```text\n🛠️ Code: This is the golden ticket! Make meaningful contributions through PRs.\n\n🧩 API extension: Build an app utilising DocsGPT API. We prefer submissions that showcase original ideas and turn the API into an AI agent.\nThey can be a completely separate repos. \nFor example: \nhttps://github.com/arc53/tg-bot-docsgpt-extenstion or \nhttps://github.com/arc53/DocsGPT-cli\n\nNon-Code Contributions:\n\n📚 Wiki: Improve our documentation, create a guide.\n\n🖥️ Design: Improve the UI/UX or design a new feature.\n```\n\n### 📝 Guidelines for Pull Requests:\n- Familiarize yourself with the current contributions and our [Roadmap](https://github.com/orgs/arc53/projects/2).\n- Before contributing check existing [issues](https://github.com/arc53/DocsGPT/issues) or [create](https://github.com/arc53/DocsGPT/issues/new/choose) an issue and wait to get assigned.\n- Once you are finished with your contribution, please fill in this [form](https://forms.gle/Npaba4n9Epfyx56S8).\n- Refer to the [Documentation](https://docs.docsgpt.cloud/).\n- Feel free to join our [Discord](https://discord.gg/vN7YFfdMpj) server. We're here to help newcomers, so don't hesitate to jump in! Join us [here](https://discord.gg/vN7YFfdMpj).\n \nThank you very much for considering contributing to DocsGPT during Hacktoberfest! 🙏 Your contributions (not just simple typos) could earn you a stylish new t-shirt.\n\nWe will publish a t-shirt design later into the October.\n" }, { "path": "LICENSE", "content": "MIT License\n\nCopyright (c) 2023 arc53\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" }, { "path": "README.md", "content": "

\n DocsGPT 🦖\n

\n\n

\n Private AI for agents, assistants and enterprise search\n

\n\n

\n DocsGPT is an open-source AI platform for building intelligent agents and assistants. Features Agent Builder, deep research tools, document analysis (PDF, Office, web content, and audio), Multi-model support (choose your provider or run locally), and rich API connectivity for agents with actionable tools and integrations. Deploy anywhere with complete privacy control.\n

\n\n
\n \n ![link to main GitHub showing Stars number](https://img.shields.io/github/stars/arc53/docsgpt?style=social)\n ![link to main GitHub showing Forks number](https://img.shields.io/github/forks/arc53/docsgpt?style=social)\n ![link to license file](https://img.shields.io/github/license/arc53/docsgpt)\n \n ![link to discord](https://img.shields.io/discord/1070046503302877216)\n ![X (formerly Twitter) URL](https://img.shields.io/twitter/follow/docsgptai)\n\n⚡️ Quickstart • ☁️ Cloud Version • 💬 Discord\n
\n📖 Documentation • 👫 Contribute • 🗞 Blog\n
\n\n
\n\n\n
\n
\n\"video-example-of-docs-gpt\"\n
\n

\n Key Features:\n

\n
    \n
  • 🗂️ Wide Format Support: Reads PDF, DOCX, CSV, XLSX, EPUB, MD, RST, HTML, MDX, JSON, PPTX, images, and audio files such as MP3, WAV, M4A, OGG, and WebM.
    • \n
  • 🎙️ Speech Workflows: Record voice input into chat, transcribe audio on the backend, and ingest meeting recordings or voice notes as searchable knowledge.
    • \n
  • 🌐 Web & Data Integration: Ingests from URLs, sitemaps, Reddit, GitHub and web crawlers.
    • \n
  • ✅ Reliable Answers: Get accurate, hallucination-free responses with source citations viewable in a clean UI.
    • \n
  • 🔑 Streamlined API Keys: Generate keys linked to your settings, documents, and models, simplifying chatbot and integration setup.
    • \n
  • 🔗 Actionable Tooling: Connect to APIs, tools, and other services to enable LLM actions.
    • \n
  • 🧩 Pre-built Integrations: Use readily available HTML/React chat widgets, search tools, Discord/Telegram bots, and more.
    • \n
  • 🔌 Flexible Deployment: Works with major LLMs (OpenAI, Google, Anthropic) and local models (Ollama, llama_cpp).
    • \n
  • 🏢 Secure & Scalable: Run privately and securely with Kubernetes support, designed for enterprise-grade reliability.
    • \n
\n\n## Roadmap\n- [x] Add OAuth 2.0 authentication for MCP ( September 2025 )\n- [x] Deep Agents ( October 2025 )\n- [x] Prompt Templating ( October 2025 )\n- [x] Full api tooling ( Dec 2025 )\n- [ ] Agent scheduling ( Jan 2026 )\n\nYou can find our full roadmap [here](https://github.com/orgs/arc53/projects/2). Please don't hesitate to contribute or create issues, it helps us improve DocsGPT!\n\n### Production Support / Help for Companies:\n\nWe're eager to provide personalized assistance when deploying your DocsGPT to a live environment.\n\n[Get a Demo :wave:](https://www.docsgpt.cloud/contact)⁠\n\n[Send Email :email:](mailto:support@docsgpt.cloud?subject=DocsGPT%20support%2Fsolutions)\n\n## Join the Lighthouse Program 🌟\n\nCalling all developers and GenAI innovators! The **DocsGPT Lighthouse Program** connects technical leaders actively deploying or extending DocsGPT in real-world scenarios. Collaborate directly with our team to shape the roadmap, access priority support, and build enterprise-ready solutions with exclusive community insights.\n\n[Learn More & Apply →](https://docs.google.com/forms/d/1KAADiJinUJ8EMQyfTXUIGyFbqINNClNR3jBNWq7DgTE)\n\n## QuickStart\n\n> [!Note]\n> Make sure you have [Docker](https://docs.docker.com/engine/install/) installed\n\nA more detailed [Quickstart](https://docs.docsgpt.cloud/quickstart) is available in our documentation\n\n1. **Clone the repository:**\n\n ```bash\n git clone https://github.com/arc53/DocsGPT.git\n cd DocsGPT\n ```\n\n**For macOS and Linux:**\n\n2. **Run the setup script:**\n\n ```bash\n ./setup.sh\n ```\n\n**For Windows:**\n\n2. **Run the PowerShell setup script:**\n\n ```powershell\n PowerShell -ExecutionPolicy Bypass -File .\\setup.ps1\n ```\n\nEither script will guide you through setting up DocsGPT. Five options available: using the public API, running locally, connecting to a local inference engine, using a cloud API provider, or build the docker image locally. Scripts will automatically configure your `.env` file and handle necessary downloads and installations based on your chosen option.\n\n**Navigate to http://localhost:5173/**\n\nTo stop DocsGPT, open a terminal in the `DocsGPT` directory and run:\n\n```bash\ndocker compose -f deployment/docker-compose.yaml down\n```\n\n(or use the specific `docker compose down` command shown after running the setup script).\n\n> [!Note]\n> For development environment setup instructions, please refer to the [Development Environment Guide](https://docs.docsgpt.cloud/Deploying/Development-Environment).\n\n## Contributing\n\nPlease refer to the [CONTRIBUTING.md](CONTRIBUTING.md) file for information about how to get involved. We welcome issues, questions, and pull requests.\n\n## Architecture\n\n![Architecture chart](https://github.com/user-attachments/assets/fc6a7841-ddfc-45e6-b5a0-d05fe648cbe2)\n\n## Project Structure\n\n- Application - Flask app (main application).\n\n- Extensions - Extensions, like react widget or discord bot.\n\n- Frontend - Frontend uses Vite and React.\n\n- Scripts - Miscellaneous scripts.\n\n## Code Of Conduct\n\nWe as members, contributors, and leaders, pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation. Please refer to the [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md) file for more information about contributing.\n\n## Many Thanks To Our Contributors⚡\n\n\n \"Contributors\"\n\n\n## License\n\nThe source code license is [MIT](https://opensource.org/license/mit/), as described in the [LICENSE](LICENSE) file.\n\n## This project is supported by:\n\n

\n \n \n \n

\n

\n \n \"color\"\n \n \n

\n" }, { "path": "SECURITY.md", "content": "# Security Policy\n\n## Supported Versions\n\nSupported Versions:\n\nCurrently, we support security patches by committing changes and bumping the version published on Github.\n\n## Reporting a Vulnerability\n\nFound a vulnerability? Please email us:\n\nsecurity@arc53.com\n\n" }, { "path": "application/Dockerfile", "content": "# Builder Stage\nFROM ubuntu:24.04 as builder\n\nENV DEBIAN_FRONTEND=noninteractive\n\nRUN apt-get update && \\\n apt-get install -y software-properties-common && \\\n add-apt-repository ppa:deadsnakes/ppa && \\\n apt-get update && \\\n apt-get install -y --no-install-recommends gcc g++ wget unzip libc6-dev python3.12 python3.12-venv python3.12-dev && \\\n rm -rf /var/lib/apt/lists/* \n\n# Verify Python installation and setup symlink\nRUN if [ -f /usr/bin/python3.12 ]; then \\\n ln -s /usr/bin/python3.12 /usr/bin/python; \\\n else \\\n echo \"Python 3.12 not found\"; exit 1; \\\n fi\n\n# Download and unzip the model\nRUN wget https://d3dg1063dc54p9.cloudfront.net/models/embeddings/mpnet-base-v2.zip && \\\n unzip mpnet-base-v2.zip -d models && \\\n rm mpnet-base-v2.zip\n\n# Install Rust\nRUN wget -q -O - https://sh.rustup.rs | sh -s -- -y\n\n# Clean up to reduce container size\nRUN apt-get remove --purge -y wget unzip && apt-get autoremove -y && rm -rf /var/lib/apt/lists/*\n\n# Copy requirements.txt\nCOPY requirements.txt .\n\n# Setup Python virtual environment\nRUN python3.12 -m venv /venv\n\n# Activate virtual environment and install Python packages\nENV PATH=\"/venv/bin:$PATH\"\n\n# Install Python packages\nRUN pip install --no-cache-dir --upgrade pip && \\\n pip install --no-cache-dir tiktoken && \\\n pip install --no-cache-dir -r requirements.txt\n\n# Final Stage\nFROM ubuntu:24.04 as final\n\nRUN apt-get update && \\\n apt-get install -y software-properties-common && \\\n add-apt-repository ppa:deadsnakes/ppa && \\\n apt-get update && apt-get install -y --no-install-recommends \\\n python3.12 \\\n libgl1 \\\n libglib2.0-0 \\\n poppler-utils \\\n && \\\n ln -s /usr/bin/python3.12 /usr/bin/python && \\\n rm -rf /var/lib/apt/lists/*\n\n# Set working directory\nWORKDIR /app\n\n# Create a non-root user: `appuser` (Feel free to choose a name)\nRUN groupadd -r appuser && \\\n useradd -r -g appuser -d /app -s /sbin/nologin -c \"Docker image user\" appuser\n\n# Copy the virtual environment and model from the builder stage\nCOPY --from=builder /venv /venv\n\nCOPY --from=builder /models /app/models\n\n# Copy your application code\nCOPY . /app/application\n\n# Change the ownership of the /app directory to the appuser\n\t\nRUN mkdir -p /app/application/inputs/local\nRUN chown -R appuser:appuser /app\n\n# Set environment variables\nENV FLASK_APP=app.py \\\n FLASK_DEBUG=true \\\n PATH=\"/venv/bin:$PATH\"\n\n# Expose the port the app runs on\nEXPOSE 7091\n\n# Switch to non-root user\nUSER appuser\n\n# Start Gunicorn\nCMD [\"gunicorn\", \"-w\", \"1\", \"--timeout\", \"120\", \"--bind\", \"0.0.0.0:7091\", \"--preload\", \"application.wsgi:app\"]\n" }, { "path": "application/__init__.py", "content": "" }, { "path": "application/agents/__init__.py", "content": "" }, { "path": "application/agents/agent_creator.py", "content": "import logging\n\nfrom application.agents.classic_agent import ClassicAgent\nfrom application.agents.react_agent import ReActAgent\nfrom application.agents.workflow_agent import WorkflowAgent\n\nlogger = logging.getLogger(__name__)\n\n\nclass AgentCreator:\n agents = {\n \"classic\": ClassicAgent,\n \"react\": ReActAgent,\n \"workflow\": WorkflowAgent,\n }\n\n @classmethod\n def create_agent(cls, type, *args, **kwargs):\n agent_class = cls.agents.get(type.lower())\n if not agent_class:\n raise ValueError(f\"No agent class found for type {type}\")\n return agent_class(*args, **kwargs)\n" }, { "path": "application/agents/base.py", "content": "import logging\nimport uuid\nfrom abc import ABC, abstractmethod\nfrom typing import Dict, Generator, List, Optional\n\nfrom bson.objectid import ObjectId\n\nfrom application.agents.tools.tool_action_parser import ToolActionParser\nfrom application.agents.tools.tool_manager import ToolManager\nfrom application.core.json_schema_utils import (\n JsonSchemaValidationError,\n normalize_json_schema_payload,\n)\nfrom application.core.mongo_db import MongoDB\nfrom application.core.settings import settings\nfrom application.llm.handlers.handler_creator import LLMHandlerCreator\nfrom application.llm.llm_creator import LLMCreator\nfrom application.logging import build_stack_data, log_activity, LogContext\nfrom application.security.encryption import decrypt_credentials\n\nlogger = logging.getLogger(__name__)\n\n\nclass BaseAgent(ABC):\n def __init__(\n self,\n endpoint: str,\n llm_name: str,\n model_id: str,\n api_key: str,\n agent_id: Optional[str] = None,\n user_api_key: Optional[str] = None,\n prompt: str = \"\",\n chat_history: Optional[List[Dict]] = None,\n retrieved_docs: Optional[List[Dict]] = None,\n decoded_token: Optional[Dict] = None,\n attachments: Optional[List[Dict]] = None,\n json_schema: Optional[Dict] = None,\n limited_token_mode: Optional[bool] = False,\n token_limit: Optional[int] = settings.DEFAULT_AGENT_LIMITS[\"token_limit\"],\n limited_request_mode: Optional[bool] = False,\n request_limit: Optional[int] = settings.DEFAULT_AGENT_LIMITS[\"request_limit\"],\n compressed_summary: Optional[str] = None,\n ):\n self.endpoint = endpoint\n self.llm_name = llm_name\n self.model_id = model_id\n self.api_key = api_key\n self.agent_id = agent_id\n self.user_api_key = user_api_key\n self.prompt = prompt\n self.decoded_token = decoded_token or {}\n self.user: str = self.decoded_token.get(\"sub\")\n self.tool_config: Dict = {}\n self.tools: List[Dict] = []\n self.tool_calls: List[Dict] = []\n self.chat_history: List[Dict] = chat_history if chat_history is not None else []\n self.llm = LLMCreator.create_llm(\n llm_name,\n api_key=api_key,\n user_api_key=user_api_key,\n decoded_token=decoded_token,\n model_id=model_id,\n agent_id=agent_id,\n )\n self.retrieved_docs = retrieved_docs or []\n self.llm_handler = LLMHandlerCreator.create_handler(\n llm_name if llm_name else \"default\"\n )\n self.attachments = attachments or []\n self.json_schema = None\n if json_schema is not None:\n try:\n self.json_schema = normalize_json_schema_payload(json_schema)\n except JsonSchemaValidationError as exc:\n logger.warning(\"Ignoring invalid JSON schema payload: %s\", exc)\n self.limited_token_mode = limited_token_mode\n self.token_limit = token_limit\n self.limited_request_mode = limited_request_mode\n self.request_limit = request_limit\n self.compressed_summary = compressed_summary\n self.current_token_count = 0\n self.context_limit_reached = False\n\n @log_activity()\n def gen(\n self, query: str, log_context: LogContext = None\n ) -> Generator[Dict, None, None]:\n yield from self._gen_inner(query, log_context)\n\n @abstractmethod\n def _gen_inner(\n self, query: str, log_context: LogContext\n ) -> Generator[Dict, None, None]:\n pass\n\n def _get_tools(self, api_key: str = None) -> Dict[str, Dict]:\n mongo = MongoDB.get_client()\n db = mongo[settings.MONGO_DB_NAME]\n agents_collection = db[\"agents\"]\n tools_collection = db[\"user_tools\"]\n\n agent_data = agents_collection.find_one({\"key\": api_key or self.user_api_key})\n tool_ids = agent_data.get(\"tools\", []) if agent_data else []\n\n tools = (\n tools_collection.find(\n {\"_id\": {\"$in\": [ObjectId(tool_id) for tool_id in tool_ids]}}\n )\n if tool_ids\n else []\n )\n tools = list(tools)\n tools_by_id = {str(tool[\"_id\"]): tool for tool in tools} if tools else {}\n\n return tools_by_id\n\n def _get_user_tools(self, user=\"local\"):\n mongo = MongoDB.get_client()\n db = mongo[settings.MONGO_DB_NAME]\n user_tools_collection = db[\"user_tools\"]\n user_tools = user_tools_collection.find({\"user\": user, \"status\": True})\n user_tools = list(user_tools)\n\n return {str(i): tool for i, tool in enumerate(user_tools)}\n\n def _build_tool_parameters(self, action):\n params = {\"type\": \"object\", \"properties\": {}, \"required\": []}\n for param_type in [\"query_params\", \"headers\", \"body\", \"parameters\"]:\n if param_type in action and action[param_type].get(\"properties\"):\n for k, v in action[param_type][\"properties\"].items():\n if v.get(\"filled_by_llm\", True):\n params[\"properties\"][k] = {\n key: value\n for key, value in v.items()\n if key not in (\"filled_by_llm\", \"value\", \"required\")\n }\n if v.get(\"required\", False):\n params[\"required\"].append(k)\n return params\n\n def _prepare_tools(self, tools_dict):\n self.tools = [\n {\n \"type\": \"function\",\n \"function\": {\n \"name\": f\"{action['name']}_{tool_id}\",\n \"description\": action[\"description\"],\n \"parameters\": self._build_tool_parameters(action),\n },\n }\n for tool_id, tool in tools_dict.items()\n if (\n (tool[\"name\"] == \"api_tool\" and \"actions\" in tool.get(\"config\", {}))\n or (tool[\"name\"] != \"api_tool\" and \"actions\" in tool)\n )\n for action in (\n tool[\"config\"][\"actions\"].values()\n if tool[\"name\"] == \"api_tool\"\n else tool[\"actions\"]\n )\n if action.get(\"active\", True)\n ]\n\n def _execute_tool_action(self, tools_dict, call):\n parser = ToolActionParser(self.llm.__class__.__name__)\n tool_id, action_name, call_args = parser.parse_args(call)\n\n call_id = getattr(call, \"id\", None) or str(uuid.uuid4())\n\n # Check if parsing failed\n\n if tool_id is None or action_name is None:\n error_message = f\"Error: Failed to parse LLM tool call. Tool name: {getattr(call, 'name', 'unknown')}\"\n logger.error(error_message)\n\n tool_call_data = {\n \"tool_name\": \"unknown\",\n \"call_id\": call_id,\n \"action_name\": getattr(call, \"name\", \"unknown\"),\n \"arguments\": call_args or {},\n \"result\": f\"Failed to parse tool call. Invalid tool name format: {getattr(call, 'name', 'unknown')}\",\n }\n yield {\"type\": \"tool_call\", \"data\": {**tool_call_data, \"status\": \"error\"}}\n self.tool_calls.append(tool_call_data)\n return \"Failed to parse tool call.\", call_id\n # Check if tool_id exists in available tools\n\n if tool_id not in tools_dict:\n error_message = f\"Error: Tool ID '{tool_id}' extracted from LLM call not found in available tools_dict. Available IDs: {list(tools_dict.keys())}\"\n logger.error(error_message)\n\n # Return error result\n\n tool_call_data = {\n \"tool_name\": \"unknown\",\n \"call_id\": call_id,\n \"action_name\": f\"{action_name}_{tool_id}\",\n \"arguments\": call_args,\n \"result\": f\"Tool with ID {tool_id} not found. Available tools: {list(tools_dict.keys())}\",\n }\n yield {\"type\": \"tool_call\", \"data\": {**tool_call_data, \"status\": \"error\"}}\n self.tool_calls.append(tool_call_data)\n return f\"Tool with ID {tool_id} not found.\", call_id\n tool_call_data = {\n \"tool_name\": tools_dict[tool_id][\"name\"],\n \"call_id\": call_id,\n \"action_name\": f\"{action_name}_{tool_id}\",\n \"arguments\": call_args,\n }\n yield {\"type\": \"tool_call\", \"data\": {**tool_call_data, \"status\": \"pending\"}}\n\n tool_data = tools_dict[tool_id]\n action_data = (\n tool_data[\"config\"][\"actions\"][action_name]\n if tool_data[\"name\"] == \"api_tool\"\n else next(\n action\n for action in tool_data[\"actions\"]\n if action[\"name\"] == action_name\n )\n )\n\n query_params, headers, body, parameters = {}, {}, {}, {}\n param_types = {\n \"query_params\": query_params,\n \"headers\": headers,\n \"body\": body,\n \"parameters\": parameters,\n }\n\n for param_type, target_dict in param_types.items():\n if param_type in action_data and action_data[param_type].get(\"properties\"):\n for param, details in action_data[param_type][\"properties\"].items():\n if (\n param not in call_args\n and \"value\" in details\n and details[\"value\"]\n ):\n target_dict[param] = details[\"value\"]\n for param, value in call_args.items():\n for param_type, target_dict in param_types.items():\n if param_type in action_data and param in action_data[param_type].get(\n \"properties\", {}\n ):\n target_dict[param] = value\n tm = ToolManager(config={})\n\n # Prepare tool_config and add tool_id for memory tools\n\n if tool_data[\"name\"] == \"api_tool\":\n action_config = tool_data[\"config\"][\"actions\"][action_name]\n tool_config = {\n \"url\": action_config[\"url\"],\n \"method\": action_config[\"method\"],\n \"headers\": headers,\n \"query_params\": query_params,\n }\n if \"body_content_type\" in action_config:\n tool_config[\"body_content_type\"] = action_config.get(\n \"body_content_type\", \"application/json\"\n )\n tool_config[\"body_encoding_rules\"] = action_config.get(\n \"body_encoding_rules\", {}\n )\n else:\n tool_config = tool_data[\"config\"].copy() if tool_data[\"config\"] else {}\n if tool_config.get(\"encrypted_credentials\") and self.user:\n decrypted = decrypt_credentials(\n tool_config[\"encrypted_credentials\"], self.user\n )\n tool_config.update(decrypted)\n tool_config[\"auth_credentials\"] = decrypted\n tool_config.pop(\"encrypted_credentials\", None)\n tool_config[\"tool_id\"] = str(tool_data.get(\"_id\", tool_id))\n if hasattr(self, \"conversation_id\") and self.conversation_id:\n tool_config[\"conversation_id\"] = self.conversation_id\n if tool_data[\"name\"] == \"mcp_tool\":\n tool_config[\"query_mode\"] = True\n tool = tm.load_tool(\n tool_data[\"name\"],\n tool_config=tool_config,\n user_id=self.user,\n )\n resolved_arguments = (\n {\"query_params\": query_params, \"headers\": headers, \"body\": body}\n if tool_data[\"name\"] == \"api_tool\"\n else parameters\n )\n if tool_data[\"name\"] == \"api_tool\":\n logger.debug(\n f\"Executing api: {action_name} with query_params: {query_params}, headers: {headers}, body: {body}\"\n )\n result = tool.execute_action(action_name, **body)\n else:\n logger.debug(f\"Executing tool: {action_name} with args: {call_args}\")\n result = tool.execute_action(action_name, **parameters)\n\n get_artifact_id = (\n getattr(tool, \"get_artifact_id\", None)\n if tool_data[\"name\"] != \"api_tool\"\n else None\n )\n\n artifact_id = None\n if callable(get_artifact_id):\n try:\n artifact_id = get_artifact_id(action_name, **parameters)\n except Exception:\n logger.exception(\n \"Failed to extract artifact_id from tool %s for action %s\",\n tool_data[\"name\"],\n action_name,\n )\n\n artifact_id = str(artifact_id).strip() if artifact_id is not None else \"\"\n if artifact_id:\n tool_call_data[\"artifact_id\"] = artifact_id\n result_full = str(result)\n tool_call_data[\"resolved_arguments\"] = resolved_arguments\n tool_call_data[\"result_full\"] = result_full\n tool_call_data[\"result\"] = (\n f\"{result_full[:50]}...\" if len(result_full) > 50 else result_full\n )\n\n stream_tool_call_data = {\n key: value\n for key, value in tool_call_data.items()\n if key not in {\"result_full\", \"resolved_arguments\"}\n }\n yield {\"type\": \"tool_call\", \"data\": {**stream_tool_call_data, \"status\": \"completed\"}}\n self.tool_calls.append(tool_call_data)\n\n return result, call_id\n\n def _get_truncated_tool_calls(self):\n return [\n {\n \"tool_name\": tool_call.get(\"tool_name\"),\n \"call_id\": tool_call.get(\"call_id\"),\n \"action_name\": tool_call.get(\"action_name\"),\n \"arguments\": tool_call.get(\"arguments\"),\n \"artifact_id\": tool_call.get(\"artifact_id\"),\n \"result\": (\n f\"{str(tool_call['result'])[:50]}...\"\n if len(str(tool_call[\"result\"])) > 50\n else tool_call[\"result\"]\n ),\n \"status\": \"completed\",\n }\n for tool_call in self.tool_calls\n ]\n\n def _calculate_current_context_tokens(self, messages: List[Dict]) -> int:\n \"\"\"\n Calculate total tokens in current context (messages).\n\n Args:\n messages: List of message dicts\n\n Returns:\n Total token count\n \"\"\"\n from application.api.answer.services.compression.token_counter import (\n TokenCounter,\n )\n\n return TokenCounter.count_message_tokens(messages)\n\n def _check_context_limit(self, messages: List[Dict]) -> bool:\n \"\"\"\n Check if we're approaching context limit (80%).\n\n Args:\n messages: Current message list\n\n Returns:\n True if at or above 80% of context limit\n \"\"\"\n from application.core.model_utils import get_token_limit\n from application.core.settings import settings\n\n try:\n # Calculate current tokens\n current_tokens = self._calculate_current_context_tokens(messages)\n self.current_token_count = current_tokens\n\n # Get context limit for model\n context_limit = get_token_limit(self.model_id)\n\n # Calculate threshold (80%)\n threshold = int(context_limit * settings.COMPRESSION_THRESHOLD_PERCENTAGE)\n\n # Check if we've reached the limit\n if current_tokens >= threshold:\n logger.warning(\n f\"Context limit approaching: {current_tokens}/{context_limit} tokens \"\n f\"({(current_tokens/context_limit)*100:.1f}%)\"\n )\n return True\n\n return False\n\n except Exception as e:\n logger.error(f\"Error checking context limit: {str(e)}\", exc_info=True)\n return False\n\n def _validate_context_size(self, messages: List[Dict]) -> None:\n \"\"\"\n Pre-flight validation before calling LLM. Logs warnings but never raises errors.\n\n Args:\n messages: Messages to be sent to LLM\n \"\"\"\n from application.core.model_utils import get_token_limit\n\n current_tokens = self._calculate_current_context_tokens(messages)\n self.current_token_count = current_tokens\n context_limit = get_token_limit(self.model_id)\n\n percentage = (current_tokens / context_limit) * 100\n\n # Log based on usage level\n if current_tokens >= context_limit:\n logger.warning(\n f\"Context at limit: {current_tokens:,}/{context_limit:,} tokens \"\n f\"({percentage:.1f}%). Model: {self.model_id}\"\n )\n elif current_tokens >= int(\n context_limit * settings.COMPRESSION_THRESHOLD_PERCENTAGE\n ):\n logger.info(\n f\"Context approaching limit: {current_tokens:,}/{context_limit:,} tokens \"\n f\"({percentage:.1f}%)\"\n )\n\n def _truncate_text_middle(self, text: str, max_tokens: int) -> str:\n \"\"\"\n Truncate text by removing content from the middle, preserving start and end.\n\n Args:\n text: Text to truncate\n max_tokens: Maximum tokens allowed\n\n Returns:\n Truncated text with middle removed if needed\n \"\"\"\n from application.utils import num_tokens_from_string\n\n current_tokens = num_tokens_from_string(text)\n if current_tokens <= max_tokens:\n return text\n\n # Estimate chars per token (roughly 4 chars per token for English)\n chars_per_token = len(text) / current_tokens if current_tokens > 0 else 4\n target_chars = int(max_tokens * chars_per_token * 0.95) # 5% safety margin\n\n if target_chars <= 0:\n return \"\"\n\n # Split: keep 40% from start, 40% from end, remove middle\n start_chars = int(target_chars * 0.4)\n end_chars = int(target_chars * 0.4)\n\n truncation_marker = \"\\n\\n[... content truncated to fit context limit ...]\\n\\n\"\n\n truncated = text[:start_chars] + truncation_marker + text[-end_chars:]\n\n logger.info(\n f\"Truncated text from {current_tokens:,} to ~{max_tokens:,} tokens \"\n f\"(removed middle section)\"\n )\n\n return truncated\n\n def _build_messages(\n self,\n system_prompt: str,\n query: str,\n ) -> List[Dict]:\n \"\"\"Build messages using pre-rendered system prompt\"\"\"\n from application.core.model_utils import get_token_limit\n from application.utils import num_tokens_from_string\n\n # Append compression summary to system prompt if present\n if self.compressed_summary:\n compression_context = (\n \"\\n\\n---\\n\\n\"\n \"This session is being continued from a previous conversation that \"\n \"has been compressed to fit within context limits. \"\n \"The conversation is summarized below:\\n\\n\"\n f\"{self.compressed_summary}\"\n )\n system_prompt = system_prompt + compression_context\n\n context_limit = get_token_limit(self.model_id)\n system_tokens = num_tokens_from_string(system_prompt)\n\n # Reserve 10% for response/tools\n safety_buffer = int(context_limit * 0.1)\n available_after_system = context_limit - system_tokens - safety_buffer\n\n # Max tokens for query: 80% of available space (leave room for history)\n max_query_tokens = int(available_after_system * 0.8)\n query_tokens = num_tokens_from_string(query)\n\n # Truncate query from middle if it exceeds 80% of available context\n if query_tokens > max_query_tokens:\n query = self._truncate_text_middle(query, max_query_tokens)\n query_tokens = num_tokens_from_string(query)\n\n # Calculate remaining budget for chat history\n available_for_history = max(available_after_system - query_tokens, 0)\n\n # Truncate chat history to fit within available budget\n working_history = self._truncate_history_to_fit(\n self.chat_history,\n available_for_history,\n )\n\n messages = [{\"role\": \"system\", \"content\": system_prompt}]\n\n for i in working_history:\n if \"prompt\" in i and \"response\" in i:\n messages.append({\"role\": \"user\", \"content\": i[\"prompt\"]})\n messages.append({\"role\": \"assistant\", \"content\": i[\"response\"]})\n if \"tool_calls\" in i:\n for tool_call in i[\"tool_calls\"]:\n call_id = tool_call.get(\"call_id\") or str(uuid.uuid4())\n\n function_call_dict = {\n \"function_call\": {\n \"name\": tool_call.get(\"action_name\"),\n \"args\": tool_call.get(\"arguments\"),\n \"call_id\": call_id,\n }\n }\n function_response_dict = {\n \"function_response\": {\n \"name\": tool_call.get(\"action_name\"),\n \"response\": {\"result\": tool_call.get(\"result\")},\n \"call_id\": call_id,\n }\n }\n\n messages.append(\n {\"role\": \"assistant\", \"content\": [function_call_dict]}\n )\n messages.append(\n {\"role\": \"tool\", \"content\": [function_response_dict]}\n )\n messages.append({\"role\": \"user\", \"content\": query})\n return messages\n\n def _truncate_history_to_fit(\n self,\n history: List[Dict],\n max_tokens: int,\n ) -> List[Dict]:\n \"\"\"\n Truncate chat history to fit within token budget, keeping most recent messages.\n\n Args:\n history: Full chat history\n max_tokens: Maximum tokens allowed for history\n\n Returns:\n Truncated history (most recent messages that fit)\n \"\"\"\n from application.utils import num_tokens_from_string\n\n if not history or max_tokens <= 0:\n return []\n\n truncated = []\n current_tokens = 0\n\n # Iterate from newest to oldest\n for message in reversed(history):\n message_tokens = 0\n\n if \"prompt\" in message and \"response\" in message:\n message_tokens += num_tokens_from_string(message[\"prompt\"])\n message_tokens += num_tokens_from_string(message[\"response\"])\n\n if \"tool_calls\" in message:\n for tool_call in message[\"tool_calls\"]:\n tool_str = (\n f\"Tool: {tool_call.get('tool_name')} | \"\n f\"Action: {tool_call.get('action_name')} | \"\n f\"Args: {tool_call.get('arguments')} | \"\n f\"Response: {tool_call.get('result')}\"\n )\n message_tokens += num_tokens_from_string(tool_str)\n\n if current_tokens + message_tokens <= max_tokens:\n current_tokens += message_tokens\n truncated.insert(0, message) # Maintain chronological order\n else:\n break\n\n if len(truncated) < len(history):\n logger.info(\n f\"Truncated chat history from {len(history)} to {len(truncated)} messages \"\n f\"to fit within {max_tokens:,} token budget\"\n )\n\n return truncated\n\n def _llm_gen(self, messages: List[Dict], log_context: Optional[LogContext] = None):\n # Pre-flight context validation - fail fast if over limit\n self._validate_context_size(messages)\n\n gen_kwargs = {\"model\": self.model_id, \"messages\": messages}\n if self.attachments:\n # Usage accounting only; stripped before provider invocation.\n gen_kwargs[\"_usage_attachments\"] = self.attachments\n\n if (\n hasattr(self.llm, \"_supports_tools\")\n and self.llm._supports_tools\n and self.tools\n ):\n gen_kwargs[\"tools\"] = self.tools\n if (\n self.json_schema\n and hasattr(self.llm, \"_supports_structured_output\")\n and self.llm._supports_structured_output()\n ):\n structured_format = self.llm.prepare_structured_output_format(\n self.json_schema\n )\n if structured_format:\n if self.llm_name == \"openai\":\n gen_kwargs[\"response_format\"] = structured_format\n elif self.llm_name == \"google\":\n gen_kwargs[\"response_schema\"] = structured_format\n resp = self.llm.gen_stream(**gen_kwargs)\n\n if log_context:\n data = build_stack_data(self.llm, exclude_attributes=[\"client\"])\n log_context.stacks.append({\"component\": \"llm\", \"data\": data})\n return resp\n\n def _llm_handler(\n self,\n resp,\n tools_dict: Dict,\n messages: List[Dict],\n log_context: Optional[LogContext] = None,\n attachments: Optional[List[Dict]] = None,\n ):\n resp = self.llm_handler.process_message_flow(\n self, resp, tools_dict, messages, attachments, True\n )\n if log_context:\n data = build_stack_data(self.llm_handler, exclude_attributes=[\"tool_calls\"])\n log_context.stacks.append({\"component\": \"llm_handler\", \"data\": data})\n return resp\n\n def _handle_response(self, response, tools_dict, messages, log_context):\n is_structured_output = (\n self.json_schema is not None\n and hasattr(self.llm, \"_supports_structured_output\")\n and self.llm._supports_structured_output()\n )\n\n if isinstance(response, str):\n answer_data = {\"answer\": response}\n if is_structured_output:\n answer_data[\"structured\"] = True\n answer_data[\"schema\"] = self.json_schema\n yield answer_data\n return\n if hasattr(response, \"message\") and getattr(response.message, \"content\", None):\n answer_data = {\"answer\": response.message.content}\n if is_structured_output:\n answer_data[\"structured\"] = True\n answer_data[\"schema\"] = self.json_schema\n yield answer_data\n return\n processed_response_gen = self._llm_handler(\n response, tools_dict, messages, log_context, self.attachments\n )\n\n for event in processed_response_gen:\n if isinstance(event, str):\n answer_data = {\"answer\": event}\n if is_structured_output:\n answer_data[\"structured\"] = True\n answer_data[\"schema\"] = self.json_schema\n yield answer_data\n elif hasattr(event, \"message\") and getattr(event.message, \"content\", None):\n answer_data = {\"answer\": event.message.content}\n if is_structured_output:\n answer_data[\"structured\"] = True\n answer_data[\"schema\"] = self.json_schema\n yield answer_data\n elif isinstance(event, dict) and \"type\" in event:\n yield event\n" }, { "path": "application/agents/classic_agent.py", "content": "import logging\nfrom typing import Dict, Generator\n\nfrom application.agents.base import BaseAgent\nfrom application.logging import LogContext\n\nlogger = logging.getLogger(__name__)\n\n\nclass ClassicAgent(BaseAgent):\n \"\"\"A simplified agent with clear execution flow\"\"\"\n\n def _gen_inner(\n self, query: str, log_context: LogContext\n ) -> Generator[Dict, None, None]:\n \"\"\"Core generator function for ClassicAgent execution flow\"\"\"\n\n tools_dict = (\n self._get_user_tools(self.user)\n if not self.user_api_key\n else self._get_tools(self.user_api_key)\n )\n self._prepare_tools(tools_dict)\n\n messages = self._build_messages(self.prompt, query)\n llm_response = self._llm_gen(messages, log_context)\n\n yield from self._handle_response(\n llm_response, tools_dict, messages, log_context\n )\n\n yield {\"sources\": self.retrieved_docs}\n yield {\"tool_calls\": self._get_truncated_tool_calls()}\n\n log_context.stacks.append(\n {\"component\": \"agent\", \"data\": {\"tool_calls\": self.tool_calls.copy()}}\n )\n" }, { "path": "application/agents/react_agent.py", "content": "import logging\nimport os\nfrom typing import Any, Dict, Generator, List\n\nfrom application.agents.base import BaseAgent\nfrom application.logging import build_stack_data, LogContext\n\nlogger = logging.getLogger(__name__)\n\nMAX_ITERATIONS_REASONING = 10\n\ncurrent_dir = os.path.dirname(\n os.path.dirname(os.path.dirname(os.path.abspath(__file__)))\n)\nwith open(\n os.path.join(current_dir, \"application/prompts\", \"react_planning_prompt.txt\"), \"r\"\n) as f:\n PLANNING_PROMPT_TEMPLATE = f.read()\nwith open(\n os.path.join(current_dir, \"application/prompts\", \"react_final_prompt.txt\"), \"r\"\n) as f:\n FINAL_PROMPT_TEMPLATE = f.read()\n\n\nclass ReActAgent(BaseAgent):\n \"\"\"\n Research and Action (ReAct) Agent - Advanced reasoning agent with iterative planning.\n\n Implements a think-act-observe loop for complex problem-solving:\n 1. Creates a strategic plan based on the query\n 2. Executes tools and gathers observations\n 3. Iteratively refines approach until satisfied\n 4. Synthesizes final answer from all observations\n \"\"\"\n\n def __init__(self, *args, **kwargs):\n super().__init__(*args, **kwargs)\n self.plan: str = \"\"\n self.observations: List[str] = []\n\n def _gen_inner(\n self, query: str, log_context: LogContext\n ) -> Generator[Dict, None, None]:\n \"\"\"Execute ReAct reasoning loop with planning, action, and observation cycles\"\"\"\n\n self._reset_state()\n\n tools_dict = (\n self._get_tools(self.user_api_key)\n if self.user_api_key\n else self._get_user_tools(self.user)\n )\n self._prepare_tools(tools_dict)\n\n for iteration in range(1, MAX_ITERATIONS_REASONING + 1):\n yield {\"thought\": f\"Reasoning... (iteration {iteration})\\n\\n\"}\n\n yield from self._planning_phase(query, log_context)\n\n if not self.plan:\n logger.warning(\n f\"ReActAgent: No plan generated in iteration {iteration}\"\n )\n break\n self.observations.append(f\"Plan (iteration {iteration}): {self.plan}\")\n\n satisfied = yield from self._execution_phase(query, tools_dict, log_context)\n\n if satisfied:\n logger.info(\"ReActAgent: Goal satisfied, stopping reasoning loop\")\n break\n yield from self._synthesis_phase(query, log_context)\n\n def _reset_state(self):\n \"\"\"Reset agent state for new query\"\"\"\n self.plan = \"\"\n self.observations = []\n\n def _planning_phase(\n self, query: str, log_context: LogContext\n ) -> Generator[Dict, None, None]:\n \"\"\"Generate strategic plan for query\"\"\"\n logger.info(\"ReActAgent: Creating plan...\")\n\n plan_prompt = self._build_planning_prompt(query)\n messages = [{\"role\": \"user\", \"content\": plan_prompt}]\n\n plan_stream = self.llm.gen_stream(\n model=self.model_id,\n messages=messages,\n tools=self.tools if self.tools else None,\n )\n\n if log_context:\n log_context.stacks.append(\n {\"component\": \"planning_llm\", \"data\": build_stack_data(self.llm)}\n )\n plan_parts = []\n for chunk in plan_stream:\n content = self._extract_content(chunk)\n if content:\n plan_parts.append(content)\n yield {\"thought\": content}\n self.plan = \"\".join(plan_parts)\n\n def _execution_phase(\n self, query: str, tools_dict: Dict, log_context: LogContext\n ) -> Generator[bool, None, None]:\n \"\"\"Execute plan with tool calls and observations\"\"\"\n execution_prompt = self._build_execution_prompt(query)\n messages = self._build_messages(execution_prompt, query)\n\n llm_response = self._llm_gen(messages, log_context)\n initial_content = self._extract_content(llm_response)\n\n if initial_content:\n self.observations.append(f\"Initial response: {initial_content}\")\n processed_response = self._llm_handler(\n llm_response, tools_dict, messages, log_context\n )\n\n for tool_call in self.tool_calls:\n observation = (\n f\"Executed: {tool_call.get('tool_name', 'Unknown')} \"\n f\"with args {tool_call.get('arguments', {})}. \"\n f\"Result: {str(tool_call.get('result', ''))[:200]}\"\n )\n self.observations.append(observation)\n final_content = self._extract_content(processed_response)\n if final_content:\n self.observations.append(f\"Response after tools: {final_content}\")\n if log_context:\n log_context.stacks.append(\n {\n \"component\": \"agent_tool_calls\",\n \"data\": {\"tool_calls\": self.tool_calls.copy()},\n }\n )\n yield {\"sources\": self.retrieved_docs}\n yield {\"tool_calls\": self._get_truncated_tool_calls()}\n\n return \"SATISFIED\" in (final_content or \"\")\n\n def _synthesis_phase(\n self, query: str, log_context: LogContext\n ) -> Generator[Dict, None, None]:\n \"\"\"Synthesize final answer from all observations\"\"\"\n logger.info(\"ReActAgent: Generating final answer...\")\n\n final_prompt = self._build_final_answer_prompt(query)\n messages = [{\"role\": \"user\", \"content\": final_prompt}]\n\n final_stream = self.llm.gen_stream(\n model=self.model_id, messages=messages, tools=None\n )\n\n if log_context:\n log_context.stacks.append(\n {\"component\": \"final_answer_llm\", \"data\": build_stack_data(self.llm)}\n )\n for chunk in final_stream:\n content = self._extract_content(chunk)\n if content:\n yield {\"answer\": content}\n\n def _build_planning_prompt(self, query: str) -> str:\n \"\"\"Build planning phase prompt\"\"\"\n prompt = PLANNING_PROMPT_TEMPLATE.replace(\"{query}\", query)\n prompt = prompt.replace(\"{prompt}\", self.prompt or \"\")\n prompt = prompt.replace(\"{summaries}\", \"\")\n prompt = prompt.replace(\"{observations}\", \"\\n\".join(self.observations))\n return prompt\n\n def _build_execution_prompt(self, query: str) -> str:\n \"\"\"Build execution phase prompt with plan and observations\"\"\"\n observations_str = \"\\n\".join(self.observations)\n\n if len(observations_str) > 20000:\n observations_str = observations_str[:20000] + \"\\n...[truncated]\"\n return (\n f\"{self.prompt or ''}\\n\\n\"\n f\"Follow this plan:\\n{self.plan}\\n\\n\"\n f\"Observations:\\n{observations_str}\\n\\n\"\n f\"If sufficient data exists to answer '{query}', respond with 'SATISFIED'. \"\n f\"Otherwise, continue executing the plan.\"\n )\n\n def _build_final_answer_prompt(self, query: str) -> str:\n \"\"\"Build final synthesis prompt\"\"\"\n observations_str = \"\\n\".join(self.observations)\n\n if len(observations_str) > 10000:\n observations_str = observations_str[:10000] + \"\\n...[truncated]\"\n logger.warning(\"ReActAgent: Observations truncated for final answer\")\n return FINAL_PROMPT_TEMPLATE.format(query=query, observations=observations_str)\n\n def _extract_content(self, response: Any) -> str:\n \"\"\"Extract text content from various LLM response formats\"\"\"\n if not response:\n return \"\"\n collected = []\n\n if isinstance(response, str):\n return response\n if hasattr(response, \"message\") and hasattr(response.message, \"content\"):\n if response.message.content:\n return response.message.content\n if hasattr(response, \"choices\") and response.choices:\n if hasattr(response.choices[0], \"message\"):\n content = response.choices[0].message.content\n if content:\n return content\n if hasattr(response, \"content\") and isinstance(response.content, list):\n if response.content and hasattr(response.content[0], \"text\"):\n return response.content[0].text\n try:\n for chunk in response:\n content_piece = \"\"\n\n if hasattr(chunk, \"choices\") and chunk.choices:\n if hasattr(chunk.choices[0], \"delta\"):\n delta_content = chunk.choices[0].delta.content\n if delta_content:\n content_piece = delta_content\n elif hasattr(chunk, \"type\") and chunk.type == \"content_block_delta\":\n if hasattr(chunk, \"delta\") and hasattr(chunk.delta, \"text\"):\n content_piece = chunk.delta.text\n elif isinstance(chunk, str):\n content_piece = chunk\n if content_piece:\n collected.append(content_piece)\n except (TypeError, AttributeError):\n logger.debug(\n f\"Response not iterable or unexpected format: {type(response)}\"\n )\n except Exception as e:\n logger.error(f\"Error extracting content: {e}\")\n return \"\".join(collected)" }, { "path": "application/agents/tools/api_body_serializer.py", "content": "import base64\nimport json\nimport logging\nfrom enum import Enum\nfrom typing import Any, Dict, Optional, Union\nfrom urllib.parse import quote, urlencode\n\nlogger = logging.getLogger(__name__)\n\n\nclass ContentType(str, Enum):\n \"\"\"Supported content types for request bodies.\"\"\"\n\n JSON = \"application/json\"\n FORM_URLENCODED = \"application/x-www-form-urlencoded\"\n MULTIPART_FORM_DATA = \"multipart/form-data\"\n TEXT_PLAIN = \"text/plain\"\n XML = \"application/xml\"\n OCTET_STREAM = \"application/octet-stream\"\n\n\nclass RequestBodySerializer:\n \"\"\"Serializes request bodies according to content-type and OpenAPI 3.1 spec.\"\"\"\n\n @staticmethod\n def serialize(\n body_data: Dict[str, Any],\n content_type: str = ContentType.JSON,\n encoding_rules: Optional[Dict[str, Dict[str, Any]]] = None,\n ) -> tuple[Union[str, bytes], Dict[str, str]]:\n \"\"\"\n Serialize body data to appropriate format.\n\n Args:\n body_data: Dictionary of body parameters\n content_type: Content-Type header value\n encoding_rules: OpenAPI Encoding Object rules per field\n\n Returns:\n Tuple of (serialized_body, updated_headers_dict)\n\n Raises:\n ValueError: If serialization fails\n \"\"\"\n if not body_data:\n return None, {}\n\n try:\n content_type_lower = content_type.lower().split(\";\")[0].strip()\n\n if content_type_lower == ContentType.JSON:\n return RequestBodySerializer._serialize_json(body_data)\n\n elif content_type_lower == ContentType.FORM_URLENCODED:\n return RequestBodySerializer._serialize_form_urlencoded(\n body_data, encoding_rules\n )\n\n elif content_type_lower == ContentType.MULTIPART_FORM_DATA:\n return RequestBodySerializer._serialize_multipart_form_data(\n body_data, encoding_rules\n )\n\n elif content_type_lower == ContentType.TEXT_PLAIN:\n return RequestBodySerializer._serialize_text_plain(body_data)\n\n elif content_type_lower == ContentType.XML:\n return RequestBodySerializer._serialize_xml(body_data)\n\n elif content_type_lower == ContentType.OCTET_STREAM:\n return RequestBodySerializer._serialize_octet_stream(body_data)\n\n else:\n logger.warning(\n f\"Unknown content type: {content_type}, treating as JSON\"\n )\n return RequestBodySerializer._serialize_json(body_data)\n\n except Exception as e:\n logger.error(f\"Error serializing body: {str(e)}\", exc_info=True)\n raise ValueError(f\"Failed to serialize request body: {str(e)}\")\n\n @staticmethod\n def _serialize_json(body_data: Dict[str, Any]) -> tuple[str, Dict[str, str]]:\n \"\"\"Serialize body as JSON per OpenAPI spec.\"\"\"\n try:\n serialized = json.dumps(\n body_data, separators=(\",\", \":\"), ensure_ascii=False\n )\n headers = {\"Content-Type\": ContentType.JSON.value}\n return serialized, headers\n except (TypeError, ValueError) as e:\n raise ValueError(f\"Failed to serialize JSON body: {str(e)}\")\n\n @staticmethod\n def _serialize_form_urlencoded(\n body_data: Dict[str, Any],\n encoding_rules: Optional[Dict[str, Dict[str, Any]]] = None,\n ) -> tuple[str, Dict[str, str]]:\n \"\"\"Serialize body as application/x-www-form-urlencoded per RFC1866/RFC3986.\"\"\"\n encoding_rules = encoding_rules or {}\n params = []\n\n for key, value in body_data.items():\n if value is None:\n continue\n\n rule = encoding_rules.get(key, {})\n style = rule.get(\"style\", \"form\")\n explode = rule.get(\"explode\", style == \"form\")\n content_type = rule.get(\"contentType\", \"text/plain\")\n\n serialized_value = RequestBodySerializer._serialize_form_value(\n value, style, explode, content_type, key\n )\n\n if isinstance(serialized_value, list):\n for sv in serialized_value:\n params.append((key, sv))\n else:\n params.append((key, serialized_value))\n\n # Use standard urlencode (replaces space with +)\n serialized = urlencode(params, safe=\"\")\n headers = {\"Content-Type\": ContentType.FORM_URLENCODED.value}\n return serialized, headers\n\n @staticmethod\n def _serialize_form_value(\n value: Any, style: str, explode: bool, content_type: str, key: str\n ) -> Union[str, list]:\n \"\"\"Serialize individual form value with encoding rules.\"\"\"\n if isinstance(value, dict):\n if content_type == \"application/json\":\n return json.dumps(value, separators=(\",\", \":\"))\n elif content_type == \"application/xml\":\n return RequestBodySerializer._dict_to_xml(value)\n else:\n if style == \"deepObject\" and explode:\n return [\n f\"{RequestBodySerializer._percent_encode(str(v))}\"\n for v in value.values()\n ]\n elif explode:\n return [\n f\"{RequestBodySerializer._percent_encode(str(v))}\"\n for v in value.values()\n ]\n else:\n pairs = [f\"{k},{v}\" for k, v in value.items()]\n return RequestBodySerializer._percent_encode(\",\".join(pairs))\n\n elif isinstance(value, (list, tuple)):\n if explode:\n return [\n RequestBodySerializer._percent_encode(str(item)) for item in value\n ]\n else:\n return RequestBodySerializer._percent_encode(\n \",\".join(str(v) for v in value)\n )\n\n else:\n return RequestBodySerializer._percent_encode(str(value))\n\n @staticmethod\n def _serialize_multipart_form_data(\n body_data: Dict[str, Any],\n encoding_rules: Optional[Dict[str, Dict[str, Any]]] = None,\n ) -> tuple[bytes, Dict[str, str]]:\n \"\"\"\n Serialize body as multipart/form-data per RFC7578.\n\n Supports file uploads and encoding rules.\n \"\"\"\n import secrets\n\n encoding_rules = encoding_rules or {}\n boundary = f\"----DocsGPT{secrets.token_hex(16)}\"\n parts = []\n\n for key, value in body_data.items():\n if value is None:\n continue\n\n rule = encoding_rules.get(key, {})\n content_type = rule.get(\"contentType\", \"text/plain\")\n headers_rule = rule.get(\"headers\", {})\n\n part = RequestBodySerializer._create_multipart_part(\n key, value, content_type, headers_rule\n )\n parts.append(part)\n\n body_bytes = f\"--{boundary}\\r\\n\".encode(\"utf-8\")\n body_bytes += f\"--{boundary}\\r\\n\".join(parts).encode(\"utf-8\")\n body_bytes += f\"\\r\\n--{boundary}--\\r\\n\".encode(\"utf-8\")\n\n headers = {\n \"Content-Type\": f\"multipart/form-data; boundary={boundary}\",\n }\n return body_bytes, headers\n\n @staticmethod\n def _create_multipart_part(\n name: str, value: Any, content_type: str, headers_rule: Dict[str, Any]\n ) -> str:\n \"\"\"Create a single multipart/form-data part.\"\"\"\n headers = [\n f'Content-Disposition: form-data; name=\"{RequestBodySerializer._percent_encode(name)}\"'\n ]\n\n if isinstance(value, bytes):\n if content_type == \"application/octet-stream\":\n value_encoded = base64.b64encode(value).decode(\"utf-8\")\n else:\n value_encoded = value.decode(\"utf-8\", errors=\"replace\")\n headers.append(f\"Content-Type: {content_type}\")\n headers.append(\"Content-Transfer-Encoding: base64\")\n elif isinstance(value, dict):\n if content_type == \"application/json\":\n value_encoded = json.dumps(value, separators=(\",\", \":\"))\n elif content_type == \"application/xml\":\n value_encoded = RequestBodySerializer._dict_to_xml(value)\n else:\n value_encoded = str(value)\n headers.append(f\"Content-Type: {content_type}\")\n elif isinstance(value, str) and content_type != \"text/plain\":\n try:\n if content_type == \"application/json\":\n json.loads(value)\n value_encoded = value\n elif content_type == \"application/xml\":\n value_encoded = value\n else:\n value_encoded = str(value)\n except json.JSONDecodeError:\n value_encoded = str(value)\n headers.append(f\"Content-Type: {content_type}\")\n else:\n value_encoded = str(value)\n if content_type != \"text/plain\":\n headers.append(f\"Content-Type: {content_type}\")\n\n part = \"\\r\\n\".join(headers) + \"\\r\\n\\r\\n\" + value_encoded + \"\\r\\n\"\n return part\n\n @staticmethod\n def _serialize_text_plain(body_data: Dict[str, Any]) -> tuple[str, Dict[str, str]]:\n \"\"\"Serialize body as plain text.\"\"\"\n if len(body_data) == 1:\n value = list(body_data.values())[0]\n return str(value), {\"Content-Type\": ContentType.TEXT_PLAIN.value}\n else:\n text = \"\\n\".join(f\"{k}: {v}\" for k, v in body_data.items())\n return text, {\"Content-Type\": ContentType.TEXT_PLAIN.value}\n\n @staticmethod\n def _serialize_xml(body_data: Dict[str, Any]) -> tuple[str, Dict[str, str]]:\n \"\"\"Serialize body as XML.\"\"\"\n xml_str = RequestBodySerializer._dict_to_xml(body_data)\n return xml_str, {\"Content-Type\": ContentType.XML.value}\n\n @staticmethod\n def _serialize_octet_stream(\n body_data: Dict[str, Any],\n ) -> tuple[bytes, Dict[str, str]]:\n \"\"\"Serialize body as binary octet stream.\"\"\"\n if isinstance(body_data, bytes):\n return body_data, {\"Content-Type\": ContentType.OCTET_STREAM.value}\n elif isinstance(body_data, str):\n return body_data.encode(\"utf-8\"), {\n \"Content-Type\": ContentType.OCTET_STREAM.value\n }\n else:\n serialized = json.dumps(body_data)\n return serialized.encode(\"utf-8\"), {\n \"Content-Type\": ContentType.OCTET_STREAM.value\n }\n\n @staticmethod\n def _percent_encode(value: str, safe_chars: str = \"\") -> str:\n \"\"\"\n Percent-encode per RFC3986.\n\n Args:\n value: String to encode\n safe_chars: Additional characters to not encode\n \"\"\"\n return quote(value, safe=safe_chars)\n\n @staticmethod\n def _dict_to_xml(data: Dict[str, Any], root_name: str = \"root\") -> str:\n \"\"\"\n Convert dict to simple XML format.\n \"\"\"\n\n def build_xml(obj: Any, name: str) -> str:\n if isinstance(obj, dict):\n inner = \"\".join(build_xml(v, k) for k, v in obj.items())\n return f\"<{name}>{inner}\"\n elif isinstance(obj, (list, tuple)):\n items = \"\".join(\n build_xml(item, f\"{name[:-1] if name.endswith('s') else name}\")\n for item in obj\n )\n return items\n else:\n return f\"<{name}>{RequestBodySerializer._escape_xml(str(obj))}\"\n\n root = build_xml(data, root_name)\n return f'{root}'\n\n @staticmethod\n def _escape_xml(value: str) -> str:\n \"\"\"Escape XML special characters.\"\"\"\n return (\n value.replace(\"&\", \"&\")\n .replace(\"<\", \"<\")\n .replace(\">\", \">\")\n .replace('\"', \""\")\n .replace(\"'\", \"'\")\n )\n" }, { "path": "application/agents/tools/api_tool.py", "content": "import json\nimport logging\nimport re\nfrom typing import Any, Dict, Optional\nfrom urllib.parse import urlencode\n\nimport requests\n\nfrom application.agents.tools.api_body_serializer import (\n ContentType,\n RequestBodySerializer,\n)\nfrom application.agents.tools.base import Tool\nfrom application.core.url_validation import validate_url, SSRFError\n\nlogger = logging.getLogger(__name__)\n\nDEFAULT_TIMEOUT = 90 # seconds\n\n\nclass APITool(Tool):\n \"\"\"\n API Tool\n A flexible tool for performing various API actions (e.g., sending messages, retrieving data) via custom user-specified APIs.\n \"\"\"\n\n def __init__(self, config):\n self.config = config\n self.url = config.get(\"url\", \"\")\n self.method = config.get(\"method\", \"GET\")\n self.headers = config.get(\"headers\", {})\n self.query_params = config.get(\"query_params\", {})\n self.body_content_type = config.get(\"body_content_type\", ContentType.JSON)\n self.body_encoding_rules = config.get(\"body_encoding_rules\", {})\n\n def execute_action(self, action_name, **kwargs):\n \"\"\"Execute an API action with the given arguments.\"\"\"\n return self._make_api_call(\n self.url,\n self.method,\n self.headers,\n self.query_params,\n kwargs,\n self.body_content_type,\n self.body_encoding_rules,\n )\n\n def _make_api_call(\n self,\n url: str,\n method: str,\n headers: Dict[str, str],\n query_params: Dict[str, Any],\n body: Dict[str, Any],\n content_type: str = ContentType.JSON,\n encoding_rules: Optional[Dict[str, Dict[str, Any]]] = None,\n ) -> Dict[str, Any]:\n \"\"\"\n Make an API call with proper body serialization and error handling.\n\n Args:\n url: API endpoint URL\n method: HTTP method (GET, POST, PUT, DELETE, PATCH, HEAD, OPTIONS)\n headers: Request headers dict\n query_params: URL query parameters\n body: Request body as dict\n content_type: Content-Type for serialization\n encoding_rules: OpenAPI encoding rules\n\n Returns:\n Dict with status_code, data, and message\n \"\"\"\n request_url = url\n request_headers = headers.copy() if headers else {}\n response = None\n\n # Validate URL to prevent SSRF attacks\n try:\n validate_url(request_url)\n except SSRFError as e:\n logger.error(f\"URL validation failed: {e}\")\n return {\n \"status_code\": None,\n \"message\": f\"URL validation error: {e}\",\n \"data\": None,\n }\n\n try:\n path_params_used = set()\n if query_params:\n for match in re.finditer(r\"\\{([^}]+)\\}\", request_url):\n param_name = match.group(1)\n if param_name in query_params:\n request_url = request_url.replace(\n f\"{{{param_name}}}\", str(query_params[param_name])\n )\n path_params_used.add(param_name)\n remaining_params = {\n k: v for k, v in query_params.items() if k not in path_params_used\n }\n if remaining_params:\n query_string = urlencode(remaining_params)\n separator = \"&\" if \"?\" in request_url else \"?\"\n request_url = f\"{request_url}{separator}{query_string}\"\n\n # Re-validate URL after parameter substitution to prevent SSRF via path params\n try:\n validate_url(request_url)\n except SSRFError as e:\n logger.error(f\"URL validation failed after parameter substitution: {e}\")\n return {\n \"status_code\": None,\n \"message\": f\"URL validation error: {e}\",\n \"data\": None,\n }\n\n # Serialize body based on content type\n\n if body and body != {}:\n try:\n serialized_body, body_headers = RequestBodySerializer.serialize(\n body, content_type, encoding_rules\n )\n request_headers.update(body_headers)\n except ValueError as e:\n logger.error(f\"Body serialization failed: {str(e)}\")\n return {\n \"status_code\": None,\n \"message\": f\"Body serialization error: {str(e)}\",\n \"data\": None,\n }\n else:\n serialized_body = None\n if \"Content-Type\" not in request_headers and method not in [\n \"GET\",\n \"HEAD\",\n \"DELETE\",\n ]:\n request_headers[\"Content-Type\"] = ContentType.JSON\n logger.debug(\n f\"API Call: {method} {request_url} | Content-Type: {request_headers.get('Content-Type', 'N/A')}\"\n )\n\n if method.upper() == \"GET\":\n response = requests.get(\n request_url, headers=request_headers, timeout=DEFAULT_TIMEOUT\n )\n elif method.upper() == \"POST\":\n response = requests.post(\n request_url,\n data=serialized_body,\n headers=request_headers,\n timeout=DEFAULT_TIMEOUT,\n )\n elif method.upper() == \"PUT\":\n response = requests.put(\n request_url,\n data=serialized_body,\n headers=request_headers,\n timeout=DEFAULT_TIMEOUT,\n )\n elif method.upper() == \"DELETE\":\n response = requests.delete(\n request_url, headers=request_headers, timeout=DEFAULT_TIMEOUT\n )\n elif method.upper() == \"PATCH\":\n response = requests.patch(\n request_url,\n data=serialized_body,\n headers=request_headers,\n timeout=DEFAULT_TIMEOUT,\n )\n elif method.upper() == \"HEAD\":\n response = requests.head(\n request_url, headers=request_headers, timeout=DEFAULT_TIMEOUT\n )\n elif method.upper() == \"OPTIONS\":\n response = requests.options(\n request_url, headers=request_headers, timeout=DEFAULT_TIMEOUT\n )\n else:\n return {\n \"status_code\": None,\n \"message\": f\"Unsupported HTTP method: {method}\",\n \"data\": None,\n }\n response.raise_for_status()\n\n data = self._parse_response(response)\n\n return {\n \"status_code\": response.status_code,\n \"data\": data,\n \"message\": \"API call successful.\",\n }\n except requests.exceptions.Timeout:\n logger.error(f\"Request timeout for {request_url}\")\n return {\n \"status_code\": None,\n \"message\": f\"Request timeout ({DEFAULT_TIMEOUT}s exceeded)\",\n \"data\": None,\n }\n except requests.exceptions.ConnectionError as e:\n logger.error(f\"Connection error: {str(e)}\")\n return {\n \"status_code\": None,\n \"message\": f\"Connection error: {str(e)}\",\n \"data\": None,\n }\n except requests.exceptions.HTTPError as e:\n logger.error(f\"HTTP error {response.status_code}: {str(e)}\")\n try:\n error_data = response.json()\n except (json.JSONDecodeError, ValueError):\n error_data = response.text\n return {\n \"status_code\": response.status_code,\n \"message\": f\"HTTP Error {response.status_code}\",\n \"data\": error_data,\n }\n except requests.exceptions.RequestException as e:\n logger.error(f\"Request failed: {str(e)}\")\n return {\n \"status_code\": response.status_code if response else None,\n \"message\": f\"API call failed: {str(e)}\",\n \"data\": None,\n }\n except Exception as e:\n logger.error(f\"Unexpected error in API call: {str(e)}\", exc_info=True)\n return {\n \"status_code\": None,\n \"message\": f\"Unexpected error: {str(e)}\",\n \"data\": None,\n }\n\n def _parse_response(self, response: requests.Response) -> Any:\n \"\"\"\n Parse response based on Content-Type header.\n\n Supports: JSON, XML, plain text, binary data.\n \"\"\"\n content_type = response.headers.get(\"Content-Type\", \"\").lower()\n\n if not response.content:\n return None\n # JSON response\n\n if \"application/json\" in content_type:\n try:\n return response.json()\n except json.JSONDecodeError as e:\n logger.warning(f\"Failed to parse JSON response: {str(e)}\")\n return response.text\n # XML response\n\n elif \"application/xml\" in content_type or \"text/xml\" in content_type:\n return response.text\n # Plain text response\n\n elif \"text/plain\" in content_type or \"text/html\" in content_type:\n return response.text\n # Binary/unknown response\n\n else:\n # Try to decode as text first, fall back to base64\n\n try:\n return response.text\n except (UnicodeDecodeError, AttributeError):\n import base64\n\n return base64.b64encode(response.content).decode(\"utf-8\")\n\n def get_actions_metadata(self):\n \"\"\"Return metadata for available actions (none for API Tool - actions are user-defined).\"\"\"\n return []\n\n def get_config_requirements(self):\n \"\"\"Return configuration requirements for the tool.\"\"\"\n return {}\n" }, { "path": "application/agents/tools/base.py", "content": "from abc import ABC, abstractmethod\n\n\nclass Tool(ABC):\n @abstractmethod\n def execute_action(self, action_name: str, **kwargs):\n pass\n\n @abstractmethod\n def get_actions_metadata(self):\n \"\"\"\n Returns a list of JSON objects describing the actions supported by the tool.\n \"\"\"\n pass\n\n @abstractmethod\n def get_config_requirements(self):\n \"\"\"\n Returns a dictionary describing the configuration requirements for the tool.\n \"\"\"\n pass\n" }, { "path": "application/agents/tools/brave.py", "content": "import logging\n\nimport requests\n\nfrom application.agents.tools.base import Tool\n\nlogger = logging.getLogger(__name__)\n\n\nclass BraveSearchTool(Tool):\n \"\"\"\n Brave Search\n A tool for performing web and image searches using the Brave Search API.\n Requires an API key for authentication.\n \"\"\"\n\n def __init__(self, config):\n self.config = config\n self.token = config.get(\"token\", \"\")\n self.base_url = \"https://api.search.brave.com/res/v1\"\n\n def execute_action(self, action_name, **kwargs):\n actions = {\n \"brave_web_search\": self._web_search,\n \"brave_image_search\": self._image_search,\n }\n\n if action_name in actions:\n return actions[action_name](**kwargs)\n else:\n raise ValueError(f\"Unknown action: {action_name}\")\n\n def _web_search(\n self,\n query,\n country=\"ALL\",\n search_lang=\"en\",\n count=10,\n offset=0,\n safesearch=\"off\",\n freshness=None,\n result_filter=None,\n extra_snippets=False,\n summary=False,\n ):\n \"\"\"\n Performs a web search using the Brave Search API.\n \"\"\"\n logger.debug(\"Performing Brave web search for: %s\", query)\n\n url = f\"{self.base_url}/web/search\"\n\n params = {\n \"q\": query,\n \"country\": country,\n \"search_lang\": search_lang,\n \"count\": min(count, 20),\n \"offset\": min(offset, 9),\n \"safesearch\": safesearch,\n }\n\n if freshness:\n params[\"freshness\"] = freshness\n if result_filter:\n params[\"result_filter\"] = result_filter\n if extra_snippets:\n params[\"extra_snippets\"] = 1\n if summary:\n params[\"summary\"] = 1\n headers = {\n \"Accept\": \"application/json\",\n \"Accept-Encoding\": \"gzip\",\n \"X-Subscription-Token\": self.token,\n }\n\n response = requests.get(url, params=params, headers=headers)\n\n if response.status_code == 200:\n return {\n \"status_code\": response.status_code,\n \"results\": response.json(),\n \"message\": \"Search completed successfully.\",\n }\n else:\n return {\n \"status_code\": response.status_code,\n \"message\": f\"Search failed with status code: {response.status_code}.\",\n }\n\n def _image_search(\n self,\n query,\n country=\"ALL\",\n search_lang=\"en\",\n count=5,\n safesearch=\"off\",\n spellcheck=False,\n ):\n \"\"\"\n Performs an image search using the Brave Search API.\n \"\"\"\n logger.debug(\"Performing Brave image search for: %s\", query)\n\n url = f\"{self.base_url}/images/search\"\n\n params = {\n \"q\": query,\n \"country\": country,\n \"search_lang\": search_lang,\n \"count\": min(count, 100), # API max is 100\n \"safesearch\": safesearch,\n \"spellcheck\": 1 if spellcheck else 0,\n }\n\n headers = {\n \"Accept\": \"application/json\",\n \"Accept-Encoding\": \"gzip\",\n \"X-Subscription-Token\": self.token,\n }\n\n response = requests.get(url, params=params, headers=headers)\n\n if response.status_code == 200:\n return {\n \"status_code\": response.status_code,\n \"results\": response.json(),\n \"message\": \"Image search completed successfully.\",\n }\n else:\n return {\n \"status_code\": response.status_code,\n \"message\": f\"Image search failed with status code: {response.status_code}.\",\n }\n\n def get_actions_metadata(self):\n return [\n {\n \"name\": \"brave_web_search\",\n \"description\": \"Perform a web search using Brave Search\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"query\": {\n \"type\": \"string\",\n \"description\": \"The search query (max 400 characters, 50 words)\",\n },\n \"search_lang\": {\n \"type\": \"string\",\n \"description\": \"The search language preference (default: en)\",\n },\n \"freshness\": {\n \"type\": \"string\",\n \"description\": \"Time filter for results (pd: last 24h, pw: last week, pm: last month, py: last year)\",\n },\n },\n \"required\": [\"query\"],\n \"additionalProperties\": False,\n },\n },\n {\n \"name\": \"brave_image_search\",\n \"description\": \"Perform an image search using Brave Search\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"query\": {\n \"type\": \"string\",\n \"description\": \"The search query (max 400 characters, 50 words)\",\n },\n \"count\": {\n \"type\": \"integer\",\n \"description\": \"Number of results to return (max 100, default: 5)\",\n },\n },\n \"required\": [\"query\"],\n \"additionalProperties\": False,\n },\n },\n ]\n\n def get_config_requirements(self):\n return {\n \"token\": {\n \"type\": \"string\",\n \"label\": \"API Key\",\n \"description\": \"Brave Search API key for authentication\",\n \"required\": True,\n \"secret\": True,\n \"order\": 1,\n },\n }\n" }, { "path": "application/agents/tools/cryptoprice.py", "content": "import requests\nfrom application.agents.tools.base import Tool\n\n\nclass CryptoPriceTool(Tool):\n \"\"\"\n CryptoPrice\n A tool for retrieving cryptocurrency prices using the CryptoCompare public API\n \"\"\"\n\n def __init__(self, config):\n self.config = config\n\n def execute_action(self, action_name, **kwargs):\n actions = {\"cryptoprice_get\": self._get_price}\n\n if action_name in actions:\n return actions[action_name](**kwargs)\n else:\n raise ValueError(f\"Unknown action: {action_name}\")\n\n def _get_price(self, symbol, currency):\n \"\"\"\n Fetches the current price of a given cryptocurrency symbol in the specified currency.\n Example:\n symbol = \"BTC\"\n currency = \"USD\"\n returns price in USD.\n \"\"\"\n url = f\"https://min-api.cryptocompare.com/data/price?fsym={symbol.upper()}&tsyms={currency.upper()}\"\n response = requests.get(url)\n if response.status_code == 200:\n data = response.json()\n if currency.upper() in data:\n return {\n \"status_code\": response.status_code,\n \"price\": data[currency.upper()],\n \"message\": f\"Price of {symbol.upper()} in {currency.upper()} retrieved successfully.\",\n }\n else:\n return {\n \"status_code\": response.status_code,\n \"message\": f\"Couldn't find price for {symbol.upper()} in {currency.upper()}.\",\n }\n else:\n return {\n \"status_code\": response.status_code,\n \"message\": \"Failed to retrieve price.\",\n }\n\n def get_actions_metadata(self):\n return [\n {\n \"name\": \"cryptoprice_get\",\n \"description\": \"Retrieve the price of a specified cryptocurrency in a given currency\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"symbol\": {\n \"type\": \"string\",\n \"description\": \"The cryptocurrency symbol (e.g. BTC)\",\n },\n \"currency\": {\n \"type\": \"string\",\n \"description\": \"The currency in which you want the price (e.g. USD)\",\n },\n },\n \"required\": [\"symbol\", \"currency\"],\n \"additionalProperties\": False,\n },\n }\n ]\n\n def get_config_requirements(self):\n # No specific configuration needed for this tool as it just queries a public endpoint\n return {}\n" }, { "path": "application/agents/tools/duckduckgo.py", "content": "import logging\nimport time\nfrom typing import Any, Dict, Optional\n\nfrom application.agents.tools.base import Tool\n\nlogger = logging.getLogger(__name__)\n\nMAX_RETRIES = 3\nRETRY_DELAY = 2.0\nDEFAULT_TIMEOUT = 15\n\n\nclass DuckDuckGoSearchTool(Tool):\n \"\"\"\n DuckDuckGo Search\n A tool for performing web and image searches using DuckDuckGo.\n \"\"\"\n\n def __init__(self, config):\n self.config = config\n self.timeout = config.get(\"timeout\", DEFAULT_TIMEOUT)\n\n def _get_ddgs_client(self):\n from ddgs import DDGS\n\n return DDGS(timeout=self.timeout)\n\n def _execute_with_retry(self, operation, operation_name: str) -> Dict[str, Any]:\n last_error = None\n for attempt in range(1, MAX_RETRIES + 1):\n try:\n results = operation()\n return {\n \"status_code\": 200,\n \"results\": list(results) if results else [],\n \"message\": f\"{operation_name} completed successfully.\",\n }\n except Exception as e:\n last_error = e\n error_str = str(e).lower()\n if \"ratelimit\" in error_str or \"429\" in error_str:\n if attempt < MAX_RETRIES:\n delay = RETRY_DELAY * attempt\n logger.warning(\n f\"{operation_name} rate limited, retrying in {delay}s (attempt {attempt}/{MAX_RETRIES})\"\n )\n time.sleep(delay)\n continue\n logger.error(f\"{operation_name} failed: {e}\")\n break\n return {\n \"status_code\": 500,\n \"results\": [],\n \"message\": f\"{operation_name} failed: {str(last_error)}\",\n }\n\n def execute_action(self, action_name, **kwargs):\n actions = {\n \"ddg_web_search\": self._web_search,\n \"ddg_image_search\": self._image_search,\n \"ddg_news_search\": self._news_search,\n }\n if action_name not in actions:\n raise ValueError(f\"Unknown action: {action_name}\")\n return actions[action_name](**kwargs)\n\n def _web_search(\n self,\n query: str,\n max_results: int = 5,\n region: str = \"wt-wt\",\n safesearch: str = \"moderate\",\n timelimit: Optional[str] = None,\n ) -> Dict[str, Any]:\n logger.info(f\"DuckDuckGo web search: {query}\")\n\n def operation():\n client = self._get_ddgs_client()\n return client.text(\n query,\n region=region,\n safesearch=safesearch,\n timelimit=timelimit,\n max_results=min(max_results, 20),\n )\n\n return self._execute_with_retry(operation, \"Web search\")\n\n def _image_search(\n self,\n query: str,\n max_results: int = 5,\n region: str = \"wt-wt\",\n safesearch: str = \"moderate\",\n timelimit: Optional[str] = None,\n ) -> Dict[str, Any]:\n logger.info(f\"DuckDuckGo image search: {query}\")\n\n def operation():\n client = self._get_ddgs_client()\n return client.images(\n query,\n region=region,\n safesearch=safesearch,\n timelimit=timelimit,\n max_results=min(max_results, 50),\n )\n\n return self._execute_with_retry(operation, \"Image search\")\n\n def _news_search(\n self,\n query: str,\n max_results: int = 5,\n region: str = \"wt-wt\",\n safesearch: str = \"moderate\",\n timelimit: Optional[str] = None,\n ) -> Dict[str, Any]:\n logger.info(f\"DuckDuckGo news search: {query}\")\n\n def operation():\n client = self._get_ddgs_client()\n return client.news(\n query,\n region=region,\n safesearch=safesearch,\n timelimit=timelimit,\n max_results=min(max_results, 20),\n )\n\n return self._execute_with_retry(operation, \"News search\")\n\n def get_actions_metadata(self):\n return [\n {\n \"name\": \"ddg_web_search\",\n \"description\": \"Search the web using DuckDuckGo. Returns titles, URLs, and snippets.\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"query\": {\n \"type\": \"string\",\n \"description\": \"Search query\",\n },\n \"max_results\": {\n \"type\": \"integer\",\n \"description\": \"Number of results (default: 5, max: 20)\",\n },\n \"region\": {\n \"type\": \"string\",\n \"description\": \"Region code (default: wt-wt for worldwide, us-en for US)\",\n },\n \"timelimit\": {\n \"type\": \"string\",\n \"description\": \"Time filter: d (day), w (week), m (month), y (year)\",\n },\n },\n \"required\": [\"query\"],\n },\n },\n {\n \"name\": \"ddg_image_search\",\n \"description\": \"Search for images using DuckDuckGo. Returns image URLs and metadata.\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"query\": {\n \"type\": \"string\",\n \"description\": \"Image search query\",\n },\n \"max_results\": {\n \"type\": \"integer\",\n \"description\": \"Number of results (default: 5, max: 50)\",\n },\n \"region\": {\n \"type\": \"string\",\n \"description\": \"Region code (default: wt-wt for worldwide)\",\n },\n },\n \"required\": [\"query\"],\n },\n },\n {\n \"name\": \"ddg_news_search\",\n \"description\": \"Search for news articles using DuckDuckGo. Returns recent news.\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"query\": {\n \"type\": \"string\",\n \"description\": \"News search query\",\n },\n \"max_results\": {\n \"type\": \"integer\",\n \"description\": \"Number of results (default: 5, max: 20)\",\n },\n \"timelimit\": {\n \"type\": \"string\",\n \"description\": \"Time filter: d (day), w (week), m (month)\",\n },\n },\n \"required\": [\"query\"],\n },\n },\n ]\n\n def get_config_requirements(self):\n return {}\n" }, { "path": "application/agents/tools/mcp_tool.py", "content": "import asyncio\nimport base64\nimport concurrent.futures\nimport json\nimport logging\nimport time\nfrom typing import Any, Dict, List, Optional\nfrom urllib.parse import parse_qs, urlparse\n\nfrom fastmcp import Client\nfrom fastmcp.client.auth import BearerAuth\nfrom fastmcp.client.transports import (\n SSETransport,\n StdioTransport,\n StreamableHttpTransport,\n)\nfrom mcp.client.auth import OAuthClientProvider, TokenStorage\nfrom mcp.shared.auth import OAuthClientInformationFull, OAuthClientMetadata, OAuthToken\nfrom pydantic import AnyHttpUrl, ValidationError\nfrom redis import Redis\n\nfrom application.agents.tools.base import Tool\nfrom application.api.user.tasks import mcp_oauth_status_task, mcp_oauth_task\nfrom application.cache import get_redis_instance\nfrom application.core.mongo_db import MongoDB\nfrom application.core.settings import settings\nfrom application.security.encryption import decrypt_credentials\n\nlogger = logging.getLogger(__name__)\n\nmongo = MongoDB.get_client()\ndb = mongo[settings.MONGO_DB_NAME]\n\n_mcp_clients_cache = {}\n\n\nclass MCPTool(Tool):\n \"\"\"\n MCP Tool\n Connect to remote Model Context Protocol (MCP) servers to access dynamic tools and resources.\n \"\"\"\n\n def __init__(self, config: Dict[str, Any], user_id: Optional[str] = None):\n \"\"\"\n Initialize the MCP Tool with configuration.\n\n Args:\n config: Dictionary containing MCP server configuration:\n - server_url: URL of the remote MCP server\n - transport_type: Transport type (auto, sse, http, stdio)\n - auth_type: Type of authentication (bearer, oauth, api_key, basic, none)\n - encrypted_credentials: Encrypted credentials (if available)\n - timeout: Request timeout in seconds (default: 30)\n - headers: Custom headers for requests\n - command: Command for STDIO transport\n - args: Arguments for STDIO transport\n - oauth_scopes: OAuth scopes for oauth auth type\n - oauth_client_name: OAuth client name for oauth auth type\n - query_mode: If True, use non-interactive OAuth (fail-fast on 401)\n user_id: User ID for decrypting credentials (required if encrypted_credentials exist)\n \"\"\"\n self.config = config\n self.user_id = user_id\n self.server_url = config.get(\"server_url\", \"\")\n self.transport_type = config.get(\"transport_type\", \"auto\")\n self.auth_type = config.get(\"auth_type\", \"none\")\n self.timeout = config.get(\"timeout\", 30)\n self.custom_headers = config.get(\"headers\", {})\n\n self.auth_credentials = {}\n if config.get(\"encrypted_credentials\") and user_id:\n self.auth_credentials = decrypt_credentials(\n config[\"encrypted_credentials\"], user_id\n )\n else:\n self.auth_credentials = config.get(\"auth_credentials\", {})\n self.oauth_scopes = config.get(\"oauth_scopes\", [])\n self.oauth_task_id = config.get(\"oauth_task_id\", None)\n self.oauth_client_name = config.get(\"oauth_client_name\", \"DocsGPT-MCP\")\n self.redirect_uri = self._resolve_redirect_uri(config.get(\"redirect_uri\"))\n\n self.available_tools = []\n self._cache_key = self._generate_cache_key()\n self._client = None\n self.query_mode = config.get(\"query_mode\", False)\n\n if self.server_url and self.auth_type != \"oauth\":\n self._setup_client()\n\n def _resolve_redirect_uri(self, configured_redirect_uri: Optional[str]) -> str:\n if configured_redirect_uri:\n return configured_redirect_uri.rstrip(\"/\")\n\n explicit = getattr(settings, \"MCP_OAUTH_REDIRECT_URI\", None)\n if explicit:\n return explicit.rstrip(\"/\")\n\n connector_base = getattr(settings, \"CONNECTOR_REDIRECT_BASE_URI\", None)\n if connector_base:\n parsed = urlparse(connector_base)\n if parsed.scheme and parsed.netloc:\n return f\"{parsed.scheme}://{parsed.netloc}/api/mcp_server/callback\"\n\n return f\"{settings.API_URL.rstrip('/')}/api/mcp_server/callback\"\n\n def _generate_cache_key(self) -> str:\n \"\"\"Generate a unique cache key for this MCP server configuration.\"\"\"\n auth_key = \"\"\n if self.auth_type == \"oauth\":\n scopes_str = \",\".join(self.oauth_scopes) if self.oauth_scopes else \"none\"\n auth_key = (\n f\"oauth:{self.oauth_client_name}:{scopes_str}:{self.redirect_uri}\"\n )\n elif self.auth_type in [\"bearer\"]:\n token = self.auth_credentials.get(\n \"bearer_token\", \"\"\n ) or self.auth_credentials.get(\"access_token\", \"\")\n auth_key = f\"bearer:{token[:10]}...\" if token else \"bearer:none\"\n elif self.auth_type == \"api_key\":\n api_key = self.auth_credentials.get(\"api_key\", \"\")\n auth_key = f\"apikey:{api_key[:10]}...\" if api_key else \"apikey:none\"\n elif self.auth_type == \"basic\":\n username = self.auth_credentials.get(\"username\", \"\")\n auth_key = f\"basic:{username}\"\n else:\n auth_key = \"none\"\n return f\"{self.server_url}#{self.transport_type}#{auth_key}\"\n\n def _setup_client(self):\n global _mcp_clients_cache\n if self._cache_key in _mcp_clients_cache:\n cached_data = _mcp_clients_cache[self._cache_key]\n if time.time() - cached_data[\"created_at\"] < 300:\n self._client = cached_data[\"client\"]\n return\n else:\n del _mcp_clients_cache[self._cache_key]\n transport = self._create_transport()\n auth = None\n\n if self.auth_type == \"oauth\":\n redis_client = get_redis_instance()\n if self.query_mode:\n auth = NonInteractiveOAuth(\n mcp_url=self.server_url,\n scopes=self.oauth_scopes,\n redis_client=redis_client,\n redirect_uri=self.redirect_uri,\n db=db,\n user_id=self.user_id,\n )\n else:\n auth = DocsGPTOAuth(\n mcp_url=self.server_url,\n scopes=self.oauth_scopes,\n redis_client=redis_client,\n redirect_uri=self.redirect_uri,\n task_id=self.oauth_task_id,\n db=db,\n user_id=self.user_id,\n )\n elif self.auth_type == \"bearer\":\n token = self.auth_credentials.get(\n \"bearer_token\", \"\"\n ) or self.auth_credentials.get(\"access_token\", \"\")\n if token:\n auth = BearerAuth(token)\n self._client = Client(transport, auth=auth)\n _mcp_clients_cache[self._cache_key] = {\n \"client\": self._client,\n \"created_at\": time.time(),\n }\n\n def _create_transport(self):\n \"\"\"Create appropriate transport based on configuration.\"\"\"\n headers = {\"Content-Type\": \"application/json\", \"User-Agent\": \"DocsGPT-MCP/1.0\"}\n headers.update(self.custom_headers)\n\n if self.auth_type == \"api_key\":\n api_key = self.auth_credentials.get(\"api_key\", \"\")\n header_name = self.auth_credentials.get(\"api_key_header\", \"X-API-Key\")\n if api_key:\n headers[header_name] = api_key\n elif self.auth_type == \"basic\":\n username = self.auth_credentials.get(\"username\", \"\")\n password = self.auth_credentials.get(\"password\", \"\")\n if username and password:\n credentials = base64.b64encode(\n f\"{username}:{password}\".encode()\n ).decode()\n headers[\"Authorization\"] = f\"Basic {credentials}\"\n if self.transport_type == \"auto\":\n if \"sse\" in self.server_url.lower() or self.server_url.endswith(\"/sse\"):\n transport_type = \"sse\"\n else:\n transport_type = \"http\"\n else:\n transport_type = self.transport_type\n if transport_type == \"stdio\":\n raise ValueError(\"STDIO transport is disabled\")\n if transport_type == \"sse\":\n headers.update({\"Accept\": \"text/event-stream\", \"Cache-Control\": \"no-cache\"})\n return SSETransport(url=self.server_url, headers=headers)\n elif transport_type == \"http\":\n return StreamableHttpTransport(url=self.server_url, headers=headers)\n elif transport_type == \"stdio\":\n command = self.config.get(\"command\", \"python\")\n args = self.config.get(\"args\", [])\n env = self.auth_credentials if self.auth_credentials else None\n return StdioTransport(command=command, args=args, env=env)\n else:\n return StreamableHttpTransport(url=self.server_url, headers=headers)\n\n def _format_tools(self, tools_response) -> List[Dict]:\n \"\"\"Format tools response to match expected format.\"\"\"\n if hasattr(tools_response, \"tools\"):\n tools = tools_response.tools\n elif isinstance(tools_response, list):\n tools = tools_response\n else:\n tools = []\n tools_dict = []\n for tool in tools:\n if hasattr(tool, \"name\"):\n tool_dict = {\n \"name\": tool.name,\n \"description\": tool.description,\n }\n if hasattr(tool, \"inputSchema\"):\n tool_dict[\"inputSchema\"] = tool.inputSchema\n tools_dict.append(tool_dict)\n elif isinstance(tool, dict):\n tools_dict.append(tool)\n else:\n if hasattr(tool, \"model_dump\"):\n tools_dict.append(tool.model_dump())\n else:\n tools_dict.append({\"name\": str(tool), \"description\": \"\"})\n return tools_dict\n\n async def _execute_with_client(self, operation: str, *args, **kwargs):\n \"\"\"Execute operation with FastMCP client.\"\"\"\n if not self._client:\n raise Exception(\"FastMCP client not initialized\")\n async with self._client:\n if operation == \"ping\":\n return await self._client.ping()\n elif operation == \"list_tools\":\n tools_response = await self._client.list_tools()\n self.available_tools = self._format_tools(tools_response)\n return self.available_tools\n elif operation == \"call_tool\":\n tool_name = args[0]\n tool_args = kwargs\n return await self._client.call_tool(tool_name, tool_args)\n elif operation == \"list_resources\":\n return await self._client.list_resources()\n elif operation == \"list_prompts\":\n return await self._client.list_prompts()\n else:\n raise Exception(f\"Unknown operation: {operation}\")\n\n _ERROR_MAP = [\n (concurrent.futures.TimeoutError, lambda op, t, _: f\"Timed out after {t}s\"),\n (ConnectionRefusedError, lambda *_: \"Connection refused\"),\n ]\n\n _ERROR_PATTERNS = {\n (\"403\", \"Forbidden\"): \"Access denied (403 Forbidden)\",\n (\"401\", \"Unauthorized\"): \"Authentication failed (401 Unauthorized)\",\n (\"ECONNREFUSED\",): \"Connection refused\",\n (\"SSL\", \"certificate\"): \"SSL/TLS error\",\n }\n\n def _run_async_operation(self, operation: str, *args, **kwargs):\n try:\n try:\n asyncio.get_running_loop()\n with concurrent.futures.ThreadPoolExecutor() as executor:\n future = executor.submit(\n self._run_in_new_loop, operation, *args, **kwargs\n )\n return future.result(timeout=self.timeout)\n except RuntimeError:\n return self._run_in_new_loop(operation, *args, **kwargs)\n except Exception as e:\n raise self._map_error(operation, e) from e\n raise self._map_error(operation, e) from e\n\n def _run_in_new_loop(self, operation, *args, **kwargs):\n loop = asyncio.new_event_loop()\n asyncio.set_event_loop(loop)\n try:\n return loop.run_until_complete(\n self._execute_with_client(operation, *args, **kwargs)\n )\n finally:\n loop.close()\n\n def _map_error(self, operation: str, exc: Exception) -> Exception:\n for exc_type, msg_fn in self._ERROR_MAP:\n if isinstance(exc, exc_type):\n return Exception(msg_fn(operation, self.timeout, exc))\n error_msg = str(exc)\n for patterns, friendly in self._ERROR_PATTERNS.items():\n if any(p.lower() in error_msg.lower() for p in patterns):\n return Exception(friendly)\n logger.error(\"MCP %s failed: %s\", operation, exc)\n return exc\n\n def discover_tools(self) -> List[Dict]:\n \"\"\"\n Discover available tools from the MCP server using FastMCP.\n\n Returns:\n List of tool definitions from the server\n \"\"\"\n if not self.server_url:\n return []\n if not self._client:\n self._setup_client()\n try:\n tools = self._run_async_operation(\"list_tools\")\n self.available_tools = tools\n return self.available_tools\n except Exception as e:\n raise Exception(f\"Failed to discover tools from MCP server: {str(e)}\")\n\n def execute_action(self, action_name: str, **kwargs) -> Any:\n if not self.server_url:\n raise Exception(\"No MCP server configured\")\n if not self._client:\n self._setup_client()\n cleaned_kwargs = {}\n for key, value in kwargs.items():\n if value == \"\" or value is None:\n continue\n cleaned_kwargs[key] = value\n try:\n result = self._run_async_operation(\n \"call_tool\", action_name, **cleaned_kwargs\n )\n return self._format_result(result)\n except Exception as e:\n error_msg = str(e)\n lower_msg = error_msg.lower()\n is_auth_error = (\n \"401\" in error_msg\n or \"unauthorized\" in lower_msg\n or \"session expired\" in lower_msg\n or \"re-authorize\" in lower_msg\n )\n if is_auth_error:\n if self.auth_type == \"oauth\":\n raise Exception(\n f\"Action '{action_name}' failed: OAuth session expired. \"\n \"Please re-authorize this MCP server in tool settings.\"\n ) from e\n global _mcp_clients_cache\n _mcp_clients_cache.pop(self._cache_key, None)\n self._client = None\n self._setup_client()\n try:\n result = self._run_async_operation(\n \"call_tool\", action_name, **cleaned_kwargs\n )\n return self._format_result(result)\n except Exception as retry_e:\n raise Exception(\n f\"Action '{action_name}' failed after re-auth attempt: {retry_e}. \"\n \"Your credentials may have expired — please re-authorize in tool settings.\"\n ) from retry_e\n raise Exception(\n f\"Failed to execute action '{action_name}': {error_msg}\"\n ) from e\n\n def _format_result(self, result) -> Dict:\n \"\"\"Format FastMCP result to match expected format.\"\"\"\n if hasattr(result, \"content\"):\n content_list = []\n for content_item in result.content:\n if hasattr(content_item, \"text\"):\n content_list.append({\"type\": \"text\", \"text\": content_item.text})\n elif hasattr(content_item, \"data\"):\n content_list.append({\"type\": \"data\", \"data\": content_item.data})\n else:\n content_list.append(\n {\"type\": \"unknown\", \"content\": str(content_item)}\n )\n return {\n \"content\": content_list,\n \"isError\": getattr(result, \"isError\", False),\n }\n else:\n return result\n\n def test_connection(self) -> Dict:\n if not self.server_url:\n return {\n \"success\": False,\n \"message\": \"No server URL configured\",\n \"tools_count\": 0,\n }\n try:\n parsed = urlparse(self.server_url)\n if parsed.scheme not in (\"http\", \"https\"):\n return {\n \"success\": False,\n \"message\": f\"Invalid URL scheme '{parsed.scheme}' — use http:// or https://\",\n \"tools_count\": 0,\n }\n except Exception:\n return {\n \"success\": False,\n \"message\": \"Invalid URL format\",\n \"tools_count\": 0,\n }\n if not self._client:\n try:\n self._setup_client()\n except Exception as e:\n return {\n \"success\": False,\n \"message\": f\"Client init failed: {str(e)}\",\n \"tools_count\": 0,\n }\n try:\n if self.auth_type == \"oauth\":\n return self._test_oauth_connection()\n else:\n return self._test_regular_connection()\n except Exception as e:\n return {\n \"success\": False,\n \"message\": f\"Connection failed: {str(e)}\",\n \"tools_count\": 0,\n }\n\n def _test_regular_connection(self) -> Dict:\n ping_ok = False\n ping_error = None\n try:\n self._run_async_operation(\"ping\")\n ping_ok = True\n except Exception as e:\n ping_error = str(e)\n\n try:\n tools = self.discover_tools()\n except Exception as e:\n return {\n \"success\": False,\n \"message\": f\"Connection failed: {ping_error or str(e)}\",\n \"tools_count\": 0,\n }\n\n if not tools and not ping_ok:\n return {\n \"success\": False,\n \"message\": f\"Connection failed: {ping_error or 'No tools found'}\",\n \"tools_count\": 0,\n }\n\n return {\n \"success\": True,\n \"message\": f\"Connected — found {len(tools)} tool{'s' if len(tools) != 1 else ''}.\",\n \"tools_count\": len(tools),\n \"tools\": [\n {\n \"name\": tool.get(\"name\", \"unknown\"),\n \"description\": tool.get(\"description\", \"\"),\n }\n for tool in tools\n ],\n }\n\n def _test_oauth_connection(self) -> Dict:\n storage = DBTokenStorage(\n server_url=self.server_url, user_id=self.user_id, db_client=db\n )\n loop = asyncio.new_event_loop()\n try:\n tokens = loop.run_until_complete(storage.get_tokens())\n finally:\n loop.close()\n\n if tokens and tokens.access_token:\n self.query_mode = True\n _mcp_clients_cache.pop(self._cache_key, None)\n self._client = None\n self._setup_client()\n try:\n tools = self.discover_tools()\n return {\n \"success\": True,\n \"message\": f\"Connected — found {len(tools)} tool{'s' if len(tools) != 1 else ''}.\",\n \"tools_count\": len(tools),\n \"tools\": [\n {\n \"name\": t.get(\"name\", \"unknown\"),\n \"description\": t.get(\"description\", \"\"),\n }\n for t in tools\n ],\n }\n except Exception as e:\n logger.warning(\"OAuth token validation failed: %s\", e)\n _mcp_clients_cache.pop(self._cache_key, None)\n self._client = None\n\n return self._start_oauth_task()\n\n def _start_oauth_task(self) -> Dict:\n task_config = self.config.copy()\n task_config.pop(\"query_mode\", None)\n result = mcp_oauth_task.delay(task_config, self.user_id)\n return {\n \"success\": False,\n \"requires_oauth\": True,\n \"task_id\": result.id,\n \"message\": \"OAuth authorization required.\",\n \"tools_count\": 0,\n }\n\n def get_actions_metadata(self) -> List[Dict]:\n \"\"\"\n Get metadata for all available actions.\n\n Returns:\n List of action metadata dictionaries\n \"\"\"\n actions = []\n for tool in self.available_tools:\n input_schema = (\n tool.get(\"inputSchema\")\n or tool.get(\"input_schema\")\n or tool.get(\"schema\")\n or tool.get(\"parameters\")\n )\n\n parameters_schema = {\n \"type\": \"object\",\n \"properties\": {},\n \"required\": [],\n }\n\n if input_schema:\n if isinstance(input_schema, dict):\n if \"properties\" in input_schema:\n parameters_schema = {\n \"type\": input_schema.get(\"type\", \"object\"),\n \"properties\": input_schema.get(\"properties\", {}),\n \"required\": input_schema.get(\"required\", []),\n }\n\n for key in [\"additionalProperties\", \"description\"]:\n if key in input_schema:\n parameters_schema[key] = input_schema[key]\n else:\n parameters_schema[\"properties\"] = input_schema\n action = {\n \"name\": tool.get(\"name\", \"\"),\n \"description\": tool.get(\"description\", \"\"),\n \"parameters\": parameters_schema,\n }\n actions.append(action)\n return actions\n\n def get_config_requirements(self) -> Dict:\n return {\n \"server_url\": {\n \"type\": \"string\",\n \"label\": \"Server URL\",\n \"description\": \"URL of the remote MCP server\",\n \"required\": True,\n \"secret\": False,\n \"order\": 1,\n },\n \"auth_type\": {\n \"type\": \"string\",\n \"label\": \"Authentication Type\",\n \"description\": \"Authentication method for the MCP server\",\n \"enum\": [\"none\", \"bearer\", \"oauth\", \"api_key\", \"basic\"],\n \"default\": \"none\",\n \"required\": True,\n \"secret\": False,\n \"order\": 2,\n },\n \"api_key\": {\n \"type\": \"string\",\n \"label\": \"API Key\",\n \"description\": \"API key for authentication\",\n \"required\": False,\n \"secret\": True,\n \"order\": 3,\n \"depends_on\": {\"auth_type\": \"api_key\"},\n },\n \"api_key_header\": {\n \"type\": \"string\",\n \"label\": \"API Key Header\",\n \"description\": \"Header name for API key (default: X-API-Key)\",\n \"default\": \"X-API-Key\",\n \"required\": False,\n \"secret\": False,\n \"order\": 4,\n \"depends_on\": {\"auth_type\": \"api_key\"},\n },\n \"bearer_token\": {\n \"type\": \"string\",\n \"label\": \"Bearer Token\",\n \"description\": \"Bearer token for authentication\",\n \"required\": False,\n \"secret\": True,\n \"order\": 3,\n \"depends_on\": {\"auth_type\": \"bearer\"},\n },\n \"username\": {\n \"type\": \"string\",\n \"label\": \"Username\",\n \"description\": \"Username for basic authentication\",\n \"required\": False,\n \"secret\": False,\n \"order\": 3,\n \"depends_on\": {\"auth_type\": \"basic\"},\n },\n \"password\": {\n \"type\": \"string\",\n \"label\": \"Password\",\n \"description\": \"Password for basic authentication\",\n \"required\": False,\n \"secret\": True,\n \"order\": 4,\n \"depends_on\": {\"auth_type\": \"basic\"},\n },\n \"oauth_scopes\": {\n \"type\": \"string\",\n \"label\": \"OAuth Scopes\",\n \"description\": \"Comma-separated OAuth scopes to request\",\n \"required\": False,\n \"secret\": False,\n \"order\": 3,\n \"depends_on\": {\"auth_type\": \"oauth\"},\n },\n \"timeout\": {\n \"type\": \"number\",\n \"label\": \"Timeout (seconds)\",\n \"description\": \"Request timeout in seconds (1-300)\",\n \"default\": 30,\n \"required\": False,\n \"secret\": False,\n \"order\": 10,\n },\n }\n\n\nclass DocsGPTOAuth(OAuthClientProvider):\n \"\"\"\n Custom OAuth handler for DocsGPT that uses frontend redirect instead of browser.\n \"\"\"\n\n def __init__(\n self,\n mcp_url: str,\n redirect_uri: str,\n redis_client: Redis | None = None,\n redis_prefix: str = \"mcp_oauth:\",\n task_id: str = None,\n scopes: str | list[str] | None = None,\n client_name: str = \"DocsGPT-MCP\",\n user_id=None,\n db=None,\n additional_client_metadata: dict[str, Any] | None = None,\n skip_redirect_validation: bool = False,\n ):\n self.redirect_uri = redirect_uri\n self.redis_client = redis_client\n self.redis_prefix = redis_prefix\n self.task_id = task_id\n self.user_id = user_id\n self.db = db\n\n parsed_url = urlparse(mcp_url)\n self.server_base_url = f\"{parsed_url.scheme}://{parsed_url.netloc}\"\n\n if isinstance(scopes, list):\n scopes = \" \".join(scopes)\n client_metadata = OAuthClientMetadata(\n client_name=client_name,\n redirect_uris=[AnyHttpUrl(redirect_uri)],\n grant_types=[\"authorization_code\", \"refresh_token\"],\n response_types=[\"code\"],\n scope=scopes,\n **(additional_client_metadata or {}),\n )\n\n storage = DBTokenStorage(\n server_url=self.server_base_url,\n user_id=self.user_id,\n db_client=self.db,\n expected_redirect_uri=None if skip_redirect_validation else redirect_uri,\n )\n\n super().__init__(\n server_url=self.server_base_url,\n client_metadata=client_metadata,\n storage=storage,\n redirect_handler=self.redirect_handler,\n callback_handler=self.callback_handler,\n )\n\n self.auth_url = None\n self.extracted_state = None\n\n def _process_auth_url(self, authorization_url: str) -> tuple[str, str]:\n \"\"\"Process authorization URL to extract state\"\"\"\n try:\n parsed_url = urlparse(authorization_url)\n query_params = parse_qs(parsed_url.query)\n\n state_params = query_params.get(\"state\", [])\n if state_params:\n state = state_params[0]\n else:\n raise ValueError(\"No state in auth URL\")\n return authorization_url, state\n except Exception as e:\n raise Exception(f\"Failed to process auth URL: {e}\")\n\n async def redirect_handler(self, authorization_url: str) -> None:\n \"\"\"Store auth URL and state in Redis for frontend to use.\"\"\"\n auth_url, state = self._process_auth_url(authorization_url)\n logger.info(\"Processed auth_url: %s, state: %s\", auth_url, state)\n self.auth_url = auth_url\n self.extracted_state = state\n\n if self.redis_client and self.extracted_state:\n key = f\"{self.redis_prefix}auth_url:{self.extracted_state}\"\n self.redis_client.setex(key, 600, auth_url)\n logger.info(\"Stored auth_url in Redis: %s\", key)\n\n if self.task_id:\n status_key = f\"mcp_oauth_status:{self.task_id}\"\n status_data = {\n \"status\": \"requires_redirect\",\n \"message\": \"Authorization required\",\n \"authorization_url\": self.auth_url,\n \"state\": self.extracted_state,\n \"requires_oauth\": True,\n \"task_id\": self.task_id,\n }\n self.redis_client.setex(status_key, 600, json.dumps(status_data))\n\n async def callback_handler(self) -> tuple[str, str | None]:\n \"\"\"Wait for auth code from Redis using the state value.\"\"\"\n if not self.redis_client or not self.extracted_state:\n raise Exception(\"Redis client or state not configured for OAuth\")\n poll_interval = 1\n max_wait_time = 300\n code_key = f\"{self.redis_prefix}code:{self.extracted_state}\"\n\n if self.task_id:\n status_key = f\"mcp_oauth_status:{self.task_id}\"\n status_data = {\n \"status\": \"awaiting_callback\",\n \"message\": \"Waiting for authorization...\",\n \"authorization_url\": self.auth_url,\n \"state\": self.extracted_state,\n \"requires_oauth\": True,\n \"task_id\": self.task_id,\n }\n self.redis_client.setex(status_key, 600, json.dumps(status_data))\n start_time = time.time()\n while time.time() - start_time < max_wait_time:\n code_data = self.redis_client.get(code_key)\n if code_data:\n code = code_data.decode()\n returned_state = self.extracted_state\n\n self.redis_client.delete(code_key)\n self.redis_client.delete(\n f\"{self.redis_prefix}auth_url:{self.extracted_state}\"\n )\n self.redis_client.delete(\n f\"{self.redis_prefix}state:{self.extracted_state}\"\n )\n\n if self.task_id:\n status_data = {\n \"status\": \"callback_received\",\n \"message\": \"Completing authentication...\",\n \"task_id\": self.task_id,\n }\n self.redis_client.setex(status_key, 600, json.dumps(status_data))\n return code, returned_state\n error_key = f\"{self.redis_prefix}error:{self.extracted_state}\"\n error_data = self.redis_client.get(error_key)\n if error_data:\n error_msg = error_data.decode()\n self.redis_client.delete(error_key)\n self.redis_client.delete(\n f\"{self.redis_prefix}auth_url:{self.extracted_state}\"\n )\n self.redis_client.delete(\n f\"{self.redis_prefix}state:{self.extracted_state}\"\n )\n raise Exception(f\"OAuth error: {error_msg}\")\n await asyncio.sleep(poll_interval)\n self.redis_client.delete(f\"{self.redis_prefix}auth_url:{self.extracted_state}\")\n self.redis_client.delete(f\"{self.redis_prefix}state:{self.extracted_state}\")\n raise Exception(\"OAuth timeout: no code received within 5 minutes\")\n\n\nclass NonInteractiveOAuth(DocsGPTOAuth):\n \"\"\"OAuth provider that fails fast on 401 instead of starting interactive auth.\n\n Used during query execution to prevent the streaming response from blocking\n while waiting for user authorization that will never come.\n \"\"\"\n\n def __init__(self, **kwargs):\n kwargs.setdefault(\"task_id\", None)\n kwargs[\"skip_redirect_validation\"] = True\n super().__init__(**kwargs)\n\n async def redirect_handler(self, authorization_url: str) -> None:\n raise Exception(\n \"OAuth session expired — please re-authorize this MCP server in tool settings.\"\n )\n\n async def callback_handler(self) -> tuple[str, str | None]:\n raise Exception(\n \"OAuth session expired — please re-authorize this MCP server in tool settings.\"\n )\n\n\nclass DBTokenStorage(TokenStorage):\n def __init__(\n self,\n server_url: str,\n user_id: str,\n db_client,\n expected_redirect_uri: Optional[str] = None,\n ):\n self.server_url = server_url\n self.user_id = user_id\n self.db_client = db_client\n self.expected_redirect_uri = expected_redirect_uri\n self.collection = db_client[\"connector_sessions\"]\n\n @staticmethod\n def get_base_url(url: str) -> str:\n parsed = urlparse(url)\n return f\"{parsed.scheme}://{parsed.netloc}\"\n\n def get_db_key(self) -> dict:\n return {\n \"server_url\": self.get_base_url(self.server_url),\n \"user_id\": self.user_id,\n }\n\n async def get_tokens(self) -> OAuthToken | None:\n doc = await asyncio.to_thread(self.collection.find_one, self.get_db_key())\n if not doc or \"tokens\" not in doc:\n return None\n try:\n return OAuthToken.model_validate(doc[\"tokens\"])\n except ValidationError as e:\n logger.error(\"Could not load tokens: %s\", e)\n return None\n\n async def set_tokens(self, tokens: OAuthToken) -> None:\n await asyncio.to_thread(\n self.collection.update_one,\n self.get_db_key(),\n {\"$set\": {\"tokens\": tokens.model_dump()}},\n True,\n )\n logger.info(\"Saved tokens for %s\", self.get_base_url(self.server_url))\n\n async def get_client_info(self) -> OAuthClientInformationFull | None:\n doc = await asyncio.to_thread(self.collection.find_one, self.get_db_key())\n if not doc or \"client_info\" not in doc:\n logger.debug(\n \"No client_info in DB for %s\", self.get_base_url(self.server_url)\n )\n return None\n try:\n client_info = OAuthClientInformationFull.model_validate(doc[\"client_info\"])\n if self.expected_redirect_uri:\n stored_uris = [\n str(uri).rstrip(\"/\") for uri in client_info.redirect_uris\n ]\n expected_uri = self.expected_redirect_uri.rstrip(\"/\")\n if expected_uri not in stored_uris:\n logger.warning(\n \"Redirect URI mismatch for %s: expected=%s stored=%s — clearing.\",\n self.get_base_url(self.server_url),\n expected_uri,\n stored_uris,\n )\n await asyncio.to_thread(\n self.collection.update_one,\n self.get_db_key(),\n {\"$unset\": {\"client_info\": \"\", \"tokens\": \"\"}},\n )\n return None\n return client_info\n except ValidationError as e:\n logger.error(\"Could not load client info: %s\", e)\n return None\n\n def _serialize_client_info(self, info: dict) -> dict:\n if \"redirect_uris\" in info and isinstance(info[\"redirect_uris\"], list):\n info[\"redirect_uris\"] = [str(u) for u in info[\"redirect_uris\"]]\n return info\n\n async def set_client_info(self, client_info: OAuthClientInformationFull) -> None:\n serialized_info = self._serialize_client_info(client_info.model_dump())\n await asyncio.to_thread(\n self.collection.update_one,\n self.get_db_key(),\n {\"$set\": {\"client_info\": serialized_info}},\n True,\n )\n logger.info(\"Saved client info for %s\", self.get_base_url(self.server_url))\n\n async def clear(self) -> None:\n await asyncio.to_thread(self.collection.delete_one, self.get_db_key())\n logger.info(\"Cleared OAuth cache for %s\", self.get_base_url(self.server_url))\n\n @classmethod\n async def clear_all(cls, db_client) -> None:\n collection = db_client[\"connector_sessions\"]\n await asyncio.to_thread(collection.delete_many, {})\n logger.info(\"Cleared all OAuth client cache data.\")\n\n\nclass MCPOAuthManager:\n \"\"\"Manager for handling MCP OAuth callbacks.\"\"\"\n\n def __init__(self, redis_client: Redis | None, redis_prefix: str = \"mcp_oauth:\"):\n self.redis_client = redis_client\n self.redis_prefix = redis_prefix\n\n def handle_oauth_callback(\n self, state: str, code: str, error: Optional[str] = None\n ) -> bool:\n \"\"\"\n Handle OAuth callback from provider.\n\n Args:\n state: The state parameter from OAuth callback\n code: The authorization code from OAuth callback\n error: Error message if OAuth failed\n\n Returns:\n True if successful, False otherwise\n \"\"\"\n try:\n if not self.redis_client or not state:\n raise Exception(\"Redis client or state not provided\")\n if error:\n error_key = f\"{self.redis_prefix}error:{state}\"\n self.redis_client.setex(error_key, 300, error)\n raise Exception(f\"OAuth error received: {error}\")\n code_key = f\"{self.redis_prefix}code:{state}\"\n self.redis_client.setex(code_key, 300, code)\n\n state_key = f\"{self.redis_prefix}state:{state}\"\n self.redis_client.setex(state_key, 300, \"completed\")\n\n return True\n except Exception as e:\n logger.error(\"Error handling OAuth callback: %s\", e)\n return False\n\n def get_oauth_status(self, task_id: str) -> Dict[str, Any]:\n \"\"\"Get current status of OAuth flow using provided task_id.\"\"\"\n if not task_id:\n return {\"status\": \"not_started\", \"message\": \"OAuth flow not started\"}\n return mcp_oauth_status_task(task_id)\n" }, { "path": "application/agents/tools/memory.py", "content": "from datetime import datetime\nfrom pathlib import Path\nfrom typing import Any, Dict, List, Optional\nimport re\nimport uuid\n\nfrom .base import Tool\nfrom application.core.mongo_db import MongoDB\nfrom application.core.settings import settings\n\n\nclass MemoryTool(Tool):\n \"\"\"Memory\n\n Stores and retrieves information across conversations through a memory file directory.\n \"\"\"\n\n def __init__(self, tool_config: Optional[Dict[str, Any]] = None, user_id: Optional[str] = None) -> None:\n \"\"\"Initialize the tool.\n\n Args:\n tool_config: Optional tool configuration. Should include:\n - tool_id: Unique identifier for this memory tool instance (from user_tools._id)\n This ensures each user's tool configuration has isolated memories\n user_id: The authenticated user's id (should come from decoded_token[\"sub\"]).\n \"\"\"\n self.user_id: Optional[str] = user_id\n\n # Get tool_id from configuration (passed from user_tools._id in production)\n # In production, tool_id is the MongoDB ObjectId string from user_tools collection\n if tool_config and \"tool_id\" in tool_config:\n self.tool_id = tool_config[\"tool_id\"]\n elif user_id:\n # Fallback for backward compatibility or testing\n self.tool_id = f\"default_{user_id}\"\n else:\n # Last resort fallback (shouldn't happen in normal use)\n self.tool_id = str(uuid.uuid4())\n\n db = MongoDB.get_client()[settings.MONGO_DB_NAME]\n self.collection = db[\"memories\"]\n\n # -----------------------------\n # Action implementations\n # -----------------------------\n def execute_action(self, action_name: str, **kwargs: Any) -> str:\n \"\"\"Execute an action by name.\n\n Args:\n action_name: One of view, create, str_replace, insert, delete, rename.\n **kwargs: Parameters for the action.\n\n Returns:\n A human-readable string result.\n \"\"\"\n if not self.user_id:\n return \"Error: MemoryTool requires a valid user_id.\"\n\n if action_name == \"view\":\n return self._view(\n kwargs.get(\"path\", \"/\"),\n kwargs.get(\"view_range\")\n )\n\n if action_name == \"create\":\n return self._create(\n kwargs.get(\"path\", \"\"),\n kwargs.get(\"file_text\", \"\")\n )\n\n if action_name == \"str_replace\":\n return self._str_replace(\n kwargs.get(\"path\", \"\"),\n kwargs.get(\"old_str\", \"\"),\n kwargs.get(\"new_str\", \"\")\n )\n\n if action_name == \"insert\":\n return self._insert(\n kwargs.get(\"path\", \"\"),\n kwargs.get(\"insert_line\", 1),\n kwargs.get(\"insert_text\", \"\")\n )\n\n if action_name == \"delete\":\n return self._delete(kwargs.get(\"path\", \"\"))\n\n if action_name == \"rename\":\n return self._rename(\n kwargs.get(\"old_path\", \"\"),\n kwargs.get(\"new_path\", \"\")\n )\n\n return f\"Unknown action: {action_name}\"\n\n def get_actions_metadata(self) -> List[Dict[str, Any]]:\n \"\"\"Return JSON metadata describing supported actions for tool schemas.\"\"\"\n return [\n {\n \"name\": \"view\",\n \"description\": \"Shows directory contents or file contents with optional line ranges.\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"path\": {\n \"type\": \"string\",\n \"description\": \"Path to file or directory (e.g., /notes.txt or /project/ or /).\"\n },\n \"view_range\": {\n \"type\": \"array\",\n \"items\": {\"type\": \"integer\"},\n \"description\": \"Optional [start_line, end_line] to view specific lines (1-indexed).\"\n }\n },\n \"required\": [\"path\"]\n },\n },\n {\n \"name\": \"create\",\n \"description\": \"Create or overwrite a file.\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"path\": {\n \"type\": \"string\",\n \"description\": \"File path to create (e.g., /notes.txt or /project/task.txt).\"\n },\n \"file_text\": {\n \"type\": \"string\",\n \"description\": \"Content to write to the file.\"\n }\n },\n \"required\": [\"path\", \"file_text\"]\n },\n },\n {\n \"name\": \"str_replace\",\n \"description\": \"Replace text in a file.\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"path\": {\n \"type\": \"string\",\n \"description\": \"File path (e.g., /notes.txt).\"\n },\n \"old_str\": {\n \"type\": \"string\",\n \"description\": \"String to find.\"\n },\n \"new_str\": {\n \"type\": \"string\",\n \"description\": \"String to replace with.\"\n }\n },\n \"required\": [\"path\", \"old_str\", \"new_str\"]\n },\n },\n {\n \"name\": \"insert\",\n \"description\": \"Insert text at a specific line in a file.\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"path\": {\n \"type\": \"string\",\n \"description\": \"File path (e.g., /notes.txt).\"\n },\n \"insert_line\": {\n \"type\": \"integer\",\n \"description\": \"Line number to insert at (1-indexed).\"\n },\n \"insert_text\": {\n \"type\": \"string\",\n \"description\": \"Text to insert.\"\n }\n },\n \"required\": [\"path\", \"insert_line\", \"insert_text\"]\n },\n },\n {\n \"name\": \"delete\",\n \"description\": \"Delete a file or directory.\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"path\": {\n \"type\": \"string\",\n \"description\": \"Path to delete (e.g., /notes.txt or /project/).\"\n }\n },\n \"required\": [\"path\"]\n },\n },\n {\n \"name\": \"rename\",\n \"description\": \"Rename or move a file/directory.\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"old_path\": {\n \"type\": \"string\",\n \"description\": \"Current path (e.g., /old.txt).\"\n },\n \"new_path\": {\n \"type\": \"string\",\n \"description\": \"New path (e.g., /new.txt).\"\n }\n },\n \"required\": [\"old_path\", \"new_path\"]\n },\n },\n ]\n\n def get_config_requirements(self) -> Dict[str, Any]:\n \"\"\"Return configuration requirements.\"\"\"\n return {}\n\n # -----------------------------\n # Path validation\n # -----------------------------\n def _validate_path(self, path: str) -> Optional[str]:\n \"\"\"Validate and normalize path.\n\n Args:\n path: User-provided path.\n\n Returns:\n Normalized path or None if invalid.\n \"\"\"\n if not path:\n return None\n\n # Remove any leading/trailing whitespace\n path = path.strip()\n\n # Preserve whether path ends with / (indicates directory)\n is_directory = path.endswith(\"/\")\n\n # Ensure path starts with / for consistency\n if not path.startswith(\"/\"):\n path = \"/\" + path\n\n # Check for directory traversal patterns\n if \"..\" in path or path.count(\"//\") > 0:\n return None\n\n # Normalize the path\n try:\n # Convert to Path object and resolve to canonical form\n normalized = str(Path(path).as_posix())\n\n # Ensure it still starts with /\n if not normalized.startswith(\"/\"):\n return None\n\n # Preserve trailing slash for directories\n if is_directory and not normalized.endswith(\"/\") and normalized != \"/\":\n normalized = normalized + \"/\"\n\n return normalized\n except Exception:\n return None\n\n # -----------------------------\n # Internal helpers\n # -----------------------------\n def _view(self, path: str, view_range: Optional[List[int]] = None) -> str:\n \"\"\"View directory contents or file contents.\"\"\"\n validated_path = self._validate_path(path)\n if not validated_path:\n return \"Error: Invalid path.\"\n\n # Check if viewing directory (ends with / or is root)\n if validated_path == \"/\" or validated_path.endswith(\"/\"):\n return self._view_directory(validated_path)\n\n # Otherwise view file\n return self._view_file(validated_path, view_range)\n\n def _view_directory(self, path: str) -> str:\n \"\"\"List files in a directory.\"\"\"\n # Ensure path ends with / for proper prefix matching\n search_path = path if path.endswith(\"/\") else path + \"/\"\n\n # Find all files that start with this directory path\n query = {\n \"user_id\": self.user_id,\n \"tool_id\": self.tool_id,\n \"path\": {\"$regex\": f\"^{re.escape(search_path)}\"}\n }\n\n docs = list(self.collection.find(query, {\"path\": 1}))\n\n if not docs:\n return f\"Directory: {path}\\n(empty)\"\n\n # Extract filenames relative to the directory\n files = []\n for doc in docs:\n file_path = doc[\"path\"]\n # Remove the directory prefix\n if file_path.startswith(search_path):\n relative = file_path[len(search_path):]\n if relative:\n files.append(relative)\n\n files.sort()\n file_list = \"\\n\".join(f\"- {f}\" for f in files)\n return f\"Directory: {path}\\n{file_list}\"\n\n def _view_file(self, path: str, view_range: Optional[List[int]] = None) -> str:\n \"\"\"View file contents with optional line range.\"\"\"\n doc = self.collection.find_one({\"user_id\": self.user_id, \"tool_id\": self.tool_id, \"path\": path})\n\n if not doc or not doc.get(\"content\"):\n return f\"Error: File not found: {path}\"\n\n content = str(doc[\"content\"])\n\n # Apply view_range if specified\n if view_range and len(view_range) == 2:\n lines = content.split(\"\\n\")\n start, end = view_range\n # Convert to 0-indexed\n start_idx = max(0, start - 1)\n end_idx = min(len(lines), end)\n\n if start_idx >= len(lines):\n return f\"Error: Line range out of bounds. File has {len(lines)} lines.\"\n\n selected_lines = lines[start_idx:end_idx]\n # Add line numbers (enumerate with 1-based start)\n numbered_lines = [f\"{i}: {line}\" for i, line in enumerate(selected_lines, start=start)]\n return \"\\n\".join(numbered_lines)\n\n return content\n\n def _create(self, path: str, file_text: str) -> str:\n \"\"\"Create or overwrite a file.\"\"\"\n validated_path = self._validate_path(path)\n if not validated_path:\n return \"Error: Invalid path.\"\n\n if validated_path == \"/\" or validated_path.endswith(\"/\"):\n return \"Error: Cannot create a file at directory path.\"\n\n self.collection.update_one(\n {\"user_id\": self.user_id, \"tool_id\": self.tool_id, \"path\": validated_path},\n {\n \"$set\": {\n \"content\": file_text,\n \"updated_at\": datetime.now()\n }\n },\n upsert=True\n )\n\n return f\"File created: {validated_path}\"\n\n def _str_replace(self, path: str, old_str: str, new_str: str) -> str:\n \"\"\"Replace text in a file.\"\"\"\n validated_path = self._validate_path(path)\n if not validated_path:\n return \"Error: Invalid path.\"\n\n if not old_str:\n return \"Error: old_str is required.\"\n\n doc = self.collection.find_one({\"user_id\": self.user_id, \"tool_id\": self.tool_id, \"path\": validated_path})\n\n if not doc or not doc.get(\"content\"):\n return f\"Error: File not found: {validated_path}\"\n\n current_content = str(doc[\"content\"])\n\n # Check if old_str exists (case-insensitive)\n if old_str.lower() not in current_content.lower():\n return f\"Error: String '{old_str}' not found in file.\"\n\n # Replace the string (case-insensitive)\n import re as regex_module\n updated_content = regex_module.sub(regex_module.escape(old_str), new_str, current_content, flags=regex_module.IGNORECASE)\n\n self.collection.update_one(\n {\"user_id\": self.user_id, \"tool_id\": self.tool_id, \"path\": validated_path},\n {\n \"$set\": {\n \"content\": updated_content,\n \"updated_at\": datetime.now()\n }\n }\n )\n\n return f\"File updated: {validated_path}\"\n\n def _insert(self, path: str, insert_line: int, insert_text: str) -> str:\n \"\"\"Insert text at a specific line.\"\"\"\n validated_path = self._validate_path(path)\n if not validated_path:\n return \"Error: Invalid path.\"\n\n if not insert_text:\n return \"Error: insert_text is required.\"\n\n doc = self.collection.find_one({\"user_id\": self.user_id, \"tool_id\": self.tool_id, \"path\": validated_path})\n\n if not doc or not doc.get(\"content\"):\n return f\"Error: File not found: {validated_path}\"\n\n current_content = str(doc[\"content\"])\n lines = current_content.split(\"\\n\")\n\n # Convert to 0-indexed\n index = insert_line - 1\n if index < 0 or index > len(lines):\n return f\"Error: Invalid line number. File has {len(lines)} lines.\"\n\n lines.insert(index, insert_text)\n updated_content = \"\\n\".join(lines)\n\n self.collection.update_one(\n {\"user_id\": self.user_id, \"tool_id\": self.tool_id, \"path\": validated_path},\n {\n \"$set\": {\n \"content\": updated_content,\n \"updated_at\": datetime.now()\n }\n }\n )\n\n return f\"Text inserted at line {insert_line} in {validated_path}\"\n\n def _delete(self, path: str) -> str:\n \"\"\"Delete a file or directory.\"\"\"\n validated_path = self._validate_path(path)\n if not validated_path:\n return \"Error: Invalid path.\"\n\n if validated_path == \"/\":\n # Delete all files for this user and tool\n result = self.collection.delete_many({\"user_id\": self.user_id, \"tool_id\": self.tool_id})\n return f\"Deleted {result.deleted_count} file(s) from memory.\"\n\n # Check if it's a directory (ends with /)\n if validated_path.endswith(\"/\"):\n # Delete all files in directory\n result = self.collection.delete_many({\n \"user_id\": self.user_id,\n \"tool_id\": self.tool_id,\n \"path\": {\"$regex\": f\"^{re.escape(validated_path)}\"}\n })\n return f\"Deleted directory and {result.deleted_count} file(s).\"\n\n # Try to delete as directory first (without trailing slash)\n # Check if any files start with this path + /\n search_path = validated_path + \"/\"\n directory_result = self.collection.delete_many({\n \"user_id\": self.user_id,\n \"tool_id\": self.tool_id,\n \"path\": {\"$regex\": f\"^{re.escape(search_path)}\"}\n })\n\n if directory_result.deleted_count > 0:\n return f\"Deleted directory and {directory_result.deleted_count} file(s).\"\n\n # Delete single file\n result = self.collection.delete_one({\n \"user_id\": self.user_id,\n \"tool_id\": self.tool_id,\n \"path\": validated_path\n })\n\n if result.deleted_count:\n return f\"Deleted: {validated_path}\"\n return f\"Error: File not found: {validated_path}\"\n\n def _rename(self, old_path: str, new_path: str) -> str:\n \"\"\"Rename or move a file/directory.\"\"\"\n validated_old = self._validate_path(old_path)\n validated_new = self._validate_path(new_path)\n\n if not validated_old or not validated_new:\n return \"Error: Invalid path.\"\n\n if validated_old == \"/\" or validated_new == \"/\":\n return \"Error: Cannot rename root directory.\"\n\n # Check if renaming a directory\n if validated_old.endswith(\"/\"):\n # Ensure validated_new also ends with / for proper path replacement\n if not validated_new.endswith(\"/\"):\n validated_new = validated_new + \"/\"\n\n # Find all files in the old directory\n docs = list(self.collection.find({\n \"user_id\": self.user_id,\n \"tool_id\": self.tool_id,\n \"path\": {\"$regex\": f\"^{re.escape(validated_old)}\"}\n }))\n\n if not docs:\n return f\"Error: Directory not found: {validated_old}\"\n\n # Update paths for all files\n for doc in docs:\n old_file_path = doc[\"path\"]\n new_file_path = old_file_path.replace(validated_old, validated_new, 1)\n\n self.collection.update_one(\n {\"_id\": doc[\"_id\"]},\n {\"$set\": {\"path\": new_file_path, \"updated_at\": datetime.now()}}\n )\n\n return f\"Renamed directory: {validated_old} -> {validated_new} ({len(docs)} files)\"\n\n # Rename single file\n doc = self.collection.find_one({\n \"user_id\": self.user_id,\n \"tool_id\": self.tool_id,\n \"path\": validated_old\n })\n\n if not doc:\n return f\"Error: File not found: {validated_old}\"\n\n # Check if new path already exists\n existing = self.collection.find_one({\n \"user_id\": self.user_id,\n \"tool_id\": self.tool_id,\n \"path\": validated_new\n })\n\n if existing:\n return f\"Error: File already exists at {validated_new}\"\n\n # Delete the old document and create a new one with the new path\n self.collection.delete_one({\"user_id\": self.user_id, \"tool_id\": self.tool_id, \"path\": validated_old})\n self.collection.insert_one({\n \"user_id\": self.user_id,\n \"tool_id\": self.tool_id,\n \"path\": validated_new,\n \"content\": doc.get(\"content\", \"\"),\n \"updated_at\": datetime.now()\n })\n\n return f\"Renamed: {validated_old} -> {validated_new}\"\n" }, { "path": "application/agents/tools/notes.py", "content": "from datetime import datetime\nfrom typing import Any, Dict, List, Optional\nimport uuid\n\nfrom .base import Tool\nfrom application.core.mongo_db import MongoDB\nfrom application.core.settings import settings\n\n\nclass NotesTool(Tool):\n \"\"\"Notepad\n\n Single note. Supports viewing, overwriting, string replacement.\n \"\"\"\n\n def __init__(self, tool_config: Optional[Dict[str, Any]] = None, user_id: Optional[str] = None) -> None:\n \"\"\"Initialize the tool.\n\n Args:\n tool_config: Optional tool configuration. Should include:\n - tool_id: Unique identifier for this notes tool instance (from user_tools._id)\n This ensures each user's tool configuration has isolated notes\n user_id: The authenticated user's id (should come from decoded_token[\"sub\"]).\n \"\"\"\n self.user_id: Optional[str] = user_id\n\n # Get tool_id from configuration (passed from user_tools._id in production)\n # In production, tool_id is the MongoDB ObjectId string from user_tools collection\n if tool_config and \"tool_id\" in tool_config:\n self.tool_id = tool_config[\"tool_id\"]\n elif user_id:\n # Fallback for backward compatibility or testing\n self.tool_id = f\"default_{user_id}\"\n else:\n # Last resort fallback (shouldn't happen in normal use)\n self.tool_id = str(uuid.uuid4())\n\n db = MongoDB.get_client()[settings.MONGO_DB_NAME]\n self.collection = db[\"notes\"]\n\n self._last_artifact_id: Optional[str] = None\n\n # -----------------------------\n # Action implementations\n # -----------------------------\n def execute_action(self, action_name: str, **kwargs: Any) -> str:\n \"\"\"Execute an action by name.\n\n Args:\n action_name: One of view, overwrite, str_replace, insert, delete.\n **kwargs: Parameters for the action.\n\n Returns:\n A human-readable string result.\n \"\"\"\n if not self.user_id:\n return \"Error: NotesTool requires a valid user_id.\"\n\n self._last_artifact_id = None\n\n if action_name == \"view\":\n return self._get_note()\n\n if action_name == \"overwrite\":\n return self._overwrite_note(kwargs.get(\"text\", \"\"))\n\n if action_name == \"str_replace\":\n return self._str_replace(kwargs.get(\"old_str\", \"\"), kwargs.get(\"new_str\", \"\"))\n\n if action_name == \"insert\":\n return self._insert(kwargs.get(\"line_number\", 1), kwargs.get(\"text\", \"\"))\n\n if action_name == \"delete\":\n return self._delete_note()\n\n return f\"Unknown action: {action_name}\"\n\n def get_actions_metadata(self) -> List[Dict[str, Any]]:\n \"\"\"Return JSON metadata describing supported actions for tool schemas.\"\"\"\n return [\n {\n \"name\": \"view\",\n \"description\": \"Retrieve the user's note.\",\n \"parameters\": {\"type\": \"object\", \"properties\": {}},\n },\n {\n \"name\": \"overwrite\",\n \"description\": \"Replace the entire note content (creates if doesn't exist).\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"text\": {\"type\": \"string\", \"description\": \"New note content.\"}\n },\n \"required\": [\"text\"],\n },\n },\n {\n \"name\": \"str_replace\",\n \"description\": \"Replace occurrences of old_str with new_str in the note.\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"old_str\": {\"type\": \"string\", \"description\": \"String to find.\"},\n \"new_str\": {\"type\": \"string\", \"description\": \"String to replace with.\"}\n },\n \"required\": [\"old_str\", \"new_str\"],\n },\n },\n {\n \"name\": \"insert\",\n \"description\": \"Insert text at the specified line number (1-indexed).\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"line_number\": {\"type\": \"integer\", \"description\": \"Line number to insert at (1-indexed).\"},\n \"text\": {\"type\": \"string\", \"description\": \"Text to insert.\"}\n },\n \"required\": [\"line_number\", \"text\"],\n },\n },\n {\n \"name\": \"delete\",\n \"description\": \"Delete the user's note.\",\n \"parameters\": {\"type\": \"object\", \"properties\": {}},\n },\n ]\n\n def get_config_requirements(self) -> Dict[str, Any]:\n \"\"\"Return configuration requirements (none for now).\"\"\"\n return {}\n\n def get_artifact_id(self, action_name: str, **kwargs: Any) -> Optional[str]:\n return self._last_artifact_id\n\n # -----------------------------\n # Internal helpers (single-note)\n # -----------------------------\n def _get_note(self) -> str:\n doc = self.collection.find_one({\"user_id\": self.user_id, \"tool_id\": self.tool_id})\n if not doc or not doc.get(\"note\"):\n return \"No note found.\"\n if doc.get(\"_id\") is not None:\n self._last_artifact_id = str(doc.get(\"_id\"))\n return str(doc[\"note\"])\n\n def _overwrite_note(self, content: str) -> str:\n content = (content or \"\").strip()\n if not content:\n return \"Note content required.\"\n result = self.collection.find_one_and_update(\n {\"user_id\": self.user_id, \"tool_id\": self.tool_id},\n {\"$set\": {\"note\": content, \"updated_at\": datetime.utcnow()}},\n upsert=True,\n return_document=True,\n )\n if result and result.get(\"_id\") is not None:\n self._last_artifact_id = str(result.get(\"_id\"))\n return \"Note saved.\"\n\n def _str_replace(self, old_str: str, new_str: str) -> str:\n if not old_str:\n return \"old_str is required.\"\n\n doc = self.collection.find_one({\"user_id\": self.user_id, \"tool_id\": self.tool_id})\n if not doc or not doc.get(\"note\"):\n return \"No note found.\"\n\n current_note = str(doc[\"note\"])\n\n # Case-insensitive search\n if old_str.lower() not in current_note.lower():\n return f\"String '{old_str}' not found in note.\"\n\n # Case-insensitive replacement\n import re\n updated_note = re.sub(re.escape(old_str), new_str, current_note, flags=re.IGNORECASE)\n\n result = self.collection.find_one_and_update(\n {\"user_id\": self.user_id, \"tool_id\": self.tool_id},\n {\"$set\": {\"note\": updated_note, \"updated_at\": datetime.utcnow()}},\n return_document=True,\n )\n if result and result.get(\"_id\") is not None:\n self._last_artifact_id = str(result.get(\"_id\"))\n return \"Note updated.\"\n\n def _insert(self, line_number: int, text: str) -> str:\n if not text:\n return \"Text is required.\"\n\n doc = self.collection.find_one({\"user_id\": self.user_id, \"tool_id\": self.tool_id})\n if not doc or not doc.get(\"note\"):\n return \"No note found.\"\n\n current_note = str(doc[\"note\"])\n lines = current_note.split(\"\\n\")\n\n # Convert to 0-indexed and validate\n index = line_number - 1\n if index < 0 or index > len(lines):\n return f\"Invalid line number. Note has {len(lines)} lines.\"\n\n lines.insert(index, text)\n updated_note = \"\\n\".join(lines)\n\n result = self.collection.find_one_and_update(\n {\"user_id\": self.user_id, \"tool_id\": self.tool_id},\n {\"$set\": {\"note\": updated_note, \"updated_at\": datetime.utcnow()}},\n return_document=True,\n )\n if result and result.get(\"_id\") is not None:\n self._last_artifact_id = str(result.get(\"_id\"))\n return \"Text inserted.\"\n\n def _delete_note(self) -> str:\n doc = self.collection.find_one_and_delete(\n {\"user_id\": self.user_id, \"tool_id\": self.tool_id}\n )\n if not doc:\n return \"No note found to delete.\"\n if doc.get(\"_id\") is not None:\n self._last_artifact_id = str(doc.get(\"_id\"))\n return \"Note deleted.\"\n" }, { "path": "application/agents/tools/ntfy.py", "content": "import requests\nfrom application.agents.tools.base import Tool\n\nclass NtfyTool(Tool):\n \"\"\"\n Ntfy Tool\n A tool for sending notifications to ntfy topics on a specified server.\n \"\"\"\n\n def __init__(self, config):\n \"\"\"\n Initialize the NtfyTool with configuration.\n\n Args:\n config (dict): Configuration dictionary containing the access token.\n \"\"\"\n self.config = config\n self.token = config.get(\"token\", \"\")\n\n def execute_action(self, action_name, **kwargs):\n \"\"\"\n Execute the specified action with given parameters.\n\n Args:\n action_name (str): Name of the action to execute.\n **kwargs: Parameters for the action, including server_url.\n\n Returns:\n dict: Result of the action with status code and message.\n\n Raises:\n ValueError: If the action name is unknown.\n \"\"\"\n actions = {\n \"ntfy_send_message\": self._send_message,\n }\n if action_name in actions:\n return actions[action_name](**kwargs)\n else:\n raise ValueError(f\"Unknown action: {action_name}\")\n\n def _send_message(self, server_url, message, topic, title=None, priority=None):\n \"\"\"\n Send a message to an ntfy topic on the specified server.\n\n Args:\n server_url (str): Base URL of the ntfy server (e.g., https://ntfy.sh).\n message (str): The message text to send.\n topic (str): The topic to send the message to.\n title (str, optional): Title of the notification.\n priority (int, optional): Priority of the notification (1-5).\n\n Returns:\n dict: Response with status code and a confirmation message.\n\n Raises:\n ValueError: If priority is not an integer between 1 and 5.\n \"\"\"\n url = f\"{server_url.rstrip('/')}/{topic}\"\n headers = {}\n if title:\n headers[\"X-Title\"] = title\n if priority:\n try:\n priority = int(priority)\n except (ValueError, TypeError):\n raise ValueError(\"Priority must be convertible to an integer\")\n if priority < 1 or priority > 5:\n raise ValueError(\"Priority must be an integer between 1 and 5\")\n headers[\"X-Priority\"] = str(priority)\n if self.token:\n headers[\"Authorization\"] = f\"Basic {self.token}\"\n data = message.encode(\"utf-8\")\n response = requests.post(url, headers=headers, data=data)\n return {\"status_code\": response.status_code, \"message\": \"Message sent\"}\n\n def get_actions_metadata(self):\n \"\"\"\n Provide metadata about available actions.\n\n Returns:\n list: List of dictionaries describing each action.\n \"\"\"\n return [\n {\n \"name\": \"ntfy_send_message\",\n \"description\": \"Send a notification to an ntfy topic\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"server_url\": {\n \"type\": \"string\",\n \"description\": \"Base URL of the ntfy server\",\n },\n \"message\": {\n \"type\": \"string\",\n \"description\": \"Text to send in the notification\",\n },\n \"topic\": {\n \"type\": \"string\",\n \"description\": \"Topic to send the notification to\",\n },\n \"title\": {\n \"type\": \"string\",\n \"description\": \"Title of the notification (optional)\",\n },\n \"priority\": {\n \"type\": \"integer\",\n \"description\": \"Priority of the notification (1-5, optional)\",\n },\n },\n \"required\": [\"server_url\", \"message\", \"topic\"],\n \"additionalProperties\": False,\n },\n },\n ]\n\n def get_config_requirements(self):\n return {\n \"token\": {\n \"type\": \"string\",\n \"label\": \"Access Token\",\n \"description\": \"Ntfy access token for authentication\",\n \"required\": True,\n \"secret\": True,\n \"order\": 1,\n },\n }" }, { "path": "application/agents/tools/postgres.py", "content": "import logging\n\nimport psycopg2\n\nfrom application.agents.tools.base import Tool\n\nlogger = logging.getLogger(__name__)\n\n\nclass PostgresTool(Tool):\n \"\"\"\n PostgreSQL Database Tool\n A tool for connecting to a PostgreSQL database using a connection string,\n executing SQL queries, and retrieving schema information.\n \"\"\"\n\n def __init__(self, config):\n self.config = config\n self.connection_string = config.get(\"token\", \"\")\n\n def execute_action(self, action_name, **kwargs):\n actions = {\n \"postgres_execute_sql\": self._execute_sql,\n \"postgres_get_schema\": self._get_schema,\n }\n if action_name not in actions:\n raise ValueError(f\"Unknown action: {action_name}\")\n return actions[action_name](**kwargs)\n\n def _execute_sql(self, sql_query):\n \"\"\"\n Executes an SQL query against the PostgreSQL database using a connection string.\n \"\"\"\n conn = None\n try:\n conn = psycopg2.connect(self.connection_string)\n cur = conn.cursor()\n cur.execute(sql_query)\n conn.commit()\n\n if sql_query.strip().lower().startswith(\"select\"):\n column_names = (\n [desc[0] for desc in cur.description] if cur.description else []\n )\n results = []\n rows = cur.fetchall()\n for row in rows:\n results.append(dict(zip(column_names, row)))\n response_data = {\"data\": results, \"column_names\": column_names}\n else:\n row_count = cur.rowcount\n response_data = {\n \"message\": f\"Query executed successfully, {row_count} rows affected.\"\n }\n\n cur.close()\n return {\n \"status_code\": 200,\n \"message\": \"SQL query executed successfully.\",\n \"response_data\": response_data,\n }\n\n except psycopg2.Error as e:\n error_message = f\"Database error: {e}\"\n logger.error(\"PostgreSQL execute_sql error: %s\", e)\n return {\n \"status_code\": 500,\n \"message\": \"Failed to execute SQL query.\",\n \"error\": error_message,\n }\n finally:\n if conn:\n conn.close()\n\n def _get_schema(self, db_name):\n \"\"\"\n Retrieves the schema of the PostgreSQL database using a connection string.\n \"\"\"\n conn = None\n try:\n conn = psycopg2.connect(self.connection_string)\n cur = conn.cursor()\n\n cur.execute(\n \"\"\"\n SELECT\n table_name,\n column_name,\n data_type,\n column_default,\n is_nullable\n FROM\n information_schema.columns\n WHERE\n table_schema = 'public'\n ORDER BY\n table_name,\n ordinal_position;\n \"\"\"\n )\n\n schema_data = {}\n for row in cur.fetchall():\n table_name, column_name, data_type, column_default, is_nullable = row\n if table_name not in schema_data:\n schema_data[table_name] = []\n schema_data[table_name].append(\n {\n \"column_name\": column_name,\n \"data_type\": data_type,\n \"column_default\": column_default,\n \"is_nullable\": is_nullable,\n }\n )\n\n cur.close()\n return {\n \"status_code\": 200,\n \"message\": \"Database schema retrieved successfully.\",\n \"schema\": schema_data,\n }\n\n except psycopg2.Error as e:\n error_message = f\"Database error: {e}\"\n logger.error(\"PostgreSQL get_schema error: %s\", e)\n return {\n \"status_code\": 500,\n \"message\": \"Failed to retrieve database schema.\",\n \"error\": error_message,\n }\n finally:\n if conn:\n conn.close()\n\n def get_actions_metadata(self):\n return [\n {\n \"name\": \"postgres_execute_sql\",\n \"description\": \"Execute an SQL query against the PostgreSQL database and return the results. Use this tool to interact with the database, e.g., retrieve specific data or perform updates. Only SELECT queries will return data, other queries will return execution status.\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"sql_query\": {\n \"type\": \"string\",\n \"description\": \"The SQL query to execute.\",\n },\n },\n \"required\": [\"sql_query\"],\n \"additionalProperties\": False,\n },\n },\n {\n \"name\": \"postgres_get_schema\",\n \"description\": \"Retrieve the schema of the PostgreSQL database, including tables and their columns. Use this to understand the database structure before executing queries. db_name is 'default' if not provided.\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"db_name\": {\n \"type\": \"string\",\n \"description\": \"The name of the database to retrieve the schema for.\",\n },\n },\n \"required\": [\"db_name\"],\n \"additionalProperties\": False,\n },\n },\n ]\n\n def get_config_requirements(self):\n return {\n \"token\": {\n \"type\": \"string\",\n \"label\": \"Connection String\",\n \"description\": \"PostgreSQL database connection string\",\n \"required\": True,\n \"secret\": True,\n \"order\": 1,\n },\n }\n" }, { "path": "application/agents/tools/read_webpage.py", "content": "import requests\nfrom markdownify import markdownify\nfrom application.agents.tools.base import Tool\nfrom application.core.url_validation import validate_url, SSRFError\n\nclass ReadWebpageTool(Tool):\n \"\"\"\n Read Webpage (browser)\n A tool to fetch the HTML content of a URL and convert it to Markdown.\n \"\"\"\n\n def __init__(self, config=None):\n \"\"\"\n Initializes the tool.\n :param config: Optional configuration dictionary. Not used by this tool.\n \"\"\"\n self.config = config\n\n def execute_action(self, action_name: str, **kwargs) -> str:\n \"\"\"\n Executes the specified action. For this tool, the only action is 'read_webpage'.\n\n :param action_name: The name of the action to execute. Should be 'read_webpage'.\n :param kwargs: Keyword arguments, must include 'url'.\n :return: The Markdown content of the webpage or an error message.\n \"\"\"\n if action_name != \"read_webpage\":\n return f\"Error: Unknown action '{action_name}'. This tool only supports 'read_webpage'.\"\n\n url = kwargs.get(\"url\")\n if not url:\n return \"Error: URL parameter is missing.\"\n\n # Validate URL to prevent SSRF attacks\n try:\n url = validate_url(url)\n except SSRFError as e:\n return f\"Error: URL validation failed - {e}\"\n\n try:\n response = requests.get(url, timeout=10, headers={'User-Agent': 'DocsGPT-Agent/1.0'})\n response.raise_for_status() # Raise an exception for HTTP errors (4xx or 5xx)\n \n html_content = response.text\n #soup = BeautifulSoup(html_content, 'html.parser')\n \n \n markdown_content = markdownify(html_content, heading_style=\"ATX\", newline_style=\"BACKSLASH\")\n \n return markdown_content\n\n except requests.exceptions.RequestException as e:\n return f\"Error fetching URL {url}: {e}\"\n except Exception as e:\n return f\"Error processing URL {url}: {e}\"\n\n def get_actions_metadata(self):\n \"\"\"\n Returns metadata for the actions supported by this tool.\n \"\"\"\n return [\n {\n \"name\": \"read_webpage\",\n \"description\": \"Fetches the HTML content of a given URL and returns it as clean Markdown text. Input must be a valid URL.\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"url\": {\n \"type\": \"string\",\n \"description\": \"The fully qualified URL of the webpage to read (e.g., 'https://www.example.com').\",\n }\n },\n \"required\": [\"url\"],\n \"additionalProperties\": False,\n },\n }\n ]\n\n def get_config_requirements(self):\n \"\"\"\n Returns a dictionary describing the configuration requirements for the tool.\n This tool does not require any specific configuration.\n \"\"\"\n return {}\n" }, { "path": "application/agents/tools/spec_parser.py", "content": "\"\"\"\nAPI Specification Parser\n\nParses OpenAPI 3.x and Swagger 2.0 specifications and converts them\nto API Tool action definitions for use in DocsGPT.\n\"\"\"\n\nimport json\nimport logging\nimport re\nfrom typing import Any, Dict, List, Optional, Tuple\n\nimport yaml\n\nlogger = logging.getLogger(__name__)\n\nSUPPORTED_METHODS = frozenset(\n {\"get\", \"post\", \"put\", \"delete\", \"patch\", \"head\", \"options\"}\n)\n\n\ndef parse_spec(spec_content: str) -> Tuple[Dict[str, Any], List[Dict[str, Any]]]:\n \"\"\"\n Parse an API specification and convert operations to action definitions.\n\n Supports OpenAPI 3.x and Swagger 2.0 formats in JSON or YAML.\n\n Args:\n spec_content: Raw specification content as string\n\n Returns:\n Tuple of (metadata dict, list of action dicts)\n\n Raises:\n ValueError: If the spec is invalid or uses an unsupported format\n \"\"\"\n spec = _load_spec(spec_content)\n _validate_spec(spec)\n\n is_swagger = \"swagger\" in spec\n metadata = _extract_metadata(spec, is_swagger)\n actions = _extract_actions(spec, is_swagger)\n\n return metadata, actions\n\n\ndef _load_spec(content: str) -> Dict[str, Any]:\n \"\"\"Parse spec content from JSON or YAML string.\"\"\"\n content = content.strip()\n if not content:\n raise ValueError(\"Empty specification content\")\n try:\n if content.startswith(\"{\"):\n return json.loads(content)\n return yaml.safe_load(content)\n except json.JSONDecodeError as e:\n raise ValueError(f\"Invalid JSON format: {e.msg}\")\n except yaml.YAMLError as e:\n raise ValueError(f\"Invalid YAML format: {e}\")\n\n\ndef _validate_spec(spec: Dict[str, Any]) -> None:\n \"\"\"Validate spec version and required fields.\"\"\"\n if not isinstance(spec, dict):\n raise ValueError(\"Specification must be a valid object\")\n openapi_version = spec.get(\"openapi\", \"\")\n swagger_version = spec.get(\"swagger\", \"\")\n\n if not (openapi_version.startswith(\"3.\") or swagger_version == \"2.0\"):\n raise ValueError(\n \"Unsupported specification version. Expected OpenAPI 3.x or Swagger 2.0\"\n )\n if \"paths\" not in spec or not spec[\"paths\"]:\n raise ValueError(\"No API paths defined in the specification\")\n\n\ndef _extract_metadata(spec: Dict[str, Any], is_swagger: bool) -> Dict[str, Any]:\n \"\"\"Extract API metadata from specification.\"\"\"\n info = spec.get(\"info\", {})\n base_url = _get_base_url(spec, is_swagger)\n\n return {\n \"title\": info.get(\"title\", \"Untitled API\"),\n \"description\": (info.get(\"description\", \"\") or \"\")[:500],\n \"version\": info.get(\"version\", \"\"),\n \"base_url\": base_url,\n }\n\n\ndef _get_base_url(spec: Dict[str, Any], is_swagger: bool) -> str:\n \"\"\"Extract base URL from spec (handles both OpenAPI 3.x and Swagger 2.0).\"\"\"\n if is_swagger:\n schemes = spec.get(\"schemes\", [\"https\"])\n host = spec.get(\"host\", \"\")\n base_path = spec.get(\"basePath\", \"\")\n if host:\n scheme = schemes[0] if schemes else \"https\"\n return f\"{scheme}://{host}{base_path}\".rstrip(\"/\")\n return \"\"\n servers = spec.get(\"servers\", [])\n if servers and isinstance(servers, list) and servers[0].get(\"url\"):\n return servers[0][\"url\"].rstrip(\"/\")\n return \"\"\n\n\ndef _extract_actions(spec: Dict[str, Any], is_swagger: bool) -> List[Dict[str, Any]]:\n \"\"\"Extract all API operations as action definitions.\"\"\"\n actions = []\n paths = spec.get(\"paths\", {})\n base_url = _get_base_url(spec, is_swagger)\n\n components = spec.get(\"components\", {})\n definitions = spec.get(\"definitions\", {})\n\n for path, path_item in paths.items():\n if not isinstance(path_item, dict):\n continue\n path_params = path_item.get(\"parameters\", [])\n\n for method in SUPPORTED_METHODS:\n operation = path_item.get(method)\n if not isinstance(operation, dict):\n continue\n try:\n action = _build_action(\n path=path,\n method=method,\n operation=operation,\n path_params=path_params,\n base_url=base_url,\n components=components,\n definitions=definitions,\n is_swagger=is_swagger,\n )\n actions.append(action)\n except Exception as e:\n logger.warning(\n f\"Failed to parse operation {method.upper()} {path}: {e}\"\n )\n continue\n return actions\n\n\ndef _build_action(\n path: str,\n method: str,\n operation: Dict[str, Any],\n path_params: List[Dict],\n base_url: str,\n components: Dict[str, Any],\n definitions: Dict[str, Any],\n is_swagger: bool,\n) -> Dict[str, Any]:\n \"\"\"Build a single action from an API operation.\"\"\"\n action_name = _generate_action_name(operation, method, path)\n full_url = f\"{base_url}{path}\" if base_url else path\n\n all_params = path_params + operation.get(\"parameters\", [])\n query_params, headers = _categorize_parameters(all_params, components, definitions)\n\n body, body_content_type = _extract_request_body(\n operation, components, definitions, is_swagger\n )\n\n description = operation.get(\"summary\", \"\") or operation.get(\"description\", \"\")\n\n return {\n \"name\": action_name,\n \"url\": full_url,\n \"method\": method.upper(),\n \"description\": (description or \"\")[:500],\n \"query_params\": {\"type\": \"object\", \"properties\": query_params},\n \"headers\": {\"type\": \"object\", \"properties\": headers},\n \"body\": {\"type\": \"object\", \"properties\": body},\n \"body_content_type\": body_content_type,\n \"active\": True,\n }\n\n\ndef _generate_action_name(operation: Dict[str, Any], method: str, path: str) -> str:\n \"\"\"Generate a valid action name from operationId or method+path.\"\"\"\n if operation.get(\"operationId\"):\n name = operation[\"operationId\"]\n else:\n path_slug = re.sub(r\"[{}]\", \"\", path)\n path_slug = re.sub(r\"[^a-zA-Z0-9]\", \"_\", path_slug)\n path_slug = re.sub(r\"_+\", \"_\", path_slug).strip(\"_\")\n name = f\"{method}_{path_slug}\"\n name = re.sub(r\"[^a-zA-Z0-9_-]\", \"_\", name)\n return name[:64]\n\n\ndef _categorize_parameters(\n parameters: List[Dict],\n components: Dict[str, Any],\n definitions: Dict[str, Any],\n) -> Tuple[Dict, Dict]:\n \"\"\"Categorize parameters into query params and headers.\"\"\"\n query_params = {}\n headers = {}\n\n for param in parameters:\n resolved = _resolve_ref(param, components, definitions)\n if not resolved or \"name\" not in resolved:\n continue\n location = resolved.get(\"in\", \"query\")\n prop = _param_to_property(resolved)\n\n if location in (\"query\", \"path\"):\n query_params[resolved[\"name\"]] = prop\n elif location == \"header\":\n headers[resolved[\"name\"]] = prop\n return query_params, headers\n\n\ndef _param_to_property(param: Dict) -> Dict[str, Any]:\n \"\"\"Convert an API parameter to an action property definition.\"\"\"\n schema = param.get(\"schema\", {})\n param_type = schema.get(\"type\", param.get(\"type\", \"string\"))\n\n mapped_type = \"integer\" if param_type in (\"integer\", \"number\") else \"string\"\n\n return {\n \"type\": mapped_type,\n \"description\": (param.get(\"description\", \"\") or \"\")[:200],\n \"value\": \"\",\n \"filled_by_llm\": param.get(\"required\", False),\n \"required\": param.get(\"required\", False),\n }\n\n\ndef _extract_request_body(\n operation: Dict[str, Any],\n components: Dict[str, Any],\n definitions: Dict[str, Any],\n is_swagger: bool,\n) -> Tuple[Dict, str]:\n \"\"\"Extract request body schema and content type.\"\"\"\n content_types = [\n \"application/json\",\n \"application/x-www-form-urlencoded\",\n \"multipart/form-data\",\n \"text/plain\",\n \"application/xml\",\n ]\n\n if is_swagger:\n consumes = operation.get(\"consumes\", [])\n body_param = next(\n (p for p in operation.get(\"parameters\", []) if p.get(\"in\") == \"body\"), None\n )\n if not body_param:\n return {}, \"application/json\"\n selected_type = consumes[0] if consumes else \"application/json\"\n schema = body_param.get(\"schema\", {})\n else:\n request_body = operation.get(\"requestBody\", {})\n if not request_body:\n return {}, \"application/json\"\n request_body = _resolve_ref(request_body, components, definitions)\n content = request_body.get(\"content\", {})\n\n selected_type = \"application/json\"\n schema = {}\n\n for ct in content_types:\n if ct in content:\n selected_type = ct\n schema = content[ct].get(\"schema\", {})\n break\n if not schema and content:\n first_type = next(iter(content))\n selected_type = first_type\n schema = content[first_type].get(\"schema\", {})\n properties = _schema_to_properties(schema, components, definitions)\n return properties, selected_type\n\n\ndef _schema_to_properties(\n schema: Dict,\n components: Dict[str, Any],\n definitions: Dict[str, Any],\n depth: int = 0,\n) -> Dict[str, Any]:\n \"\"\"Convert schema to action body properties (limited depth to prevent recursion).\"\"\"\n if depth > 3:\n return {}\n schema = _resolve_ref(schema, components, definitions)\n if not schema or not isinstance(schema, dict):\n return {}\n properties = {}\n schema_type = schema.get(\"type\", \"object\")\n\n if schema_type == \"object\":\n required_fields = set(schema.get(\"required\", []))\n for prop_name, prop_schema in schema.get(\"properties\", {}).items():\n resolved = _resolve_ref(prop_schema, components, definitions)\n if not isinstance(resolved, dict):\n continue\n prop_type = resolved.get(\"type\", \"string\")\n mapped_type = \"integer\" if prop_type in (\"integer\", \"number\") else \"string\"\n\n properties[prop_name] = {\n \"type\": mapped_type,\n \"description\": (resolved.get(\"description\", \"\") or \"\")[:200],\n \"value\": \"\",\n \"filled_by_llm\": prop_name in required_fields,\n \"required\": prop_name in required_fields,\n }\n return properties\n\n\ndef _resolve_ref(\n obj: Any,\n components: Dict[str, Any],\n definitions: Dict[str, Any],\n) -> Optional[Dict]:\n \"\"\"Resolve $ref references in the specification.\"\"\"\n if not isinstance(obj, dict):\n return obj if isinstance(obj, dict) else None\n if \"$ref\" not in obj:\n return obj\n ref_path = obj[\"$ref\"]\n\n if ref_path.startswith(\"#/components/\"):\n parts = ref_path.replace(\"#/components/\", \"\").split(\"/\")\n return _traverse_path(components, parts)\n elif ref_path.startswith(\"#/definitions/\"):\n parts = ref_path.replace(\"#/definitions/\", \"\").split(\"/\")\n return _traverse_path(definitions, parts)\n logger.debug(f\"Unsupported ref path: {ref_path}\")\n return None\n\n\ndef _traverse_path(obj: Dict, parts: List[str]) -> Optional[Dict]:\n \"\"\"Traverse a nested dictionary using path parts.\"\"\"\n try:\n for part in parts:\n obj = obj[part]\n return obj if isinstance(obj, dict) else None\n except (KeyError, TypeError):\n return None\n" }, { "path": "application/agents/tools/telegram.py", "content": "import logging\n\nimport requests\n\nfrom application.agents.tools.base import Tool\n\nlogger = logging.getLogger(__name__)\n\n\nclass TelegramTool(Tool):\n \"\"\"\n Telegram Bot\n A flexible Telegram tool for performing various actions (e.g., sending messages, images).\n Requires a bot token and chat ID for configuration\n \"\"\"\n\n def __init__(self, config):\n self.config = config\n self.token = config.get(\"token\", \"\")\n\n def execute_action(self, action_name, **kwargs):\n actions = {\n \"telegram_send_message\": self._send_message,\n \"telegram_send_image\": self._send_image,\n }\n if action_name not in actions:\n raise ValueError(f\"Unknown action: {action_name}\")\n return actions[action_name](**kwargs)\n\n def _send_message(self, text, chat_id):\n logger.debug(\"Sending Telegram message to chat_id=%s\", chat_id)\n url = f\"https://api.telegram.org/bot{self.token}/sendMessage\"\n payload = {\"chat_id\": chat_id, \"text\": text}\n response = requests.post(url, data=payload)\n return {\"status_code\": response.status_code, \"message\": \"Message sent\"}\n\n def _send_image(self, image_url, chat_id):\n logger.debug(\"Sending Telegram image to chat_id=%s\", chat_id)\n url = f\"https://api.telegram.org/bot{self.token}/sendPhoto\"\n payload = {\"chat_id\": chat_id, \"photo\": image_url}\n response = requests.post(url, data=payload)\n return {\"status_code\": response.status_code, \"message\": \"Image sent\"}\n\n def get_actions_metadata(self):\n return [\n {\n \"name\": \"telegram_send_message\",\n \"description\": \"Send a notification to Telegram chat\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"text\": {\n \"type\": \"string\",\n \"description\": \"Text to send in the notification\",\n },\n \"chat_id\": {\n \"type\": \"string\",\n \"description\": \"Chat ID to send the notification to\",\n },\n },\n \"required\": [\"text\"],\n \"additionalProperties\": False,\n },\n },\n {\n \"name\": \"telegram_send_image\",\n \"description\": \"Send an image to the Telegram chat\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"image_url\": {\n \"type\": \"string\",\n \"description\": \"URL of the image to send\",\n },\n \"chat_id\": {\n \"type\": \"string\",\n \"description\": \"Chat ID to send the image to\",\n },\n },\n \"required\": [\"image_url\"],\n \"additionalProperties\": False,\n },\n },\n ]\n\n def get_config_requirements(self):\n return {\n \"token\": {\n \"type\": \"string\",\n \"label\": \"Bot Token\",\n \"description\": \"Telegram bot token for authentication\",\n \"required\": True,\n \"secret\": True,\n \"order\": 1,\n },\n }\n" }, { "path": "application/agents/tools/todo_list.py", "content": "from datetime import datetime\nfrom typing import Any, Dict, List, Optional\nimport uuid\n\nfrom .base import Tool\nfrom application.core.mongo_db import MongoDB\nfrom application.core.settings import settings\n\n\nclass TodoListTool(Tool):\n \"\"\"Todo List\n\n Manages todo items for users. Supports creating, viewing, updating, and deleting todos.\n \"\"\"\n\n def __init__(self, tool_config: Optional[Dict[str, Any]] = None, user_id: Optional[str] = None) -> None:\n \"\"\"Initialize the tool.\n\n Args:\n tool_config: Optional tool configuration. Should include:\n - tool_id: Unique identifier for this todo list tool instance (from user_tools._id)\n This ensures each user's tool configuration has isolated todos\n user_id: The authenticated user's id (should come from decoded_token[\"sub\"]).\n \"\"\"\n self.user_id: Optional[str] = user_id\n\n # Get tool_id from configuration (passed from user_tools._id in production)\n # In production, tool_id is the MongoDB ObjectId string from user_tools collection\n if tool_config and \"tool_id\" in tool_config:\n self.tool_id = tool_config[\"tool_id\"]\n elif user_id:\n # Fallback for backward compatibility or testing\n self.tool_id = f\"default_{user_id}\"\n else:\n # Last resort fallback (shouldn't happen in normal use)\n self.tool_id = str(uuid.uuid4())\n\n db = MongoDB.get_client()[settings.MONGO_DB_NAME]\n self.collection = db[\"todos\"]\n\n self._last_artifact_id: Optional[str] = None\n\n # -----------------------------\n # Action implementations\n # -----------------------------\n def execute_action(self, action_name: str, **kwargs: Any) -> str:\n \"\"\"Execute an action by name.\n\n Args:\n action_name: One of list, create, get, update, complete, delete.\n **kwargs: Parameters for the action.\n\n Returns:\n A human-readable string result.\n \"\"\"\n if not self.user_id:\n return \"Error: TodoListTool requires a valid user_id.\"\n\n self._last_artifact_id = None\n\n if action_name == \"list\":\n return self._list()\n\n if action_name == \"create\":\n return self._create(kwargs.get(\"title\", \"\"))\n\n if action_name == \"get\":\n return self._get(kwargs.get(\"todo_id\"))\n\n if action_name == \"update\":\n return self._update(\n kwargs.get(\"todo_id\"),\n kwargs.get(\"title\", \"\")\n )\n\n if action_name == \"complete\":\n return self._complete(kwargs.get(\"todo_id\"))\n\n if action_name == \"delete\":\n return self._delete(kwargs.get(\"todo_id\"))\n\n return f\"Unknown action: {action_name}\"\n\n def get_actions_metadata(self) -> List[Dict[str, Any]]:\n \"\"\"Return JSON metadata describing supported actions for tool schemas.\"\"\"\n return [\n {\n \"name\": \"list\",\n \"description\": \"List all todos for the user.\",\n \"parameters\": {\"type\": \"object\", \"properties\": {}},\n },\n {\n \"name\": \"create\",\n \"description\": \"Create a new todo item.\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"title\": {\n \"type\": \"string\",\n \"description\": \"Title of the todo item.\"\n }\n },\n \"required\": [\"title\"],\n },\n },\n {\n \"name\": \"get\",\n \"description\": \"Get a specific todo by ID.\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"todo_id\": {\n \"type\": \"integer\",\n \"description\": \"The ID of the todo to retrieve.\"\n }\n },\n \"required\": [\"todo_id\"],\n },\n },\n {\n \"name\": \"update\",\n \"description\": \"Update a todo's title by ID.\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"todo_id\": {\n \"type\": \"integer\",\n \"description\": \"The ID of the todo to update.\"\n },\n \"title\": {\n \"type\": \"string\",\n \"description\": \"The new title for the todo.\"\n }\n },\n \"required\": [\"todo_id\", \"title\"],\n },\n },\n {\n \"name\": \"complete\",\n \"description\": \"Mark a todo as completed.\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"todo_id\": {\n \"type\": \"integer\",\n \"description\": \"The ID of the todo to mark as completed.\"\n }\n },\n \"required\": [\"todo_id\"],\n },\n },\n {\n \"name\": \"delete\",\n \"description\": \"Delete a specific todo by ID.\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"todo_id\": {\n \"type\": \"integer\",\n \"description\": \"The ID of the todo to delete.\"\n }\n },\n \"required\": [\"todo_id\"],\n },\n },\n ]\n\n def get_config_requirements(self) -> Dict[str, Any]:\n \"\"\"Return configuration requirements.\"\"\"\n return {}\n\n def get_artifact_id(self, action_name: str, **kwargs: Any) -> Optional[str]:\n return self._last_artifact_id\n\n # -----------------------------\n # Internal helpers\n # -----------------------------\n def _coerce_todo_id(self, value: Optional[Any]) -> Optional[int]:\n \"\"\"Convert todo identifiers to sequential integers.\"\"\"\n if value is None:\n return None\n\n if isinstance(value, int):\n return value if value > 0 else None\n\n if isinstance(value, str):\n stripped = value.strip()\n if stripped.isdigit():\n numeric_value = int(stripped)\n return numeric_value if numeric_value > 0 else None\n\n return None\n\n def _get_next_todo_id(self) -> int:\n \"\"\"Get the next sequential todo_id for this user and tool.\n\n Returns a simple integer (1, 2, 3, ...) scoped to this user/tool.\n With 5-10 todos max, scanning is negligible.\n \"\"\"\n query = {\"user_id\": self.user_id, \"tool_id\": self.tool_id}\n todos = list(self.collection.find(query, {\"todo_id\": 1}))\n\n # Find the maximum todo_id\n max_id = 0\n for todo in todos:\n todo_id = self._coerce_todo_id(todo.get(\"todo_id\"))\n if todo_id is not None:\n max_id = max(max_id, todo_id)\n\n return max_id + 1\n\n def _list(self) -> str:\n \"\"\"List all todos for the user.\"\"\"\n query = {\"user_id\": self.user_id, \"tool_id\": self.tool_id}\n todos = list(self.collection.find(query))\n\n if not todos:\n return \"No todos found.\"\n\n result_lines = [\"Todos:\"]\n for doc in todos:\n todo_id = doc.get(\"todo_id\")\n title = doc.get(\"title\", \"Untitled\")\n status = doc.get(\"status\", \"open\")\n\n line = f\"[{todo_id}] {title} ({status})\"\n result_lines.append(line)\n\n return \"\\n\".join(result_lines)\n\n def _create(self, title: str) -> str:\n \"\"\"Create a new todo item.\"\"\"\n title = (title or \"\").strip()\n if not title:\n return \"Error: Title is required.\"\n\n now = datetime.now()\n todo_id = self._get_next_todo_id()\n\n doc = {\n \"todo_id\": todo_id,\n \"user_id\": self.user_id,\n \"tool_id\": self.tool_id,\n \"title\": title,\n \"status\": \"open\",\n \"created_at\": now,\n \"updated_at\": now,\n }\n insert_result = self.collection.insert_one(doc)\n inserted_id = getattr(insert_result, \"inserted_id\", None) or doc.get(\"_id\")\n if inserted_id is not None:\n self._last_artifact_id = str(inserted_id)\n return f\"Todo created with ID {todo_id}: {title}\"\n\n def _get(self, todo_id: Optional[Any]) -> str:\n \"\"\"Get a specific todo by ID.\"\"\"\n parsed_todo_id = self._coerce_todo_id(todo_id)\n if parsed_todo_id is None:\n return \"Error: todo_id must be a positive integer.\"\n\n query = {\"user_id\": self.user_id, \"tool_id\": self.tool_id, \"todo_id\": parsed_todo_id}\n doc = self.collection.find_one(query)\n\n if not doc:\n return f\"Error: Todo with ID {parsed_todo_id} not found.\"\n\n if doc.get(\"_id\") is not None:\n self._last_artifact_id = str(doc.get(\"_id\"))\n\n title = doc.get(\"title\", \"Untitled\")\n status = doc.get(\"status\", \"open\")\n\n result = f\"Todo [{parsed_todo_id}]:\\nTitle: {title}\\nStatus: {status}\"\n\n return result\n\n def _update(self, todo_id: Optional[Any], title: str) -> str:\n \"\"\"Update a todo's title by ID.\"\"\"\n parsed_todo_id = self._coerce_todo_id(todo_id)\n if parsed_todo_id is None:\n return \"Error: todo_id must be a positive integer.\"\n\n title = (title or \"\").strip()\n if not title:\n return \"Error: Title is required.\"\n\n query = {\"user_id\": self.user_id, \"tool_id\": self.tool_id, \"todo_id\": parsed_todo_id}\n doc = self.collection.find_one_and_update(\n query,\n {\"$set\": {\"title\": title, \"updated_at\": datetime.now()}},\n )\n if not doc:\n return f\"Error: Todo with ID {parsed_todo_id} not found.\"\n\n if doc.get(\"_id\") is not None:\n self._last_artifact_id = str(doc.get(\"_id\"))\n\n return f\"Todo {parsed_todo_id} updated to: {title}\"\n\n def _complete(self, todo_id: Optional[Any]) -> str:\n \"\"\"Mark a todo as completed.\"\"\"\n parsed_todo_id = self._coerce_todo_id(todo_id)\n if parsed_todo_id is None:\n return \"Error: todo_id must be a positive integer.\"\n\n query = {\"user_id\": self.user_id, \"tool_id\": self.tool_id, \"todo_id\": parsed_todo_id}\n doc = self.collection.find_one_and_update(\n query,\n {\"$set\": {\"status\": \"completed\", \"updated_at\": datetime.now()}},\n )\n if not doc:\n return f\"Error: Todo with ID {parsed_todo_id} not found.\"\n\n if doc.get(\"_id\") is not None:\n self._last_artifact_id = str(doc.get(\"_id\"))\n\n return f\"Todo {parsed_todo_id} marked as completed.\"\n\n def _delete(self, todo_id: Optional[Any]) -> str:\n \"\"\"Delete a specific todo by ID.\"\"\"\n parsed_todo_id = self._coerce_todo_id(todo_id)\n if parsed_todo_id is None:\n return \"Error: todo_id must be a positive integer.\"\n\n query = {\"user_id\": self.user_id, \"tool_id\": self.tool_id, \"todo_id\": parsed_todo_id}\n doc = self.collection.find_one_and_delete(query)\n if not doc:\n return f\"Error: Todo with ID {parsed_todo_id} not found.\"\n\n if doc.get(\"_id\") is not None:\n self._last_artifact_id = str(doc.get(\"_id\"))\n\n return f\"Todo {parsed_todo_id} deleted.\"\n" }, { "path": "application/agents/tools/tool_action_parser.py", "content": "import json\nimport logging\n\nlogger = logging.getLogger(__name__)\n\n\nclass ToolActionParser:\n def __init__(self, llm_type):\n self.llm_type = llm_type\n self.parsers = {\n \"OpenAILLM\": self._parse_openai_llm,\n \"GoogleLLM\": self._parse_google_llm,\n }\n\n def parse_args(self, call):\n parser = self.parsers.get(self.llm_type, self._parse_openai_llm)\n return parser(call)\n\n def _parse_openai_llm(self, call):\n try:\n call_args = json.loads(call.arguments)\n tool_parts = call.name.split(\"_\")\n\n # If the tool name doesn't contain an underscore, it's likely a hallucinated tool\n if len(tool_parts) < 2:\n logger.warning(\n f\"Invalid tool name format: {call.name}. Expected format: action_name_tool_id\"\n )\n return None, None, None\n\n tool_id = tool_parts[-1]\n action_name = \"_\".join(tool_parts[:-1])\n\n # Validate that tool_id looks like a numerical ID\n if not tool_id.isdigit():\n logger.warning(\n f\"Tool ID '{tool_id}' is not numerical. This might be a hallucinated tool call.\"\n )\n\n except (AttributeError, TypeError, json.JSONDecodeError) as e:\n logger.error(f\"Error parsing OpenAI LLM call: {e}\")\n return None, None, None\n return tool_id, action_name, call_args\n\n def _parse_google_llm(self, call):\n try:\n call_args = call.arguments\n tool_parts = call.name.split(\"_\")\n\n # If the tool name doesn't contain an underscore, it's likely a hallucinated tool\n if len(tool_parts) < 2:\n logger.warning(\n f\"Invalid tool name format: {call.name}. Expected format: action_name_tool_id\"\n )\n return None, None, None\n\n tool_id = tool_parts[-1]\n action_name = \"_\".join(tool_parts[:-1])\n\n # Validate that tool_id looks like a numerical ID\n if not tool_id.isdigit():\n logger.warning(\n f\"Tool ID '{tool_id}' is not numerical. This might be a hallucinated tool call.\"\n )\n\n except (AttributeError, TypeError) as e:\n logger.error(f\"Error parsing Google LLM call: {e}\")\n return None, None, None\n return tool_id, action_name, call_args\n" }, { "path": "application/agents/tools/tool_manager.py", "content": "import importlib\nimport inspect\nimport os\nimport pkgutil\n\nfrom application.agents.tools.base import Tool\n\n\nclass ToolManager:\n def __init__(self, config):\n self.config = config\n self.tools = {}\n self.load_tools()\n\n def load_tools(self):\n tools_dir = os.path.join(os.path.dirname(__file__))\n for finder, name, ispkg in pkgutil.iter_modules([tools_dir]):\n if name == \"base\" or name.startswith(\"__\"):\n continue\n module = importlib.import_module(f\"application.agents.tools.{name}\")\n for member_name, obj in inspect.getmembers(module, inspect.isclass):\n if issubclass(obj, Tool) and obj is not Tool:\n tool_config = self.config.get(name, {})\n self.tools[name] = obj(tool_config)\n\n def load_tool(self, tool_name, tool_config, user_id=None):\n self.config[tool_name] = tool_config\n module = importlib.import_module(f\"application.agents.tools.{tool_name}\")\n for member_name, obj in inspect.getmembers(module, inspect.isclass):\n if issubclass(obj, Tool) and obj is not Tool:\n if tool_name in {\"mcp_tool\", \"notes\", \"memory\", \"todo_list\"} and user_id:\n return obj(tool_config, user_id)\n else:\n return obj(tool_config)\n\n def execute_action(self, tool_name, action_name, user_id=None, **kwargs):\n if tool_name not in self.tools:\n raise ValueError(f\"Tool '{tool_name}' not loaded\")\n if tool_name in {\"mcp_tool\", \"memory\", \"todo_list\", \"notes\"} and user_id:\n tool_config = self.config.get(tool_name, {})\n tool = self.load_tool(tool_name, tool_config, user_id)\n return tool.execute_action(action_name, **kwargs)\n return self.tools[tool_name].execute_action(action_name, **kwargs)\n\n def get_all_actions_metadata(self):\n metadata = []\n for tool in self.tools.values():\n metadata.extend(tool.get_actions_metadata())\n return metadata\n" }, { "path": "application/agents/workflow_agent.py", "content": "import logging\nfrom datetime import datetime, timezone\nfrom typing import Any, Dict, Generator, Optional\n\nfrom application.agents.base import BaseAgent\nfrom application.agents.workflows.schemas import (\n ExecutionStatus,\n Workflow,\n WorkflowEdge,\n WorkflowGraph,\n WorkflowNode,\n WorkflowRun,\n)\nfrom application.agents.workflows.workflow_engine import WorkflowEngine\nfrom application.core.mongo_db import MongoDB\nfrom application.core.settings import settings\nfrom application.logging import log_activity, LogContext\n\nlogger = logging.getLogger(__name__)\n\n\nclass WorkflowAgent(BaseAgent):\n \"\"\"A specialized agent that executes predefined workflows.\"\"\"\n\n def __init__(\n self,\n *args,\n workflow_id: Optional[str] = None,\n workflow: Optional[Dict[str, Any]] = None,\n workflow_owner: Optional[str] = None,\n **kwargs,\n ):\n super().__init__(*args, **kwargs)\n self.workflow_id = workflow_id\n self.workflow_owner = workflow_owner\n self._workflow_data = workflow\n self._engine: Optional[WorkflowEngine] = None\n\n @log_activity()\n def gen(\n self, query: str, log_context: LogContext = None\n ) -> Generator[Dict[str, str], None, None]:\n yield from self._gen_inner(query, log_context)\n\n def _gen_inner(\n self, query: str, log_context: LogContext\n ) -> Generator[Dict[str, str], None, None]:\n graph = self._load_workflow_graph()\n if not graph:\n yield {\"type\": \"error\", \"error\": \"Failed to load workflow configuration.\"}\n return\n self._engine = WorkflowEngine(graph, self)\n yield from self._engine.execute({}, query)\n self._save_workflow_run(query)\n\n def _load_workflow_graph(self) -> Optional[WorkflowGraph]:\n if self._workflow_data:\n return self._parse_embedded_workflow()\n if self.workflow_id:\n return self._load_from_database()\n return None\n\n def _parse_embedded_workflow(self) -> Optional[WorkflowGraph]:\n try:\n nodes_data = self._workflow_data.get(\"nodes\", [])\n edges_data = self._workflow_data.get(\"edges\", [])\n\n workflow = Workflow(\n name=self._workflow_data.get(\"name\", \"Embedded Workflow\"),\n description=self._workflow_data.get(\"description\"),\n )\n\n nodes = []\n for n in nodes_data:\n node_config = n.get(\"data\", {})\n nodes.append(\n WorkflowNode(\n id=n[\"id\"],\n workflow_id=self.workflow_id or \"embedded\",\n type=n[\"type\"],\n title=n.get(\"title\", \"Node\"),\n description=n.get(\"description\"),\n position=n.get(\"position\", {\"x\": 0, \"y\": 0}),\n config=node_config,\n )\n )\n edges = []\n for e in edges_data:\n edges.append(\n WorkflowEdge(\n id=e[\"id\"],\n workflow_id=self.workflow_id or \"embedded\",\n source=e.get(\"source\") or e.get(\"source_id\"),\n target=e.get(\"target\") or e.get(\"target_id\"),\n sourceHandle=e.get(\"sourceHandle\") or e.get(\"source_handle\"),\n targetHandle=e.get(\"targetHandle\") or e.get(\"target_handle\"),\n )\n )\n return WorkflowGraph(workflow=workflow, nodes=nodes, edges=edges)\n except Exception as e:\n logger.error(f\"Invalid embedded workflow: {e}\")\n return None\n\n def _load_from_database(self) -> Optional[WorkflowGraph]:\n try:\n from bson.objectid import ObjectId\n\n if not self.workflow_id or not ObjectId.is_valid(self.workflow_id):\n logger.error(f\"Invalid workflow ID: {self.workflow_id}\")\n return None\n owner_id = self.workflow_owner\n if not owner_id and isinstance(self.decoded_token, dict):\n owner_id = self.decoded_token.get(\"sub\")\n if not owner_id:\n logger.error(\n f\"Workflow owner not available for workflow load: {self.workflow_id}\"\n )\n return None\n\n mongo = MongoDB.get_client()\n db = mongo[settings.MONGO_DB_NAME]\n\n workflows_coll = db[\"workflows\"]\n workflow_nodes_coll = db[\"workflow_nodes\"]\n workflow_edges_coll = db[\"workflow_edges\"]\n\n workflow_doc = workflows_coll.find_one(\n {\"_id\": ObjectId(self.workflow_id), \"user\": owner_id}\n )\n if not workflow_doc:\n logger.error(\n f\"Workflow {self.workflow_id} not found or inaccessible for user {owner_id}\"\n )\n return None\n workflow = Workflow(**workflow_doc)\n graph_version = workflow_doc.get(\"current_graph_version\", 1)\n try:\n graph_version = int(graph_version)\n if graph_version <= 0:\n graph_version = 1\n except (ValueError, TypeError):\n graph_version = 1\n\n nodes_docs = list(\n workflow_nodes_coll.find(\n {\"workflow_id\": self.workflow_id, \"graph_version\": graph_version}\n )\n )\n if not nodes_docs and graph_version == 1:\n nodes_docs = list(\n workflow_nodes_coll.find(\n {\n \"workflow_id\": self.workflow_id,\n \"graph_version\": {\"$exists\": False},\n }\n )\n )\n nodes = [WorkflowNode(**doc) for doc in nodes_docs]\n\n edges_docs = list(\n workflow_edges_coll.find(\n {\"workflow_id\": self.workflow_id, \"graph_version\": graph_version}\n )\n )\n if not edges_docs and graph_version == 1:\n edges_docs = list(\n workflow_edges_coll.find(\n {\n \"workflow_id\": self.workflow_id,\n \"graph_version\": {\"$exists\": False},\n }\n )\n )\n edges = [WorkflowEdge(**doc) for doc in edges_docs]\n\n return WorkflowGraph(workflow=workflow, nodes=nodes, edges=edges)\n except Exception as e:\n logger.error(f\"Failed to load workflow from database: {e}\")\n return None\n\n def _save_workflow_run(self, query: str) -> None:\n if not self._engine:\n return\n try:\n mongo = MongoDB.get_client()\n db = mongo[settings.MONGO_DB_NAME]\n workflow_runs_coll = db[\"workflow_runs\"]\n\n run = WorkflowRun(\n workflow_id=self.workflow_id or \"unknown\",\n status=self._determine_run_status(),\n inputs={\"query\": query},\n outputs=self._serialize_state(self._engine.state),\n steps=self._engine.get_execution_summary(),\n created_at=datetime.now(timezone.utc),\n completed_at=datetime.now(timezone.utc),\n )\n\n workflow_runs_coll.insert_one(run.to_mongo_doc())\n except Exception as e:\n logger.error(f\"Failed to save workflow run: {e}\")\n\n def _determine_run_status(self) -> ExecutionStatus:\n if not self._engine or not self._engine.execution_log:\n return ExecutionStatus.COMPLETED\n for log in self._engine.execution_log:\n if log.get(\"status\") == ExecutionStatus.FAILED.value:\n return ExecutionStatus.FAILED\n return ExecutionStatus.COMPLETED\n\n def _serialize_state(self, state: Dict[str, Any]) -> Dict[str, Any]:\n serialized: Dict[str, Any] = {}\n for key, value in state.items():\n serialized[key] = self._serialize_state_value(value)\n return serialized\n\n def _serialize_state_value(self, value: Any) -> Any:\n if isinstance(value, dict):\n return {\n str(dict_key): self._serialize_state_value(dict_value)\n for dict_key, dict_value in value.items()\n }\n if isinstance(value, list):\n return [self._serialize_state_value(item) for item in value]\n if isinstance(value, tuple):\n return [self._serialize_state_value(item) for item in value]\n if isinstance(value, datetime):\n return value.isoformat()\n if isinstance(value, (str, int, float, bool, type(None))):\n return value\n return str(value)\n" }, { "path": "application/agents/workflows/cel_evaluator.py", "content": "from typing import Any, Dict\n\nimport celpy\nimport celpy.celtypes\n\n\nclass CelEvaluationError(Exception):\n pass\n\n\ndef _convert_value(value: Any) -> Any:\n if isinstance(value, bool):\n return celpy.celtypes.BoolType(value)\n if isinstance(value, int):\n return celpy.celtypes.IntType(value)\n if isinstance(value, float):\n return celpy.celtypes.DoubleType(value)\n if isinstance(value, str):\n return celpy.celtypes.StringType(value)\n if isinstance(value, list):\n return celpy.celtypes.ListType([_convert_value(item) for item in value])\n if isinstance(value, dict):\n return celpy.celtypes.MapType(\n {celpy.celtypes.StringType(k): _convert_value(v) for k, v in value.items()}\n )\n if value is None:\n return celpy.celtypes.BoolType(False)\n return celpy.celtypes.StringType(str(value))\n\n\ndef build_activation(state: Dict[str, Any]) -> Dict[str, Any]:\n return {k: _convert_value(v) for k, v in state.items()}\n\n\ndef evaluate_cel(expression: str, state: Dict[str, Any]) -> Any:\n if not expression or not expression.strip():\n raise CelEvaluationError(\"Empty expression\")\n try:\n env = celpy.Environment()\n ast = env.compile(expression)\n program = env.program(ast)\n activation = build_activation(state)\n result = program.evaluate(activation)\n except celpy.CELEvalError as exc:\n raise CelEvaluationError(f\"CEL evaluation error: {exc}\") from exc\n except Exception as exc:\n raise CelEvaluationError(f\"CEL error: {exc}\") from exc\n return cel_to_python(result)\n\n\ndef cel_to_python(value: Any) -> Any:\n if isinstance(value, celpy.celtypes.BoolType):\n return bool(value)\n if isinstance(value, celpy.celtypes.IntType):\n return int(value)\n if isinstance(value, celpy.celtypes.DoubleType):\n return float(value)\n if isinstance(value, celpy.celtypes.StringType):\n return str(value)\n if isinstance(value, celpy.celtypes.ListType):\n return [cel_to_python(item) for item in value]\n if isinstance(value, celpy.celtypes.MapType):\n return {str(k): cel_to_python(v) for k, v in value.items()}\n return value\n" }, { "path": "application/agents/workflows/node_agent.py", "content": "\"\"\"Workflow Node Agents - defines specialized agents for workflow nodes.\"\"\"\n\nfrom typing import Any, Dict, List, Optional, Type\n\nfrom application.agents.base import BaseAgent\nfrom application.agents.classic_agent import ClassicAgent\nfrom application.agents.react_agent import ReActAgent\nfrom application.agents.workflows.schemas import AgentType\n\n\nclass ToolFilterMixin:\n \"\"\"Mixin that filters fetched tools to only those specified in tool_ids.\"\"\"\n\n _allowed_tool_ids: List[str]\n\n def _get_user_tools(self, user: str = \"local\") -> Dict[str, Dict[str, Any]]:\n all_tools = super()._get_user_tools(user)\n if not self._allowed_tool_ids:\n return {}\n filtered_tools = {\n tool_id: tool\n for tool_id, tool in all_tools.items()\n if str(tool.get(\"_id\", \"\")) in self._allowed_tool_ids\n }\n return filtered_tools\n\n def _get_tools(self, api_key: str = None) -> Dict[str, Dict[str, Any]]:\n all_tools = super()._get_tools(api_key)\n if not self._allowed_tool_ids:\n return {}\n filtered_tools = {\n tool_id: tool\n for tool_id, tool in all_tools.items()\n if str(tool.get(\"_id\", \"\")) in self._allowed_tool_ids\n }\n return filtered_tools\n\n\nclass WorkflowNodeClassicAgent(ToolFilterMixin, ClassicAgent):\n\n def __init__(\n self,\n endpoint: str,\n llm_name: str,\n model_id: str,\n api_key: str,\n tool_ids: Optional[List[str]] = None,\n **kwargs,\n ):\n super().__init__(\n endpoint=endpoint,\n llm_name=llm_name,\n model_id=model_id,\n api_key=api_key,\n **kwargs,\n )\n self._allowed_tool_ids = tool_ids or []\n\n\nclass WorkflowNodeReActAgent(ToolFilterMixin, ReActAgent):\n\n def __init__(\n self,\n endpoint: str,\n llm_name: str,\n model_id: str,\n api_key: str,\n tool_ids: Optional[List[str]] = None,\n **kwargs,\n ):\n super().__init__(\n endpoint=endpoint,\n llm_name=llm_name,\n model_id=model_id,\n api_key=api_key,\n **kwargs,\n )\n self._allowed_tool_ids = tool_ids or []\n\n\nclass WorkflowNodeAgentFactory:\n\n _agents: Dict[AgentType, Type[BaseAgent]] = {\n AgentType.CLASSIC: WorkflowNodeClassicAgent,\n AgentType.REACT: WorkflowNodeReActAgent,\n }\n\n @classmethod\n def create(\n cls,\n agent_type: AgentType,\n endpoint: str,\n llm_name: str,\n model_id: str,\n api_key: str,\n tool_ids: Optional[List[str]] = None,\n **kwargs,\n ) -> BaseAgent:\n agent_class = cls._agents.get(agent_type)\n if not agent_class:\n raise ValueError(f\"Unsupported agent type: {agent_type}\")\n return agent_class(\n endpoint=endpoint,\n llm_name=llm_name,\n model_id=model_id,\n api_key=api_key,\n tool_ids=tool_ids,\n **kwargs,\n )\n" }, { "path": "application/agents/workflows/schemas.py", "content": "from datetime import datetime, timezone\nfrom enum import Enum\nfrom typing import Any, Dict, List, Literal, Optional, Union\n\nfrom bson import ObjectId\nfrom pydantic import BaseModel, ConfigDict, Field, field_validator\n\n\nclass NodeType(str, Enum):\n START = \"start\"\n END = \"end\"\n AGENT = \"agent\"\n NOTE = \"note\"\n STATE = \"state\"\n CONDITION = \"condition\"\n\n\nclass AgentType(str, Enum):\n CLASSIC = \"classic\"\n REACT = \"react\"\n\n\nclass ExecutionStatus(str, Enum):\n PENDING = \"pending\"\n RUNNING = \"running\"\n COMPLETED = \"completed\"\n FAILED = \"failed\"\n\n\nclass Position(BaseModel):\n model_config = ConfigDict(extra=\"forbid\")\n x: float = 0.0\n y: float = 0.0\n\n\nclass AgentNodeConfig(BaseModel):\n model_config = ConfigDict(extra=\"allow\")\n agent_type: AgentType = AgentType.CLASSIC\n llm_name: Optional[str] = None\n system_prompt: str = \"You are a helpful assistant.\"\n prompt_template: str = \"\"\n output_variable: Optional[str] = None\n stream_to_user: bool = True\n tools: List[str] = Field(default_factory=list)\n sources: List[str] = Field(default_factory=list)\n chunks: str = \"2\"\n retriever: str = \"\"\n model_id: Optional[str] = None\n json_schema: Optional[Dict[str, Any]] = None\n\n\nclass ConditionCase(BaseModel):\n model_config = ConfigDict(extra=\"forbid\", populate_by_name=True)\n name: Optional[str] = None\n expression: str = \"\"\n source_handle: str = Field(..., alias=\"sourceHandle\")\n\n\nclass ConditionNodeConfig(BaseModel):\n model_config = ConfigDict(extra=\"allow\")\n mode: Literal[\"simple\", \"advanced\"] = \"simple\"\n cases: List[ConditionCase] = Field(default_factory=list)\n\n\nclass StateOperation(BaseModel):\n model_config = ConfigDict(extra=\"forbid\")\n expression: str = \"\"\n target_variable: str = \"\"\n\n\nclass WorkflowEdgeCreate(BaseModel):\n model_config = ConfigDict(populate_by_name=True)\n id: str\n workflow_id: str\n source_id: str = Field(..., alias=\"source\")\n target_id: str = Field(..., alias=\"target\")\n source_handle: Optional[str] = Field(None, alias=\"sourceHandle\")\n target_handle: Optional[str] = Field(None, alias=\"targetHandle\")\n\n\nclass WorkflowEdge(WorkflowEdgeCreate):\n mongo_id: Optional[str] = Field(None, alias=\"_id\")\n\n @field_validator(\"mongo_id\", mode=\"before\")\n @classmethod\n def convert_objectid(cls, v: Any) -> Optional[str]:\n if isinstance(v, ObjectId):\n return str(v)\n return v\n\n def to_mongo_doc(self) -> Dict[str, Any]:\n return {\n \"id\": self.id,\n \"workflow_id\": self.workflow_id,\n \"source_id\": self.source_id,\n \"target_id\": self.target_id,\n \"source_handle\": self.source_handle,\n \"target_handle\": self.target_handle,\n }\n\n\nclass WorkflowNodeCreate(BaseModel):\n model_config = ConfigDict(extra=\"allow\")\n id: str\n workflow_id: str\n type: NodeType\n title: str = \"Node\"\n description: Optional[str] = None\n position: Position = Field(default_factory=Position)\n config: Dict[str, Any] = Field(default_factory=dict)\n\n @field_validator(\"position\", mode=\"before\")\n @classmethod\n def parse_position(cls, v: Union[Dict[str, float], Position]) -> Position:\n if isinstance(v, dict):\n return Position(**v)\n return v\n\n\nclass WorkflowNode(WorkflowNodeCreate):\n mongo_id: Optional[str] = Field(None, alias=\"_id\")\n\n @field_validator(\"mongo_id\", mode=\"before\")\n @classmethod\n def convert_objectid(cls, v: Any) -> Optional[str]:\n if isinstance(v, ObjectId):\n return str(v)\n return v\n\n def to_mongo_doc(self) -> Dict[str, Any]:\n return {\n \"id\": self.id,\n \"workflow_id\": self.workflow_id,\n \"type\": self.type.value,\n \"title\": self.title,\n \"description\": self.description,\n \"position\": self.position.model_dump(),\n \"config\": self.config,\n }\n\n\nclass WorkflowCreate(BaseModel):\n model_config = ConfigDict(extra=\"allow\")\n name: str = \"New Workflow\"\n description: Optional[str] = None\n user: Optional[str] = None\n\n\nclass Workflow(WorkflowCreate):\n id: Optional[str] = Field(None, alias=\"_id\")\n created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))\n updated_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))\n\n @field_validator(\"id\", mode=\"before\")\n @classmethod\n def convert_objectid(cls, v: Any) -> Optional[str]:\n if isinstance(v, ObjectId):\n return str(v)\n return v\n\n def to_mongo_doc(self) -> Dict[str, Any]:\n return {\n \"name\": self.name,\n \"description\": self.description,\n \"user\": self.user,\n \"created_at\": self.created_at,\n \"updated_at\": self.updated_at,\n }\n\n\nclass WorkflowGraph(BaseModel):\n workflow: Workflow\n nodes: List[WorkflowNode] = Field(default_factory=list)\n edges: List[WorkflowEdge] = Field(default_factory=list)\n\n def get_node_by_id(self, node_id: str) -> Optional[WorkflowNode]:\n for node in self.nodes:\n if node.id == node_id:\n return node\n return None\n\n def get_start_node(self) -> Optional[WorkflowNode]:\n for node in self.nodes:\n if node.type == NodeType.START:\n return node\n return None\n\n def get_outgoing_edges(self, node_id: str) -> List[WorkflowEdge]:\n return [edge for edge in self.edges if edge.source_id == node_id]\n\n\nclass NodeExecutionLog(BaseModel):\n model_config = ConfigDict(extra=\"forbid\")\n node_id: str\n node_type: str\n status: ExecutionStatus\n started_at: datetime\n completed_at: Optional[datetime] = None\n error: Optional[str] = None\n state_snapshot: Dict[str, Any] = Field(default_factory=dict)\n\n\nclass WorkflowRunCreate(BaseModel):\n workflow_id: str\n inputs: Dict[str, str] = Field(default_factory=dict)\n\n\nclass WorkflowRun(BaseModel):\n model_config = ConfigDict(extra=\"allow\")\n id: Optional[str] = Field(None, alias=\"_id\")\n workflow_id: str\n status: ExecutionStatus = ExecutionStatus.PENDING\n inputs: Dict[str, str] = Field(default_factory=dict)\n outputs: Dict[str, Any] = Field(default_factory=dict)\n steps: List[NodeExecutionLog] = Field(default_factory=list)\n created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))\n completed_at: Optional[datetime] = None\n\n @field_validator(\"id\", mode=\"before\")\n @classmethod\n def convert_objectid(cls, v: Any) -> Optional[str]:\n if isinstance(v, ObjectId):\n return str(v)\n return v\n\n def to_mongo_doc(self) -> Dict[str, Any]:\n return {\n \"workflow_id\": self.workflow_id,\n \"status\": self.status.value,\n \"inputs\": self.inputs,\n \"outputs\": self.outputs,\n \"steps\": [step.model_dump() for step in self.steps],\n \"created_at\": self.created_at,\n \"completed_at\": self.completed_at,\n }\n" }, { "path": "application/agents/workflows/workflow_engine.py", "content": "import json\nimport logging\nfrom datetime import datetime, timezone\nfrom typing import Any, Dict, Generator, List, Optional, TYPE_CHECKING\n\nfrom application.agents.workflows.cel_evaluator import CelEvaluationError, evaluate_cel\nfrom application.agents.workflows.node_agent import WorkflowNodeAgentFactory\nfrom application.agents.workflows.schemas import (\n AgentNodeConfig,\n ConditionNodeConfig,\n ExecutionStatus,\n NodeExecutionLog,\n NodeType,\n WorkflowGraph,\n WorkflowNode,\n)\nfrom application.core.json_schema_utils import (\n JsonSchemaValidationError,\n normalize_json_schema_payload,\n)\nfrom application.error import sanitize_api_error\nfrom application.templates.namespaces import NamespaceManager\nfrom application.templates.template_engine import TemplateEngine, TemplateRenderError\n\ntry:\n import jsonschema\nexcept ImportError: # pragma: no cover - optional dependency in some deployments.\n jsonschema = None\n\nif TYPE_CHECKING:\n from application.agents.base import BaseAgent\nlogger = logging.getLogger(__name__)\n\nStateValue = Any\nWorkflowState = Dict[str, StateValue]\nTEMPLATE_RESERVED_NAMESPACES = {\"agent\", \"system\", \"source\", \"tools\", \"passthrough\"}\n\n\nclass WorkflowEngine:\n MAX_EXECUTION_STEPS = 50\n\n def __init__(self, graph: WorkflowGraph, agent: \"BaseAgent\"):\n self.graph = graph\n self.agent = agent\n self.state: WorkflowState = {}\n self.execution_log: List[Dict[str, Any]] = []\n self._condition_result: Optional[str] = None\n self._template_engine = TemplateEngine()\n self._namespace_manager = NamespaceManager()\n\n def execute(\n self, initial_inputs: WorkflowState, query: str\n ) -> Generator[Dict[str, str], None, None]:\n self._initialize_state(initial_inputs, query)\n\n start_node = self.graph.get_start_node()\n if not start_node:\n yield {\"type\": \"error\", \"error\": \"No start node found in workflow.\"}\n return\n current_node_id: Optional[str] = start_node.id\n steps = 0\n\n while current_node_id and steps < self.MAX_EXECUTION_STEPS:\n node = self.graph.get_node_by_id(current_node_id)\n if not node:\n yield {\"type\": \"error\", \"error\": f\"Node {current_node_id} not found.\"}\n break\n log_entry = self._create_log_entry(node)\n\n yield {\n \"type\": \"workflow_step\",\n \"node_id\": node.id,\n \"node_type\": node.type.value,\n \"node_title\": node.title,\n \"status\": \"running\",\n }\n\n try:\n yield from self._execute_node(node)\n log_entry[\"status\"] = ExecutionStatus.COMPLETED.value\n log_entry[\"completed_at\"] = datetime.now(timezone.utc)\n\n output_key = f\"node_{node.id}_output\"\n node_output = self.state.get(output_key)\n\n yield {\n \"type\": \"workflow_step\",\n \"node_id\": node.id,\n \"node_type\": node.type.value,\n \"node_title\": node.title,\n \"status\": \"completed\",\n \"state_snapshot\": dict(self.state),\n \"output\": node_output,\n }\n except Exception as e:\n logger.error(f\"Error executing node {node.id}: {e}\", exc_info=True)\n log_entry[\"status\"] = ExecutionStatus.FAILED.value\n log_entry[\"error\"] = str(e)\n log_entry[\"completed_at\"] = datetime.now(timezone.utc)\n log_entry[\"state_snapshot\"] = dict(self.state)\n self.execution_log.append(log_entry)\n\n user_friendly_error = sanitize_api_error(e)\n yield {\n \"type\": \"workflow_step\",\n \"node_id\": node.id,\n \"node_type\": node.type.value,\n \"node_title\": node.title,\n \"status\": \"failed\",\n \"state_snapshot\": dict(self.state),\n \"error\": user_friendly_error,\n }\n yield {\"type\": \"error\", \"error\": user_friendly_error}\n break\n log_entry[\"state_snapshot\"] = dict(self.state)\n self.execution_log.append(log_entry)\n\n if node.type == NodeType.END:\n break\n current_node_id = self._get_next_node_id(current_node_id)\n if current_node_id is None and node.type != NodeType.END:\n logger.warning(\n f\"Branch ended at node '{node.title}' ({node.id}) without reaching an end node\"\n )\n steps += 1\n if steps >= self.MAX_EXECUTION_STEPS:\n logger.warning(\n f\"Workflow reached max steps limit ({self.MAX_EXECUTION_STEPS})\"\n )\n\n def _initialize_state(self, initial_inputs: WorkflowState, query: str) -> None:\n self.state.update(initial_inputs)\n self.state[\"query\"] = query\n self.state[\"chat_history\"] = str(self.agent.chat_history)\n\n def _create_log_entry(self, node: WorkflowNode) -> Dict[str, Any]:\n return {\n \"node_id\": node.id,\n \"node_type\": node.type.value,\n \"started_at\": datetime.now(timezone.utc),\n \"completed_at\": None,\n \"status\": ExecutionStatus.RUNNING.value,\n \"error\": None,\n \"state_snapshot\": {},\n }\n\n def _get_next_node_id(self, current_node_id: str) -> Optional[str]:\n node = self.graph.get_node_by_id(current_node_id)\n edges = self.graph.get_outgoing_edges(current_node_id)\n if not edges:\n return None\n\n if node and node.type == NodeType.CONDITION and self._condition_result:\n target_handle = self._condition_result\n self._condition_result = None\n for edge in edges:\n if edge.source_handle == target_handle:\n return edge.target_id\n return None\n\n return edges[0].target_id\n\n def _execute_node(\n self, node: WorkflowNode\n ) -> Generator[Dict[str, str], None, None]:\n logger.info(f\"Executing node {node.id} ({node.type.value})\")\n\n node_handlers = {\n NodeType.START: self._execute_start_node,\n NodeType.NOTE: self._execute_note_node,\n NodeType.AGENT: self._execute_agent_node,\n NodeType.STATE: self._execute_state_node,\n NodeType.CONDITION: self._execute_condition_node,\n NodeType.END: self._execute_end_node,\n }\n\n handler = node_handlers.get(node.type)\n if handler:\n yield from handler(node)\n\n def _execute_start_node(\n self, node: WorkflowNode\n ) -> Generator[Dict[str, str], None, None]:\n yield from ()\n\n def _execute_note_node(\n self, node: WorkflowNode\n ) -> Generator[Dict[str, str], None, None]:\n yield from ()\n\n def _execute_agent_node(\n self, node: WorkflowNode\n ) -> Generator[Dict[str, str], None, None]:\n from application.core.model_utils import (\n get_api_key_for_provider,\n get_model_capabilities,\n get_provider_from_model_id,\n )\n\n node_config = AgentNodeConfig(**node.config.get(\"config\", node.config))\n\n if node_config.prompt_template:\n formatted_prompt = self._format_template(node_config.prompt_template)\n else:\n formatted_prompt = self.state.get(\"query\", \"\")\n node_json_schema = self._normalize_node_json_schema(\n node_config.json_schema, node.title\n )\n node_model_id = node_config.model_id or self.agent.model_id\n node_llm_name = (\n node_config.llm_name\n or get_provider_from_model_id(node_model_id or \"\")\n or self.agent.llm_name\n )\n node_api_key = get_api_key_for_provider(node_llm_name) or self.agent.api_key\n\n if node_json_schema and node_model_id:\n model_capabilities = get_model_capabilities(node_model_id)\n if model_capabilities and not model_capabilities.get(\n \"supports_structured_output\", False\n ):\n raise ValueError(\n f'Model \"{node_model_id}\" does not support structured output for node \"{node.title}\"'\n )\n\n node_agent = WorkflowNodeAgentFactory.create(\n agent_type=node_config.agent_type,\n endpoint=self.agent.endpoint,\n llm_name=node_llm_name,\n model_id=node_model_id,\n api_key=node_api_key,\n tool_ids=node_config.tools,\n prompt=node_config.system_prompt,\n chat_history=self.agent.chat_history,\n decoded_token=self.agent.decoded_token,\n json_schema=node_json_schema,\n )\n\n full_response_parts: List[str] = []\n structured_response_parts: List[str] = []\n has_structured_response = False\n first_chunk = True\n for event in node_agent.gen(formatted_prompt):\n if \"answer\" in event:\n chunk = str(event[\"answer\"])\n full_response_parts.append(chunk)\n if event.get(\"structured\"):\n has_structured_response = True\n structured_response_parts.append(chunk)\n if node_config.stream_to_user:\n if first_chunk and hasattr(self, \"_has_streamed\"):\n yield {\"answer\": \"\\n\\n\"}\n first_chunk = False\n yield event\n\n if node_config.stream_to_user:\n self._has_streamed = True\n\n full_response = \"\".join(full_response_parts).strip()\n output_value: Any = full_response\n if has_structured_response:\n structured_response = \"\".join(structured_response_parts).strip()\n response_to_parse = structured_response or full_response\n parsed_success, parsed_structured = self._parse_structured_output(\n response_to_parse\n )\n output_value = parsed_structured if parsed_success else response_to_parse\n if node_json_schema:\n self._validate_structured_output(node_json_schema, output_value)\n elif node_json_schema:\n parsed_success, parsed_structured = self._parse_structured_output(\n full_response\n )\n if not parsed_success:\n raise ValueError(\n \"Structured output was expected but response was not valid JSON\"\n )\n output_value = parsed_structured\n self._validate_structured_output(node_json_schema, output_value)\n\n default_output_key = f\"node_{node.id}_output\"\n self.state[default_output_key] = output_value\n\n if node_config.output_variable:\n self.state[node_config.output_variable] = output_value\n\n def _execute_state_node(\n self, node: WorkflowNode\n ) -> Generator[Dict[str, str], None, None]:\n config = node.config.get(\"config\", node.config)\n for op in config.get(\"operations\", []):\n expression = op.get(\"expression\", \"\")\n target_variable = op.get(\"target_variable\", \"\")\n if expression and target_variable:\n self.state[target_variable] = evaluate_cel(expression, self.state)\n yield from ()\n\n def _execute_condition_node(\n self, node: WorkflowNode\n ) -> Generator[Dict[str, str], None, None]:\n config = ConditionNodeConfig(**node.config.get(\"config\", node.config))\n matched_handle = None\n\n for case in config.cases:\n if not case.expression.strip():\n continue\n try:\n if evaluate_cel(case.expression, self.state):\n matched_handle = case.source_handle\n break\n except CelEvaluationError:\n continue\n\n self._condition_result = matched_handle or \"else\"\n yield from ()\n\n def _execute_end_node(\n self, node: WorkflowNode\n ) -> Generator[Dict[str, str], None, None]:\n config = node.config.get(\"config\", node.config)\n output_template = str(config.get(\"output_template\", \"\"))\n if output_template:\n formatted_output = self._format_template(output_template)\n yield {\"answer\": formatted_output}\n\n def _parse_structured_output(self, raw_response: str) -> tuple[bool, Optional[Any]]:\n normalized_response = raw_response.strip()\n if not normalized_response:\n return False, None\n\n try:\n return True, json.loads(normalized_response)\n except json.JSONDecodeError:\n logger.warning(\n \"Workflow agent returned structured output that was not valid JSON\"\n )\n return False, None\n\n def _normalize_node_json_schema(\n self, schema: Optional[Dict[str, Any]], node_title: str\n ) -> Optional[Dict[str, Any]]:\n if schema is None:\n return None\n try:\n return normalize_json_schema_payload(schema)\n except JsonSchemaValidationError as exc:\n raise ValueError(\n f'Invalid JSON schema for node \"{node_title}\": {exc}'\n ) from exc\n\n def _validate_structured_output(self, schema: Dict[str, Any], output_value: Any) -> None:\n if jsonschema is None:\n logger.warning(\n \"jsonschema package is not available, skipping structured output validation\"\n )\n return\n\n try:\n normalized_schema = normalize_json_schema_payload(schema)\n except JsonSchemaValidationError as exc:\n raise ValueError(f\"Invalid JSON schema: {exc}\") from exc\n\n try:\n jsonschema.validate(instance=output_value, schema=normalized_schema)\n except jsonschema.exceptions.ValidationError as exc:\n raise ValueError(f\"Structured output did not match schema: {exc.message}\") from exc\n except jsonschema.exceptions.SchemaError as exc:\n raise ValueError(f\"Invalid JSON schema: {exc.message}\") from exc\n\n def _format_template(self, template: str) -> str:\n context = self._build_template_context()\n try:\n return self._template_engine.render(template, context)\n except TemplateRenderError as e:\n logger.warning(\n \"Workflow template rendering failed, using raw template: %s\", str(e)\n )\n return template\n\n def _build_template_context(self) -> Dict[str, Any]:\n docs, docs_together = self._get_source_template_data()\n passthrough_data = (\n self.state.get(\"passthrough\")\n if isinstance(self.state.get(\"passthrough\"), dict)\n else None\n )\n tools_data = (\n self.state.get(\"tools\") if isinstance(self.state.get(\"tools\"), dict) else None\n )\n\n context = self._namespace_manager.build_context(\n user_id=getattr(self.agent, \"user\", None),\n request_id=getattr(self.agent, \"request_id\", None),\n passthrough_data=passthrough_data,\n docs=docs,\n docs_together=docs_together,\n tools_data=tools_data,\n )\n\n agent_context: Dict[str, Any] = {}\n for key, value in self.state.items():\n if not isinstance(key, str):\n continue\n normalized_key = key.strip()\n if not normalized_key:\n continue\n agent_context[normalized_key] = value\n\n context[\"agent\"] = agent_context\n\n # Keep legacy top-level variables working while namespaced variables are adopted.\n for key, value in agent_context.items():\n if key in TEMPLATE_RESERVED_NAMESPACES:\n context[f\"agent_{key}\"] = value\n continue\n if key not in context:\n context[key] = value\n\n return context\n\n def _get_source_template_data(self) -> tuple[Optional[List[Dict[str, Any]]], Optional[str]]:\n docs = getattr(self.agent, \"retrieved_docs\", None)\n if not isinstance(docs, list) or len(docs) == 0:\n return None, None\n\n docs_together_parts: List[str] = []\n for doc in docs:\n if not isinstance(doc, dict):\n continue\n text = doc.get(\"text\")\n if not isinstance(text, str):\n continue\n\n filename = doc.get(\"filename\") or doc.get(\"title\") or doc.get(\"source\")\n if isinstance(filename, str) and filename.strip():\n docs_together_parts.append(f\"{filename}\\n{text}\")\n else:\n docs_together_parts.append(text)\n\n docs_together = \"\\n\\n\".join(docs_together_parts) if docs_together_parts else None\n return docs, docs_together\n\n def get_execution_summary(self) -> List[NodeExecutionLog]:\n return [\n NodeExecutionLog(\n node_id=log[\"node_id\"],\n node_type=log[\"node_type\"],\n status=ExecutionStatus(log[\"status\"]),\n started_at=log[\"started_at\"],\n completed_at=log.get(\"completed_at\"),\n error=log.get(\"error\"),\n state_snapshot=log.get(\"state_snapshot\", {}),\n )\n for log in self.execution_log\n ]\n" }, { "path": "application/api/__init__.py", "content": "from flask_restx import Api\n\napi = Api(\n version=\"1.0\",\n title=\"DocsGPT API\",\n description=\"API for DocsGPT\",\n)\n" }, { "path": "application/api/answer/__init__.py", "content": "from flask import Blueprint\n\nfrom application.api import api\nfrom application.api.answer.routes.answer import AnswerResource\nfrom application.api.answer.routes.base import answer_ns\nfrom application.api.answer.routes.search import SearchResource\nfrom application.api.answer.routes.stream import StreamResource\n\n\nanswer = Blueprint(\"answer\", __name__)\n\napi.add_namespace(answer_ns)\n\n\ndef init_answer_routes():\n api.add_resource(StreamResource, \"/stream\")\n api.add_resource(AnswerResource, \"/api/answer\")\n api.add_resource(SearchResource, \"/api/search\")\n\n\ninit_answer_routes()\n" }, { "path": "application/api/answer/routes/__init__.py", "content": "" }, { "path": "application/api/answer/routes/answer.py", "content": "import logging\nimport traceback\n\nfrom flask import make_response, request\nfrom flask_restx import fields, Resource\n\nfrom application.api import api\n\nfrom application.api.answer.routes.base import answer_ns, BaseAnswerResource\n\nfrom application.api.answer.services.stream_processor import StreamProcessor\n\nlogger = logging.getLogger(__name__)\n\n\n@answer_ns.route(\"/api/answer\")\nclass AnswerResource(Resource, BaseAnswerResource):\n def __init__(self, *args, **kwargs):\n Resource.__init__(self, *args, **kwargs)\n BaseAnswerResource.__init__(self)\n\n answer_model = answer_ns.model(\n \"AnswerModel\",\n {\n \"question\": fields.String(\n required=True, description=\"Question to be asked\"\n ),\n \"history\": fields.List(\n fields.String,\n required=False,\n description=\"Conversation history (only for new conversations)\",\n ),\n \"conversation_id\": fields.String(\n required=False,\n description=\"Existing conversation ID (loads history)\",\n ),\n \"prompt_id\": fields.String(\n required=False, default=\"default\", description=\"Prompt ID\"\n ),\n \"chunks\": fields.Integer(\n required=False, default=2, description=\"Number of chunks\"\n ),\n \"retriever\": fields.String(required=False, description=\"Retriever type\"),\n \"api_key\": fields.String(required=False, description=\"API key\"),\n \"agent_id\": fields.String(required=False, description=\"Agent ID\"),\n \"active_docs\": fields.String(\n required=False, description=\"Active documents\"\n ),\n \"isNoneDoc\": fields.Boolean(\n required=False, description=\"Flag indicating if no document is used\"\n ),\n \"save_conversation\": fields.Boolean(\n required=False,\n default=True,\n description=\"Whether to save the conversation\",\n ),\n \"model_id\": fields.String(\n required=False,\n description=\"Model ID to use for this request\",\n ),\n \"passthrough\": fields.Raw(\n required=False,\n description=\"Dynamic parameters to inject into prompt template\",\n ),\n },\n )\n\n @api.expect(answer_model)\n @api.doc(description=\"Provide a response based on the question and retriever\")\n def post(self):\n data = request.get_json()\n if error := self.validate_request(data):\n return error\n decoded_token = getattr(request, \"decoded_token\", None)\n processor = StreamProcessor(data, decoded_token)\n try:\n processor.initialize()\n if not processor.decoded_token:\n return make_response({\"error\": \"Unauthorized\"}, 401)\n\n docs_together, docs_list = processor.pre_fetch_docs(\n data.get(\"question\", \"\")\n )\n tools_data = processor.pre_fetch_tools()\n\n agent = processor.create_agent(\n docs_together=docs_together,\n docs=docs_list,\n tools_data=tools_data,\n )\n\n if error := self.check_usage(processor.agent_config):\n return error\n\n stream = self.complete_stream(\n question=data[\"question\"],\n agent=agent,\n conversation_id=processor.conversation_id,\n user_api_key=processor.agent_config.get(\"user_api_key\"),\n decoded_token=processor.decoded_token,\n isNoneDoc=data.get(\"isNoneDoc\"),\n index=None,\n should_save_conversation=data.get(\"save_conversation\", True),\n agent_id=processor.agent_id,\n is_shared_usage=processor.is_shared_usage,\n shared_token=processor.shared_token,\n model_id=processor.model_id,\n )\n stream_result = self.process_response_stream(stream)\n\n if len(stream_result) == 7:\n (\n conversation_id,\n response,\n sources,\n tool_calls,\n thought,\n error,\n structured_info,\n ) = stream_result\n else:\n conversation_id, response, sources, tool_calls, thought, error = (\n stream_result\n )\n structured_info = None\n\n if error:\n return make_response({\"error\": error}, 400)\n result = {\n \"conversation_id\": conversation_id,\n \"answer\": response,\n \"sources\": sources,\n \"tool_calls\": tool_calls,\n \"thought\": thought,\n }\n\n if structured_info:\n result.update(structured_info)\n except Exception as e:\n logger.error(\n f\"/api/answer - error: {str(e)} - traceback: {traceback.format_exc()}\",\n extra={\"error\": str(e), \"traceback\": traceback.format_exc()},\n )\n return make_response({\"error\": \"An error occurred processing your request\"}, 500)\n return make_response(result, 200)\n" }, { "path": "application/api/answer/routes/base.py", "content": "import datetime\nimport json\nimport logging\nfrom typing import Any, Dict, Generator, List, Optional\n\nfrom flask import jsonify, make_response, Response\nfrom flask_restx import Namespace\n\nfrom application.api.answer.services.conversation_service import ConversationService\nfrom application.core.model_utils import (\n get_api_key_for_provider,\n get_default_model_id,\n get_provider_from_model_id,\n)\n\nfrom application.core.mongo_db import MongoDB\nfrom application.core.settings import settings\nfrom application.error import sanitize_api_error\nfrom application.llm.llm_creator import LLMCreator\nfrom application.utils import check_required_fields\n\nlogger = logging.getLogger(__name__)\n\n\nanswer_ns = Namespace(\"answer\", description=\"Answer related operations\", path=\"/\")\n\n\nclass BaseAnswerResource:\n \"\"\"Shared base class for answer endpoints\"\"\"\n\n def __init__(self):\n mongo = MongoDB.get_client()\n db = mongo[settings.MONGO_DB_NAME]\n self.db = db\n self.user_logs_collection = db[\"user_logs\"]\n self.default_model_id = get_default_model_id()\n self.conversation_service = ConversationService()\n\n def validate_request(\n self, data: Dict[str, Any], require_conversation_id: bool = False\n ) -> Optional[Response]:\n \"\"\"Common request validation\"\"\"\n required_fields = [\"question\"]\n if require_conversation_id:\n required_fields.append(\"conversation_id\")\n if missing_fields := check_required_fields(data, required_fields):\n return missing_fields\n return None\n\n @staticmethod\n def _prepare_tool_calls_for_logging(\n tool_calls: Optional[List[Dict[str, Any]]], max_chars: int = 10000\n ) -> List[Dict[str, Any]]:\n if not tool_calls:\n return []\n\n prepared = []\n for tool_call in tool_calls:\n if not isinstance(tool_call, dict):\n prepared.append({\"result\": str(tool_call)[:max_chars]})\n continue\n\n item = dict(tool_call)\n for key in (\"result\", \"result_full\"):\n value = item.get(key)\n if isinstance(value, str) and len(value) > max_chars:\n item[key] = value[:max_chars]\n prepared.append(item)\n return prepared\n\n def check_usage(self, agent_config: Dict) -> Optional[Response]:\n \"\"\"Check if there is a usage limit and if it is exceeded\n\n Args:\n agent_config: The config dict of agent instance\n\n Returns:\n None or Response if either of limits exceeded.\n\n \"\"\"\n api_key = agent_config.get(\"user_api_key\")\n if not api_key:\n return None\n agents_collection = self.db[\"agents\"]\n agent = agents_collection.find_one({\"key\": api_key})\n\n if not agent:\n return make_response(\n jsonify({\"success\": False, \"message\": \"Invalid API key.\"}), 401\n )\n limited_token_mode_raw = agent.get(\"limited_token_mode\", False)\n limited_request_mode_raw = agent.get(\"limited_request_mode\", False)\n\n limited_token_mode = (\n limited_token_mode_raw\n if isinstance(limited_token_mode_raw, bool)\n else limited_token_mode_raw == \"True\"\n )\n limited_request_mode = (\n limited_request_mode_raw\n if isinstance(limited_request_mode_raw, bool)\n else limited_request_mode_raw == \"True\"\n )\n\n token_limit = int(\n agent.get(\"token_limit\", settings.DEFAULT_AGENT_LIMITS[\"token_limit\"])\n )\n request_limit = int(\n agent.get(\"request_limit\", settings.DEFAULT_AGENT_LIMITS[\"request_limit\"])\n )\n\n token_usage_collection = self.db[\"token_usage\"]\n\n end_date = datetime.datetime.now()\n start_date = end_date - datetime.timedelta(hours=24)\n\n match_query = {\n \"timestamp\": {\"$gte\": start_date, \"$lte\": end_date},\n \"api_key\": api_key,\n }\n\n if limited_token_mode:\n token_pipeline = [\n {\"$match\": match_query},\n {\n \"$group\": {\n \"_id\": None,\n \"total_tokens\": {\n \"$sum\": {\"$add\": [\"$prompt_tokens\", \"$generated_tokens\"]}\n },\n }\n },\n ]\n token_result = list(token_usage_collection.aggregate(token_pipeline))\n daily_token_usage = token_result[0][\"total_tokens\"] if token_result else 0\n else:\n daily_token_usage = 0\n if limited_request_mode:\n daily_request_usage = token_usage_collection.count_documents(match_query)\n else:\n daily_request_usage = 0\n if not limited_token_mode and not limited_request_mode:\n return None\n token_exceeded = (\n limited_token_mode and token_limit > 0 and daily_token_usage >= token_limit\n )\n request_exceeded = (\n limited_request_mode\n and request_limit > 0\n and daily_request_usage >= request_limit\n )\n\n if token_exceeded or request_exceeded:\n return make_response(\n jsonify(\n {\n \"success\": False,\n \"message\": \"Exceeding usage limit, please try again later.\",\n }\n ),\n 429,\n )\n return None\n\n def complete_stream(\n self,\n question: str,\n agent: Any,\n conversation_id: Optional[str],\n user_api_key: Optional[str],\n decoded_token: Dict[str, Any],\n isNoneDoc: bool = False,\n index: Optional[int] = None,\n should_save_conversation: bool = True,\n attachment_ids: Optional[List[str]] = None,\n agent_id: Optional[str] = None,\n is_shared_usage: bool = False,\n shared_token: Optional[str] = None,\n model_id: Optional[str] = None,\n ) -> Generator[str, None, None]:\n \"\"\"\n Generator function that streams the complete conversation response.\n\n Args:\n question: The user's question\n agent: The agent instance\n retriever: The retriever instance\n conversation_id: Existing conversation ID\n user_api_key: User's API key if any\n decoded_token: Decoded JWT token\n isNoneDoc: Flag for document-less responses\n index: Index of message to update\n should_save_conversation: Whether to persist the conversation\n attachment_ids: List of attachment IDs\n agent_id: ID of agent used\n is_shared_usage: Flag for shared agent usage\n shared_token: Token for shared agent\n model_id: Model ID used for the request\n retrieved_docs: Pre-fetched documents for sources (optional)\n\n Yields:\n Server-sent event strings\n \"\"\"\n try:\n response_full, thought, source_log_docs, tool_calls = \"\", \"\", [], []\n is_structured = False\n schema_info = None\n structured_chunks = []\n\n for line in agent.gen(query=question):\n if \"answer\" in line:\n response_full += str(line[\"answer\"])\n if line.get(\"structured\"):\n is_structured = True\n schema_info = line.get(\"schema\")\n structured_chunks.append(line[\"answer\"])\n else:\n data = json.dumps({\"type\": \"answer\", \"answer\": line[\"answer\"]})\n yield f\"data: {data}\\n\\n\"\n elif \"sources\" in line:\n truncated_sources = []\n source_log_docs = line[\"sources\"]\n for source in line[\"sources\"]:\n truncated_source = source.copy()\n if \"text\" in truncated_source:\n truncated_source[\"text\"] = (\n truncated_source[\"text\"][:100].strip() + \"...\"\n )\n truncated_sources.append(truncated_source)\n if truncated_sources:\n data = json.dumps(\n {\"type\": \"source\", \"source\": truncated_sources}\n )\n yield f\"data: {data}\\n\\n\"\n elif \"tool_calls\" in line:\n tool_calls = line[\"tool_calls\"]\n data = json.dumps({\"type\": \"tool_calls\", \"tool_calls\": tool_calls})\n yield f\"data: {data}\\n\\n\"\n elif \"thought\" in line:\n thought += line[\"thought\"]\n data = json.dumps({\"type\": \"thought\", \"thought\": line[\"thought\"]})\n yield f\"data: {data}\\n\\n\"\n elif \"type\" in line:\n if line.get(\"type\") == \"error\":\n sanitized_error = {\n \"type\": \"error\",\n \"error\": sanitize_api_error(line.get(\"error\", \"An error occurred\"))\n }\n data = json.dumps(sanitized_error)\n else:\n data = json.dumps(line)\n yield f\"data: {data}\\n\\n\"\n if is_structured and structured_chunks:\n structured_data = {\n \"type\": \"structured_answer\",\n \"answer\": response_full,\n \"structured\": True,\n \"schema\": schema_info,\n }\n data = json.dumps(structured_data)\n yield f\"data: {data}\\n\\n\"\n if isNoneDoc:\n for doc in source_log_docs:\n doc[\"source\"] = \"None\"\n provider = (\n get_provider_from_model_id(model_id)\n if model_id\n else settings.LLM_PROVIDER\n )\n system_api_key = get_api_key_for_provider(provider or settings.LLM_PROVIDER)\n\n llm = LLMCreator.create_llm(\n provider or settings.LLM_PROVIDER,\n api_key=system_api_key,\n user_api_key=user_api_key,\n decoded_token=decoded_token,\n model_id=model_id,\n agent_id=agent_id,\n )\n\n if should_save_conversation:\n conversation_id = self.conversation_service.save_conversation(\n conversation_id,\n question,\n response_full,\n thought,\n source_log_docs,\n tool_calls,\n llm,\n model_id or self.default_model_id,\n decoded_token,\n index=index,\n api_key=user_api_key,\n agent_id=agent_id,\n is_shared_usage=is_shared_usage,\n shared_token=shared_token,\n attachment_ids=attachment_ids,\n )\n # Persist compression metadata/summary if it exists and wasn't saved mid-execution\n compression_meta = getattr(agent, \"compression_metadata\", None)\n compression_saved = getattr(agent, \"compression_saved\", False)\n if conversation_id and compression_meta and not compression_saved:\n try:\n self.conversation_service.update_compression_metadata(\n conversation_id, compression_meta\n )\n self.conversation_service.append_compression_message(\n conversation_id, compression_meta\n )\n agent.compression_saved = True\n logger.info(\n f\"Persisted compression metadata for conversation {conversation_id}\"\n )\n except Exception as e:\n logger.error(\n f\"Failed to persist compression metadata: {str(e)}\",\n exc_info=True,\n )\n else:\n conversation_id = None\n id_data = {\"type\": \"id\", \"id\": str(conversation_id)}\n data = json.dumps(id_data)\n yield f\"data: {data}\\n\\n\"\n\n tool_calls_for_logging = self._prepare_tool_calls_for_logging(\n getattr(agent, \"tool_calls\", tool_calls) or tool_calls\n )\n\n log_data = {\n \"action\": \"stream_answer\",\n \"level\": \"info\",\n \"user\": decoded_token.get(\"sub\"),\n \"api_key\": user_api_key,\n \"agent_id\": agent_id,\n \"question\": question,\n \"response\": response_full,\n \"sources\": source_log_docs,\n \"tool_calls\": tool_calls_for_logging,\n \"attachments\": attachment_ids,\n \"timestamp\": datetime.datetime.now(datetime.timezone.utc),\n }\n if is_structured:\n log_data[\"structured_output\"] = True\n if schema_info:\n log_data[\"schema\"] = schema_info\n # Clean up text fields to be no longer than 10000 characters\n\n for key, value in log_data.items():\n if isinstance(value, str) and len(value) > 10000:\n log_data[key] = value[:10000]\n self.user_logs_collection.insert_one(log_data)\n\n data = json.dumps({\"type\": \"end\"})\n yield f\"data: {data}\\n\\n\"\n except GeneratorExit:\n logger.info(f\"Stream aborted by client for question: {question[:50]}... \")\n # Save partial response\n\n if should_save_conversation and response_full:\n try:\n if isNoneDoc:\n for doc in source_log_docs:\n doc[\"source\"] = \"None\"\n llm = LLMCreator.create_llm(\n settings.LLM_PROVIDER,\n api_key=settings.API_KEY,\n user_api_key=user_api_key,\n decoded_token=decoded_token,\n agent_id=agent_id,\n )\n self.conversation_service.save_conversation(\n conversation_id,\n question,\n response_full,\n thought,\n source_log_docs,\n tool_calls,\n llm,\n model_id or self.default_model_id,\n decoded_token,\n index=index,\n api_key=user_api_key,\n agent_id=agent_id,\n is_shared_usage=is_shared_usage,\n shared_token=shared_token,\n attachment_ids=attachment_ids,\n )\n compression_meta = getattr(agent, \"compression_metadata\", None)\n compression_saved = getattr(agent, \"compression_saved\", False)\n if conversation_id and compression_meta and not compression_saved:\n try:\n self.conversation_service.update_compression_metadata(\n conversation_id, compression_meta\n )\n self.conversation_service.append_compression_message(\n conversation_id, compression_meta\n )\n agent.compression_saved = True\n logger.info(\n f\"Persisted compression metadata for conversation {conversation_id} (partial stream)\"\n )\n except Exception as e:\n logger.error(\n f\"Failed to persist compression metadata (partial stream): {str(e)}\",\n exc_info=True,\n )\n except Exception as e:\n logger.error(\n f\"Error saving partial response: {str(e)}\", exc_info=True\n )\n raise\n except Exception as e:\n logger.error(f\"Error in stream: {str(e)}\", exc_info=True)\n data = json.dumps(\n {\n \"type\": \"error\",\n \"error\": \"Please try again later. We apologize for any inconvenience.\",\n }\n )\n yield f\"data: {data}\\n\\n\"\n return\n\n def process_response_stream(self, stream):\n \"\"\"Process the stream response for non-streaming endpoint\"\"\"\n conversation_id = \"\"\n response_full = \"\"\n source_log_docs = []\n tool_calls = []\n thought = \"\"\n stream_ended = False\n is_structured = False\n schema_info = None\n\n for line in stream:\n try:\n event_data = line.replace(\"data: \", \"\").strip()\n event = json.loads(event_data)\n\n if event[\"type\"] == \"id\":\n conversation_id = event[\"id\"]\n elif event[\"type\"] == \"answer\":\n response_full += event[\"answer\"]\n elif event[\"type\"] == \"structured_answer\":\n response_full = event[\"answer\"]\n is_structured = True\n schema_info = event.get(\"schema\")\n elif event[\"type\"] == \"source\":\n source_log_docs = event[\"source\"]\n elif event[\"type\"] == \"tool_calls\":\n tool_calls = event[\"tool_calls\"]\n elif event[\"type\"] == \"thought\":\n thought = event[\"thought\"]\n elif event[\"type\"] == \"error\":\n logger.error(f\"Error from stream: {event['error']}\")\n return None, None, None, None, event[\"error\"], None\n elif event[\"type\"] == \"end\":\n stream_ended = True\n except (json.JSONDecodeError, KeyError) as e:\n logger.warning(f\"Error parsing stream event: {e}, line: {line}\")\n continue\n if not stream_ended:\n logger.error(\"Stream ended unexpectedly without an 'end' event.\")\n return None, None, None, None, \"Stream ended unexpectedly\", None\n result = (\n conversation_id,\n response_full,\n source_log_docs,\n tool_calls,\n thought,\n None,\n )\n\n if is_structured:\n result = result + ({\"structured\": True, \"schema\": schema_info},)\n return result\n\n def error_stream_generate(self, err_response):\n data = json.dumps({\"type\": \"error\", \"error\": err_response})\n yield f\"data: {data}\\n\\n\"\n" }, { "path": "application/api/answer/routes/search.py", "content": "import logging\nfrom typing import Any, Dict, List\n\nfrom flask import make_response, request\nfrom flask_restx import fields, Resource\n\nfrom bson.dbref import DBRef\n\nfrom application.api.answer.routes.base import answer_ns\nfrom application.core.mongo_db import MongoDB\nfrom application.core.settings import settings\nfrom application.vectorstore.vector_creator import VectorCreator\n\nlogger = logging.getLogger(__name__)\n\n\n@answer_ns.route(\"/api/search\")\nclass SearchResource(Resource):\n \"\"\"Fast search endpoint for retrieving relevant documents\"\"\"\n\n def __init__(self, *args, **kwargs):\n super().__init__(*args, **kwargs)\n mongo = MongoDB.get_client()\n self.db = mongo[settings.MONGO_DB_NAME]\n self.agents_collection = self.db[\"agents\"]\n\n search_model = answer_ns.model(\n \"SearchModel\",\n {\n \"question\": fields.String(\n required=True, description=\"Search query\"\n ),\n \"api_key\": fields.String(\n required=True, description=\"API key for authentication\"\n ),\n \"chunks\": fields.Integer(\n required=False, default=5, description=\"Number of results to return\"\n ),\n },\n )\n\n def _get_sources_from_api_key(self, api_key: str) -> List[str]:\n \"\"\"Get source IDs connected to the API key/agent.\n\n \"\"\"\n agent_data = self.agents_collection.find_one({\"key\": api_key})\n if not agent_data:\n return []\n\n source_ids = []\n\n # Handle multiple sources (only if non-empty)\n sources = agent_data.get(\"sources\", [])\n if sources and isinstance(sources, list) and len(sources) > 0:\n for source_ref in sources:\n # Skip \"default\" - it's a placeholder, not an actual vectorstore\n if source_ref == \"default\":\n continue\n elif isinstance(source_ref, DBRef):\n source_doc = self.db.dereference(source_ref)\n if source_doc:\n source_ids.append(str(source_doc[\"_id\"]))\n\n # Handle single source (legacy) - check if sources was empty or didn't yield results\n if not source_ids:\n source = agent_data.get(\"source\")\n if isinstance(source, DBRef):\n source_doc = self.db.dereference(source)\n if source_doc:\n source_ids.append(str(source_doc[\"_id\"]))\n # Skip \"default\" - it's a placeholder, not an actual vectorstore\n elif source and source != \"default\":\n source_ids.append(source)\n\n return source_ids\n\n def _search_vectorstores(\n self, query: str, source_ids: List[str], chunks: int\n ) -> List[Dict[str, Any]]:\n \"\"\"Search across vectorstores and return results\"\"\"\n if not source_ids:\n return []\n\n results = []\n chunks_per_source = max(1, chunks // len(source_ids))\n seen_texts = set()\n\n for source_id in source_ids:\n if not source_id or not source_id.strip():\n continue\n\n try:\n docsearch = VectorCreator.create_vectorstore(\n settings.VECTOR_STORE, source_id, settings.EMBEDDINGS_KEY\n )\n docs = docsearch.search(query, k=chunks_per_source * 2)\n\n for doc in docs:\n if len(results) >= chunks:\n break\n\n if hasattr(doc, \"page_content\") and hasattr(doc, \"metadata\"):\n page_content = doc.page_content\n metadata = doc.metadata\n else:\n page_content = doc.get(\"text\", doc.get(\"page_content\", \"\"))\n metadata = doc.get(\"metadata\", {})\n\n # Skip duplicates\n text_hash = hash(page_content[:200])\n if text_hash in seen_texts:\n continue\n seen_texts.add(text_hash)\n\n title = metadata.get(\n \"title\", metadata.get(\"post_title\", \"\")\n )\n if not isinstance(title, str):\n title = str(title) if title else \"\"\n\n # Clean up title\n if title:\n title = title.split(\"/\")[-1]\n else:\n # Use filename or first part of content as title\n title = metadata.get(\"filename\", page_content[:50] + \"...\")\n\n source = metadata.get(\"source\", source_id)\n\n results.append({\n \"text\": page_content,\n \"title\": title,\n \"source\": source,\n })\n\n if len(results) >= chunks:\n break\n\n except Exception as e:\n logger.error(\n f\"Error searching vectorstore {source_id}: {e}\",\n exc_info=True,\n )\n continue\n\n return results[:chunks]\n\n @answer_ns.expect(search_model)\n @answer_ns.doc(description=\"Search for relevant documents based on query\")\n def post(self):\n data = request.get_json()\n\n question = data.get(\"question\")\n api_key = data.get(\"api_key\")\n chunks = data.get(\"chunks\", 5)\n\n if not question:\n return make_response({\"error\": \"question is required\"}, 400)\n\n if not api_key:\n return make_response({\"error\": \"api_key is required\"}, 400)\n\n # Validate API key\n agent = self.agents_collection.find_one({\"key\": api_key})\n if not agent:\n return make_response({\"error\": \"Invalid API key\"}, 401)\n\n try:\n # Get sources connected to this API key\n source_ids = self._get_sources_from_api_key(api_key)\n\n if not source_ids:\n return make_response([], 200)\n\n # Perform search\n results = self._search_vectorstores(question, source_ids, chunks)\n\n return make_response(results, 200)\n\n except Exception as e:\n logger.error(\n f\"/api/search - error: {str(e)}\",\n extra={\"error\": str(e)},\n exc_info=True,\n )\n return make_response({\"error\": \"Search failed\"}, 500)\n" }, { "path": "application/api/answer/routes/stream.py", "content": "import logging\nimport traceback\n\nfrom flask import request, Response\nfrom flask_restx import fields, Resource\n\nfrom application.api import api\n\nfrom application.api.answer.routes.base import answer_ns, BaseAnswerResource\n\nfrom application.api.answer.services.stream_processor import StreamProcessor\n\nlogger = logging.getLogger(__name__)\n\n\n@answer_ns.route(\"/stream\")\nclass StreamResource(Resource, BaseAnswerResource):\n def __init__(self, *args, **kwargs):\n Resource.__init__(self, *args, **kwargs)\n BaseAnswerResource.__init__(self)\n\n stream_model = answer_ns.model(\n \"StreamModel\",\n {\n \"question\": fields.String(\n required=True, description=\"Question to be asked\"\n ),\n \"history\": fields.List(\n fields.String,\n required=False,\n description=\"Conversation history (only for new conversations)\",\n ),\n \"conversation_id\": fields.String(\n required=False,\n description=\"Existing conversation ID (loads history)\",\n ),\n \"prompt_id\": fields.String(\n required=False, default=\"default\", description=\"Prompt ID\"\n ),\n \"chunks\": fields.Integer(\n required=False, default=2, description=\"Number of chunks\"\n ),\n \"retriever\": fields.String(required=False, description=\"Retriever type\"),\n \"api_key\": fields.String(required=False, description=\"API key\"),\n \"agent_id\": fields.String(required=False, description=\"Agent ID\"),\n \"active_docs\": fields.String(\n required=False, description=\"Active documents\"\n ),\n \"isNoneDoc\": fields.Boolean(\n required=False, description=\"Flag indicating if no document is used\"\n ),\n \"index\": fields.Integer(\n required=False, description=\"Index of the query to update\"\n ),\n \"save_conversation\": fields.Boolean(\n required=False,\n default=True,\n description=\"Whether to save the conversation\",\n ),\n \"model_id\": fields.String(\n required=False,\n description=\"Model ID to use for this request\",\n ),\n \"attachments\": fields.List(\n fields.String, required=False, description=\"List of attachment IDs\"\n ),\n \"passthrough\": fields.Raw(\n required=False,\n description=\"Dynamic parameters to inject into prompt template\",\n ),\n },\n )\n\n @api.expect(stream_model)\n @api.doc(description=\"Stream a response based on the question and retriever\")\n def post(self):\n data = request.get_json()\n if error := self.validate_request(data, \"index\" in data):\n return error\n decoded_token = getattr(request, \"decoded_token\", None)\n processor = StreamProcessor(data, decoded_token)\n try:\n processor.initialize()\n if not processor.decoded_token:\n return Response(\n self.error_stream_generate(\"Unauthorized\"),\n status=401,\n mimetype=\"text/event-stream\",\n )\n\n docs_together, docs_list = processor.pre_fetch_docs(data[\"question\"])\n tools_data = processor.pre_fetch_tools()\n\n agent = processor.create_agent(\n docs_together=docs_together, docs=docs_list, tools_data=tools_data\n )\n\n if error := self.check_usage(processor.agent_config):\n return error\n return Response(\n self.complete_stream(\n question=data[\"question\"],\n agent=agent,\n conversation_id=processor.conversation_id,\n user_api_key=processor.agent_config.get(\"user_api_key\"),\n decoded_token=processor.decoded_token,\n isNoneDoc=data.get(\"isNoneDoc\"),\n index=data.get(\"index\"),\n should_save_conversation=data.get(\"save_conversation\", True),\n attachment_ids=data.get(\"attachments\", []),\n agent_id=processor.agent_id,\n is_shared_usage=processor.is_shared_usage,\n shared_token=processor.shared_token,\n model_id=processor.model_id,\n ),\n mimetype=\"text/event-stream\",\n )\n except ValueError as e:\n message = \"Malformed request body\"\n logger.error(\n f\"/stream - error: {message} - specific error: {str(e)} - traceback: {traceback.format_exc()}\",\n extra={\"error\": str(e), \"traceback\": traceback.format_exc()},\n )\n return Response(\n self.error_stream_generate(message),\n status=400,\n mimetype=\"text/event-stream\",\n )\n except Exception as e:\n logger.error(\n f\"/stream - error: {str(e)} - traceback: {traceback.format_exc()}\",\n extra={\"error\": str(e), \"traceback\": traceback.format_exc()},\n )\n return Response(\n self.error_stream_generate(\"Unknown error occurred\"),\n status=400,\n mimetype=\"text/event-stream\",\n )\n" }, { "path": "application/api/answer/services/__init__.py", "content": "" }, { "path": "application/api/answer/services/compression/__init__.py", "content": "\"\"\"\nCompression module for managing conversation context compression.\n\n\"\"\"\n\nfrom application.api.answer.services.compression.orchestrator import (\n CompressionOrchestrator,\n)\nfrom application.api.answer.services.compression.service import CompressionService\nfrom application.api.answer.services.compression.types import (\n CompressionResult,\n CompressionMetadata,\n)\n\n__all__ = [\n \"CompressionOrchestrator\",\n \"CompressionService\",\n \"CompressionResult\",\n \"CompressionMetadata\",\n]\n" }, { "path": "application/api/answer/services/compression/message_builder.py", "content": "\"\"\"Message reconstruction utilities for compression.\"\"\"\n\nimport logging\nimport uuid\nfrom typing import Dict, List, Optional\n\nlogger = logging.getLogger(__name__)\n\n\nclass MessageBuilder:\n \"\"\"Builds message arrays from compressed context.\"\"\"\n\n @staticmethod\n def build_from_compressed_context(\n system_prompt: str,\n compressed_summary: Optional[str],\n recent_queries: List[Dict],\n include_tool_calls: bool = False,\n context_type: str = \"pre_request\",\n ) -> List[Dict]:\n \"\"\"\n Build messages from compressed context.\n\n Args:\n system_prompt: Original system prompt\n compressed_summary: Compressed summary (if any)\n recent_queries: Recent uncompressed queries\n include_tool_calls: Whether to include tool calls from history\n context_type: Type of context ('pre_request' or 'mid_execution')\n\n Returns:\n List of message dicts ready for LLM\n \"\"\"\n # Append compression summary to system prompt if present\n if compressed_summary:\n system_prompt = MessageBuilder._append_compression_context(\n system_prompt, compressed_summary, context_type\n )\n\n messages = [{\"role\": \"system\", \"content\": system_prompt}]\n\n # Add recent history\n for query in recent_queries:\n if \"prompt\" in query and \"response\" in query:\n messages.append({\"role\": \"user\", \"content\": query[\"prompt\"]})\n messages.append({\"role\": \"assistant\", \"content\": query[\"response\"]})\n\n # Add tool calls from history if present\n if include_tool_calls and \"tool_calls\" in query:\n for tool_call in query[\"tool_calls\"]:\n call_id = tool_call.get(\"call_id\") or str(uuid.uuid4())\n\n function_call_dict = {\n \"function_call\": {\n \"name\": tool_call.get(\"action_name\"),\n \"args\": tool_call.get(\"arguments\"),\n \"call_id\": call_id,\n }\n }\n function_response_dict = {\n \"function_response\": {\n \"name\": tool_call.get(\"action_name\"),\n \"response\": {\"result\": tool_call.get(\"result\")},\n \"call_id\": call_id,\n }\n }\n\n messages.append(\n {\"role\": \"assistant\", \"content\": [function_call_dict]}\n )\n messages.append(\n {\"role\": \"tool\", \"content\": [function_response_dict]}\n )\n\n # If no recent queries (everything was compressed), add a continuation user message\n if len(recent_queries) == 0 and compressed_summary:\n messages.append({\n \"role\": \"user\",\n \"content\": \"Please continue with the remaining tasks based on the context above.\"\n })\n logger.info(\"Added continuation user message to maintain proper turn-taking after full compression\")\n\n return messages\n\n @staticmethod\n def _append_compression_context(\n system_prompt: str, compressed_summary: str, context_type: str = \"pre_request\"\n ) -> str:\n \"\"\"\n Append compression context to system prompt.\n\n Args:\n system_prompt: Original system prompt\n compressed_summary: Summary to append\n context_type: Type of compression context\n\n Returns:\n Updated system prompt\n \"\"\"\n # Remove existing compression context if present\n if \"This session is being continued\" in system_prompt or \"Context window limit reached\" in system_prompt:\n parts = system_prompt.split(\"\\n\\n---\\n\\n\")\n system_prompt = parts[0]\n\n # Build appropriate context message based on type\n if context_type == \"mid_execution\":\n context_message = (\n \"\\n\\n---\\n\\n\"\n \"Context window limit reached during execution. \"\n \"Previous conversation has been compressed to fit within limits. \"\n \"The conversation is summarized below:\\n\\n\"\n f\"{compressed_summary}\"\n )\n else: # pre_request\n context_message = (\n \"\\n\\n---\\n\\n\"\n \"This session is being continued from a previous conversation that \"\n \"has been compressed to fit within context limits. \"\n \"The conversation is summarized below:\\n\\n\"\n f\"{compressed_summary}\"\n )\n\n return system_prompt + context_message\n\n @staticmethod\n def rebuild_messages_after_compression(\n messages: List[Dict],\n compressed_summary: Optional[str],\n recent_queries: List[Dict],\n include_current_execution: bool = False,\n include_tool_calls: bool = False,\n ) -> Optional[List[Dict]]:\n \"\"\"\n Rebuild the message list after compression so tool execution can continue.\n\n Args:\n messages: Original message list\n compressed_summary: Compressed summary\n recent_queries: Recent uncompressed queries\n include_current_execution: Whether to preserve current execution messages\n include_tool_calls: Whether to include tool calls from history\n\n Returns:\n Rebuilt message list or None if failed\n \"\"\"\n # Find the system message\n system_message = next(\n (msg for msg in messages if msg.get(\"role\") == \"system\"), None\n )\n if not system_message:\n logger.warning(\"No system message found in messages list\")\n return None\n\n # Update system message with compressed summary\n if compressed_summary:\n content = system_message.get(\"content\", \"\")\n system_message[\"content\"] = MessageBuilder._append_compression_context(\n content, compressed_summary, \"mid_execution\"\n )\n logger.info(\n \"Appended compression summary to system prompt (truncated): %s\",\n (\n compressed_summary[:500] + \"...\"\n if len(compressed_summary) > 500\n else compressed_summary\n ),\n )\n\n rebuilt_messages = [system_message]\n\n # Add recent history from compressed context\n for query in recent_queries:\n if \"prompt\" in query and \"response\" in query:\n rebuilt_messages.append({\"role\": \"user\", \"content\": query[\"prompt\"]})\n rebuilt_messages.append(\n {\"role\": \"assistant\", \"content\": query[\"response\"]}\n )\n\n # Add tool calls from history if present\n if include_tool_calls and \"tool_calls\" in query:\n for tool_call in query[\"tool_calls\"]:\n call_id = tool_call.get(\"call_id\") or str(uuid.uuid4())\n\n function_call_dict = {\n \"function_call\": {\n \"name\": tool_call.get(\"action_name\"),\n \"args\": tool_call.get(\"arguments\"),\n \"call_id\": call_id,\n }\n }\n function_response_dict = {\n \"function_response\": {\n \"name\": tool_call.get(\"action_name\"),\n \"response\": {\"result\": tool_call.get(\"result\")},\n \"call_id\": call_id,\n }\n }\n\n rebuilt_messages.append(\n {\"role\": \"assistant\", \"content\": [function_call_dict]}\n )\n rebuilt_messages.append(\n {\"role\": \"tool\", \"content\": [function_response_dict]}\n )\n\n # If no recent queries (everything was compressed), add a continuation user message\n if len(recent_queries) == 0 and compressed_summary:\n rebuilt_messages.append({\n \"role\": \"user\",\n \"content\": \"Please continue with the remaining tasks based on the context above.\"\n })\n logger.info(\"Added continuation user message to maintain proper turn-taking after full compression\")\n\n if include_current_execution:\n # Preserve any messages that were added during the current execution cycle\n recent_msg_count = 1 # system message\n for query in recent_queries:\n if \"prompt\" in query and \"response\" in query:\n recent_msg_count += 2\n if \"tool_calls\" in query:\n recent_msg_count += len(query[\"tool_calls\"]) * 2\n\n if len(messages) > recent_msg_count:\n current_execution_messages = messages[recent_msg_count:]\n rebuilt_messages.extend(current_execution_messages)\n logger.info(\n f\"Preserved {len(current_execution_messages)} messages from current execution cycle\"\n )\n\n logger.info(\n f\"Messages rebuilt: {len(messages)} → {len(rebuilt_messages)} messages. \"\n f\"Ready to continue tool execution.\"\n )\n return rebuilt_messages\n" }, { "path": "application/api/answer/services/compression/orchestrator.py", "content": "\"\"\"High-level compression orchestration.\"\"\"\n\nimport logging\nfrom typing import Any, Dict, Optional\n\nfrom application.api.answer.services.compression.service import CompressionService\nfrom application.api.answer.services.compression.threshold_checker import (\n CompressionThresholdChecker,\n)\nfrom application.api.answer.services.compression.types import CompressionResult\nfrom application.api.answer.services.conversation_service import ConversationService\nfrom application.core.model_utils import (\n get_api_key_for_provider,\n get_provider_from_model_id,\n)\nfrom application.core.settings import settings\nfrom application.llm.llm_creator import LLMCreator\n\nlogger = logging.getLogger(__name__)\n\n\nclass CompressionOrchestrator:\n \"\"\"\n Facade for compression operations.\n\n Coordinates between all compression components and provides\n a simple interface for callers.\n \"\"\"\n\n def __init__(\n self,\n conversation_service: ConversationService,\n threshold_checker: Optional[CompressionThresholdChecker] = None,\n ):\n \"\"\"\n Initialize orchestrator.\n\n Args:\n conversation_service: Service for DB operations\n threshold_checker: Custom threshold checker (optional)\n \"\"\"\n self.conversation_service = conversation_service\n self.threshold_checker = threshold_checker or CompressionThresholdChecker()\n\n def compress_if_needed(\n self,\n conversation_id: str,\n user_id: str,\n model_id: str,\n decoded_token: Dict[str, Any],\n current_query_tokens: int = 500,\n ) -> CompressionResult:\n \"\"\"\n Check if compression is needed and perform it if so.\n\n This is the main entry point for compression operations.\n\n Args:\n conversation_id: Conversation ID\n user_id: User ID\n model_id: Model being used for conversation\n decoded_token: User's decoded JWT token\n current_query_tokens: Estimated tokens for current query\n\n Returns:\n CompressionResult with summary and recent queries\n \"\"\"\n try:\n # Load conversation\n conversation = self.conversation_service.get_conversation(\n conversation_id, user_id\n )\n\n if not conversation:\n logger.warning(\n f\"Conversation {conversation_id} not found for user {user_id}\"\n )\n return CompressionResult.failure(\"Conversation not found\")\n\n # Check if compression is needed\n if not self.threshold_checker.should_compress(\n conversation, model_id, current_query_tokens\n ):\n # No compression needed, return full history\n queries = conversation.get(\"queries\", [])\n return CompressionResult.success_no_compression(queries)\n\n # Perform compression\n return self._perform_compression(\n conversation_id, conversation, model_id, decoded_token\n )\n\n except Exception as e:\n logger.error(\n f\"Error in compress_if_needed: {str(e)}\", exc_info=True\n )\n return CompressionResult.failure(str(e))\n\n def _perform_compression(\n self,\n conversation_id: str,\n conversation: Dict[str, Any],\n model_id: str,\n decoded_token: Dict[str, Any],\n ) -> CompressionResult:\n \"\"\"\n Perform the actual compression operation.\n\n Args:\n conversation_id: Conversation ID\n conversation: Conversation document\n model_id: Model ID for conversation\n decoded_token: User token\n\n Returns:\n CompressionResult\n \"\"\"\n try:\n # Determine which model to use for compression\n compression_model = (\n settings.COMPRESSION_MODEL_OVERRIDE\n if settings.COMPRESSION_MODEL_OVERRIDE\n else model_id\n )\n\n # Get provider and API key for compression model\n provider = get_provider_from_model_id(compression_model)\n api_key = get_api_key_for_provider(provider)\n\n # Create compression LLM\n compression_llm = LLMCreator.create_llm(\n provider,\n api_key=api_key,\n user_api_key=None,\n decoded_token=decoded_token,\n model_id=compression_model,\n agent_id=conversation.get(\"agent_id\"),\n )\n\n # Create compression service with DB update capability\n compression_service = CompressionService(\n llm=compression_llm,\n model_id=compression_model,\n conversation_service=self.conversation_service,\n )\n\n # Compress all queries up to the latest\n queries_count = len(conversation.get(\"queries\", []))\n compress_up_to = queries_count - 1\n\n if compress_up_to < 0:\n logger.warning(\"No queries to compress\")\n return CompressionResult.success_no_compression([])\n\n logger.info(\n f\"Initiating compression for conversation {conversation_id}: \"\n f\"compressing all {queries_count} queries (0-{compress_up_to})\"\n )\n\n # Perform compression and save to DB\n metadata = compression_service.compress_and_save(\n conversation_id, conversation, compress_up_to\n )\n\n logger.info(\n f\"Compression successful - ratio: {metadata.compression_ratio:.1f}x, \"\n f\"saved {metadata.original_token_count - metadata.compressed_token_count} tokens\"\n )\n\n # Reload conversation with updated metadata\n conversation = self.conversation_service.get_conversation(\n conversation_id, user_id=decoded_token.get(\"sub\")\n )\n\n # Get compressed context\n compressed_summary, recent_queries = (\n compression_service.get_compressed_context(conversation)\n )\n\n return CompressionResult.success_with_compression(\n compressed_summary, recent_queries, metadata\n )\n\n except Exception as e:\n logger.error(f\"Error performing compression: {str(e)}\", exc_info=True)\n return CompressionResult.failure(str(e))\n\n def compress_mid_execution(\n self,\n conversation_id: str,\n user_id: str,\n model_id: str,\n decoded_token: Dict[str, Any],\n current_conversation: Optional[Dict[str, Any]] = None,\n ) -> CompressionResult:\n \"\"\"\n Perform compression during tool execution.\n\n Args:\n conversation_id: Conversation ID\n user_id: User ID\n model_id: Model ID\n decoded_token: User token\n current_conversation: Pre-loaded conversation (optional)\n\n Returns:\n CompressionResult\n \"\"\"\n try:\n # Load conversation if not provided\n if current_conversation:\n conversation = current_conversation\n else:\n conversation = self.conversation_service.get_conversation(\n conversation_id, user_id\n )\n\n if not conversation:\n logger.warning(\n f\"Could not load conversation {conversation_id} for mid-execution compression\"\n )\n return CompressionResult.failure(\"Conversation not found\")\n\n # Perform compression\n return self._perform_compression(\n conversation_id, conversation, model_id, decoded_token\n )\n\n except Exception as e:\n logger.error(\n f\"Error in mid-execution compression: {str(e)}\", exc_info=True\n )\n return CompressionResult.failure(str(e))\n" }, { "path": "application/api/answer/services/compression/prompt_builder.py", "content": "\"\"\"Compression prompt building logic.\"\"\"\n\nimport logging\nfrom pathlib import Path\nfrom typing import Any, Dict, List, Optional\n\nlogger = logging.getLogger(__name__)\n\n\nclass CompressionPromptBuilder:\n \"\"\"Builds prompts for LLM compression calls.\"\"\"\n\n def __init__(self, version: str = \"v1.0\"):\n \"\"\"\n Initialize prompt builder.\n\n Args:\n version: Prompt template version to use\n \"\"\"\n self.version = version\n self.system_prompt = self._load_prompt(version)\n\n def _load_prompt(self, version: str) -> str:\n \"\"\"\n Load prompt template from file.\n\n Args:\n version: Version string (e.g., 'v1.0')\n\n Returns:\n Prompt template content\n\n Raises:\n FileNotFoundError: If prompt template file doesn't exist\n \"\"\"\n current_dir = Path(__file__).resolve().parents[4]\n prompt_path = current_dir / \"prompts\" / \"compression\" / f\"{version}.txt\"\n\n try:\n with open(prompt_path, \"r\") as f:\n return f.read()\n except FileNotFoundError:\n logger.error(f\"Compression prompt template not found: {prompt_path}\")\n raise FileNotFoundError(\n f\"Compression prompt template '{version}' not found at {prompt_path}. \"\n f\"Please ensure the template file exists.\"\n )\n\n def build_prompt(\n self,\n queries: List[Dict[str, Any]],\n existing_compressions: Optional[List[Dict[str, Any]]] = None,\n ) -> List[Dict[str, str]]:\n \"\"\"\n Build messages for compression LLM call.\n\n Args:\n queries: List of query objects to compress\n existing_compressions: List of previous compression points\n\n Returns:\n List of message dicts for LLM\n \"\"\"\n # Build conversation text\n conversation_text = self._format_conversation(queries)\n\n # Add existing compression context if present\n existing_compression_context = \"\"\n if existing_compressions and len(existing_compressions) > 0:\n existing_compression_context = (\n \"\\n\\nIMPORTANT: This conversation has been compressed before. \"\n \"Previous compression summaries:\\n\\n\"\n )\n for i, comp in enumerate(existing_compressions):\n existing_compression_context += (\n f\"--- Compression {i + 1} (up to message {comp.get('query_index', 'unknown')}) ---\\n\"\n f\"{comp.get('compressed_summary', '')}\\n\\n\"\n )\n existing_compression_context += (\n \"Your task is to create a NEW summary that incorporates the context from \"\n \"previous compressions AND the new messages below. The final summary should \"\n \"be comprehensive and include all important information from both previous \"\n \"compressions and new messages.\\n\\n\"\n )\n\n user_prompt = (\n f\"{existing_compression_context}\"\n f\"Here is the conversation to summarize:\\n\\n\"\n f\"{conversation_text}\"\n )\n\n messages = [\n {\"role\": \"system\", \"content\": self.system_prompt},\n {\"role\": \"user\", \"content\": user_prompt},\n ]\n\n return messages\n\n def _format_conversation(self, queries: List[Dict[str, Any]]) -> str:\n \"\"\"\n Format conversation queries into readable text for compression.\n\n Args:\n queries: List of query objects\n\n Returns:\n Formatted conversation text\n \"\"\"\n conversation_lines = []\n\n for i, query in enumerate(queries):\n conversation_lines.append(f\"--- Message {i + 1} ---\")\n conversation_lines.append(f\"User: {query.get('prompt', '')}\")\n\n # Add tool calls if present\n tool_calls = query.get(\"tool_calls\", [])\n if tool_calls:\n conversation_lines.append(\"\\nTool Calls:\")\n for tc in tool_calls:\n tool_name = tc.get(\"tool_name\", \"unknown\")\n action_name = tc.get(\"action_name\", \"unknown\")\n arguments = tc.get(\"arguments\", {})\n result = tc.get(\"result\", \"\")\n if result is None:\n result = \"\"\n status = tc.get(\"status\", \"unknown\")\n\n # Include full tool result for complete compression context\n conversation_lines.append(\n f\" - {tool_name}.{action_name}({arguments}) \"\n f\"[{status}] → {result}\"\n )\n\n # Add agent thought if present\n thought = query.get(\"thought\", \"\")\n if thought:\n conversation_lines.append(f\"\\nAgent Thought: {thought}\")\n\n # Add assistant response\n conversation_lines.append(f\"\\nAssistant: {query.get('response', '')}\")\n\n # Add sources if present\n sources = query.get(\"sources\", [])\n if sources:\n conversation_lines.append(f\"\\nSources Used: {len(sources)} documents\")\n\n conversation_lines.append(\"\") # Empty line between messages\n\n return \"\\n\".join(conversation_lines)\n" }, { "path": "application/api/answer/services/compression/service.py", "content": "\"\"\"Core compression service with simplified responsibilities.\"\"\"\n\nimport logging\nimport re\nfrom datetime import datetime, timezone\nfrom typing import Any, Dict, List, Optional\n\nfrom application.api.answer.services.compression.prompt_builder import (\n CompressionPromptBuilder,\n)\nfrom application.api.answer.services.compression.token_counter import TokenCounter\nfrom application.api.answer.services.compression.types import (\n CompressionMetadata,\n)\nfrom application.core.settings import settings\n\nlogger = logging.getLogger(__name__)\n\n\nclass CompressionService:\n \"\"\"\n Service for compressing conversation history.\n\n Handles DB updates.\n \"\"\"\n\n def __init__(\n self,\n llm,\n model_id: str,\n conversation_service=None,\n prompt_builder: Optional[CompressionPromptBuilder] = None,\n ):\n \"\"\"\n Initialize compression service.\n\n Args:\n llm: LLM instance to use for compression\n model_id: Model ID for compression\n conversation_service: Service for DB operations (optional, for DB updates)\n prompt_builder: Custom prompt builder (optional)\n \"\"\"\n self.llm = llm\n self.model_id = model_id\n self.conversation_service = conversation_service\n self.prompt_builder = prompt_builder or CompressionPromptBuilder(\n version=settings.COMPRESSION_PROMPT_VERSION\n )\n\n def compress_conversation(\n self,\n conversation: Dict[str, Any],\n compress_up_to_index: int,\n ) -> CompressionMetadata:\n \"\"\"\n Compress conversation history up to specified index.\n\n Args:\n conversation: Full conversation document\n compress_up_to_index: Last query index to include in compression\n\n Returns:\n CompressionMetadata with compression details\n\n Raises:\n ValueError: If compress_up_to_index is invalid\n \"\"\"\n try:\n queries = conversation.get(\"queries\", [])\n\n if compress_up_to_index < 0 or compress_up_to_index >= len(queries):\n raise ValueError(\n f\"Invalid compress_up_to_index: {compress_up_to_index} \"\n f\"(conversation has {len(queries)} queries)\"\n )\n\n # Get queries to compress\n queries_to_compress = queries[: compress_up_to_index + 1]\n\n # Check if there are existing compressions\n existing_compressions = conversation.get(\"compression_metadata\", {}).get(\n \"compression_points\", []\n )\n\n if existing_compressions:\n logger.info(\n f\"Found {len(existing_compressions)} previous compression(s) - \"\n f\"will incorporate into new summary\"\n )\n\n # Calculate original token count\n original_tokens = TokenCounter.count_query_tokens(queries_to_compress)\n\n # Log tool call stats\n self._log_tool_call_stats(queries_to_compress)\n\n # Build compression prompt\n messages = self.prompt_builder.build_prompt(\n queries_to_compress, existing_compressions\n )\n\n # Call LLM to generate compression\n logger.info(\n f\"Starting compression: {len(queries_to_compress)} queries \"\n f\"(messages 0-{compress_up_to_index}, {original_tokens} tokens) \"\n f\"using model {self.model_id}\"\n )\n\n response = self.llm.gen(\n model=self.model_id, messages=messages, max_tokens=4000\n )\n\n # Extract summary from response\n compressed_summary = self._extract_summary(response)\n\n # Calculate compressed token count\n compressed_tokens = TokenCounter.count_message_tokens(\n [{\"content\": compressed_summary}]\n )\n\n # Calculate compression ratio\n compression_ratio = (\n original_tokens / compressed_tokens if compressed_tokens > 0 else 0\n )\n\n logger.info(\n f\"Compression complete: {original_tokens} → {compressed_tokens} tokens \"\n f\"({compression_ratio:.1f}x compression)\"\n )\n\n # Build compression metadata\n compression_metadata = CompressionMetadata(\n timestamp=datetime.now(timezone.utc),\n query_index=compress_up_to_index,\n compressed_summary=compressed_summary,\n original_token_count=original_tokens,\n compressed_token_count=compressed_tokens,\n compression_ratio=compression_ratio,\n model_used=self.model_id,\n compression_prompt_version=self.prompt_builder.version,\n )\n\n return compression_metadata\n\n except Exception as e:\n logger.error(f\"Error compressing conversation: {str(e)}\", exc_info=True)\n raise\n\n def compress_and_save(\n self,\n conversation_id: str,\n conversation: Dict[str, Any],\n compress_up_to_index: int,\n ) -> CompressionMetadata:\n \"\"\"\n Compress conversation and save to database.\n\n Args:\n conversation_id: Conversation ID\n conversation: Full conversation document\n compress_up_to_index: Last query index to include\n\n Returns:\n CompressionMetadata\n\n Raises:\n ValueError: If conversation_service not provided or invalid index\n \"\"\"\n if not self.conversation_service:\n raise ValueError(\n \"conversation_service required for compress_and_save operation\"\n )\n\n # Perform compression\n metadata = self.compress_conversation(conversation, compress_up_to_index)\n\n # Save to database\n self.conversation_service.update_compression_metadata(\n conversation_id, metadata.to_dict()\n )\n\n logger.info(f\"Compression metadata saved to database for {conversation_id}\")\n\n return metadata\n\n def get_compressed_context(\n self, conversation: Dict[str, Any]\n ) -> tuple[Optional[str], List[Dict[str, Any]]]:\n \"\"\"\n Get compressed summary + recent uncompressed messages.\n\n Args:\n conversation: Full conversation document\n\n Returns:\n (compressed_summary, recent_messages)\n \"\"\"\n try:\n compression_metadata = conversation.get(\"compression_metadata\", {})\n\n if not compression_metadata.get(\"is_compressed\"):\n logger.debug(\"No compression metadata found - using full history\")\n queries = conversation.get(\"queries\", [])\n if queries is None:\n logger.error(\"Conversation queries is None - returning empty list\")\n return None, []\n return None, queries\n\n compression_points = compression_metadata.get(\"compression_points\", [])\n\n if not compression_points:\n logger.debug(\"No compression points found - using full history\")\n queries = conversation.get(\"queries\", [])\n if queries is None:\n logger.error(\"Conversation queries is None - returning empty list\")\n return None, []\n return None, queries\n\n # Get the most recent compression point\n latest_compression = compression_points[-1]\n compressed_summary = latest_compression.get(\"compressed_summary\")\n last_compressed_index = latest_compression.get(\"query_index\")\n compressed_tokens = latest_compression.get(\"compressed_token_count\", 0)\n original_tokens = latest_compression.get(\"original_token_count\", 0)\n\n # Get only messages after compression point\n queries = conversation.get(\"queries\", [])\n total_queries = len(queries)\n recent_queries = queries[last_compressed_index + 1 :]\n\n logger.info(\n f\"Using compressed context: summary ({compressed_tokens} tokens, \"\n f\"compressed from {original_tokens}) + {len(recent_queries)} recent messages \"\n f\"(messages {last_compressed_index + 1}-{total_queries - 1})\"\n )\n\n return compressed_summary, recent_queries\n\n except Exception as e:\n logger.error(\n f\"Error getting compressed context: {str(e)}\", exc_info=True\n )\n queries = conversation.get(\"queries\", [])\n if queries is None:\n return None, []\n return None, queries\n\n def _extract_summary(self, llm_response: str) -> str:\n \"\"\"\n Extract clean summary from LLM response.\n\n Args:\n llm_response: Raw LLM response\n\n Returns:\n Cleaned summary text\n \"\"\"\n try:\n # Try to extract content within tags\n summary_match = re.search(\n r\"(.*?)\", llm_response, re.DOTALL\n )\n\n if summary_match:\n summary = summary_match.group(1).strip()\n else:\n # If no summary tags, remove analysis tags and use the rest\n summary = re.sub(\n r\".*?\", \"\", llm_response, flags=re.DOTALL\n ).strip()\n\n return summary\n\n except Exception as e:\n logger.warning(f\"Error extracting summary: {str(e)}, using full response\")\n return llm_response\n\n def _log_tool_call_stats(self, queries: List[Dict[str, Any]]) -> None:\n \"\"\"Log statistics about tool calls in queries.\"\"\"\n total_tool_calls = 0\n total_tool_result_chars = 0\n tool_call_breakdown = {}\n\n for q in queries:\n for tc in q.get(\"tool_calls\", []):\n total_tool_calls += 1\n tool_name = tc.get(\"tool_name\", \"unknown\")\n action_name = tc.get(\"action_name\", \"unknown\")\n key = f\"{tool_name}.{action_name}\"\n tool_call_breakdown[key] = tool_call_breakdown.get(key, 0) + 1\n\n # Track total tool result size\n result = tc.get(\"result\", \"\")\n if result:\n total_tool_result_chars += len(str(result))\n\n if total_tool_calls > 0:\n tool_breakdown_str = \", \".join(\n f\"{tool}({count})\"\n for tool, count in sorted(tool_call_breakdown.items())\n )\n tool_result_kb = total_tool_result_chars / 1024\n logger.info(\n f\"Tool call breakdown: {tool_breakdown_str} \"\n f\"(total result size: {tool_result_kb:.1f} KB, {total_tool_result_chars:,} chars)\"\n )\n" }, { "path": "application/api/answer/services/compression/threshold_checker.py", "content": "\"\"\"Compression threshold checking logic.\"\"\"\n\nimport logging\nfrom typing import Any, Dict\n\nfrom application.core.model_utils import get_token_limit\nfrom application.core.settings import settings\nfrom application.api.answer.services.compression.token_counter import TokenCounter\n\nlogger = logging.getLogger(__name__)\n\n\nclass CompressionThresholdChecker:\n \"\"\"Determines if compression is needed based on token thresholds.\"\"\"\n\n def __init__(self, threshold_percentage: float = None):\n \"\"\"\n Initialize threshold checker.\n\n Args:\n threshold_percentage: Percentage of context to use as threshold\n (defaults to settings.COMPRESSION_THRESHOLD_PERCENTAGE)\n \"\"\"\n self.threshold_percentage = (\n threshold_percentage or settings.COMPRESSION_THRESHOLD_PERCENTAGE\n )\n\n def should_compress(\n self,\n conversation: Dict[str, Any],\n model_id: str,\n current_query_tokens: int = 500,\n ) -> bool:\n \"\"\"\n Determine if compression is needed.\n\n Args:\n conversation: Full conversation document\n model_id: Target model for this request\n current_query_tokens: Estimated tokens for current query\n\n Returns:\n True if tokens >= threshold% of context window\n \"\"\"\n try:\n # Calculate total tokens in conversation\n total_tokens = TokenCounter.count_conversation_tokens(conversation)\n total_tokens += current_query_tokens\n\n # Get context window limit for model\n context_limit = get_token_limit(model_id)\n\n # Calculate threshold\n threshold = int(context_limit * self.threshold_percentage)\n\n compression_needed = total_tokens >= threshold\n percentage_used = (total_tokens / context_limit) * 100\n\n if compression_needed:\n logger.warning(\n f\"COMPRESSION TRIGGERED: {total_tokens} tokens / {context_limit} limit \"\n f\"({percentage_used:.1f}% used, threshold: {self.threshold_percentage * 100:.0f}%)\"\n )\n else:\n logger.info(\n f\"Compression check: {total_tokens}/{context_limit} tokens \"\n f\"({percentage_used:.1f}% used, threshold: {self.threshold_percentage * 100:.0f}%) - No compression needed\"\n )\n\n return compression_needed\n\n except Exception as e:\n logger.error(f\"Error checking compression need: {str(e)}\", exc_info=True)\n return False\n\n def check_message_tokens(self, messages: list, model_id: str) -> bool:\n \"\"\"\n Check if message list exceeds threshold.\n\n Args:\n messages: List of message dicts\n model_id: Target model\n\n Returns:\n True if at or above threshold\n \"\"\"\n try:\n current_tokens = TokenCounter.count_message_tokens(messages)\n context_limit = get_token_limit(model_id)\n threshold = int(context_limit * self.threshold_percentage)\n\n if current_tokens >= threshold:\n logger.warning(\n f\"Message context limit approaching: {current_tokens}/{context_limit} tokens \"\n f\"({(current_tokens/context_limit)*100:.1f}%)\"\n )\n return True\n\n return False\n\n except Exception as e:\n logger.error(f\"Error checking message tokens: {str(e)}\", exc_info=True)\n return False\n" }, { "path": "application/api/answer/services/compression/token_counter.py", "content": "\"\"\"Token counting utilities for compression.\"\"\"\n\nimport logging\nfrom typing import Any, Dict, List\n\nfrom application.utils import num_tokens_from_string\nfrom application.core.settings import settings\n\nlogger = logging.getLogger(__name__)\n\n\nclass TokenCounter:\n \"\"\"Centralized token counting for conversations and messages.\"\"\"\n\n @staticmethod\n def count_message_tokens(messages: List[Dict]) -> int:\n \"\"\"\n Calculate total tokens in a list of messages.\n\n Args:\n messages: List of message dicts with 'content' field\n\n Returns:\n Total token count\n \"\"\"\n total_tokens = 0\n for message in messages:\n content = message.get(\"content\", \"\")\n if isinstance(content, str):\n total_tokens += num_tokens_from_string(content)\n elif isinstance(content, list):\n # Handle structured content (tool calls, etc.)\n for item in content:\n if isinstance(item, dict):\n total_tokens += num_tokens_from_string(str(item))\n return total_tokens\n\n @staticmethod\n def count_query_tokens(\n queries: List[Dict[str, Any]], include_tool_calls: bool = True\n ) -> int:\n \"\"\"\n Count tokens across multiple query objects.\n\n Args:\n queries: List of query objects from conversation\n include_tool_calls: Whether to count tool call tokens\n\n Returns:\n Total token count\n \"\"\"\n total_tokens = 0\n\n for query in queries:\n # Count prompt and response tokens\n if \"prompt\" in query:\n total_tokens += num_tokens_from_string(query[\"prompt\"])\n if \"response\" in query:\n total_tokens += num_tokens_from_string(query[\"response\"])\n if \"thought\" in query:\n total_tokens += num_tokens_from_string(query.get(\"thought\", \"\"))\n\n # Count tool call tokens\n if include_tool_calls and \"tool_calls\" in query:\n for tool_call in query[\"tool_calls\"]:\n tool_call_string = (\n f\"Tool: {tool_call.get('tool_name')} | \"\n f\"Action: {tool_call.get('action_name')} | \"\n f\"Args: {tool_call.get('arguments')} | \"\n f\"Response: {tool_call.get('result')}\"\n )\n total_tokens += num_tokens_from_string(tool_call_string)\n\n return total_tokens\n\n @staticmethod\n def count_conversation_tokens(\n conversation: Dict[str, Any], include_system_prompt: bool = False\n ) -> int:\n \"\"\"\n Calculate total tokens in a conversation.\n\n Args:\n conversation: Conversation document\n include_system_prompt: Whether to include system prompt in count\n\n Returns:\n Total token count\n \"\"\"\n try:\n queries = conversation.get(\"queries\", [])\n total_tokens = TokenCounter.count_query_tokens(queries)\n\n # Add system prompt tokens if requested\n if include_system_prompt:\n # Rough estimate for system prompt\n total_tokens += settings.RESERVED_TOKENS.get(\"system_prompt\", 500)\n\n return total_tokens\n\n except Exception as e:\n logger.error(f\"Error calculating conversation tokens: {str(e)}\")\n return 0\n" }, { "path": "application/api/answer/services/compression/types.py", "content": "\"\"\"Type definitions for compression module.\"\"\"\n\nfrom dataclasses import dataclass, field\nfrom datetime import datetime\nfrom typing import Any, Dict, List, Optional\n\n\n@dataclass\nclass CompressionMetadata:\n \"\"\"Metadata about a compression operation.\"\"\"\n\n timestamp: datetime\n query_index: int\n compressed_summary: str\n original_token_count: int\n compressed_token_count: int\n compression_ratio: float\n model_used: str\n compression_prompt_version: str\n\n def to_dict(self) -> Dict[str, Any]:\n \"\"\"Convert to dictionary for DB storage.\"\"\"\n return {\n \"timestamp\": self.timestamp,\n \"query_index\": self.query_index,\n \"compressed_summary\": self.compressed_summary,\n \"original_token_count\": self.original_token_count,\n \"compressed_token_count\": self.compressed_token_count,\n \"compression_ratio\": self.compression_ratio,\n \"model_used\": self.model_used,\n \"compression_prompt_version\": self.compression_prompt_version,\n }\n\n\n@dataclass\nclass CompressionResult:\n \"\"\"Result of a compression operation.\"\"\"\n\n success: bool\n compressed_summary: Optional[str] = None\n recent_queries: List[Dict[str, Any]] = field(default_factory=list)\n metadata: Optional[CompressionMetadata] = None\n error: Optional[str] = None\n compression_performed: bool = False\n\n @classmethod\n def success_with_compression(\n cls, summary: str, queries: List[Dict], metadata: CompressionMetadata\n ) -> \"CompressionResult\":\n \"\"\"Create a successful result with compression.\"\"\"\n return cls(\n success=True,\n compressed_summary=summary,\n recent_queries=queries,\n metadata=metadata,\n compression_performed=True,\n )\n\n @classmethod\n def success_no_compression(cls, queries: List[Dict]) -> \"CompressionResult\":\n \"\"\"Create a successful result without compression needed.\"\"\"\n return cls(\n success=True,\n recent_queries=queries,\n compression_performed=False,\n )\n\n @classmethod\n def failure(cls, error: str) -> \"CompressionResult\":\n \"\"\"Create a failure result.\"\"\"\n return cls(success=False, error=error, compression_performed=False)\n\n def as_history(self) -> List[Dict[str, str]]:\n \"\"\"\n Convert recent queries to history format.\n\n Returns:\n List of prompt/response dicts\n \"\"\"\n return [\n {\"prompt\": q[\"prompt\"], \"response\": q[\"response\"]}\n for q in self.recent_queries\n ]\n" }, { "path": "application/api/answer/services/conversation_service.py", "content": "import logging\nfrom datetime import datetime, timezone\nfrom typing import Any, Dict, List, Optional\n\nfrom application.core.mongo_db import MongoDB\n\nfrom application.core.settings import settings\nfrom bson import ObjectId\n\n\nlogger = logging.getLogger(__name__)\n\n\nclass ConversationService:\n def __init__(self):\n mongo = MongoDB.get_client()\n db = mongo[settings.MONGO_DB_NAME]\n self.conversations_collection = db[\"conversations\"]\n self.agents_collection = db[\"agents\"]\n\n def get_conversation(\n self, conversation_id: str, user_id: str\n ) -> Optional[Dict[str, Any]]:\n \"\"\"Retrieve a conversation with proper access control\"\"\"\n if not conversation_id or not user_id:\n return None\n try:\n conversation = self.conversations_collection.find_one(\n {\n \"_id\": ObjectId(conversation_id),\n \"$or\": [{\"user\": user_id}, {\"shared_with\": user_id}],\n }\n )\n\n if not conversation:\n logger.warning(\n f\"Conversation not found or unauthorized - ID: {conversation_id}, User: {user_id}\"\n )\n return None\n conversation[\"_id\"] = str(conversation[\"_id\"])\n return conversation\n except Exception as e:\n logger.error(f\"Error fetching conversation: {str(e)}\", exc_info=True)\n return None\n\n def save_conversation(\n self,\n conversation_id: Optional[str],\n question: str,\n response: str,\n thought: str,\n sources: List[Dict[str, Any]],\n tool_calls: List[Dict[str, Any]],\n llm: Any,\n model_id: str,\n decoded_token: Dict[str, Any],\n index: Optional[int] = None,\n api_key: Optional[str] = None,\n agent_id: Optional[str] = None,\n is_shared_usage: bool = False,\n shared_token: Optional[str] = None,\n attachment_ids: Optional[List[str]] = None,\n ) -> str:\n \"\"\"Save or update a conversation in the database\"\"\"\n if decoded_token is None:\n raise ValueError(\"Invalid or missing authentication token\")\n user_id = decoded_token.get(\"sub\")\n if not user_id:\n raise ValueError(\"User ID not found in token\")\n current_time = datetime.now(timezone.utc)\n\n # clean up in sources array such that we save max 1k characters for text part\n for source in sources:\n if \"text\" in source and isinstance(source[\"text\"], str):\n source[\"text\"] = source[\"text\"][:1000]\n\n if conversation_id is not None and index is not None:\n # Update existing conversation with new query\n\n result = self.conversations_collection.update_one(\n {\n \"_id\": ObjectId(conversation_id),\n \"user\": user_id,\n f\"queries.{index}\": {\"$exists\": True},\n },\n {\n \"$set\": {\n f\"queries.{index}.prompt\": question,\n f\"queries.{index}.response\": response,\n f\"queries.{index}.thought\": thought,\n f\"queries.{index}.sources\": sources,\n f\"queries.{index}.tool_calls\": tool_calls,\n f\"queries.{index}.timestamp\": current_time,\n f\"queries.{index}.attachments\": attachment_ids,\n f\"queries.{index}.model_id\": model_id,\n }\n },\n )\n\n if result.matched_count == 0:\n raise ValueError(\"Conversation not found or unauthorized\")\n self.conversations_collection.update_one(\n {\n \"_id\": ObjectId(conversation_id),\n \"user\": user_id,\n f\"queries.{index}\": {\"$exists\": True},\n },\n {\"$push\": {\"queries\": {\"$each\": [], \"$slice\": index + 1}}},\n )\n return conversation_id\n elif conversation_id:\n # Append new message to existing conversation\n\n result = self.conversations_collection.update_one(\n {\"_id\": ObjectId(conversation_id), \"user\": user_id},\n {\n \"$push\": {\n \"queries\": {\n \"prompt\": question,\n \"response\": response,\n \"thought\": thought,\n \"sources\": sources,\n \"tool_calls\": tool_calls,\n \"timestamp\": current_time,\n \"attachments\": attachment_ids,\n \"model_id\": model_id,\n }\n }\n },\n )\n\n if result.matched_count == 0:\n raise ValueError(\"Conversation not found or unauthorized\")\n return conversation_id\n else:\n # Create new conversation\n\n messages_summary = [\n {\n \"role\": \"system\",\n \"content\": \"You are a helpful assistant that creates concise conversation titles. \"\n \"Summarize conversations in 3 words or less using the same language as the user.\",\n },\n {\n \"role\": \"user\",\n \"content\": \"Summarise following conversation in no more than 3 words, \"\n \"respond ONLY with the summary, use the same language as the \"\n \"user query \\n\\nUser: \" + question + \"\\n\\n\" + \"AI: \" + response,\n },\n ]\n\n completion = llm.gen(\n model=model_id, messages=messages_summary, max_tokens=500\n )\n\n if not completion or not completion.strip():\n completion = question[:50] if question else \"New Conversation\"\n\n conversation_data = {\n \"user\": user_id,\n \"date\": current_time,\n \"name\": completion,\n \"queries\": [\n {\n \"prompt\": question,\n \"response\": response,\n \"thought\": thought,\n \"sources\": sources,\n \"tool_calls\": tool_calls,\n \"timestamp\": current_time,\n \"attachments\": attachment_ids,\n \"model_id\": model_id,\n }\n ],\n }\n\n if api_key:\n if agent_id:\n conversation_data[\"agent_id\"] = agent_id\n if is_shared_usage:\n conversation_data[\"is_shared_usage\"] = is_shared_usage\n conversation_data[\"shared_token\"] = shared_token\n agent = self.agents_collection.find_one({\"key\": api_key})\n if agent:\n conversation_data[\"api_key\"] = agent[\"key\"]\n result = self.conversations_collection.insert_one(conversation_data)\n return str(result.inserted_id)\n\n def update_compression_metadata(\n self, conversation_id: str, compression_metadata: Dict[str, Any]\n ) -> None:\n \"\"\"\n Update conversation with compression metadata.\n\n Uses $push with $slice to keep only the most recent compression points,\n preventing unbounded array growth. Since each compression incorporates\n previous compressions, older points become redundant.\n\n Args:\n conversation_id: Conversation ID\n compression_metadata: Compression point data\n \"\"\"\n try:\n self.conversations_collection.update_one(\n {\"_id\": ObjectId(conversation_id)},\n {\n \"$set\": {\n \"compression_metadata.is_compressed\": True,\n \"compression_metadata.last_compression_at\": compression_metadata.get(\n \"timestamp\"\n ),\n },\n \"$push\": {\n \"compression_metadata.compression_points\": {\n \"$each\": [compression_metadata],\n \"$slice\": -settings.COMPRESSION_MAX_HISTORY_POINTS,\n }\n },\n },\n )\n logger.info(\n f\"Updated compression metadata for conversation {conversation_id}\"\n )\n except Exception as e:\n logger.error(\n f\"Error updating compression metadata: {str(e)}\", exc_info=True\n )\n raise\n\n def append_compression_message(\n self, conversation_id: str, compression_metadata: Dict[str, Any]\n ) -> None:\n \"\"\"\n Append a synthetic compression summary entry into the conversation history.\n This makes the summary visible in the DB alongside normal queries.\n \"\"\"\n try:\n summary = compression_metadata.get(\"compressed_summary\", \"\")\n if not summary:\n return\n timestamp = compression_metadata.get(\"timestamp\", datetime.now(timezone.utc))\n\n self.conversations_collection.update_one(\n {\"_id\": ObjectId(conversation_id)},\n {\n \"$push\": {\n \"queries\": {\n \"prompt\": \"[Context Compression Summary]\",\n \"response\": summary,\n \"thought\": \"\",\n \"sources\": [],\n \"tool_calls\": [],\n \"timestamp\": timestamp,\n \"attachments\": [],\n \"model_id\": compression_metadata.get(\"model_used\"),\n }\n }\n },\n )\n logger.info(f\"Appended compression summary to conversation {conversation_id}\")\n except Exception as e:\n logger.error(\n f\"Error appending compression summary: {str(e)}\", exc_info=True\n )\n\n def get_compression_metadata(\n self, conversation_id: str\n ) -> Optional[Dict[str, Any]]:\n \"\"\"\n Get compression metadata for a conversation.\n\n Args:\n conversation_id: Conversation ID\n\n Returns:\n Compression metadata dict or None\n \"\"\"\n try:\n conversation = self.conversations_collection.find_one(\n {\"_id\": ObjectId(conversation_id)}, {\"compression_metadata\": 1}\n )\n return conversation.get(\"compression_metadata\") if conversation else None\n except Exception as e:\n logger.error(\n f\"Error getting compression metadata: {str(e)}\", exc_info=True\n )\n return None\n" }, { "path": "application/api/answer/services/prompt_renderer.py", "content": "import logging\nfrom typing import Any, Dict, Optional\n\nfrom application.templates.namespaces import NamespaceManager\n\nfrom application.templates.template_engine import TemplateEngine, TemplateRenderError\n\nlogger = logging.getLogger(__name__)\n\n\nclass PromptRenderer:\n \"\"\"Service for rendering prompts with dynamic context using namespaces\"\"\"\n\n def __init__(self):\n self.template_engine = TemplateEngine()\n self.namespace_manager = NamespaceManager()\n\n def render_prompt(\n self,\n prompt_content: str,\n user_id: Optional[str] = None,\n request_id: Optional[str] = None,\n passthrough_data: Optional[Dict[str, Any]] = None,\n docs: Optional[list] = None,\n docs_together: Optional[str] = None,\n tools_data: Optional[Dict[str, Any]] = None,\n **kwargs,\n ) -> str:\n \"\"\"\n Render prompt with full context from all namespaces.\n\n Args:\n prompt_content: Raw prompt template string\n user_id: Current user identifier\n request_id: Unique request identifier\n passthrough_data: Parameters from web request\n docs: RAG retrieved documents\n docs_together: Concatenated document content\n tools_data: Pre-fetched tool results organized by tool name\n **kwargs: Additional parameters for namespace builders\n\n Returns:\n Rendered prompt string with all variables substituted\n\n Raises:\n TemplateRenderError: If template rendering fails\n \"\"\"\n if not prompt_content:\n return \"\"\n\n uses_template = self._uses_template_syntax(prompt_content)\n\n if not uses_template:\n return self._apply_legacy_substitutions(prompt_content, docs_together)\n\n try:\n context = self.namespace_manager.build_context(\n user_id=user_id,\n request_id=request_id,\n passthrough_data=passthrough_data,\n docs=docs,\n docs_together=docs_together,\n tools_data=tools_data,\n **kwargs,\n )\n\n return self.template_engine.render(prompt_content, context)\n except TemplateRenderError:\n raise\n except Exception as e:\n error_msg = f\"Prompt rendering failed: {str(e)}\"\n logger.error(error_msg)\n raise TemplateRenderError(error_msg) from e\n\n def _uses_template_syntax(self, prompt_content: str) -> bool:\n \"\"\"Check if prompt uses Jinja2 template syntax\"\"\"\n return \"{{\" in prompt_content and \"}}\" in prompt_content\n\n def _apply_legacy_substitutions(\n self, prompt_content: str, docs_together: Optional[str] = None\n ) -> str:\n \"\"\"\n Apply backward-compatible substitutions for old prompt format.\n\n Handles legacy {summaries} and {query} placeholders during transition period.\n \"\"\"\n if docs_together:\n prompt_content = prompt_content.replace(\"{summaries}\", docs_together)\n return prompt_content\n\n def validate_template(self, prompt_content: str) -> bool:\n \"\"\"Validate prompt template syntax\"\"\"\n return self.template_engine.validate_template(prompt_content)\n\n def extract_variables(self, prompt_content: str) -> set[str]:\n \"\"\"Extract all variable names from prompt template\"\"\"\n return self.template_engine.extract_variables(prompt_content)\n" }, { "path": "application/api/answer/services/stream_processor.py", "content": "import datetime\nimport json\nimport logging\nimport os\nfrom pathlib import Path\nfrom typing import Any, Dict, Optional, Set\n\nfrom bson.dbref import DBRef\n\nfrom bson.objectid import ObjectId\n\nfrom application.agents.agent_creator import AgentCreator\nfrom application.api.answer.services.compression import CompressionOrchestrator\nfrom application.api.answer.services.compression.token_counter import TokenCounter\nfrom application.api.answer.services.conversation_service import ConversationService\nfrom application.api.answer.services.prompt_renderer import PromptRenderer\nfrom application.core.model_utils import (\n get_api_key_for_provider,\n get_default_model_id,\n get_provider_from_model_id,\n validate_model_id,\n)\nfrom application.core.mongo_db import MongoDB\nfrom application.core.settings import settings\nfrom application.retriever.retriever_creator import RetrieverCreator\nfrom application.utils import (\n calculate_doc_token_budget,\n limit_chat_history,\n)\n\nlogger = logging.getLogger(__name__)\n\n\ndef get_prompt(prompt_id: str, prompts_collection=None) -> str:\n \"\"\"\n Get a prompt by preset name or MongoDB ID\n \"\"\"\n current_dir = Path(__file__).resolve().parents[3]\n prompts_dir = current_dir / \"prompts\"\n\n preset_mapping = {\n \"default\": \"chat_combine_default.txt\",\n \"creative\": \"chat_combine_creative.txt\",\n \"strict\": \"chat_combine_strict.txt\",\n \"reduce\": \"chat_reduce_prompt.txt\",\n }\n\n if prompt_id in preset_mapping:\n file_path = os.path.join(prompts_dir, preset_mapping[prompt_id])\n try:\n with open(file_path, \"r\") as f:\n return f.read()\n except FileNotFoundError:\n raise FileNotFoundError(f\"Prompt file not found: {file_path}\")\n try:\n if prompts_collection is None:\n mongo = MongoDB.get_client()\n db = mongo[settings.MONGO_DB_NAME]\n prompts_collection = db[\"prompts\"]\n prompt_doc = prompts_collection.find_one({\"_id\": ObjectId(prompt_id)})\n if not prompt_doc:\n raise ValueError(f\"Prompt with ID {prompt_id} not found\")\n return prompt_doc[\"content\"]\n except Exception as e:\n raise ValueError(f\"Invalid prompt ID: {prompt_id}\") from e\n\n\nclass StreamProcessor:\n def __init__(\n self, request_data: Dict[str, Any], decoded_token: Optional[Dict[str, Any]]\n ):\n mongo = MongoDB.get_client()\n self.db = mongo[settings.MONGO_DB_NAME]\n self.agents_collection = self.db[\"agents\"]\n self.attachments_collection = self.db[\"attachments\"]\n self.prompts_collection = self.db[\"prompts\"]\n\n self.data = request_data\n self.decoded_token = decoded_token\n self.initial_user_id = (\n self.decoded_token.get(\"sub\") if self.decoded_token is not None else None\n )\n self.conversation_id = self.data.get(\"conversation_id\")\n self.source = {}\n self.all_sources = []\n self.attachments = []\n self.history = []\n self.retrieved_docs = []\n self.agent_config = {}\n self.retriever_config = {}\n self.is_shared_usage = False\n self.shared_token = None\n self.agent_id = self.data.get(\"agent_id\")\n self.model_id: Optional[str] = None\n self.conversation_service = ConversationService()\n self.compression_orchestrator = CompressionOrchestrator(\n self.conversation_service\n )\n self.prompt_renderer = PromptRenderer()\n self._prompt_content: Optional[str] = None\n self._required_tool_actions: Optional[Dict[str, Set[Optional[str]]]] = None\n self.compressed_summary: Optional[str] = None\n self.compressed_summary_tokens: int = 0\n\n def initialize(self):\n \"\"\"Initialize all required components for processing\"\"\"\n self._configure_agent()\n self._validate_and_set_model()\n self._configure_source()\n self._configure_retriever()\n self._load_conversation_history()\n self._process_attachments()\n\n def _load_conversation_history(self):\n \"\"\"Load conversation history either from DB or request\"\"\"\n if self.conversation_id and self.initial_user_id:\n conversation = self.conversation_service.get_conversation(\n self.conversation_id, self.initial_user_id\n )\n if not conversation:\n raise ValueError(\"Conversation not found or unauthorized\")\n\n # Check if compression is enabled and needed\n if settings.ENABLE_CONVERSATION_COMPRESSION:\n self._handle_compression(conversation)\n else:\n # Original behavior - load all history\n self.history = [\n {\"prompt\": query[\"prompt\"], \"response\": query[\"response\"]}\n for query in conversation.get(\"queries\", [])\n ]\n else:\n self.history = limit_chat_history(\n json.loads(self.data.get(\"history\", \"[]\")), model_id=self.model_id\n )\n\n def _handle_compression(self, conversation: Dict[str, Any]):\n \"\"\"\n Handle conversation compression logic using orchestrator.\n\n Args:\n conversation: Full conversation document\n \"\"\"\n try:\n # Use orchestrator to handle all compression logic\n result = self.compression_orchestrator.compress_if_needed(\n conversation_id=self.conversation_id,\n user_id=self.initial_user_id,\n model_id=self.model_id,\n decoded_token=self.decoded_token,\n )\n\n if not result.success:\n logger.error(f\"Compression failed: {result.error}, using full history\")\n self.history = [\n {\"prompt\": query[\"prompt\"], \"response\": query[\"response\"]}\n for query in conversation.get(\"queries\", [])\n ]\n return\n\n # Set compressed summary if compression was performed\n if result.compression_performed and result.compressed_summary:\n self.compressed_summary = result.compressed_summary\n self.compressed_summary_tokens = TokenCounter.count_message_tokens(\n [{\"content\": result.compressed_summary}]\n )\n logger.info(\n f\"Using compressed summary ({self.compressed_summary_tokens} tokens) \"\n f\"+ {len(result.recent_queries)} recent messages\"\n )\n\n # Build history from recent queries\n self.history = result.as_history()\n\n except Exception as e:\n logger.error(\n f\"Error handling compression, falling back to standard history: {str(e)}\",\n exc_info=True,\n )\n # Fallback to original behavior\n self.history = [\n {\"prompt\": query[\"prompt\"], \"response\": query[\"response\"]}\n for query in conversation.get(\"queries\", [])\n ]\n\n def _process_attachments(self):\n \"\"\"Process any attachments in the request\"\"\"\n attachment_ids = self.data.get(\"attachments\", [])\n self.attachments = self._get_attachments_content(\n attachment_ids, self.initial_user_id\n )\n\n def _get_attachments_content(self, attachment_ids, user_id):\n \"\"\"\n Retrieve content from attachment documents based on their IDs.\n \"\"\"\n if not attachment_ids:\n return []\n attachments = []\n for attachment_id in attachment_ids:\n try:\n attachment_doc = self.attachments_collection.find_one(\n {\"_id\": ObjectId(attachment_id), \"user\": user_id}\n )\n\n if attachment_doc:\n attachments.append(attachment_doc)\n except Exception as e:\n logger.error(\n f\"Error retrieving attachment {attachment_id}: {e}\", exc_info=True\n )\n return attachments\n\n def _validate_and_set_model(self):\n \"\"\"Validate and set model_id from request\"\"\"\n from application.core.model_settings import ModelRegistry\n\n requested_model = self.data.get(\"model_id\")\n\n if requested_model:\n if not validate_model_id(requested_model):\n registry = ModelRegistry.get_instance()\n available_models = [m.id for m in registry.get_enabled_models()]\n raise ValueError(\n f\"Invalid model_id '{requested_model}'. \"\n f\"Available models: {', '.join(available_models[:5])}\"\n + (\n f\" and {len(available_models) - 5} more\"\n if len(available_models) > 5\n else \"\"\n )\n )\n self.model_id = requested_model\n else:\n # Check if agent has a default model configured\n agent_default_model = self.agent_config.get(\"default_model_id\", \"\")\n if agent_default_model and validate_model_id(agent_default_model):\n self.model_id = agent_default_model\n else:\n self.model_id = get_default_model_id()\n\n def _get_agent_key(self, agent_id: Optional[str], user_id: Optional[str]) -> tuple:\n \"\"\"Get API key for agent with access control\"\"\"\n if not agent_id:\n return None, False, None\n try:\n agent = self.agents_collection.find_one({\"_id\": ObjectId(agent_id)})\n if agent is None:\n raise Exception(\"Agent not found\")\n is_owner = agent.get(\"user\") == user_id\n is_shared_with_user = agent.get(\n \"shared_publicly\", False\n ) or user_id in agent.get(\"shared_with\", [])\n\n if not (is_owner or is_shared_with_user):\n raise Exception(\"Unauthorized access to the agent\")\n if is_owner:\n self.agents_collection.update_one(\n {\"_id\": ObjectId(agent_id)},\n {\n \"$set\": {\n \"lastUsedAt\": datetime.datetime.now(datetime.timezone.utc)\n }\n },\n )\n return str(agent[\"key\"]), not is_owner, agent.get(\"shared_token\")\n except Exception as e:\n logger.error(f\"Error in get_agent_key: {str(e)}\", exc_info=True)\n raise\n\n def _get_data_from_api_key(self, api_key: str) -> Dict[str, Any]:\n data = self.agents_collection.find_one({\"key\": api_key})\n if not data:\n raise Exception(\"Invalid API Key, please generate a new key\", 401)\n source = data.get(\"source\")\n if isinstance(source, DBRef):\n source_doc = self.db.dereference(source)\n if source_doc:\n data[\"source\"] = str(source_doc[\"_id\"])\n data[\"retriever\"] = source_doc.get(\"retriever\", data.get(\"retriever\"))\n data[\"chunks\"] = source_doc.get(\"chunks\", data.get(\"chunks\"))\n else:\n data[\"source\"] = None\n elif source == \"default\":\n data[\"source\"] = \"default\"\n else:\n data[\"source\"] = None\n # Handle multiple sources\n\n sources = data.get(\"sources\", [])\n if sources and isinstance(sources, list):\n sources_list = []\n for i, source_ref in enumerate(sources):\n if source_ref == \"default\":\n processed_source = {\n \"id\": \"default\",\n \"retriever\": \"classic\",\n \"chunks\": data.get(\"chunks\", \"2\"),\n }\n sources_list.append(processed_source)\n elif isinstance(source_ref, DBRef):\n source_doc = self.db.dereference(source_ref)\n if source_doc:\n processed_source = {\n \"id\": str(source_doc[\"_id\"]),\n \"retriever\": source_doc.get(\"retriever\", \"classic\"),\n \"chunks\": source_doc.get(\"chunks\", data.get(\"chunks\", \"2\")),\n }\n sources_list.append(processed_source)\n data[\"sources\"] = sources_list\n else:\n data[\"sources\"] = []\n\n # Preserve model configuration from agent\n data[\"default_model_id\"] = data.get(\"default_model_id\", \"\")\n\n return data\n\n def _configure_source(self):\n \"\"\"Configure the source based on agent data\"\"\"\n api_key = self.data.get(\"api_key\") or self.agent_key\n\n if api_key:\n agent_data = self._get_data_from_api_key(api_key)\n\n if agent_data.get(\"sources\") and len(agent_data[\"sources\"]) > 0:\n source_ids = [\n source[\"id\"] for source in agent_data[\"sources\"] if source.get(\"id\")\n ]\n if source_ids:\n self.source = {\"active_docs\": source_ids}\n else:\n self.source = {}\n self.all_sources = agent_data[\"sources\"]\n elif agent_data.get(\"source\"):\n self.source = {\"active_docs\": agent_data[\"source\"]}\n self.all_sources = [\n {\n \"id\": agent_data[\"source\"],\n \"retriever\": agent_data.get(\"retriever\", \"classic\"),\n }\n ]\n else:\n self.source = {}\n self.all_sources = []\n return\n if \"active_docs\" in self.data:\n self.source = {\"active_docs\": self.data[\"active_docs\"]}\n return\n self.source = {}\n self.all_sources = []\n\n def _resolve_agent_id(self) -> Optional[str]:\n \"\"\"Resolve agent_id from request, then fall back to conversation context.\"\"\"\n request_agent_id = self.data.get(\"agent_id\")\n if request_agent_id:\n return str(request_agent_id)\n\n if not self.conversation_id or not self.initial_user_id:\n return None\n\n try:\n conversation = self.conversation_service.get_conversation(\n self.conversation_id, self.initial_user_id\n )\n except Exception:\n return None\n\n if not conversation:\n return None\n\n conversation_agent_id = conversation.get(\"agent_id\")\n if conversation_agent_id:\n return str(conversation_agent_id)\n\n return None\n\n def _configure_agent(self):\n \"\"\"Configure the agent based on request data\"\"\"\n agent_id = self._resolve_agent_id()\n\n self.agent_key, self.is_shared_usage, self.shared_token = self._get_agent_key(\n agent_id, self.initial_user_id\n )\n self.agent_id = str(agent_id) if agent_id else None\n\n api_key = self.data.get(\"api_key\")\n if api_key:\n data_key = self._get_data_from_api_key(api_key)\n if data_key.get(\"_id\"):\n self.agent_id = str(data_key.get(\"_id\"))\n self.agent_config.update(\n {\n \"prompt_id\": data_key.get(\"prompt_id\", \"default\"),\n \"agent_type\": data_key.get(\"agent_type\", settings.AGENT_NAME),\n \"user_api_key\": api_key,\n \"json_schema\": data_key.get(\"json_schema\"),\n \"default_model_id\": data_key.get(\"default_model_id\", \"\"),\n }\n )\n self.initial_user_id = data_key.get(\"user\")\n self.decoded_token = {\"sub\": data_key.get(\"user\")}\n if data_key.get(\"source\"):\n self.source = {\"active_docs\": data_key[\"source\"]}\n if data_key.get(\"workflow\"):\n self.agent_config[\"workflow\"] = data_key[\"workflow\"]\n self.agent_config[\"workflow_owner\"] = data_key.get(\"user\")\n if data_key.get(\"retriever\"):\n self.retriever_config[\"retriever_name\"] = data_key[\"retriever\"]\n if data_key.get(\"chunks\") is not None:\n try:\n self.retriever_config[\"chunks\"] = int(data_key[\"chunks\"])\n except (ValueError, TypeError):\n logger.warning(\n f\"Invalid chunks value: {data_key['chunks']}, using default value 2\"\n )\n self.retriever_config[\"chunks\"] = 2\n elif self.agent_key:\n data_key = self._get_data_from_api_key(self.agent_key)\n if data_key.get(\"_id\"):\n self.agent_id = str(data_key.get(\"_id\"))\n self.agent_config.update(\n {\n \"prompt_id\": data_key.get(\"prompt_id\", \"default\"),\n \"agent_type\": data_key.get(\"agent_type\", settings.AGENT_NAME),\n \"user_api_key\": self.agent_key,\n \"json_schema\": data_key.get(\"json_schema\"),\n \"default_model_id\": data_key.get(\"default_model_id\", \"\"),\n }\n )\n self.decoded_token = (\n self.decoded_token\n if self.is_shared_usage\n else {\"sub\": data_key.get(\"user\")}\n )\n if data_key.get(\"source\"):\n self.source = {\"active_docs\": data_key[\"source\"]}\n if data_key.get(\"workflow\"):\n self.agent_config[\"workflow\"] = data_key[\"workflow\"]\n self.agent_config[\"workflow_owner\"] = data_key.get(\"user\")\n if data_key.get(\"retriever\"):\n self.retriever_config[\"retriever_name\"] = data_key[\"retriever\"]\n if data_key.get(\"chunks\") is not None:\n try:\n self.retriever_config[\"chunks\"] = int(data_key[\"chunks\"])\n except (ValueError, TypeError):\n logger.warning(\n f\"Invalid chunks value: {data_key['chunks']}, using default value 2\"\n )\n self.retriever_config[\"chunks\"] = 2\n else:\n agent_type = settings.AGENT_NAME\n if self.data.get(\"workflow\") and isinstance(\n self.data.get(\"workflow\"), dict\n ):\n agent_type = \"workflow\"\n self.agent_config[\"workflow\"] = self.data[\"workflow\"]\n if isinstance(self.decoded_token, dict):\n self.agent_config[\"workflow_owner\"] = self.decoded_token.get(\"sub\")\n\n self.agent_config.update(\n {\n \"prompt_id\": self.data.get(\"prompt_id\", \"default\"),\n \"agent_type\": agent_type,\n \"user_api_key\": None,\n \"json_schema\": None,\n \"default_model_id\": \"\",\n }\n )\n\n def _configure_retriever(self):\n doc_token_limit = calculate_doc_token_budget(model_id=self.model_id)\n\n self.retriever_config = {\n \"retriever_name\": self.data.get(\"retriever\", \"classic\"),\n \"chunks\": int(self.data.get(\"chunks\", 2)),\n \"doc_token_limit\": doc_token_limit,\n }\n\n api_key = self.data.get(\"api_key\") or self.agent_key\n if not api_key and \"isNoneDoc\" in self.data and self.data[\"isNoneDoc\"]:\n self.retriever_config[\"chunks\"] = 0\n\n def create_retriever(self):\n return RetrieverCreator.create_retriever(\n self.retriever_config[\"retriever_name\"],\n source=self.source,\n chat_history=self.history,\n prompt=get_prompt(self.agent_config[\"prompt_id\"], self.prompts_collection),\n chunks=self.retriever_config[\"chunks\"],\n doc_token_limit=self.retriever_config.get(\"doc_token_limit\", 50000),\n model_id=self.model_id,\n user_api_key=self.agent_config[\"user_api_key\"],\n agent_id=self.agent_id,\n decoded_token=self.decoded_token,\n )\n\n def pre_fetch_docs(self, question: str) -> tuple[Optional[str], Optional[list]]:\n \"\"\"Pre-fetch documents for template rendering before agent creation\"\"\"\n if self.data.get(\"isNoneDoc\", False) and not self.agent_id:\n logger.info(\"Pre-fetch skipped: isNoneDoc=True\")\n return None, None\n try:\n retriever = self.create_retriever()\n logger.info(\n f\"Pre-fetching docs with chunks={retriever.chunks}, doc_token_limit={retriever.doc_token_limit}\"\n )\n docs = retriever.search(question)\n logger.info(f\"Pre-fetch retrieved {len(docs) if docs else 0} documents\")\n\n if not docs:\n logger.info(\"Pre-fetch: No documents returned from search\")\n return None, None\n self.retrieved_docs = docs\n\n docs_with_filenames = []\n for doc in docs:\n filename = doc.get(\"filename\") or doc.get(\"title\") or doc.get(\"source\")\n if filename:\n chunk_header = str(filename)\n docs_with_filenames.append(f\"{chunk_header}\\n{doc['text']}\")\n else:\n docs_with_filenames.append(doc[\"text\"])\n docs_together = \"\\n\\n\".join(docs_with_filenames)\n\n logger.info(f\"Pre-fetch docs_together size: {len(docs_together)} chars\")\n\n return docs_together, docs\n except Exception as e:\n logger.error(f\"Failed to pre-fetch docs: {str(e)}\", exc_info=True)\n return None, None\n\n def pre_fetch_tools(self) -> Optional[Dict[str, Any]]:\n \"\"\"Pre-fetch tool data for template rendering before agent creation\n\n Can be controlled via:\n 1. Global setting: ENABLE_TOOL_PREFETCH in .env\n 2. Per-request: disable_tool_prefetch in request data\n \"\"\"\n if not settings.ENABLE_TOOL_PREFETCH:\n logger.info(\n \"Tool pre-fetching disabled globally via ENABLE_TOOL_PREFETCH setting\"\n )\n return None\n\n if self.data.get(\"disable_tool_prefetch\", False):\n logger.info(\"Tool pre-fetching disabled for this request\")\n return None\n\n required_tool_actions = self._get_required_tool_actions()\n filtering_enabled = required_tool_actions is not None\n\n try:\n user_tools_collection = self.db[\"user_tools\"]\n user_id = self.initial_user_id or \"local\"\n\n user_tools = list(\n user_tools_collection.find({\"user\": user_id, \"status\": True})\n )\n\n if not user_tools:\n return None\n\n tools_data = {}\n\n for tool_doc in user_tools:\n tool_name = tool_doc.get(\"name\")\n tool_id = str(tool_doc.get(\"_id\"))\n\n if filtering_enabled:\n required_actions_by_name = required_tool_actions.get(\n tool_name, set()\n )\n required_actions_by_id = required_tool_actions.get(tool_id, set())\n\n required_actions = required_actions_by_name | required_actions_by_id\n\n if not required_actions:\n continue\n else:\n required_actions = None\n\n tool_data = self._fetch_tool_data(tool_doc, required_actions)\n if tool_data:\n tools_data[tool_name] = tool_data\n tools_data[tool_id] = tool_data\n\n return tools_data if tools_data else None\n except Exception as e:\n logger.warning(f\"Failed to pre-fetch tools: {type(e).__name__}\")\n return None\n\n def _fetch_tool_data(\n self,\n tool_doc: Dict[str, Any],\n required_actions: Optional[Set[Optional[str]]],\n ) -> Optional[Dict[str, Any]]:\n \"\"\"Fetch and execute tool actions with saved parameters\"\"\"\n try:\n from application.agents.tools.tool_manager import ToolManager\n\n tool_name = tool_doc.get(\"name\")\n tool_config = tool_doc.get(\"config\", {}).copy()\n tool_config[\"tool_id\"] = str(tool_doc[\"_id\"])\n\n tool_manager = ToolManager(config={tool_name: tool_config})\n user_id = self.initial_user_id or \"local\"\n tool = tool_manager.load_tool(tool_name, tool_config, user_id=user_id)\n\n if not tool:\n logger.debug(f\"Tool '{tool_name}' failed to load\")\n return None\n\n tool_actions = tool.get_actions_metadata()\n if not tool_actions:\n logger.debug(f\"Tool '{tool_name}' has no actions\")\n return None\n\n saved_actions = tool_doc.get(\"actions\", [])\n\n include_all_actions = required_actions is None or (\n required_actions and None in required_actions\n )\n allowed_actions: Set[str] = (\n {action for action in required_actions if isinstance(action, str)}\n if required_actions\n else set()\n )\n\n action_results = {}\n for action_meta in tool_actions:\n action_name = action_meta.get(\"name\")\n if action_name is None:\n continue\n if (\n not include_all_actions\n and allowed_actions\n and action_name not in allowed_actions\n ):\n continue\n\n try:\n saved_action = None\n for sa in saved_actions:\n if sa.get(\"name\") == action_name:\n saved_action = sa\n break\n\n action_params = action_meta.get(\"parameters\", {})\n properties = action_params.get(\"properties\", {})\n\n kwargs = {}\n for param_name, param_spec in properties.items():\n if saved_action:\n saved_props = saved_action.get(\"parameters\", {}).get(\n \"properties\", {}\n )\n if param_name in saved_props:\n param_value = saved_props[param_name].get(\"value\")\n if param_value is not None:\n kwargs[param_name] = param_value\n continue\n\n if param_name in tool_config:\n kwargs[param_name] = tool_config[param_name]\n elif \"default\" in param_spec:\n kwargs[param_name] = param_spec[\"default\"]\n\n result = tool.execute_action(action_name, **kwargs)\n action_results[action_name] = result\n except Exception as e:\n logger.debug(\n f\"Action '{action_name}' execution failed: {type(e).__name__}\"\n )\n continue\n\n return action_results if action_results else None\n\n except Exception as e:\n logger.debug(f\"Tool pre-fetch failed for '{tool_name}': {type(e).__name__}\")\n return None\n\n def _get_prompt_content(self) -> Optional[str]:\n \"\"\"Retrieve and cache the raw prompt content for the current agent configuration.\"\"\"\n if self._prompt_content is not None:\n return self._prompt_content\n prompt_id = (\n self.agent_config.get(\"prompt_id\")\n if isinstance(self.agent_config, dict)\n else None\n )\n if not prompt_id:\n return None\n try:\n self._prompt_content = get_prompt(prompt_id, self.prompts_collection)\n except ValueError as e:\n logger.debug(f\"Invalid prompt ID '{prompt_id}': {str(e)}\")\n self._prompt_content = None\n except Exception as e:\n logger.debug(f\"Failed to fetch prompt '{prompt_id}': {type(e).__name__}\")\n self._prompt_content = None\n return self._prompt_content\n\n def _get_required_tool_actions(self) -> Optional[Dict[str, Set[Optional[str]]]]:\n \"\"\"Determine which tool actions are referenced in the prompt template\"\"\"\n if self._required_tool_actions is not None:\n return self._required_tool_actions\n\n prompt_content = self._get_prompt_content()\n if prompt_content is None:\n return None\n\n if \"{{\" not in prompt_content or \"}}\" not in prompt_content:\n self._required_tool_actions = {}\n return self._required_tool_actions\n\n try:\n from application.templates.template_engine import TemplateEngine\n\n template_engine = TemplateEngine()\n usages = template_engine.extract_tool_usages(prompt_content)\n self._required_tool_actions = usages\n return self._required_tool_actions\n except Exception as e:\n logger.debug(f\"Failed to extract tool usages: {type(e).__name__}\")\n self._required_tool_actions = {}\n return self._required_tool_actions\n\n def _fetch_memory_tool_data(\n self, tool_doc: Dict[str, Any]\n ) -> Optional[Dict[str, Any]]:\n \"\"\"Fetch memory tool data for pre-injection into prompt\"\"\"\n try:\n tool_config = tool_doc.get(\"config\", {}).copy()\n tool_config[\"tool_id\"] = str(tool_doc[\"_id\"])\n\n from application.agents.tools.memory import MemoryTool\n\n memory_tool = MemoryTool(tool_config, self.initial_user_id)\n\n root_view = memory_tool.execute_action(\"view\", path=\"/\")\n\n if \"Error:\" in root_view or not root_view.strip():\n return None\n\n return {\"root\": root_view, \"available\": True}\n except Exception as e:\n logger.warning(f\"Failed to fetch memory tool data: {str(e)}\")\n return None\n\n def create_agent(\n self,\n docs_together: Optional[str] = None,\n docs: Optional[list] = None,\n tools_data: Optional[Dict[str, Any]] = None,\n ):\n \"\"\"Create and return the configured agent with rendered prompt\"\"\"\n raw_prompt = self._get_prompt_content()\n if raw_prompt is None:\n raw_prompt = get_prompt(\n self.agent_config[\"prompt_id\"], self.prompts_collection\n )\n self._prompt_content = raw_prompt\n\n rendered_prompt = self.prompt_renderer.render_prompt(\n prompt_content=raw_prompt,\n user_id=self.initial_user_id,\n request_id=self.data.get(\"request_id\"),\n passthrough_data=self.data.get(\"passthrough\"),\n docs=docs,\n docs_together=docs_together,\n tools_data=tools_data,\n )\n\n provider = (\n get_provider_from_model_id(self.model_id)\n if self.model_id\n else settings.LLM_PROVIDER\n )\n system_api_key = get_api_key_for_provider(provider or settings.LLM_PROVIDER)\n\n agent_type = self.agent_config[\"agent_type\"]\n\n # Base agent kwargs\n agent_kwargs = {\n \"endpoint\": \"stream\",\n \"llm_name\": provider or settings.LLM_PROVIDER,\n \"model_id\": self.model_id,\n \"api_key\": system_api_key,\n \"agent_id\": self.agent_id,\n \"user_api_key\": self.agent_config[\"user_api_key\"],\n \"prompt\": rendered_prompt,\n \"chat_history\": self.history,\n \"retrieved_docs\": self.retrieved_docs,\n \"decoded_token\": self.decoded_token,\n \"attachments\": self.attachments,\n \"json_schema\": self.agent_config.get(\"json_schema\"),\n \"compressed_summary\": self.compressed_summary,\n }\n\n # Workflow-specific kwargs for workflow agents\n if agent_type == \"workflow\":\n workflow_config = self.agent_config.get(\"workflow\")\n if isinstance(workflow_config, str):\n agent_kwargs[\"workflow_id\"] = workflow_config\n elif isinstance(workflow_config, dict):\n agent_kwargs[\"workflow\"] = workflow_config\n workflow_owner = self.agent_config.get(\"workflow_owner\")\n if workflow_owner:\n agent_kwargs[\"workflow_owner\"] = workflow_owner\n\n agent = AgentCreator.create_agent(agent_type, **agent_kwargs)\n\n agent.conversation_id = self.conversation_id\n agent.initial_user_id = self.initial_user_id\n\n return agent\n" }, { "path": "application/api/connector/routes.py", "content": "import base64\nimport datetime\nimport html\nimport json\nimport uuid\nfrom urllib.parse import urlencode\n\n\nfrom bson.objectid import ObjectId\nfrom flask import (\n Blueprint,\n current_app,\n jsonify,\n make_response,\n request\n)\nfrom flask_restx import fields, Namespace, Resource\n\n\nfrom application.api.user.tasks import (\n ingest_connector_task,\n)\nfrom application.core.mongo_db import MongoDB\nfrom application.core.settings import settings\nfrom application.api import api\n\n\nfrom application.parser.connectors.connector_creator import ConnectorCreator\n\n\nmongo = MongoDB.get_client()\ndb = mongo[settings.MONGO_DB_NAME]\nsources_collection = db[\"sources\"]\nsessions_collection = db[\"connector_sessions\"]\n\nconnector = Blueprint(\"connector\", __name__)\nconnectors_ns = Namespace(\"connectors\", description=\"Connector operations\", path=\"/\")\napi.add_namespace(connectors_ns)\n\n# Fixed callback status path to prevent open redirect\nCALLBACK_STATUS_PATH = \"/api/connectors/callback-status\"\n\n\ndef build_callback_redirect(params: dict) -> str:\n \"\"\"Build a safe redirect URL to the callback status page.\n\n Uses a fixed path and properly URL-encodes all parameters\n to prevent URL injection and open redirect vulnerabilities.\n \"\"\"\n return f\"{CALLBACK_STATUS_PATH}?{urlencode(params)}\"\n\n\n\n@connectors_ns.route(\"/api/connectors/auth\")\nclass ConnectorAuth(Resource):\n @api.doc(description=\"Get connector OAuth authorization URL\", params={\"provider\": \"Connector provider (e.g., google_drive)\"})\n def get(self):\n try:\n provider = request.args.get('provider') or request.args.get('source')\n if not provider:\n return make_response(jsonify({\"success\": False, \"error\": \"Missing provider\"}), 400)\n\n if not ConnectorCreator.is_supported(provider):\n return make_response(jsonify({\"success\": False, \"error\": f\"Unsupported provider: {provider}\"}), 400)\n\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False, \"error\": \"Unauthorized\"}), 401)\n user_id = decoded_token.get('sub')\n\n now = datetime.datetime.now(datetime.timezone.utc)\n result = sessions_collection.insert_one({\n \"provider\": provider,\n \"user\": user_id,\n \"status\": \"pending\",\n \"created_at\": now\n })\n state_dict = {\n \"provider\": provider,\n \"object_id\": str(result.inserted_id)\n }\n state = base64.urlsafe_b64encode(json.dumps(state_dict).encode()).decode()\n\n auth = ConnectorCreator.create_auth(provider)\n authorization_url = auth.get_authorization_url(state=state)\n return make_response(jsonify({\n \"success\": True,\n \"authorization_url\": authorization_url,\n \"state\": state\n }), 200)\n except Exception as e:\n current_app.logger.error(f\"Error generating connector auth URL: {e}\", exc_info=True)\n return make_response(jsonify({\"success\": False, \"error\": \"Failed to generate authorization URL\"}), 500)\n\n\n@connectors_ns.route(\"/api/connectors/callback\")\nclass ConnectorsCallback(Resource):\n @api.doc(description=\"Handle OAuth callback for external connectors\")\n def get(self):\n \"\"\"Handle OAuth callback for external connectors\"\"\"\n try:\n from application.parser.connectors.connector_creator import ConnectorCreator\n from flask import request, redirect\n\n authorization_code = request.args.get('code')\n state = request.args.get('state')\n error = request.args.get('error')\n\n state_dict = json.loads(base64.urlsafe_b64decode(state.encode()).decode())\n provider = state_dict.get(\"provider\")\n state_object_id = state_dict.get(\"object_id\")\n\n # Validate provider\n if not provider or not isinstance(provider, str) or not ConnectorCreator.is_supported(provider):\n return redirect(build_callback_redirect({\n \"status\": \"error\",\n \"message\": \"Invalid provider\"\n }))\n\n if error:\n if error == \"access_denied\":\n return redirect(build_callback_redirect({\n \"status\": \"cancelled\",\n \"message\": \"Authentication was cancelled. You can try again if you'd like to connect your account.\",\n \"provider\": provider\n }))\n else:\n current_app.logger.warning(f\"OAuth error in callback: {error}\")\n return redirect(build_callback_redirect({\n \"status\": \"error\",\n \"message\": \"Authentication failed. Please try again and make sure to grant all requested permissions.\",\n \"provider\": provider\n }))\n\n if not authorization_code:\n return redirect(build_callback_redirect({\n \"status\": \"error\",\n \"message\": \"Authentication failed. Please try again and make sure to grant all requested permissions.\",\n \"provider\": provider\n }))\n\n try:\n auth = ConnectorCreator.create_auth(provider)\n token_info = auth.exchange_code_for_tokens(authorization_code)\n\n session_token = str(uuid.uuid4())\n\n try:\n if provider == \"google_drive\":\n credentials = auth.create_credentials_from_token_info(token_info)\n service = auth.build_drive_service(credentials)\n user_info = service.about().get(fields=\"user\").execute()\n user_email = user_info.get('user', {}).get('emailAddress', 'Connected User')\n else:\n user_email = token_info.get('user_info', {}).get('email', 'Connected User')\n\n except Exception as e:\n current_app.logger.warning(f\"Could not get user info: {e}\")\n user_email = 'Connected User'\n\n sanitized_token_info = auth.sanitize_token_info(token_info)\n\n sessions_collection.find_one_and_update(\n {\"_id\": ObjectId(state_object_id), \"provider\": provider},\n {\n \"$set\": {\n \"session_token\": session_token,\n \"token_info\": sanitized_token_info,\n \"user_email\": user_email,\n \"status\": \"authorized\"\n }\n }\n )\n\n # Redirect to success page with session token and user email\n return redirect(build_callback_redirect({\n \"status\": \"success\",\n \"message\": \"Authentication successful\",\n \"provider\": provider,\n \"session_token\": session_token,\n \"user_email\": user_email\n }))\n\n except Exception as e:\n current_app.logger.error(f\"Error exchanging code for tokens: {str(e)}\", exc_info=True)\n return redirect(build_callback_redirect({\n \"status\": \"error\",\n \"message\": \"Authentication failed. Please try again and make sure to grant all requested permissions.\",\n \"provider\": provider\n }))\n\n except Exception as e:\n current_app.logger.error(f\"Error handling connector callback: {e}\")\n return redirect(build_callback_redirect({\n \"status\": \"error\",\n \"message\": \"Authentication failed. Please try again and make sure to grant all requested permissions.\"\n }))\n\n\n@connectors_ns.route(\"/api/connectors/files\")\nclass ConnectorFiles(Resource):\n @api.expect(api.model(\"ConnectorFilesModel\", {\n \"provider\": fields.String(required=True),\n \"session_token\": fields.String(required=True),\n \"folder_id\": fields.String(required=False),\n \"limit\": fields.Integer(required=False),\n \"page_token\": fields.String(required=False),\n \"search_query\": fields.String(required=False),\n }))\n @api.doc(description=\"List files from a connector provider (supports pagination and search)\")\n def post(self):\n try:\n data = request.get_json()\n provider = data.get('provider')\n session_token = data.get('session_token')\n limit = data.get('limit', 10)\n\n if not provider or not session_token:\n return make_response(jsonify({\"success\": False, \"error\": \"provider and session_token are required\"}), 400)\n\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False, \"error\": \"Unauthorized\"}), 401)\n user = decoded_token.get('sub')\n session = sessions_collection.find_one({\"session_token\": session_token, \"user\": user})\n if not session:\n return make_response(jsonify({\"success\": False, \"error\": \"Invalid or unauthorized session\"}), 401)\n\n loader = ConnectorCreator.create_connector(provider, session_token)\n\n generic_keys = {'provider', 'session_token'}\n input_config = {\n k: v for k, v in data.items() if k not in generic_keys\n }\n input_config['list_only'] = True\n \n documents = loader.load_data(input_config)\n\n files = []\n for doc in documents[:limit]:\n metadata = doc.extra_info\n modified_time = metadata.get('modified_time')\n if modified_time:\n date_part = modified_time.split('T')[0]\n time_part = modified_time.split('T')[1].split('.')[0].split('Z')[0]\n formatted_time = f\"{date_part} {time_part}\"\n else:\n formatted_time = None\n\n files.append({\n 'id': doc.doc_id,\n 'name': metadata.get('file_name', 'Unknown File'),\n 'type': metadata.get('mime_type', 'unknown'),\n 'size': metadata.get('size', None),\n 'modifiedTime': formatted_time,\n 'isFolder': metadata.get('is_folder', False)\n })\n\n next_token = getattr(loader, 'next_page_token', None)\n has_more = bool(next_token)\n\n return make_response(jsonify({\n \"success\": True, \n \"files\": files, \n \"total\": len(files), \n \"next_page_token\": next_token, \n \"has_more\": has_more\n }), 200)\n except Exception as e:\n current_app.logger.error(f\"Error loading connector files: {e}\", exc_info=True)\n return make_response(jsonify({\"success\": False, \"error\": \"Failed to load files\"}), 500)\n\n\n@connectors_ns.route(\"/api/connectors/validate-session\")\nclass ConnectorValidateSession(Resource):\n @api.expect(api.model(\"ConnectorValidateSessionModel\", {\"provider\": fields.String(required=True), \"session_token\": fields.String(required=True)}))\n @api.doc(description=\"Validate connector session token and return user info and access token\")\n def post(self):\n try:\n data = request.get_json()\n provider = data.get('provider')\n session_token = data.get('session_token')\n if not provider or not session_token:\n return make_response(jsonify({\"success\": False, \"error\": \"provider and session_token are required\"}), 400)\n\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False, \"error\": \"Unauthorized\"}), 401)\n user = decoded_token.get('sub')\n\n session = sessions_collection.find_one({\"session_token\": session_token, \"user\": user})\n if not session or \"token_info\" not in session:\n return make_response(jsonify({\"success\": False, \"error\": \"Invalid or expired session\"}), 401)\n\n token_info = session[\"token_info\"]\n auth = ConnectorCreator.create_auth(provider)\n is_expired = auth.is_token_expired(token_info)\n\n if is_expired and token_info.get('refresh_token'):\n try:\n refreshed_token_info = auth.refresh_access_token(token_info.get('refresh_token'))\n sanitized_token_info = auth.sanitize_token_info(refreshed_token_info)\n sessions_collection.update_one(\n {\"session_token\": session_token},\n {\"$set\": {\"token_info\": sanitized_token_info}}\n )\n token_info = sanitized_token_info\n is_expired = False\n except Exception as refresh_error:\n current_app.logger.error(f\"Failed to refresh token: {refresh_error}\")\n \n if is_expired:\n return make_response(jsonify({\n \"success\": False,\n \"expired\": True,\n \"error\": \"Session token has expired. Please reconnect.\"\n }), 401)\n\n _base_fields = {\"access_token\", \"refresh_token\", \"token_uri\", \"expiry\"}\n provider_extras = {k: v for k, v in token_info.items() if k not in _base_fields}\n\n response_data = {\n \"success\": True,\n \"expired\": False,\n \"user_email\": session.get('user_email', 'Connected User'),\n \"access_token\": token_info.get('access_token'),\n **provider_extras,\n }\n\n return make_response(jsonify(response_data), 200)\n except Exception as e:\n current_app.logger.error(f\"Error validating connector session: {e}\", exc_info=True)\n return make_response(jsonify({\"success\": False, \"error\": \"Failed to validate session\"}), 500)\n\n\n@connectors_ns.route(\"/api/connectors/disconnect\")\nclass ConnectorDisconnect(Resource):\n @api.expect(api.model(\"ConnectorDisconnectModel\", {\"provider\": fields.String(required=True), \"session_token\": fields.String(required=False)}))\n @api.doc(description=\"Disconnect a connector session\")\n def post(self):\n try:\n data = request.get_json()\n provider = data.get('provider')\n session_token = data.get('session_token')\n if not provider:\n return make_response(jsonify({\"success\": False, \"error\": \"provider is required\"}), 400)\n\n\n if session_token:\n sessions_collection.delete_one({\"session_token\": session_token})\n \n return make_response(jsonify({\"success\": True}), 200)\n except Exception as e:\n current_app.logger.error(f\"Error disconnecting connector session: {e}\", exc_info=True)\n return make_response(jsonify({\"success\": False, \"error\": \"Failed to disconnect session\"}), 500)\n\n\n@connectors_ns.route(\"/api/connectors/sync\")\nclass ConnectorSync(Resource):\n @api.expect(\n api.model(\n \"ConnectorSyncModel\",\n {\n \"source_id\": fields.String(required=True, description=\"Source ID to sync\"),\n \"session_token\": fields.String(required=True, description=\"Authentication token\")\n },\n )\n )\n @api.doc(description=\"Sync connector source to check for modifications\")\n def post(self):\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False}), 401)\n\n try:\n data = request.get_json()\n source_id = data.get('source_id')\n session_token = data.get('session_token')\n\n if not all([source_id, session_token]):\n return make_response(\n jsonify({\n \"success\": False,\n \"error\": \"source_id and session_token are required\"\n }), \n 400\n )\n source = sources_collection.find_one({\"_id\": ObjectId(source_id)})\n if not source:\n return make_response(\n jsonify({\n \"success\": False,\n \"error\": \"Source not found\"\n }), \n 404\n )\n\n if source.get('user') != decoded_token.get('sub'):\n return make_response(\n jsonify({\n \"success\": False,\n \"error\": \"Unauthorized access to source\"\n }), \n 403\n )\n\n remote_data = {}\n try:\n if source.get('remote_data'):\n remote_data = json.loads(source.get('remote_data'))\n except json.JSONDecodeError:\n current_app.logger.error(f\"Invalid remote_data format for source {source_id}\")\n remote_data = {}\n\n source_type = remote_data.get('provider')\n if not source_type:\n return make_response(\n jsonify({\n \"success\": False,\n \"error\": \"Source provider not found in remote_data\"\n }), \n 400\n )\n\n # Extract configuration from remote_data\n file_ids = remote_data.get('file_ids', [])\n folder_ids = remote_data.get('folder_ids', [])\n recursive = remote_data.get('recursive', True)\n\n # Start the sync task\n task = ingest_connector_task.delay(\n job_name=source.get('name'),\n user=decoded_token.get('sub'),\n source_type=source_type,\n session_token=session_token,\n file_ids=file_ids,\n folder_ids=folder_ids,\n recursive=recursive,\n retriever=source.get('retriever', 'classic'),\n operation_mode=\"sync\",\n doc_id=source_id,\n sync_frequency=source.get('sync_frequency', 'never')\n )\n\n return make_response(\n jsonify({\n \"success\": True,\n \"task_id\": task.id\n }), \n 200\n )\n\n except Exception as err:\n current_app.logger.error(\n f\"Error syncing connector source: {err}\",\n exc_info=True\n )\n return make_response(\n jsonify({\n \"success\": False,\n \"error\": \"Failed to sync connector source\"\n }),\n 400\n )\n\n\n@connectors_ns.route(\"/api/connectors/callback-status\")\nclass ConnectorCallbackStatus(Resource):\n @api.doc(description=\"Return HTML page with connector authentication status\")\n def get(self):\n \"\"\"Return HTML page with connector authentication status\"\"\"\n try:\n # Validate and sanitize status to a known value\n status_raw = request.args.get('status', 'error')\n status = status_raw if status_raw in ('success', 'error', 'cancelled') else 'error'\n\n # Escape all user-controlled values for HTML context\n message = html.escape(request.args.get('message', ''))\n provider_raw = request.args.get('provider', 'connector')\n provider = html.escape(provider_raw.replace('_', ' ').title())\n session_token = request.args.get('session_token', '')\n user_email = html.escape(request.args.get('user_email', ''))\n\n def safe_js_string(value: str) -> str:\n \"\"\"Safely encode a string for embedding in inline JavaScript.\"\"\"\n js_encoded = json.dumps(value)\n return js_encoded.replace('\n \n \n {provider} Authentication\n \n \n \n \n
\n

{provider} Authentication

\n
\n

{message}

\n {f'

Connected as: {user_email}

' if status == 'success' else ''}\n
\n

You can close this window. {f\"Your {provider} is now connected and ready to use.\" if status == 'success' else \"Feel free to close this window.\"}

\n
\n \n \n \"\"\"\n\n return make_response(html_content, 200, {'Content-Type': 'text/html'})\n except Exception as e:\n current_app.logger.error(f\"Error rendering callback status page: {e}\")\n return make_response(\"Authentication error occurred\", 500, {'Content-Type': 'text/html'})\n\n\n" }, { "path": "application/api/internal/__init__.py", "content": "" }, { "path": "application/api/internal/routes.py", "content": "import os\nimport datetime\nimport json\nfrom flask import Blueprint, request, send_from_directory, jsonify\nfrom werkzeug.utils import secure_filename\nfrom bson.objectid import ObjectId\nimport logging\nfrom application.core.mongo_db import MongoDB\nfrom application.core.settings import settings\nfrom application.storage.storage_creator import StorageCreator\n\n\nlogger = logging.getLogger(__name__)\nmongo = MongoDB.get_client()\ndb = mongo[settings.MONGO_DB_NAME]\nconversations_collection = db[\"conversations\"]\nsources_collection = db[\"sources\"]\n\ncurrent_dir = os.path.dirname(\n os.path.dirname(os.path.dirname(os.path.abspath(__file__)))\n)\n\n\ninternal = Blueprint(\"internal\", __name__)\n\n\n@internal.before_request\ndef verify_internal_key():\n \"\"\"Verify INTERNAL_KEY for all internal endpoint requests.\"\"\"\n if settings.INTERNAL_KEY:\n internal_key = request.headers.get(\"X-Internal-Key\")\n if not internal_key or internal_key != settings.INTERNAL_KEY:\n logger.warning(f\"Unauthorized internal API access attempt from {request.remote_addr}\")\n return jsonify({\"error\": \"Unauthorized\", \"message\": \"Invalid or missing internal key\"}), 401\n\n\n@internal.route(\"/api/download\", methods=[\"get\"])\ndef download_file():\n user = secure_filename(request.args.get(\"user\"))\n job_name = secure_filename(request.args.get(\"name\"))\n filename = secure_filename(request.args.get(\"file\"))\n save_dir = os.path.join(current_dir, settings.UPLOAD_FOLDER, user, job_name)\n return send_from_directory(save_dir, filename, as_attachment=True)\n\n\n@internal.route(\"/api/upload_index\", methods=[\"POST\"])\ndef upload_index_files():\n \"\"\"Upload two files(index.faiss, index.pkl) to the user's folder.\"\"\"\n if \"user\" not in request.form:\n return {\"status\": \"no user\"}\n user = request.form[\"user\"] \n if \"name\" not in request.form:\n return {\"status\": \"no name\"}\n job_name = request.form[\"name\"]\n tokens = request.form[\"tokens\"]\n retriever = request.form[\"retriever\"]\n id = request.form[\"id\"]\n type = request.form[\"type\"]\n remote_data = request.form[\"remote_data\"] if \"remote_data\" in request.form else None\n sync_frequency = request.form[\"sync_frequency\"] if \"sync_frequency\" in request.form else None\n \n file_path = request.form.get(\"file_path\")\n directory_structure = request.form.get(\"directory_structure\")\n file_name_map = request.form.get(\"file_name_map\")\n \n if directory_structure:\n try:\n directory_structure = json.loads(directory_structure)\n except Exception:\n logger.error(\"Error parsing directory_structure\")\n directory_structure = {}\n else:\n directory_structure = {}\n if file_name_map:\n try:\n file_name_map = json.loads(file_name_map)\n except Exception:\n logger.error(\"Error parsing file_name_map\")\n file_name_map = None\n else:\n file_name_map = None\n\n storage = StorageCreator.get_storage()\n index_base_path = f\"indexes/{id}\"\n \n if settings.VECTOR_STORE == \"faiss\":\n if \"file_faiss\" not in request.files:\n logger.error(\"No file_faiss part\")\n return {\"status\": \"no file\"}\n file_faiss = request.files[\"file_faiss\"]\n if file_faiss.filename == \"\":\n return {\"status\": \"no file name\"}\n if \"file_pkl\" not in request.files:\n logger.error(\"No file_pkl part\")\n return {\"status\": \"no file\"}\n file_pkl = request.files[\"file_pkl\"]\n if file_pkl.filename == \"\":\n return {\"status\": \"no file name\"}\n\n # Save index files to storage\n faiss_storage_path = f\"{index_base_path}/index.faiss\"\n pkl_storage_path = f\"{index_base_path}/index.pkl\"\n storage.save_file(file_faiss, faiss_storage_path)\n storage.save_file(file_pkl, pkl_storage_path)\n\n\n existing_entry = sources_collection.find_one({\"_id\": ObjectId(id)})\n if existing_entry:\n update_fields = {\n \"user\": user,\n \"name\": job_name,\n \"language\": job_name,\n \"date\": datetime.datetime.now(),\n \"model\": settings.EMBEDDINGS_NAME,\n \"type\": type,\n \"tokens\": tokens,\n \"retriever\": retriever,\n \"remote_data\": remote_data,\n \"sync_frequency\": sync_frequency,\n \"file_path\": file_path,\n \"directory_structure\": directory_structure,\n }\n if file_name_map is not None:\n update_fields[\"file_name_map\"] = file_name_map\n sources_collection.update_one(\n {\"_id\": ObjectId(id)},\n {\"$set\": update_fields},\n )\n else:\n insert_doc = {\n \"_id\": ObjectId(id),\n \"user\": user,\n \"name\": job_name,\n \"language\": job_name,\n \"date\": datetime.datetime.now(),\n \"model\": settings.EMBEDDINGS_NAME,\n \"type\": type,\n \"tokens\": tokens,\n \"retriever\": retriever,\n \"remote_data\": remote_data,\n \"sync_frequency\": sync_frequency,\n \"file_path\": file_path,\n \"directory_structure\": directory_structure,\n }\n if file_name_map is not None:\n insert_doc[\"file_name_map\"] = file_name_map\n sources_collection.insert_one(insert_doc)\n return {\"status\": \"ok\"}\n" }, { "path": "application/api/user/__init__.py", "content": "\"\"\"User API module - provides all user-related API endpoints\"\"\"\n\nfrom .routes import user\n\n__all__ = [\"user\"]\n" }, { "path": "application/api/user/agents/__init__.py", "content": "\"\"\"Agents module.\"\"\"\n\nfrom .routes import agents_ns\nfrom .sharing import agents_sharing_ns\nfrom .webhooks import agents_webhooks_ns\nfrom .folders import agents_folders_ns\n\n__all__ = [\"agents_ns\", \"agents_sharing_ns\", \"agents_webhooks_ns\", \"agents_folders_ns\"]\n" }, { "path": "application/api/user/agents/folders.py", "content": "\"\"\"\nAgent folders management routes.\nProvides virtual folder organization for agents (Google Drive-like structure).\n\"\"\"\n\nimport datetime\nfrom bson.objectid import ObjectId\nfrom flask import current_app, jsonify, make_response, request\nfrom flask_restx import Namespace, Resource, fields\n\nfrom application.api import api\nfrom application.api.user.base import (\n agent_folders_collection,\n agents_collection,\n)\n\nagents_folders_ns = Namespace(\n \"agents_folders\", description=\"Agent folder management\", path=\"/api/agents/folders\"\n)\n\n\ndef _folder_error_response(message: str, err: Exception):\n current_app.logger.error(f\"{message}: {err}\", exc_info=True)\n return make_response(jsonify({\"success\": False, \"message\": message}), 400)\n\n\n@agents_folders_ns.route(\"/\")\nclass AgentFolders(Resource):\n @api.doc(description=\"Get all folders for the user\")\n def get(self):\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False}), 401)\n user = decoded_token.get(\"sub\")\n try:\n folders = list(agent_folders_collection.find({\"user\": user}))\n result = [\n {\n \"id\": str(f[\"_id\"]),\n \"name\": f[\"name\"],\n \"parent_id\": f.get(\"parent_id\"),\n \"created_at\": f.get(\"created_at\", \"\").isoformat() if f.get(\"created_at\") else None,\n \"updated_at\": f.get(\"updated_at\", \"\").isoformat() if f.get(\"updated_at\") else None,\n }\n for f in folders\n ]\n return make_response(jsonify({\"folders\": result}), 200)\n except Exception as err:\n return _folder_error_response(\"Failed to fetch folders\", err)\n\n @api.doc(description=\"Create a new folder\")\n @api.expect(\n api.model(\n \"CreateFolder\",\n {\n \"name\": fields.String(required=True, description=\"Folder name\"),\n \"parent_id\": fields.String(required=False, description=\"Parent folder ID\"),\n },\n )\n )\n def post(self):\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False}), 401)\n user = decoded_token.get(\"sub\")\n data = request.get_json()\n if not data or not data.get(\"name\"):\n return make_response(jsonify({\"success\": False, \"message\": \"Folder name is required\"}), 400)\n\n parent_id = data.get(\"parent_id\")\n if parent_id:\n parent = agent_folders_collection.find_one({\"_id\": ObjectId(parent_id), \"user\": user})\n if not parent:\n return make_response(jsonify({\"success\": False, \"message\": \"Parent folder not found\"}), 404)\n\n try:\n now = datetime.datetime.now(datetime.timezone.utc)\n folder = {\n \"user\": user,\n \"name\": data[\"name\"],\n \"parent_id\": parent_id,\n \"created_at\": now,\n \"updated_at\": now,\n }\n result = agent_folders_collection.insert_one(folder)\n return make_response(\n jsonify({\"id\": str(result.inserted_id), \"name\": data[\"name\"], \"parent_id\": parent_id}),\n 201,\n )\n except Exception as err:\n return _folder_error_response(\"Failed to create folder\", err)\n\n\n@agents_folders_ns.route(\"/\")\nclass AgentFolder(Resource):\n @api.doc(description=\"Get a specific folder with its agents\")\n def get(self, folder_id):\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False}), 401)\n user = decoded_token.get(\"sub\")\n try:\n folder = agent_folders_collection.find_one({\"_id\": ObjectId(folder_id), \"user\": user})\n if not folder:\n return make_response(jsonify({\"success\": False, \"message\": \"Folder not found\"}), 404)\n \n agents = list(agents_collection.find({\"user\": user, \"folder_id\": folder_id}))\n agents_list = [\n {\"id\": str(a[\"_id\"]), \"name\": a[\"name\"], \"description\": a.get(\"description\", \"\")}\n for a in agents\n ]\n subfolders = list(agent_folders_collection.find({\"user\": user, \"parent_id\": folder_id}))\n subfolders_list = [{\"id\": str(sf[\"_id\"]), \"name\": sf[\"name\"]} for sf in subfolders]\n\n return make_response(\n jsonify({\n \"id\": str(folder[\"_id\"]),\n \"name\": folder[\"name\"],\n \"parent_id\": folder.get(\"parent_id\"),\n \"agents\": agents_list,\n \"subfolders\": subfolders_list,\n }),\n 200,\n )\n except Exception as err:\n return _folder_error_response(\"Failed to fetch folder\", err)\n\n @api.doc(description=\"Update a folder\")\n def put(self, folder_id):\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False}), 401)\n user = decoded_token.get(\"sub\")\n data = request.get_json()\n if not data:\n return make_response(jsonify({\"success\": False, \"message\": \"No data provided\"}), 400)\n\n try:\n update_fields = {\"updated_at\": datetime.datetime.now(datetime.timezone.utc)}\n if \"name\" in data:\n update_fields[\"name\"] = data[\"name\"]\n if \"parent_id\" in data:\n if data[\"parent_id\"] == folder_id:\n return make_response(jsonify({\"success\": False, \"message\": \"Cannot set folder as its own parent\"}), 400)\n update_fields[\"parent_id\"] = data[\"parent_id\"]\n\n result = agent_folders_collection.update_one(\n {\"_id\": ObjectId(folder_id), \"user\": user}, {\"$set\": update_fields}\n )\n if result.matched_count == 0:\n return make_response(jsonify({\"success\": False, \"message\": \"Folder not found\"}), 404)\n return make_response(jsonify({\"success\": True}), 200)\n except Exception as err:\n return _folder_error_response(\"Failed to update folder\", err)\n\n @api.doc(description=\"Delete a folder\")\n def delete(self, folder_id):\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False}), 401)\n user = decoded_token.get(\"sub\")\n try:\n agents_collection.update_many(\n {\"user\": user, \"folder_id\": folder_id}, {\"$unset\": {\"folder_id\": \"\"}}\n )\n agent_folders_collection.update_many(\n {\"user\": user, \"parent_id\": folder_id}, {\"$unset\": {\"parent_id\": \"\"}}\n )\n result = agent_folders_collection.delete_one({\"_id\": ObjectId(folder_id), \"user\": user})\n if result.deleted_count == 0:\n return make_response(jsonify({\"success\": False, \"message\": \"Folder not found\"}), 404)\n return make_response(jsonify({\"success\": True}), 200)\n except Exception as err:\n return _folder_error_response(\"Failed to delete folder\", err)\n\n\n@agents_folders_ns.route(\"/move_agent\")\nclass MoveAgentToFolder(Resource):\n @api.doc(description=\"Move an agent to a folder or remove from folder\")\n @api.expect(\n api.model(\n \"MoveAgent\",\n {\n \"agent_id\": fields.String(required=True, description=\"Agent ID to move\"),\n \"folder_id\": fields.String(required=False, description=\"Target folder ID (null to remove from folder)\"),\n },\n )\n )\n def post(self):\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False}), 401)\n user = decoded_token.get(\"sub\")\n data = request.get_json()\n if not data or not data.get(\"agent_id\"):\n return make_response(jsonify({\"success\": False, \"message\": \"Agent ID is required\"}), 400)\n\n agent_id = data[\"agent_id\"]\n folder_id = data.get(\"folder_id\")\n\n try:\n agent = agents_collection.find_one({\"_id\": ObjectId(agent_id), \"user\": user})\n if not agent:\n return make_response(jsonify({\"success\": False, \"message\": \"Agent not found\"}), 404)\n\n if folder_id:\n folder = agent_folders_collection.find_one({\"_id\": ObjectId(folder_id), \"user\": user})\n if not folder:\n return make_response(jsonify({\"success\": False, \"message\": \"Folder not found\"}), 404)\n agents_collection.update_one(\n {\"_id\": ObjectId(agent_id)}, {\"$set\": {\"folder_id\": folder_id}}\n )\n else:\n agents_collection.update_one(\n {\"_id\": ObjectId(agent_id)}, {\"$unset\": {\"folder_id\": \"\"}}\n )\n\n return make_response(jsonify({\"success\": True}), 200)\n except Exception as err:\n return _folder_error_response(\"Failed to move agent\", err)\n\n\n@agents_folders_ns.route(\"/bulk_move\")\nclass BulkMoveAgents(Resource):\n @api.doc(description=\"Move multiple agents to a folder\")\n @api.expect(\n api.model(\n \"BulkMoveAgents\",\n {\n \"agent_ids\": fields.List(fields.String, required=True, description=\"List of agent IDs\"),\n \"folder_id\": fields.String(required=False, description=\"Target folder ID\"),\n },\n )\n )\n def post(self):\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False}), 401)\n user = decoded_token.get(\"sub\")\n data = request.get_json()\n if not data or not data.get(\"agent_ids\"):\n return make_response(jsonify({\"success\": False, \"message\": \"Agent IDs are required\"}), 400)\n\n agent_ids = data[\"agent_ids\"]\n folder_id = data.get(\"folder_id\")\n\n try:\n if folder_id:\n folder = agent_folders_collection.find_one({\"_id\": ObjectId(folder_id), \"user\": user})\n if not folder:\n return make_response(jsonify({\"success\": False, \"message\": \"Folder not found\"}), 404)\n\n object_ids = [ObjectId(aid) for aid in agent_ids]\n if folder_id:\n agents_collection.update_many(\n {\"_id\": {\"$in\": object_ids}, \"user\": user},\n {\"$set\": {\"folder_id\": folder_id}},\n )\n else:\n agents_collection.update_many(\n {\"_id\": {\"$in\": object_ids}, \"user\": user},\n {\"$unset\": {\"folder_id\": \"\"}},\n )\n return make_response(jsonify({\"success\": True}), 200)\n except Exception as err:\n return _folder_error_response(\"Failed to move agents\", err)\n" }, { "path": "application/api/user/agents/routes.py", "content": "\"\"\"Agent management routes.\"\"\"\n\nimport datetime\nimport json\nimport uuid\n\nfrom bson.dbref import DBRef\nfrom bson.objectid import ObjectId\nfrom flask import current_app, jsonify, make_response, request\nfrom flask_restx import fields, Namespace, Resource\n\nfrom application.api import api\nfrom application.api.user.base import (\n agent_folders_collection,\n agents_collection,\n db,\n ensure_user_doc,\n handle_image_upload,\n resolve_tool_details,\n storage,\n users_collection,\n workflow_edges_collection,\n workflow_nodes_collection,\n workflows_collection,\n)\nfrom application.core.json_schema_utils import (\n JsonSchemaValidationError,\n normalize_json_schema_payload,\n)\nfrom application.core.settings import settings\nfrom application.utils import (\n check_required_fields,\n generate_image_url,\n validate_required_fields,\n)\n\n\nagents_ns = Namespace(\"agents\", description=\"Agent management operations\", path=\"/api\")\n\n\nAGENT_TYPE_SCHEMAS = {\n \"classic\": {\n \"required_published\": [\n \"name\",\n \"description\",\n \"chunks\",\n \"retriever\",\n \"prompt_id\",\n ],\n \"required_draft\": [\"name\"],\n \"validate_published\": [\"name\", \"description\", \"prompt_id\"],\n \"validate_draft\": [],\n \"require_source\": True,\n \"fields\": [\n \"user\",\n \"name\",\n \"description\",\n \"agent_type\",\n \"status\",\n \"key\",\n \"image\",\n \"source\",\n \"sources\",\n \"chunks\",\n \"retriever\",\n \"prompt_id\",\n \"tools\",\n \"json_schema\",\n \"models\",\n \"default_model_id\",\n \"folder_id\",\n \"limited_token_mode\",\n \"token_limit\",\n \"limited_request_mode\",\n \"request_limit\",\n \"createdAt\",\n \"updatedAt\",\n \"lastUsedAt\",\n ],\n },\n \"workflow\": {\n \"required_published\": [\"name\", \"workflow\"],\n \"required_draft\": [\"name\"],\n \"validate_published\": [\"name\", \"workflow\"],\n \"validate_draft\": [],\n \"fields\": [\n \"user\",\n \"name\",\n \"description\",\n \"agent_type\",\n \"status\",\n \"key\",\n \"workflow\",\n \"folder_id\",\n \"limited_token_mode\",\n \"token_limit\",\n \"limited_request_mode\",\n \"request_limit\",\n \"createdAt\",\n \"updatedAt\",\n \"lastUsedAt\",\n ],\n },\n}\n\nAGENT_TYPE_SCHEMAS[\"react\"] = AGENT_TYPE_SCHEMAS[\"classic\"]\nAGENT_TYPE_SCHEMAS[\"openai\"] = AGENT_TYPE_SCHEMAS[\"classic\"]\n\n\ndef normalize_workflow_reference(workflow_value):\n \"\"\"Normalize workflow references from form/json payloads.\"\"\"\n if workflow_value is None:\n return None\n if isinstance(workflow_value, dict):\n return (\n workflow_value.get(\"id\")\n or workflow_value.get(\"_id\")\n or workflow_value.get(\"workflow_id\")\n )\n if isinstance(workflow_value, str):\n value = workflow_value.strip()\n if not value:\n return \"\"\n try:\n parsed = json.loads(value)\n if isinstance(parsed, str):\n return parsed.strip()\n if isinstance(parsed, dict):\n return (\n parsed.get(\"id\") or parsed.get(\"_id\") or parsed.get(\"workflow_id\")\n )\n except json.JSONDecodeError:\n pass\n return value\n return str(workflow_value)\n\n\ndef validate_workflow_access(workflow_value, user, required=False):\n \"\"\"Validate workflow reference and ensure ownership.\"\"\"\n workflow_id = normalize_workflow_reference(workflow_value)\n if not workflow_id:\n if required:\n return None, make_response(\n jsonify({\"success\": False, \"message\": \"Workflow is required\"}), 400\n )\n return None, None\n if not ObjectId.is_valid(workflow_id):\n return None, make_response(\n jsonify({\"success\": False, \"message\": \"Invalid workflow ID format\"}), 400\n )\n workflow = workflows_collection.find_one({\"_id\": ObjectId(workflow_id), \"user\": user})\n if not workflow:\n return None, make_response(\n jsonify({\"success\": False, \"message\": \"Workflow not found\"}), 404\n )\n return workflow_id, None\n\n\ndef build_agent_document(\n data, user, key, agent_type, image_url=None, source_field=None, sources_list=None\n):\n \"\"\"Build agent document based on agent type schema.\"\"\"\n\n if not agent_type or agent_type not in AGENT_TYPE_SCHEMAS:\n agent_type = \"classic\"\n schema = AGENT_TYPE_SCHEMAS.get(agent_type, AGENT_TYPE_SCHEMAS[\"classic\"])\n allowed_fields = set(schema[\"fields\"])\n\n now = datetime.datetime.now(datetime.timezone.utc)\n base_doc = {\n \"user\": user,\n \"name\": data.get(\"name\"),\n \"description\": data.get(\"description\", \"\"),\n \"agent_type\": agent_type,\n \"status\": data.get(\"status\"),\n \"key\": key,\n \"createdAt\": now,\n \"updatedAt\": now,\n \"lastUsedAt\": None,\n }\n\n if agent_type == \"workflow\":\n base_doc[\"workflow\"] = data.get(\"workflow\")\n base_doc[\"folder_id\"] = data.get(\"folder_id\")\n else:\n base_doc.update(\n {\n \"image\": image_url or \"\",\n \"source\": source_field or \"\",\n \"sources\": sources_list or [],\n \"chunks\": data.get(\"chunks\", \"\"),\n \"retriever\": data.get(\"retriever\", \"\"),\n \"prompt_id\": data.get(\"prompt_id\", \"\"),\n \"tools\": data.get(\"tools\", []),\n \"json_schema\": data.get(\"json_schema\"),\n \"models\": data.get(\"models\", []),\n \"default_model_id\": data.get(\"default_model_id\", \"\"),\n \"folder_id\": data.get(\"folder_id\"),\n }\n )\n if \"limited_token_mode\" in allowed_fields:\n base_doc[\"limited_token_mode\"] = (\n data.get(\"limited_token_mode\") == \"True\"\n if isinstance(data.get(\"limited_token_mode\"), str)\n else bool(data.get(\"limited_token_mode\", False))\n )\n if \"token_limit\" in allowed_fields:\n base_doc[\"token_limit\"] = int(\n data.get(\"token_limit\", settings.DEFAULT_AGENT_LIMITS[\"token_limit\"])\n )\n if \"limited_request_mode\" in allowed_fields:\n base_doc[\"limited_request_mode\"] = (\n data.get(\"limited_request_mode\") == \"True\"\n if isinstance(data.get(\"limited_request_mode\"), str)\n else bool(data.get(\"limited_request_mode\", False))\n )\n if \"request_limit\" in allowed_fields:\n base_doc[\"request_limit\"] = int(\n data.get(\"request_limit\", settings.DEFAULT_AGENT_LIMITS[\"request_limit\"])\n )\n return {k: v for k, v in base_doc.items() if k in allowed_fields}\n\n\n@agents_ns.route(\"/get_agent\")\nclass GetAgent(Resource):\n @api.doc(params={\"id\": \"Agent ID\"}, description=\"Get agent by ID\")\n def get(self):\n if not (decoded_token := request.decoded_token):\n return {\"success\": False}, 401\n if not (agent_id := request.args.get(\"id\")):\n return {\"success\": False, \"message\": \"ID required\"}, 400\n try:\n agent = agents_collection.find_one(\n {\"_id\": ObjectId(agent_id), \"user\": decoded_token[\"sub\"]}\n )\n if not agent:\n return {\"status\": \"Not found\"}, 404\n data = {\n \"id\": str(agent[\"_id\"]),\n \"name\": agent[\"name\"],\n \"description\": agent.get(\"description\", \"\"),\n \"image\": (\n generate_image_url(agent[\"image\"]) if agent.get(\"image\") else \"\"\n ),\n \"source\": (\n str(source_doc[\"_id\"])\n if isinstance(agent.get(\"source\"), DBRef)\n and (source_doc := db.dereference(agent.get(\"source\")))\n else \"\"\n ),\n \"sources\": [\n (\n str(db.dereference(source_ref)[\"_id\"])\n if isinstance(source_ref, DBRef) and db.dereference(source_ref)\n else source_ref\n )\n for source_ref in agent.get(\"sources\", [])\n if (isinstance(source_ref, DBRef) and db.dereference(source_ref))\n or source_ref == \"default\"\n ],\n \"chunks\": agent.get(\"chunks\", \"2\"),\n \"retriever\": agent.get(\"retriever\", \"\"),\n \"prompt_id\": agent.get(\"prompt_id\", \"\"),\n \"tools\": agent.get(\"tools\", []),\n \"tool_details\": resolve_tool_details(agent.get(\"tools\", [])),\n \"agent_type\": agent.get(\"agent_type\", \"\"),\n \"status\": agent.get(\"status\", \"\"),\n \"json_schema\": agent.get(\"json_schema\"),\n \"limited_token_mode\": agent.get(\"limited_token_mode\", False),\n \"token_limit\": agent.get(\n \"token_limit\", settings.DEFAULT_AGENT_LIMITS[\"token_limit\"]\n ),\n \"limited_request_mode\": agent.get(\"limited_request_mode\", False),\n \"request_limit\": agent.get(\n \"request_limit\", settings.DEFAULT_AGENT_LIMITS[\"request_limit\"]\n ),\n \"created_at\": agent.get(\"createdAt\", \"\"),\n \"updated_at\": agent.get(\"updatedAt\", \"\"),\n \"last_used_at\": agent.get(\"lastUsedAt\", \"\"),\n \"key\": (\n f\"{agent['key'][:4]}...{agent['key'][-4:]}\"\n if \"key\" in agent\n else \"\"\n ),\n \"pinned\": agent.get(\"pinned\", False),\n \"shared\": agent.get(\"shared_publicly\", False),\n \"shared_metadata\": agent.get(\"shared_metadata\", {}),\n \"shared_token\": agent.get(\"shared_token\", \"\"),\n \"models\": agent.get(\"models\", []),\n \"default_model_id\": agent.get(\"default_model_id\", \"\"),\n \"folder_id\": agent.get(\"folder_id\"),\n \"workflow\": agent.get(\"workflow\"),\n }\n return make_response(jsonify(data), 200)\n except Exception as e:\n current_app.logger.error(f\"Agent fetch error: {e}\", exc_info=True)\n return {\"success\": False}, 400\n\n\n@agents_ns.route(\"/get_agents\")\nclass GetAgents(Resource):\n @api.doc(description=\"Retrieve agents for the user\")\n def get(self):\n if not (decoded_token := request.decoded_token):\n return {\"success\": False}, 401\n user = decoded_token.get(\"sub\")\n try:\n user_doc = ensure_user_doc(user)\n pinned_ids = set(user_doc.get(\"agent_preferences\", {}).get(\"pinned\", []))\n\n agents = agents_collection.find({\"user\": user})\n list_agents = [\n {\n \"id\": str(agent[\"_id\"]),\n \"name\": agent[\"name\"],\n \"description\": agent.get(\"description\", \"\"),\n \"image\": (\n generate_image_url(agent[\"image\"]) if agent.get(\"image\") else \"\"\n ),\n \"source\": (\n str(source_doc[\"_id\"])\n if isinstance(agent.get(\"source\"), DBRef)\n and (source_doc := db.dereference(agent.get(\"source\")))\n else (\n agent.get(\"source\", \"\")\n if agent.get(\"source\") == \"default\"\n else \"\"\n )\n ),\n \"sources\": [\n (\n source_ref\n if source_ref == \"default\"\n else str(db.dereference(source_ref)[\"_id\"])\n )\n for source_ref in agent.get(\"sources\", [])\n if source_ref == \"default\"\n or (\n isinstance(source_ref, DBRef) and db.dereference(source_ref)\n )\n ],\n \"chunks\": agent.get(\"chunks\", \"2\"),\n \"retriever\": agent.get(\"retriever\", \"\"),\n \"prompt_id\": agent.get(\"prompt_id\", \"\"),\n \"tools\": agent.get(\"tools\", []),\n \"tool_details\": resolve_tool_details(agent.get(\"tools\", [])),\n \"agent_type\": agent.get(\"agent_type\", \"\"),\n \"status\": agent.get(\"status\", \"\"),\n \"json_schema\": agent.get(\"json_schema\"),\n \"limited_token_mode\": agent.get(\"limited_token_mode\", False),\n \"token_limit\": agent.get(\n \"token_limit\", settings.DEFAULT_AGENT_LIMITS[\"token_limit\"]\n ),\n \"limited_request_mode\": agent.get(\"limited_request_mode\", False),\n \"request_limit\": agent.get(\n \"request_limit\", settings.DEFAULT_AGENT_LIMITS[\"request_limit\"]\n ),\n \"created_at\": agent.get(\"createdAt\", \"\"),\n \"updated_at\": agent.get(\"updatedAt\", \"\"),\n \"last_used_at\": agent.get(\"lastUsedAt\", \"\"),\n \"key\": (\n f\"{agent['key'][:4]}...{agent['key'][-4:]}\"\n if \"key\" in agent\n else \"\"\n ),\n \"pinned\": str(agent[\"_id\"]) in pinned_ids,\n \"shared\": agent.get(\"shared_publicly\", False),\n \"shared_metadata\": agent.get(\"shared_metadata\", {}),\n \"shared_token\": agent.get(\"shared_token\", \"\"),\n \"models\": agent.get(\"models\", []),\n \"default_model_id\": agent.get(\"default_model_id\", \"\"),\n \"folder_id\": agent.get(\"folder_id\"),\n \"workflow\": agent.get(\"workflow\"),\n }\n for agent in agents\n if \"source\" in agent\n or \"retriever\" in agent\n or agent.get(\"agent_type\") == \"workflow\"\n ]\n except Exception as err:\n current_app.logger.error(f\"Error retrieving agents: {err}\", exc_info=True)\n return make_response(jsonify({\"success\": False}), 400)\n return make_response(jsonify(list_agents), 200)\n\n\n@agents_ns.route(\"/create_agent\")\nclass CreateAgent(Resource):\n create_agent_model = api.model(\n \"CreateAgentModel\",\n {\n \"name\": fields.String(required=True, description=\"Name of the agent\"),\n \"description\": fields.String(\n required=True, description=\"Description of the agent\"\n ),\n \"image\": fields.Raw(\n required=False, description=\"Image file upload\", type=\"file\"\n ),\n \"source\": fields.String(\n required=False, description=\"Source ID (legacy single source)\"\n ),\n \"sources\": fields.List(\n fields.String,\n required=False,\n description=\"List of source identifiers for multiple sources\",\n ),\n \"chunks\": fields.Integer(required=False, description=\"Chunks count\"),\n \"retriever\": fields.String(required=False, description=\"Retriever ID\"),\n \"prompt_id\": fields.String(required=False, description=\"Prompt ID\"),\n \"tools\": fields.List(\n fields.String, required=False, description=\"List of tool identifiers\"\n ),\n \"agent_type\": fields.String(\n required=False,\n description=\"Type of the agent (classic, react, workflow). Defaults to 'classic' for backwards compatibility.\",\n ),\n \"status\": fields.String(\n required=True, description=\"Status of the agent (draft or published)\"\n ),\n \"workflow\": fields.String(\n required=False, description=\"Workflow ID for workflow-type agents\"\n ),\n \"json_schema\": fields.Raw(\n required=False,\n description=\"JSON schema for enforcing structured output format\",\n ),\n \"limited_token_mode\": fields.Boolean(\n required=False, description=\"Whether the agent is in limited token mode\"\n ),\n \"token_limit\": fields.Integer(\n required=False, description=\"Token limit for the agent in limited mode\"\n ),\n \"limited_request_mode\": fields.Boolean(\n required=False,\n description=\"Whether the agent is in limited request mode\",\n ),\n \"request_limit\": fields.Integer(\n required=False,\n description=\"Request limit for the agent in limited mode\",\n ),\n \"models\": fields.List(\n fields.String,\n required=False,\n description=\"List of available model IDs for this agent\",\n ),\n \"default_model_id\": fields.String(\n required=False, description=\"Default model ID for this agent\"\n ),\n \"folder_id\": fields.String(\n required=False, description=\"Folder ID to organize the agent\"\n ),\n },\n )\n\n @api.expect(create_agent_model)\n @api.doc(description=\"Create a new agent\")\n def post(self):\n if not (decoded_token := request.decoded_token):\n return {\"success\": False}, 401\n user = decoded_token.get(\"sub\")\n if request.content_type == \"application/json\":\n data = request.get_json()\n else:\n data = request.form.to_dict()\n if \"tools\" in data:\n try:\n data[\"tools\"] = json.loads(data[\"tools\"])\n except json.JSONDecodeError:\n data[\"tools\"] = []\n if \"sources\" in data:\n try:\n data[\"sources\"] = json.loads(data[\"sources\"])\n except json.JSONDecodeError:\n data[\"sources\"] = []\n if \"json_schema\" in data:\n try:\n data[\"json_schema\"] = json.loads(data[\"json_schema\"])\n except json.JSONDecodeError:\n data[\"json_schema\"] = None\n if \"models\" in data:\n try:\n data[\"models\"] = json.loads(data[\"models\"])\n except json.JSONDecodeError:\n data[\"models\"] = []\n print(f\"Received data: {data}\")\n\n # Validate and normalize JSON schema if provided\n if \"json_schema\" in data:\n try:\n data[\"json_schema\"] = normalize_json_schema_payload(\n data.get(\"json_schema\")\n )\n except JsonSchemaValidationError as exc:\n return make_response(\n jsonify({\"success\": False, \"message\": f\"JSON schema {exc}\"}),\n 400,\n )\n if data.get(\"status\") not in [\"draft\", \"published\"]:\n return make_response(\n jsonify(\n {\n \"success\": False,\n \"message\": \"Status must be either 'draft' or 'published'\",\n }\n ),\n 400,\n )\n agent_type = data.get(\"agent_type\", \"\")\n # Default to classic schema for empty or unknown agent types\n\n if not agent_type or agent_type not in AGENT_TYPE_SCHEMAS:\n schema = AGENT_TYPE_SCHEMAS[\"classic\"]\n # Set agent_type to classic if it was empty\n\n if not agent_type:\n agent_type = \"classic\"\n else:\n schema = AGENT_TYPE_SCHEMAS[agent_type]\n is_published = data.get(\"status\") == \"published\"\n if agent_type == \"workflow\":\n workflow_id, workflow_error = validate_workflow_access(\n data.get(\"workflow\"), user, required=is_published\n )\n if workflow_error:\n return workflow_error\n data[\"workflow\"] = workflow_id\n if data.get(\"status\") == \"published\":\n required_fields = schema[\"required_published\"]\n validate_fields = schema[\"validate_published\"]\n\n if (\n schema.get(\"require_source\")\n and not data.get(\"source\")\n and not data.get(\"sources\")\n ):\n return make_response(\n jsonify(\n {\n \"success\": False,\n \"message\": \"Either 'source' or 'sources' field is required for published agents\",\n }\n ),\n 400,\n )\n else:\n required_fields = schema[\"required_draft\"]\n validate_fields = schema[\"validate_draft\"]\n missing_fields = check_required_fields(data, required_fields)\n invalid_fields = validate_required_fields(data, validate_fields)\n if missing_fields:\n return missing_fields\n if invalid_fields:\n return invalid_fields\n image_url, error = handle_image_upload(request, \"\", user, storage)\n if error:\n return make_response(\n jsonify({\"success\": False, \"message\": \"Image upload failed\"}), 400\n )\n folder_id = data.get(\"folder_id\")\n if folder_id:\n if not ObjectId.is_valid(folder_id):\n return make_response(\n jsonify({\"success\": False, \"message\": \"Invalid folder ID format\"}),\n 400,\n )\n folder = agent_folders_collection.find_one(\n {\"_id\": ObjectId(folder_id), \"user\": user}\n )\n if not folder:\n return make_response(\n jsonify({\"success\": False, \"message\": \"Folder not found\"}), 404\n )\n try:\n key = str(uuid.uuid4()) if data.get(\"status\") == \"published\" else \"\"\n\n sources_list = []\n source_field = \"\"\n if data.get(\"sources\") and len(data.get(\"sources\", [])) > 0:\n for source_id in data.get(\"sources\", []):\n if source_id == \"default\":\n sources_list.append(\"default\")\n elif ObjectId.is_valid(source_id):\n sources_list.append(DBRef(\"sources\", ObjectId(source_id)))\n else:\n source_value = data.get(\"source\", \"\")\n if source_value == \"default\":\n source_field = \"default\"\n elif ObjectId.is_valid(source_value):\n source_field = DBRef(\"sources\", ObjectId(source_value))\n new_agent = build_agent_document(\n data, user, key, agent_type, image_url, source_field, sources_list\n )\n\n if agent_type != \"workflow\":\n if new_agent.get(\"chunks\") == \"\":\n new_agent[\"chunks\"] = \"2\"\n if (\n new_agent.get(\"source\") == \"\"\n and new_agent.get(\"retriever\") == \"\"\n and not new_agent.get(\"sources\")\n ):\n new_agent[\"retriever\"] = \"classic\"\n resp = agents_collection.insert_one(new_agent)\n new_id = str(resp.inserted_id)\n except Exception as err:\n current_app.logger.error(f\"Error creating agent: {err}\", exc_info=True)\n return make_response(jsonify({\"success\": False}), 400)\n return make_response(jsonify({\"id\": new_id, \"key\": key}), 201)\n\n\n@agents_ns.route(\"/update_agent/\")\nclass UpdateAgent(Resource):\n update_agent_model = api.model(\n \"UpdateAgentModel\",\n {\n \"name\": fields.String(required=True, description=\"New name of the agent\"),\n \"description\": fields.String(\n required=True, description=\"New description of the agent\"\n ),\n \"image\": fields.String(\n required=False, description=\"New image URL or identifier\"\n ),\n \"source\": fields.String(\n required=False, description=\"Source ID (legacy single source)\"\n ),\n \"sources\": fields.List(\n fields.String,\n required=False,\n description=\"List of source identifiers for multiple sources\",\n ),\n \"chunks\": fields.Integer(required=False, description=\"Chunks count\"),\n \"retriever\": fields.String(required=False, description=\"Retriever ID\"),\n \"prompt_id\": fields.String(required=False, description=\"Prompt ID\"),\n \"tools\": fields.List(\n fields.String, required=False, description=\"List of tool identifiers\"\n ),\n \"agent_type\": fields.String(\n required=False,\n description=\"Type of the agent (classic, react, workflow). Defaults to 'classic' for backwards compatibility.\",\n ),\n \"status\": fields.String(\n required=True, description=\"Status of the agent (draft or published)\"\n ),\n \"workflow\": fields.String(\n required=False, description=\"Workflow ID for workflow-type agents\"\n ),\n \"json_schema\": fields.Raw(\n required=False,\n description=\"JSON schema for enforcing structured output format\",\n ),\n \"limited_token_mode\": fields.Boolean(\n required=False, description=\"Whether the agent is in limited token mode\"\n ),\n \"token_limit\": fields.Integer(\n required=False, description=\"Token limit for the agent in limited mode\"\n ),\n \"limited_request_mode\": fields.Boolean(\n require=False,\n description=\"Whether the agent is in limited request mode\",\n ),\n \"request_limit\": fields.Integer(\n required=False,\n description=\"Request limit for the agent in limited mode\",\n ),\n \"models\": fields.List(\n fields.String,\n required=False,\n description=\"List of available model IDs for this agent\",\n ),\n \"default_model_id\": fields.String(\n required=False, description=\"Default model ID for this agent\"\n ),\n \"folder_id\": fields.String(\n required=False, description=\"Folder ID to organize the agent\"\n ),\n },\n )\n\n @api.expect(update_agent_model)\n @api.doc(description=\"Update an existing agent\")\n def put(self, agent_id):\n if not (decoded_token := request.decoded_token):\n return make_response(\n jsonify({\"success\": False, \"message\": \"Unauthorized\"}), 401\n )\n user = decoded_token.get(\"sub\")\n\n if not ObjectId.is_valid(agent_id):\n return make_response(\n jsonify({\"success\": False, \"message\": \"Invalid agent ID format\"}), 400\n )\n oid = ObjectId(agent_id)\n\n try:\n if request.content_type and \"application/json\" in request.content_type:\n data = request.get_json()\n else:\n data = request.form.to_dict()\n json_fields = [\"tools\", \"sources\", \"json_schema\", \"models\"]\n for field in json_fields:\n if field in data and data[field]:\n try:\n data[field] = json.loads(data[field])\n except json.JSONDecodeError:\n return make_response(\n jsonify(\n {\n \"success\": False,\n \"message\": f\"Invalid JSON format for field: {field}\",\n }\n ),\n 400,\n )\n if data.get(\"json_schema\") == \"\":\n data[\"json_schema\"] = None\n except Exception as err:\n current_app.logger.error(\n f\"Error parsing request data: {err}\", exc_info=True\n )\n return make_response(\n jsonify({\"success\": False, \"message\": \"Invalid request data\"}), 400\n )\n try:\n existing_agent = agents_collection.find_one({\"_id\": oid, \"user\": user})\n except Exception as err:\n current_app.logger.error(\n f\"Error finding agent {agent_id}: {err}\", exc_info=True\n )\n return make_response(\n jsonify({\"success\": False, \"message\": \"Database error finding agent\"}),\n 500,\n )\n if not existing_agent:\n return make_response(\n jsonify(\n {\"success\": False, \"message\": \"Agent not found or not authorized\"}\n ),\n 404,\n )\n image_url, error = handle_image_upload(\n request, existing_agent.get(\"image\", \"\"), user, storage\n )\n if error:\n return error\n update_fields = {}\n allowed_fields = [\n \"name\",\n \"description\",\n \"image\",\n \"source\",\n \"sources\",\n \"chunks\",\n \"retriever\",\n \"prompt_id\",\n \"tools\",\n \"agent_type\",\n \"status\",\n \"json_schema\",\n \"limited_token_mode\",\n \"token_limit\",\n \"limited_request_mode\",\n \"request_limit\",\n \"models\",\n \"default_model_id\",\n \"folder_id\",\n \"workflow\",\n ]\n\n for field in allowed_fields:\n if field not in data:\n continue\n if field == \"status\":\n new_status = data.get(\"status\")\n if new_status not in [\"draft\", \"published\"]:\n return make_response(\n jsonify(\n {\n \"success\": False,\n \"message\": \"Invalid status value. Must be 'draft' or 'published'\",\n }\n ),\n 400,\n )\n update_fields[field] = new_status\n elif field == \"source\":\n source_id = data.get(\"source\")\n if source_id == \"default\":\n update_fields[field] = \"default\"\n elif source_id and ObjectId.is_valid(source_id):\n update_fields[field] = DBRef(\"sources\", ObjectId(source_id))\n elif source_id:\n return make_response(\n jsonify(\n {\n \"success\": False,\n \"message\": f\"Invalid source ID format: {source_id}\",\n }\n ),\n 400,\n )\n else:\n update_fields[field] = \"\"\n elif field == \"sources\":\n sources_list = data.get(\"sources\", [])\n if sources_list and isinstance(sources_list, list):\n valid_sources = []\n for source_id in sources_list:\n if source_id == \"default\":\n valid_sources.append(\"default\")\n elif ObjectId.is_valid(source_id):\n valid_sources.append(DBRef(\"sources\", ObjectId(source_id)))\n else:\n return make_response(\n jsonify(\n {\n \"success\": False,\n \"message\": f\"Invalid source ID in list: {source_id}\",\n }\n ),\n 400,\n )\n update_fields[field] = valid_sources\n else:\n update_fields[field] = []\n elif field == \"chunks\":\n chunks_value = data.get(\"chunks\")\n if chunks_value == \"\" or chunks_value is None:\n update_fields[field] = \"2\"\n else:\n try:\n chunks_int = int(chunks_value)\n if chunks_int < 0:\n return make_response(\n jsonify(\n {\n \"success\": False,\n \"message\": \"Chunks value must be a non-negative integer\",\n }\n ),\n 400,\n )\n update_fields[field] = str(chunks_int)\n except (ValueError, TypeError):\n return make_response(\n jsonify(\n {\n \"success\": False,\n \"message\": f\"Invalid chunks value: {chunks_value}\",\n }\n ),\n 400,\n )\n elif field == \"tools\":\n tools_list = data.get(\"tools\", [])\n if isinstance(tools_list, list):\n update_fields[field] = tools_list\n else:\n return make_response(\n jsonify(\n {\n \"success\": False,\n \"message\": \"Tools must be a list\",\n }\n ),\n 400,\n )\n elif field == \"json_schema\":\n json_schema = data.get(\"json_schema\")\n if json_schema is not None:\n try:\n update_fields[field] = normalize_json_schema_payload(\n json_schema\n )\n except JsonSchemaValidationError as exc:\n return make_response(\n jsonify({\"success\": False, \"message\": f\"JSON schema {exc}\"}),\n 400,\n )\n else:\n update_fields[field] = None\n elif field == \"limited_token_mode\":\n raw_value = data.get(\"limited_token_mode\", False)\n bool_value = (\n raw_value == \"True\"\n if isinstance(raw_value, str)\n else bool(raw_value)\n )\n update_fields[field] = bool_value\n\n if bool_value and data.get(\"token_limit\") is None:\n return make_response(\n jsonify(\n {\n \"success\": False,\n \"message\": \"Token limit must be provided when limited token mode is enabled\",\n }\n ),\n 400,\n )\n elif field == \"limited_request_mode\":\n raw_value = data.get(\"limited_request_mode\", False)\n bool_value = (\n raw_value == \"True\"\n if isinstance(raw_value, str)\n else bool(raw_value)\n )\n update_fields[field] = bool_value\n\n if bool_value and data.get(\"request_limit\") is None:\n return make_response(\n jsonify(\n {\n \"success\": False,\n \"message\": \"Request limit must be provided when limited request mode is enabled\",\n }\n ),\n 400,\n )\n elif field == \"token_limit\":\n token_limit = data.get(\"token_limit\")\n update_fields[field] = int(token_limit) if token_limit else 0\n\n # Validate consistency with mode\n\n if update_fields[field] > 0 and not data.get(\"limited_token_mode\"):\n return make_response(\n jsonify(\n {\n \"success\": False,\n \"message\": \"Token limit cannot be set when limited token mode is disabled\",\n }\n ),\n 400,\n )\n elif field == \"request_limit\":\n request_limit = data.get(\"request_limit\")\n update_fields[field] = int(request_limit) if request_limit else 0\n\n if update_fields[field] > 0 and not data.get(\"limited_request_mode\"):\n return make_response(\n jsonify(\n {\n \"success\": False,\n \"message\": \"Request limit cannot be set when limited request mode is disabled\",\n }\n ),\n 400,\n )\n elif field == \"folder_id\":\n folder_id = data.get(\"folder_id\")\n if folder_id:\n if not ObjectId.is_valid(folder_id):\n return make_response(\n jsonify(\n {\n \"success\": False,\n \"message\": \"Invalid folder ID format\",\n }\n ),\n 400,\n )\n folder = agent_folders_collection.find_one(\n {\"_id\": ObjectId(folder_id), \"user\": user}\n )\n if not folder:\n return make_response(\n jsonify({\"success\": False, \"message\": \"Folder not found\"}),\n 404,\n )\n update_fields[field] = folder_id\n else:\n update_fields[field] = None\n elif field == \"workflow\":\n workflow_required = (\n data.get(\"status\", existing_agent.get(\"status\")) == \"published\"\n and data.get(\"agent_type\", existing_agent.get(\"agent_type\"))\n == \"workflow\"\n )\n workflow_id, workflow_error = validate_workflow_access(\n data.get(\"workflow\"), user, required=workflow_required\n )\n if workflow_error:\n return workflow_error\n update_fields[field] = workflow_id\n else:\n value = data[field]\n if field in [\"name\", \"description\", \"prompt_id\", \"agent_type\"]:\n if not value or not str(value).strip():\n return make_response(\n jsonify(\n {\n \"success\": False,\n \"message\": f\"Field '{field}' cannot be empty\",\n }\n ),\n 400,\n )\n update_fields[field] = value\n if image_url:\n update_fields[\"image\"] = image_url\n if not update_fields:\n return make_response(\n jsonify(\n {\n \"success\": False,\n \"message\": \"No valid update data provided\",\n }\n ),\n 400,\n )\n newly_generated_key = None\n final_status = update_fields.get(\"status\", existing_agent.get(\"status\"))\n agent_type = update_fields.get(\"agent_type\", existing_agent.get(\"agent_type\"))\n\n if final_status == \"published\":\n if agent_type == \"workflow\":\n required_published_fields = {\n \"name\": \"Agent name\",\n }\n missing_published_fields = []\n for req_field, field_label in required_published_fields.items():\n final_value = update_fields.get(\n req_field, existing_agent.get(req_field)\n )\n if not final_value:\n missing_published_fields.append(field_label)\n\n workflow_id = update_fields.get(\"workflow\", existing_agent.get(\"workflow\"))\n if not workflow_id:\n missing_published_fields.append(\"Workflow\")\n elif not ObjectId.is_valid(workflow_id):\n missing_published_fields.append(\"Valid workflow\")\n else:\n workflow = workflows_collection.find_one(\n {\"_id\": ObjectId(workflow_id), \"user\": user}\n )\n if not workflow:\n missing_published_fields.append(\"Workflow access\")\n\n if missing_published_fields:\n return make_response(\n jsonify(\n {\n \"success\": False,\n \"message\": f\"Cannot publish workflow agent. Missing required fields: {', '.join(missing_published_fields)}\",\n }\n ),\n 400,\n )\n else:\n required_published_fields = {\n \"name\": \"Agent name\",\n \"description\": \"Agent description\",\n \"chunks\": \"Chunks count\",\n \"prompt_id\": \"Prompt\",\n \"agent_type\": \"Agent type\",\n }\n\n missing_published_fields = []\n for req_field, field_label in required_published_fields.items():\n final_value = update_fields.get(\n req_field, existing_agent.get(req_field)\n )\n if not final_value:\n missing_published_fields.append(field_label)\n source_val = update_fields.get(\"source\", existing_agent.get(\"source\"))\n sources_val = update_fields.get(\n \"sources\", existing_agent.get(\"sources\", [])\n )\n\n has_valid_source = (\n isinstance(source_val, DBRef)\n or source_val == \"default\"\n or (isinstance(sources_val, list) and len(sources_val) > 0)\n )\n\n if not has_valid_source:\n missing_published_fields.append(\"Source\")\n if missing_published_fields:\n return make_response(\n jsonify(\n {\n \"success\": False,\n \"message\": f\"Cannot publish agent. Missing or invalid required fields: {', '.join(missing_published_fields)}\",\n }\n ),\n 400,\n )\n if not existing_agent.get(\"key\"):\n newly_generated_key = str(uuid.uuid4())\n update_fields[\"key\"] = newly_generated_key\n update_fields[\"updatedAt\"] = datetime.datetime.now(datetime.timezone.utc)\n\n try:\n result = agents_collection.update_one(\n {\"_id\": oid, \"user\": user}, {\"$set\": update_fields}\n )\n\n if result.matched_count == 0:\n return make_response(\n jsonify(\n {\n \"success\": False,\n \"message\": \"Agent not found or update failed\",\n }\n ),\n 404,\n )\n if result.modified_count == 0 and result.matched_count == 1:\n return make_response(\n jsonify(\n {\n \"success\": True,\n \"message\": \"No changes detected\",\n \"id\": agent_id,\n }\n ),\n 200,\n )\n except Exception as err:\n current_app.logger.error(\n f\"Error updating agent {agent_id}: {err}\", exc_info=True\n )\n return make_response(\n jsonify({\"success\": False, \"message\": \"Database error during update\"}),\n 500,\n )\n response_data = {\n \"success\": True,\n \"id\": agent_id,\n \"message\": \"Agent updated successfully\",\n }\n if newly_generated_key:\n response_data[\"key\"] = newly_generated_key\n return make_response(jsonify(response_data), 200)\n\n\n@agents_ns.route(\"/delete_agent\")\nclass DeleteAgent(Resource):\n @api.doc(params={\"id\": \"ID of the agent\"}, description=\"Delete an agent by ID\")\n def delete(self):\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False}), 401)\n user = decoded_token.get(\"sub\")\n agent_id = request.args.get(\"id\")\n if not agent_id:\n return make_response(\n jsonify({\"success\": False, \"message\": \"ID is required\"}), 400\n )\n try:\n deleted_agent = agents_collection.find_one_and_delete(\n {\"_id\": ObjectId(agent_id), \"user\": user}\n )\n if not deleted_agent:\n return make_response(\n jsonify({\"success\": False, \"message\": \"Agent not found\"}), 404\n )\n deleted_id = str(deleted_agent[\"_id\"])\n\n if deleted_agent.get(\"agent_type\") == \"workflow\" and deleted_agent.get(\n \"workflow\"\n ):\n workflow_id = normalize_workflow_reference(deleted_agent.get(\"workflow\"))\n if workflow_id and ObjectId.is_valid(workflow_id):\n workflow_oid = ObjectId(workflow_id)\n owned_workflow = workflows_collection.find_one(\n {\"_id\": workflow_oid, \"user\": user}, {\"_id\": 1}\n )\n if owned_workflow:\n workflow_nodes_collection.delete_many({\"workflow_id\": workflow_id})\n workflow_edges_collection.delete_many({\"workflow_id\": workflow_id})\n workflows_collection.delete_one({\"_id\": workflow_oid, \"user\": user})\n else:\n current_app.logger.warning(\n f\"Skipping workflow cleanup for non-owned workflow {workflow_id}\"\n )\n elif workflow_id:\n current_app.logger.warning(\n f\"Skipping workflow cleanup for invalid workflow id {workflow_id}\"\n )\n\n except Exception as err:\n current_app.logger.error(f\"Error deleting agent: {err}\", exc_info=True)\n return make_response(jsonify({\"success\": False}), 400)\n return make_response(jsonify({\"id\": deleted_id}), 200)\n\n\n@agents_ns.route(\"/pinned_agents\")\nclass PinnedAgents(Resource):\n @api.doc(description=\"Get pinned agents for the user\")\n def get(self):\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False}), 401)\n user_id = decoded_token.get(\"sub\")\n\n try:\n user_doc = ensure_user_doc(user_id)\n pinned_ids = user_doc.get(\"agent_preferences\", {}).get(\"pinned\", [])\n\n if not pinned_ids:\n return make_response(jsonify([]), 200)\n pinned_object_ids = [ObjectId(agent_id) for agent_id in pinned_ids]\n\n pinned_agents_cursor = agents_collection.find(\n {\"_id\": {\"$in\": pinned_object_ids}}\n )\n pinned_agents = list(pinned_agents_cursor)\n existing_ids = {str(agent[\"_id\"]) for agent in pinned_agents}\n\n # Clean up any stale pinned IDs\n\n stale_ids = [\n agent_id for agent_id in pinned_ids if agent_id not in existing_ids\n ]\n if stale_ids:\n users_collection.update_one(\n {\"user_id\": user_id},\n {\"$pullAll\": {\"agent_preferences.pinned\": stale_ids}},\n )\n list_pinned_agents = [\n {\n \"id\": str(agent[\"_id\"]),\n \"name\": agent.get(\"name\", \"\"),\n \"description\": agent.get(\"description\", \"\"),\n \"image\": (\n generate_image_url(agent[\"image\"]) if agent.get(\"image\") else \"\"\n ),\n \"source\": (\n str(db.dereference(agent[\"source\"])[\"_id\"])\n if \"source\" in agent\n and agent[\"source\"]\n and isinstance(agent[\"source\"], DBRef)\n and db.dereference(agent[\"source\"]) is not None\n else \"\"\n ),\n \"chunks\": agent.get(\"chunks\", \"\"),\n \"retriever\": agent.get(\"retriever\", \"\"),\n \"prompt_id\": agent.get(\"prompt_id\", \"\"),\n \"tools\": agent.get(\"tools\", []),\n \"tool_details\": resolve_tool_details(agent.get(\"tools\", [])),\n \"agent_type\": agent.get(\"agent_type\", \"\"),\n \"status\": agent.get(\"status\", \"\"),\n \"created_at\": agent.get(\"createdAt\", \"\"),\n \"updated_at\": agent.get(\"updatedAt\", \"\"),\n \"last_used_at\": agent.get(\"lastUsedAt\", \"\"),\n \"key\": (\n f\"{agent['key'][:4]}...{agent['key'][-4:]}\"\n if \"key\" in agent\n else \"\"\n ),\n \"pinned\": True,\n }\n for agent in pinned_agents\n if \"source\" in agent or \"retriever\" in agent\n ]\n except Exception as err:\n current_app.logger.error(f\"Error retrieving pinned agents: {err}\")\n return make_response(jsonify({\"success\": False}), 400)\n return make_response(jsonify(list_pinned_agents), 200)\n\n\n@agents_ns.route(\"/template_agents\")\nclass GetTemplateAgents(Resource):\n @api.doc(description=\"Get template/premade agents\")\n def get(self):\n try:\n template_agents = agents_collection.find({\"user\": \"system\"})\n template_agents = [\n {\n \"id\": str(agent[\"_id\"]),\n \"name\": agent[\"name\"],\n \"description\": agent[\"description\"],\n \"image\": agent.get(\"image\", \"\"),\n }\n for agent in template_agents\n ]\n return make_response(jsonify(template_agents), 200)\n except Exception as e:\n current_app.logger.error(f\"Template agents fetch error: {e}\", exc_info=True)\n return make_response(jsonify({\"success\": False}), 400)\n\n\n@agents_ns.route(\"/adopt_agent\")\nclass AdoptAgent(Resource):\n @api.doc(params={\"id\": \"Agent ID\"}, description=\"Adopt an agent by ID\")\n def post(self):\n if not (decoded_token := request.decoded_token):\n return make_response(jsonify({\"success\": False}), 401)\n if not (agent_id := request.args.get(\"id\")):\n return make_response(\n jsonify({\"success\": False, \"message\": \"ID required\"}), 400\n )\n try:\n agent = agents_collection.find_one(\n {\"_id\": ObjectId(agent_id), \"user\": \"system\"}\n )\n if not agent:\n return make_response(jsonify({\"status\": \"Not found\"}), 404)\n new_agent = agent.copy()\n new_agent.pop(\"_id\", None)\n new_agent[\"user\"] = decoded_token[\"sub\"]\n new_agent[\"status\"] = \"published\"\n new_agent[\"lastUsedAt\"] = datetime.datetime.now(datetime.timezone.utc)\n new_agent[\"key\"] = str(uuid.uuid4())\n insert_result = agents_collection.insert_one(new_agent)\n\n response_agent = new_agent.copy()\n response_agent.pop(\"_id\", None)\n response_agent[\"id\"] = str(insert_result.inserted_id)\n response_agent[\"tool_details\"] = resolve_tool_details(\n response_agent.get(\"tools\", [])\n )\n if isinstance(response_agent.get(\"source\"), DBRef):\n response_agent[\"source\"] = str(response_agent[\"source\"].id)\n return make_response(\n jsonify({\"success\": True, \"agent\": response_agent}), 200\n )\n except Exception as e:\n current_app.logger.error(f\"Agent adopt error: {e}\", exc_info=True)\n return make_response(jsonify({\"success\": False}), 400)\n\n\n@agents_ns.route(\"/pin_agent\")\nclass PinAgent(Resource):\n @api.doc(params={\"id\": \"ID of the agent\"}, description=\"Pin or unpin an agent\")\n def post(self):\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False}), 401)\n user_id = decoded_token.get(\"sub\")\n agent_id = request.args.get(\"id\")\n\n if not agent_id:\n return make_response(\n jsonify({\"success\": False, \"message\": \"ID is required\"}), 400\n )\n try:\n agent = agents_collection.find_one({\"_id\": ObjectId(agent_id)})\n if not agent:\n return make_response(\n jsonify({\"success\": False, \"message\": \"Agent not found\"}), 404\n )\n user_doc = ensure_user_doc(user_id)\n pinned_list = user_doc.get(\"agent_preferences\", {}).get(\"pinned\", [])\n\n if agent_id in pinned_list:\n users_collection.update_one(\n {\"user_id\": user_id},\n {\"$pull\": {\"agent_preferences.pinned\": agent_id}},\n )\n action = \"unpinned\"\n else:\n users_collection.update_one(\n {\"user_id\": user_id},\n {\"$addToSet\": {\"agent_preferences.pinned\": agent_id}},\n )\n action = \"pinned\"\n except Exception as err:\n current_app.logger.error(f\"Error pinning/unpinning agent: {err}\")\n return make_response(\n jsonify({\"success\": False, \"message\": \"Server error\"}), 500\n )\n return make_response(jsonify({\"success\": True, \"action\": action}), 200)\n\n\n@agents_ns.route(\"/remove_shared_agent\")\nclass RemoveSharedAgent(Resource):\n @api.doc(\n params={\"id\": \"ID of the shared agent\"},\n description=\"Remove a shared agent from the current user's shared list\",\n )\n def delete(self):\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False}), 401)\n user_id = decoded_token.get(\"sub\")\n agent_id = request.args.get(\"id\")\n\n if not agent_id:\n return make_response(\n jsonify({\"success\": False, \"message\": \"ID is required\"}), 400\n )\n try:\n agent = agents_collection.find_one(\n {\"_id\": ObjectId(agent_id), \"shared_publicly\": True}\n )\n if not agent:\n return make_response(\n jsonify({\"success\": False, \"message\": \"Shared agent not found\"}),\n 404,\n )\n ensure_user_doc(user_id)\n users_collection.update_one(\n {\"user_id\": user_id},\n {\n \"$pull\": {\n \"agent_preferences.shared_with_me\": agent_id,\n \"agent_preferences.pinned\": agent_id,\n }\n },\n )\n\n return make_response(jsonify({\"success\": True, \"action\": \"removed\"}), 200)\n except Exception as err:\n current_app.logger.error(f\"Error removing shared agent: {err}\")\n return make_response(\n jsonify({\"success\": False, \"message\": \"Server error\"}), 500\n )\n" }, { "path": "application/api/user/agents/sharing.py", "content": "\"\"\"Agent management sharing functionality.\"\"\"\n\nimport datetime\nimport secrets\n\nfrom bson import DBRef\nfrom bson.objectid import ObjectId\nfrom flask import current_app, jsonify, make_response, request\nfrom flask_restx import fields, Namespace, Resource\n\nfrom application.api import api\nfrom application.core.settings import settings\nfrom application.api.user.base import (\n agents_collection,\n db,\n ensure_user_doc,\n resolve_tool_details,\n user_tools_collection,\n users_collection,\n)\nfrom application.utils import generate_image_url\n\nagents_sharing_ns = Namespace(\n \"agents\", description=\"Agent management operations\", path=\"/api\"\n)\n\n\n@agents_sharing_ns.route(\"/shared_agent\")\nclass SharedAgent(Resource):\n @api.doc(\n params={\n \"token\": \"Shared token of the agent\",\n },\n description=\"Get a shared agent by token or ID\",\n )\n def get(self):\n shared_token = request.args.get(\"token\")\n\n if not shared_token:\n return make_response(\n jsonify({\"success\": False, \"message\": \"Token or ID is required\"}), 400\n )\n try:\n query = {\n \"shared_publicly\": True,\n \"shared_token\": shared_token,\n }\n shared_agent = agents_collection.find_one(query)\n if not shared_agent:\n return make_response(\n jsonify({\"success\": False, \"message\": \"Shared agent not found\"}),\n 404,\n )\n agent_id = str(shared_agent[\"_id\"])\n data = {\n \"id\": agent_id,\n \"user\": shared_agent.get(\"user\", \"\"),\n \"name\": shared_agent.get(\"name\", \"\"),\n \"image\": (\n generate_image_url(shared_agent[\"image\"])\n if shared_agent.get(\"image\")\n else \"\"\n ),\n \"description\": shared_agent.get(\"description\", \"\"),\n \"source\": (\n str(source_doc[\"_id\"])\n if isinstance(shared_agent.get(\"source\"), DBRef)\n and (source_doc := db.dereference(shared_agent.get(\"source\")))\n else \"\"\n ),\n \"chunks\": shared_agent.get(\"chunks\", \"0\"),\n \"retriever\": shared_agent.get(\"retriever\", \"classic\"),\n \"prompt_id\": shared_agent.get(\"prompt_id\", \"default\"),\n \"tools\": shared_agent.get(\"tools\", []),\n \"tool_details\": resolve_tool_details(shared_agent.get(\"tools\", [])),\n \"agent_type\": shared_agent.get(\"agent_type\", \"\"),\n \"status\": shared_agent.get(\"status\", \"\"),\n \"json_schema\": shared_agent.get(\"json_schema\"),\n \"limited_token_mode\": shared_agent.get(\"limited_token_mode\", False),\n \"token_limit\": shared_agent.get(\"token_limit\", settings.DEFAULT_AGENT_LIMITS[\"token_limit\"]),\n \"limited_request_mode\": shared_agent.get(\"limited_request_mode\", False),\n \"request_limit\": shared_agent.get(\"request_limit\", settings.DEFAULT_AGENT_LIMITS[\"request_limit\"]),\n \"created_at\": shared_agent.get(\"createdAt\", \"\"),\n \"updated_at\": shared_agent.get(\"updatedAt\", \"\"),\n \"shared\": shared_agent.get(\"shared_publicly\", False),\n \"shared_token\": shared_agent.get(\"shared_token\", \"\"),\n \"shared_metadata\": shared_agent.get(\"shared_metadata\", {}),\n }\n\n if data[\"tools\"]:\n enriched_tools = []\n for tool in data[\"tools\"]:\n tool_data = user_tools_collection.find_one({\"_id\": ObjectId(tool)})\n if tool_data:\n enriched_tools.append(tool_data.get(\"name\", \"\"))\n data[\"tools\"] = enriched_tools\n decoded_token = getattr(request, \"decoded_token\", None)\n if decoded_token:\n user_id = decoded_token.get(\"sub\")\n owner_id = shared_agent.get(\"user\")\n\n if user_id != owner_id:\n ensure_user_doc(user_id)\n users_collection.update_one(\n {\"user_id\": user_id},\n {\"$addToSet\": {\"agent_preferences.shared_with_me\": agent_id}},\n )\n return make_response(jsonify(data), 200)\n except Exception as err:\n current_app.logger.error(f\"Error retrieving shared agent: {err}\")\n return make_response(jsonify({\"success\": False}), 400)\n\n\n@agents_sharing_ns.route(\"/shared_agents\")\nclass SharedAgents(Resource):\n @api.doc(description=\"Get shared agents explicitly shared with the user\")\n def get(self):\n try:\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False}), 401)\n user_id = decoded_token.get(\"sub\")\n\n user_doc = ensure_user_doc(user_id)\n shared_with_ids = user_doc.get(\"agent_preferences\", {}).get(\n \"shared_with_me\", []\n )\n shared_object_ids = [ObjectId(id) for id in shared_with_ids]\n\n shared_agents_cursor = agents_collection.find(\n {\"_id\": {\"$in\": shared_object_ids}, \"shared_publicly\": True}\n )\n shared_agents = list(shared_agents_cursor)\n\n found_ids_set = {str(agent[\"_id\"]) for agent in shared_agents}\n stale_ids = [id for id in shared_with_ids if id not in found_ids_set]\n if stale_ids:\n users_collection.update_one(\n {\"user_id\": user_id},\n {\"$pullAll\": {\"agent_preferences.shared_with_me\": stale_ids}},\n )\n pinned_ids = set(user_doc.get(\"agent_preferences\", {}).get(\"pinned\", []))\n\n list_shared_agents = [\n {\n \"id\": str(agent[\"_id\"]),\n \"name\": agent.get(\"name\", \"\"),\n \"description\": agent.get(\"description\", \"\"),\n \"image\": (\n generate_image_url(agent[\"image\"]) if agent.get(\"image\") else \"\"\n ),\n \"tools\": agent.get(\"tools\", []),\n \"tool_details\": resolve_tool_details(agent.get(\"tools\", [])),\n \"agent_type\": agent.get(\"agent_type\", \"\"),\n \"status\": agent.get(\"status\", \"\"),\n \"json_schema\": agent.get(\"json_schema\"),\n \"limited_token_mode\": agent.get(\"limited_token_mode\", False),\n \"token_limit\": agent.get(\"token_limit\", settings.DEFAULT_AGENT_LIMITS[\"token_limit\"]),\n \"limited_request_mode\": agent.get(\"limited_request_mode\", False),\n \"request_limit\": agent.get(\"request_limit\", settings.DEFAULT_AGENT_LIMITS[\"request_limit\"]),\n \"created_at\": agent.get(\"createdAt\", \"\"),\n \"updated_at\": agent.get(\"updatedAt\", \"\"),\n \"pinned\": str(agent[\"_id\"]) in pinned_ids,\n \"shared\": agent.get(\"shared_publicly\", False),\n \"shared_token\": agent.get(\"shared_token\", \"\"),\n \"shared_metadata\": agent.get(\"shared_metadata\", {}),\n }\n for agent in shared_agents\n ]\n\n return make_response(jsonify(list_shared_agents), 200)\n except Exception as err:\n current_app.logger.error(f\"Error retrieving shared agents: {err}\")\n return make_response(jsonify({\"success\": False}), 400)\n\n\n@agents_sharing_ns.route(\"/share_agent\")\nclass ShareAgent(Resource):\n @api.expect(\n api.model(\n \"ShareAgentModel\",\n {\n \"id\": fields.String(required=True, description=\"ID of the agent\"),\n \"shared\": fields.Boolean(\n required=True, description=\"Share or unshare the agent\"\n ),\n \"username\": fields.String(\n required=False, description=\"Name of the user\"\n ),\n },\n )\n )\n @api.doc(description=\"Share or unshare an agent\")\n def put(self):\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False}), 401)\n user = decoded_token.get(\"sub\")\n\n data = request.get_json()\n if not data:\n return make_response(\n jsonify({\"success\": False, \"message\": \"Missing JSON body\"}), 400\n )\n agent_id = data.get(\"id\")\n shared = data.get(\"shared\")\n username = data.get(\"username\", \"\")\n\n if not agent_id:\n return make_response(\n jsonify({\"success\": False, \"message\": \"ID is required\"}), 400\n )\n if shared is None:\n return make_response(\n jsonify(\n {\n \"success\": False,\n \"message\": \"Shared parameter is required and must be true or false\",\n }\n ),\n 400,\n )\n try:\n try:\n agent_oid = ObjectId(agent_id)\n except Exception:\n return make_response(\n jsonify({\"success\": False, \"message\": \"Invalid agent ID\"}), 400\n )\n agent = agents_collection.find_one({\"_id\": agent_oid, \"user\": user})\n if not agent:\n return make_response(\n jsonify({\"success\": False, \"message\": \"Agent not found\"}), 404\n )\n if shared:\n shared_metadata = {\n \"shared_by\": username,\n \"shared_at\": datetime.datetime.now(datetime.timezone.utc),\n }\n shared_token = secrets.token_urlsafe(32)\n agents_collection.update_one(\n {\"_id\": agent_oid, \"user\": user},\n {\n \"$set\": {\n \"shared_publicly\": shared,\n \"shared_metadata\": shared_metadata,\n \"shared_token\": shared_token,\n }\n },\n )\n else:\n agents_collection.update_one(\n {\"_id\": agent_oid, \"user\": user},\n {\"$set\": {\"shared_publicly\": shared, \"shared_token\": None}},\n {\"$unset\": {\"shared_metadata\": \"\"}},\n )\n except Exception as err:\n current_app.logger.error(f\"Error sharing/unsharing agent: {err}\", exc_info=True)\n return make_response(jsonify({\"success\": False, \"error\": \"Failed to update agent sharing status\"}), 400)\n shared_token = shared_token if shared else None\n return make_response(\n jsonify({\"success\": True, \"shared_token\": shared_token}), 200\n )\n" }, { "path": "application/api/user/agents/webhooks.py", "content": "\"\"\"Agent management webhook handlers.\"\"\"\n\nimport secrets\n\nfrom bson.objectid import ObjectId\nfrom flask import current_app, jsonify, make_response, request\nfrom flask_restx import Namespace, Resource\n\nfrom application.api import api\nfrom application.api.user.base import agents_collection, require_agent\nfrom application.api.user.tasks import process_agent_webhook\nfrom application.core.settings import settings\n\n\nagents_webhooks_ns = Namespace(\n \"agents\", description=\"Agent management operations\", path=\"/api\"\n)\n\n\n@agents_webhooks_ns.route(\"/agent_webhook\")\nclass AgentWebhook(Resource):\n @api.doc(\n params={\"id\": \"ID of the agent\"},\n description=\"Generate webhook URL for the agent\",\n )\n def get(self):\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False}), 401)\n user = decoded_token.get(\"sub\")\n agent_id = request.args.get(\"id\")\n if not agent_id:\n return make_response(\n jsonify({\"success\": False, \"message\": \"ID is required\"}), 400\n )\n try:\n agent = agents_collection.find_one(\n {\"_id\": ObjectId(agent_id), \"user\": user}\n )\n if not agent:\n return make_response(\n jsonify({\"success\": False, \"message\": \"Agent not found\"}), 404\n )\n webhook_token = agent.get(\"incoming_webhook_token\")\n if not webhook_token:\n webhook_token = secrets.token_urlsafe(32)\n agents_collection.update_one(\n {\"_id\": ObjectId(agent_id), \"user\": user},\n {\"$set\": {\"incoming_webhook_token\": webhook_token}},\n )\n base_url = settings.API_URL.rstrip(\"/\")\n full_webhook_url = f\"{base_url}/api/webhooks/agents/{webhook_token}\"\n except Exception as err:\n current_app.logger.error(\n f\"Error generating webhook URL: {err}\", exc_info=True\n )\n return make_response(\n jsonify({\"success\": False, \"message\": \"Error generating webhook URL\"}),\n 400,\n )\n return make_response(\n jsonify({\"success\": True, \"webhook_url\": full_webhook_url}), 200\n )\n\n\n@agents_webhooks_ns.route(\"/webhooks/agents/\")\nclass AgentWebhookListener(Resource):\n method_decorators = [require_agent]\n\n def _enqueue_webhook_task(self, agent_id_str, payload, source_method):\n if not payload:\n current_app.logger.warning(\n f\"Webhook ({source_method}) received for agent {agent_id_str} with empty payload.\"\n )\n current_app.logger.info(\n f\"Incoming {source_method} webhook for agent {agent_id_str}. Enqueuing task with payload: {payload}\"\n )\n\n try:\n task = process_agent_webhook.delay(\n agent_id=agent_id_str,\n payload=payload,\n )\n current_app.logger.info(\n f\"Task {task.id} enqueued for agent {agent_id_str} ({source_method}).\"\n )\n return make_response(jsonify({\"success\": True, \"task_id\": task.id}), 200)\n except Exception as err:\n current_app.logger.error(\n f\"Error enqueuing webhook task ({source_method}) for agent {agent_id_str}: {err}\",\n exc_info=True,\n )\n return make_response(\n jsonify({\"success\": False, \"message\": \"Error processing webhook\"}), 500\n )\n\n @api.doc(\n description=\"Webhook listener for agent events (POST). Expects JSON payload, which is used to trigger processing.\",\n )\n def post(self, webhook_token, agent, agent_id_str):\n payload = request.get_json()\n if payload is None:\n return make_response(\n jsonify(\n {\n \"success\": False,\n \"message\": \"Invalid or missing JSON data in request body\",\n }\n ),\n 400,\n )\n return self._enqueue_webhook_task(agent_id_str, payload, source_method=\"POST\")\n\n @api.doc(\n description=\"Webhook listener for agent events (GET). Uses URL query parameters as payload to trigger processing.\",\n )\n def get(self, webhook_token, agent, agent_id_str):\n payload = request.args.to_dict(flat=True)\n return self._enqueue_webhook_task(agent_id_str, payload, source_method=\"GET\")\n" }, { "path": "application/api/user/analytics/__init__.py", "content": "\"\"\"Analytics module.\"\"\"\n\nfrom .routes import analytics_ns\n\n__all__ = [\"analytics_ns\"]\n" }, { "path": "application/api/user/analytics/routes.py", "content": "\"\"\"Analytics and reporting routes.\"\"\"\n\nimport datetime\n\nfrom bson.objectid import ObjectId\nfrom flask import current_app, jsonify, make_response, request\nfrom flask_restx import fields, Namespace, Resource\n\nfrom application.api import api\nfrom application.api.user.base import (\n agents_collection,\n conversations_collection,\n generate_date_range,\n generate_hourly_range,\n generate_minute_range,\n token_usage_collection,\n user_logs_collection,\n)\n\nanalytics_ns = Namespace(\n \"analytics\", description=\"Analytics and reporting operations\", path=\"/api\"\n)\n\n\n@analytics_ns.route(\"/get_message_analytics\")\nclass GetMessageAnalytics(Resource):\n get_message_analytics_model = api.model(\n \"GetMessageAnalyticsModel\",\n {\n \"api_key_id\": fields.String(required=False, description=\"API Key ID\"),\n \"filter_option\": fields.String(\n required=False,\n description=\"Filter option for analytics\",\n default=\"last_30_days\",\n enum=[\n \"last_hour\",\n \"last_24_hour\",\n \"last_7_days\",\n \"last_15_days\",\n \"last_30_days\",\n ],\n ),\n },\n )\n\n @api.expect(get_message_analytics_model)\n @api.doc(description=\"Get message analytics based on filter option\")\n def post(self):\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False}), 401)\n user = decoded_token.get(\"sub\")\n data = request.get_json()\n api_key_id = data.get(\"api_key_id\")\n filter_option = data.get(\"filter_option\", \"last_30_days\")\n\n try:\n api_key = (\n agents_collection.find_one({\"_id\": ObjectId(api_key_id), \"user\": user})[\n \"key\"\n ]\n if api_key_id\n else None\n )\n except Exception as err:\n current_app.logger.error(f\"Error getting API key: {err}\", exc_info=True)\n return make_response(jsonify({\"success\": False}), 400)\n end_date = datetime.datetime.now(datetime.timezone.utc)\n\n if filter_option == \"last_hour\":\n start_date = end_date - datetime.timedelta(hours=1)\n group_format = \"%Y-%m-%d %H:%M:00\"\n elif filter_option == \"last_24_hour\":\n start_date = end_date - datetime.timedelta(hours=24)\n group_format = \"%Y-%m-%d %H:00\"\n else:\n if filter_option in [\"last_7_days\", \"last_15_days\", \"last_30_days\"]:\n filter_days = (\n 6\n if filter_option == \"last_7_days\"\n else 14 if filter_option == \"last_15_days\" else 29\n )\n else:\n return make_response(\n jsonify({\"success\": False, \"message\": \"Invalid option\"}), 400\n )\n start_date = end_date - datetime.timedelta(days=filter_days)\n start_date = start_date.replace(hour=0, minute=0, second=0, microsecond=0)\n end_date = end_date.replace(\n hour=23, minute=59, second=59, microsecond=999999\n )\n group_format = \"%Y-%m-%d\"\n try:\n match_stage = {\n \"$match\": {\n \"user\": user,\n }\n }\n if api_key:\n match_stage[\"$match\"][\"api_key\"] = api_key\n pipeline = [\n match_stage,\n {\"$unwind\": \"$queries\"},\n {\n \"$match\": {\n \"queries.timestamp\": {\"$gte\": start_date, \"$lte\": end_date}\n }\n },\n {\n \"$group\": {\n \"_id\": {\n \"$dateToString\": {\n \"format\": group_format,\n \"date\": \"$queries.timestamp\",\n }\n },\n \"count\": {\"$sum\": 1},\n }\n },\n {\"$sort\": {\"_id\": 1}},\n ]\n\n message_data = conversations_collection.aggregate(pipeline)\n\n if filter_option == \"last_hour\":\n intervals = generate_minute_range(start_date, end_date)\n elif filter_option == \"last_24_hour\":\n intervals = generate_hourly_range(start_date, end_date)\n else:\n intervals = generate_date_range(start_date, end_date)\n daily_messages = {interval: 0 for interval in intervals}\n\n for entry in message_data:\n daily_messages[entry[\"_id\"]] = entry[\"count\"]\n except Exception as err:\n current_app.logger.error(\n f\"Error getting message analytics: {err}\", exc_info=True\n )\n return make_response(jsonify({\"success\": False}), 400)\n return make_response(\n jsonify({\"success\": True, \"messages\": daily_messages}), 200\n )\n\n\n@analytics_ns.route(\"/get_token_analytics\")\nclass GetTokenAnalytics(Resource):\n get_token_analytics_model = api.model(\n \"GetTokenAnalyticsModel\",\n {\n \"api_key_id\": fields.String(required=False, description=\"API Key ID\"),\n \"filter_option\": fields.String(\n required=False,\n description=\"Filter option for analytics\",\n default=\"last_30_days\",\n enum=[\n \"last_hour\",\n \"last_24_hour\",\n \"last_7_days\",\n \"last_15_days\",\n \"last_30_days\",\n ],\n ),\n },\n )\n\n @api.expect(get_token_analytics_model)\n @api.doc(description=\"Get token analytics data\")\n def post(self):\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False}), 401)\n user = decoded_token.get(\"sub\")\n data = request.get_json()\n api_key_id = data.get(\"api_key_id\")\n filter_option = data.get(\"filter_option\", \"last_30_days\")\n\n try:\n api_key = (\n agents_collection.find_one({\"_id\": ObjectId(api_key_id), \"user\": user})[\n \"key\"\n ]\n if api_key_id\n else None\n )\n except Exception as err:\n current_app.logger.error(f\"Error getting API key: {err}\", exc_info=True)\n return make_response(jsonify({\"success\": False}), 400)\n end_date = datetime.datetime.now(datetime.timezone.utc)\n\n if filter_option == \"last_hour\":\n start_date = end_date - datetime.timedelta(hours=1)\n group_format = \"%Y-%m-%d %H:%M:00\"\n group_stage = {\n \"$group\": {\n \"_id\": {\n \"minute\": {\n \"$dateToString\": {\n \"format\": group_format,\n \"date\": \"$timestamp\",\n }\n }\n },\n \"total_tokens\": {\n \"$sum\": {\"$add\": [\"$prompt_tokens\", \"$generated_tokens\"]}\n },\n }\n }\n elif filter_option == \"last_24_hour\":\n start_date = end_date - datetime.timedelta(hours=24)\n group_format = \"%Y-%m-%d %H:00\"\n group_stage = {\n \"$group\": {\n \"_id\": {\n \"hour\": {\n \"$dateToString\": {\n \"format\": group_format,\n \"date\": \"$timestamp\",\n }\n }\n },\n \"total_tokens\": {\n \"$sum\": {\"$add\": [\"$prompt_tokens\", \"$generated_tokens\"]}\n },\n }\n }\n else:\n if filter_option in [\"last_7_days\", \"last_15_days\", \"last_30_days\"]:\n filter_days = (\n 6\n if filter_option == \"last_7_days\"\n else (14 if filter_option == \"last_15_days\" else 29)\n )\n else:\n return make_response(\n jsonify({\"success\": False, \"message\": \"Invalid option\"}), 400\n )\n start_date = end_date - datetime.timedelta(days=filter_days)\n start_date = start_date.replace(hour=0, minute=0, second=0, microsecond=0)\n end_date = end_date.replace(\n hour=23, minute=59, second=59, microsecond=999999\n )\n group_format = \"%Y-%m-%d\"\n group_stage = {\n \"$group\": {\n \"_id\": {\n \"day\": {\n \"$dateToString\": {\n \"format\": group_format,\n \"date\": \"$timestamp\",\n }\n }\n },\n \"total_tokens\": {\n \"$sum\": {\"$add\": [\"$prompt_tokens\", \"$generated_tokens\"]}\n },\n }\n }\n try:\n match_stage = {\n \"$match\": {\n \"user_id\": user,\n \"timestamp\": {\"$gte\": start_date, \"$lte\": end_date},\n }\n }\n if api_key:\n match_stage[\"$match\"][\"api_key\"] = api_key\n token_usage_data = token_usage_collection.aggregate(\n [\n match_stage,\n group_stage,\n {\"$sort\": {\"_id\": 1}},\n ]\n )\n\n if filter_option == \"last_hour\":\n intervals = generate_minute_range(start_date, end_date)\n elif filter_option == \"last_24_hour\":\n intervals = generate_hourly_range(start_date, end_date)\n else:\n intervals = generate_date_range(start_date, end_date)\n daily_token_usage = {interval: 0 for interval in intervals}\n\n for entry in token_usage_data:\n if filter_option == \"last_hour\":\n daily_token_usage[entry[\"_id\"][\"minute\"]] = entry[\"total_tokens\"]\n elif filter_option == \"last_24_hour\":\n daily_token_usage[entry[\"_id\"][\"hour\"]] = entry[\"total_tokens\"]\n else:\n daily_token_usage[entry[\"_id\"][\"day\"]] = entry[\"total_tokens\"]\n except Exception as err:\n current_app.logger.error(\n f\"Error getting token analytics: {err}\", exc_info=True\n )\n return make_response(jsonify({\"success\": False}), 400)\n return make_response(\n jsonify({\"success\": True, \"token_usage\": daily_token_usage}), 200\n )\n\n\n@analytics_ns.route(\"/get_feedback_analytics\")\nclass GetFeedbackAnalytics(Resource):\n get_feedback_analytics_model = api.model(\n \"GetFeedbackAnalyticsModel\",\n {\n \"api_key_id\": fields.String(required=False, description=\"API Key ID\"),\n \"filter_option\": fields.String(\n required=False,\n description=\"Filter option for analytics\",\n default=\"last_30_days\",\n enum=[\n \"last_hour\",\n \"last_24_hour\",\n \"last_7_days\",\n \"last_15_days\",\n \"last_30_days\",\n ],\n ),\n },\n )\n\n @api.expect(get_feedback_analytics_model)\n @api.doc(description=\"Get feedback analytics data\")\n def post(self):\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False}), 401)\n user = decoded_token.get(\"sub\")\n data = request.get_json()\n api_key_id = data.get(\"api_key_id\")\n filter_option = data.get(\"filter_option\", \"last_30_days\")\n\n try:\n api_key = (\n agents_collection.find_one({\"_id\": ObjectId(api_key_id), \"user\": user})[\n \"key\"\n ]\n if api_key_id\n else None\n )\n except Exception as err:\n current_app.logger.error(f\"Error getting API key: {err}\", exc_info=True)\n return make_response(jsonify({\"success\": False}), 400)\n end_date = datetime.datetime.now(datetime.timezone.utc)\n\n if filter_option == \"last_hour\":\n start_date = end_date - datetime.timedelta(hours=1)\n group_format = \"%Y-%m-%d %H:%M:00\"\n date_field = {\n \"$dateToString\": {\n \"format\": group_format,\n \"date\": \"$queries.feedback_timestamp\",\n }\n }\n elif filter_option == \"last_24_hour\":\n start_date = end_date - datetime.timedelta(hours=24)\n group_format = \"%Y-%m-%d %H:00\"\n date_field = {\n \"$dateToString\": {\n \"format\": group_format,\n \"date\": \"$queries.feedback_timestamp\",\n }\n }\n else:\n if filter_option in [\"last_7_days\", \"last_15_days\", \"last_30_days\"]:\n filter_days = (\n 6\n if filter_option == \"last_7_days\"\n else (14 if filter_option == \"last_15_days\" else 29)\n )\n else:\n return make_response(\n jsonify({\"success\": False, \"message\": \"Invalid option\"}), 400\n )\n start_date = end_date - datetime.timedelta(days=filter_days)\n start_date = start_date.replace(hour=0, minute=0, second=0, microsecond=0)\n end_date = end_date.replace(\n hour=23, minute=59, second=59, microsecond=999999\n )\n group_format = \"%Y-%m-%d\"\n date_field = {\n \"$dateToString\": {\n \"format\": group_format,\n \"date\": \"$queries.feedback_timestamp\",\n }\n }\n try:\n match_stage = {\n \"$match\": {\n \"queries.feedback_timestamp\": {\n \"$gte\": start_date,\n \"$lte\": end_date,\n },\n \"queries.feedback\": {\"$exists\": True},\n }\n }\n if api_key:\n match_stage[\"$match\"][\"api_key\"] = api_key\n pipeline = [\n match_stage,\n {\"$unwind\": \"$queries\"},\n {\"$match\": {\"queries.feedback\": {\"$exists\": True}}},\n {\n \"$group\": {\n \"_id\": {\"time\": date_field, \"feedback\": \"$queries.feedback\"},\n \"count\": {\"$sum\": 1},\n }\n },\n {\n \"$group\": {\n \"_id\": \"$_id.time\",\n \"positive\": {\n \"$sum\": {\n \"$cond\": [\n {\"$eq\": [\"$_id.feedback\", \"LIKE\"]},\n \"$count\",\n 0,\n ]\n }\n },\n \"negative\": {\n \"$sum\": {\n \"$cond\": [\n {\"$eq\": [\"$_id.feedback\", \"DISLIKE\"]},\n \"$count\",\n 0,\n ]\n }\n },\n }\n },\n {\"$sort\": {\"_id\": 1}},\n ]\n\n feedback_data = conversations_collection.aggregate(pipeline)\n\n if filter_option == \"last_hour\":\n intervals = generate_minute_range(start_date, end_date)\n elif filter_option == \"last_24_hour\":\n intervals = generate_hourly_range(start_date, end_date)\n else:\n intervals = generate_date_range(start_date, end_date)\n daily_feedback = {\n interval: {\"positive\": 0, \"negative\": 0} for interval in intervals\n }\n\n for entry in feedback_data:\n daily_feedback[entry[\"_id\"]] = {\n \"positive\": entry[\"positive\"],\n \"negative\": entry[\"negative\"],\n }\n except Exception as err:\n current_app.logger.error(\n f\"Error getting feedback analytics: {err}\", exc_info=True\n )\n return make_response(jsonify({\"success\": False}), 400)\n return make_response(\n jsonify({\"success\": True, \"feedback\": daily_feedback}), 200\n )\n\n\n@analytics_ns.route(\"/get_user_logs\")\nclass GetUserLogs(Resource):\n get_user_logs_model = api.model(\n \"GetUserLogsModel\",\n {\n \"page\": fields.Integer(\n required=False,\n description=\"Page number for pagination\",\n default=1,\n ),\n \"api_key_id\": fields.String(required=False, description=\"API Key ID\"),\n \"page_size\": fields.Integer(\n required=False,\n description=\"Number of logs per page\",\n default=10,\n ),\n },\n )\n\n @api.expect(get_user_logs_model)\n @api.doc(description=\"Get user logs with pagination\")\n def post(self):\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False}), 401)\n user = decoded_token.get(\"sub\")\n data = request.get_json()\n page = int(data.get(\"page\", 1))\n api_key_id = data.get(\"api_key_id\")\n page_size = int(data.get(\"page_size\", 10))\n skip = (page - 1) * page_size\n\n try:\n api_key = (\n agents_collection.find_one({\"_id\": ObjectId(api_key_id)})[\"key\"]\n if api_key_id\n else None\n )\n except Exception as err:\n current_app.logger.error(f\"Error getting API key: {err}\", exc_info=True)\n return make_response(jsonify({\"success\": False}), 400)\n query = {\"user\": user}\n if api_key:\n query = {\"api_key\": api_key}\n items_cursor = (\n user_logs_collection.find(query)\n .sort(\"timestamp\", -1)\n .skip(skip)\n .limit(page_size + 1)\n )\n items = list(items_cursor)\n\n results = [\n {\n \"id\": str(item.get(\"_id\")),\n \"action\": item.get(\"action\"),\n \"level\": item.get(\"level\"),\n \"user\": item.get(\"user\"),\n \"question\": item.get(\"question\"),\n \"sources\": item.get(\"sources\"),\n \"retriever_params\": item.get(\"retriever_params\"),\n \"timestamp\": item.get(\"timestamp\"),\n }\n for item in items[:page_size]\n ]\n\n has_more = len(items) > page_size\n\n return make_response(\n jsonify(\n {\n \"success\": True,\n \"logs\": results,\n \"page\": page,\n \"page_size\": page_size,\n \"has_more\": has_more,\n }\n ),\n 200,\n )\n" }, { "path": "application/api/user/attachments/__init__.py", "content": "\"\"\"Attachments module.\"\"\"\n\nfrom .routes import attachments_ns\n\n__all__ = [\"attachments_ns\"]\n" }, { "path": "application/api/user/attachments/routes.py", "content": "\"\"\"File attachments and media routes.\"\"\"\n\nimport os\nimport tempfile\nfrom pathlib import Path\n\nfrom bson.objectid import ObjectId\nfrom flask import current_app, jsonify, make_response, request\nfrom flask_restx import fields, Namespace, Resource\n\nfrom application.api import api\nfrom application.cache import get_redis_instance\nfrom application.core.settings import settings\nfrom application.stt.constants import (\n SUPPORTED_AUDIO_EXTENSIONS,\n SUPPORTED_AUDIO_MIME_TYPES,\n)\nfrom application.stt.upload_limits import (\n AudioFileTooLargeError,\n build_stt_file_size_limit_message,\n enforce_audio_file_size_limit,\n is_audio_filename,\n)\nfrom application.stt.live_session import (\n apply_live_stt_hypothesis,\n create_live_stt_session,\n delete_live_stt_session,\n finalize_live_stt_session,\n get_live_stt_transcript_text,\n load_live_stt_session,\n save_live_stt_session,\n)\nfrom application.stt.stt_creator import STTCreator\nfrom application.tts.tts_creator import TTSCreator\nfrom application.utils import safe_filename\n\n\nattachments_ns = Namespace(\n \"attachments\", description=\"File attachments and media operations\", path=\"/api\"\n)\n\n\ndef _resolve_authenticated_user():\n decoded_token = getattr(request, \"decoded_token\", None)\n api_key = request.form.get(\"api_key\") or request.args.get(\"api_key\")\n\n if decoded_token:\n return safe_filename(decoded_token.get(\"sub\"))\n\n if api_key:\n from application.api.user.base import agents_collection\n\n agent = agents_collection.find_one({\"key\": api_key})\n if not agent:\n return make_response(\n jsonify({\"success\": False, \"message\": \"Invalid API key\"}), 401\n )\n return safe_filename(agent.get(\"user\"))\n\n return None\n\n\ndef _get_uploaded_file_size(file) -> int:\n try:\n current_position = file.stream.tell()\n file.stream.seek(0, os.SEEK_END)\n size_bytes = file.stream.tell()\n file.stream.seek(current_position)\n return size_bytes\n except Exception:\n return 0\n\n\ndef _is_supported_audio_mimetype(mimetype: str) -> bool:\n if not mimetype:\n return True\n normalized = mimetype.split(\";\")[0].strip().lower()\n return normalized.startswith(\"audio/\") or normalized in SUPPORTED_AUDIO_MIME_TYPES\n\n\ndef _enforce_uploaded_audio_size_limit(file, filename: str) -> None:\n if not is_audio_filename(filename):\n return\n size_bytes = _get_uploaded_file_size(file)\n if size_bytes:\n enforce_audio_file_size_limit(size_bytes)\n\n\ndef _get_store_attachment_user_error(exc: Exception) -> str:\n if isinstance(exc, AudioFileTooLargeError):\n return build_stt_file_size_limit_message()\n return \"Failed to process file\"\n\n\ndef _require_live_stt_redis():\n redis_client = get_redis_instance()\n if redis_client:\n return redis_client\n return make_response(\n jsonify({\"success\": False, \"message\": \"Live transcription is unavailable\"}),\n 503,\n )\n\n\ndef _parse_bool_form_value(value: str | None) -> bool:\n if value is None:\n return False\n return value.strip().lower() in {\"1\", \"true\", \"yes\", \"on\"}\n\n\n@attachments_ns.route(\"/store_attachment\")\nclass StoreAttachment(Resource):\n @api.expect(\n api.model(\n \"AttachmentModel\",\n {\n \"file\": fields.Raw(required=True, description=\"File(s) to upload\"),\n \"api_key\": fields.String(\n required=False, description=\"API key (optional)\"\n ),\n },\n )\n )\n @api.doc(\n description=\"Stores one or multiple attachments without vectorization or training. Supports user or API key authentication.\"\n )\n def post(self):\n auth_user = _resolve_authenticated_user()\n if hasattr(auth_user, \"status_code\"):\n return auth_user\n \n files = request.files.getlist(\"file\")\n if not files:\n single_file = request.files.get(\"file\")\n if single_file:\n files = [single_file]\n \n if not files or all(f.filename == \"\" for f in files):\n return make_response(\n jsonify({\"status\": \"error\", \"message\": \"Missing file(s)\"}),\n 400,\n )\n \n user = auth_user\n if not user:\n return make_response(\n jsonify({\"success\": False, \"message\": \"Authentication required\"}), 401\n )\n \n try:\n from application.api.user.tasks import store_attachment\n from application.api.user.base import storage\n\n tasks = []\n errors = []\n original_file_count = len(files)\n \n for idx, file in enumerate(files):\n try:\n attachment_id = ObjectId()\n original_filename = safe_filename(os.path.basename(file.filename))\n _enforce_uploaded_audio_size_limit(file, original_filename)\n relative_path = f\"{settings.UPLOAD_FOLDER}/{user}/attachments/{str(attachment_id)}/{original_filename}\"\n\n metadata = storage.save_file(file, relative_path)\n file_info = {\n \"filename\": original_filename,\n \"attachment_id\": str(attachment_id),\n \"path\": relative_path,\n \"metadata\": metadata,\n }\n\n task = store_attachment.delay(file_info, user)\n tasks.append({\n \"task_id\": task.id,\n \"filename\": original_filename,\n \"attachment_id\": str(attachment_id),\n \"upload_index\": idx,\n })\n except Exception as file_err:\n current_app.logger.error(f\"Error processing file {idx} ({file.filename}): {file_err}\", exc_info=True)\n errors.append({\n \"upload_index\": idx,\n \"filename\": file.filename,\n \"error\": _get_store_attachment_user_error(file_err),\n })\n \n if not tasks:\n if errors and all(\n error.get(\"error\") == build_stt_file_size_limit_message()\n for error in errors\n ):\n return make_response(\n jsonify(\n {\n \"success\": False,\n \"message\": build_stt_file_size_limit_message(),\n \"errors\": errors,\n }\n ),\n 413,\n )\n return make_response(\n jsonify({\"status\": \"error\", \"message\": \"No valid files to upload\"}),\n 400,\n )\n \n if original_file_count == 1 and len(tasks) == 1:\n current_app.logger.info(\"Returning single task_id response\")\n return make_response(\n jsonify(\n {\n \"success\": True,\n \"task_id\": tasks[0][\"task_id\"],\n \"message\": \"File uploaded successfully. Processing started.\",\n }\n ),\n 200,\n )\n else:\n response_data = {\n \"success\": True,\n \"tasks\": tasks,\n \"message\": f\"{len(tasks)} file(s) uploaded successfully. Processing started.\",\n }\n if errors:\n response_data[\"errors\"] = errors\n response_data[\"message\"] += f\" {len(errors)} file(s) failed.\"\n \n return make_response(\n jsonify(response_data),\n 200,\n )\n except Exception as err:\n current_app.logger.error(f\"Error storing attachment: {err}\", exc_info=True)\n return make_response(jsonify({\"success\": False, \"error\": \"Failed to store attachment\"}), 400)\n\n\n@attachments_ns.route(\"/stt\")\nclass SpeechToText(Resource):\n @api.expect(\n api.model(\n \"SpeechToTextModel\",\n {\n \"file\": fields.Raw(required=True, description=\"Audio file\"),\n \"language\": fields.String(\n required=False, description=\"Optional transcription language hint\"\n ),\n },\n )\n )\n @api.doc(description=\"Transcribe an uploaded audio file\")\n def post(self):\n auth_user = _resolve_authenticated_user()\n if hasattr(auth_user, \"status_code\"):\n return auth_user\n if not auth_user:\n return make_response(\n jsonify({\"success\": False, \"message\": \"Authentication required\"}),\n 401,\n )\n\n file = request.files.get(\"file\")\n if not file or file.filename == \"\":\n return make_response(\n jsonify({\"success\": False, \"message\": \"Missing file\"}),\n 400,\n )\n\n filename = safe_filename(os.path.basename(file.filename))\n suffix = Path(filename).suffix.lower()\n if suffix not in SUPPORTED_AUDIO_EXTENSIONS:\n return make_response(\n jsonify({\"success\": False, \"message\": \"Unsupported audio format\"}),\n 400,\n )\n\n if not _is_supported_audio_mimetype(file.mimetype or \"\"):\n return make_response(\n jsonify({\"success\": False, \"message\": \"Unsupported audio MIME type\"}),\n 400,\n )\n\n try:\n _enforce_uploaded_audio_size_limit(file, filename)\n except AudioFileTooLargeError:\n return make_response(\n jsonify(\n {\n \"success\": False,\n \"message\": build_stt_file_size_limit_message(),\n }\n ),\n 413,\n )\n\n temp_path = None\n try:\n with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as temp_file:\n file.save(temp_file.name)\n temp_path = Path(temp_file.name)\n\n stt_instance = STTCreator.create_stt(settings.STT_PROVIDER)\n transcript = stt_instance.transcribe(\n temp_path,\n language=request.form.get(\"language\") or settings.STT_LANGUAGE,\n timestamps=settings.STT_ENABLE_TIMESTAMPS,\n diarize=settings.STT_ENABLE_DIARIZATION,\n )\n return make_response(jsonify({\"success\": True, **transcript}), 200)\n except Exception as err:\n current_app.logger.error(f\"Error transcribing audio: {err}\", exc_info=True)\n return make_response(\n jsonify({\"success\": False, \"message\": \"Failed to transcribe audio\"}),\n 400,\n )\n finally:\n if temp_path and temp_path.exists():\n temp_path.unlink()\n\n\n@attachments_ns.route(\"/stt/live/start\")\nclass LiveSpeechToTextStart(Resource):\n @api.doc(description=\"Start a live speech-to-text session\")\n def post(self):\n auth_user = _resolve_authenticated_user()\n if hasattr(auth_user, \"status_code\"):\n return auth_user\n if not auth_user:\n return make_response(\n jsonify({\"success\": False, \"message\": \"Authentication required\"}),\n 401,\n )\n\n redis_client = _require_live_stt_redis()\n if hasattr(redis_client, \"status_code\"):\n return redis_client\n\n payload = request.get_json(silent=True) or {}\n session_state = create_live_stt_session(\n user=auth_user,\n language=payload.get(\"language\") or settings.STT_LANGUAGE,\n )\n save_live_stt_session(redis_client, session_state)\n\n return make_response(\n jsonify(\n {\n \"success\": True,\n \"session_id\": session_state[\"session_id\"],\n \"language\": session_state.get(\"language\"),\n \"committed_text\": \"\",\n \"mutable_text\": \"\",\n \"previous_hypothesis\": \"\",\n \"latest_hypothesis\": \"\",\n \"finalized_text\": \"\",\n \"pending_text\": \"\",\n \"transcript_text\": \"\",\n }\n ),\n 200,\n )\n\n\n@attachments_ns.route(\"/stt/live/chunk\")\nclass LiveSpeechToTextChunk(Resource):\n @api.expect(\n api.model(\n \"LiveSpeechToTextChunkModel\",\n {\n \"session_id\": fields.String(\n required=True, description=\"Live transcription session ID\"\n ),\n \"chunk_index\": fields.Integer(\n required=True, description=\"Sequential chunk index\"\n ),\n \"is_silence\": fields.Boolean(\n required=False,\n description=\"Whether the latest capture window was mostly silence\",\n ),\n \"file\": fields.Raw(required=True, description=\"Audio chunk\"),\n },\n )\n )\n @api.doc(description=\"Transcribe a chunk for a live speech-to-text session\")\n def post(self):\n auth_user = _resolve_authenticated_user()\n if hasattr(auth_user, \"status_code\"):\n return auth_user\n if not auth_user:\n return make_response(\n jsonify({\"success\": False, \"message\": \"Authentication required\"}),\n 401,\n )\n\n redis_client = _require_live_stt_redis()\n if hasattr(redis_client, \"status_code\"):\n return redis_client\n\n session_id = request.form.get(\"session_id\", \"\").strip()\n if not session_id:\n return make_response(\n jsonify({\"success\": False, \"message\": \"Missing session_id\"}),\n 400,\n )\n\n session_state = load_live_stt_session(redis_client, session_id)\n if not session_state:\n return make_response(\n jsonify(\n {\n \"success\": False,\n \"message\": \"Live transcription session not found\",\n }\n ),\n 404,\n )\n\n if safe_filename(str(session_state.get(\"user\", \"\"))) != auth_user:\n return make_response(\n jsonify({\"success\": False, \"message\": \"Forbidden\"}),\n 403,\n )\n\n chunk_index_raw = request.form.get(\"chunk_index\", \"\").strip()\n if chunk_index_raw == \"\":\n return make_response(\n jsonify({\"success\": False, \"message\": \"Missing chunk_index\"}),\n 400,\n )\n\n try:\n chunk_index = int(chunk_index_raw)\n except ValueError:\n return make_response(\n jsonify({\"success\": False, \"message\": \"Invalid chunk_index\"}),\n 400,\n )\n is_silence = _parse_bool_form_value(request.form.get(\"is_silence\"))\n\n file = request.files.get(\"file\")\n if not file or file.filename == \"\":\n return make_response(\n jsonify({\"success\": False, \"message\": \"Missing file\"}),\n 400,\n )\n\n filename = safe_filename(os.path.basename(file.filename))\n suffix = Path(filename).suffix.lower()\n if suffix not in SUPPORTED_AUDIO_EXTENSIONS:\n return make_response(\n jsonify({\"success\": False, \"message\": \"Unsupported audio format\"}),\n 400,\n )\n\n if not _is_supported_audio_mimetype(file.mimetype or \"\"):\n return make_response(\n jsonify({\"success\": False, \"message\": \"Unsupported audio MIME type\"}),\n 400,\n )\n\n try:\n _enforce_uploaded_audio_size_limit(file, filename)\n except AudioFileTooLargeError:\n return make_response(\n jsonify(\n {\n \"success\": False,\n \"message\": build_stt_file_size_limit_message(),\n }\n ),\n 413,\n )\n\n temp_path = None\n try:\n with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as temp_file:\n file.save(temp_file.name)\n temp_path = Path(temp_file.name)\n\n session_language = session_state.get(\"language\") or settings.STT_LANGUAGE\n stt_instance = STTCreator.create_stt(settings.STT_PROVIDER)\n transcript = stt_instance.transcribe(\n temp_path,\n language=session_language,\n timestamps=False,\n diarize=False,\n )\n if not session_state.get(\"language\") and transcript.get(\"language\"):\n session_state[\"language\"] = transcript[\"language\"]\n\n try:\n apply_live_stt_hypothesis(\n session_state,\n str(transcript.get(\"text\", \"\")),\n chunk_index,\n is_silence=is_silence,\n )\n except ValueError:\n current_app.logger.warning(\n \"Invalid live transcription chunk\",\n exc_info=True,\n )\n return make_response(\n jsonify(\n {\n \"success\": False,\n \"message\": \"Invalid live transcription chunk\",\n }\n ),\n 409,\n )\n save_live_stt_session(redis_client, session_state)\n\n return make_response(\n jsonify(\n {\n \"success\": True,\n \"session_id\": session_id,\n \"chunk_index\": chunk_index,\n \"chunk_text\": transcript.get(\"text\", \"\"),\n \"is_silence\": is_silence,\n \"language\": session_state.get(\"language\"),\n \"committed_text\": session_state.get(\"committed_text\", \"\"),\n \"mutable_text\": session_state.get(\"mutable_text\", \"\"),\n \"previous_hypothesis\": session_state.get(\n \"previous_hypothesis\", \"\"\n ),\n \"latest_hypothesis\": session_state.get(\n \"latest_hypothesis\", \"\"\n ),\n \"finalized_text\": session_state.get(\"committed_text\", \"\"),\n \"pending_text\": session_state.get(\"mutable_text\", \"\"),\n \"transcript_text\": get_live_stt_transcript_text(session_state),\n }\n ),\n 200,\n )\n except Exception as err:\n current_app.logger.error(\n f\"Error transcribing live audio chunk: {err}\", exc_info=True\n )\n return make_response(\n jsonify({\"success\": False, \"message\": \"Failed to transcribe audio\"}),\n 400,\n )\n finally:\n if temp_path and temp_path.exists():\n temp_path.unlink()\n\n\n@attachments_ns.route(\"/stt/live/finish\")\nclass LiveSpeechToTextFinish(Resource):\n @api.doc(description=\"Finish a live speech-to-text session\")\n def post(self):\n auth_user = _resolve_authenticated_user()\n if hasattr(auth_user, \"status_code\"):\n return auth_user\n if not auth_user:\n return make_response(\n jsonify({\"success\": False, \"message\": \"Authentication required\"}),\n 401,\n )\n\n redis_client = _require_live_stt_redis()\n if hasattr(redis_client, \"status_code\"):\n return redis_client\n\n payload = request.get_json(silent=True) or {}\n session_id = str(payload.get(\"session_id\", \"\")).strip()\n if not session_id:\n return make_response(\n jsonify({\"success\": False, \"message\": \"Missing session_id\"}),\n 400,\n )\n\n session_state = load_live_stt_session(redis_client, session_id)\n if not session_state:\n return make_response(\n jsonify(\n {\n \"success\": False,\n \"message\": \"Live transcription session not found\",\n }\n ),\n 404,\n )\n\n if safe_filename(str(session_state.get(\"user\", \"\"))) != auth_user:\n return make_response(\n jsonify({\"success\": False, \"message\": \"Forbidden\"}),\n 403,\n )\n\n final_text = finalize_live_stt_session(session_state)\n delete_live_stt_session(redis_client, session_id)\n\n return make_response(\n jsonify(\n {\n \"success\": True,\n \"session_id\": session_id,\n \"language\": session_state.get(\"language\"),\n \"text\": final_text,\n }\n ),\n 200,\n )\n\n\n@attachments_ns.route(\"/images/\")\nclass ServeImage(Resource):\n @api.doc(description=\"Serve an image from storage\")\n def get(self, image_path):\n try:\n from application.api.user.base import storage\n\n file_obj = storage.get_file(image_path)\n extension = image_path.split(\".\")[-1].lower()\n content_type = f\"image/{extension}\"\n if extension == \"jpg\":\n content_type = \"image/jpeg\"\n response = make_response(file_obj.read())\n response.headers.set(\"Content-Type\", content_type)\n response.headers.set(\"Cache-Control\", \"max-age=86400\")\n\n return response\n except FileNotFoundError:\n return make_response(\n jsonify({\"success\": False, \"message\": \"Image not found\"}), 404\n )\n except Exception as e:\n current_app.logger.error(f\"Error serving image: {e}\")\n return make_response(\n jsonify({\"success\": False, \"message\": \"Error retrieving image\"}), 500\n )\n\n\n@attachments_ns.route(\"/tts\")\nclass TextToSpeech(Resource):\n tts_model = api.model(\n \"TextToSpeechModel\",\n {\n \"text\": fields.String(\n required=True, description=\"Text to be synthesized as audio\"\n ),\n },\n )\n\n @api.expect(tts_model)\n @api.doc(description=\"Synthesize audio speech from text\")\n def post(self):\n data = request.get_json()\n text = data[\"text\"]\n try:\n tts_instance = TTSCreator.create_tts(settings.TTS_PROVIDER)\n audio_base64, detected_language = tts_instance.text_to_speech(text)\n return make_response(\n jsonify(\n {\n \"success\": True,\n \"audio_base64\": audio_base64,\n \"lang\": detected_language,\n }\n ),\n 200,\n )\n except Exception as err:\n current_app.logger.error(f\"Error synthesizing audio: {err}\", exc_info=True)\n return make_response(jsonify({\"success\": False}), 400)\n" }, { "path": "application/api/user/base.py", "content": "\"\"\"\nShared utilities, database connections, and helper functions for user API routes.\n\"\"\"\n\nimport datetime\nimport os\nimport uuid\nfrom functools import wraps\nfrom typing import Optional, Tuple\n\nfrom bson.objectid import ObjectId\nfrom flask import current_app, jsonify, make_response, Response\nfrom pymongo import ReturnDocument\nfrom werkzeug.utils import secure_filename\n\nfrom application.core.mongo_db import MongoDB\nfrom application.core.settings import settings\nfrom application.storage.storage_creator import StorageCreator\nfrom application.vectorstore.vector_creator import VectorCreator\n\n\nstorage = StorageCreator.get_storage()\n\n\nmongo = MongoDB.get_client()\ndb = mongo[settings.MONGO_DB_NAME]\n\n\nconversations_collection = db[\"conversations\"]\nsources_collection = db[\"sources\"]\nprompts_collection = db[\"prompts\"]\nfeedback_collection = db[\"feedback\"]\nagents_collection = db[\"agents\"]\nagent_folders_collection = db[\"agent_folders\"]\ntoken_usage_collection = db[\"token_usage\"]\nshared_conversations_collections = db[\"shared_conversations\"]\nusers_collection = db[\"users\"]\nuser_logs_collection = db[\"user_logs\"]\nuser_tools_collection = db[\"user_tools\"]\nattachments_collection = db[\"attachments\"]\nworkflow_runs_collection = db[\"workflow_runs\"]\nworkflows_collection = db[\"workflows\"]\nworkflow_nodes_collection = db[\"workflow_nodes\"]\nworkflow_edges_collection = db[\"workflow_edges\"]\n\n\ntry:\n agents_collection.create_index(\n [(\"shared\", 1)],\n name=\"shared_index\",\n background=True,\n )\n users_collection.create_index(\"user_id\", unique=True)\n workflows_collection.create_index(\n [(\"user\", 1)], name=\"workflow_user_index\", background=True\n )\n workflow_nodes_collection.create_index(\n [(\"workflow_id\", 1)], name=\"node_workflow_index\", background=True\n )\n workflow_nodes_collection.create_index(\n [(\"workflow_id\", 1), (\"graph_version\", 1)],\n name=\"node_workflow_graph_version_index\",\n background=True,\n )\n workflow_edges_collection.create_index(\n [(\"workflow_id\", 1)], name=\"edge_workflow_index\", background=True\n )\n workflow_edges_collection.create_index(\n [(\"workflow_id\", 1), (\"graph_version\", 1)],\n name=\"edge_workflow_graph_version_index\",\n background=True,\n )\nexcept Exception as e:\n print(\"Error creating indexes:\", e)\ncurrent_dir = os.path.dirname(\n os.path.dirname(os.path.dirname(os.path.abspath(__file__)))\n)\n\n\ndef generate_minute_range(start_date, end_date):\n \"\"\"Generate a dictionary with minute-level time ranges.\"\"\"\n return {\n (start_date + datetime.timedelta(minutes=i)).strftime(\"%Y-%m-%d %H:%M:00\"): 0\n for i in range(int((end_date - start_date).total_seconds() // 60) + 1)\n }\n\n\ndef generate_hourly_range(start_date, end_date):\n \"\"\"Generate a dictionary with hourly time ranges.\"\"\"\n return {\n (start_date + datetime.timedelta(hours=i)).strftime(\"%Y-%m-%d %H:00\"): 0\n for i in range(int((end_date - start_date).total_seconds() // 3600) + 1)\n }\n\n\ndef generate_date_range(start_date, end_date):\n \"\"\"Generate a dictionary with daily date ranges.\"\"\"\n return {\n (start_date + datetime.timedelta(days=i)).strftime(\"%Y-%m-%d\"): 0\n for i in range((end_date - start_date).days + 1)\n }\n\n\ndef ensure_user_doc(user_id):\n \"\"\"\n Ensure user document exists with proper agent preferences structure.\n\n Args:\n user_id: The user ID to ensure\n\n Returns:\n The user document\n \"\"\"\n default_prefs = {\n \"pinned\": [],\n \"shared_with_me\": [],\n }\n\n user_doc = users_collection.find_one_and_update(\n {\"user_id\": user_id},\n {\"$setOnInsert\": {\"agent_preferences\": default_prefs}},\n upsert=True,\n return_document=ReturnDocument.AFTER,\n )\n\n prefs = user_doc.get(\"agent_preferences\", {})\n updates = {}\n if \"pinned\" not in prefs:\n updates[\"agent_preferences.pinned\"] = []\n if \"shared_with_me\" not in prefs:\n updates[\"agent_preferences.shared_with_me\"] = []\n if updates:\n users_collection.update_one({\"user_id\": user_id}, {\"$set\": updates})\n user_doc = users_collection.find_one({\"user_id\": user_id})\n return user_doc\n\n\ndef resolve_tool_details(tool_ids):\n \"\"\"\n Resolve tool IDs to their details.\n\n Args:\n tool_ids: List of tool IDs\n\n Returns:\n List of tool details with id, name, and display_name\n \"\"\"\n tools = user_tools_collection.find(\n {\"_id\": {\"$in\": [ObjectId(tid) for tid in tool_ids]}}\n )\n return [\n {\n \"id\": str(tool[\"_id\"]),\n \"name\": tool.get(\"name\", \"\"),\n \"display_name\": tool.get(\"displayName\", tool.get(\"name\", \"\")),\n }\n for tool in tools\n ]\n\n\ndef get_vector_store(source_id):\n \"\"\"\n Get the Vector Store for a given source ID.\n\n Args:\n source_id (str): source id of the document\n\n Returns:\n Vector store instance\n \"\"\"\n store = VectorCreator.create_vectorstore(\n settings.VECTOR_STORE,\n source_id=source_id,\n embeddings_key=os.getenv(\"EMBEDDINGS_KEY\"),\n )\n return store\n\n\ndef handle_image_upload(\n request, existing_url: str, user: str, storage, base_path: str = \"attachments/\"\n) -> Tuple[str, Optional[Response]]:\n \"\"\"\n Handle image file upload from request.\n\n Args:\n request: Flask request object\n existing_url: Existing image URL (fallback)\n user: User ID\n storage: Storage instance\n base_path: Base path for upload\n\n Returns:\n Tuple of (image_url, error_response)\n \"\"\"\n image_url = existing_url\n\n if \"image\" in request.files:\n file = request.files[\"image\"]\n if file.filename != \"\":\n filename = secure_filename(file.filename)\n upload_path = f\"{settings.UPLOAD_FOLDER.rstrip('/')}/{user}/{base_path.rstrip('/')}/{uuid.uuid4()}_{filename}\"\n try:\n storage.save_file(file, upload_path, storage_class=\"STANDARD\")\n image_url = upload_path\n except Exception as e:\n current_app.logger.error(f\"Error uploading image: {e}\")\n return None, make_response(\n jsonify({\"success\": False, \"message\": \"Image upload failed\"}),\n 400,\n )\n return image_url, None\n\n\ndef require_agent(func):\n \"\"\"\n Decorator to require valid agent webhook token.\n\n Args:\n func: Function to decorate\n\n Returns:\n Wrapped function\n \"\"\"\n\n @wraps(func)\n def wrapper(*args, **kwargs):\n webhook_token = kwargs.get(\"webhook_token\")\n if not webhook_token:\n return make_response(\n jsonify({\"success\": False, \"message\": \"Webhook token missing\"}), 400\n )\n agent = agents_collection.find_one(\n {\"incoming_webhook_token\": webhook_token}, {\"_id\": 1}\n )\n if not agent:\n current_app.logger.warning(\n f\"Webhook attempt with invalid token: {webhook_token}\"\n )\n return make_response(\n jsonify({\"success\": False, \"message\": \"Agent not found\"}), 404\n )\n kwargs[\"agent\"] = agent\n kwargs[\"agent_id_str\"] = str(agent[\"_id\"])\n return func(*args, **kwargs)\n\n return wrapper\n" }, { "path": "application/api/user/conversations/__init__.py", "content": "\"\"\"Conversation management module.\"\"\"\n\nfrom .routes import conversations_ns\n\n__all__ = [\"conversations_ns\"]\n" }, { "path": "application/api/user/conversations/routes.py", "content": "\"\"\"Conversation management routes.\"\"\"\n\nimport datetime\n\nfrom bson.objectid import ObjectId\nfrom flask import current_app, jsonify, make_response, request\nfrom flask_restx import fields, Namespace, Resource\n\nfrom application.api import api\nfrom application.api.user.base import attachments_collection, conversations_collection\nfrom application.utils import check_required_fields\n\nconversations_ns = Namespace(\n \"conversations\", description=\"Conversation management operations\", path=\"/api\"\n)\n\n\n@conversations_ns.route(\"/delete_conversation\")\nclass DeleteConversation(Resource):\n @api.doc(\n description=\"Deletes a conversation by ID\",\n params={\"id\": \"The ID of the conversation to delete\"},\n )\n def post(self):\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False}), 401)\n conversation_id = request.args.get(\"id\")\n if not conversation_id:\n return make_response(\n jsonify({\"success\": False, \"message\": \"ID is required\"}), 400\n )\n try:\n conversations_collection.delete_one(\n {\"_id\": ObjectId(conversation_id), \"user\": decoded_token[\"sub\"]}\n )\n except Exception as err:\n current_app.logger.error(\n f\"Error deleting conversation: {err}\", exc_info=True\n )\n return make_response(jsonify({\"success\": False}), 400)\n return make_response(jsonify({\"success\": True}), 200)\n\n\n@conversations_ns.route(\"/delete_all_conversations\")\nclass DeleteAllConversations(Resource):\n @api.doc(\n description=\"Deletes all conversations for a specific user\",\n )\n def get(self):\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False}), 401)\n user_id = decoded_token.get(\"sub\")\n try:\n conversations_collection.delete_many({\"user\": user_id})\n except Exception as err:\n current_app.logger.error(\n f\"Error deleting all conversations: {err}\", exc_info=True\n )\n return make_response(jsonify({\"success\": False}), 400)\n return make_response(jsonify({\"success\": True}), 200)\n\n\n@conversations_ns.route(\"/get_conversations\")\nclass GetConversations(Resource):\n @api.doc(\n description=\"Retrieve a list of the latest 30 conversations (excluding API key conversations)\",\n )\n def get(self):\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False}), 401)\n try:\n conversations = (\n conversations_collection.find(\n {\n \"$or\": [\n {\"api_key\": {\"$exists\": False}},\n {\"agent_id\": {\"$exists\": True}},\n ],\n \"user\": decoded_token.get(\"sub\"),\n }\n )\n .sort(\"date\", -1)\n .limit(30)\n )\n\n list_conversations = [\n {\n \"id\": str(conversation[\"_id\"]),\n \"name\": conversation[\"name\"],\n \"agent_id\": conversation.get(\"agent_id\", None),\n \"is_shared_usage\": conversation.get(\"is_shared_usage\", False),\n \"shared_token\": conversation.get(\"shared_token\", None),\n }\n for conversation in conversations\n ]\n except Exception as err:\n current_app.logger.error(\n f\"Error retrieving conversations: {err}\", exc_info=True\n )\n return make_response(jsonify({\"success\": False}), 400)\n return make_response(jsonify(list_conversations), 200)\n\n\n@conversations_ns.route(\"/get_single_conversation\")\nclass GetSingleConversation(Resource):\n @api.doc(\n description=\"Retrieve a single conversation by ID\",\n params={\"id\": \"The conversation ID\"},\n )\n def get(self):\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False}), 401)\n conversation_id = request.args.get(\"id\")\n if not conversation_id:\n return make_response(\n jsonify({\"success\": False, \"message\": \"ID is required\"}), 400\n )\n try:\n conversation = conversations_collection.find_one(\n {\"_id\": ObjectId(conversation_id), \"user\": decoded_token.get(\"sub\")}\n )\n if not conversation:\n return make_response(jsonify({\"status\": \"not found\"}), 404)\n # Process queries to include attachment names\n\n queries = conversation[\"queries\"]\n for query in queries:\n if \"attachments\" in query and query[\"attachments\"]:\n attachment_details = []\n for attachment_id in query[\"attachments\"]:\n try:\n attachment = attachments_collection.find_one(\n {\"_id\": ObjectId(attachment_id)}\n )\n if attachment:\n attachment_details.append(\n {\n \"id\": str(attachment[\"_id\"]),\n \"fileName\": attachment.get(\n \"filename\", \"Unknown file\"\n ),\n }\n )\n except Exception as e:\n current_app.logger.error(\n f\"Error retrieving attachment {attachment_id}: {e}\",\n exc_info=True,\n )\n query[\"attachments\"] = attachment_details\n except Exception as err:\n current_app.logger.error(\n f\"Error retrieving conversation: {err}\", exc_info=True\n )\n return make_response(jsonify({\"success\": False}), 400)\n data = {\n \"queries\": queries,\n \"agent_id\": conversation.get(\"agent_id\"),\n \"is_shared_usage\": conversation.get(\"is_shared_usage\", False),\n \"shared_token\": conversation.get(\"shared_token\", None),\n }\n return make_response(jsonify(data), 200)\n\n\n@conversations_ns.route(\"/update_conversation_name\")\nclass UpdateConversationName(Resource):\n @api.expect(\n api.model(\n \"UpdateConversationModel\",\n {\n \"id\": fields.String(required=True, description=\"Conversation ID\"),\n \"name\": fields.String(\n required=True, description=\"New name of the conversation\"\n ),\n },\n )\n )\n @api.doc(\n description=\"Updates the name of a conversation\",\n )\n def post(self):\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False}), 401)\n data = request.get_json()\n required_fields = [\"id\", \"name\"]\n missing_fields = check_required_fields(data, required_fields)\n if missing_fields:\n return missing_fields\n try:\n conversations_collection.update_one(\n {\"_id\": ObjectId(data[\"id\"]), \"user\": decoded_token.get(\"sub\")},\n {\"$set\": {\"name\": data[\"name\"]}},\n )\n except Exception as err:\n current_app.logger.error(\n f\"Error updating conversation name: {err}\", exc_info=True\n )\n return make_response(jsonify({\"success\": False}), 400)\n return make_response(jsonify({\"success\": True}), 200)\n\n\n@conversations_ns.route(\"/feedback\")\nclass SubmitFeedback(Resource):\n @api.expect(\n api.model(\n \"FeedbackModel\",\n {\n \"question\": fields.String(\n required=False, description=\"The user question\"\n ),\n \"answer\": fields.String(required=False, description=\"The AI answer\"),\n \"feedback\": fields.String(required=True, description=\"User feedback\"),\n \"question_index\": fields.Integer(\n required=True,\n description=\"The question number in that particular conversation\",\n ),\n \"conversation_id\": fields.String(\n required=True, description=\"id of the particular conversation\"\n ),\n \"api_key\": fields.String(description=\"Optional API key\"),\n },\n )\n )\n @api.doc(\n description=\"Submit feedback for a conversation\",\n )\n def post(self):\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False}), 401)\n data = request.get_json()\n required_fields = [\"feedback\", \"conversation_id\", \"question_index\"]\n missing_fields = check_required_fields(data, required_fields)\n if missing_fields:\n return missing_fields\n try:\n if data[\"feedback\"] is None:\n # Remove feedback and feedback_timestamp if feedback is null\n\n conversations_collection.update_one(\n {\n \"_id\": ObjectId(data[\"conversation_id\"]),\n \"user\": decoded_token.get(\"sub\"),\n f\"queries.{data['question_index']}\": {\"$exists\": True},\n },\n {\n \"$unset\": {\n f\"queries.{data['question_index']}.feedback\": \"\",\n f\"queries.{data['question_index']}.feedback_timestamp\": \"\",\n }\n },\n )\n else:\n # Set feedback and feedback_timestamp if feedback has a value\n\n conversations_collection.update_one(\n {\n \"_id\": ObjectId(data[\"conversation_id\"]),\n \"user\": decoded_token.get(\"sub\"),\n f\"queries.{data['question_index']}\": {\"$exists\": True},\n },\n {\n \"$set\": {\n f\"queries.{data['question_index']}.feedback\": data[\n \"feedback\"\n ],\n f\"queries.{data['question_index']}.feedback_timestamp\": datetime.datetime.now(\n datetime.timezone.utc\n ),\n }\n },\n )\n except Exception as err:\n current_app.logger.error(f\"Error submitting feedback: {err}\", exc_info=True)\n return make_response(jsonify({\"success\": False}), 400)\n return make_response(jsonify({\"success\": True}), 200)\n" }, { "path": "application/api/user/models/__init__.py", "content": "from .routes import models_ns\n\n__all__ = [\"models_ns\"]\n" }, { "path": "application/api/user/models/routes.py", "content": "from flask import current_app, jsonify, make_response\nfrom flask_restx import Namespace, Resource\n\nfrom application.core.model_settings import ModelRegistry\n\nmodels_ns = Namespace(\"models\", description=\"Available models\", path=\"/api\")\n\n\n@models_ns.route(\"/models\")\nclass ModelsListResource(Resource):\n def get(self):\n \"\"\"Get list of available models with their capabilities.\"\"\"\n try:\n registry = ModelRegistry.get_instance()\n models = registry.get_enabled_models()\n\n response = {\n \"models\": [model.to_dict() for model in models],\n \"default_model_id\": registry.default_model_id,\n \"count\": len(models),\n }\n except Exception as err:\n current_app.logger.error(f\"Error fetching models: {err}\", exc_info=True)\n return make_response(jsonify({\"success\": False}), 500)\n return make_response(jsonify(response), 200)\n" }, { "path": "application/api/user/prompts/__init__.py", "content": "\"\"\"Prompts module.\"\"\"\n\nfrom .routes import prompts_ns\n\n__all__ = [\"prompts_ns\"]\n" }, { "path": "application/api/user/prompts/routes.py", "content": "\"\"\"Prompt management routes.\"\"\"\n\nimport os\n\nfrom bson.objectid import ObjectId\nfrom flask import current_app, jsonify, make_response, request\nfrom flask_restx import fields, Namespace, Resource\n\nfrom application.api import api\nfrom application.api.user.base import current_dir, prompts_collection\nfrom application.utils import check_required_fields\n\nprompts_ns = Namespace(\n \"prompts\", description=\"Prompt management operations\", path=\"/api\"\n)\n\n\n@prompts_ns.route(\"/create_prompt\")\nclass CreatePrompt(Resource):\n create_prompt_model = api.model(\n \"CreatePromptModel\",\n {\n \"content\": fields.String(\n required=True, description=\"Content of the prompt\"\n ),\n \"name\": fields.String(required=True, description=\"Name of the prompt\"),\n },\n )\n\n @api.expect(create_prompt_model)\n @api.doc(description=\"Create a new prompt\")\n def post(self):\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False}), 401)\n data = request.get_json()\n required_fields = [\"content\", \"name\"]\n missing_fields = check_required_fields(data, required_fields)\n if missing_fields:\n return missing_fields\n user = decoded_token.get(\"sub\")\n try:\n\n resp = prompts_collection.insert_one(\n {\n \"name\": data[\"name\"],\n \"content\": data[\"content\"],\n \"user\": user,\n }\n )\n new_id = str(resp.inserted_id)\n except Exception as err:\n current_app.logger.error(f\"Error creating prompt: {err}\", exc_info=True)\n return make_response(jsonify({\"success\": False}), 400)\n return make_response(jsonify({\"id\": new_id}), 200)\n\n\n@prompts_ns.route(\"/get_prompts\")\nclass GetPrompts(Resource):\n @api.doc(description=\"Get all prompts for the user\")\n def get(self):\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False}), 401)\n user = decoded_token.get(\"sub\")\n try:\n prompts = prompts_collection.find({\"user\": user})\n list_prompts = [\n {\"id\": \"default\", \"name\": \"default\", \"type\": \"public\"},\n {\"id\": \"creative\", \"name\": \"creative\", \"type\": \"public\"},\n {\"id\": \"strict\", \"name\": \"strict\", \"type\": \"public\"},\n ]\n\n for prompt in prompts:\n list_prompts.append(\n {\n \"id\": str(prompt[\"_id\"]),\n \"name\": prompt[\"name\"],\n \"type\": \"private\",\n }\n )\n except Exception as err:\n current_app.logger.error(f\"Error retrieving prompts: {err}\", exc_info=True)\n return make_response(jsonify({\"success\": False}), 400)\n return make_response(jsonify(list_prompts), 200)\n\n\n@prompts_ns.route(\"/get_single_prompt\")\nclass GetSinglePrompt(Resource):\n @api.doc(params={\"id\": \"ID of the prompt\"}, description=\"Get a single prompt by ID\")\n def get(self):\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False}), 401)\n user = decoded_token.get(\"sub\")\n prompt_id = request.args.get(\"id\")\n if not prompt_id:\n return make_response(\n jsonify({\"success\": False, \"message\": \"ID is required\"}), 400\n )\n try:\n if prompt_id == \"default\":\n with open(\n os.path.join(current_dir, \"prompts\", \"chat_combine_default.txt\"),\n \"r\",\n ) as f:\n chat_combine_template = f.read()\n return make_response(jsonify({\"content\": chat_combine_template}), 200)\n elif prompt_id == \"creative\":\n with open(\n os.path.join(current_dir, \"prompts\", \"chat_combine_creative.txt\"),\n \"r\",\n ) as f:\n chat_reduce_creative = f.read()\n return make_response(jsonify({\"content\": chat_reduce_creative}), 200)\n elif prompt_id == \"strict\":\n with open(\n os.path.join(current_dir, \"prompts\", \"chat_combine_strict.txt\"), \"r\"\n ) as f:\n chat_reduce_strict = f.read()\n return make_response(jsonify({\"content\": chat_reduce_strict}), 200)\n prompt = prompts_collection.find_one(\n {\"_id\": ObjectId(prompt_id), \"user\": user}\n )\n except Exception as err:\n current_app.logger.error(f\"Error retrieving prompt: {err}\", exc_info=True)\n return make_response(jsonify({\"success\": False}), 400)\n return make_response(jsonify({\"content\": prompt[\"content\"]}), 200)\n\n\n@prompts_ns.route(\"/delete_prompt\")\nclass DeletePrompt(Resource):\n delete_prompt_model = api.model(\n \"DeletePromptModel\",\n {\"id\": fields.String(required=True, description=\"Prompt ID to delete\")},\n )\n\n @api.expect(delete_prompt_model)\n @api.doc(description=\"Delete a prompt by ID\")\n def post(self):\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False}), 401)\n user = decoded_token.get(\"sub\")\n data = request.get_json()\n required_fields = [\"id\"]\n missing_fields = check_required_fields(data, required_fields)\n if missing_fields:\n return missing_fields\n try:\n prompts_collection.delete_one({\"_id\": ObjectId(data[\"id\"]), \"user\": user})\n except Exception as err:\n current_app.logger.error(f\"Error deleting prompt: {err}\", exc_info=True)\n return make_response(jsonify({\"success\": False}), 400)\n return make_response(jsonify({\"success\": True}), 200)\n\n\n@prompts_ns.route(\"/update_prompt\")\nclass UpdatePrompt(Resource):\n update_prompt_model = api.model(\n \"UpdatePromptModel\",\n {\n \"id\": fields.String(required=True, description=\"Prompt ID to update\"),\n \"name\": fields.String(required=True, description=\"New name of the prompt\"),\n \"content\": fields.String(\n required=True, description=\"New content of the prompt\"\n ),\n },\n )\n\n @api.expect(update_prompt_model)\n @api.doc(description=\"Update an existing prompt\")\n def post(self):\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False}), 401)\n user = decoded_token.get(\"sub\")\n data = request.get_json()\n required_fields = [\"id\", \"name\", \"content\"]\n missing_fields = check_required_fields(data, required_fields)\n if missing_fields:\n return missing_fields\n try:\n prompts_collection.update_one(\n {\"_id\": ObjectId(data[\"id\"]), \"user\": user},\n {\"$set\": {\"name\": data[\"name\"], \"content\": data[\"content\"]}},\n )\n except Exception as err:\n current_app.logger.error(f\"Error updating prompt: {err}\", exc_info=True)\n return make_response(jsonify({\"success\": False}), 400)\n return make_response(jsonify({\"success\": True}), 200)\n" }, { "path": "application/api/user/routes.py", "content": "\"\"\"\nMain user API routes - registers all namespace modules.\n\"\"\"\n\nfrom flask import Blueprint\n\nfrom application.api import api\nfrom .agents import agents_ns, agents_sharing_ns, agents_webhooks_ns, agents_folders_ns\nfrom .analytics import analytics_ns\nfrom .attachments import attachments_ns\nfrom .conversations import conversations_ns\nfrom .models import models_ns\nfrom .prompts import prompts_ns\nfrom .sharing import sharing_ns\nfrom .sources import sources_chunks_ns, sources_ns, sources_upload_ns\nfrom .tools import tools_mcp_ns, tools_ns\nfrom .workflows import workflows_ns\n\n\nuser = Blueprint(\"user\", __name__)\n\n# Analytics\napi.add_namespace(analytics_ns)\n\n# Attachments\napi.add_namespace(attachments_ns)\n\n# Conversations\napi.add_namespace(conversations_ns)\n\n# Models\napi.add_namespace(models_ns)\n\n# Agents (main, sharing, webhooks, folders)\napi.add_namespace(agents_ns)\napi.add_namespace(agents_sharing_ns)\napi.add_namespace(agents_webhooks_ns)\napi.add_namespace(agents_folders_ns)\n\n# Prompts\napi.add_namespace(prompts_ns)\n\n# Sharing\napi.add_namespace(sharing_ns)\n\n# Sources (main, chunks, upload)\napi.add_namespace(sources_ns)\napi.add_namespace(sources_chunks_ns)\napi.add_namespace(sources_upload_ns)\n\n# Tools (main, MCP)\napi.add_namespace(tools_ns)\napi.add_namespace(tools_mcp_ns)\n\n# Workflows\napi.add_namespace(workflows_ns)\n" }, { "path": "application/api/user/sharing/__init__.py", "content": "\"\"\"Sharing module.\"\"\"\n\nfrom .routes import sharing_ns\n\n__all__ = [\"sharing_ns\"]\n" }, { "path": "application/api/user/sharing/routes.py", "content": "\"\"\"Conversation sharing routes.\"\"\"\n\nimport uuid\n\nfrom bson.binary import Binary, UuidRepresentation\nfrom bson.dbref import DBRef\nfrom bson.objectid import ObjectId\nfrom flask import current_app, jsonify, make_response, request\nfrom flask_restx import fields, inputs, Namespace, Resource\n\nfrom application.api import api\nfrom application.api.user.base import (\n agents_collection,\n attachments_collection,\n conversations_collection,\n shared_conversations_collections,\n)\nfrom application.utils import check_required_fields\n\nsharing_ns = Namespace(\n \"sharing\", description=\"Conversation sharing operations\", path=\"/api\"\n)\n\n\n@sharing_ns.route(\"/share\")\nclass ShareConversation(Resource):\n share_conversation_model = api.model(\n \"ShareConversationModel\",\n {\n \"conversation_id\": fields.String(\n required=True, description=\"Conversation ID\"\n ),\n \"user\": fields.String(description=\"User ID (optional)\"),\n \"prompt_id\": fields.String(description=\"Prompt ID (optional)\"),\n \"chunks\": fields.Integer(description=\"Chunks count (optional)\"),\n },\n )\n\n @api.expect(share_conversation_model)\n @api.doc(description=\"Share a conversation\")\n def post(self):\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False}), 401)\n user = decoded_token.get(\"sub\")\n data = request.get_json()\n required_fields = [\"conversation_id\"]\n missing_fields = check_required_fields(data, required_fields)\n if missing_fields:\n return missing_fields\n is_promptable = request.args.get(\"isPromptable\", type=inputs.boolean)\n if is_promptable is None:\n return make_response(\n jsonify({\"success\": False, \"message\": \"isPromptable is required\"}), 400\n )\n conversation_id = data[\"conversation_id\"]\n\n try:\n conversation = conversations_collection.find_one(\n {\"_id\": ObjectId(conversation_id)}\n )\n if conversation is None:\n return make_response(\n jsonify(\n {\n \"status\": \"error\",\n \"message\": \"Conversation does not exist\",\n }\n ),\n 404,\n )\n current_n_queries = len(conversation[\"queries\"])\n explicit_binary = Binary.from_uuid(\n uuid.uuid4(), UuidRepresentation.STANDARD\n )\n\n if is_promptable:\n prompt_id = data.get(\"prompt_id\", \"default\")\n chunks = data.get(\"chunks\", \"2\")\n\n name = conversation[\"name\"] + \"(shared)\"\n new_api_key_data = {\n \"prompt_id\": prompt_id,\n \"chunks\": chunks,\n \"user\": user,\n }\n\n if \"source\" in data and ObjectId.is_valid(data[\"source\"]):\n new_api_key_data[\"source\"] = DBRef(\n \"sources\", ObjectId(data[\"source\"])\n )\n if \"retriever\" in data:\n new_api_key_data[\"retriever\"] = data[\"retriever\"]\n pre_existing_api_document = agents_collection.find_one(new_api_key_data)\n if pre_existing_api_document:\n api_uuid = pre_existing_api_document[\"key\"]\n pre_existing = shared_conversations_collections.find_one(\n {\n \"conversation_id\": ObjectId(conversation_id),\n \"isPromptable\": is_promptable,\n \"first_n_queries\": current_n_queries,\n \"user\": user,\n \"api_key\": api_uuid,\n }\n )\n if pre_existing is not None:\n return make_response(\n jsonify(\n {\n \"success\": True,\n \"identifier\": str(pre_existing[\"uuid\"].as_uuid()),\n }\n ),\n 200,\n )\n else:\n shared_conversations_collections.insert_one(\n {\n \"uuid\": explicit_binary,\n \"conversation_id\": ObjectId(conversation_id),\n \"isPromptable\": is_promptable,\n \"first_n_queries\": current_n_queries,\n \"user\": user,\n \"api_key\": api_uuid,\n }\n )\n return make_response(\n jsonify(\n {\n \"success\": True,\n \"identifier\": str(explicit_binary.as_uuid()),\n }\n ),\n 201,\n )\n else:\n api_uuid = str(uuid.uuid4())\n new_api_key_data[\"key\"] = api_uuid\n new_api_key_data[\"name\"] = name\n\n if \"source\" in data and ObjectId.is_valid(data[\"source\"]):\n new_api_key_data[\"source\"] = DBRef(\n \"sources\", ObjectId(data[\"source\"])\n )\n if \"retriever\" in data:\n new_api_key_data[\"retriever\"] = data[\"retriever\"]\n agents_collection.insert_one(new_api_key_data)\n shared_conversations_collections.insert_one(\n {\n \"uuid\": explicit_binary,\n \"conversation_id\": ObjectId(conversation_id),\n \"isPromptable\": is_promptable,\n \"first_n_queries\": current_n_queries,\n \"user\": user,\n \"api_key\": api_uuid,\n }\n )\n return make_response(\n jsonify(\n {\n \"success\": True,\n \"identifier\": str(explicit_binary.as_uuid()),\n }\n ),\n 201,\n )\n pre_existing = shared_conversations_collections.find_one(\n {\n \"conversation_id\": ObjectId(conversation_id),\n \"isPromptable\": is_promptable,\n \"first_n_queries\": current_n_queries,\n \"user\": user,\n }\n )\n if pre_existing is not None:\n return make_response(\n jsonify(\n {\n \"success\": True,\n \"identifier\": str(pre_existing[\"uuid\"].as_uuid()),\n }\n ),\n 200,\n )\n else:\n shared_conversations_collections.insert_one(\n {\n \"uuid\": explicit_binary,\n \"conversation_id\": ObjectId(conversation_id),\n \"isPromptable\": is_promptable,\n \"first_n_queries\": current_n_queries,\n \"user\": user,\n }\n )\n return make_response(\n jsonify(\n {\"success\": True, \"identifier\": str(explicit_binary.as_uuid())}\n ),\n 201,\n )\n except Exception as err:\n current_app.logger.error(\n f\"Error sharing conversation: {err}\", exc_info=True\n )\n return make_response(jsonify({\"success\": False}), 400)\n\n\n@sharing_ns.route(\"/shared_conversation/\")\nclass GetPubliclySharedConversations(Resource):\n @api.doc(description=\"Get publicly shared conversations by identifier\")\n def get(self, identifier: str):\n try:\n query_uuid = Binary.from_uuid(\n uuid.UUID(identifier), UuidRepresentation.STANDARD\n )\n shared = shared_conversations_collections.find_one({\"uuid\": query_uuid})\n conversation_queries = []\n\n if (\n shared\n and \"conversation_id\" in shared\n ):\n # Handle DBRef (legacy), ObjectId, dict, and string formats for conversation_id\n conversation_id = shared[\"conversation_id\"]\n if isinstance(conversation_id, DBRef):\n conversation_id = conversation_id.id\n elif isinstance(conversation_id, dict):\n # Handle dict representation of DBRef (e.g., {\"$ref\": \"...\", \"$id\": \"...\"})\n if \"$id\" in conversation_id:\n conv_id = conversation_id[\"$id\"]\n # $id might be a dict like {\"$oid\": \"...\"} or a string\n if isinstance(conv_id, dict) and \"$oid\" in conv_id:\n conversation_id = ObjectId(conv_id[\"$oid\"])\n else:\n conversation_id = ObjectId(conv_id)\n elif \"_id\" in conversation_id:\n conversation_id = ObjectId(conversation_id[\"_id\"])\n elif isinstance(conversation_id, str):\n conversation_id = ObjectId(conversation_id)\n conversation = conversations_collection.find_one(\n {\"_id\": conversation_id}\n )\n if conversation is None:\n return make_response(\n jsonify(\n {\n \"success\": False,\n \"error\": \"might have broken url or the conversation does not exist\",\n }\n ),\n 404,\n )\n conversation_queries = conversation[\"queries\"][\n : (shared[\"first_n_queries\"])\n ]\n\n for query in conversation_queries:\n if \"attachments\" in query and query[\"attachments\"]:\n attachment_details = []\n for attachment_id in query[\"attachments\"]:\n try:\n attachment = attachments_collection.find_one(\n {\"_id\": ObjectId(attachment_id)}\n )\n if attachment:\n attachment_details.append(\n {\n \"id\": str(attachment[\"_id\"]),\n \"fileName\": attachment.get(\n \"filename\", \"Unknown file\"\n ),\n }\n )\n except Exception as e:\n current_app.logger.error(\n f\"Error retrieving attachment {attachment_id}: {e}\",\n exc_info=True,\n )\n query[\"attachments\"] = attachment_details\n else:\n return make_response(\n jsonify(\n {\n \"success\": False,\n \"error\": \"might have broken url or the conversation does not exist\",\n }\n ),\n 404,\n )\n date = conversation[\"_id\"].generation_time.isoformat()\n res = {\n \"success\": True,\n \"queries\": conversation_queries,\n \"title\": conversation[\"name\"],\n \"timestamp\": date,\n }\n if shared[\"isPromptable\"] and \"api_key\" in shared:\n res[\"api_key\"] = shared[\"api_key\"]\n return make_response(jsonify(res), 200)\n except Exception as err:\n current_app.logger.error(\n f\"Error getting shared conversation: {err}\", exc_info=True\n )\n return make_response(jsonify({\"success\": False}), 400)\n" }, { "path": "application/api/user/sources/__init__.py", "content": "\"\"\"Sources module.\"\"\"\n\nfrom .chunks import sources_chunks_ns\nfrom .routes import sources_ns\nfrom .upload import sources_upload_ns\n\n__all__ = [\"sources_ns\", \"sources_chunks_ns\", \"sources_upload_ns\"]\n" }, { "path": "application/api/user/sources/chunks.py", "content": "\"\"\"Source document management chunk management.\"\"\"\n\nfrom bson.objectid import ObjectId\nfrom flask import current_app, jsonify, make_response, request\nfrom flask_restx import fields, Namespace, Resource\n\nfrom application.api import api\nfrom application.api.user.base import get_vector_store, sources_collection\nfrom application.utils import check_required_fields, num_tokens_from_string\n\nsources_chunks_ns = Namespace(\n \"sources\", description=\"Source document management operations\", path=\"/api\"\n)\n\n\n@sources_chunks_ns.route(\"/get_chunks\")\nclass GetChunks(Resource):\n @api.doc(\n description=\"Retrieves chunks from a document, optionally filtered by file path and search term\",\n params={\n \"id\": \"The document ID\",\n \"page\": \"Page number for pagination\",\n \"per_page\": \"Number of chunks per page\",\n \"path\": \"Optional: Filter chunks by relative file path\",\n \"search\": \"Optional: Search term to filter chunks by title or content\",\n },\n )\n def get(self):\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False}), 401)\n user = decoded_token.get(\"sub\")\n doc_id = request.args.get(\"id\")\n page = int(request.args.get(\"page\", 1))\n per_page = int(request.args.get(\"per_page\", 10))\n path = request.args.get(\"path\")\n search_term = request.args.get(\"search\", \"\").strip().lower()\n\n if not ObjectId.is_valid(doc_id):\n return make_response(jsonify({\"error\": \"Invalid doc_id\"}), 400)\n doc = sources_collection.find_one({\"_id\": ObjectId(doc_id), \"user\": user})\n if not doc:\n return make_response(\n jsonify({\"error\": \"Document not found or access denied\"}), 404\n )\n try:\n store = get_vector_store(doc_id)\n chunks = store.get_chunks()\n\n filtered_chunks = []\n for chunk in chunks:\n metadata = chunk.get(\"metadata\", {})\n\n # Filter by path if provided\n\n if path:\n chunk_source = metadata.get(\"source\", \"\")\n chunk_file_path = metadata.get(\"file_path\", \"\")\n # Check if the chunk matches the requested path\n # For file uploads: source ends with path (e.g., \"inputs/.../file.pdf\" ends with \"file.pdf\")\n # For crawlers: file_path ends with path (e.g., \"guides/setup.md\" ends with \"setup.md\")\n source_match = chunk_source and chunk_source.endswith(path)\n file_path_match = chunk_file_path and chunk_file_path.endswith(path)\n\n if not (source_match or file_path_match):\n continue\n # Filter by search term if provided\n\n if search_term:\n text_match = search_term in chunk.get(\"text\", \"\").lower()\n title_match = search_term in metadata.get(\"title\", \"\").lower()\n\n if not (text_match or title_match):\n continue\n filtered_chunks.append(chunk)\n chunks = filtered_chunks\n\n total_chunks = len(chunks)\n start = (page - 1) * per_page\n end = start + per_page\n paginated_chunks = chunks[start:end]\n\n return make_response(\n jsonify(\n {\n \"page\": page,\n \"per_page\": per_page,\n \"total\": total_chunks,\n \"chunks\": paginated_chunks,\n \"path\": path if path else None,\n \"search\": search_term if search_term else None,\n }\n ),\n 200,\n )\n except Exception as e:\n current_app.logger.error(f\"Error getting chunks: {e}\", exc_info=True)\n return make_response(jsonify({\"success\": False}), 500)\n\n\n@sources_chunks_ns.route(\"/add_chunk\")\nclass AddChunk(Resource):\n @api.expect(\n api.model(\n \"AddChunkModel\",\n {\n \"id\": fields.String(required=True, description=\"Document ID\"),\n \"text\": fields.String(required=True, description=\"Text of the chunk\"),\n \"metadata\": fields.Raw(\n required=False,\n description=\"Metadata associated with the chunk\",\n ),\n },\n )\n )\n @api.doc(\n description=\"Adds a new chunk to the document\",\n )\n def post(self):\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False}), 401)\n user = decoded_token.get(\"sub\")\n data = request.get_json()\n required_fields = [\"id\", \"text\"]\n missing_fields = check_required_fields(data, required_fields)\n if missing_fields:\n return missing_fields\n doc_id = data.get(\"id\")\n text = data.get(\"text\")\n metadata = data.get(\"metadata\", {})\n token_count = num_tokens_from_string(text)\n metadata[\"token_count\"] = token_count\n\n if not ObjectId.is_valid(doc_id):\n return make_response(jsonify({\"error\": \"Invalid doc_id\"}), 400)\n doc = sources_collection.find_one({\"_id\": ObjectId(doc_id), \"user\": user})\n if not doc:\n return make_response(\n jsonify({\"error\": \"Document not found or access denied\"}), 404\n )\n try:\n store = get_vector_store(doc_id)\n chunk_id = store.add_chunk(text, metadata)\n return make_response(\n jsonify({\"message\": \"Chunk added successfully\", \"chunk_id\": chunk_id}),\n 201,\n )\n except Exception as e:\n current_app.logger.error(f\"Error adding chunk: {e}\", exc_info=True)\n return make_response(jsonify({\"success\": False}), 500)\n\n\n@sources_chunks_ns.route(\"/delete_chunk\")\nclass DeleteChunk(Resource):\n @api.doc(\n description=\"Deletes a specific chunk from the document.\",\n params={\"id\": \"The document ID\", \"chunk_id\": \"The ID of the chunk to delete\"},\n )\n def delete(self):\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False}), 401)\n user = decoded_token.get(\"sub\")\n doc_id = request.args.get(\"id\")\n chunk_id = request.args.get(\"chunk_id\")\n\n if not ObjectId.is_valid(doc_id):\n return make_response(jsonify({\"error\": \"Invalid doc_id\"}), 400)\n doc = sources_collection.find_one({\"_id\": ObjectId(doc_id), \"user\": user})\n if not doc:\n return make_response(\n jsonify({\"error\": \"Document not found or access denied\"}), 404\n )\n try:\n store = get_vector_store(doc_id)\n deleted = store.delete_chunk(chunk_id)\n if deleted:\n return make_response(\n jsonify({\"message\": \"Chunk deleted successfully\"}), 200\n )\n else:\n return make_response(\n jsonify({\"message\": \"Chunk not found or could not be deleted\"}),\n 404,\n )\n except Exception as e:\n current_app.logger.error(f\"Error deleting chunk: {e}\", exc_info=True)\n return make_response(jsonify({\"success\": False}), 500)\n\n\n@sources_chunks_ns.route(\"/update_chunk\")\nclass UpdateChunk(Resource):\n @api.expect(\n api.model(\n \"UpdateChunkModel\",\n {\n \"id\": fields.String(required=True, description=\"Document ID\"),\n \"chunk_id\": fields.String(\n required=True, description=\"Chunk ID to update\"\n ),\n \"text\": fields.String(\n required=False, description=\"New text of the chunk\"\n ),\n \"metadata\": fields.Raw(\n required=False,\n description=\"Updated metadata associated with the chunk\",\n ),\n },\n )\n )\n @api.doc(\n description=\"Updates an existing chunk in the document.\",\n )\n def put(self):\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False}), 401)\n user = decoded_token.get(\"sub\")\n data = request.get_json()\n required_fields = [\"id\", \"chunk_id\"]\n missing_fields = check_required_fields(data, required_fields)\n if missing_fields:\n return missing_fields\n doc_id = data.get(\"id\")\n chunk_id = data.get(\"chunk_id\")\n text = data.get(\"text\")\n metadata = data.get(\"metadata\")\n\n if text is not None:\n token_count = num_tokens_from_string(text)\n if metadata is None:\n metadata = {}\n metadata[\"token_count\"] = token_count\n if not ObjectId.is_valid(doc_id):\n return make_response(jsonify({\"error\": \"Invalid doc_id\"}), 400)\n doc = sources_collection.find_one({\"_id\": ObjectId(doc_id), \"user\": user})\n if not doc:\n return make_response(\n jsonify({\"error\": \"Document not found or access denied\"}), 404\n )\n try:\n store = get_vector_store(doc_id)\n\n chunks = store.get_chunks()\n existing_chunk = next((c for c in chunks if c[\"doc_id\"] == chunk_id), None)\n if not existing_chunk:\n return make_response(jsonify({\"error\": \"Chunk not found\"}), 404)\n new_text = text if text is not None else existing_chunk[\"text\"]\n\n if metadata is not None:\n new_metadata = existing_chunk[\"metadata\"].copy()\n new_metadata.update(metadata)\n else:\n new_metadata = existing_chunk[\"metadata\"].copy()\n if text is not None:\n new_metadata[\"token_count\"] = num_tokens_from_string(new_text)\n try:\n new_chunk_id = store.add_chunk(new_text, new_metadata)\n\n deleted = store.delete_chunk(chunk_id)\n if not deleted:\n current_app.logger.warning(\n f\"Failed to delete old chunk {chunk_id}, but new chunk {new_chunk_id} was created\"\n )\n return make_response(\n jsonify(\n {\n \"message\": \"Chunk updated successfully\",\n \"chunk_id\": new_chunk_id,\n \"original_chunk_id\": chunk_id,\n }\n ),\n 200,\n )\n except Exception as add_error:\n current_app.logger.error(f\"Failed to add updated chunk: {add_error}\")\n return make_response(\n jsonify({\"error\": \"Failed to update chunk - addition failed\"}), 500\n )\n except Exception as e:\n current_app.logger.error(f\"Error updating chunk: {e}\", exc_info=True)\n return make_response(jsonify({\"success\": False}), 500)\n" }, { "path": "application/api/user/sources/routes.py", "content": "\"\"\"Source document management routes.\"\"\"\n\nimport json\nimport math\n\nfrom bson.objectid import ObjectId\nfrom flask import current_app, jsonify, make_response, redirect, request\nfrom flask_restx import fields, Namespace, Resource\n\nfrom application.api import api\nfrom application.api.user.base import sources_collection\nfrom application.api.user.tasks import sync_source\nfrom application.core.settings import settings\nfrom application.storage.storage_creator import StorageCreator\nfrom application.utils import check_required_fields\nfrom application.vectorstore.vector_creator import VectorCreator\n\n\nsources_ns = Namespace(\n \"sources\", description=\"Source document management operations\", path=\"/api\"\n)\n\n\ndef _get_provider_from_remote_data(remote_data):\n if not remote_data:\n return None\n if isinstance(remote_data, dict):\n return remote_data.get(\"provider\")\n if isinstance(remote_data, str):\n try:\n remote_data_obj = json.loads(remote_data)\n except Exception:\n return None\n if isinstance(remote_data_obj, dict):\n return remote_data_obj.get(\"provider\")\n return None\n\n\n@sources_ns.route(\"/sources\")\nclass CombinedJson(Resource):\n @api.doc(description=\"Provide JSON file with combined available indexes\")\n def get(self):\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False}), 401)\n user = decoded_token.get(\"sub\")\n data = [\n {\n \"name\": \"Default\",\n \"date\": \"default\",\n \"model\": settings.EMBEDDINGS_NAME,\n \"location\": \"remote\",\n \"tokens\": \"\",\n \"retriever\": \"classic\",\n }\n ]\n\n try:\n for index in sources_collection.find({\"user\": user}).sort(\"date\", -1):\n provider = _get_provider_from_remote_data(index.get(\"remote_data\"))\n data.append(\n {\n \"id\": str(index[\"_id\"]),\n \"name\": index.get(\"name\"),\n \"date\": index.get(\"date\"),\n \"model\": settings.EMBEDDINGS_NAME,\n \"location\": \"local\",\n \"tokens\": index.get(\"tokens\", \"\"),\n \"retriever\": index.get(\"retriever\", \"classic\"),\n \"syncFrequency\": index.get(\"sync_frequency\", \"\"),\n \"provider\": provider,\n \"is_nested\": bool(index.get(\"directory_structure\")),\n \"type\": index.get(\n \"type\", \"file\"\n ), # Add type field with default \"file\"\n }\n )\n except Exception as err:\n current_app.logger.error(f\"Error retrieving sources: {err}\", exc_info=True)\n return make_response(jsonify({\"success\": False}), 400)\n return make_response(jsonify(data), 200)\n\n\n@sources_ns.route(\"/sources/paginated\")\nclass PaginatedSources(Resource):\n @api.doc(description=\"Get document with pagination, sorting and filtering\")\n def get(self):\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False}), 401)\n user = decoded_token.get(\"sub\")\n sort_field = request.args.get(\"sort\", \"date\") # Default to 'date'\n sort_order = request.args.get(\"order\", \"desc\") # Default to 'desc'\n page = int(request.args.get(\"page\", 1)) # Default to 1\n rows_per_page = int(request.args.get(\"rows\", 10)) # Default to 10\n # add .strip() to remove leading and trailing whitespaces\n\n search_term = request.args.get(\n \"search\", \"\"\n ).strip() # add search for filter documents\n\n # Prepare query for filtering\n\n query = {\"user\": user}\n if search_term:\n query[\"name\"] = {\n \"$regex\": search_term,\n \"$options\": \"i\", # using case-insensitive search\n }\n total_documents = sources_collection.count_documents(query)\n total_pages = max(1, math.ceil(total_documents / rows_per_page))\n page = min(\n max(1, page), total_pages\n ) # add this to make sure page inbound is within the range\n sort_order = 1 if sort_order == \"asc\" else -1\n skip = (page - 1) * rows_per_page\n\n try:\n documents = (\n sources_collection.find(query)\n .sort(sort_field, sort_order)\n .skip(skip)\n .limit(rows_per_page)\n )\n\n paginated_docs = []\n for doc in documents:\n provider = _get_provider_from_remote_data(doc.get(\"remote_data\"))\n doc_data = {\n \"id\": str(doc[\"_id\"]),\n \"name\": doc.get(\"name\", \"\"),\n \"date\": doc.get(\"date\", \"\"),\n \"model\": settings.EMBEDDINGS_NAME,\n \"location\": \"local\",\n \"tokens\": doc.get(\"tokens\", \"\"),\n \"retriever\": doc.get(\"retriever\", \"classic\"),\n \"syncFrequency\": doc.get(\"sync_frequency\", \"\"),\n \"provider\": provider,\n \"isNested\": bool(doc.get(\"directory_structure\")),\n \"type\": doc.get(\"type\", \"file\"),\n }\n paginated_docs.append(doc_data)\n response = {\n \"total\": total_documents,\n \"totalPages\": total_pages,\n \"currentPage\": page,\n \"paginated\": paginated_docs,\n }\n return make_response(jsonify(response), 200)\n except Exception as err:\n current_app.logger.error(\n f\"Error retrieving paginated sources: {err}\", exc_info=True\n )\n return make_response(jsonify({\"success\": False}), 400)\n\n\n@sources_ns.route(\"/delete_by_ids\")\nclass DeleteByIds(Resource):\n @api.doc(\n description=\"Deletes documents from the vector store by IDs\",\n params={\"path\": \"Comma-separated list of IDs\"},\n )\n def get(self):\n ids = request.args.get(\"path\")\n if not ids:\n return make_response(\n jsonify({\"success\": False, \"message\": \"Missing required fields\"}), 400\n )\n try:\n result = sources_collection.delete_index(ids=ids)\n if result:\n return make_response(jsonify({\"success\": True}), 200)\n except Exception as err:\n current_app.logger.error(f\"Error deleting indexes: {err}\", exc_info=True)\n return make_response(jsonify({\"success\": False}), 400)\n return make_response(jsonify({\"success\": False}), 400)\n\n\n@sources_ns.route(\"/delete_old\")\nclass DeleteOldIndexes(Resource):\n @api.doc(\n description=\"Deletes old indexes and associated files\",\n params={\"source_id\": \"The source ID to delete\"},\n )\n def get(self):\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False}), 401)\n source_id = request.args.get(\"source_id\")\n if not source_id:\n return make_response(\n jsonify({\"success\": False, \"message\": \"Missing required fields\"}), 400\n )\n doc = sources_collection.find_one(\n {\"_id\": ObjectId(source_id), \"user\": decoded_token.get(\"sub\")}\n )\n if not doc:\n return make_response(jsonify({\"status\": \"not found\"}), 404)\n storage = StorageCreator.get_storage()\n\n try:\n # Delete vector index\n\n if settings.VECTOR_STORE == \"faiss\":\n index_path = f\"indexes/{str(doc['_id'])}\"\n if storage.file_exists(f\"{index_path}/index.faiss\"):\n storage.delete_file(f\"{index_path}/index.faiss\")\n if storage.file_exists(f\"{index_path}/index.pkl\"):\n storage.delete_file(f\"{index_path}/index.pkl\")\n else:\n vectorstore = VectorCreator.create_vectorstore(\n settings.VECTOR_STORE, source_id=str(doc[\"_id\"])\n )\n vectorstore.delete_index()\n if \"file_path\" in doc and doc[\"file_path\"]:\n file_path = doc[\"file_path\"]\n if storage.is_directory(file_path):\n files = storage.list_files(file_path)\n for f in files:\n storage.delete_file(f)\n else:\n storage.delete_file(file_path)\n except FileNotFoundError:\n pass\n except Exception as err:\n current_app.logger.error(\n f\"Error deleting files and indexes: {err}\", exc_info=True\n )\n return make_response(jsonify({\"success\": False}), 400)\n sources_collection.delete_one({\"_id\": ObjectId(source_id)})\n return make_response(jsonify({\"success\": True}), 200)\n\n\n@sources_ns.route(\"/combine\")\nclass RedirectToSources(Resource):\n @api.doc(\n description=\"Redirects /api/combine to /api/sources for backward compatibility\"\n )\n def get(self):\n return redirect(\"/api/sources\", code=301)\n\n\n@sources_ns.route(\"/manage_sync\")\nclass ManageSync(Resource):\n manage_sync_model = api.model(\n \"ManageSyncModel\",\n {\n \"source_id\": fields.String(required=True, description=\"Source ID\"),\n \"sync_frequency\": fields.String(\n required=True,\n description=\"Sync frequency (never, daily, weekly, monthly)\",\n ),\n },\n )\n\n @api.expect(manage_sync_model)\n @api.doc(description=\"Manage sync frequency for sources\")\n def post(self):\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False}), 401)\n user = decoded_token.get(\"sub\")\n data = request.get_json() or {}\n required_fields = [\"source_id\", \"sync_frequency\"]\n missing_fields = check_required_fields(data, required_fields)\n if missing_fields:\n return missing_fields\n source_id = data[\"source_id\"]\n sync_frequency = data[\"sync_frequency\"]\n\n if sync_frequency not in [\"never\", \"daily\", \"weekly\", \"monthly\"]:\n return make_response(\n jsonify({\"success\": False, \"message\": \"Invalid frequency\"}), 400\n )\n update_data = {\"$set\": {\"sync_frequency\": sync_frequency}}\n try:\n sources_collection.update_one(\n {\n \"_id\": ObjectId(source_id),\n \"user\": user,\n },\n update_data,\n )\n except Exception as err:\n current_app.logger.error(\n f\"Error updating sync frequency: {err}\", exc_info=True\n )\n return make_response(jsonify({\"success\": False}), 400)\n return make_response(jsonify({\"success\": True}), 200)\n\n\n@sources_ns.route(\"/sync_source\")\nclass SyncSource(Resource):\n sync_source_model = api.model(\n \"SyncSourceModel\",\n {\"source_id\": fields.String(required=True, description=\"Source ID\")},\n )\n\n @api.expect(sync_source_model)\n @api.doc(description=\"Trigger an immediate sync for a source\")\n def post(self):\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False}), 401)\n user = decoded_token.get(\"sub\")\n data = request.get_json()\n required_fields = [\"source_id\"]\n missing_fields = check_required_fields(data, required_fields)\n if missing_fields:\n return missing_fields\n source_id = data[\"source_id\"]\n if not ObjectId.is_valid(source_id):\n return make_response(\n jsonify({\"success\": False, \"message\": \"Invalid source ID\"}), 400\n )\n doc = sources_collection.find_one(\n {\"_id\": ObjectId(source_id), \"user\": user}\n )\n if not doc:\n return make_response(\n jsonify({\"success\": False, \"message\": \"Source not found\"}), 404\n )\n source_type = doc.get(\"type\", \"\")\n if source_type.startswith(\"connector\"):\n return make_response(\n jsonify(\n {\n \"success\": False,\n \"message\": \"Connector sources must be synced via /api/connectors/sync\",\n }\n ),\n 400,\n )\n source_data = doc.get(\"remote_data\")\n if not source_data:\n return make_response(\n jsonify({\"success\": False, \"message\": \"Source is not syncable\"}), 400\n )\n try:\n task = sync_source.delay(\n source_data=source_data,\n job_name=doc.get(\"name\", \"\"),\n user=user,\n loader=source_type,\n sync_frequency=doc.get(\"sync_frequency\", \"never\"),\n retriever=doc.get(\"retriever\", \"classic\"),\n doc_id=source_id,\n )\n except Exception as err:\n current_app.logger.error(\n f\"Error starting sync for source {source_id}: {err}\",\n exc_info=True,\n )\n return make_response(jsonify({\"success\": False}), 400)\n return make_response(jsonify({\"success\": True, \"task_id\": task.id}), 200)\n\n\n@sources_ns.route(\"/directory_structure\")\nclass DirectoryStructure(Resource):\n @api.doc(\n description=\"Get the directory structure for a document\",\n params={\"id\": \"The document ID\"},\n )\n def get(self):\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False}), 401)\n user = decoded_token.get(\"sub\")\n doc_id = request.args.get(\"id\")\n\n if not doc_id:\n return make_response(jsonify({\"error\": \"Document ID is required\"}), 400)\n if not ObjectId.is_valid(doc_id):\n return make_response(jsonify({\"error\": \"Invalid document ID\"}), 400)\n try:\n doc = sources_collection.find_one({\"_id\": ObjectId(doc_id), \"user\": user})\n if not doc:\n return make_response(\n jsonify({\"error\": \"Document not found or access denied\"}), 404\n )\n directory_structure = doc.get(\"directory_structure\", {})\n base_path = doc.get(\"file_path\", \"\")\n\n provider = None\n remote_data = doc.get(\"remote_data\")\n try:\n if isinstance(remote_data, str) and remote_data:\n remote_data_obj = json.loads(remote_data)\n provider = remote_data_obj.get(\"provider\")\n except Exception as e:\n current_app.logger.warning(\n f\"Failed to parse remote_data for doc {doc_id}: {e}\"\n )\n return make_response(\n jsonify(\n {\n \"success\": True,\n \"directory_structure\": directory_structure,\n \"base_path\": base_path,\n \"provider\": provider,\n }\n ),\n 200,\n )\n except Exception as e:\n current_app.logger.error(\n f\"Error retrieving directory structure: {e}\", exc_info=True\n )\n return make_response(jsonify({\"success\": False, \"error\": \"Failed to retrieve directory structure\"}), 500)\n" }, { "path": "application/api/user/sources/upload.py", "content": "\"\"\"Source document management upload functionality.\"\"\"\n\nimport json\nimport os\nimport tempfile\nimport zipfile\n\nfrom bson.objectid import ObjectId\nfrom flask import current_app, jsonify, make_response, request\nfrom flask_restx import fields, Namespace, Resource\n\nfrom application.api import api\nfrom application.api.user.base import sources_collection\nfrom application.api.user.tasks import ingest, ingest_connector_task, ingest_remote\nfrom application.core.settings import settings\nfrom application.parser.connectors.connector_creator import ConnectorCreator\nfrom application.parser.file.constants import SUPPORTED_SOURCE_EXTENSIONS\nfrom application.storage.storage_creator import StorageCreator\nfrom application.stt.upload_limits import (\n AudioFileTooLargeError,\n build_stt_file_size_limit_message,\n enforce_audio_file_size_limit,\n is_audio_filename,\n)\nfrom application.utils import check_required_fields, safe_filename\n\n\nsources_upload_ns = Namespace(\n \"sources\", description=\"Source document management operations\", path=\"/api\"\n)\n\n\ndef _enforce_audio_path_size_limit(file_path: str, filename: str) -> None:\n if not is_audio_filename(filename):\n return\n enforce_audio_file_size_limit(os.path.getsize(file_path))\n\n\n@sources_upload_ns.route(\"/upload\")\nclass UploadFile(Resource):\n @api.expect(\n api.model(\n \"UploadModel\",\n {\n \"user\": fields.String(required=True, description=\"User ID\"),\n \"name\": fields.String(required=True, description=\"Job name\"),\n \"file\": fields.Raw(required=True, description=\"File(s) to upload\"),\n },\n )\n )\n @api.doc(\n description=\"Uploads a file to be vectorized and indexed\",\n )\n def post(self):\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False}), 401)\n data = request.form\n files = request.files.getlist(\"file\")\n required_fields = [\"user\", \"name\"]\n missing_fields = check_required_fields(data, required_fields)\n if missing_fields or not files or all(file.filename == \"\" for file in files):\n return make_response(\n jsonify(\n {\n \"status\": \"error\",\n \"message\": \"Missing required fields or files\",\n }\n ),\n 400,\n )\n user = decoded_token.get(\"sub\")\n job_name = request.form[\"name\"]\n\n # Create safe versions for filesystem operations\n\n safe_user = safe_filename(user)\n dir_name = safe_filename(job_name)\n base_path = f\"{settings.UPLOAD_FOLDER}/{safe_user}/{dir_name}\"\n file_name_map = {}\n\n try:\n storage = StorageCreator.get_storage()\n\n for file in files:\n original_filename = os.path.basename(file.filename)\n safe_file = safe_filename(original_filename)\n if original_filename:\n file_name_map[safe_file] = original_filename\n\n with tempfile.TemporaryDirectory() as temp_dir:\n temp_file_path = os.path.join(temp_dir, safe_file)\n file.save(temp_file_path)\n _enforce_audio_path_size_limit(temp_file_path, safe_file)\n\n # Only extract actual .zip files, not Office formats (.docx, .xlsx, .pptx)\n # which are technically zip archives but should be processed as-is\n is_office_format = safe_file.lower().endswith(\n (\".docx\", \".xlsx\", \".pptx\", \".odt\", \".ods\", \".odp\", \".epub\")\n )\n if zipfile.is_zipfile(temp_file_path) and not is_office_format:\n try:\n with zipfile.ZipFile(temp_file_path, \"r\") as zip_ref:\n zip_ref.extractall(path=temp_dir)\n\n # Walk through extracted files and upload them\n\n for root, _, files in os.walk(temp_dir):\n for extracted_file in files:\n if (\n os.path.join(root, extracted_file)\n == temp_file_path\n ):\n continue\n rel_path = os.path.relpath(\n os.path.join(root, extracted_file), temp_dir\n )\n storage_path = f\"{base_path}/{rel_path}\"\n _enforce_audio_path_size_limit(\n os.path.join(root, extracted_file),\n extracted_file,\n )\n\n with open(\n os.path.join(root, extracted_file), \"rb\"\n ) as f:\n storage.save_file(f, storage_path)\n except Exception as e:\n current_app.logger.error(\n f\"Error extracting zip: {e}\", exc_info=True\n )\n # If zip extraction fails, save the original zip file\n\n file_path = f\"{base_path}/{safe_file}\"\n with open(temp_file_path, \"rb\") as f:\n storage.save_file(f, file_path)\n else:\n # For non-zip files, save directly\n\n file_path = f\"{base_path}/{safe_file}\"\n with open(temp_file_path, \"rb\") as f:\n storage.save_file(f, file_path)\n task = ingest.delay(\n settings.UPLOAD_FOLDER,\n list(SUPPORTED_SOURCE_EXTENSIONS),\n job_name,\n user,\n file_path=base_path,\n filename=dir_name,\n file_name_map=file_name_map,\n )\n except AudioFileTooLargeError:\n return make_response(\n jsonify(\n {\n \"success\": False,\n \"message\": build_stt_file_size_limit_message(),\n }\n ),\n 413,\n )\n except Exception as err:\n current_app.logger.error(f\"Error uploading file: {err}\", exc_info=True)\n return make_response(jsonify({\"success\": False}), 400)\n return make_response(jsonify({\"success\": True, \"task_id\": task.id}), 200)\n\n\n@sources_upload_ns.route(\"/remote\")\nclass UploadRemote(Resource):\n @api.expect(\n api.model(\n \"RemoteUploadModel\",\n {\n \"user\": fields.String(required=True, description=\"User ID\"),\n \"source\": fields.String(\n required=True, description=\"Source of the data\"\n ),\n \"name\": fields.String(required=True, description=\"Job name\"),\n \"data\": fields.String(required=True, description=\"Data to process\"),\n \"repo_url\": fields.String(description=\"GitHub repository URL\"),\n },\n )\n )\n @api.doc(\n description=\"Uploads remote source for vectorization\",\n )\n def post(self):\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False}), 401)\n data = request.form\n required_fields = [\"user\", \"source\", \"name\", \"data\"]\n missing_fields = check_required_fields(data, required_fields)\n if missing_fields:\n return missing_fields\n try:\n config = json.loads(data[\"data\"])\n source_data = None\n\n if data[\"source\"] == \"github\":\n source_data = config.get(\"repo_url\")\n elif data[\"source\"] in [\"crawler\", \"url\"]:\n source_data = config.get(\"url\")\n elif data[\"source\"] == \"reddit\":\n source_data = config\n elif data[\"source\"] == \"s3\":\n source_data = config\n elif data[\"source\"] in ConnectorCreator.get_supported_connectors():\n session_token = config.get(\"session_token\")\n if not session_token:\n return make_response(\n jsonify(\n {\n \"success\": False,\n \"error\": f\"Missing session_token in {data['source']} configuration\",\n }\n ),\n 400,\n )\n # Process file_ids\n\n file_ids = config.get(\"file_ids\", [])\n if isinstance(file_ids, str):\n file_ids = [id.strip() for id in file_ids.split(\",\") if id.strip()]\n elif not isinstance(file_ids, list):\n file_ids = []\n # Process folder_ids\n\n folder_ids = config.get(\"folder_ids\", [])\n if isinstance(folder_ids, str):\n folder_ids = [\n id.strip() for id in folder_ids.split(\",\") if id.strip()\n ]\n elif not isinstance(folder_ids, list):\n folder_ids = []\n config[\"file_ids\"] = file_ids\n config[\"folder_ids\"] = folder_ids\n\n task = ingest_connector_task.delay(\n job_name=data[\"name\"],\n user=decoded_token.get(\"sub\"),\n source_type=data[\"source\"],\n session_token=session_token,\n file_ids=file_ids,\n folder_ids=folder_ids,\n recursive=config.get(\"recursive\", False),\n retriever=config.get(\"retriever\", \"classic\"),\n )\n return make_response(\n jsonify({\"success\": True, \"task_id\": task.id}), 200\n )\n task = ingest_remote.delay(\n source_data=source_data,\n job_name=data[\"name\"],\n user=decoded_token.get(\"sub\"),\n loader=data[\"source\"],\n )\n except Exception as err:\n current_app.logger.error(\n f\"Error uploading remote source: {err}\", exc_info=True\n )\n return make_response(jsonify({\"success\": False}), 400)\n return make_response(jsonify({\"success\": True, \"task_id\": task.id}), 200)\n\n\n@sources_upload_ns.route(\"/manage_source_files\")\nclass ManageSourceFiles(Resource):\n @api.expect(\n api.model(\n \"ManageSourceFilesModel\",\n {\n \"source_id\": fields.String(\n required=True, description=\"Source ID to modify\"\n ),\n \"operation\": fields.String(\n required=True,\n description=\"Operation: 'add', 'remove', or 'remove_directory'\",\n ),\n \"file_paths\": fields.List(\n fields.String,\n required=False,\n description=\"File paths to remove (for remove operation)\",\n ),\n \"directory_path\": fields.String(\n required=False,\n description=\"Directory path to remove (for remove_directory operation)\",\n ),\n \"file\": fields.Raw(\n required=False, description=\"Files to add (for add operation)\"\n ),\n \"parent_dir\": fields.String(\n required=False,\n description=\"Parent directory path relative to source root\",\n ),\n },\n )\n )\n @api.doc(\n description=\"Add files, remove files, or remove directories from an existing source\",\n )\n def post(self):\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(\n jsonify({\"success\": False, \"message\": \"Unauthorized\"}), 401\n )\n user = decoded_token.get(\"sub\")\n source_id = request.form.get(\"source_id\")\n operation = request.form.get(\"operation\")\n\n if not source_id or not operation:\n return make_response(\n jsonify(\n {\n \"success\": False,\n \"message\": \"source_id and operation are required\",\n }\n ),\n 400,\n )\n if operation not in [\"add\", \"remove\", \"remove_directory\"]:\n return make_response(\n jsonify(\n {\n \"success\": False,\n \"message\": \"operation must be 'add', 'remove', or 'remove_directory'\",\n }\n ),\n 400,\n )\n try:\n ObjectId(source_id)\n except Exception:\n return make_response(\n jsonify({\"success\": False, \"message\": \"Invalid source ID format\"}), 400\n )\n try:\n source = sources_collection.find_one(\n {\"_id\": ObjectId(source_id), \"user\": user}\n )\n if not source:\n return make_response(\n jsonify(\n {\n \"success\": False,\n \"message\": \"Source not found or access denied\",\n }\n ),\n 404,\n )\n except Exception as err:\n current_app.logger.error(f\"Error finding source: {err}\", exc_info=True)\n return make_response(\n jsonify({\"success\": False, \"message\": \"Database error\"}), 500\n )\n try:\n storage = StorageCreator.get_storage()\n source_file_path = source.get(\"file_path\", \"\")\n parent_dir = request.form.get(\"parent_dir\", \"\")\n file_name_map = source.get(\"file_name_map\") or {}\n if isinstance(file_name_map, str):\n try:\n file_name_map = json.loads(file_name_map)\n except Exception:\n file_name_map = {}\n if not isinstance(file_name_map, dict):\n file_name_map = {}\n\n if parent_dir and (parent_dir.startswith(\"/\") or \"..\" in parent_dir):\n return make_response(\n jsonify(\n {\"success\": False, \"message\": \"Invalid parent directory path\"}\n ),\n 400,\n )\n if operation == \"add\":\n files = request.files.getlist(\"file\")\n if not files or all(file.filename == \"\" for file in files):\n return make_response(\n jsonify(\n {\n \"success\": False,\n \"message\": \"No files provided for add operation\",\n }\n ),\n 400,\n )\n added_files = []\n map_updated = False\n\n target_dir = source_file_path\n if parent_dir:\n target_dir = f\"{source_file_path}/{parent_dir}\"\n for file in files:\n if file.filename:\n original_filename = os.path.basename(file.filename)\n safe_filename_str = safe_filename(original_filename)\n file_path = f\"{target_dir}/{safe_filename_str}\"\n\n # Save file to storage\n\n storage.save_file(file, file_path)\n added_files.append(safe_filename_str)\n if original_filename:\n relative_key = (\n f\"{parent_dir}/{safe_filename_str}\"\n if parent_dir\n else safe_filename_str\n )\n file_name_map[relative_key] = original_filename\n map_updated = True\n\n if map_updated:\n sources_collection.update_one(\n {\"_id\": ObjectId(source_id)},\n {\"$set\": {\"file_name_map\": file_name_map}},\n )\n # Trigger re-ingestion pipeline\n\n from application.api.user.tasks import reingest_source_task\n\n task = reingest_source_task.delay(source_id=source_id, user=user)\n\n return make_response(\n jsonify(\n {\n \"success\": True,\n \"message\": f\"Added {len(added_files)} files\",\n \"added_files\": added_files,\n \"parent_dir\": parent_dir,\n \"reingest_task_id\": task.id,\n }\n ),\n 200,\n )\n elif operation == \"remove\":\n file_paths_str = request.form.get(\"file_paths\")\n if not file_paths_str:\n return make_response(\n jsonify(\n {\n \"success\": False,\n \"message\": \"file_paths required for remove operation\",\n }\n ),\n 400,\n )\n try:\n file_paths = (\n json.loads(file_paths_str)\n if isinstance(file_paths_str, str)\n else file_paths_str\n )\n except Exception:\n return make_response(\n jsonify(\n {\"success\": False, \"message\": \"Invalid file_paths format\"}\n ),\n 400,\n )\n # Remove files from storage and directory structure\n\n removed_files = []\n map_updated = False\n for file_path in file_paths:\n full_path = f\"{source_file_path}/{file_path}\"\n\n # Remove from storage\n\n if storage.file_exists(full_path):\n storage.delete_file(full_path)\n removed_files.append(file_path)\n if file_path in file_name_map:\n file_name_map.pop(file_path, None)\n map_updated = True\n\n if map_updated and isinstance(file_name_map, dict):\n sources_collection.update_one(\n {\"_id\": ObjectId(source_id)},\n {\"$set\": {\"file_name_map\": file_name_map}},\n )\n # Trigger re-ingestion pipeline\n\n from application.api.user.tasks import reingest_source_task\n\n task = reingest_source_task.delay(source_id=source_id, user=user)\n\n return make_response(\n jsonify(\n {\n \"success\": True,\n \"message\": f\"Removed {len(removed_files)} files\",\n \"removed_files\": removed_files,\n \"reingest_task_id\": task.id,\n }\n ),\n 200,\n )\n elif operation == \"remove_directory\":\n directory_path = request.form.get(\"directory_path\")\n if not directory_path:\n return make_response(\n jsonify(\n {\n \"success\": False,\n \"message\": \"directory_path required for remove_directory operation\",\n }\n ),\n 400,\n )\n # Validate directory path (prevent path traversal)\n\n if directory_path.startswith(\"/\") or \"..\" in directory_path:\n current_app.logger.warning(\n f\"Invalid directory path attempted for removal. \"\n f\"User: {user}, Source ID: {source_id}, Directory path: {directory_path}\"\n )\n return make_response(\n jsonify(\n {\"success\": False, \"message\": \"Invalid directory path\"}\n ),\n 400,\n )\n full_directory_path = (\n f\"{source_file_path}/{directory_path}\"\n if directory_path\n else source_file_path\n )\n\n if not storage.is_directory(full_directory_path):\n current_app.logger.warning(\n f\"Directory not found or is not a directory for removal. \"\n f\"User: {user}, Source ID: {source_id}, Directory path: {directory_path}, \"\n f\"Full path: {full_directory_path}\"\n )\n return make_response(\n jsonify(\n {\n \"success\": False,\n \"message\": \"Directory not found or is not a directory\",\n }\n ),\n 404,\n )\n success = storage.remove_directory(full_directory_path)\n\n if not success:\n current_app.logger.error(\n f\"Failed to remove directory from storage. \"\n f\"User: {user}, Source ID: {source_id}, Directory path: {directory_path}, \"\n f\"Full path: {full_directory_path}\"\n )\n return make_response(\n jsonify(\n {\"success\": False, \"message\": \"Failed to remove directory\"}\n ),\n 500,\n )\n current_app.logger.info(\n f\"Successfully removed directory. \"\n f\"User: {user}, Source ID: {source_id}, Directory path: {directory_path}, \"\n f\"Full path: {full_directory_path}\"\n )\n if directory_path and file_name_map:\n prefix = f\"{directory_path.rstrip('/')}/\"\n keys_to_remove = [\n key\n for key in file_name_map.keys()\n if key == directory_path or key.startswith(prefix)\n ]\n if keys_to_remove:\n for key in keys_to_remove:\n file_name_map.pop(key, None)\n sources_collection.update_one(\n {\"_id\": ObjectId(source_id)},\n {\"$set\": {\"file_name_map\": file_name_map}},\n )\n\n # Trigger re-ingestion pipeline\n\n from application.api.user.tasks import reingest_source_task\n\n task = reingest_source_task.delay(source_id=source_id, user=user)\n\n return make_response(\n jsonify(\n {\n \"success\": True,\n \"message\": f\"Successfully removed directory: {directory_path}\",\n \"removed_directory\": directory_path,\n \"reingest_task_id\": task.id,\n }\n ),\n 200,\n )\n except Exception as err:\n error_context = f\"operation={operation}, user={user}, source_id={source_id}\"\n if operation == \"remove_directory\":\n directory_path = request.form.get(\"directory_path\", \"\")\n error_context += f\", directory_path={directory_path}\"\n elif operation == \"remove\":\n file_paths_str = request.form.get(\"file_paths\", \"\")\n error_context += f\", file_paths={file_paths_str}\"\n elif operation == \"add\":\n parent_dir = request.form.get(\"parent_dir\", \"\")\n error_context += f\", parent_dir={parent_dir}\"\n current_app.logger.error(\n f\"Error managing source files: {err} ({error_context})\", exc_info=True\n )\n return make_response(\n jsonify({\"success\": False, \"message\": \"Operation failed\"}), 500\n )\n\n\n@sources_upload_ns.route(\"/task_status\")\nclass TaskStatus(Resource):\n task_status_model = api.model(\n \"TaskStatusModel\",\n {\"task_id\": fields.String(required=True, description=\"Task ID\")},\n )\n\n @api.expect(task_status_model)\n @api.doc(description=\"Get celery job status\")\n def get(self):\n task_id = request.args.get(\"task_id\")\n if not task_id:\n return make_response(\n jsonify({\"success\": False, \"message\": \"Task ID is required\"}), 400\n )\n try:\n from application.celery_init import celery\n\n task = celery.AsyncResult(task_id)\n task_meta = task.info\n print(f\"Task status: {task.status}\")\n\n if task.status == \"PENDING\":\n inspect = celery.control.inspect()\n active_workers = inspect.ping()\n if not active_workers:\n raise ConnectionError(\"Service unavailable\")\n\n if not isinstance(\n task_meta, (dict, list, str, int, float, bool, type(None))\n ):\n task_meta = str(task_meta) # Convert to a string representation\n except ConnectionError as err:\n current_app.logger.error(f\"Connection error getting task status: {err}\")\n return make_response(\n jsonify({\"success\": False, \"message\": \"Service unavailable\"}), 503\n )\n except Exception as err:\n current_app.logger.error(f\"Error getting task status: {err}\", exc_info=True)\n return make_response(jsonify({\"success\": False}), 400)\n return make_response(jsonify({\"status\": task.status, \"result\": task_meta}), 200)\n" }, { "path": "application/api/user/tasks.py", "content": "from datetime import timedelta\n\nfrom application.celery_init import celery\nfrom application.worker import (\n agent_webhook_worker,\n attachment_worker,\n ingest_worker,\n mcp_oauth,\n mcp_oauth_status,\n remote_worker,\n sync,\n sync_worker,\n)\n\n\n@celery.task(bind=True)\ndef ingest(\n self, directory, formats, job_name, user, file_path, filename, file_name_map=None\n):\n resp = ingest_worker(\n self,\n directory,\n formats,\n job_name,\n file_path,\n filename,\n user,\n file_name_map=file_name_map,\n )\n return resp\n\n\n@celery.task(bind=True)\ndef ingest_remote(self, source_data, job_name, user, loader):\n resp = remote_worker(self, source_data, job_name, user, loader)\n return resp\n\n\n@celery.task(bind=True)\ndef reingest_source_task(self, source_id, user):\n from application.worker import reingest_source_worker\n\n resp = reingest_source_worker(self, source_id, user)\n return resp\n\n\n@celery.task(bind=True)\ndef schedule_syncs(self, frequency):\n resp = sync_worker(self, frequency)\n return resp\n\n\n@celery.task(bind=True)\ndef sync_source(\n self,\n source_data,\n job_name,\n user,\n loader,\n sync_frequency,\n retriever,\n doc_id,\n):\n resp = sync(\n self,\n source_data,\n job_name,\n user,\n loader,\n sync_frequency,\n retriever,\n doc_id,\n )\n return resp\n\n\n@celery.task(bind=True)\ndef store_attachment(self, file_info, user):\n resp = attachment_worker(self, file_info, user)\n return resp\n\n\n@celery.task(bind=True)\ndef process_agent_webhook(self, agent_id, payload):\n resp = agent_webhook_worker(self, agent_id, payload)\n return resp\n\n\n@celery.task(bind=True)\ndef ingest_connector_task(\n self,\n job_name,\n user,\n source_type,\n session_token=None,\n file_ids=None,\n folder_ids=None,\n recursive=True,\n retriever=\"classic\",\n operation_mode=\"upload\",\n doc_id=None,\n sync_frequency=\"never\",\n):\n from application.worker import ingest_connector\n\n resp = ingest_connector(\n self,\n job_name,\n user,\n source_type,\n session_token=session_token,\n file_ids=file_ids,\n folder_ids=folder_ids,\n recursive=recursive,\n retriever=retriever,\n operation_mode=operation_mode,\n doc_id=doc_id,\n sync_frequency=sync_frequency,\n )\n return resp\n\n\n@celery.on_after_configure.connect\ndef setup_periodic_tasks(sender, **kwargs):\n sender.add_periodic_task(\n timedelta(days=1),\n schedule_syncs.s(\"daily\"),\n )\n sender.add_periodic_task(\n timedelta(weeks=1),\n schedule_syncs.s(\"weekly\"),\n )\n sender.add_periodic_task(\n timedelta(days=30),\n schedule_syncs.s(\"monthly\"),\n )\n\n\n@celery.task(bind=True)\ndef mcp_oauth_task(self, config, user):\n resp = mcp_oauth(self, config, user)\n return resp\n\n\n@celery.task(bind=True)\ndef mcp_oauth_status_task(self, task_id):\n resp = mcp_oauth_status(self, task_id)\n return resp\n" }, { "path": "application/api/user/tools/__init__.py", "content": "\"\"\"Tools module.\"\"\"\n\nfrom .mcp import tools_mcp_ns\nfrom .routes import tools_ns\n\n__all__ = [\"tools_ns\", \"tools_mcp_ns\"]\n" }, { "path": "application/api/user/tools/mcp.py", "content": "\"\"\"Tool management MCP server integration.\"\"\"\n\nimport json\nfrom urllib.parse import urlencode, urlparse\n\nfrom bson.objectid import ObjectId\nfrom flask import current_app, jsonify, make_response, redirect, request\nfrom flask_restx import Namespace, Resource, fields\n\nfrom application.agents.tools.mcp_tool import MCPOAuthManager, MCPTool\nfrom application.api import api\nfrom application.api.user.base import user_tools_collection\nfrom application.api.user.tools.routes import transform_actions\nfrom application.cache import get_redis_instance\nfrom application.core.mongo_db import MongoDB\nfrom application.core.settings import settings\nfrom application.security.encryption import decrypt_credentials, encrypt_credentials\nfrom application.utils import check_required_fields\n\ntools_mcp_ns = Namespace(\"tools\", description=\"Tool management operations\", path=\"/api\")\n\n_mongo = MongoDB.get_client()\n_db = _mongo[settings.MONGO_DB_NAME]\n_connector_sessions = _db[\"connector_sessions\"]\n\n_ALLOWED_TRANSPORTS = {\"auto\", \"sse\", \"http\"}\n\n\ndef _sanitize_mcp_transport(config):\n \"\"\"Normalise and validate the transport_type field.\n\n Strips ``command`` / ``args`` keys that are only valid for local STDIO\n transports and returns the cleaned transport type string.\n \"\"\"\n transport_type = (config.get(\"transport_type\") or \"auto\").lower()\n if transport_type not in _ALLOWED_TRANSPORTS:\n raise ValueError(f\"Unsupported transport_type: {transport_type}\")\n config.pop(\"command\", None)\n config.pop(\"args\", None)\n config[\"transport_type\"] = transport_type\n return transport_type\n\n\ndef _extract_auth_credentials(config):\n \"\"\"Build an ``auth_credentials`` dict from the raw MCP config.\"\"\"\n auth_credentials = {}\n auth_type = config.get(\"auth_type\", \"none\")\n\n if auth_type == \"api_key\":\n if config.get(\"api_key\"):\n auth_credentials[\"api_key\"] = config[\"api_key\"]\n if config.get(\"api_key_header\"):\n auth_credentials[\"api_key_header\"] = config[\"api_key_header\"]\n elif auth_type == \"bearer\":\n if config.get(\"bearer_token\"):\n auth_credentials[\"bearer_token\"] = config[\"bearer_token\"]\n elif auth_type == \"basic\":\n if config.get(\"username\"):\n auth_credentials[\"username\"] = config[\"username\"]\n if config.get(\"password\"):\n auth_credentials[\"password\"] = config[\"password\"]\n\n return auth_credentials\n\n\n@tools_mcp_ns.route(\"/mcp_server/test\")\nclass TestMCPServerConfig(Resource):\n @api.expect(\n api.model(\n \"MCPServerTestModel\",\n {\n \"config\": fields.Raw(\n required=True, description=\"MCP server configuration to test\"\n ),\n },\n )\n )\n @api.doc(description=\"Test MCP server connection with provided configuration\")\n def post(self):\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False}), 401)\n user = decoded_token.get(\"sub\")\n data = request.get_json()\n\n required_fields = [\"config\"]\n missing_fields = check_required_fields(data, required_fields)\n if missing_fields:\n return missing_fields\n try:\n config = data[\"config\"]\n try:\n _sanitize_mcp_transport(config)\n except ValueError:\n return make_response(\n jsonify({\"success\": False, \"error\": \"Unsupported transport_type\"}),\n 400,\n )\n\n auth_credentials = _extract_auth_credentials(config)\n test_config = config.copy()\n test_config[\"auth_credentials\"] = auth_credentials\n\n mcp_tool = MCPTool(config=test_config, user_id=user)\n result = mcp_tool.test_connection()\n\n if result.get(\"requires_oauth\"):\n return make_response(jsonify(result), 200)\n\n if not result.get(\"success\") and \"message\" in result:\n current_app.logger.error(\n f\"MCP connection test failed: {result.get('message')}\"\n )\n result[\"message\"] = \"Connection test failed\"\n\n return make_response(jsonify(result), 200)\n except Exception as e:\n current_app.logger.error(f\"Error testing MCP server: {e}\", exc_info=True)\n return make_response(\n jsonify({\"success\": False, \"error\": \"Connection test failed\"}),\n 500,\n )\n\n\n@tools_mcp_ns.route(\"/mcp_server/save\")\nclass MCPServerSave(Resource):\n @api.expect(\n api.model(\n \"MCPServerSaveModel\",\n {\n \"id\": fields.String(\n required=False, description=\"Tool ID for updates (optional)\"\n ),\n \"displayName\": fields.String(\n required=True, description=\"Display name for the MCP server\"\n ),\n \"config\": fields.Raw(\n required=True, description=\"MCP server configuration\"\n ),\n \"status\": fields.Boolean(\n required=False, default=True, description=\"Tool status\"\n ),\n },\n )\n )\n @api.doc(description=\"Create or update MCP server with automatic tool discovery\")\n def post(self):\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False}), 401)\n user = decoded_token.get(\"sub\")\n data = request.get_json()\n\n required_fields = [\"displayName\", \"config\"]\n missing_fields = check_required_fields(data, required_fields)\n if missing_fields:\n return missing_fields\n try:\n config = data[\"config\"]\n try:\n _sanitize_mcp_transport(config)\n except ValueError:\n return make_response(\n jsonify({\"success\": False, \"error\": \"Unsupported transport_type\"}),\n 400,\n )\n\n auth_credentials = _extract_auth_credentials(config)\n auth_type = config.get(\"auth_type\", \"none\")\n mcp_config = config.copy()\n mcp_config[\"auth_credentials\"] = auth_credentials\n\n if auth_type == \"oauth\":\n if not config.get(\"oauth_task_id\"):\n return make_response(\n jsonify(\n {\n \"success\": False,\n \"error\": \"Connection not authorized. Please complete the OAuth authorization first.\",\n }\n ),\n 400,\n )\n redis_client = get_redis_instance()\n manager = MCPOAuthManager(redis_client)\n result = manager.get_oauth_status(config[\"oauth_task_id\"])\n if not result.get(\"status\") == \"completed\":\n return make_response(\n jsonify(\n {\n \"success\": False,\n \"error\": \"OAuth failed or not completed. Please try authorizing again.\",\n }\n ),\n 400,\n )\n actions_metadata = result.get(\"tools\", [])\n elif auth_type == \"none\" or auth_credentials:\n mcp_tool = MCPTool(config=mcp_config, user_id=user)\n mcp_tool.discover_tools()\n actions_metadata = mcp_tool.get_actions_metadata()\n else:\n raise Exception(\n \"No valid credentials provided for the selected authentication type\"\n )\n storage_config = config.copy()\n\n tool_id = data.get(\"id\")\n existing_encrypted = None\n if tool_id:\n existing_doc = user_tools_collection.find_one(\n {\"_id\": ObjectId(tool_id), \"user\": user, \"name\": \"mcp_tool\"}\n )\n if existing_doc:\n existing_encrypted = existing_doc.get(\"config\", {}).get(\n \"encrypted_credentials\"\n )\n\n if auth_credentials:\n if existing_encrypted:\n existing_secrets = decrypt_credentials(existing_encrypted, user)\n existing_secrets.update(auth_credentials)\n auth_credentials = existing_secrets\n storage_config[\"encrypted_credentials\"] = encrypt_credentials(\n auth_credentials, user\n )\n elif existing_encrypted:\n storage_config[\"encrypted_credentials\"] = existing_encrypted\n\n for field in [\n \"api_key\",\n \"bearer_token\",\n \"username\",\n \"password\",\n \"api_key_header\",\n \"redirect_uri\",\n ]:\n storage_config.pop(field, None)\n transformed_actions = transform_actions(actions_metadata)\n tool_data = {\n \"name\": \"mcp_tool\",\n \"displayName\": data[\"displayName\"],\n \"customName\": data[\"displayName\"],\n \"description\": f\"MCP Server: {storage_config.get('server_url', 'Unknown')}\",\n \"config\": storage_config,\n \"actions\": transformed_actions,\n \"status\": data.get(\"status\", True),\n \"user\": user,\n }\n\n if tool_id:\n result = user_tools_collection.update_one(\n {\"_id\": ObjectId(tool_id), \"user\": user, \"name\": \"mcp_tool\"},\n {\"$set\": {k: v for k, v in tool_data.items() if k != \"user\"}},\n )\n if result.matched_count == 0:\n return make_response(\n jsonify(\n {\n \"success\": False,\n \"error\": \"Tool not found or access denied\",\n }\n ),\n 404,\n )\n response_data = {\n \"success\": True,\n \"id\": tool_id,\n \"message\": f\"MCP server updated successfully! Discovered {len(transformed_actions)} tools.\",\n \"tools_count\": len(transformed_actions),\n }\n else:\n result = user_tools_collection.insert_one(tool_data)\n tool_id = str(result.inserted_id)\n response_data = {\n \"success\": True,\n \"id\": tool_id,\n \"message\": f\"MCP server created successfully! Discovered {len(transformed_actions)} tools.\",\n \"tools_count\": len(transformed_actions),\n }\n return make_response(jsonify(response_data), 200)\n except Exception as e:\n current_app.logger.error(f\"Error saving MCP server: {e}\", exc_info=True)\n return make_response(\n jsonify({\"success\": False, \"error\": \"Failed to save MCP server\"}),\n 500,\n )\n\n\n@tools_mcp_ns.route(\"/mcp_server/callback\")\nclass MCPOAuthCallback(Resource):\n @api.expect(\n api.model(\n \"MCPServerCallbackModel\",\n {\n \"code\": fields.String(required=True, description=\"Authorization code\"),\n \"state\": fields.String(required=True, description=\"State parameter\"),\n \"error\": fields.String(\n required=False, description=\"Error message (if any)\"\n ),\n },\n )\n )\n @api.doc(\n description=\"Handle OAuth callback by providing the authorization code and state\"\n )\n def get(self):\n code = request.args.get(\"code\")\n state = request.args.get(\"state\")\n error = request.args.get(\"error\")\n\n if error:\n params = {\n \"status\": \"error\",\n \"message\": f\"OAuth error: {error}. Please try again and make sure to grant all requested permissions, including offline access.\",\n \"provider\": \"mcp_tool\",\n }\n return redirect(f\"/api/connectors/callback-status?{urlencode(params)}\")\n if not code or not state:\n return redirect(\n \"/api/connectors/callback-status?status=error&message=Authorization+code+or+state+not+provided.+Please+complete+the+authorization+process+and+make+sure+to+grant+offline+access.&provider=mcp_tool\"\n )\n try:\n redis_client = get_redis_instance()\n if not redis_client:\n return redirect(\n \"/api/connectors/callback-status?status=error&message=Internal+server+error:+Redis+not+available.&provider=mcp_tool\"\n )\n manager = MCPOAuthManager(redis_client)\n success = manager.handle_oauth_callback(state, code, error)\n if success:\n return redirect(\n \"/api/connectors/callback-status?status=success&message=Authorization+code+received+successfully.+You+can+close+this+window.&provider=mcp_tool\"\n )\n else:\n return redirect(\n \"/api/connectors/callback-status?status=error&message=OAuth+callback+failed.&provider=mcp_tool\"\n )\n except Exception as e:\n current_app.logger.error(\n f\"Error handling MCP OAuth callback: {str(e)}\", exc_info=True\n )\n return redirect(\n \"/api/connectors/callback-status?status=error&message=Internal+server+error.&provider=mcp_tool\"\n )\n\n\n@tools_mcp_ns.route(\"/mcp_server/oauth_status/\")\nclass MCPOAuthStatus(Resource):\n def get(self, task_id):\n try:\n redis_client = get_redis_instance()\n status_key = f\"mcp_oauth_status:{task_id}\"\n status_data = redis_client.get(status_key)\n\n if status_data:\n status = json.loads(status_data)\n if \"tools\" in status and isinstance(status[\"tools\"], list):\n status[\"tools\"] = [\n {\n \"name\": t.get(\"name\", \"unknown\"),\n \"description\": t.get(\"description\", \"\"),\n }\n for t in status[\"tools\"]\n ]\n return make_response(\n jsonify({\"success\": True, \"task_id\": task_id, **status})\n )\n else:\n return make_response(\n jsonify(\n {\n \"success\": True,\n \"task_id\": task_id,\n \"status\": \"pending\",\n \"message\": \"Waiting for OAuth to start...\",\n }\n ),\n 200,\n )\n except Exception as e:\n current_app.logger.error(\n f\"Error getting OAuth status for task {task_id}: {str(e)}\",\n exc_info=True,\n )\n return make_response(\n jsonify(\n {\n \"success\": False,\n \"error\": \"Failed to get OAuth status\",\n \"task_id\": task_id,\n }\n ),\n 500,\n )\n\n\n@tools_mcp_ns.route(\"/mcp_server/auth_status\")\nclass MCPAuthStatus(Resource):\n @api.doc(\n description=\"Batch check auth status for all MCP tools. \"\n \"Lightweight DB-only check — no network calls to MCP servers.\"\n )\n def get(self):\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False}), 401)\n user = decoded_token.get(\"sub\")\n try:\n mcp_tools = list(\n user_tools_collection.find(\n {\"user\": user, \"name\": \"mcp_tool\"},\n {\"_id\": 1, \"config\": 1},\n )\n )\n if not mcp_tools:\n return make_response(jsonify({\"success\": True, \"statuses\": {}}), 200)\n\n oauth_server_urls = {}\n statuses = {}\n for tool in mcp_tools:\n tool_id = str(tool[\"_id\"])\n config = tool.get(\"config\", {})\n auth_type = config.get(\"auth_type\", \"none\")\n if auth_type == \"oauth\":\n server_url = config.get(\"server_url\", \"\")\n if server_url:\n parsed = urlparse(server_url)\n base_url = f\"{parsed.scheme}://{parsed.netloc}\"\n oauth_server_urls[tool_id] = base_url\n else:\n statuses[tool_id] = \"needs_auth\"\n else:\n statuses[tool_id] = \"configured\"\n\n if oauth_server_urls:\n unique_urls = list(set(oauth_server_urls.values()))\n sessions = list(\n _connector_sessions.find(\n {\"user_id\": user, \"server_url\": {\"$in\": unique_urls}},\n {\"server_url\": 1, \"tokens\": 1},\n )\n )\n url_has_tokens = {\n doc[\"server_url\"]: bool(doc.get(\"tokens\", {}).get(\"access_token\"))\n for doc in sessions\n }\n for tool_id, base_url in oauth_server_urls.items():\n if url_has_tokens.get(base_url):\n statuses[tool_id] = \"connected\"\n else:\n statuses[tool_id] = \"needs_auth\"\n\n return make_response(jsonify({\"success\": True, \"statuses\": statuses}), 200)\n except Exception as e:\n current_app.logger.error(\n \"Error checking MCP auth status: %s\", e, exc_info=True\n )\n return make_response(\n jsonify({\"success\": False, \"error\": \"Failed to check auth status\"}),\n 500,\n )\n" }, { "path": "application/api/user/tools/routes.py", "content": "\"\"\"Tool management routes.\"\"\"\n\nfrom bson.objectid import ObjectId\nfrom flask import current_app, jsonify, make_response, request\nfrom flask_restx import fields, Namespace, Resource\n\nfrom application.agents.tools.spec_parser import parse_spec\nfrom application.agents.tools.tool_manager import ToolManager\nfrom application.api import api\nfrom application.api.user.base import user_tools_collection\nfrom application.security.encryption import decrypt_credentials, encrypt_credentials\nfrom application.utils import check_required_fields, validate_function_name\n\ntool_config = {}\ntool_manager = ToolManager(config=tool_config)\n\n\ndef _encrypt_secret_fields(config, config_requirements, user_id):\n secret_keys = [\n key for key, spec in config_requirements.items()\n if spec.get(\"secret\") and key in config and config[key]\n ]\n if not secret_keys:\n return config\n\n storage_config = config.copy()\n secret_values = {k: config[k] for k in secret_keys}\n storage_config[\"encrypted_credentials\"] = encrypt_credentials(secret_values, user_id)\n for key in secret_keys:\n storage_config.pop(key, None)\n return storage_config\n\n\ndef _validate_config(config, config_requirements, has_existing_secrets=False):\n errors = {}\n for key, spec in config_requirements.items():\n depends_on = spec.get(\"depends_on\")\n if depends_on:\n if not all(config.get(dk) == dv for dk, dv in depends_on.items()):\n continue\n if spec.get(\"required\") and not config.get(key):\n if has_existing_secrets and spec.get(\"secret\"):\n continue\n errors[key] = f\"{spec.get('label', key)} is required\"\n value = config.get(key)\n if value is not None and value != \"\":\n if spec.get(\"type\") == \"number\":\n try:\n num = float(value)\n if key == \"timeout\" and (num < 1 or num > 300):\n errors[key] = \"Timeout must be between 1 and 300\"\n except (ValueError, TypeError):\n errors[key] = f\"{spec.get('label', key)} must be a number\"\n if spec.get(\"enum\") and value not in spec[\"enum\"]:\n errors[key] = f\"Invalid value for {spec.get('label', key)}\"\n return errors\n\n\ndef _merge_secrets_on_update(new_config, existing_config, config_requirements, user_id):\n \"\"\"Merge incoming config with existing encrypted secrets and re-encrypt.\n\n For updates, the client may omit unchanged secret values. This helper\n decrypts any previously stored secrets, overlays whatever the client *did*\n send, strips plain-text secrets from the stored config, and re-encrypts\n the merged result.\n\n Returns the final ``config`` dict ready for persistence.\n \"\"\"\n secret_keys = [\n key for key, spec in config_requirements.items()\n if spec.get(\"secret\")\n ]\n\n if not secret_keys:\n return new_config\n\n existing_secrets = {}\n if \"encrypted_credentials\" in existing_config:\n existing_secrets = decrypt_credentials(\n existing_config[\"encrypted_credentials\"], user_id\n )\n\n merged_secrets = existing_secrets.copy()\n for key in secret_keys:\n if key in new_config and new_config[key]:\n merged_secrets[key] = new_config[key]\n\n # Start from existing non-secret values, then overlay incoming non-secrets\n storage_config = {\n k: v for k, v in existing_config.items()\n if k not in secret_keys and k != \"encrypted_credentials\"\n }\n storage_config.update(\n {k: v for k, v in new_config.items() if k not in secret_keys}\n )\n\n if merged_secrets:\n storage_config[\"encrypted_credentials\"] = encrypt_credentials(\n merged_secrets, user_id\n )\n else:\n storage_config.pop(\"encrypted_credentials\", None)\n\n storage_config.pop(\"has_encrypted_credentials\", None)\n return storage_config\n\n\ndef transform_actions(actions_metadata):\n \"\"\"Set default flags on action metadata for storage.\n\n Marks each action as active, sets ``filled_by_llm`` and ``value`` on every\n parameter property. Used by both the generic create_tool and MCP save routes.\n \"\"\"\n transformed = []\n for action in actions_metadata:\n action[\"active\"] = True\n if \"parameters\" in action:\n props = action[\"parameters\"].get(\"properties\", {})\n for param_details in props.values():\n param_details[\"filled_by_llm\"] = True\n param_details[\"value\"] = \"\"\n transformed.append(action)\n return transformed\n\n\ntools_ns = Namespace(\"tools\", description=\"Tool management operations\", path=\"/api\")\n\n\n@tools_ns.route(\"/available_tools\")\nclass AvailableTools(Resource):\n @api.doc(description=\"Get available tools for a user\")\n def get(self):\n try:\n tools_metadata = []\n for tool_name, tool_instance in tool_manager.tools.items():\n doc = tool_instance.__doc__.strip()\n lines = doc.split(\"\\n\", 1)\n name = lines[0].strip()\n description = lines[1].strip() if len(lines) > 1 else \"\"\n config_req = tool_instance.get_config_requirements()\n actions = tool_instance.get_actions_metadata()\n tools_metadata.append(\n {\n \"name\": tool_name,\n \"displayName\": name,\n \"description\": description,\n \"configRequirements\": config_req,\n \"actions\": actions,\n }\n )\n except Exception as err:\n current_app.logger.error(\n f\"Error getting available tools: {err}\", exc_info=True\n )\n return make_response(jsonify({\"success\": False}), 400)\n return make_response(jsonify({\"success\": True, \"data\": tools_metadata}), 200)\n\n\n@tools_ns.route(\"/get_tools\")\nclass GetTools(Resource):\n @api.doc(description=\"Get tools created by a user\")\n def get(self):\n try:\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False}), 401)\n user = decoded_token.get(\"sub\")\n tools = user_tools_collection.find({\"user\": user})\n user_tools = []\n for tool in tools:\n tool_copy = {**tool}\n tool_copy[\"id\"] = str(tool[\"_id\"])\n tool_copy.pop(\"_id\", None)\n\n config_req = tool_copy.get(\"configRequirements\", {})\n if not config_req:\n tool_instance = tool_manager.tools.get(tool_copy.get(\"name\"))\n if tool_instance:\n config_req = tool_instance.get_config_requirements()\n tool_copy[\"configRequirements\"] = config_req\n\n has_secrets = any(\n spec.get(\"secret\") for spec in config_req.values()\n ) if config_req else False\n if has_secrets and \"encrypted_credentials\" in tool_copy.get(\"config\", {}):\n tool_copy[\"config\"][\"has_encrypted_credentials\"] = True\n tool_copy[\"config\"].pop(\"encrypted_credentials\", None)\n\n user_tools.append(tool_copy)\n except Exception as err:\n current_app.logger.error(f\"Error getting user tools: {err}\", exc_info=True)\n return make_response(jsonify({\"success\": False}), 400)\n return make_response(jsonify({\"success\": True, \"tools\": user_tools}), 200)\n\n\n@tools_ns.route(\"/create_tool\")\nclass CreateTool(Resource):\n @api.expect(\n api.model(\n \"CreateToolModel\",\n {\n \"name\": fields.String(required=True, description=\"Name of the tool\"),\n \"displayName\": fields.String(\n required=True, description=\"Display name for the tool\"\n ),\n \"description\": fields.String(\n required=True, description=\"Tool description\"\n ),\n \"config\": fields.Raw(\n required=True, description=\"Configuration of the tool\"\n ),\n \"customName\": fields.String(\n required=False, description=\"Custom name for the tool\"\n ),\n \"status\": fields.Boolean(\n required=True, description=\"Status of the tool\"\n ),\n },\n )\n )\n @api.doc(description=\"Create a new tool\")\n def post(self):\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False}), 401)\n user = decoded_token.get(\"sub\")\n data = request.get_json()\n required_fields = [\n \"name\",\n \"displayName\",\n \"description\",\n \"config\",\n \"status\",\n ]\n missing_fields = check_required_fields(data, required_fields)\n if missing_fields:\n return missing_fields\n try:\n tool_instance = tool_manager.tools.get(data[\"name\"])\n if not tool_instance:\n return make_response(\n jsonify({\"success\": False, \"message\": \"Tool not found\"}), 404\n )\n actions_metadata = tool_instance.get_actions_metadata()\n transformed_actions = transform_actions(actions_metadata)\n except Exception as err:\n current_app.logger.error(\n f\"Error getting tool actions: {err}\", exc_info=True\n )\n return make_response(jsonify({\"success\": False}), 400)\n try:\n config_requirements = tool_instance.get_config_requirements()\n if config_requirements:\n validation_errors = _validate_config(\n data[\"config\"], config_requirements\n )\n if validation_errors:\n return make_response(\n jsonify(\n {\n \"success\": False,\n \"message\": \"Validation failed\",\n \"errors\": validation_errors,\n }\n ),\n 400,\n )\n storage_config = _encrypt_secret_fields(\n data[\"config\"], config_requirements, user\n )\n new_tool = {\n \"user\": user,\n \"name\": data[\"name\"],\n \"displayName\": data[\"displayName\"],\n \"description\": data[\"description\"],\n \"customName\": data.get(\"customName\", \"\"),\n \"actions\": transformed_actions,\n \"config\": storage_config,\n \"configRequirements\": config_requirements,\n \"status\": data[\"status\"],\n }\n resp = user_tools_collection.insert_one(new_tool)\n new_id = str(resp.inserted_id)\n except Exception as err:\n current_app.logger.error(f\"Error creating tool: {err}\", exc_info=True)\n return make_response(jsonify({\"success\": False}), 400)\n return make_response(jsonify({\"id\": new_id}), 200)\n\n\n@tools_ns.route(\"/update_tool\")\nclass UpdateTool(Resource):\n @api.expect(\n api.model(\n \"UpdateToolModel\",\n {\n \"id\": fields.String(required=True, description=\"Tool ID\"),\n \"name\": fields.String(description=\"Name of the tool\"),\n \"displayName\": fields.String(description=\"Display name for the tool\"),\n \"customName\": fields.String(description=\"Custom name for the tool\"),\n \"description\": fields.String(description=\"Tool description\"),\n \"config\": fields.Raw(description=\"Configuration of the tool\"),\n \"actions\": fields.List(\n fields.Raw, description=\"Actions the tool can perform\"\n ),\n \"status\": fields.Boolean(description=\"Status of the tool\"),\n },\n )\n )\n @api.doc(description=\"Update a tool by ID\")\n def post(self):\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False}), 401)\n user = decoded_token.get(\"sub\")\n data = request.get_json()\n required_fields = [\"id\"]\n missing_fields = check_required_fields(data, required_fields)\n if missing_fields:\n return missing_fields\n try:\n update_data = {}\n if \"name\" in data:\n update_data[\"name\"] = data[\"name\"]\n if \"displayName\" in data:\n update_data[\"displayName\"] = data[\"displayName\"]\n if \"customName\" in data:\n update_data[\"customName\"] = data[\"customName\"]\n if \"description\" in data:\n update_data[\"description\"] = data[\"description\"]\n if \"actions\" in data:\n update_data[\"actions\"] = data[\"actions\"]\n if \"config\" in data:\n if \"actions\" in data[\"config\"]:\n for action_name in list(data[\"config\"][\"actions\"].keys()):\n if not validate_function_name(action_name):\n return make_response(\n jsonify(\n {\n \"success\": False,\n \"message\": f\"Invalid function name '{action_name}'. Function names must match pattern '^[a-zA-Z0-9_-]+$'.\",\n \"param\": \"tools[].function.name\",\n }\n ),\n 400,\n )\n tool_doc = user_tools_collection.find_one(\n {\"_id\": ObjectId(data[\"id\"]), \"user\": user}\n )\n if not tool_doc:\n return make_response(\n jsonify({\"success\": False, \"message\": \"Tool not found\"}),\n 404,\n )\n tool_name = tool_doc.get(\"name\", data.get(\"name\"))\n tool_instance = tool_manager.tools.get(tool_name)\n config_requirements = (\n tool_instance.get_config_requirements() if tool_instance else {}\n )\n existing_config = tool_doc.get(\"config\", {})\n has_existing_secrets = \"encrypted_credentials\" in existing_config\n\n if config_requirements:\n validation_errors = _validate_config(\n data[\"config\"], config_requirements,\n has_existing_secrets=has_existing_secrets,\n )\n if validation_errors:\n return make_response(\n jsonify({\n \"success\": False,\n \"message\": \"Validation failed\",\n \"errors\": validation_errors,\n }),\n 400,\n )\n\n update_data[\"config\"] = _merge_secrets_on_update(\n data[\"config\"], existing_config, config_requirements, user\n )\n if \"status\" in data:\n update_data[\"status\"] = data[\"status\"]\n user_tools_collection.update_one(\n {\"_id\": ObjectId(data[\"id\"]), \"user\": user},\n {\"$set\": update_data},\n )\n except Exception as err:\n current_app.logger.error(f\"Error updating tool: {err}\", exc_info=True)\n return make_response(jsonify({\"success\": False}), 400)\n return make_response(jsonify({\"success\": True}), 200)\n\n\n@tools_ns.route(\"/update_tool_config\")\nclass UpdateToolConfig(Resource):\n @api.expect(\n api.model(\n \"UpdateToolConfigModel\",\n {\n \"id\": fields.String(required=True, description=\"Tool ID\"),\n \"config\": fields.Raw(\n required=True, description=\"Configuration of the tool\"\n ),\n },\n )\n )\n @api.doc(description=\"Update the configuration of a tool\")\n def post(self):\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False}), 401)\n user = decoded_token.get(\"sub\")\n data = request.get_json()\n required_fields = [\"id\", \"config\"]\n missing_fields = check_required_fields(data, required_fields)\n if missing_fields:\n return missing_fields\n try:\n tool_doc = user_tools_collection.find_one(\n {\"_id\": ObjectId(data[\"id\"]), \"user\": user}\n )\n if not tool_doc:\n return make_response(jsonify({\"success\": False}), 404)\n\n tool_name = tool_doc.get(\"name\")\n tool_instance = tool_manager.tools.get(tool_name)\n config_requirements = (\n tool_instance.get_config_requirements() if tool_instance else {}\n )\n existing_config = tool_doc.get(\"config\", {})\n has_existing_secrets = \"encrypted_credentials\" in existing_config\n\n if config_requirements:\n validation_errors = _validate_config(\n data[\"config\"], config_requirements,\n has_existing_secrets=has_existing_secrets,\n )\n if validation_errors:\n return make_response(\n jsonify({\n \"success\": False,\n \"message\": \"Validation failed\",\n \"errors\": validation_errors,\n }),\n 400,\n )\n\n final_config = _merge_secrets_on_update(\n data[\"config\"], existing_config, config_requirements, user\n )\n\n user_tools_collection.update_one(\n {\"_id\": ObjectId(data[\"id\"]), \"user\": user},\n {\"$set\": {\"config\": final_config}},\n )\n except Exception as err:\n current_app.logger.error(\n f\"Error updating tool config: {err}\", exc_info=True\n )\n return make_response(jsonify({\"success\": False}), 400)\n return make_response(jsonify({\"success\": True}), 200)\n\n\n@tools_ns.route(\"/update_tool_actions\")\nclass UpdateToolActions(Resource):\n @api.expect(\n api.model(\n \"UpdateToolActionsModel\",\n {\n \"id\": fields.String(required=True, description=\"Tool ID\"),\n \"actions\": fields.List(\n fields.Raw,\n required=True,\n description=\"Actions the tool can perform\",\n ),\n },\n )\n )\n @api.doc(description=\"Update the actions of a tool\")\n def post(self):\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False}), 401)\n user = decoded_token.get(\"sub\")\n data = request.get_json()\n required_fields = [\"id\", \"actions\"]\n missing_fields = check_required_fields(data, required_fields)\n if missing_fields:\n return missing_fields\n try:\n user_tools_collection.update_one(\n {\"_id\": ObjectId(data[\"id\"]), \"user\": user},\n {\"$set\": {\"actions\": data[\"actions\"]}},\n )\n except Exception as err:\n current_app.logger.error(\n f\"Error updating tool actions: {err}\", exc_info=True\n )\n return make_response(jsonify({\"success\": False}), 400)\n return make_response(jsonify({\"success\": True}), 200)\n\n\n@tools_ns.route(\"/update_tool_status\")\nclass UpdateToolStatus(Resource):\n @api.expect(\n api.model(\n \"UpdateToolStatusModel\",\n {\n \"id\": fields.String(required=True, description=\"Tool ID\"),\n \"status\": fields.Boolean(\n required=True, description=\"Status of the tool\"\n ),\n },\n )\n )\n @api.doc(description=\"Update the status of a tool\")\n def post(self):\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False}), 401)\n user = decoded_token.get(\"sub\")\n data = request.get_json()\n required_fields = [\"id\", \"status\"]\n missing_fields = check_required_fields(data, required_fields)\n if missing_fields:\n return missing_fields\n try:\n user_tools_collection.update_one(\n {\"_id\": ObjectId(data[\"id\"]), \"user\": user},\n {\"$set\": {\"status\": data[\"status\"]}},\n )\n except Exception as err:\n current_app.logger.error(\n f\"Error updating tool status: {err}\", exc_info=True\n )\n return make_response(jsonify({\"success\": False}), 400)\n return make_response(jsonify({\"success\": True}), 200)\n\n\n@tools_ns.route(\"/delete_tool\")\nclass DeleteTool(Resource):\n @api.expect(\n api.model(\n \"DeleteToolModel\",\n {\"id\": fields.String(required=True, description=\"Tool ID\")},\n )\n )\n @api.doc(description=\"Delete a tool by ID\")\n def post(self):\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False}), 401)\n user = decoded_token.get(\"sub\")\n data = request.get_json()\n required_fields = [\"id\"]\n missing_fields = check_required_fields(data, required_fields)\n if missing_fields:\n return missing_fields\n try:\n result = user_tools_collection.delete_one(\n {\"_id\": ObjectId(data[\"id\"]), \"user\": user}\n )\n if result.deleted_count == 0:\n return make_response(\n jsonify({\"success\": False, \"message\": \"Tool not found\"}), 404\n )\n except Exception as err:\n current_app.logger.error(f\"Error deleting tool: {err}\", exc_info=True)\n return make_response(jsonify({\"success\": False}), 400)\n return make_response(jsonify({\"success\": True}), 200)\n\n\n@tools_ns.route(\"/parse_spec\")\nclass ParseSpec(Resource):\n @api.doc(\n description=\"Parse an API specification (OpenAPI 3.x or Swagger 2.0) and return actions\"\n )\n def post(self):\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False}), 401)\n if \"file\" in request.files:\n file = request.files[\"file\"]\n if not file.filename:\n return make_response(\n jsonify({\"success\": False, \"message\": \"No file selected\"}), 400\n )\n try:\n spec_content = file.read().decode(\"utf-8\")\n except UnicodeDecodeError:\n return make_response(\n jsonify({\"success\": False, \"message\": \"Invalid file encoding\"}), 400\n )\n elif request.is_json:\n data = request.get_json()\n spec_content = data.get(\"spec_content\", \"\")\n else:\n return make_response(\n jsonify({\"success\": False, \"message\": \"No spec provided\"}), 400\n )\n if not spec_content or not spec_content.strip():\n return make_response(\n jsonify({\"success\": False, \"message\": \"Empty spec content\"}), 400\n )\n try:\n metadata, actions = parse_spec(spec_content)\n return make_response(\n jsonify(\n {\n \"success\": True,\n \"metadata\": metadata,\n \"actions\": actions,\n }\n ),\n 200,\n )\n except ValueError as e:\n current_app.logger.error(f\"Spec validation error: {e}\")\n return make_response(jsonify({\"success\": False, \"error\": \"Invalid specification format\"}), 400)\n except Exception as err:\n current_app.logger.error(f\"Error parsing spec: {err}\", exc_info=True)\n return make_response(jsonify({\"success\": False, \"error\": \"Failed to parse specification\"}), 500)\n\n\n@tools_ns.route(\"/artifact/\")\nclass GetArtifact(Resource):\n @api.doc(description=\"Get artifact data by artifact ID. Returns all todos for the tool when fetching a todo artifact.\")\n def get(self, artifact_id: str):\n decoded_token = request.decoded_token\n if not decoded_token:\n return make_response(jsonify({\"success\": False}), 401)\n user_id = decoded_token.get(\"sub\")\n\n try:\n obj_id = ObjectId(artifact_id)\n except Exception:\n return make_response(\n jsonify({\"success\": False, \"message\": \"Invalid artifact ID\"}), 400\n )\n\n from application.core.mongo_db import MongoDB\n from application.core.settings import settings\n\n db = MongoDB.get_client()[settings.MONGO_DB_NAME]\n\n note_doc = db[\"notes\"].find_one({\"_id\": obj_id, \"user_id\": user_id})\n if note_doc:\n content = note_doc.get(\"note\", \"\")\n line_count = len(content.split(\"\\n\")) if content else 0\n artifact = {\n \"artifact_type\": \"note\",\n \"data\": {\n \"content\": content,\n \"line_count\": line_count,\n \"updated_at\": (\n note_doc[\"updated_at\"].isoformat()\n if note_doc.get(\"updated_at\")\n else None\n ),\n },\n }\n return make_response(jsonify({\"success\": True, \"artifact\": artifact}), 200)\n\n todo_doc = db[\"todos\"].find_one({\"_id\": obj_id, \"user_id\": user_id})\n if todo_doc:\n tool_id = todo_doc.get(\"tool_id\")\n query = {\"user_id\": user_id, \"tool_id\": tool_id}\n all_todos = list(db[\"todos\"].find(query))\n items = []\n open_count = 0\n completed_count = 0\n for t in all_todos:\n status = t.get(\"status\", \"open\")\n if status == \"open\":\n open_count += 1\n elif status == \"completed\":\n completed_count += 1\n items.append({\n \"todo_id\": t.get(\"todo_id\"),\n \"title\": t.get(\"title\", \"\"),\n \"status\": status,\n \"created_at\": (\n t[\"created_at\"].isoformat() if t.get(\"created_at\") else None\n ),\n \"updated_at\": (\n t[\"updated_at\"].isoformat() if t.get(\"updated_at\") else None\n ),\n })\n artifact = {\n \"artifact_type\": \"todo_list\",\n \"data\": {\n \"items\": items,\n \"total_count\": len(items),\n \"open_count\": open_count,\n \"completed_count\": completed_count,\n },\n }\n return make_response(jsonify({\"success\": True, \"artifact\": artifact}), 200)\n\n return make_response(\n jsonify({\"success\": False, \"message\": \"Artifact not found\"}), 404\n )\n" }, { "path": "application/api/user/utils.py", "content": "\"\"\"Centralized utilities for API routes.\"\"\"\n\nfrom functools import wraps\nfrom typing import Any, Callable, Dict, List, Optional, Tuple\n\nfrom bson.errors import InvalidId\nfrom bson.objectid import ObjectId\nfrom flask import (\n Response,\n current_app,\n has_app_context,\n jsonify,\n make_response,\n request,\n)\nfrom pymongo.collection import Collection\n\n\ndef get_user_id() -> Optional[str]:\n \"\"\"\n Extract user ID from decoded JWT token.\n\n Returns:\n User ID string or None if not authenticated\n \"\"\"\n decoded_token = getattr(request, \"decoded_token\", None)\n return decoded_token.get(\"sub\") if decoded_token else None\n\n\ndef require_auth(func: Callable) -> Callable:\n \"\"\"\n Decorator to require authentication for route handlers.\n\n Usage:\n @require_auth\n def get(self):\n user_id = get_user_id()\n ...\n \"\"\"\n\n @wraps(func)\n def wrapper(*args, **kwargs):\n user_id = get_user_id()\n if not user_id:\n return error_response(\"Unauthorized\", 401)\n return func(*args, **kwargs)\n\n return wrapper\n\n\ndef success_response(\n data: Optional[Dict[str, Any]] = None, status: int = 200\n) -> Response:\n \"\"\"\n Create a standardized success response.\n\n Args:\n data: Optional data dictionary to include in response\n status: HTTP status code (default: 200)\n\n Returns:\n Flask Response object\n\n Example:\n return success_response({\"users\": [...], \"total\": 10})\n \"\"\"\n response = {\"success\": True}\n if data:\n response.update(data)\n return make_response(jsonify(response), status)\n\n\ndef error_response(message: str, status: int = 400, **kwargs) -> Response:\n \"\"\"\n Create a standardized error response.\n\n Args:\n message: Error message string\n status: HTTP status code (default: 400)\n **kwargs: Additional fields to include in response\n\n Returns:\n Flask Response object\n\n Example:\n return error_response(\"Resource not found\", 404)\n return error_response(\"Invalid input\", 400, errors=[\"field1\", \"field2\"])\n \"\"\"\n response = {\"success\": False, \"message\": message}\n response.update(kwargs)\n return make_response(jsonify(response), status)\n\n\ndef validate_object_id(\n id_string: str, resource_name: str = \"Resource\"\n) -> Tuple[Optional[ObjectId], Optional[Response]]:\n \"\"\"\n Validate and convert string to ObjectId.\n\n Args:\n id_string: String to convert\n resource_name: Name of resource for error message\n\n Returns:\n Tuple of (ObjectId or None, error_response or None)\n\n Example:\n obj_id, error = validate_object_id(workflow_id, \"Workflow\")\n if error:\n return error\n \"\"\"\n try:\n return ObjectId(id_string), None\n except (InvalidId, TypeError):\n return None, error_response(f\"Invalid {resource_name} ID format\")\n\n\ndef validate_pagination(\n default_limit: int = 20, max_limit: int = 100\n) -> Tuple[int, int, Optional[Response]]:\n \"\"\"\n Extract and validate pagination parameters from request.\n\n Args:\n default_limit: Default items per page\n max_limit: Maximum allowed items per page\n\n Returns:\n Tuple of (limit, skip, error_response or None)\n\n Example:\n limit, skip, error = validate_pagination()\n if error:\n return error\n \"\"\"\n try:\n limit = min(int(request.args.get(\"limit\", default_limit)), max_limit)\n skip = int(request.args.get(\"skip\", 0))\n if limit < 1 or skip < 0:\n return 0, 0, error_response(\"Invalid pagination parameters\")\n return limit, skip, None\n except ValueError:\n return 0, 0, error_response(\"Invalid pagination parameters\")\n\n\ndef check_resource_ownership(\n collection: Collection,\n resource_id: ObjectId,\n user_id: str,\n resource_name: str = \"Resource\",\n) -> Tuple[Optional[Dict], Optional[Response]]:\n \"\"\"\n Check if resource exists and belongs to user.\n\n Args:\n collection: MongoDB collection\n resource_id: Resource ObjectId\n user_id: User ID string\n resource_name: Name of resource for error messages\n\n Returns:\n Tuple of (resource_dict or None, error_response or None)\n\n Example:\n workflow, error = check_resource_ownership(\n workflows_collection,\n workflow_id,\n user_id,\n \"Workflow\"\n )\n if error:\n return error\n \"\"\"\n resource = collection.find_one({\"_id\": resource_id, \"user\": user_id})\n if not resource:\n return None, error_response(f\"{resource_name} not found\", 404)\n return resource, None\n\n\ndef serialize_object_id(\n obj: Dict[str, Any], id_field: str = \"_id\", new_field: str = \"id\"\n) -> Dict[str, Any]:\n \"\"\"\n Convert ObjectId to string in a dictionary.\n\n Args:\n obj: Dictionary containing ObjectId\n id_field: Field name containing ObjectId\n new_field: New field name for string ID\n\n Returns:\n Modified dictionary\n\n Example:\n user = serialize_object_id(user_doc)\n # user[\"id\"] = \"507f1f77bcf86cd799439011\"\n \"\"\"\n if id_field in obj:\n obj[new_field] = str(obj[id_field])\n if id_field != new_field:\n obj.pop(id_field, None)\n return obj\n\n\ndef serialize_list(items: List[Dict], serializer: Callable[[Dict], Dict]) -> List[Dict]:\n \"\"\"\n Apply serializer function to list of items.\n\n Args:\n items: List of dictionaries\n serializer: Function to apply to each item\n\n Returns:\n List of serialized items\n\n Example:\n workflows = serialize_list(workflow_docs, serialize_workflow)\n \"\"\"\n return [serializer(item) for item in items]\n\n\ndef paginated_response(\n collection: Collection,\n query: Dict[str, Any],\n serializer: Callable[[Dict], Dict],\n limit: int,\n skip: int,\n sort_field: str = \"created_at\",\n sort_order: int = -1,\n response_key: str = \"items\",\n) -> Response:\n \"\"\"\n Create paginated response for collection query.\n\n Args:\n collection: MongoDB collection\n query: Query dictionary\n serializer: Function to serialize each item\n limit: Items per page\n skip: Number of items to skip\n sort_field: Field to sort by\n sort_order: Sort order (1=asc, -1=desc)\n response_key: Key name for items in response\n\n Returns:\n Flask Response with paginated data\n\n Example:\n return paginated_response(\n workflows_collection,\n {\"user\": user_id},\n serialize_workflow,\n limit, skip,\n response_key=\"workflows\"\n )\n \"\"\"\n items = list(\n collection.find(query).sort(sort_field, sort_order).skip(skip).limit(limit)\n )\n total = collection.count_documents(query)\n\n return success_response(\n {\n response_key: serialize_list(items, serializer),\n \"total\": total,\n \"limit\": limit,\n \"skip\": skip,\n }\n )\n\n\ndef require_fields(required: List[str]) -> Callable:\n \"\"\"\n Decorator to validate required fields in request JSON.\n\n Args:\n required: List of required field names\n\n Returns:\n Decorator function\n\n Example:\n @require_fields([\"name\", \"description\"])\n def post(self):\n data = request.get_json()\n ...\n \"\"\"\n\n def decorator(func: Callable) -> Callable:\n @wraps(func)\n def wrapper(*args, **kwargs):\n data = request.get_json()\n if not data:\n return error_response(\"Request body required\")\n missing = [field for field in required if not data.get(field)]\n if missing:\n return error_response(f\"Missing required fields: {', '.join(missing)}\")\n return func(*args, **kwargs)\n\n return wrapper\n\n return decorator\n\n\ndef safe_db_operation(\n operation: Callable, error_message: str = \"Database operation failed\"\n) -> Tuple[Any, Optional[Response]]:\n \"\"\"\n Safely execute database operation with error handling.\n\n Args:\n operation: Function to execute\n error_message: Error message if operation fails\n\n Returns:\n Tuple of (result or None, error_response or None)\n\n Example:\n result, error = safe_db_operation(\n lambda: collection.insert_one(doc),\n \"Failed to create resource\"\n )\n if error:\n return error\n \"\"\"\n try:\n result = operation()\n return result, None\n except Exception as err:\n if has_app_context():\n current_app.logger.error(f\"{error_message}: {err}\", exc_info=True)\n return None, error_response(error_message)\n\n\ndef validate_enum(\n value: Any, allowed: List[Any], field_name: str\n) -> Optional[Response]:\n \"\"\"\n Validate that value is in allowed list.\n\n Args:\n value: Value to validate\n allowed: List of allowed values\n field_name: Field name for error message\n\n Returns:\n error_response if invalid, None if valid\n\n Example:\n error = validate_enum(status, [\"draft\", \"published\"], \"status\")\n if error:\n return error\n \"\"\"\n if value not in allowed:\n allowed_str = \", \".join(f\"'{v}'\" for v in allowed)\n return error_response(f\"Invalid {field_name}. Must be one of: {allowed_str}\")\n return None\n\n\ndef extract_sort_params(\n default_field: str = \"created_at\",\n default_order: str = \"desc\",\n allowed_fields: Optional[List[str]] = None,\n) -> Tuple[str, int]:\n \"\"\"\n Extract and validate sort parameters from request.\n\n Args:\n default_field: Default sort field\n default_order: Default sort order (\"asc\" or \"desc\")\n allowed_fields: List of allowed sort fields (None = no validation)\n\n Returns:\n Tuple of (sort_field, sort_order)\n\n Example:\n sort_field, sort_order = extract_sort_params(\n allowed_fields=[\"name\", \"date\", \"status\"]\n )\n \"\"\"\n sort_field = request.args.get(\"sort\", default_field)\n sort_order_str = request.args.get(\"order\", default_order).lower()\n\n if allowed_fields and sort_field not in allowed_fields:\n sort_field = default_field\n sort_order = -1 if sort_order_str == \"desc\" else 1\n return sort_field, sort_order\n" }, { "path": "application/api/user/workflows/__init__.py", "content": "from .routes import workflows_ns\n\n__all__ = [\"workflows_ns\"]\n" }, { "path": "application/api/user/workflows/routes.py", "content": "\"\"\"Workflow management routes.\"\"\"\n\nfrom datetime import datetime, timezone\nfrom typing import Any, Dict, List, Optional, Set\n\nfrom flask import current_app, request\nfrom flask_restx import Namespace, Resource\n\nfrom application.api.user.base import (\n workflow_edges_collection,\n workflow_nodes_collection,\n workflows_collection,\n)\nfrom application.core.json_schema_utils import (\n JsonSchemaValidationError,\n normalize_json_schema_payload,\n)\nfrom application.core.model_utils import get_model_capabilities\nfrom application.api.user.utils import (\n check_resource_ownership,\n error_response,\n get_user_id,\n require_auth,\n require_fields,\n safe_db_operation,\n success_response,\n validate_object_id,\n)\n\nworkflows_ns = Namespace(\"workflows\", path=\"/api\")\n\n\ndef _workflow_error_response(message: str, err: Exception):\n current_app.logger.error(f\"{message}: {err}\", exc_info=True)\n return error_response(message)\n\n\ndef serialize_workflow(w: Dict) -> Dict:\n \"\"\"Serialize workflow document to API response format.\"\"\"\n return {\n \"id\": str(w[\"_id\"]),\n \"name\": w.get(\"name\"),\n \"description\": w.get(\"description\"),\n \"created_at\": w[\"created_at\"].isoformat() if w.get(\"created_at\") else None,\n \"updated_at\": w[\"updated_at\"].isoformat() if w.get(\"updated_at\") else None,\n }\n\n\ndef serialize_node(n: Dict) -> Dict:\n \"\"\"Serialize workflow node document to API response format.\"\"\"\n return {\n \"id\": n[\"id\"],\n \"type\": n[\"type\"],\n \"title\": n.get(\"title\"),\n \"description\": n.get(\"description\"),\n \"position\": n.get(\"position\"),\n \"data\": n.get(\"config\", {}),\n }\n\n\ndef serialize_edge(e: Dict) -> Dict:\n \"\"\"Serialize workflow edge document to API response format.\"\"\"\n return {\n \"id\": e[\"id\"],\n \"source\": e.get(\"source_id\"),\n \"target\": e.get(\"target_id\"),\n \"sourceHandle\": e.get(\"source_handle\"),\n \"targetHandle\": e.get(\"target_handle\"),\n }\n\n\ndef get_workflow_graph_version(workflow: Dict) -> int:\n \"\"\"Get current graph version with legacy fallback.\"\"\"\n raw_version = workflow.get(\"current_graph_version\", 1)\n try:\n version = int(raw_version)\n return version if version > 0 else 1\n except (ValueError, TypeError):\n return 1\n\n\ndef fetch_graph_documents(collection, workflow_id: str, graph_version: int) -> List[Dict]:\n \"\"\"Fetch graph docs for active version, with fallback for legacy unversioned data.\"\"\"\n docs = list(\n collection.find({\"workflow_id\": workflow_id, \"graph_version\": graph_version})\n )\n if docs:\n return docs\n if graph_version == 1:\n return list(\n collection.find(\n {\"workflow_id\": workflow_id, \"graph_version\": {\"$exists\": False}}\n )\n )\n return docs\n\n\ndef validate_json_schema_payload(\n json_schema: Any,\n) -> tuple[Optional[Dict[str, Any]], Optional[str]]:\n \"\"\"Validate and normalize optional JSON schema payload for structured output.\"\"\"\n if json_schema is None:\n return None, None\n try:\n return normalize_json_schema_payload(json_schema), None\n except JsonSchemaValidationError as exc:\n return None, str(exc)\n\n\ndef normalize_agent_node_json_schemas(nodes: List[Dict]) -> List[Dict]:\n \"\"\"Normalize agent-node JSON schema payloads before persistence.\"\"\"\n normalized_nodes: List[Dict] = []\n for node in nodes:\n if not isinstance(node, dict):\n normalized_nodes.append(node)\n continue\n\n normalized_node = dict(node)\n if normalized_node.get(\"type\") != \"agent\":\n normalized_nodes.append(normalized_node)\n continue\n\n raw_config = normalized_node.get(\"data\")\n if not isinstance(raw_config, dict) or \"json_schema\" not in raw_config:\n normalized_nodes.append(normalized_node)\n continue\n\n normalized_config = dict(raw_config)\n try:\n normalized_config[\"json_schema\"] = normalize_json_schema_payload(\n raw_config.get(\"json_schema\")\n )\n except JsonSchemaValidationError:\n # Validation runs before normalization; keep original on unexpected shape.\n normalized_config[\"json_schema\"] = raw_config.get(\"json_schema\")\n normalized_node[\"data\"] = normalized_config\n normalized_nodes.append(normalized_node)\n\n return normalized_nodes\n\n\ndef validate_workflow_structure(nodes: List[Dict], edges: List[Dict]) -> List[str]:\n \"\"\"Validate workflow graph structure.\"\"\"\n errors = []\n\n if not nodes:\n errors.append(\"Workflow must have at least one node\")\n return errors\n\n start_nodes = [n for n in nodes if n.get(\"type\") == \"start\"]\n if len(start_nodes) != 1:\n errors.append(\"Workflow must have exactly one start node\")\n\n end_nodes = [n for n in nodes if n.get(\"type\") == \"end\"]\n if not end_nodes:\n errors.append(\"Workflow must have at least one end node\")\n\n node_ids = {n.get(\"id\") for n in nodes}\n node_map = {n.get(\"id\"): n for n in nodes}\n end_ids = {n.get(\"id\") for n in end_nodes}\n\n for edge in edges:\n source_id = edge.get(\"source\")\n target_id = edge.get(\"target\")\n if source_id not in node_ids:\n errors.append(f\"Edge references non-existent source: {source_id}\")\n if target_id not in node_ids:\n errors.append(f\"Edge references non-existent target: {target_id}\")\n\n if start_nodes:\n start_id = start_nodes[0].get(\"id\")\n if not any(e.get(\"source\") == start_id for e in edges):\n errors.append(\"Start node must have at least one outgoing edge\")\n\n condition_nodes = [n for n in nodes if n.get(\"type\") == \"condition\"]\n for cnode in condition_nodes:\n cnode_id = cnode.get(\"id\")\n cnode_title = cnode.get(\"title\", cnode_id)\n outgoing = [e for e in edges if e.get(\"source\") == cnode_id]\n if len(outgoing) < 2:\n errors.append(\n f\"Condition node '{cnode_title}' must have at least 2 outgoing edges\"\n )\n node_data = cnode.get(\"data\", {}) or {}\n cases = node_data.get(\"cases\", [])\n if not isinstance(cases, list):\n cases = []\n if not cases or not any(\n isinstance(c, dict) and str(c.get(\"expression\", \"\")).strip() for c in cases\n ):\n errors.append(\n f\"Condition node '{cnode_title}' must have at least one case with an expression\"\n )\n\n case_handles: Set[str] = set()\n duplicate_case_handles: Set[str] = set()\n for case in cases:\n if not isinstance(case, dict):\n continue\n raw_handle = case.get(\"sourceHandle\", \"\")\n handle = raw_handle.strip() if isinstance(raw_handle, str) else \"\"\n if not handle:\n errors.append(\n f\"Condition node '{cnode_title}' has a case without a branch handle\"\n )\n continue\n if handle in case_handles:\n duplicate_case_handles.add(handle)\n case_handles.add(handle)\n\n for handle in duplicate_case_handles:\n errors.append(\n f\"Condition node '{cnode_title}' has duplicate case handle '{handle}'\"\n )\n\n outgoing_by_handle: Dict[str, List[Dict]] = {}\n for out_edge in outgoing:\n raw_handle = out_edge.get(\"sourceHandle\", \"\")\n handle = raw_handle.strip() if isinstance(raw_handle, str) else \"\"\n outgoing_by_handle.setdefault(handle, []).append(out_edge)\n\n for handle, handle_edges in outgoing_by_handle.items():\n if not handle:\n errors.append(\n f\"Condition node '{cnode_title}' has an outgoing edge without sourceHandle\"\n )\n continue\n if handle != \"else\" and handle not in case_handles:\n errors.append(\n f\"Condition node '{cnode_title}' has a connection from unknown branch '{handle}'\"\n )\n if len(handle_edges) > 1:\n errors.append(\n f\"Condition node '{cnode_title}' has multiple outgoing edges from branch '{handle}'\"\n )\n\n if \"else\" not in outgoing_by_handle:\n errors.append(f\"Condition node '{cnode_title}' must have an 'else' branch\")\n\n for case in cases:\n if not isinstance(case, dict):\n continue\n raw_handle = case.get(\"sourceHandle\", \"\")\n handle = raw_handle.strip() if isinstance(raw_handle, str) else \"\"\n if not handle:\n continue\n\n raw_expression = case.get(\"expression\", \"\")\n has_expression = isinstance(raw_expression, str) and bool(\n raw_expression.strip()\n )\n has_outgoing = bool(outgoing_by_handle.get(handle))\n if has_expression and not has_outgoing:\n errors.append(\n f\"Condition node '{cnode_title}' case '{handle}' has an expression but no outgoing edge\"\n )\n if not has_expression and has_outgoing:\n errors.append(\n f\"Condition node '{cnode_title}' case '{handle}' has an outgoing edge but no expression\"\n )\n\n for handle, handle_edges in outgoing_by_handle.items():\n if not handle:\n continue\n for out_edge in handle_edges:\n target = out_edge.get(\"target\")\n if target and not _can_reach_end(target, edges, node_map, end_ids):\n errors.append(\n f\"Branch '{handle}' of condition '{cnode_title}' \"\n f\"must eventually reach an end node\"\n )\n\n agent_nodes = [n for n in nodes if n.get(\"type\") == \"agent\"]\n for agent_node in agent_nodes:\n agent_title = agent_node.get(\"title\", agent_node.get(\"id\", \"unknown\"))\n raw_config = agent_node.get(\"data\", {}) or {}\n if not isinstance(raw_config, dict):\n errors.append(f\"Agent node '{agent_title}' has invalid configuration\")\n continue\n normalized_schema, schema_error = validate_json_schema_payload(\n raw_config.get(\"json_schema\")\n )\n has_json_schema = normalized_schema is not None\n\n model_id = raw_config.get(\"model_id\")\n if has_json_schema and isinstance(model_id, str) and model_id.strip():\n capabilities = get_model_capabilities(model_id.strip())\n if capabilities and not capabilities.get(\"supports_structured_output\", False):\n errors.append(\n f\"Agent node '{agent_title}' selected model does not support structured output\"\n )\n if schema_error:\n errors.append(f\"Agent node '{agent_title}' JSON schema {schema_error}\")\n\n for node in nodes:\n if not node.get(\"id\"):\n errors.append(\"All nodes must have an id\")\n if not node.get(\"type\"):\n errors.append(f\"Node {node.get('id', 'unknown')} must have a type\")\n\n return errors\n\n\ndef _can_reach_end(\n node_id: str, edges: List[Dict], node_map: Dict, end_ids: set, visited: set = None\n) -> bool:\n if visited is None:\n visited = set()\n if node_id in end_ids:\n return True\n if node_id in visited or node_id not in node_map:\n return False\n visited.add(node_id)\n outgoing = [e.get(\"target\") for e in edges if e.get(\"source\") == node_id]\n return any(_can_reach_end(t, edges, node_map, end_ids, visited) for t in outgoing if t)\n\n\ndef create_workflow_nodes(\n workflow_id: str, nodes_data: List[Dict], graph_version: int\n) -> None:\n \"\"\"Insert workflow nodes into database.\"\"\"\n if nodes_data:\n workflow_nodes_collection.insert_many(\n [\n {\n \"id\": n[\"id\"],\n \"workflow_id\": workflow_id,\n \"graph_version\": graph_version,\n \"type\": n[\"type\"],\n \"title\": n.get(\"title\", \"\"),\n \"description\": n.get(\"description\", \"\"),\n \"position\": n.get(\"position\", {\"x\": 0, \"y\": 0}),\n \"config\": n.get(\"data\", {}),\n }\n for n in nodes_data\n ]\n )\n\n\ndef create_workflow_edges(\n workflow_id: str, edges_data: List[Dict], graph_version: int\n) -> None:\n \"\"\"Insert workflow edges into database.\"\"\"\n if edges_data:\n workflow_edges_collection.insert_many(\n [\n {\n \"id\": e[\"id\"],\n \"workflow_id\": workflow_id,\n \"graph_version\": graph_version,\n \"source_id\": e.get(\"source\"),\n \"target_id\": e.get(\"target\"),\n \"source_handle\": e.get(\"sourceHandle\"),\n \"target_handle\": e.get(\"targetHandle\"),\n }\n for e in edges_data\n ]\n )\n\n\n@workflows_ns.route(\"/workflows\")\nclass WorkflowList(Resource):\n\n @require_auth\n @require_fields([\"name\"])\n def post(self):\n \"\"\"Create a new workflow with nodes and edges.\"\"\"\n user_id = get_user_id()\n data = request.get_json()\n\n name = data.get(\"name\", \"\").strip()\n nodes_data = data.get(\"nodes\", [])\n edges_data = data.get(\"edges\", [])\n\n validation_errors = validate_workflow_structure(nodes_data, edges_data)\n if validation_errors:\n return error_response(\n \"Workflow validation failed\", errors=validation_errors\n )\n nodes_data = normalize_agent_node_json_schemas(nodes_data)\n\n now = datetime.now(timezone.utc)\n workflow_doc = {\n \"name\": name,\n \"description\": data.get(\"description\", \"\"),\n \"user\": user_id,\n \"created_at\": now,\n \"updated_at\": now,\n \"current_graph_version\": 1,\n }\n\n result, error = safe_db_operation(\n lambda: workflows_collection.insert_one(workflow_doc),\n \"Failed to create workflow\",\n )\n if error:\n return error\n\n workflow_id = str(result.inserted_id)\n\n try:\n create_workflow_nodes(workflow_id, nodes_data, 1)\n create_workflow_edges(workflow_id, edges_data, 1)\n except Exception as err:\n workflow_nodes_collection.delete_many({\"workflow_id\": workflow_id})\n workflow_edges_collection.delete_many({\"workflow_id\": workflow_id})\n workflows_collection.delete_one({\"_id\": result.inserted_id})\n return _workflow_error_response(\"Failed to create workflow structure\", err)\n\n return success_response({\"id\": workflow_id}, 201)\n\n\n@workflows_ns.route(\"/workflows/\")\nclass WorkflowDetail(Resource):\n\n @require_auth\n def get(self, workflow_id: str):\n \"\"\"Get workflow details with nodes and edges.\"\"\"\n user_id = get_user_id()\n obj_id, error = validate_object_id(workflow_id, \"Workflow\")\n if error:\n return error\n\n workflow, error = check_resource_ownership(\n workflows_collection, obj_id, user_id, \"Workflow\"\n )\n if error:\n return error\n\n graph_version = get_workflow_graph_version(workflow)\n nodes = fetch_graph_documents(\n workflow_nodes_collection, workflow_id, graph_version\n )\n edges = fetch_graph_documents(\n workflow_edges_collection, workflow_id, graph_version\n )\n\n return success_response(\n {\n \"workflow\": serialize_workflow(workflow),\n \"nodes\": [serialize_node(n) for n in nodes],\n \"edges\": [serialize_edge(e) for e in edges],\n }\n )\n\n @require_auth\n @require_fields([\"name\"])\n def put(self, workflow_id: str):\n \"\"\"Update workflow and replace nodes/edges.\"\"\"\n user_id = get_user_id()\n obj_id, error = validate_object_id(workflow_id, \"Workflow\")\n if error:\n return error\n\n workflow, error = check_resource_ownership(\n workflows_collection, obj_id, user_id, \"Workflow\"\n )\n if error:\n return error\n\n data = request.get_json()\n name = data.get(\"name\", \"\").strip()\n nodes_data = data.get(\"nodes\", [])\n edges_data = data.get(\"edges\", [])\n\n validation_errors = validate_workflow_structure(nodes_data, edges_data)\n if validation_errors:\n return error_response(\n \"Workflow validation failed\", errors=validation_errors\n )\n nodes_data = normalize_agent_node_json_schemas(nodes_data)\n\n current_graph_version = get_workflow_graph_version(workflow)\n next_graph_version = current_graph_version + 1\n try:\n create_workflow_nodes(workflow_id, nodes_data, next_graph_version)\n create_workflow_edges(workflow_id, edges_data, next_graph_version)\n except Exception as err:\n workflow_nodes_collection.delete_many(\n {\"workflow_id\": workflow_id, \"graph_version\": next_graph_version}\n )\n workflow_edges_collection.delete_many(\n {\"workflow_id\": workflow_id, \"graph_version\": next_graph_version}\n )\n return _workflow_error_response(\"Failed to update workflow structure\", err)\n\n now = datetime.now(timezone.utc)\n _, error = safe_db_operation(\n lambda: workflows_collection.update_one(\n {\"_id\": obj_id},\n {\n \"$set\": {\n \"name\": name,\n \"description\": data.get(\"description\", \"\"),\n \"updated_at\": now,\n \"current_graph_version\": next_graph_version,\n }\n },\n ),\n \"Failed to update workflow\",\n )\n if error:\n workflow_nodes_collection.delete_many(\n {\"workflow_id\": workflow_id, \"graph_version\": next_graph_version}\n )\n workflow_edges_collection.delete_many(\n {\"workflow_id\": workflow_id, \"graph_version\": next_graph_version}\n )\n return error\n\n try:\n workflow_nodes_collection.delete_many(\n {\"workflow_id\": workflow_id, \"graph_version\": {\"$ne\": next_graph_version}}\n )\n workflow_edges_collection.delete_many(\n {\"workflow_id\": workflow_id, \"graph_version\": {\"$ne\": next_graph_version}}\n )\n except Exception as cleanup_err:\n current_app.logger.warning(\n f\"Failed to clean old workflow graph versions for {workflow_id}: {cleanup_err}\"\n )\n\n return success_response()\n\n @require_auth\n def delete(self, workflow_id: str):\n \"\"\"Delete workflow and its graph.\"\"\"\n user_id = get_user_id()\n obj_id, error = validate_object_id(workflow_id, \"Workflow\")\n if error:\n return error\n\n workflow, error = check_resource_ownership(\n workflows_collection, obj_id, user_id, \"Workflow\"\n )\n if error:\n return error\n\n try:\n workflow_nodes_collection.delete_many({\"workflow_id\": workflow_id})\n workflow_edges_collection.delete_many({\"workflow_id\": workflow_id})\n workflows_collection.delete_one({\"_id\": workflow[\"_id\"], \"user\": user_id})\n except Exception as err:\n return _workflow_error_response(\"Failed to delete workflow\", err)\n\n return success_response()\n" }, { "path": "application/app.py", "content": "import os\nimport platform\nimport uuid\n\nimport dotenv\nfrom flask import Flask, jsonify, redirect, request\nfrom jose import jwt\n\nfrom application.auth import handle_auth\n\nfrom application.core.logging_config import setup_logging\n\nsetup_logging()\n\nfrom application.api import api # noqa: E402\nfrom application.api.answer import answer # noqa: E402\nfrom application.api.internal.routes import internal # noqa: E402\nfrom application.api.user.routes import user # noqa: E402\nfrom application.api.connector.routes import connector # noqa: E402\nfrom application.celery_init import celery # noqa: E402\nfrom application.core.settings import settings # noqa: E402\nfrom application.stt.upload_limits import ( # noqa: E402\n build_stt_file_size_limit_message,\n should_reject_stt_request,\n)\n\n\nif platform.system() == \"Windows\":\n import pathlib\n\n pathlib.PosixPath = pathlib.WindowsPath\ndotenv.load_dotenv()\n\napp = Flask(__name__)\napp.register_blueprint(user)\napp.register_blueprint(answer)\napp.register_blueprint(internal)\napp.register_blueprint(connector)\napp.config.update(\n UPLOAD_FOLDER=\"inputs\",\n CELERY_BROKER_URL=settings.CELERY_BROKER_URL,\n CELERY_RESULT_BACKEND=settings.CELERY_RESULT_BACKEND,\n MONGO_URI=settings.MONGO_URI,\n)\ncelery.config_from_object(\"application.celeryconfig\")\napi.init_app(app)\n\nif settings.AUTH_TYPE in (\"simple_jwt\", \"session_jwt\") and not settings.JWT_SECRET_KEY:\n key_file = \".jwt_secret_key\"\n try:\n with open(key_file, \"r\") as f:\n settings.JWT_SECRET_KEY = f.read().strip()\n except FileNotFoundError:\n new_key = os.urandom(32).hex()\n with open(key_file, \"w\") as f:\n f.write(new_key)\n settings.JWT_SECRET_KEY = new_key\n except Exception as e:\n raise RuntimeError(f\"Failed to setup JWT_SECRET_KEY: {e}\")\nSIMPLE_JWT_TOKEN = None\nif settings.AUTH_TYPE == \"simple_jwt\":\n payload = {\"sub\": \"local\"}\n SIMPLE_JWT_TOKEN = jwt.encode(payload, settings.JWT_SECRET_KEY, algorithm=\"HS256\")\n print(f\"Generated Simple JWT Token: {SIMPLE_JWT_TOKEN}\")\n\n\n@app.route(\"/\")\ndef home():\n if request.remote_addr in (\"0.0.0.0\", \"127.0.0.1\", \"localhost\", \"172.18.0.1\"):\n return redirect(\"http://localhost:5173\")\n else:\n return \"Welcome to DocsGPT Backend!\"\n\n\n@app.route(\"/api/config\")\ndef get_config():\n response = {\n \"auth_type\": settings.AUTH_TYPE,\n \"requires_auth\": settings.AUTH_TYPE in [\"simple_jwt\", \"session_jwt\"],\n }\n return jsonify(response)\n\n\n@app.route(\"/api/generate_token\")\ndef generate_token():\n if settings.AUTH_TYPE == \"session_jwt\":\n new_user_id = str(uuid.uuid4())\n token = jwt.encode(\n {\"sub\": new_user_id}, settings.JWT_SECRET_KEY, algorithm=\"HS256\"\n )\n return jsonify({\"token\": token})\n return jsonify({\"error\": \"Token generation not allowed in current auth mode\"}), 400\n\n\n@app.before_request\ndef enforce_stt_request_size_limits():\n if request.method == \"OPTIONS\":\n return None\n if should_reject_stt_request(request.path, request.content_length):\n return (\n jsonify(\n {\n \"success\": False,\n \"message\": build_stt_file_size_limit_message(),\n }\n ),\n 413,\n )\n return None\n\n\n@app.before_request\ndef authenticate_request():\n if request.method == \"OPTIONS\":\n return \"\", 200\n decoded_token = handle_auth(request)\n if not decoded_token:\n request.decoded_token = None\n elif \"error\" in decoded_token:\n return jsonify(decoded_token), 401\n else:\n request.decoded_token = decoded_token\n\n\n@app.after_request\ndef after_request(response):\n response.headers.add(\"Access-Control-Allow-Origin\", \"*\")\n response.headers.add(\"Access-Control-Allow-Headers\", \"Content-Type, Authorization\")\n response.headers.add(\n \"Access-Control-Allow-Methods\", \"GET, POST, PUT, DELETE, OPTIONS\"\n )\n return response\n\n\nif __name__ == \"__main__\":\n app.run(debug=settings.FLASK_DEBUG_MODE, port=7091)\n" }, { "path": "application/auth.py", "content": "from jose import jwt\n\nfrom application.core.settings import settings\n\n\ndef handle_auth(request, data={}):\n if settings.AUTH_TYPE in [\"simple_jwt\", \"session_jwt\"]:\n jwt_token = request.headers.get(\"Authorization\")\n if not jwt_token:\n return None\n\n jwt_token = jwt_token.replace(\"Bearer \", \"\")\n\n try:\n decoded_token = jwt.decode(\n jwt_token,\n settings.JWT_SECRET_KEY,\n algorithms=[\"HS256\"],\n options={\"verify_exp\": False},\n )\n return decoded_token\n except Exception:\n return {\n \"message\": \"Authentication error: invalid token\",\n \"error\": \"invalid_token\",\n }\n else:\n return {\"sub\": \"local\"}\n" }, { "path": "application/cache.py", "content": "import json\nimport logging\nimport time\nfrom threading import Lock\n\nimport redis\n\nfrom application.core.settings import settings\nfrom application.utils import get_hash\n\nlogger = logging.getLogger(__name__)\n\n_redis_instance = None\n_redis_creation_failed = False\n_instance_lock = Lock()\n\ndef get_redis_instance():\n global _redis_instance, _redis_creation_failed\n if _redis_instance is None and not _redis_creation_failed:\n with _instance_lock:\n if _redis_instance is None and not _redis_creation_failed:\n try:\n _redis_instance = redis.Redis.from_url(\n settings.CACHE_REDIS_URL, socket_connect_timeout=2\n )\n except ValueError as e:\n logger.error(f\"Invalid Redis URL: {e}\")\n _redis_creation_failed = True # Stop future attempts\n _redis_instance = None\n except redis.ConnectionError as e:\n logger.error(f\"Redis connection error: {e}\")\n _redis_instance = None # Keep trying for connection errors\n return _redis_instance\n\n\ndef gen_cache_key(messages, model=\"docgpt\", tools=None):\n if not all(isinstance(msg, dict) for msg in messages):\n raise ValueError(\"All messages must be dictionaries.\")\n messages_str = json.dumps(messages)\n tools_str = json.dumps(str(tools)) if tools else \"\"\n combined = f\"{model}_{messages_str}_{tools_str}\"\n cache_key = get_hash(combined)\n return cache_key\n\n\ndef gen_cache(func):\n def wrapper(self, model, messages, stream, tools=None, *args, **kwargs):\n if tools is not None:\n return func(self, model, messages, stream, tools, *args, **kwargs)\n \n try:\n cache_key = gen_cache_key(messages, model, tools)\n except ValueError as e:\n logger.error(f\"Cache key generation failed: {e}\")\n return func(self, model, messages, stream, tools, *args, **kwargs)\n\n redis_client = get_redis_instance()\n if redis_client:\n try:\n cached_response = redis_client.get(cache_key)\n if cached_response:\n return cached_response.decode(\"utf-8\")\n except Exception as e:\n logger.error(f\"Error getting cached response: {e}\", exc_info=True)\n\n result = func(self, model, messages, stream, tools, *args, **kwargs)\n if redis_client and isinstance(result, str):\n try:\n redis_client.set(cache_key, result, ex=1800)\n except Exception as e:\n logger.error(f\"Error setting cache: {e}\", exc_info=True)\n\n return result\n\n return wrapper\n\n\ndef stream_cache(func):\n def wrapper(self, model, messages, stream, tools=None, *args, **kwargs):\n if tools is not None:\n yield from func(self, model, messages, stream, tools, *args, **kwargs)\n return\n \n try:\n cache_key = gen_cache_key(messages, model, tools)\n except ValueError as e:\n logger.error(f\"Cache key generation failed: {e}\")\n yield from func(self, model, messages, stream, tools, *args, **kwargs)\n return\n\n redis_client = get_redis_instance()\n if redis_client:\n try:\n cached_response = redis_client.get(cache_key)\n if cached_response:\n logger.info(f\"Cache hit for stream key: {cache_key}\")\n cached_response = json.loads(cached_response.decode(\"utf-8\"))\n for chunk in cached_response:\n yield chunk\n time.sleep(0.03) # Simulate streaming delay\n return\n except Exception as e:\n logger.error(f\"Error getting cached stream: {e}\", exc_info=True)\n\n stream_cache_data = []\n for chunk in func(self, model, messages, stream, tools, *args, **kwargs):\n yield chunk\n stream_cache_data.append(str(chunk))\n\n if redis_client:\n try:\n redis_client.set(cache_key, json.dumps(stream_cache_data), ex=1800)\n logger.info(f\"Stream cache saved for key: {cache_key}\")\n except Exception as e:\n logger.error(f\"Error setting stream cache: {e}\", exc_info=True)\n\n return wrapper\n" }, { "path": "application/celery_init.py", "content": "from celery import Celery\nfrom application.core.settings import settings\nfrom celery.signals import setup_logging\n\n\ndef make_celery(app_name=__name__):\n celery = Celery(\n app_name,\n broker=settings.CELERY_BROKER_URL,\n backend=settings.CELERY_RESULT_BACKEND,\n )\n celery.conf.update(settings)\n return celery\n\n\n@setup_logging.connect\ndef config_loggers(*args, **kwargs):\n from application.core.logging_config import setup_logging\n\n setup_logging()\n\n\ncelery = make_celery()\ncelery.config_from_object(\"application.celeryconfig\")\n" }, { "path": "application/celeryconfig.py", "content": "import os\n\nbroker_url = os.getenv(\"CELERY_BROKER_URL\")\nresult_backend = os.getenv(\"CELERY_RESULT_BACKEND\")\n\ntask_serializer = 'json'\nresult_serializer = 'json'\naccept_content = ['json']\n\n# Autodiscover tasks\nimports = ('application.api.user.tasks',)\n" }, { "path": "application/core/__init__.py", "content": "" }, { "path": "application/core/json_schema_utils.py", "content": "from typing import Any, Dict, Optional\n\n\nclass JsonSchemaValidationError(ValueError):\n \"\"\"Raised when a JSON schema payload is invalid.\"\"\"\n\n\ndef normalize_json_schema_payload(json_schema: Any) -> Optional[Dict[str, Any]]:\n \"\"\"\n Normalize accepted JSON schema payload shapes to a plain schema object.\n\n Accepted inputs:\n - None\n - A raw schema object with a top-level \"type\"\n - A wrapped payload with a top-level \"schema\" object\n \"\"\"\n if json_schema is None:\n return None\n\n if not isinstance(json_schema, dict):\n raise JsonSchemaValidationError(\"must be a valid JSON object\")\n\n wrapped_schema = json_schema.get(\"schema\")\n if wrapped_schema is not None:\n if not isinstance(wrapped_schema, dict):\n raise JsonSchemaValidationError('field \"schema\" must be a valid JSON object')\n return wrapped_schema\n\n if \"type\" not in json_schema:\n raise JsonSchemaValidationError(\n 'must include either a \"type\" or \"schema\" field'\n )\n\n return json_schema\n" }, { "path": "application/core/logging_config.py", "content": "from logging.config import dictConfig\n\ndef setup_logging():\n dictConfig({\n 'version': 1,\n 'formatters': {\n 'default': {\n 'format': '[%(asctime)s] %(levelname)s in %(module)s: %(message)s',\n }\n },\n \"handlers\": {\n \"console\": {\n \"class\": \"logging.StreamHandler\",\n \"stream\": \"ext://sys.stdout\",\n \"formatter\": \"default\",\n }\n },\n 'root': {\n 'level': 'INFO',\n 'handlers': ['console'],\n },\n })" }, { "path": "application/core/model_configs.py", "content": "\"\"\"\nModel configurations for all supported LLM providers.\n\"\"\"\n\nfrom application.core.model_settings import (\n AvailableModel,\n ModelCapabilities,\n ModelProvider,\n)\n\n# Base image attachment types supported by most vision-capable LLMs\nIMAGE_ATTACHMENTS = [\n \"image/png\",\n \"image/jpeg\",\n \"image/jpg\",\n \"image/webp\",\n \"image/gif\",\n]\n\n# PDF excluded: most OpenAI-compatible endpoints don't support native PDF uploads.\n# When excluded, PDFs are synthetically processed by converting pages to images.\nOPENAI_ATTACHMENTS = IMAGE_ATTACHMENTS\n\nGOOGLE_ATTACHMENTS = [\"application/pdf\"] + IMAGE_ATTACHMENTS\n\nANTHROPIC_ATTACHMENTS = IMAGE_ATTACHMENTS\n\nOPENROUTER_ATTACHMENTS = IMAGE_ATTACHMENTS\n\n\nOPENAI_MODELS = [\n AvailableModel(\n id=\"gpt-5.1\",\n provider=ModelProvider.OPENAI,\n display_name=\"GPT-5.1\",\n description=\"Flagship model with enhanced reasoning, coding, and agentic capabilities\",\n capabilities=ModelCapabilities(\n supports_tools=True,\n supports_structured_output=True,\n supported_attachment_types=OPENAI_ATTACHMENTS,\n context_window=200000,\n ),\n ),\n AvailableModel(\n id=\"gpt-5-mini\",\n provider=ModelProvider.OPENAI,\n display_name=\"GPT-5 Mini\",\n description=\"Faster, cost-effective variant of GPT-5.1\",\n capabilities=ModelCapabilities(\n supports_tools=True,\n supports_structured_output=True,\n supported_attachment_types=OPENAI_ATTACHMENTS,\n context_window=200000,\n ),\n )\n]\n\n\nANTHROPIC_MODELS = [\n AvailableModel(\n id=\"claude-3-5-sonnet-20241022\",\n provider=ModelProvider.ANTHROPIC,\n display_name=\"Claude 3.5 Sonnet (Latest)\",\n description=\"Latest Claude 3.5 Sonnet with enhanced capabilities\",\n capabilities=ModelCapabilities(\n supports_tools=True,\n supported_attachment_types=ANTHROPIC_ATTACHMENTS,\n context_window=200000,\n ),\n ),\n AvailableModel(\n id=\"claude-3-5-sonnet\",\n provider=ModelProvider.ANTHROPIC,\n display_name=\"Claude 3.5 Sonnet\",\n description=\"Balanced performance and capability\",\n capabilities=ModelCapabilities(\n supports_tools=True,\n supported_attachment_types=ANTHROPIC_ATTACHMENTS,\n context_window=200000,\n ),\n ),\n AvailableModel(\n id=\"claude-3-opus\",\n provider=ModelProvider.ANTHROPIC,\n display_name=\"Claude 3 Opus\",\n description=\"Most capable Claude model\",\n capabilities=ModelCapabilities(\n supports_tools=True,\n supported_attachment_types=ANTHROPIC_ATTACHMENTS,\n context_window=200000,\n ),\n ),\n AvailableModel(\n id=\"claude-3-haiku\",\n provider=ModelProvider.ANTHROPIC,\n display_name=\"Claude 3 Haiku\",\n description=\"Fastest Claude model\",\n capabilities=ModelCapabilities(\n supports_tools=True,\n supported_attachment_types=ANTHROPIC_ATTACHMENTS,\n context_window=200000,\n ),\n ),\n]\n\n\nGOOGLE_MODELS = [\n AvailableModel(\n id=\"gemini-flash-latest\",\n provider=ModelProvider.GOOGLE,\n display_name=\"Gemini Flash (Latest)\",\n description=\"Latest experimental Gemini model\",\n capabilities=ModelCapabilities(\n supports_tools=True,\n supports_structured_output=True,\n supported_attachment_types=GOOGLE_ATTACHMENTS,\n context_window=int(1e6),\n ),\n ),\n AvailableModel(\n id=\"gemini-flash-lite-latest\",\n provider=ModelProvider.GOOGLE,\n display_name=\"Gemini Flash Lite (Latest)\",\n description=\"Fast with huge context window\",\n capabilities=ModelCapabilities(\n supports_tools=True,\n supports_structured_output=True,\n supported_attachment_types=GOOGLE_ATTACHMENTS,\n context_window=int(1e6),\n ),\n ),\n AvailableModel(\n id=\"gemini-3-pro-preview\",\n provider=ModelProvider.GOOGLE,\n display_name=\"Gemini 3 Pro\",\n description=\"Most capable Gemini model\",\n capabilities=ModelCapabilities(\n supports_tools=True,\n supports_structured_output=True,\n supported_attachment_types=GOOGLE_ATTACHMENTS,\n context_window=2000000,\n ),\n ),\n]\n\n\nGROQ_MODELS = [\n AvailableModel(\n id=\"llama-3.3-70b-versatile\",\n provider=ModelProvider.GROQ,\n display_name=\"Llama 3.3 70B\",\n description=\"Latest Llama model with high-speed inference\",\n capabilities=ModelCapabilities(\n supports_tools=True,\n context_window=128000,\n ),\n ),\n AvailableModel(\n id=\"openai/gpt-oss-120b\",\n provider=ModelProvider.GROQ,\n display_name=\"GPT-OSS 120B\",\n description=\"Open-source GPT model optimized for speed\",\n capabilities=ModelCapabilities(\n supports_tools=True,\n context_window=128000,\n ),\n ),\n]\n\n\nOPENROUTER_MODELS = [\n AvailableModel(\n id=\"qwen/qwen3-coder:free\",\n provider=ModelProvider.OPENROUTER,\n display_name=\"Qwen 3 Coder\",\n description=\"Latest Qwen model with high-speed inference\",\n capabilities=ModelCapabilities(\n supports_tools=True,\n context_window=128000,\n supported_attachment_types=OPENROUTER_ATTACHMENTS\n ),\n ),\n AvailableModel(\n id=\"google/gemma-3-27b-it:free\",\n provider=ModelProvider.OPENROUTER,\n display_name=\"Gemma 3 27B\",\n description=\"Latest Gemma model with high-speed inference\",\n capabilities=ModelCapabilities(\n supports_tools=True,\n context_window=128000,\n supported_attachment_types=OPENROUTER_ATTACHMENTS\n ),\n ),\n]\n\nAZURE_OPENAI_MODELS = [\n AvailableModel(\n id=\"azure-gpt-4\",\n provider=ModelProvider.AZURE_OPENAI,\n display_name=\"Azure OpenAI GPT-4\",\n description=\"Azure-hosted GPT model\",\n capabilities=ModelCapabilities(\n supports_tools=True,\n supports_structured_output=True,\n supported_attachment_types=OPENAI_ATTACHMENTS,\n context_window=8192,\n ),\n ),\n]\n\n\ndef create_custom_openai_model(model_name: str, base_url: str) -> AvailableModel:\n \"\"\"Create a custom OpenAI-compatible model (e.g., LM Studio, Ollama).\"\"\"\n return AvailableModel(\n id=model_name,\n provider=ModelProvider.OPENAI,\n display_name=model_name,\n description=f\"Custom OpenAI-compatible model at {base_url}\",\n base_url=base_url,\n capabilities=ModelCapabilities(\n supports_tools=True,\n supported_attachment_types=OPENAI_ATTACHMENTS,\n ),\n )\n" }, { "path": "application/core/model_settings.py", "content": "import logging\nfrom dataclasses import dataclass, field\nfrom enum import Enum\nfrom typing import Dict, List, Optional\n\nlogger = logging.getLogger(__name__)\n\n\nclass ModelProvider(str, Enum):\n OPENAI = \"openai\"\n OPENROUTER = \"openrouter\"\n AZURE_OPENAI = \"azure_openai\"\n ANTHROPIC = \"anthropic\"\n GROQ = \"groq\"\n GOOGLE = \"google\"\n HUGGINGFACE = \"huggingface\"\n LLAMA_CPP = \"llama.cpp\"\n DOCSGPT = \"docsgpt\"\n PREMAI = \"premai\"\n SAGEMAKER = \"sagemaker\"\n NOVITA = \"novita\"\n\n\n@dataclass\nclass ModelCapabilities:\n supports_tools: bool = False\n supports_structured_output: bool = False\n supports_streaming: bool = True\n supported_attachment_types: List[str] = field(default_factory=list)\n context_window: int = 128000\n input_cost_per_token: Optional[float] = None\n output_cost_per_token: Optional[float] = None\n\n\n@dataclass\nclass AvailableModel:\n id: str\n provider: ModelProvider\n display_name: str\n description: str = \"\"\n capabilities: ModelCapabilities = field(default_factory=ModelCapabilities)\n enabled: bool = True\n base_url: Optional[str] = None\n\n def to_dict(self) -> Dict:\n result = {\n \"id\": self.id,\n \"provider\": self.provider.value,\n \"display_name\": self.display_name,\n \"description\": self.description,\n \"supported_attachment_types\": self.capabilities.supported_attachment_types,\n \"supports_tools\": self.capabilities.supports_tools,\n \"supports_structured_output\": self.capabilities.supports_structured_output,\n \"supports_streaming\": self.capabilities.supports_streaming,\n \"context_window\": self.capabilities.context_window,\n \"enabled\": self.enabled,\n }\n if self.base_url:\n result[\"base_url\"] = self.base_url\n return result\n\n\nclass ModelRegistry:\n _instance = None\n _initialized = False\n\n def __new__(cls):\n if cls._instance is None:\n cls._instance = super().__new__(cls)\n return cls._instance\n\n def __init__(self):\n if not ModelRegistry._initialized:\n self.models: Dict[str, AvailableModel] = {}\n self.default_model_id: Optional[str] = None\n self._load_models()\n ModelRegistry._initialized = True\n\n @classmethod\n def get_instance(cls) -> \"ModelRegistry\":\n return cls()\n\n def _load_models(self):\n from application.core.settings import settings\n\n self.models.clear()\n\n # Skip DocsGPT model if using custom OpenAI-compatible endpoint\n if not settings.OPENAI_BASE_URL:\n self._add_docsgpt_models(settings)\n if (\n settings.OPENAI_API_KEY\n or (settings.LLM_PROVIDER == \"openai\" and settings.API_KEY)\n or settings.OPENAI_BASE_URL\n ):\n self._add_openai_models(settings)\n if settings.OPENAI_API_BASE or (\n settings.LLM_PROVIDER == \"azure_openai\" and settings.API_KEY\n ):\n self._add_azure_openai_models(settings)\n if settings.ANTHROPIC_API_KEY or (\n settings.LLM_PROVIDER == \"anthropic\" and settings.API_KEY\n ):\n self._add_anthropic_models(settings)\n if settings.GOOGLE_API_KEY or (\n settings.LLM_PROVIDER == \"google\" and settings.API_KEY\n ):\n self._add_google_models(settings)\n if settings.GROQ_API_KEY or (\n settings.LLM_PROVIDER == \"groq\" and settings.API_KEY\n ):\n self._add_groq_models(settings)\n if settings.OPEN_ROUTER_API_KEY or (\n settings.LLM_PROVIDER == \"openrouter\" and settings.API_KEY\n ):\n self._add_openrouter_models(settings)\n if settings.HUGGINGFACE_API_KEY or (\n settings.LLM_PROVIDER == \"huggingface\" and settings.API_KEY\n ):\n self._add_huggingface_models(settings)\n # Default model selection\n if settings.LLM_NAME:\n # Parse LLM_NAME (may be comma-separated)\n model_names = self._parse_model_names(settings.LLM_NAME)\n # First model in the list becomes default\n for model_name in model_names:\n if model_name in self.models:\n self.default_model_id = model_name\n break\n # Backward compat: try exact match if no parsed model found\n if not self.default_model_id and settings.LLM_NAME in self.models:\n self.default_model_id = settings.LLM_NAME\n\n if not self.default_model_id:\n if settings.LLM_PROVIDER and settings.API_KEY:\n for model_id, model in self.models.items():\n if model.provider.value == settings.LLM_PROVIDER:\n self.default_model_id = model_id\n break\n\n if not self.default_model_id and self.models:\n self.default_model_id = next(iter(self.models.keys()))\n logger.info(\n f\"ModelRegistry loaded {len(self.models)} models, default: {self.default_model_id}\"\n )\n\n def _add_openai_models(self, settings):\n from application.core.model_configs import (\n OPENAI_MODELS,\n create_custom_openai_model,\n )\n\n # Check if using local OpenAI-compatible endpoint (Ollama, LM Studio, etc.)\n using_local_endpoint = bool(\n settings.OPENAI_BASE_URL and settings.OPENAI_BASE_URL.strip()\n )\n\n if using_local_endpoint:\n # When OPENAI_BASE_URL is set, ONLY register custom models from LLM_NAME\n # Do NOT add standard OpenAI models (gpt-5.1, etc.)\n if settings.LLM_NAME:\n model_names = self._parse_model_names(settings.LLM_NAME)\n for model_name in model_names:\n custom_model = create_custom_openai_model(\n model_name, settings.OPENAI_BASE_URL\n )\n self.models[model_name] = custom_model\n logger.info(\n f\"Registered custom OpenAI model: {model_name} at {settings.OPENAI_BASE_URL}\"\n )\n else:\n # Standard OpenAI API usage - add standard models if API key is valid\n if settings.OPENAI_API_KEY:\n for model in OPENAI_MODELS:\n self.models[model.id] = model\n\n def _add_azure_openai_models(self, settings):\n from application.core.model_configs import AZURE_OPENAI_MODELS\n\n if settings.LLM_PROVIDER == \"azure_openai\" and settings.LLM_NAME:\n for model in AZURE_OPENAI_MODELS:\n if model.id == settings.LLM_NAME:\n self.models[model.id] = model\n return\n for model in AZURE_OPENAI_MODELS:\n self.models[model.id] = model\n\n def _add_anthropic_models(self, settings):\n from application.core.model_configs import ANTHROPIC_MODELS\n\n if settings.ANTHROPIC_API_KEY:\n for model in ANTHROPIC_MODELS:\n self.models[model.id] = model\n return\n if settings.LLM_PROVIDER == \"anthropic\" and settings.LLM_NAME:\n for model in ANTHROPIC_MODELS:\n if model.id == settings.LLM_NAME:\n self.models[model.id] = model\n return\n for model in ANTHROPIC_MODELS:\n self.models[model.id] = model\n\n def _add_google_models(self, settings):\n from application.core.model_configs import GOOGLE_MODELS\n\n if settings.GOOGLE_API_KEY:\n for model in GOOGLE_MODELS:\n self.models[model.id] = model\n return\n if settings.LLM_PROVIDER == \"google\" and settings.LLM_NAME:\n for model in GOOGLE_MODELS:\n if model.id == settings.LLM_NAME:\n self.models[model.id] = model\n return\n for model in GOOGLE_MODELS:\n self.models[model.id] = model\n\n def _add_groq_models(self, settings):\n from application.core.model_configs import GROQ_MODELS\n\n if settings.GROQ_API_KEY:\n for model in GROQ_MODELS:\n self.models[model.id] = model\n return\n if settings.LLM_PROVIDER == \"groq\" and settings.LLM_NAME:\n for model in GROQ_MODELS:\n if model.id == settings.LLM_NAME:\n self.models[model.id] = model\n return\n for model in GROQ_MODELS:\n self.models[model.id] = model\n \n def _add_openrouter_models(self, settings):\n from application.core.model_configs import OPENROUTER_MODELS\n\n if settings.OPEN_ROUTER_API_KEY:\n for model in OPENROUTER_MODELS:\n self.models[model.id] = model\n return\n if settings.LLM_PROVIDER == \"openrouter\" and settings.LLM_NAME:\n for model in OPENROUTER_MODELS:\n if model.id == settings.LLM_NAME:\n self.models[model.id] = model\n return\n for model in OPENROUTER_MODELS:\n self.models[model.id] = model\n\n def _add_docsgpt_models(self, settings):\n model_id = \"docsgpt-local\"\n model = AvailableModel(\n id=model_id,\n provider=ModelProvider.DOCSGPT,\n display_name=\"DocsGPT Model\",\n description=\"Local model\",\n capabilities=ModelCapabilities(\n supports_tools=False,\n supported_attachment_types=[],\n ),\n )\n self.models[model_id] = model\n\n def _add_huggingface_models(self, settings):\n model_id = \"huggingface-local\"\n model = AvailableModel(\n id=model_id,\n provider=ModelProvider.HUGGINGFACE,\n display_name=\"Hugging Face Model\",\n description=\"Local Hugging Face model\",\n capabilities=ModelCapabilities(\n supports_tools=False,\n supported_attachment_types=[],\n ),\n )\n self.models[model_id] = model\n\n def _parse_model_names(self, llm_name: str) -> List[str]:\n \"\"\"\n Parse LLM_NAME which may contain comma-separated model names.\n E.g., 'deepseek-r1:1.5b,gemma:2b' -> ['deepseek-r1:1.5b', 'gemma:2b']\n \"\"\"\n if not llm_name:\n return []\n return [name.strip() for name in llm_name.split(\",\") if name.strip()]\n\n def get_model(self, model_id: str) -> Optional[AvailableModel]:\n return self.models.get(model_id)\n\n def get_all_models(self) -> List[AvailableModel]:\n return list(self.models.values())\n\n def get_enabled_models(self) -> List[AvailableModel]:\n return [m for m in self.models.values() if m.enabled]\n\n def model_exists(self, model_id: str) -> bool:\n return model_id in self.models\n" }, { "path": "application/core/model_utils.py", "content": "from typing import Any, Dict, Optional\n\nfrom application.core.model_settings import ModelRegistry\n\n\ndef get_api_key_for_provider(provider: str) -> Optional[str]:\n \"\"\"Get the appropriate API key for a provider\"\"\"\n from application.core.settings import settings\n\n provider_key_map = {\n \"openai\": settings.OPENAI_API_KEY,\n \"openrouter\": settings.OPEN_ROUTER_API_KEY,\n \"anthropic\": settings.ANTHROPIC_API_KEY,\n \"google\": settings.GOOGLE_API_KEY,\n \"groq\": settings.GROQ_API_KEY,\n \"huggingface\": settings.HUGGINGFACE_API_KEY,\n \"azure_openai\": settings.API_KEY,\n \"docsgpt\": None,\n \"llama.cpp\": None,\n }\n\n provider_key = provider_key_map.get(provider)\n if provider_key:\n return provider_key\n return settings.API_KEY\n\n\ndef get_all_available_models() -> Dict[str, Dict[str, Any]]:\n \"\"\"Get all available models with metadata for API response\"\"\"\n registry = ModelRegistry.get_instance()\n return {model.id: model.to_dict() for model in registry.get_enabled_models()}\n\n\ndef validate_model_id(model_id: str) -> bool:\n \"\"\"Check if a model ID exists in registry\"\"\"\n registry = ModelRegistry.get_instance()\n return registry.model_exists(model_id)\n\n\ndef get_model_capabilities(model_id: str) -> Optional[Dict[str, Any]]:\n \"\"\"Get capabilities for a specific model\"\"\"\n registry = ModelRegistry.get_instance()\n model = registry.get_model(model_id)\n if model:\n return {\n \"supported_attachment_types\": model.capabilities.supported_attachment_types,\n \"supports_tools\": model.capabilities.supports_tools,\n \"supports_structured_output\": model.capabilities.supports_structured_output,\n \"context_window\": model.capabilities.context_window,\n }\n return None\n\n\ndef get_default_model_id() -> str:\n \"\"\"Get the system default model ID\"\"\"\n registry = ModelRegistry.get_instance()\n return registry.default_model_id\n\n\ndef get_provider_from_model_id(model_id: str) -> Optional[str]:\n \"\"\"Get the provider name for a given model_id\"\"\"\n registry = ModelRegistry.get_instance()\n model = registry.get_model(model_id)\n if model:\n return model.provider.value\n return None\n\n\ndef get_token_limit(model_id: str) -> int:\n \"\"\"\n Get context window (token limit) for a model.\n Returns model's context_window or default 128000 if model not found.\n \"\"\"\n from application.core.settings import settings\n\n registry = ModelRegistry.get_instance()\n model = registry.get_model(model_id)\n if model:\n return model.capabilities.context_window\n return settings.DEFAULT_LLM_TOKEN_LIMIT\n\n\ndef get_base_url_for_model(model_id: str) -> Optional[str]:\n \"\"\"\n Get the custom base_url for a specific model if configured.\n Returns None if no custom base_url is set.\n \"\"\"\n registry = ModelRegistry.get_instance()\n model = registry.get_model(model_id)\n if model:\n return model.base_url\n return None\n" }, { "path": "application/core/mongo_db.py", "content": "from application.core.settings import settings\nfrom pymongo import MongoClient\n\n\nclass MongoDB:\n _client = None\n\n @classmethod\n def get_client(cls):\n \"\"\"\n Get the MongoDB client instance, creating it if necessary.\n \"\"\"\n if cls._client is None:\n cls._client = MongoClient(settings.MONGO_URI)\n return cls._client\n\n @classmethod\n def close_client(cls):\n \"\"\"\n Close the MongoDB client connection.\n \"\"\"\n if cls._client is not None:\n cls._client.close()\n cls._client = None\n" }, { "path": "application/core/settings.py", "content": "import os\nfrom pathlib import Path\nfrom typing import Optional\n\nfrom pydantic import field_validator\nfrom pydantic_settings import BaseSettings, SettingsConfigDict\n\ncurrent_dir = os.path.dirname(\n os.path.dirname(os.path.dirname(os.path.abspath(__file__)))\n)\n\n\nclass Settings(BaseSettings):\n model_config = SettingsConfigDict(extra=\"ignore\")\n\n AUTH_TYPE: Optional[str] = None # simple_jwt, session_jwt, or None\n LLM_PROVIDER: str = \"docsgpt\"\n LLM_NAME: Optional[str] = (\n None # if LLM_PROVIDER is openai, LLM_NAME can be gpt-4 or gpt-3.5-turbo\n )\n EMBEDDINGS_NAME: str = \"huggingface_sentence-transformers/all-mpnet-base-v2\"\n EMBEDDINGS_BASE_URL: Optional[str] = None # Remote embeddings API URL (OpenAI-compatible)\n EMBEDDINGS_KEY: Optional[str] = (\n None # api key for embeddings (if using openai, just copy API_KEY)\n )\n \n CELERY_BROKER_URL: str = \"redis://localhost:6379/0\"\n CELERY_RESULT_BACKEND: str = \"redis://localhost:6379/1\"\n MONGO_URI: str = \"mongodb://localhost:27017/docsgpt\"\n MONGO_DB_NAME: str = \"docsgpt\"\n LLM_PATH: str = os.path.join(current_dir, \"models/docsgpt-7b-f16.gguf\")\n DEFAULT_MAX_HISTORY: int = 150\n DEFAULT_LLM_TOKEN_LIMIT: int = 128000 # Fallback when model not found in registry\n RESERVED_TOKENS: dict = {\n \"system_prompt\": 500,\n \"current_query\": 500,\n \"safety_buffer\": 1000,\n }\n DEFAULT_AGENT_LIMITS: dict = {\n \"token_limit\": 50000,\n \"request_limit\": 500,\n }\n UPLOAD_FOLDER: str = \"inputs\"\n PARSE_PDF_AS_IMAGE: bool = False\n PARSE_IMAGE_REMOTE: bool = False\n DOCLING_OCR_ENABLED: bool = False # Enable OCR for docling parsers (PDF, images)\n DOCLING_OCR_ATTACHMENTS_ENABLED: bool = False # Enable OCR for docling when parsing attachments\n VECTOR_STORE: str = (\n \"faiss\" # \"faiss\" or \"elasticsearch\" or \"qdrant\" or \"milvus\" or \"lancedb\" or \"pgvector\"\n )\n RETRIEVERS_ENABLED: list = [\"classic_rag\"]\n AGENT_NAME: str = \"classic\"\n FALLBACK_LLM_PROVIDER: Optional[str] = None # provider for fallback llm\n FALLBACK_LLM_NAME: Optional[str] = None # model name for fallback llm\n FALLBACK_LLM_API_KEY: Optional[str] = None # api key for fallback llm\n\n # Google Drive integration\n GOOGLE_CLIENT_ID: Optional[str] = (\n None # Replace with your actual Google OAuth client ID\n )\n GOOGLE_CLIENT_SECRET: Optional[str] = (\n None # Replace with your actual Google OAuth client secret\n )\n CONNECTOR_REDIRECT_BASE_URI: Optional[str] = (\n \"http://127.0.0.1:7091/api/connectors/callback\" ##add redirect url as it is to your provider's console(gcp)\n )\n\n # Microsoft Entra ID (Azure AD) integration\n MICROSOFT_CLIENT_ID: Optional[str] = None # Azure AD Application (client) ID\n MICROSOFT_CLIENT_SECRET: Optional[str] = None # Azure AD Application client secret\n MICROSOFT_TENANT_ID: Optional[str] = \"common\" # Azure AD Tenant ID (or 'common' for multi-tenant)\n MICROSOFT_AUTHORITY: Optional[str] = None # e.g., \"https://login.microsoftonline.com/{tenant_id}\"\n\n # GitHub source\n GITHUB_ACCESS_TOKEN: Optional[str] = None # PAT token with read repo access\n\n # LLM Cache\n CACHE_REDIS_URL: str = \"redis://localhost:6379/2\"\n\n API_URL: str = \"http://localhost:7091\" # backend url for celery worker\n MCP_OAUTH_REDIRECT_URI: Optional[str] = None # public callback URL for MCP OAuth\n INTERNAL_KEY: Optional[str] = None # internal api key for worker-to-backend auth\n\n API_KEY: Optional[str] = None # LLM api key (used by LLM_PROVIDER)\n\n # Provider-specific API keys (for multi-model support)\n OPENAI_API_KEY: Optional[str] = None\n ANTHROPIC_API_KEY: Optional[str] = None\n GOOGLE_API_KEY: Optional[str] = None\n GROQ_API_KEY: Optional[str] = None\n HUGGINGFACE_API_KEY: Optional[str] = None\n OPEN_ROUTER_API_KEY: Optional[str] = None\n\n OPENAI_API_BASE: Optional[str] = None # azure openai api base url\n OPENAI_API_VERSION: Optional[str] = None # azure openai api version\n AZURE_DEPLOYMENT_NAME: Optional[str] = None # azure deployment name for answering\n AZURE_EMBEDDINGS_DEPLOYMENT_NAME: Optional[str] = (\n None # azure deployment name for embeddings\n )\n OPENAI_BASE_URL: Optional[str] = (\n None # openai base url for open ai compatable models\n )\n\n # elasticsearch\n ELASTIC_CLOUD_ID: Optional[str] = None # cloud id for elasticsearch\n ELASTIC_USERNAME: Optional[str] = None # username for elasticsearch\n ELASTIC_PASSWORD: Optional[str] = None # password for elasticsearch\n ELASTIC_URL: Optional[str] = None # url for elasticsearch\n ELASTIC_INDEX: Optional[str] = \"docsgpt\" # index name for elasticsearch\n\n # SageMaker config\n SAGEMAKER_ENDPOINT: Optional[str] = None # SageMaker endpoint name\n SAGEMAKER_REGION: Optional[str] = None # SageMaker region name\n SAGEMAKER_ACCESS_KEY: Optional[str] = None # SageMaker access key\n SAGEMAKER_SECRET_KEY: Optional[str] = None # SageMaker secret key\n\n # prem ai project id\n PREMAI_PROJECT_ID: Optional[str] = None\n\n # Qdrant vectorstore config\n QDRANT_COLLECTION_NAME: Optional[str] = \"docsgpt\"\n QDRANT_LOCATION: Optional[str] = None\n QDRANT_URL: Optional[str] = None\n QDRANT_PORT: Optional[int] = 6333\n QDRANT_GRPC_PORT: int = 6334\n QDRANT_PREFER_GRPC: bool = False\n QDRANT_HTTPS: Optional[bool] = None\n QDRANT_API_KEY: Optional[str] = None\n QDRANT_PREFIX: Optional[str] = None\n QDRANT_TIMEOUT: Optional[float] = None\n QDRANT_HOST: Optional[str] = None\n QDRANT_PATH: Optional[str] = None\n QDRANT_DISTANCE_FUNC: str = \"Cosine\"\n\n # PGVector vectorstore config\n PGVECTOR_CONNECTION_STRING: Optional[str] = None\n # Milvus vectorstore config\n MILVUS_COLLECTION_NAME: Optional[str] = \"docsgpt\"\n MILVUS_URI: Optional[str] = \"./milvus_local.db\" # milvus lite version as default\n MILVUS_TOKEN: Optional[str] = \"\"\n\n # LanceDB vectorstore config\n LANCEDB_PATH: str = \"./data/lancedb\" # Path where LanceDB stores its local data\n LANCEDB_TABLE_NAME: Optional[str] = (\n \"docsgpts\" # Name of the table to use for storing vectors\n )\n\n FLASK_DEBUG_MODE: bool = False\n STORAGE_TYPE: str = \"local\" # local or s3\n URL_STRATEGY: str = \"backend\" # backend or s3\n\n JWT_SECRET_KEY: str = \"\"\n\n # Encryption settings\n ENCRYPTION_SECRET_KEY: str = \"default-docsgpt-encryption-key\"\n\n TTS_PROVIDER: str = \"google_tts\" # google_tts or elevenlabs\n ELEVENLABS_API_KEY: Optional[str] = None\n STT_PROVIDER: str = \"openai\" # openai or faster_whisper\n OPENAI_STT_MODEL: str = \"gpt-4o-mini-transcribe\"\n STT_LANGUAGE: Optional[str] = None\n STT_MAX_FILE_SIZE_MB: int = 50\n STT_ENABLE_TIMESTAMPS: bool = False\n STT_ENABLE_DIARIZATION: bool = False\n\n # Tool pre-fetch settings\n ENABLE_TOOL_PREFETCH: bool = True\n\n # Conversation Compression Settings\n ENABLE_CONVERSATION_COMPRESSION: bool = True\n COMPRESSION_THRESHOLD_PERCENTAGE: float = 0.8 # Trigger at 80% of context\n COMPRESSION_MODEL_OVERRIDE: Optional[str] = None # Use different model for compression\n COMPRESSION_PROMPT_VERSION: str = \"v1.0\" # Track prompt iterations\n COMPRESSION_MAX_HISTORY_POINTS: int = 3 # Keep only last N compression points to prevent DB bloat\n\n @field_validator(\n \"API_KEY\",\n \"OPENAI_API_KEY\",\n \"ANTHROPIC_API_KEY\",\n \"GOOGLE_API_KEY\",\n \"GROQ_API_KEY\",\n \"HUGGINGFACE_API_KEY\",\n \"EMBEDDINGS_KEY\",\n \"FALLBACK_LLM_API_KEY\",\n \"QDRANT_API_KEY\",\n \"ELEVENLABS_API_KEY\",\n \"INTERNAL_KEY\",\n mode=\"before\",\n )\n @classmethod\n def normalize_api_key(cls, v: Optional[str]) -> Optional[str]:\n \"\"\"\n Normalize API keys: convert 'None', 'none', empty strings,\n and whitespace-only strings to actual None.\n Handles Pydantic loading 'None' from .env as string \"None\".\n \"\"\"\n if v is None:\n return None\n if not isinstance(v, str):\n return v\n stripped = v.strip()\n if stripped == \"\" or stripped.lower() == \"none\":\n return None\n return stripped\n\n\n# Project root is one level above application/\npath = Path(__file__).parent.parent.parent.absolute()\nsettings = Settings(_env_file=path.joinpath(\".env\"), _env_file_encoding=\"utf-8\")\n" }, { "path": "application/core/url_validation.py", "content": "\"\"\"\nURL validation utilities to prevent SSRF (Server-Side Request Forgery) attacks.\n\nThis module provides functions to validate URLs before making HTTP requests,\nblocking access to internal networks, cloud metadata services, and other\npotentially dangerous endpoints.\n\"\"\"\n\nimport ipaddress\nimport socket\nfrom urllib.parse import urlparse\nfrom typing import Optional, Set\n\n\nclass SSRFError(Exception):\n \"\"\"Raised when a URL fails SSRF validation.\"\"\"\n pass\n\n\n# Blocked hostnames that should never be accessed\nBLOCKED_HOSTNAMES: Set[str] = {\n \"localhost\",\n \"localhost.localdomain\",\n \"metadata.google.internal\",\n \"metadata\",\n}\n\n# Cloud metadata IP addresses (AWS, GCP, Azure, etc.)\nMETADATA_IPS: Set[str] = {\n \"169.254.169.254\", # AWS, GCP, Azure metadata\n \"169.254.170.2\", # AWS ECS task metadata\n \"fd00:ec2::254\", # AWS IPv6 metadata\n}\n\n# Allowed schemes for external requests\nALLOWED_SCHEMES: Set[str] = {\"http\", \"https\"}\n\n\ndef is_private_ip(ip_str: str) -> bool:\n \"\"\"\n Check if an IP address is private, loopback, or link-local.\n\n Args:\n ip_str: IP address as a string\n\n Returns:\n True if the IP is private/internal, False otherwise\n \"\"\"\n try:\n ip = ipaddress.ip_address(ip_str)\n return (\n ip.is_private or\n ip.is_loopback or\n ip.is_link_local or\n ip.is_reserved or\n ip.is_multicast or\n ip.is_unspecified\n )\n except ValueError:\n # If we can't parse it as an IP, return False\n return False\n\n\ndef is_metadata_ip(ip_str: str) -> bool:\n \"\"\"\n Check if an IP address is a cloud metadata service IP.\n\n Args:\n ip_str: IP address as a string\n\n Returns:\n True if the IP is a metadata service, False otherwise\n \"\"\"\n return ip_str in METADATA_IPS\n\n\ndef resolve_hostname(hostname: str) -> Optional[str]:\n \"\"\"\n Resolve a hostname to an IP address.\n\n Args:\n hostname: The hostname to resolve\n\n Returns:\n The resolved IP address, or None if resolution fails\n \"\"\"\n try:\n return socket.gethostbyname(hostname)\n except socket.gaierror:\n return None\n\n\ndef validate_url(url: str, allow_localhost: bool = False) -> str:\n \"\"\"\n Validate a URL to prevent SSRF attacks.\n\n This function checks that:\n 1. The URL has an allowed scheme (http or https)\n 2. The hostname is not a blocked hostname\n 3. The resolved IP is not a private/internal IP\n 4. The resolved IP is not a cloud metadata service\n\n Args:\n url: The URL to validate\n allow_localhost: If True, allow localhost connections (for testing only)\n\n Returns:\n The validated URL (with scheme added if missing)\n\n Raises:\n SSRFError: If the URL fails validation\n \"\"\"\n # Ensure URL has a scheme\n if not urlparse(url).scheme:\n url = \"http://\" + url\n\n parsed = urlparse(url)\n\n # Check scheme\n if parsed.scheme not in ALLOWED_SCHEMES:\n raise SSRFError(f\"URL scheme '{parsed.scheme}' is not allowed. Only HTTP(S) is permitted.\")\n\n hostname = parsed.hostname\n if not hostname:\n raise SSRFError(\"URL must have a valid hostname.\")\n\n hostname_lower = hostname.lower()\n\n # Check blocked hostnames\n if hostname_lower in BLOCKED_HOSTNAMES and not allow_localhost:\n raise SSRFError(f\"Access to '{hostname}' is not allowed.\")\n\n # Check if hostname is an IP address directly\n try:\n ip = ipaddress.ip_address(hostname)\n ip_str = str(ip)\n\n if is_metadata_ip(ip_str):\n raise SSRFError(\"Access to cloud metadata services is not allowed.\")\n\n if is_private_ip(ip_str) and not allow_localhost:\n raise SSRFError(\"Access to private/internal IP addresses is not allowed.\")\n\n return url\n except ValueError:\n # Not an IP address, it's a hostname - resolve it\n pass\n\n # Resolve hostname and check the IP\n resolved_ip = resolve_hostname(hostname)\n if resolved_ip is None:\n raise SSRFError(f\"Unable to resolve hostname: {hostname}\")\n\n if is_metadata_ip(resolved_ip):\n raise SSRFError(\"Access to cloud metadata services is not allowed.\")\n\n if is_private_ip(resolved_ip) and not allow_localhost:\n raise SSRFError(\"Access to private/internal networks is not allowed.\")\n\n return url\n\n\ndef validate_url_safe(url: str, allow_localhost: bool = False) -> tuple[bool, str, Optional[str]]:\n \"\"\"\n Validate a URL and return a tuple with validation result.\n\n This is a non-throwing version of validate_url for cases where\n you want to handle validation failures gracefully.\n\n Args:\n url: The URL to validate\n allow_localhost: If True, allow localhost connections (for testing only)\n\n Returns:\n Tuple of (is_valid, validated_url_or_original, error_message_or_none)\n \"\"\"\n try:\n validated = validate_url(url, allow_localhost)\n return (True, validated, None)\n except SSRFError as e:\n return (False, url, str(e))\n" }, { "path": "application/error.py", "content": "from flask import jsonify\nfrom werkzeug.http import HTTP_STATUS_CODES\n\n\ndef response_error(code_status, message=None):\n payload = {'error': HTTP_STATUS_CODES.get(code_status, \"something went wrong\")}\n if message:\n payload['message'] = message\n response = jsonify(payload)\n response.status_code = code_status\n return response\n\n\ndef bad_request(status_code=400, message=''):\n return response_error(code_status=status_code, message=message)\n\n\ndef sanitize_api_error(error) -> str:\n \"\"\"\n Convert technical API errors to user-friendly messages.\n Works with both Exception objects and error message strings.\n \"\"\"\n error_str = str(error).lower()\n if \"503\" in error_str or \"unavailable\" in error_str or \"high demand\" in error_str:\n return \"The AI service is temporarily unavailable due to high demand. Please try again in a moment.\"\n if \"429\" in error_str or \"rate limit\" in error_str or \"quota\" in error_str:\n return \"Rate limit exceeded. Please wait a moment before trying again.\"\n if \"401\" in error_str or \"unauthorized\" in error_str or \"invalid api key\" in error_str:\n return \"Authentication error. Please check your API configuration.\"\n if \"timeout\" in error_str or \"timed out\" in error_str:\n return \"The request timed out. Please try again.\"\n if \"connection\" in error_str or \"network\" in error_str:\n return \"Network error. Please check your connection and try again.\"\n original = str(error)\n if len(original) > 200 or \"{\" in original or \"traceback\" in error_str:\n return \"An error occurred while processing your request. Please try again later.\"\n return original\n" }, { "path": "application/llm/__init__.py", "content": "" }, { "path": "application/llm/anthropic.py", "content": "import base64\nimport logging\n\nfrom anthropic import AI_PROMPT, Anthropic, HUMAN_PROMPT\n\nfrom application.core.settings import settings\nfrom application.llm.base import BaseLLM\nfrom application.storage.storage_creator import StorageCreator\n\nlogger = logging.getLogger(__name__)\n\n\nclass AnthropicLLM(BaseLLM):\n\n def __init__(self, api_key=None, user_api_key=None, base_url=None, *args, **kwargs):\n\n super().__init__(*args, **kwargs)\n self.api_key = api_key or settings.ANTHROPIC_API_KEY or settings.API_KEY\n self.user_api_key = user_api_key\n\n # Use custom base_url if provided\n if base_url:\n self.anthropic = Anthropic(api_key=self.api_key, base_url=base_url)\n else:\n self.anthropic = Anthropic(api_key=self.api_key)\n\n self.HUMAN_PROMPT = HUMAN_PROMPT\n self.AI_PROMPT = AI_PROMPT\n self.storage = StorageCreator.get_storage()\n\n def _raw_gen(\n self,\n baseself,\n model,\n messages,\n stream=False,\n tools=None,\n max_tokens=300,\n **kwargs,\n ):\n context = messages[0][\"content\"]\n user_question = messages[-1][\"content\"]\n prompt = f\"### Context \\n {context} \\n ### Question \\n {user_question}\"\n if stream:\n return self.gen_stream(model, prompt, stream, max_tokens, **kwargs)\n completion = self.anthropic.completions.create(\n model=model,\n max_tokens_to_sample=max_tokens,\n stream=stream,\n prompt=f\"{self.HUMAN_PROMPT} {prompt}{self.AI_PROMPT}\",\n )\n return completion.completion\n\n def _raw_gen_stream(\n self,\n baseself,\n model,\n messages,\n stream=True,\n tools=None,\n max_tokens=300,\n **kwargs,\n ):\n context = messages[0][\"content\"]\n user_question = messages[-1][\"content\"]\n prompt = f\"### Context \\n {context} \\n ### Question \\n {user_question}\"\n stream_response = self.anthropic.completions.create(\n model=model,\n prompt=f\"{self.HUMAN_PROMPT} {prompt}{self.AI_PROMPT}\",\n max_tokens_to_sample=max_tokens,\n stream=True,\n )\n\n try:\n for completion in stream_response:\n yield completion.completion\n finally:\n if hasattr(stream_response, \"close\"):\n stream_response.close()\n\n def get_supported_attachment_types(self):\n \"\"\"\n Return a list of MIME types supported by Anthropic Claude for file uploads.\n Claude supports images but not PDFs natively.\n PDFs are synthetically supported via PDF-to-image conversion in the handler.\n\n Returns:\n list: List of supported MIME types\n \"\"\"\n return [\n \"image/png\",\n \"image/jpeg\",\n \"image/jpg\",\n \"image/webp\",\n \"image/gif\",\n ]\n\n def prepare_messages_with_attachments(self, messages, attachments=None):\n \"\"\"\n Process attachments for Anthropic Claude API.\n Formats images using Claude's vision message format.\n\n Args:\n messages (list): List of message dictionaries.\n attachments (list): List of attachment dictionaries with content and metadata.\n\n Returns:\n list: Messages formatted with image content for Claude API.\n \"\"\"\n if not attachments:\n return messages\n\n prepared_messages = messages.copy()\n\n # Find the last user message to attach images to\n user_message_index = None\n for i in range(len(prepared_messages) - 1, -1, -1):\n if prepared_messages[i].get(\"role\") == \"user\":\n user_message_index = i\n break\n\n if user_message_index is None:\n user_message = {\"role\": \"user\", \"content\": []}\n prepared_messages.append(user_message)\n user_message_index = len(prepared_messages) - 1\n\n # Convert content to list format if it's a string\n if isinstance(prepared_messages[user_message_index].get(\"content\"), str):\n text_content = prepared_messages[user_message_index][\"content\"]\n prepared_messages[user_message_index][\"content\"] = [\n {\"type\": \"text\", \"text\": text_content}\n ]\n elif not isinstance(prepared_messages[user_message_index].get(\"content\"), list):\n prepared_messages[user_message_index][\"content\"] = []\n\n for attachment in attachments:\n mime_type = attachment.get(\"mime_type\")\n\n if mime_type and mime_type.startswith(\"image/\"):\n try:\n # Check if this is a pre-converted image (from PDF-to-image conversion)\n # These have 'data' key with base64 already\n if \"data\" in attachment:\n base64_image = attachment[\"data\"]\n else:\n base64_image = self._get_base64_image(attachment)\n\n # Claude uses a specific format for images\n prepared_messages[user_message_index][\"content\"].append(\n {\n \"type\": \"image\",\n \"source\": {\n \"type\": \"base64\",\n \"media_type\": mime_type,\n \"data\": base64_image,\n },\n }\n )\n\n except Exception as e:\n logger.error(\n f\"Error processing image attachment: {e}\", exc_info=True\n )\n if \"content\" in attachment:\n prepared_messages[user_message_index][\"content\"].append(\n {\n \"type\": \"text\",\n \"text\": f\"[Image could not be processed: {attachment.get('path', 'unknown')}]\",\n }\n )\n\n return prepared_messages\n\n def _get_base64_image(self, attachment):\n \"\"\"\n Convert an image file to base64 encoding.\n\n Args:\n attachment (dict): Attachment dictionary with path and metadata.\n\n Returns:\n str: Base64-encoded image data.\n \"\"\"\n file_path = attachment.get(\"path\")\n if not file_path:\n raise ValueError(\"No file path provided in attachment\")\n try:\n with self.storage.get_file(file_path) as image_file:\n return base64.b64encode(image_file.read()).decode(\"utf-8\")\n except FileNotFoundError:\n raise FileNotFoundError(f\"File not found: {file_path}\")\n" }, { "path": "application/llm/base.py", "content": "import logging\nfrom abc import ABC, abstractmethod\n\nfrom application.cache import gen_cache, stream_cache\n\nfrom application.core.settings import settings\nfrom application.usage import gen_token_usage, stream_token_usage\n\nlogger = logging.getLogger(__name__)\n\n\nclass BaseLLM(ABC):\n def __init__(\n self,\n decoded_token=None,\n agent_id=None,\n model_id=None,\n base_url=None,\n ):\n self.decoded_token = decoded_token\n self.agent_id = str(agent_id) if agent_id else None\n self.model_id = model_id\n self.base_url = base_url\n self.token_usage = {\"prompt_tokens\": 0, \"generated_tokens\": 0}\n self._fallback_llm = None\n self._fallback_sequence_index = 0\n\n @property\n def fallback_llm(self):\n \"\"\"Lazy-loaded fallback LLM from FALLBACK_* settings.\"\"\"\n if self._fallback_llm is None and settings.FALLBACK_LLM_PROVIDER:\n try:\n from application.llm.llm_creator import LLMCreator\n\n self._fallback_llm = LLMCreator.create_llm(\n settings.FALLBACK_LLM_PROVIDER,\n api_key=settings.FALLBACK_LLM_API_KEY or settings.API_KEY,\n user_api_key=getattr(self, \"user_api_key\", None),\n decoded_token=self.decoded_token,\n model_id=settings.FALLBACK_LLM_NAME,\n agent_id=self.agent_id,\n )\n logger.info(\n f\"Fallback LLM initialized: {settings.FALLBACK_LLM_PROVIDER}/{settings.FALLBACK_LLM_NAME}\"\n )\n except Exception as e:\n logger.error(\n f\"Failed to initialize fallback LLM: {str(e)}\", exc_info=True\n )\n return self._fallback_llm\n\n @staticmethod\n def _remove_null_values(args_dict):\n if not isinstance(args_dict, dict):\n return args_dict\n return {k: v for k, v in args_dict.items() if v is not None}\n\n def _execute_with_fallback(\n self, method_name: str, decorators: list, *args, **kwargs\n ):\n \"\"\"\n Execute method with fallback support.\n\n Args:\n method_name: Name of the raw method ('_raw_gen' or '_raw_gen_stream')\n decorators: List of decorators to apply\n *args: Positional arguments\n **kwargs: Keyword arguments\n \"\"\"\n\n def decorated_method():\n method = getattr(self, method_name)\n for decorator in decorators:\n method = decorator(method)\n return method(self, *args, **kwargs)\n\n try:\n return decorated_method()\n except Exception as e:\n if not self.fallback_llm:\n logger.error(f\"Primary LLM failed and no fallback configured: {str(e)}\")\n raise\n logger.warning(\n f\"Primary LLM failed. Falling back to {settings.FALLBACK_LLM_PROVIDER}/{settings.FALLBACK_LLM_NAME}. Error: {str(e)}\"\n )\n\n fallback_method = getattr(\n self.fallback_llm, method_name.replace(\"_raw_\", \"\")\n )\n return fallback_method(*args, **kwargs)\n\n def gen(self, model, messages, stream=False, tools=None, *args, **kwargs):\n decorators = [gen_token_usage, gen_cache]\n return self._execute_with_fallback(\n \"_raw_gen\",\n decorators,\n model=model,\n messages=messages,\n stream=stream,\n tools=tools,\n *args,\n **kwargs,\n )\n\n def gen_stream(self, model, messages, stream=True, tools=None, *args, **kwargs):\n decorators = [stream_cache, stream_token_usage]\n return self._execute_with_fallback(\n \"_raw_gen_stream\",\n decorators,\n model=model,\n messages=messages,\n stream=stream,\n tools=tools,\n *args,\n **kwargs,\n )\n\n @abstractmethod\n def _raw_gen(self, model, messages, stream, tools, *args, **kwargs):\n pass\n\n @abstractmethod\n def _raw_gen_stream(self, model, messages, stream, *args, **kwargs):\n pass\n\n def supports_tools(self):\n return hasattr(self, \"_supports_tools\") and callable(\n getattr(self, \"_supports_tools\")\n )\n\n def _supports_tools(self):\n raise NotImplementedError(\"Subclass must implement _supports_tools method\")\n\n def supports_structured_output(self):\n \"\"\"Check if the LLM supports structured output/JSON schema enforcement\"\"\"\n return hasattr(self, \"_supports_structured_output\") and callable(\n getattr(self, \"_supports_structured_output\")\n )\n\n def _supports_structured_output(self):\n return False\n\n def prepare_structured_output_format(self, json_schema):\n \"\"\"Prepare structured output format specific to the LLM provider\"\"\"\n _ = json_schema\n return None\n\n def get_supported_attachment_types(self):\n \"\"\"\n Return a list of MIME types supported by this LLM for file uploads.\n\n Returns:\n list: List of supported MIME types\n \"\"\"\n return []\n" }, { "path": "application/llm/docsgpt_provider.py", "content": "from application.core.settings import settings\nfrom application.llm.openai import OpenAILLM\n\nDOCSGPT_API_KEY = \"sk-docsgpt-public\"\nDOCSGPT_BASE_URL = \"https://oai.arc53.com\"\nDOCSGPT_MODEL = \"docsgpt\"\n\nclass DocsGPTAPILLM(OpenAILLM):\n def __init__(self, api_key=None, user_api_key=None, base_url=None, *args, **kwargs):\n super().__init__(\n api_key=DOCSGPT_API_KEY,\n user_api_key=user_api_key,\n base_url=DOCSGPT_BASE_URL,\n *args,\n **kwargs,\n )\n\n def _raw_gen(\n self,\n baseself,\n model,\n messages,\n stream=False,\n tools=None,\n engine=settings.AZURE_DEPLOYMENT_NAME,\n response_format=None,\n **kwargs,\n ):\n return super()._raw_gen(\n baseself,\n DOCSGPT_MODEL,\n messages,\n stream=stream,\n tools=tools,\n engine=engine,\n response_format=response_format,\n **kwargs,\n )\n\n def _raw_gen_stream(\n self,\n baseself,\n model,\n messages,\n stream=True,\n tools=None,\n engine=settings.AZURE_DEPLOYMENT_NAME,\n response_format=None,\n **kwargs,\n ):\n return super()._raw_gen_stream(\n baseself,\n DOCSGPT_MODEL,\n messages,\n stream=stream,\n tools=tools,\n engine=engine,\n response_format=response_format,\n **kwargs,\n )\n" }, { "path": "application/llm/google_ai.py", "content": "import logging\n\nfrom google import genai\nfrom google.genai import types\n\nfrom application.core.settings import settings\n\nfrom application.llm.base import BaseLLM\nfrom application.storage.storage_creator import StorageCreator\n\n\nclass GoogleLLM(BaseLLM):\n def __init__(\n self, api_key=None, user_api_key=None, decoded_token=None, *args, **kwargs\n ):\n super().__init__(decoded_token=decoded_token, *args, **kwargs)\n self.api_key = api_key or settings.GOOGLE_API_KEY or settings.API_KEY\n self.user_api_key = user_api_key\n\n self.client = genai.Client(api_key=self.api_key)\n self.storage = StorageCreator.get_storage()\n\n def get_supported_attachment_types(self):\n \"\"\"\n Return a list of MIME types supported by Google Gemini for file uploads.\n\n Returns:\n list: List of supported MIME types\n \"\"\"\n return [\n \"application/pdf\",\n \"image/png\",\n \"image/jpeg\",\n \"image/jpg\",\n \"image/webp\",\n \"image/gif\",\n \"application/pdf\",\n \"image/png\",\n \"image/jpeg\",\n \"image/jpg\",\n \"image/webp\",\n \"image/gif\",\n ]\n\n def prepare_messages_with_attachments(self, messages, attachments=None):\n \"\"\"\n Process attachments using Google AI's file API for more efficient handling.\n\n Args:\n messages (list): List of message dictionaries.\n attachments (list): List of attachment dictionaries with content and metadata.\n\n Returns:\n list: Messages formatted with file references for Google AI API.\n \"\"\"\n if not attachments:\n return messages\n prepared_messages = messages.copy()\n\n # Find the user message to attach files to the last one\n\n user_message_index = None\n for i in range(len(prepared_messages) - 1, -1, -1):\n if prepared_messages[i].get(\"role\") == \"user\":\n user_message_index = i\n break\n if user_message_index is None:\n user_message = {\"role\": \"user\", \"content\": []}\n prepared_messages.append(user_message)\n user_message_index = len(prepared_messages) - 1\n if isinstance(prepared_messages[user_message_index].get(\"content\"), str):\n text_content = prepared_messages[user_message_index][\"content\"]\n prepared_messages[user_message_index][\"content\"] = [\n {\"type\": \"text\", \"text\": text_content}\n ]\n elif not isinstance(prepared_messages[user_message_index].get(\"content\"), list):\n prepared_messages[user_message_index][\"content\"] = []\n files = []\n for attachment in attachments:\n mime_type = attachment.get(\"mime_type\")\n\n if mime_type in self.get_supported_attachment_types():\n try:\n file_uri = self._upload_file_to_google(attachment)\n logging.info(\n f\"GoogleLLM: Successfully uploaded file, got URI: {file_uri}\"\n )\n files.append({\"file_uri\": file_uri, \"mime_type\": mime_type})\n except Exception as e:\n logging.error(\n f\"GoogleLLM: Error uploading file: {e}\", exc_info=True\n )\n if \"content\" in attachment:\n prepared_messages[user_message_index][\"content\"].append(\n {\n \"type\": \"text\",\n \"text\": f\"[File could not be processed: {attachment.get('path', 'unknown')}]\",\n }\n )\n if files:\n logging.info(f\"GoogleLLM: Adding {len(files)} files to message\")\n prepared_messages[user_message_index][\"content\"].append({\"files\": files})\n return prepared_messages\n\n def _upload_file_to_google(self, attachment):\n \"\"\"\n Upload a file to Google AI and return the file URI.\n\n Args:\n attachment (dict): Attachment dictionary with path and metadata.\n\n Returns:\n str: Google AI file URI for the uploaded file.\n \"\"\"\n if \"google_file_uri\" in attachment:\n return attachment[\"google_file_uri\"]\n file_path = attachment.get(\"path\")\n if not file_path:\n raise ValueError(\"No file path provided in attachment\")\n if not self.storage.file_exists(file_path):\n raise FileNotFoundError(f\"File not found: {file_path}\")\n try:\n file_uri = self.storage.process_file(\n file_path,\n lambda local_path, **kwargs: self.client.files.upload(\n file=local_path\n ).uri,\n )\n\n from application.core.mongo_db import MongoDB\n\n mongo = MongoDB.get_client()\n db = mongo[settings.MONGO_DB_NAME]\n attachments_collection = db[\"attachments\"]\n if \"_id\" in attachment:\n attachments_collection.update_one(\n {\"_id\": attachment[\"_id\"]}, {\"$set\": {\"google_file_uri\": file_uri}}\n )\n return file_uri\n except Exception as e:\n logging.error(f\"Error uploading file to Google AI: {e}\", exc_info=True)\n raise\n\n def _clean_messages_google(self, messages):\n \"\"\"\n Convert OpenAI format messages to Google AI format and collect system prompts.\n\n Returns:\n tuple[list[types.Content], Optional[str]]: cleaned messages and optional\n combined system instruction.\n \"\"\"\n cleaned_messages = []\n system_instructions = []\n\n def _extract_system_text(content):\n if isinstance(content, str):\n return content\n if isinstance(content, list):\n parts = []\n for item in content:\n if (\n isinstance(item, dict)\n and \"text\" in item\n and item[\"text\"] is not None\n ):\n parts.append(item[\"text\"])\n return \"\\n\".join(parts)\n return \"\"\n\n for message in messages:\n role = message.get(\"role\")\n content = message.get(\"content\")\n\n # Gemini only accepts user/model in the contents list.\n if role == \"system\":\n sys_text = _extract_system_text(content)\n if sys_text:\n system_instructions.append(sys_text)\n continue\n\n if role == \"assistant\":\n role = \"model\"\n elif role == \"tool\":\n role = \"model\"\n parts = []\n if role and content is not None:\n if isinstance(content, str):\n parts = [types.Part.from_text(text=content)]\n elif isinstance(content, list):\n for item in content:\n if \"text\" in item:\n parts.append(types.Part.from_text(text=item[\"text\"]))\n elif \"function_call\" in item:\n # Remove null values from args to avoid API errors\n\n cleaned_args = self._remove_null_values(\n item[\"function_call\"][\"args\"]\n )\n # Create function call part with thought_signature if present\n # For Gemini 3 models, we need to include thought_signature\n if \"thought_signature\" in item:\n # Use Part constructor with functionCall and thoughtSignature\n parts.append(\n types.Part(\n functionCall=types.FunctionCall(\n name=item[\"function_call\"][\"name\"],\n args=cleaned_args,\n ),\n thoughtSignature=item[\"thought_signature\"],\n )\n )\n else:\n # Use helper method when no thought_signature\n parts.append(\n types.Part.from_function_call(\n name=item[\"function_call\"][\"name\"],\n args=cleaned_args,\n )\n )\n elif \"function_response\" in item:\n parts.append(\n types.Part.from_function_response(\n name=item[\"function_response\"][\"name\"],\n response=item[\"function_response\"][\"response\"],\n )\n )\n elif \"files\" in item:\n for file_data in item[\"files\"]:\n parts.append(\n types.Part.from_uri(\n file_uri=file_data[\"file_uri\"],\n mime_type=file_data[\"mime_type\"],\n )\n )\n else:\n raise ValueError(\n f\"Unexpected content dictionary format:{item}\"\n )\n else:\n raise ValueError(f\"Unexpected content type: {type(content)}\")\n if parts:\n cleaned_messages.append(types.Content(role=role, parts=parts))\n system_instruction = (\n \"\\n\\n\".join(system_instructions) if system_instructions else None\n )\n return cleaned_messages, system_instruction\n\n def _clean_schema(self, schema_obj):\n \"\"\"\n Recursively remove unsupported fields from schema objects\n and validate required properties.\n \"\"\"\n if not isinstance(schema_obj, dict):\n return schema_obj\n allowed_fields = {\n \"type\",\n \"description\",\n \"items\",\n \"properties\",\n \"required\",\n \"enum\",\n \"pattern\",\n \"minimum\",\n \"maximum\",\n \"nullable\",\n \"default\",\n }\n\n cleaned = {}\n for key, value in schema_obj.items():\n if key not in allowed_fields:\n continue\n elif key == \"type\" and isinstance(value, str):\n cleaned[key] = value.upper()\n elif isinstance(value, dict):\n cleaned[key] = self._clean_schema(value)\n elif isinstance(value, list):\n cleaned[key] = [self._clean_schema(item) for item in value]\n else:\n cleaned[key] = value\n # Validate that required properties actually exist in properties\n\n if \"required\" in cleaned and \"properties\" in cleaned:\n valid_required = []\n properties_keys = set(cleaned[\"properties\"].keys())\n for required_prop in cleaned[\"required\"]:\n if required_prop in properties_keys:\n valid_required.append(required_prop)\n if valid_required:\n cleaned[\"required\"] = valid_required\n else:\n cleaned.pop(\"required\", None)\n elif \"required\" in cleaned and \"properties\" not in cleaned:\n cleaned.pop(\"required\", None)\n return cleaned\n\n def _clean_tools_format(self, tools_list):\n \"\"\"Convert OpenAI format tools to Google AI format.\"\"\"\n genai_tools = []\n for tool_data in tools_list:\n if tool_data[\"type\"] == \"function\":\n function = tool_data[\"function\"]\n parameters = function[\"parameters\"]\n properties = parameters.get(\"properties\", {})\n\n if properties:\n cleaned_properties = {}\n for k, v in properties.items():\n cleaned_properties[k] = self._clean_schema(v)\n genai_function = dict(\n name=function[\"name\"],\n description=function[\"description\"],\n parameters={\n \"type\": \"OBJECT\",\n \"properties\": cleaned_properties,\n \"required\": (\n parameters[\"required\"]\n if \"required\" in parameters\n else []\n ),\n },\n )\n else:\n genai_function = dict(\n name=function[\"name\"],\n description=function[\"description\"],\n )\n genai_tool = types.Tool(function_declarations=[genai_function])\n genai_tools.append(genai_tool)\n return genai_tools\n\n def _extract_preview_from_message(self, message):\n \"\"\"Get a short, human-readable preview from the last message.\"\"\"\n try:\n if hasattr(message, \"parts\"):\n for part in reversed(message.parts):\n if getattr(part, \"text\", None):\n return part.text\n function_call = getattr(part, \"function_call\", None)\n if function_call:\n name = getattr(function_call, \"name\", \"\") or \"function_call\"\n return f\"function_call:{name}\"\n function_response = getattr(part, \"function_response\", None)\n if function_response:\n name = (\n getattr(function_response, \"name\", \"\")\n or \"function_response\"\n )\n return f\"function_response:{name}\"\n if isinstance(message, dict):\n content = message.get(\"content\")\n if isinstance(content, str):\n return content\n if isinstance(content, list):\n for item in reversed(content):\n if isinstance(item, str):\n return item\n if isinstance(item, dict):\n if item.get(\"text\"):\n return item[\"text\"]\n if item.get(\"function_call\"):\n fn = item[\"function_call\"]\n if isinstance(fn, dict):\n name = fn.get(\"name\") or \"function_call\"\n return f\"function_call:{name}\"\n return \"function_call\"\n if item.get(\"function_response\"):\n resp = item[\"function_response\"]\n if isinstance(resp, dict):\n name = resp.get(\"name\") or \"function_response\"\n return f\"function_response:{name}\"\n return \"function_response\"\n if \"text\" in message and isinstance(message[\"text\"], str):\n return message[\"text\"]\n except Exception:\n pass\n return str(message)\n\n def _summarize_messages_for_log(self, messages, preview_chars=20):\n \"\"\"Return a compact summary for logging to avoid huge payloads.\"\"\"\n message_count = len(messages) if messages else 0\n last_preview = \"\"\n if messages:\n last_preview = self._extract_preview_from_message(messages[-1]) or \"\"\n last_preview = str(last_preview).replace(\"\\n\", \" \")\n if len(last_preview) > preview_chars:\n last_preview = f\"{last_preview[:preview_chars]}...\"\n return f\"count={message_count}, last='{last_preview}'\"\n\n @staticmethod\n def _get_text_value(part):\n \"\"\"Get text from both SDK objects and dict-shaped test doubles.\"\"\"\n if isinstance(part, dict):\n value = part.get(\"text\")\n return value if isinstance(value, str) else \"\"\n value = getattr(part, \"text\", None)\n return value if isinstance(value, str) else \"\"\n\n @staticmethod\n def _is_thought_part(part):\n \"\"\"Detect Gemini thinking parts when available.\"\"\"\n if isinstance(part, dict):\n return bool(part.get(\"thought\"))\n return bool(getattr(part, \"thought\", False))\n\n def _raw_gen(\n self,\n baseself,\n model,\n messages,\n stream=False,\n tools=None,\n formatting=\"openai\",\n response_schema=None,\n **kwargs,\n ):\n \"\"\"Generate content using Google AI API without streaming.\"\"\"\n client = genai.Client(api_key=self.api_key)\n system_instruction = None\n if formatting == \"openai\":\n messages, system_instruction = self._clean_messages_google(messages)\n config = types.GenerateContentConfig()\n if system_instruction:\n config.system_instruction = system_instruction\n if tools:\n cleaned_tools = self._clean_tools_format(tools)\n config.tools = cleaned_tools\n # Add response schema for structured output if provided\n\n if response_schema:\n config.response_schema = response_schema\n config.response_mime_type = \"application/json\"\n response = client.models.generate_content(\n model=model,\n contents=messages,\n config=config,\n )\n\n if tools:\n return response\n else:\n return response.text\n\n def _raw_gen_stream(\n self,\n baseself,\n model,\n messages,\n stream=True,\n tools=None,\n formatting=\"openai\",\n response_schema=None,\n **kwargs,\n ):\n \"\"\"Generate content using Google AI API with streaming.\"\"\"\n client = genai.Client(api_key=self.api_key)\n system_instruction = None\n if formatting == \"openai\":\n messages, system_instruction = self._clean_messages_google(messages)\n config = types.GenerateContentConfig()\n if system_instruction:\n config.system_instruction = system_instruction\n if tools:\n cleaned_tools = self._clean_tools_format(tools)\n config.tools = cleaned_tools\n\n if response_schema:\n config.response_schema = response_schema\n config.response_mime_type = \"application/json\"\n # Check if we have both tools and file attachments\n\n has_attachments = False\n for message in messages:\n for part in message.parts:\n if hasattr(part, \"file_data\") and part.file_data is not None:\n has_attachments = True\n break\n if has_attachments:\n break\n messages_summary = self._summarize_messages_for_log(messages)\n logging.info(\n \"GoogleLLM: Starting stream generation. Model: %s, Messages: %s, Has attachments: %s\",\n model,\n messages_summary,\n has_attachments,\n )\n\n response = client.models.generate_content_stream(\n model=model,\n contents=messages,\n config=config,\n )\n\n try:\n for chunk in response:\n if hasattr(chunk, \"candidates\") and chunk.candidates:\n for candidate in chunk.candidates:\n if candidate.content and candidate.content.parts:\n for part in candidate.content.parts:\n if part.function_call:\n yield part\n continue\n\n part_text = self._get_text_value(part)\n if not part_text:\n continue\n\n if self._is_thought_part(part):\n yield {\"type\": \"thought\", \"thought\": part_text}\n else:\n yield part_text\n elif hasattr(chunk, \"text\"):\n chunk_text = self._get_text_value(chunk)\n if chunk_text:\n if self._is_thought_part(chunk):\n yield {\"type\": \"thought\", \"thought\": chunk_text}\n else:\n yield chunk_text\n except Exception as e:\n logging.error(f\"GoogleLLM: Stream error: {e}\", exc_info=True)\n raise\n finally:\n if hasattr(response, \"close\"):\n response.close()\n\n def _supports_tools(self):\n \"\"\"Return whether this LLM supports function calling.\"\"\"\n return True\n\n def _supports_structured_output(self):\n \"\"\"Return whether this LLM supports structured JSON output.\"\"\"\n return True\n\n def prepare_structured_output_format(self, json_schema):\n \"\"\"Convert JSON schema to Google AI structured output format.\"\"\"\n if not json_schema:\n return None\n type_map = {\n \"object\": \"OBJECT\",\n \"array\": \"ARRAY\",\n \"string\": \"STRING\",\n \"integer\": \"INTEGER\",\n \"number\": \"NUMBER\",\n \"boolean\": \"BOOLEAN\",\n }\n\n def convert(schema):\n if not isinstance(schema, dict):\n return schema\n result = {}\n schema_type = schema.get(\"type\")\n if schema_type:\n result[\"type\"] = type_map.get(schema_type.lower(), schema_type.upper())\n for key in [\n \"description\",\n \"nullable\",\n \"enum\",\n \"minItems\",\n \"maxItems\",\n \"required\",\n \"propertyOrdering\",\n ]:\n if key in schema:\n result[key] = schema[key]\n if \"format\" in schema:\n format_value = schema[\"format\"]\n if schema_type == \"string\":\n if format_value == \"date\":\n result[\"format\"] = \"date-time\"\n elif format_value in [\"enum\", \"date-time\"]:\n result[\"format\"] = format_value\n else:\n result[\"format\"] = format_value\n if \"properties\" in schema:\n result[\"properties\"] = {\n k: convert(v) for k, v in schema[\"properties\"].items()\n }\n if \"propertyOrdering\" not in result and result.get(\"type\") == \"OBJECT\":\n result[\"propertyOrdering\"] = list(result[\"properties\"].keys())\n if \"items\" in schema:\n result[\"items\"] = convert(schema[\"items\"])\n for field in [\"anyOf\", \"oneOf\", \"allOf\"]:\n if field in schema:\n result[field] = [convert(s) for s in schema[field]]\n return result\n\n try:\n return convert(json_schema)\n except Exception as e:\n logging.error(\n f\"Error preparing structured output format for Google: {e}\",\n exc_info=True,\n )\n return None\n" }, { "path": "application/llm/groq.py", "content": "from application.core.settings import settings\nfrom application.llm.openai import OpenAILLM\n\nGROQ_BASE_URL = \"https://api.groq.com/openai/v1\"\n\n\nclass GroqLLM(OpenAILLM):\n def __init__(self, api_key=None, user_api_key=None, base_url=None, *args, **kwargs):\n super().__init__(\n api_key=api_key or settings.GROQ_API_KEY or settings.API_KEY,\n user_api_key=user_api_key,\n base_url=base_url or GROQ_BASE_URL,\n *args,\n **kwargs,\n )\n" }, { "path": "application/llm/handlers/__init__.py", "content": "" }, { "path": "application/llm/handlers/base.py", "content": "import logging\nimport uuid\nfrom abc import ABC, abstractmethod\nfrom dataclasses import dataclass\nfrom typing import Any, Dict, Generator, List, Optional, Union\n\nfrom application.logging import build_stack_data\n\nlogger = logging.getLogger(__name__)\n\n\n@dataclass\nclass ToolCall:\n \"\"\"Represents a tool/function call from the LLM.\"\"\"\n\n id: str\n name: str\n arguments: Union[str, Dict]\n index: Optional[int] = None\n thought_signature: Optional[str] = None\n\n @classmethod\n def from_dict(cls, data: Dict) -> \"ToolCall\":\n \"\"\"Create ToolCall from dictionary.\"\"\"\n return cls(\n id=data.get(\"id\", \"\"),\n name=data.get(\"name\", \"\"),\n arguments=data.get(\"arguments\", {}),\n index=data.get(\"index\"),\n )\n\n\n@dataclass\nclass LLMResponse:\n \"\"\"Represents a response from the LLM.\"\"\"\n\n content: str\n tool_calls: List[ToolCall]\n finish_reason: str\n raw_response: Any\n\n @property\n def requires_tool_call(self) -> bool:\n \"\"\"Check if the response requires tool calls.\"\"\"\n return bool(self.tool_calls) and self.finish_reason == \"tool_calls\"\n\n\nclass LLMHandler(ABC):\n \"\"\"Abstract base class for LLM handlers.\"\"\"\n\n def __init__(self):\n self.llm_calls = []\n self.tool_calls = []\n\n @abstractmethod\n def parse_response(self, response: Any) -> LLMResponse:\n \"\"\"Parse raw LLM response into standardized format.\"\"\"\n pass\n\n @abstractmethod\n def create_tool_message(self, tool_call: ToolCall, result: Any) -> Dict:\n \"\"\"Create a tool result message for the conversation history.\"\"\"\n pass\n\n @abstractmethod\n def _iterate_stream(self, response: Any) -> Generator:\n \"\"\"Iterate through streaming response chunks.\"\"\"\n pass\n\n def process_message_flow(\n self,\n agent,\n initial_response,\n tools_dict: Dict,\n messages: List[Dict],\n attachments: Optional[List] = None,\n stream: bool = False,\n ) -> Union[str, Generator]:\n \"\"\"\n Main orchestration method for processing LLM message flow.\n\n Args:\n agent: The agent instance\n initial_response: Initial LLM response\n tools_dict: Dictionary of available tools\n messages: Conversation history\n attachments: Optional attachments\n stream: Whether to use streaming\n\n Returns:\n Final response or generator for streaming\n \"\"\"\n messages = self.prepare_messages(agent, messages, attachments)\n\n if stream:\n return self.handle_streaming(agent, initial_response, tools_dict, messages)\n else:\n return self.handle_non_streaming(\n agent, initial_response, tools_dict, messages\n )\n\n def prepare_messages(\n self, agent, messages: List[Dict], attachments: Optional[List] = None\n ) -> List[Dict]:\n \"\"\"\n Prepare messages with attachments and provider-specific formatting.\n\n\n Args:\n agent: The agent instance\n messages: Original messages\n attachments: List of attachments\n\n Returns:\n Prepared messages list\n \"\"\"\n if not attachments:\n return messages\n logger.info(f\"Preparing messages with {len(attachments)} attachments\")\n supported_types = agent.llm.get_supported_attachment_types()\n\n # Check if provider supports images but not PDF (synthetic PDF support)\n supports_images = any(t.startswith(\"image/\") for t in supported_types)\n supports_pdf = \"application/pdf\" in supported_types\n\n # Process attachments, converting PDFs to images if needed\n processed_attachments = []\n for attachment in attachments:\n mime_type = attachment.get(\"mime_type\")\n\n # Synthetic PDF support: convert PDF to images if LLM supports images but not PDF\n if mime_type == \"application/pdf\" and supports_images and not supports_pdf:\n logger.info(\n f\"Converting PDF to images for synthetic PDF support: {attachment.get('path', 'unknown')}\"\n )\n try:\n converted_images = self._convert_pdf_to_images(attachment)\n processed_attachments.extend(converted_images)\n logger.info(\n f\"Converted PDF to {len(converted_images)} images\"\n )\n except Exception as e:\n logger.error(\n f\"Failed to convert PDF to images, falling back to text: {e}\"\n )\n # Fall back to treating as unsupported (text extraction)\n processed_attachments.append(attachment)\n else:\n processed_attachments.append(attachment)\n\n supported_attachments = [\n a for a in processed_attachments if a.get(\"mime_type\") in supported_types\n ]\n unsupported_attachments = [\n a for a in processed_attachments if a.get(\"mime_type\") not in supported_types\n ]\n\n # Process supported attachments with the LLM's custom method\n\n if supported_attachments:\n logger.info(\n f\"Processing {len(supported_attachments)} supported attachments\"\n )\n messages = agent.llm.prepare_messages_with_attachments(\n messages, supported_attachments\n )\n # Process unsupported attachments with default method\n\n if unsupported_attachments:\n logger.info(\n f\"Processing {len(unsupported_attachments)} unsupported attachments\"\n )\n messages = self._append_unsupported_attachments(\n messages, unsupported_attachments\n )\n return messages\n\n def _convert_pdf_to_images(self, attachment: Dict) -> List[Dict]:\n \"\"\"\n Convert a PDF attachment to a list of image attachments.\n\n This enables synthetic PDF support for LLMs that support images but not PDFs.\n\n Args:\n attachment: PDF attachment dictionary with 'path' and optional 'content'\n\n Returns:\n List of image attachment dictionaries with 'data', 'mime_type', and 'page'\n \"\"\"\n from application.utils import convert_pdf_to_images\n from application.storage.storage_creator import StorageCreator\n\n file_path = attachment.get(\"path\")\n if not file_path:\n raise ValueError(\"No file path provided in PDF attachment\")\n\n storage = StorageCreator.get_storage()\n\n # Convert PDF to images\n images_data = convert_pdf_to_images(\n file_path=file_path,\n storage=storage,\n max_pages=20,\n dpi=150,\n )\n\n return images_data\n\n def _append_unsupported_attachments(\n self, messages: List[Dict], attachments: List[Dict]\n ) -> List[Dict]:\n \"\"\"\n Default method to append unsupported attachment content to system prompt.\n\n Args:\n messages: Current messages\n attachments: List of unsupported attachments\n\n Returns:\n Updated messages list\n \"\"\"\n prepared_messages = messages.copy()\n attachment_texts = []\n\n for attachment in attachments:\n logger.info(f\"Adding attachment {attachment.get('id')} to context\")\n if \"content\" in attachment:\n attachment_texts.append(\n f\"Attached file content:\\n\\n{attachment['content']}\"\n )\n if attachment_texts:\n combined_text = \"\\n\\n\".join(attachment_texts)\n\n system_msg = next(\n (msg for msg in prepared_messages if msg.get(\"role\") == \"system\"),\n {\"role\": \"system\", \"content\": \"\"},\n )\n\n if system_msg not in prepared_messages:\n prepared_messages.insert(0, system_msg)\n system_msg[\"content\"] += f\"\\n\\n{combined_text}\"\n return prepared_messages\n\n def _prune_messages_minimal(self, messages: List[Dict]) -> Optional[List[Dict]]:\n \"\"\"\n Build a minimal context: system prompt + latest user message only.\n Drops all tool/function messages to shrink context aggressively.\n \"\"\"\n system_message = next((m for m in messages if m.get(\"role\") == \"system\"), None)\n if not system_message:\n logger.warning(\"Cannot prune messages minimally: missing system message.\")\n return None\n last_non_system = None\n for m in reversed(messages):\n if m.get(\"role\") == \"user\":\n last_non_system = m\n break\n if not last_non_system and m.get(\"role\") not in (\"system\", None):\n last_non_system = m\n if not last_non_system:\n logger.warning(\"Cannot prune messages minimally: missing user/assistant messages.\")\n return None\n logger.info(\"Pruning context to system + latest user/assistant message to proceed.\")\n return [system_message, last_non_system]\n\n def _extract_text_from_content(self, content: Any) -> str:\n \"\"\"\n Convert message content (str or list of parts) to plain text for compression.\n \"\"\"\n if isinstance(content, str):\n return content\n if isinstance(content, list):\n parts_text = []\n for item in content:\n if isinstance(item, dict):\n if \"text\" in item and item[\"text\"] is not None:\n parts_text.append(str(item[\"text\"]))\n elif \"function_call\" in item or \"function_response\" in item:\n # Keep serialized function calls/responses so the compressor sees actions\n parts_text.append(str(item))\n elif \"files\" in item:\n parts_text.append(str(item))\n return \"\\n\".join(parts_text)\n return \"\"\n\n def _build_conversation_from_messages(self, messages: List[Dict]) -> Optional[Dict]:\n \"\"\"\n Build a conversation-like dict from current messages so we can compress\n even when the conversation isn't persisted yet. Includes tool calls/results.\n \"\"\"\n queries = []\n current_prompt = None\n current_tool_calls = {}\n\n def _commit_query(response_text: str):\n nonlocal current_prompt, current_tool_calls\n if current_prompt is None and not response_text:\n return\n tool_calls_list = list(current_tool_calls.values())\n queries.append(\n {\n \"prompt\": current_prompt or \"\",\n \"response\": response_text,\n \"tool_calls\": tool_calls_list,\n }\n )\n current_prompt = None\n current_tool_calls = {}\n\n for message in messages:\n role = message.get(\"role\")\n content = message.get(\"content\")\n\n if role == \"user\":\n current_prompt = self._extract_text_from_content(content)\n\n elif role in {\"assistant\", \"model\"}:\n # If this assistant turn contains tool calls, collect them; otherwise commit a response.\n if isinstance(content, list):\n for item in content:\n if \"function_call\" in item:\n fc = item[\"function_call\"]\n call_id = fc.get(\"call_id\") or str(uuid.uuid4())\n current_tool_calls[call_id] = {\n \"tool_name\": \"unknown_tool\",\n \"action_name\": fc.get(\"name\"),\n \"arguments\": fc.get(\"args\"),\n \"result\": None,\n \"status\": \"called\",\n \"call_id\": call_id,\n }\n elif \"function_response\" in item:\n fr = item[\"function_response\"]\n call_id = fr.get(\"call_id\") or str(uuid.uuid4())\n current_tool_calls[call_id] = {\n \"tool_name\": \"unknown_tool\",\n \"action_name\": fr.get(\"name\"),\n \"arguments\": None,\n \"result\": fr.get(\"response\", {}).get(\"result\"),\n \"status\": \"completed\",\n \"call_id\": call_id,\n }\n # No direct assistant text here; continue to next message\n continue\n\n response_text = self._extract_text_from_content(content)\n _commit_query(response_text)\n\n elif role == \"tool\":\n # Attach tool outputs to the latest pending tool call if possible\n tool_text = self._extract_text_from_content(content)\n # Attempt to parse function_response style\n call_id = None\n if isinstance(content, list):\n for item in content:\n if \"function_response\" in item and item[\"function_response\"].get(\"call_id\"):\n call_id = item[\"function_response\"][\"call_id\"]\n break\n if call_id and call_id in current_tool_calls:\n current_tool_calls[call_id][\"result\"] = tool_text\n current_tool_calls[call_id][\"status\"] = \"completed\"\n elif queries:\n queries[-1].setdefault(\"tool_calls\", []).append(\n {\n \"tool_name\": \"unknown_tool\",\n \"action_name\": \"unknown_action\",\n \"arguments\": {},\n \"result\": tool_text,\n \"status\": \"completed\",\n }\n )\n\n # If there's an unfinished prompt with tool_calls but no response yet, commit it\n if current_prompt is not None or current_tool_calls:\n _commit_query(response_text=\"\")\n\n if not queries:\n return None\n\n return {\n \"queries\": queries,\n \"compression_metadata\": {\n \"is_compressed\": False,\n \"compression_points\": [],\n },\n }\n\n def _rebuild_messages_after_compression(\n self,\n messages: List[Dict],\n compressed_summary: Optional[str],\n recent_queries: List[Dict],\n include_current_execution: bool = False,\n include_tool_calls: bool = False,\n ) -> Optional[List[Dict]]:\n \"\"\"\n Rebuild the message list after compression so tool execution can continue.\n\n Delegates to MessageBuilder for the actual reconstruction.\n \"\"\"\n from application.api.answer.services.compression.message_builder import (\n MessageBuilder,\n )\n\n return MessageBuilder.rebuild_messages_after_compression(\n messages=messages,\n compressed_summary=compressed_summary,\n recent_queries=recent_queries,\n include_current_execution=include_current_execution,\n include_tool_calls=include_tool_calls,\n )\n\n def _perform_mid_execution_compression(\n self, agent, messages: List[Dict]\n ) -> tuple[bool, Optional[List[Dict]]]:\n \"\"\"\n Perform compression during tool execution and rebuild messages.\n\n Uses the new orchestrator for simplified compression.\n\n Args:\n agent: The agent instance\n messages: Current conversation messages\n\n Returns:\n (success: bool, rebuilt_messages: Optional[List[Dict]])\n \"\"\"\n try:\n from application.api.answer.services.compression import (\n CompressionOrchestrator,\n )\n from application.api.answer.services.conversation_service import (\n ConversationService,\n )\n\n conversation_service = ConversationService()\n orchestrator = CompressionOrchestrator(conversation_service)\n\n # Get conversation from database (may be None for new sessions)\n conversation = conversation_service.get_conversation(\n agent.conversation_id, agent.initial_user_id\n )\n\n if conversation:\n # Merge current in-flight messages (including tool calls)\n conversation_from_msgs = self._build_conversation_from_messages(messages)\n if conversation_from_msgs:\n conversation = conversation_from_msgs\n else:\n logger.warning(\n \"Could not load conversation for compression; attempting in-memory compression\"\n )\n return self._perform_in_memory_compression(agent, messages)\n\n # Use orchestrator to perform compression\n result = orchestrator.compress_mid_execution(\n conversation_id=agent.conversation_id,\n user_id=agent.initial_user_id,\n model_id=agent.model_id,\n decoded_token=getattr(agent, \"decoded_token\", {}),\n current_conversation=conversation,\n )\n\n if not result.success:\n logger.warning(f\"Mid-execution compression failed: {result.error}\")\n # Try minimal pruning as fallback\n pruned = self._prune_messages_minimal(messages)\n if pruned:\n agent.context_limit_reached = False\n agent.current_token_count = 0\n return True, pruned\n return False, None\n\n if not result.compression_performed:\n logger.warning(\"Compression not performed\")\n return False, None\n\n # Check if compression actually reduced tokens\n if result.metadata:\n if result.metadata.compressed_token_count >= result.metadata.original_token_count:\n logger.warning(\n \"Compression did not reduce token count; falling back to minimal pruning\"\n )\n pruned = self._prune_messages_minimal(messages)\n if pruned:\n agent.context_limit_reached = False\n agent.current_token_count = 0\n return True, pruned\n return False, None\n\n logger.info(\n f\"Mid-execution compression successful - ratio: {result.metadata.compression_ratio:.1f}x, \"\n f\"saved {result.metadata.original_token_count - result.metadata.compressed_token_count} tokens\"\n )\n\n # Also store the compression summary as a visible message\n if result.metadata:\n conversation_service.append_compression_message(\n agent.conversation_id, result.metadata.to_dict()\n )\n\n # Update agent's compressed summary for downstream persistence\n agent.compressed_summary = result.compressed_summary\n agent.compression_metadata = result.metadata.to_dict() if result.metadata else None\n agent.compression_saved = False\n\n # Reset the context limit flag so tools can continue\n agent.context_limit_reached = False\n agent.current_token_count = 0\n\n # Rebuild messages\n rebuilt_messages = self._rebuild_messages_after_compression(\n messages,\n result.compressed_summary,\n result.recent_queries,\n include_current_execution=False,\n include_tool_calls=False,\n )\n\n if rebuilt_messages is None:\n return False, None\n\n return True, rebuilt_messages\n\n except Exception as e:\n logger.error(\n f\"Error performing mid-execution compression: {str(e)}\", exc_info=True\n )\n return False, None\n\n def _perform_in_memory_compression(\n self, agent, messages: List[Dict]\n ) -> tuple[bool, Optional[List[Dict]]]:\n \"\"\"\n Fallback compression path when the conversation is not yet persisted.\n\n Uses CompressionService directly without DB persistence.\n \"\"\"\n try:\n from application.api.answer.services.compression.service import (\n CompressionService,\n )\n from application.core.model_utils import (\n get_api_key_for_provider,\n get_provider_from_model_id,\n )\n from application.core.settings import settings\n from application.llm.llm_creator import LLMCreator\n\n conversation = self._build_conversation_from_messages(messages)\n if not conversation:\n logger.warning(\n \"Cannot perform in-memory compression: no user/assistant turns found\"\n )\n return False, None\n\n compression_model = (\n settings.COMPRESSION_MODEL_OVERRIDE\n if settings.COMPRESSION_MODEL_OVERRIDE\n else agent.model_id\n )\n provider = get_provider_from_model_id(compression_model)\n api_key = get_api_key_for_provider(provider)\n compression_llm = LLMCreator.create_llm(\n provider,\n api_key,\n getattr(agent, \"user_api_key\", None),\n getattr(agent, \"decoded_token\", None),\n model_id=compression_model,\n agent_id=getattr(agent, \"agent_id\", None),\n )\n\n # Create service without DB persistence capability\n compression_service = CompressionService(\n llm=compression_llm,\n model_id=compression_model,\n conversation_service=None, # No DB updates for in-memory\n )\n\n queries_count = len(conversation.get(\"queries\", []))\n compress_up_to = queries_count - 1\n\n if compress_up_to < 0 or queries_count == 0:\n logger.warning(\"Not enough queries to compress in-memory context\")\n return False, None\n\n metadata = compression_service.compress_conversation(\n conversation,\n compress_up_to_index=compress_up_to,\n )\n\n # If compression doesn't reduce tokens, fall back to minimal pruning\n if (\n metadata.compressed_token_count\n >= metadata.original_token_count\n ):\n logger.warning(\n \"In-memory compression did not reduce token count; falling back to minimal pruning\"\n )\n pruned = self._prune_messages_minimal(messages)\n if pruned:\n agent.context_limit_reached = False\n agent.current_token_count = 0\n return True, pruned\n return False, None\n\n # Attach metadata to synthetic conversation\n conversation[\"compression_metadata\"] = {\n \"is_compressed\": True,\n \"compression_points\": [metadata.to_dict()],\n }\n\n compressed_summary, recent_queries = (\n compression_service.get_compressed_context(conversation)\n )\n\n agent.compressed_summary = compressed_summary\n agent.compression_metadata = metadata.to_dict()\n agent.compression_saved = False\n agent.context_limit_reached = False\n agent.current_token_count = 0\n\n rebuilt_messages = self._rebuild_messages_after_compression(\n messages,\n compressed_summary,\n recent_queries,\n include_current_execution=False,\n include_tool_calls=False,\n )\n if rebuilt_messages is None:\n return False, None\n\n logger.info(\n f\"In-memory compression successful - ratio: {metadata.compression_ratio:.1f}x, \"\n f\"saved {metadata.original_token_count - metadata.compressed_token_count} tokens\"\n )\n return True, rebuilt_messages\n\n except Exception as e:\n logger.error(\n f\"Error performing in-memory compression: {str(e)}\", exc_info=True\n )\n return False, None\n\n def handle_tool_calls(\n self, agent, tool_calls: List[ToolCall], tools_dict: Dict, messages: List[Dict]\n ) -> Generator:\n \"\"\"\n Execute tool calls and update conversation history.\n\n Args:\n agent: The agent instance\n tool_calls: List of tool calls to execute\n tools_dict: Available tools dictionary\n messages: Current conversation history\n\n Returns:\n Updated messages list\n \"\"\"\n updated_messages = messages.copy()\n\n for i, call in enumerate(tool_calls):\n # Check context limit before executing tool call\n if hasattr(agent, '_check_context_limit') and agent._check_context_limit(updated_messages):\n # Context limit reached - attempt mid-execution compression\n compression_attempted = False\n compression_successful = False\n\n try:\n from application.core.settings import settings\n compression_enabled = settings.ENABLE_CONVERSATION_COMPRESSION\n except Exception:\n compression_enabled = False\n\n if compression_enabled:\n compression_attempted = True\n try:\n logger.info(\n f\"Context limit reached with {len(tool_calls) - i} remaining tool calls. \"\n f\"Attempting mid-execution compression...\"\n )\n\n # Trigger mid-execution compression (DB-backed if available, otherwise in-memory)\n compression_successful, rebuilt_messages = self._perform_mid_execution_compression(\n agent, updated_messages\n )\n\n if compression_successful and rebuilt_messages is not None:\n # Update the messages list with rebuilt compressed version\n updated_messages = rebuilt_messages\n\n # Yield compression success message\n yield {\n \"type\": \"info\",\n \"data\": {\n \"message\": \"Context window limit reached. Compressed conversation history to continue processing.\"\n }\n }\n\n logger.info(\n f\"Mid-execution compression successful. Continuing with {len(tool_calls) - i} remaining tool calls.\"\n )\n # Proceed to execute the current tool call with the reduced context\n else:\n logger.warning(\"Mid-execution compression attempted but failed. Skipping remaining tools.\")\n except Exception as e:\n logger.error(f\"Error during mid-execution compression: {str(e)}\", exc_info=True)\n compression_attempted = True\n compression_successful = False\n\n # If compression wasn't attempted or failed, skip remaining tools\n if not compression_successful:\n if i == 0:\n # Special case: limit reached before executing any tools\n # This can happen when previous tool responses pushed context over limit\n if compression_attempted:\n logger.warning(\n f\"Context limit reached before executing any tools. \"\n f\"Compression attempted but failed. \"\n f\"Skipping all {len(tool_calls)} pending tool call(s). \"\n f\"This typically occurs when previous tool responses contained large amounts of data.\"\n )\n else:\n logger.warning(\n f\"Context limit reached before executing any tools. \"\n f\"Skipping all {len(tool_calls)} pending tool call(s). \"\n f\"This typically occurs when previous tool responses contained large amounts of data. \"\n f\"Consider enabling compression or using a model with larger context window.\"\n )\n else:\n # Normal case: executed some tools, now stopping\n tool_word = \"tool call\" if i == 1 else \"tool calls\"\n remaining = len(tool_calls) - i\n remaining_word = \"tool call\" if remaining == 1 else \"tool calls\"\n if compression_attempted:\n logger.warning(\n f\"Context limit reached after executing {i} {tool_word}. \"\n f\"Compression attempted but failed. \"\n f\"Skipping remaining {remaining} {remaining_word}.\"\n )\n else:\n logger.warning(\n f\"Context limit reached after executing {i} {tool_word}. \"\n f\"Skipping remaining {remaining} {remaining_word}. \"\n f\"Consider enabling compression or using a model with larger context window.\"\n )\n\n # Mark remaining tools as skipped\n for remaining_call in tool_calls[i:]:\n skip_message = {\n \"type\": \"tool_call\",\n \"data\": {\n \"tool_name\": \"system\",\n \"call_id\": remaining_call.id,\n \"action_name\": remaining_call.name,\n \"arguments\": {},\n \"result\": \"Skipped: Context limit reached. Too many tool calls in conversation.\",\n \"status\": \"skipped\"\n }\n }\n yield skip_message\n\n # Set flag on agent\n agent.context_limit_reached = True\n break\n try:\n self.tool_calls.append(call)\n tool_executor_gen = agent._execute_tool_action(tools_dict, call)\n while True:\n try:\n yield next(tool_executor_gen)\n except StopIteration as e:\n tool_response, call_id = e.value\n break\n \n function_call_content = {\n \"function_call\": {\n \"name\": call.name,\n \"args\": call.arguments,\n \"call_id\": call_id,\n }\n }\n # Include thought_signature for Google Gemini 3 models\n # It should be at the same level as function_call, not inside it\n if call.thought_signature:\n function_call_content[\"thought_signature\"] = call.thought_signature\n updated_messages.append(\n {\n \"role\": \"assistant\",\n \"content\": [function_call_content],\n }\n )\n\n\n updated_messages.append(self.create_tool_message(call, tool_response))\n except Exception as e:\n logger.error(f\"Error executing tool: {str(e)}\", exc_info=True)\n error_call = ToolCall(\n id=call.id, name=call.name, arguments=call.arguments\n )\n error_response = f\"Error executing tool: {str(e)}\"\n error_message = self.create_tool_message(error_call, error_response)\n updated_messages.append(error_message)\n\n call_parts = call.name.split(\"_\")\n if len(call_parts) >= 2:\n tool_id = call_parts[-1] # Last part is tool ID (e.g., \"1\")\n action_name = \"_\".join(call_parts[:-1])\n tool_name = tools_dict.get(tool_id, {}).get(\"name\", \"unknown_tool\")\n full_action_name = f\"{action_name}_{tool_id}\"\n else:\n tool_name = \"unknown_tool\"\n action_name = call.name\n full_action_name = call.name\n yield {\n \"type\": \"tool_call\",\n \"data\": {\n \"tool_name\": tool_name,\n \"call_id\": call.id,\n \"action_name\": full_action_name,\n \"arguments\": call.arguments,\n \"error\": error_response,\n \"status\": \"error\",\n },\n }\n return updated_messages\n\n def handle_non_streaming(\n self, agent, response: Any, tools_dict: Dict, messages: List[Dict]\n ) -> Generator:\n \"\"\"\n Handle non-streaming response flow.\n\n Args:\n agent: The agent instance\n response: Current LLM response\n tools_dict: Available tools dictionary\n messages: Conversation history\n\n Returns:\n Final response after processing all tool calls\n \"\"\"\n parsed = self.parse_response(response)\n self.llm_calls.append(build_stack_data(agent.llm))\n\n while parsed.requires_tool_call:\n tool_handler_gen = self.handle_tool_calls(\n agent, parsed.tool_calls, tools_dict, messages\n )\n while True:\n try:\n yield next(tool_handler_gen)\n except StopIteration as e:\n messages = e.value\n break\n response = agent.llm.gen(\n model=agent.model_id, messages=messages, tools=agent.tools\n )\n parsed = self.parse_response(response)\n self.llm_calls.append(build_stack_data(agent.llm))\n return parsed.content\n\n def handle_streaming(\n self, agent, response: Any, tools_dict: Dict, messages: List[Dict]\n ) -> Generator:\n \"\"\"\n Handle streaming response flow.\n\n Args:\n agent: The agent instance\n response: Current LLM response\n tools_dict: Available tools dictionary\n messages: Conversation history\n\n Yields:\n Streaming response chunks\n \"\"\"\n buffer = \"\"\n tool_calls = {}\n\n for chunk in self._iterate_stream(response):\n if isinstance(chunk, dict) and chunk.get(\"type\") == \"thought\":\n yield chunk\n continue\n if isinstance(chunk, str):\n yield chunk\n continue\n parsed = self.parse_response(chunk)\n\n if parsed.tool_calls:\n for call in parsed.tool_calls:\n if call.index not in tool_calls:\n tool_calls[call.index] = call\n else:\n existing = tool_calls[call.index]\n if call.id:\n existing.id = call.id\n if call.name:\n existing.name = call.name\n if call.arguments:\n if existing.arguments is None:\n existing.arguments = call.arguments\n else:\n existing.arguments += call.arguments\n # Preserve thought_signature for Google Gemini 3 models\n if call.thought_signature:\n existing.thought_signature = call.thought_signature\n if parsed.finish_reason == \"tool_calls\":\n tool_handler_gen = self.handle_tool_calls(\n agent, list(tool_calls.values()), tools_dict, messages\n )\n while True:\n try:\n yield next(tool_handler_gen)\n except StopIteration as e:\n messages = e.value\n break\n tool_calls = {}\n\n # Check if context limit was reached during tool execution\n if hasattr(agent, 'context_limit_reached') and agent.context_limit_reached:\n # Add system message warning about context limit\n messages.append({\n \"role\": \"system\",\n \"content\": (\n \"WARNING: Context window limit has been reached. \"\n \"Please provide a final response to the user without making additional tool calls. \"\n \"Summarize the work completed so far.\"\n )\n })\n logger.info(\"Context limit reached - instructing agent to wrap up\")\n\n response = agent.llm.gen_stream(\n model=agent.model_id, messages=messages, tools=agent.tools if not agent.context_limit_reached else None\n )\n self.llm_calls.append(build_stack_data(agent.llm))\n\n yield from self.handle_streaming(agent, response, tools_dict, messages)\n return\n if parsed.content:\n buffer += parsed.content\n yield buffer\n buffer = \"\"\n if parsed.finish_reason == \"stop\":\n return\n" }, { "path": "application/llm/handlers/google.py", "content": "import uuid\nfrom typing import Any, Dict, Generator\n\nfrom application.llm.handlers.base import LLMHandler, LLMResponse, ToolCall\n\n\nclass GoogleLLMHandler(LLMHandler):\n \"\"\"Handler for Google's GenAI API.\"\"\"\n\n def parse_response(self, response: Any) -> LLMResponse:\n \"\"\"Parse Google response into standardized format.\"\"\"\n\n if isinstance(response, str):\n return LLMResponse(\n content=response,\n tool_calls=[],\n finish_reason=\"stop\",\n raw_response=response,\n )\n if hasattr(response, \"candidates\"):\n parts = response.candidates[0].content.parts if response.candidates else []\n tool_calls = []\n for idx, part in enumerate(parts):\n if hasattr(part, \"function_call\") and part.function_call is not None:\n has_sig = hasattr(part, \"thought_signature\") and part.thought_signature is not None\n thought_sig = part.thought_signature if has_sig else None\n tool_calls.append(\n ToolCall(\n id=str(uuid.uuid4()),\n name=part.function_call.name,\n arguments=part.function_call.args,\n index=idx,\n thought_signature=thought_sig,\n )\n )\n\n content = \" \".join(\n part.text\n for part in parts\n if hasattr(part, \"text\") and part.text is not None\n )\n return LLMResponse(\n content=content,\n tool_calls=tool_calls,\n finish_reason=\"tool_calls\" if tool_calls else \"stop\",\n raw_response=response,\n )\n else:\n # This branch handles individual Part objects from streaming responses\n tool_calls = []\n if hasattr(response, \"function_call\") and response.function_call is not None:\n has_sig = hasattr(response, \"thought_signature\") and response.thought_signature is not None\n thought_sig = response.thought_signature if has_sig else None\n tool_calls.append(\n ToolCall(\n id=str(uuid.uuid4()),\n name=response.function_call.name,\n arguments=response.function_call.args,\n thought_signature=thought_sig,\n )\n )\n return LLMResponse(\n content=response.text if hasattr(response, \"text\") else \"\",\n tool_calls=tool_calls,\n finish_reason=\"tool_calls\" if tool_calls else \"stop\",\n raw_response=response,\n )\n\n def create_tool_message(self, tool_call: ToolCall, result: Any) -> Dict:\n \"\"\"Create Google-style tool message.\"\"\"\n\n return {\n \"role\": \"model\",\n \"content\": [\n {\n \"function_response\": {\n \"name\": tool_call.name,\n \"response\": {\"result\": result},\n }\n }\n ],\n }\n\n def _iterate_stream(self, response: Any) -> Generator:\n \"\"\"Iterate through Google streaming response.\"\"\"\n for chunk in response:\n yield chunk\n" }, { "path": "application/llm/handlers/handler_creator.py", "content": "from application.llm.handlers.base import LLMHandler\nfrom application.llm.handlers.google import GoogleLLMHandler\nfrom application.llm.handlers.openai import OpenAILLMHandler\n\n\nclass LLMHandlerCreator:\n handlers = {\n \"openai\": OpenAILLMHandler,\n \"google\": GoogleLLMHandler,\n \"default\": OpenAILLMHandler,\n }\n\n @classmethod\n def create_handler(cls, llm_type: str, *args, **kwargs) -> LLMHandler:\n handler_class = cls.handlers.get(llm_type.lower())\n if not handler_class:\n handler_class = OpenAILLMHandler\n return handler_class(*args, **kwargs)\n" }, { "path": "application/llm/handlers/openai.py", "content": "from typing import Any, Dict, Generator\n\nfrom application.llm.handlers.base import LLMHandler, LLMResponse, ToolCall\n\n\nclass OpenAILLMHandler(LLMHandler):\n \"\"\"Handler for OpenAI API.\"\"\"\n\n def parse_response(self, response: Any) -> LLMResponse:\n \"\"\"Parse OpenAI response into standardized format.\"\"\"\n if isinstance(response, str):\n return LLMResponse(\n content=response,\n tool_calls=[],\n finish_reason=\"stop\",\n raw_response=response,\n )\n\n message = getattr(response, \"message\", None) or getattr(response, \"delta\", None)\n\n tool_calls = []\n if hasattr(message, \"tool_calls\"):\n tool_calls = [\n ToolCall(\n id=getattr(tc, \"id\", \"\"),\n name=getattr(tc.function, \"name\", \"\"),\n arguments=getattr(tc.function, \"arguments\", \"\"),\n index=getattr(tc, \"index\", None),\n )\n for tc in message.tool_calls or []\n ]\n return LLMResponse(\n content=getattr(message, \"content\", \"\"),\n tool_calls=tool_calls,\n finish_reason=getattr(response, \"finish_reason\", \"\"),\n raw_response=response,\n )\n\n def create_tool_message(self, tool_call: ToolCall, result: Any) -> Dict:\n \"\"\"Create OpenAI-style tool message.\"\"\"\n return {\n \"role\": \"tool\",\n \"content\": [\n {\n \"function_response\": {\n \"name\": tool_call.name,\n \"response\": {\"result\": result},\n \"call_id\": tool_call.id,\n }\n }\n ],\n }\n\n def _iterate_stream(self, response: Any) -> Generator:\n \"\"\"Iterate through OpenAI streaming response.\"\"\"\n for chunk in response:\n yield chunk\n" }, { "path": "application/llm/llama_cpp.py", "content": "from application.llm.base import BaseLLM\nfrom application.core.settings import settings\nimport threading\n\n\nclass LlamaSingleton:\n _instances = {}\n _lock = threading.Lock() # Add a lock for thread synchronization\n\n @classmethod\n def get_instance(cls, llm_name):\n if llm_name not in cls._instances:\n try:\n from llama_cpp import Llama\n except ImportError:\n raise ImportError(\n \"Please install llama_cpp using pip install llama-cpp-python\"\n )\n cls._instances[llm_name] = Llama(model_path=llm_name, n_ctx=2048)\n return cls._instances[llm_name]\n\n @classmethod\n def query_model(cls, llm, prompt, **kwargs):\n with cls._lock:\n return llm(prompt, **kwargs)\n\n\nclass LlamaCpp(BaseLLM):\n def __init__(\n self,\n api_key=None,\n user_api_key=None,\n llm_name=settings.LLM_PATH,\n *args,\n **kwargs,\n ):\n super().__init__(*args, **kwargs)\n self.api_key = api_key\n self.user_api_key = user_api_key\n self.llama = LlamaSingleton.get_instance(llm_name)\n\n def _raw_gen(self, baseself, model, messages, stream=False, **kwargs):\n context = messages[0][\"content\"]\n user_question = messages[-1][\"content\"]\n prompt = f\"### Instruction \\n {user_question} \\n ### Context \\n {context} \\n ### Answer \\n\"\n result = LlamaSingleton.query_model(\n self.llama, prompt, max_tokens=150, echo=False\n )\n return result[\"choices\"][0][\"text\"].split(\"### Answer \\n\")[-1]\n\n def _raw_gen_stream(self, baseself, model, messages, stream=True, **kwargs):\n context = messages[0][\"content\"]\n user_question = messages[-1][\"content\"]\n prompt = f\"### Instruction \\n {user_question} \\n ### Context \\n {context} \\n ### Answer \\n\"\n result = LlamaSingleton.query_model(\n self.llama, prompt, max_tokens=150, echo=False, stream=stream\n )\n for item in result:\n for choice in item[\"choices\"]:\n yield choice[\"text\"]\n" }, { "path": "application/llm/llm_creator.py", "content": "import logging\n\nfrom application.llm.anthropic import AnthropicLLM\nfrom application.llm.docsgpt_provider import DocsGPTAPILLM\nfrom application.llm.google_ai import GoogleLLM\nfrom application.llm.groq import GroqLLM\nfrom application.llm.llama_cpp import LlamaCpp\nfrom application.llm.novita import NovitaLLM\nfrom application.llm.openai import AzureOpenAILLM, OpenAILLM\nfrom application.llm.premai import PremAILLM\nfrom application.llm.sagemaker import SagemakerAPILLM\nfrom application.llm.open_router import OpenRouterLLM\n\nlogger = logging.getLogger(__name__)\n\n\nclass LLMCreator:\n llms = {\n \"openai\": OpenAILLM,\n \"azure_openai\": AzureOpenAILLM,\n \"sagemaker\": SagemakerAPILLM,\n \"llama.cpp\": LlamaCpp,\n \"anthropic\": AnthropicLLM,\n \"docsgpt\": DocsGPTAPILLM,\n \"premai\": PremAILLM,\n \"groq\": GroqLLM,\n \"google\": GoogleLLM,\n \"novita\": NovitaLLM,\n \"openrouter\": OpenRouterLLM,\n }\n\n @classmethod\n def create_llm(\n cls,\n type,\n api_key,\n user_api_key,\n decoded_token,\n model_id=None,\n agent_id=None,\n *args,\n **kwargs,\n ):\n from application.core.model_utils import get_base_url_for_model\n\n llm_class = cls.llms.get(type.lower())\n if not llm_class:\n raise ValueError(f\"No LLM class found for type {type}\")\n\n # Extract base_url from model configuration if model_id is provided\n base_url = None\n if model_id:\n base_url = get_base_url_for_model(model_id)\n\n return llm_class(\n api_key,\n user_api_key,\n decoded_token=decoded_token,\n model_id=model_id,\n agent_id=agent_id,\n base_url=base_url,\n *args,\n **kwargs,\n )\n" }, { "path": "application/llm/novita.py", "content": "from application.core.settings import settings\nfrom application.llm.openai import OpenAILLM\n\nNOVITA_BASE_URL = \"https://api.novita.ai/v3/openai\"\n\n\nclass NovitaLLM(OpenAILLM):\n def __init__(self, api_key=None, user_api_key=None, base_url=None, *args, **kwargs):\n super().__init__(\n api_key=api_key or settings.API_KEY,\n user_api_key=user_api_key,\n base_url=base_url or NOVITA_BASE_URL,\n *args,\n **kwargs,\n )\n" }, { "path": "application/llm/open_router.py", "content": "from application.core.settings import settings\nfrom application.llm.openai import OpenAILLM\n\nOPEN_ROUTER_BASE_URL = \"https://openrouter.ai/api/v1\"\n\n\nclass OpenRouterLLM(OpenAILLM):\n def __init__(self, api_key=None, user_api_key=None, base_url=None, *args, **kwargs):\n super().__init__(\n api_key=api_key or settings.OPEN_ROUTER_API_KEY or settings.API_KEY,\n user_api_key=user_api_key,\n base_url=base_url or OPEN_ROUTER_BASE_URL,\n *args,\n **kwargs,\n )\n" }, { "path": "application/llm/openai.py", "content": "import base64\nimport json\nimport logging\n\nfrom openai import OpenAI\n\nfrom application.core.settings import settings\nfrom application.llm.base import BaseLLM\nfrom application.storage.storage_creator import StorageCreator\n\n\ndef _truncate_base64_for_logging(messages):\n \"\"\"\n Create a copy of messages with base64 data truncated for readable logging.\n\n Args:\n messages: List of message dicts\n\n Returns:\n Copy of messages with truncated base64 content\n \"\"\"\n import copy\n\n def truncate_content(content):\n if isinstance(content, str):\n # Check if it looks like a data URL with base64\n if content.startswith(\"data:\") and \";base64,\" in content:\n prefix_end = content.index(\";base64,\") + len(\";base64,\")\n prefix = content[:prefix_end]\n return f\"{prefix}[BASE64_DATA_TRUNCATED, length={len(content) - prefix_end}]\"\n return content\n elif isinstance(content, list):\n return [truncate_item(item) for item in content]\n elif isinstance(content, dict):\n return {k: truncate_content(v) for k, v in content.items()}\n return content\n\n def truncate_item(item):\n if isinstance(item, dict):\n result = {}\n for k, v in item.items():\n if k == \"url\" and isinstance(v, str) and \";base64,\" in v:\n prefix_end = v.index(\";base64,\") + len(\";base64,\")\n prefix = v[:prefix_end]\n result[k] = f\"{prefix}[BASE64_DATA_TRUNCATED, length={len(v) - prefix_end}]\"\n elif k == \"data\" and isinstance(v, str) and len(v) > 100:\n result[k] = f\"[BASE64_DATA_TRUNCATED, length={len(v)}]\"\n else:\n result[k] = truncate_content(v)\n return result\n return truncate_content(item)\n\n truncated = []\n for msg in messages:\n msg_copy = copy.copy(msg)\n if \"content\" in msg_copy:\n msg_copy[\"content\"] = truncate_content(msg_copy[\"content\"])\n truncated.append(msg_copy)\n\n return truncated\n\n\nclass OpenAILLM(BaseLLM):\n\n def __init__(self, api_key=None, user_api_key=None, base_url=None, *args, **kwargs):\n\n super().__init__(*args, **kwargs)\n self.api_key = api_key or settings.OPENAI_API_KEY or settings.API_KEY\n self.user_api_key = user_api_key\n\n # Priority: 1) Parameter base_url, 2) Settings OPENAI_BASE_URL, 3) Default\n effective_base_url = None\n if base_url and isinstance(base_url, str) and base_url.strip():\n effective_base_url = base_url\n elif (\n isinstance(settings.OPENAI_BASE_URL, str)\n and settings.OPENAI_BASE_URL.strip()\n ):\n effective_base_url = settings.OPENAI_BASE_URL\n else:\n effective_base_url = \"https://api.openai.com/v1\"\n\n self.client = OpenAI(api_key=self.api_key, base_url=effective_base_url)\n self.storage = StorageCreator.get_storage()\n\n def _clean_messages_openai(self, messages):\n cleaned_messages = []\n for message in messages:\n role = message.get(\"role\")\n content = message.get(\"content\")\n\n if role == \"model\":\n role = \"assistant\"\n if role and content is not None:\n if isinstance(content, str):\n cleaned_messages.append({\"role\": role, \"content\": content})\n elif isinstance(content, list):\n # Collect all content parts into a single message\n content_parts = []\n\n for item in content:\n if \"function_call\" in item:\n # Function calls need their own message\n cleaned_args = self._remove_null_values(\n item[\"function_call\"][\"args\"]\n )\n tool_call = {\n \"id\": item[\"function_call\"][\"call_id\"],\n \"type\": \"function\",\n \"function\": {\n \"name\": item[\"function_call\"][\"name\"],\n \"arguments\": json.dumps(cleaned_args),\n },\n }\n cleaned_messages.append(\n {\n \"role\": \"assistant\",\n \"content\": None,\n \"tool_calls\": [tool_call],\n }\n )\n elif \"function_response\" in item:\n # Function responses need their own message\n cleaned_messages.append(\n {\n \"role\": \"tool\",\n \"tool_call_id\": item[\"function_response\"][\n \"call_id\"\n ],\n \"content\": json.dumps(\n item[\"function_response\"][\"response\"][\"result\"]\n ),\n }\n )\n elif isinstance(item, dict):\n # Collect content parts (text, images, files) into a single message\n if \"type\" in item and item[\"type\"] == \"text\" and \"text\" in item:\n content_parts.append(item)\n elif \"type\" in item and item[\"type\"] == \"file\" and \"file\" in item:\n content_parts.append(item)\n elif \"type\" in item and item[\"type\"] == \"image_url\" and \"image_url\" in item:\n content_parts.append(item)\n elif \"text\" in item and \"type\" not in item:\n # Legacy format: {\"text\": \"...\"} without type\n content_parts.append({\"type\": \"text\", \"text\": item[\"text\"]})\n\n # Add the collected content parts as a single message\n if content_parts:\n cleaned_messages.append({\"role\": role, \"content\": content_parts})\n else:\n raise ValueError(f\"Unexpected content type: {type(content)}\")\n return cleaned_messages\n\n @staticmethod\n def _normalize_reasoning_value(value):\n \"\"\"Normalize reasoning payloads from OpenAI-compatible stream chunks.\"\"\"\n if value is None:\n return \"\"\n if isinstance(value, str):\n return value\n if isinstance(value, list):\n return \"\".join(\n OpenAILLM._normalize_reasoning_value(item) for item in value\n )\n if isinstance(value, dict):\n for key in (\"text\", \"content\", \"value\", \"reasoning_content\", \"reasoning\"):\n normalized = OpenAILLM._normalize_reasoning_value(value.get(key))\n if normalized:\n return normalized\n return \"\"\n\n for attr in (\"text\", \"content\", \"value\"):\n if hasattr(value, attr):\n normalized = OpenAILLM._normalize_reasoning_value(getattr(value, attr))\n if normalized:\n return normalized\n return \"\"\n\n @classmethod\n def _extract_reasoning_text(cls, delta):\n \"\"\"Extract reasoning/thinking tokens from OpenAI-compatible delta chunks.\"\"\"\n if delta is None:\n return \"\"\n\n for key in (\n \"reasoning_content\",\n \"reasoning\",\n \"thinking\",\n \"thinking_content\",\n ):\n value = getattr(delta, key, None)\n if value is None and isinstance(delta, dict):\n value = delta.get(key)\n normalized = cls._normalize_reasoning_value(value)\n if normalized:\n return normalized\n return \"\"\n\n def _raw_gen(\n self,\n baseself,\n model,\n messages,\n stream=False,\n tools=None,\n engine=settings.AZURE_DEPLOYMENT_NAME,\n response_format=None,\n **kwargs,\n ):\n messages = self._clean_messages_openai(messages)\n logging.info(f\"Cleaned messages: {_truncate_base64_for_logging(messages)}\")\n\n # Convert max_tokens to max_completion_tokens for newer models\n if \"max_tokens\" in kwargs:\n kwargs[\"max_completion_tokens\"] = kwargs.pop(\"max_tokens\")\n\n request_params = {\n \"model\": model,\n \"messages\": messages,\n \"stream\": stream,\n **kwargs,\n }\n\n if tools:\n request_params[\"tools\"] = tools\n if response_format:\n request_params[\"response_format\"] = response_format\n response = self.client.chat.completions.create(**request_params)\n logging.info(f\"OpenAI response: {response}\")\n if tools:\n return response.choices[0]\n else:\n return response.choices[0].message.content\n\n def _raw_gen_stream(\n self,\n baseself,\n model,\n messages,\n stream=True,\n tools=None,\n engine=settings.AZURE_DEPLOYMENT_NAME,\n response_format=None,\n **kwargs,\n ):\n messages = self._clean_messages_openai(messages)\n logging.info(f\"Cleaned messages: {_truncate_base64_for_logging(messages)}\")\n\n # Convert max_tokens to max_completion_tokens for newer models\n if \"max_tokens\" in kwargs:\n kwargs[\"max_completion_tokens\"] = kwargs.pop(\"max_tokens\")\n\n request_params = {\n \"model\": model,\n \"messages\": messages,\n \"stream\": stream,\n **kwargs,\n }\n\n if tools:\n request_params[\"tools\"] = tools\n if response_format:\n request_params[\"response_format\"] = response_format\n response = self.client.chat.completions.create(**request_params)\n\n try:\n for line in response:\n logging.debug(f\"OpenAI stream line: {line}\")\n if not getattr(line, \"choices\", None):\n continue\n\n choice = line.choices[0]\n delta = getattr(choice, \"delta\", None)\n reasoning_text = self._extract_reasoning_text(delta)\n if reasoning_text:\n yield {\"type\": \"thought\", \"thought\": reasoning_text}\n\n content = getattr(delta, \"content\", None)\n if isinstance(content, str) and content:\n yield content\n continue\n\n has_tool_calls = bool(getattr(delta, \"tool_calls\", None))\n finish_reason = getattr(choice, \"finish_reason\", None)\n\n # Yield non-content chunks only when needed for tool-call handling.\n if has_tool_calls or finish_reason == \"tool_calls\":\n yield choice\n finally:\n if hasattr(response, \"close\"):\n response.close()\n\n def _supports_tools(self):\n return True\n\n def _supports_structured_output(self):\n return True\n\n def prepare_structured_output_format(self, json_schema):\n if not json_schema:\n return None\n try:\n\n def add_additional_properties_false(schema_obj):\n if isinstance(schema_obj, dict):\n schema_copy = schema_obj.copy()\n\n if schema_copy.get(\"type\") == \"object\":\n schema_copy[\"additionalProperties\"] = False\n # Ensure 'required' includes all properties for OpenAI strict mode\n\n if \"properties\" in schema_copy:\n schema_copy[\"required\"] = list(\n schema_copy[\"properties\"].keys()\n )\n for key, value in schema_copy.items():\n if key == \"properties\" and isinstance(value, dict):\n schema_copy[key] = {\n prop_name: add_additional_properties_false(prop_schema)\n for prop_name, prop_schema in value.items()\n }\n elif key == \"items\" and isinstance(value, dict):\n schema_copy[key] = add_additional_properties_false(value)\n elif key in [\"anyOf\", \"oneOf\", \"allOf\"] and isinstance(\n value, list\n ):\n schema_copy[key] = [\n add_additional_properties_false(sub_schema)\n for sub_schema in value\n ]\n return schema_copy\n return schema_obj\n\n processed_schema = add_additional_properties_false(json_schema)\n\n result = {\n \"type\": \"json_schema\",\n \"json_schema\": {\n \"name\": processed_schema.get(\"name\", \"response\"),\n \"description\": processed_schema.get(\n \"description\", \"Structured response\"\n ),\n \"schema\": processed_schema,\n \"strict\": True,\n },\n }\n\n return result\n except Exception as e:\n logging.error(f\"Error preparing structured output format: {e}\")\n return None\n\n def get_supported_attachment_types(self):\n \"\"\"\n Return a list of MIME types supported by OpenAI for file uploads.\n\n This reads from the model config to ensure consistency.\n If no model config found, falls back to images only (safest default).\n\n Returns:\n list: List of supported MIME types\n \"\"\"\n from application.core.model_configs import OPENAI_ATTACHMENTS\n return OPENAI_ATTACHMENTS\n\n def prepare_messages_with_attachments(self, messages, attachments=None):\n \"\"\"\n Process attachments using OpenAI's file API for more efficient handling.\n\n Args:\n messages (list): List of message dictionaries.\n attachments (list): List of attachment dictionaries with content and metadata.\n\n Returns:\n list: Messages formatted with file references for OpenAI API.\n \"\"\"\n if not attachments:\n return messages\n prepared_messages = messages.copy()\n\n # Find the user message to attach file_id to the last one\n\n user_message_index = None\n for i in range(len(prepared_messages) - 1, -1, -1):\n if prepared_messages[i].get(\"role\") == \"user\":\n user_message_index = i\n break\n if user_message_index is None:\n user_message = {\"role\": \"user\", \"content\": []}\n prepared_messages.append(user_message)\n user_message_index = len(prepared_messages) - 1\n if isinstance(prepared_messages[user_message_index].get(\"content\"), str):\n text_content = prepared_messages[user_message_index][\"content\"]\n prepared_messages[user_message_index][\"content\"] = [\n {\"type\": \"text\", \"text\": text_content}\n ]\n elif not isinstance(prepared_messages[user_message_index].get(\"content\"), list):\n prepared_messages[user_message_index][\"content\"] = []\n for attachment in attachments:\n mime_type = attachment.get(\"mime_type\")\n logging.info(f\"Processing attachment with mime_type: {mime_type}, has_data: {'data' in attachment}, has_path: {'path' in attachment}\")\n\n if mime_type and mime_type.startswith(\"image/\"):\n try:\n # Check if this is a pre-converted image (from PDF-to-image conversion)\n if \"data\" in attachment:\n base64_image = attachment[\"data\"]\n else:\n base64_image = self._get_base64_image(attachment)\n\n prepared_messages[user_message_index][\"content\"].append(\n {\n \"type\": \"image_url\",\n \"image_url\": {\n \"url\": f\"data:{mime_type};base64,{base64_image}\"\n },\n }\n )\n\n except Exception as e:\n logging.error(\n f\"Error processing image attachment: {e}\", exc_info=True\n )\n if \"content\" in attachment:\n prepared_messages[user_message_index][\"content\"].append(\n {\n \"type\": \"text\",\n \"text\": f\"[Image could not be processed: {attachment.get('path', 'unknown')}]\",\n }\n )\n # Handle PDFs using the file API\n\n elif mime_type == \"application/pdf\":\n logging.info(f\"Attempting to upload PDF to OpenAI: {attachment.get('path', 'unknown')}\")\n try:\n file_id = self._upload_file_to_openai(attachment)\n prepared_messages[user_message_index][\"content\"].append(\n {\"type\": \"file\", \"file\": {\"file_id\": file_id}}\n )\n except Exception as e:\n logging.error(f\"Error uploading PDF to OpenAI: {e}\", exc_info=True)\n if \"content\" in attachment:\n prepared_messages[user_message_index][\"content\"].append(\n {\n \"type\": \"text\",\n \"text\": f\"File content:\\n\\n{attachment['content']}\",\n }\n )\n else:\n logging.warning(f\"Unsupported attachment type in OpenAI provider: {mime_type}\")\n return prepared_messages\n\n def _get_base64_image(self, attachment):\n \"\"\"\n Convert an image file to base64 encoding.\n\n Args:\n attachment (dict): Attachment dictionary with path and metadata.\n\n Returns:\n str: Base64-encoded image data.\n \"\"\"\n file_path = attachment.get(\"path\")\n if not file_path:\n raise ValueError(\"No file path provided in attachment\")\n try:\n with self.storage.get_file(file_path) as image_file:\n return base64.b64encode(image_file.read()).decode(\"utf-8\")\n except FileNotFoundError:\n raise FileNotFoundError(f\"File not found: {file_path}\")\n\n def _upload_file_to_openai(self, attachment):\n \"\"\"\n Upload a file to OpenAI and return the file_id.\n\n Args:\n attachment (dict): Attachment dictionary with path and metadata.\n Expected keys:\n - path: Path to the file\n - id: Optional MongoDB ID for caching\n\n Returns:\n str: OpenAI file_id for the uploaded file.\n \"\"\"\n import logging\n\n if \"openai_file_id\" in attachment:\n return attachment[\"openai_file_id\"]\n file_path = attachment.get(\"path\")\n\n if not self.storage.file_exists(file_path):\n raise FileNotFoundError(f\"File not found: {file_path}\")\n try:\n file_id = self.storage.process_file(\n file_path,\n lambda local_path, **kwargs: self.client.files.create(\n file=open(local_path, \"rb\"), purpose=\"assistants\"\n ).id,\n )\n\n from application.core.mongo_db import MongoDB\n\n mongo = MongoDB.get_client()\n db = mongo[settings.MONGO_DB_NAME]\n attachments_collection = db[\"attachments\"]\n if \"_id\" in attachment:\n attachments_collection.update_one(\n {\"_id\": attachment[\"_id\"]}, {\"$set\": {\"openai_file_id\": file_id}}\n )\n return file_id\n except Exception as e:\n logging.error(f\"Error uploading file to OpenAI: {e}\", exc_info=True)\n raise\n\n\nclass AzureOpenAILLM(OpenAILLM):\n\n def __init__(self, api_key, user_api_key, *args, **kwargs):\n\n super().__init__(api_key)\n self.api_base = (settings.OPENAI_API_BASE,)\n self.api_version = (settings.OPENAI_API_VERSION,)\n self.deployment_name = (settings.AZURE_DEPLOYMENT_NAME,)\n from openai import AzureOpenAI\n\n self.client = AzureOpenAI(\n api_key=api_key,\n api_version=settings.OPENAI_API_VERSION,\n azure_endpoint=settings.OPENAI_API_BASE,\n )\n" }, { "path": "application/llm/premai.py", "content": "from application.llm.base import BaseLLM\nfrom application.core.settings import settings\n\n\nclass PremAILLM(BaseLLM):\n\n def __init__(self, api_key=None, user_api_key=None, *args, **kwargs):\n from premai import Prem\n\n super().__init__(*args, **kwargs)\n self.client = Prem(api_key=api_key)\n self.api_key = api_key\n self.user_api_key = user_api_key\n self.project_id = settings.PREMAI_PROJECT_ID\n\n def _raw_gen(self, baseself, model, messages, stream=False, **kwargs):\n response = self.client.chat.completions.create(\n model=model,\n project_id=self.project_id,\n messages=messages,\n stream=stream,\n **kwargs\n )\n\n return response.choices[0].message[\"content\"]\n\n def _raw_gen_stream(self, baseself, model, messages, stream=True, **kwargs):\n response = self.client.chat.completions.create(\n model=model,\n project_id=self.project_id,\n messages=messages,\n stream=stream,\n **kwargs\n )\n\n for line in response:\n if line.choices[0].delta[\"content\"] is not None:\n yield line.choices[0].delta[\"content\"]\n" }, { "path": "application/llm/sagemaker.py", "content": "from application.llm.base import BaseLLM\nfrom application.core.settings import settings\nimport json\nimport io\n\n\nclass LineIterator:\n \"\"\"\n A helper class for parsing the byte stream input.\n\n The output of the model will be in the following format:\n ```\n b'{\"outputs\": [\" a\"]}\\n'\n b'{\"outputs\": [\" challenging\"]}\\n'\n b'{\"outputs\": [\" problem\"]}\\n'\n ...\n ```\n\n While usually each PayloadPart event from the event stream will contain a byte array\n with a full json, this is not guaranteed and some of the json objects may be split across\n PayloadPart events. For example:\n ```\n {'PayloadPart': {'Bytes': b'{\"outputs\": '}}\n {'PayloadPart': {'Bytes': b'[\" problem\"]}\\n'}}\n ```\n\n This class accounts for this by concatenating bytes written via the 'write' function\n and then exposing a method which will return lines (ending with a '\\n' character) within\n the buffer via the 'scan_lines' function. It maintains the position of the last read\n position to ensure that previous bytes are not exposed again.\n \"\"\"\n\n def __init__(self, stream):\n self.byte_iterator = iter(stream)\n self.buffer = io.BytesIO()\n self.read_pos = 0\n\n def __iter__(self):\n return self\n\n def __next__(self):\n while True:\n self.buffer.seek(self.read_pos)\n line = self.buffer.readline()\n if line and line[-1] == ord(\"\\n\"):\n self.read_pos += len(line)\n return line[:-1]\n try:\n chunk = next(self.byte_iterator)\n except StopIteration:\n if self.read_pos < self.buffer.getbuffer().nbytes:\n continue\n raise\n if \"PayloadPart\" not in chunk:\n print(\"Unknown event type:\" + chunk)\n continue\n self.buffer.seek(0, io.SEEK_END)\n self.buffer.write(chunk[\"PayloadPart\"][\"Bytes\"])\n\n\nclass SagemakerAPILLM(BaseLLM):\n\n def __init__(self, api_key=None, user_api_key=None, *args, **kwargs):\n import boto3\n\n runtime = boto3.client(\n \"runtime.sagemaker\",\n aws_access_key_id=\"xxx\",\n aws_secret_access_key=\"xxx\",\n region_name=\"us-west-2\",\n )\n\n super().__init__(*args, **kwargs)\n self.api_key = api_key\n self.user_api_key = user_api_key\n self.endpoint = settings.SAGEMAKER_ENDPOINT\n self.runtime = runtime\n\n def _raw_gen(self, baseself, model, messages, stream=False, tools=None, **kwargs):\n context = messages[0][\"content\"]\n user_question = messages[-1][\"content\"]\n prompt = f\"### Instruction \\n {user_question} \\n ### Context \\n {context} \\n ### Answer \\n\"\n\n # Construct payload for endpoint\n payload = {\n \"inputs\": prompt,\n \"stream\": False,\n \"parameters\": {\n \"do_sample\": True,\n \"temperature\": 0.1,\n \"max_new_tokens\": 30,\n \"repetition_penalty\": 1.03,\n \"stop\": [\"\", \"###\"],\n },\n }\n body_bytes = json.dumps(payload).encode(\"utf-8\")\n\n # Invoke the endpoint\n response = self.runtime.invoke_endpoint(\n EndpointName=self.endpoint, ContentType=\"application/json\", Body=body_bytes\n )\n result = json.loads(response[\"Body\"].read().decode())\n import sys\n\n print(result[0][\"generated_text\"], file=sys.stderr)\n return result[0][\"generated_text\"][len(prompt) :]\n\n def _raw_gen_stream(self, baseself, model, messages, stream=True, tools=None, **kwargs):\n context = messages[0][\"content\"]\n user_question = messages[-1][\"content\"]\n prompt = f\"### Instruction \\n {user_question} \\n ### Context \\n {context} \\n ### Answer \\n\"\n\n # Construct payload for endpoint\n payload = {\n \"inputs\": prompt,\n \"stream\": True,\n \"parameters\": {\n \"do_sample\": True,\n \"temperature\": 0.1,\n \"max_new_tokens\": 512,\n \"repetition_penalty\": 1.03,\n \"stop\": [\"\", \"###\"],\n },\n }\n body_bytes = json.dumps(payload).encode(\"utf-8\")\n\n # Invoke the endpoint\n response = self.runtime.invoke_endpoint_with_response_stream(\n EndpointName=self.endpoint, ContentType=\"application/json\", Body=body_bytes\n )\n # result = json.loads(response['Body'].read().decode())\n event_stream = response[\"Body\"]\n start_json = b\"{\"\n for line in LineIterator(event_stream):\n if line != b\"\" and start_json in line:\n # print(line)\n data = json.loads(line[line.find(start_json) :].decode(\"utf-8\"))\n if data[\"token\"][\"text\"] not in [\"\", \"###\"]:\n print(data[\"token\"][\"text\"], end=\"\")\n yield data[\"token\"][\"text\"]\n" }, { "path": "application/logging.py", "content": "import datetime\nimport functools\nimport inspect\n\nimport logging\nimport uuid\nfrom typing import Any, Callable, Dict, Generator, List\n\nfrom application.core.mongo_db import MongoDB\nfrom application.core.settings import settings\n\nlogging.basicConfig(\n level=logging.INFO, format=\"%(asctime)s - %(levelname)s - %(message)s\"\n)\n\n\nclass LogContext:\n def __init__(self, endpoint, activity_id, user, api_key, query):\n self.endpoint = endpoint\n self.activity_id = activity_id\n self.user = user\n self.api_key = api_key\n self.query = query\n self.stacks = []\n\n\ndef build_stack_data(\n obj: Any,\n include_attributes: List[str] = None,\n exclude_attributes: List[str] = None,\n custom_data: Dict = None,\n) -> Dict:\n if obj is None:\n raise ValueError(\"The 'obj' parameter cannot be None\")\n data = {}\n if include_attributes is None:\n include_attributes = []\n for name, value in inspect.getmembers(obj):\n if (\n not name.startswith(\"_\")\n and not inspect.ismethod(value)\n and not inspect.isfunction(value)\n ):\n include_attributes.append(name)\n for attr_name in include_attributes:\n if exclude_attributes and attr_name in exclude_attributes:\n continue\n try:\n attr_value = getattr(obj, attr_name)\n if attr_value is not None:\n if isinstance(attr_value, (int, float, str, bool)):\n data[attr_name] = attr_value\n elif isinstance(attr_value, list):\n if all(isinstance(item, dict) for item in attr_value):\n data[attr_name] = attr_value\n elif all(hasattr(item, \"__dict__\") for item in attr_value):\n data[attr_name] = [item.__dict__ for item in attr_value]\n else:\n data[attr_name] = [str(item) for item in attr_value]\n elif isinstance(attr_value, dict):\n data[attr_name] = {k: str(v) for k, v in attr_value.items()}\n except AttributeError as e:\n logging.warning(f\"AttributeError while accessing {attr_name}: {e}\")\n except AttributeError:\n pass\n if custom_data:\n data.update(custom_data)\n return data\n\n\ndef log_activity() -> Callable:\n def decorator(func: Callable) -> Callable:\n @functools.wraps(func)\n def wrapper(*args: Any, **kwargs: Any) -> Any:\n activity_id = str(uuid.uuid4())\n data = build_stack_data(args[0])\n endpoint = data.get(\"endpoint\", \"\")\n user = data.get(\"user\", \"local\")\n api_key = data.get(\"user_api_key\", \"\")\n query = kwargs.get(\"query\", getattr(args[0], \"query\", \"\"))\n\n context = LogContext(endpoint, activity_id, user, api_key, query)\n kwargs[\"log_context\"] = context\n\n logging.info(\n f\"Starting activity: {endpoint} - {activity_id} - User: {user}\"\n )\n\n generator = func(*args, **kwargs)\n yield from _consume_and_log(generator, context)\n\n return wrapper\n\n return decorator\n\n\ndef _consume_and_log(generator: Generator, context: \"LogContext\"):\n try:\n for item in generator:\n yield item\n except Exception as e:\n logging.exception(f\"Error in {context.endpoint} - {context.activity_id}: {e}\")\n context.stacks.append({\"component\": \"error\", \"data\": {\"message\": str(e)}})\n _log_to_mongodb(\n endpoint=context.endpoint,\n activity_id=context.activity_id,\n user=context.user,\n api_key=context.api_key,\n query=context.query,\n stacks=context.stacks,\n level=\"error\",\n )\n raise\n finally:\n _log_to_mongodb(\n endpoint=context.endpoint,\n activity_id=context.activity_id,\n user=context.user,\n api_key=context.api_key,\n query=context.query,\n stacks=context.stacks,\n level=\"info\",\n )\n\n\ndef _log_to_mongodb(\n endpoint: str,\n activity_id: str,\n user: str,\n api_key: str,\n query: str,\n stacks: List[Dict],\n level: str,\n) -> None:\n try:\n mongo = MongoDB.get_client()\n db = mongo[settings.MONGO_DB_NAME]\n user_logs_collection = db[\"stack_logs\"]\n \n\n\n log_entry = {\n \"endpoint\": endpoint,\n \"id\": activity_id,\n \"level\": level,\n \"user\": user,\n \"api_key\": api_key,\n \"query\": query,\n \"stacks\": stacks,\n \"timestamp\": datetime.datetime.now(datetime.timezone.utc),\n }\n # clean up text fields to be no longer than 10000 characters\n for key, value in log_entry.items():\n if isinstance(value, str) and len(value) > 10000:\n log_entry[key] = value[:10000]\n \n user_logs_collection.insert_one(log_entry)\n logging.debug(f\"Logged activity to MongoDB: {activity_id}\")\n\n except Exception as e:\n logging.error(f\"Failed to log to MongoDB: {e}\", exc_info=True)\n" }, { "path": "application/parser/__init__.py", "content": "\n" }, { "path": "application/parser/chunking.py", "content": "import re\nfrom typing import List, Tuple\nimport logging\nfrom application.parser.schema.base import Document\nfrom application.utils import get_encoding\n\nlogger = logging.getLogger(__name__)\n\nclass Chunker:\n def __init__(\n self,\n chunking_strategy: str = \"classic_chunk\",\n max_tokens: int = 2000,\n min_tokens: int = 150,\n duplicate_headers: bool = False,\n ):\n if chunking_strategy not in [\"classic_chunk\"]:\n raise ValueError(f\"Unsupported chunking strategy: {chunking_strategy}\")\n self.chunking_strategy = chunking_strategy\n self.max_tokens = max_tokens\n self.min_tokens = min_tokens\n self.duplicate_headers = duplicate_headers\n self.encoding = get_encoding()\n\n def separate_header_and_body(self, text: str) -> Tuple[str, str]:\n header_pattern = r\"^(.*?\\n){3}\"\n match = re.match(header_pattern, text)\n if match:\n header = match.group(0)\n body = text[len(header):]\n else:\n header, body = \"\", text # No header, treat entire text as body\n return header, body\n\n\n \n def split_document(self, doc: Document) -> List[Document]:\n split_docs = []\n header, body = self.separate_header_and_body(doc.text)\n header_tokens = self.encoding.encode(header) if header else []\n body_tokens = self.encoding.encode(body)\n\n current_position = 0\n part_index = 0\n while current_position < len(body_tokens):\n end_position = current_position + self.max_tokens - len(header_tokens)\n chunk_tokens = (header_tokens + body_tokens[current_position:end_position]\n if self.duplicate_headers or part_index == 0 else body_tokens[current_position:end_position])\n chunk_text = self.encoding.decode(chunk_tokens)\n new_doc = Document(\n text=chunk_text,\n doc_id=f\"{doc.doc_id}-{part_index}\",\n embedding=doc.embedding,\n extra_info={**(doc.extra_info or {}), \"token_count\": len(chunk_tokens)}\n )\n split_docs.append(new_doc)\n current_position = end_position\n part_index += 1\n header_tokens = []\n return split_docs\n\n def classic_chunk(self, documents: List[Document]) -> List[Document]:\n processed_docs = []\n i = 0\n while i < len(documents):\n doc = documents[i]\n tokens = self.encoding.encode(doc.text)\n token_count = len(tokens)\n\n if self.min_tokens <= token_count <= self.max_tokens:\n doc.extra_info = doc.extra_info or {}\n doc.extra_info[\"token_count\"] = token_count\n processed_docs.append(doc)\n i += 1\n elif token_count < self.min_tokens:\n \n doc.extra_info = doc.extra_info or {}\n doc.extra_info[\"token_count\"] = token_count\n processed_docs.append(doc)\n i += 1\n else:\n # Split large documents\n processed_docs.extend(self.split_document(doc))\n i += 1\n return processed_docs\n\n def chunk(\n self,\n documents: List[Document]\n ) -> List[Document]:\n if self.chunking_strategy == \"classic_chunk\":\n return self.classic_chunk(documents)\n else:\n raise ValueError(\"Unsupported chunking strategy\")\n" }, { "path": "application/parser/connectors/__init__.py", "content": "\"\"\"\nExternal knowledge base connectors for DocsGPT.\n\nThis module contains connectors for external knowledge bases and document storage systems\nthat require authentication and specialized handling, separate from simple web scrapers.\n\"\"\"\n\nfrom .base import BaseConnectorAuth, BaseConnectorLoader\nfrom .connector_creator import ConnectorCreator\nfrom .google_drive import GoogleDriveAuth, GoogleDriveLoader\n\n__all__ = [\n 'BaseConnectorAuth',\n 'BaseConnectorLoader',\n 'ConnectorCreator',\n 'GoogleDriveAuth',\n 'GoogleDriveLoader'\n]\n" }, { "path": "application/parser/connectors/base.py", "content": "\"\"\"\nBase classes for external knowledge base connectors.\n\nThis module provides minimal abstract base classes that define the essential\ninterface for external knowledge base connectors.\n\"\"\"\n\nfrom abc import ABC, abstractmethod\nfrom typing import Any, Dict, List, Optional\n\nfrom application.parser.schema.base import Document\n\n\nclass BaseConnectorAuth(ABC):\n \"\"\"\n Abstract base class for connector authentication.\n \n Defines the minimal interface that all connector authentication\n implementations must follow.\n \"\"\"\n \n @abstractmethod\n def get_authorization_url(self, state: Optional[str] = None) -> str:\n \"\"\"\n Generate authorization URL for OAuth flows.\n \n Args:\n state: Optional state parameter for CSRF protection\n \n Returns:\n Authorization URL\n \"\"\"\n pass\n \n @abstractmethod\n def exchange_code_for_tokens(self, authorization_code: str) -> Dict[str, Any]:\n \"\"\"\n Exchange authorization code for access tokens.\n \n Args:\n authorization_code: Authorization code from OAuth callback\n \n Returns:\n Dictionary containing token information\n \"\"\"\n pass\n \n @abstractmethod\n def refresh_access_token(self, refresh_token: str) -> Dict[str, Any]:\n \"\"\"\n Refresh an expired access token.\n \n Args:\n refresh_token: Refresh token\n \n Returns:\n Dictionary containing refreshed token information\n \"\"\"\n pass\n \n @abstractmethod\n def is_token_expired(self, token_info: Dict[str, Any]) -> bool:\n \"\"\"\n Check if a token is expired.\n\n Args:\n token_info: Token information dictionary\n\n Returns:\n True if token is expired, False otherwise\n \"\"\"\n pass\n\n def sanitize_token_info(self, token_info: Dict[str, Any], **extra_fields) -> Dict[str, Any]:\n \"\"\"Extract the fields safe to persist in the session store.\n \"\"\"\n return {\n \"access_token\": token_info.get(\"access_token\"),\n \"refresh_token\": token_info.get(\"refresh_token\"),\n \"token_uri\": token_info.get(\"token_uri\"),\n \"expiry\": token_info.get(\"expiry\"),\n **extra_fields,\n }\n\n\nclass BaseConnectorLoader(ABC):\n \"\"\"\n Abstract base class for connector loaders.\n \n Defines the minimal interface that all connector loader\n implementations must follow.\n \"\"\"\n \n @abstractmethod\n def __init__(self, session_token: str):\n \"\"\"\n Initialize the connector loader.\n \n Args:\n session_token: Authentication session token\n \"\"\"\n pass\n \n @abstractmethod\n def load_data(self, inputs: Dict[str, Any]) -> List[Document]:\n \"\"\"\n Load documents from the external knowledge base.\n \n Args:\n inputs: Configuration dictionary containing:\n - file_ids: Optional list of specific file IDs to load\n - folder_ids: Optional list of folder IDs to browse/download\n - limit: Maximum number of items to return\n - list_only: If True, return metadata without content\n - recursive: Whether to recursively process folders\n \n Returns:\n List of Document objects\n \"\"\"\n pass\n \n @abstractmethod\n def download_to_directory(self, local_dir: str, source_config: Dict[str, Any] = None) -> Dict[str, Any]:\n \"\"\"\n Download files/folders to a local directory.\n \n Args:\n local_dir: Local directory path to download files to\n source_config: Configuration for what to download\n \n Returns:\n Dictionary containing download results:\n - files_downloaded: Number of files downloaded\n - directory_path: Path where files were downloaded\n - empty_result: Whether no files were downloaded\n - source_type: Type of connector\n - config_used: Configuration that was used\n - error: Error message if download failed (optional)\n \"\"\"\n pass\n" }, { "path": "application/parser/connectors/connector_creator.py", "content": "from application.parser.connectors.google_drive.loader import GoogleDriveLoader\nfrom application.parser.connectors.google_drive.auth import GoogleDriveAuth\nfrom application.parser.connectors.share_point.auth import SharePointAuth\nfrom application.parser.connectors.share_point.loader import SharePointLoader\n\n\nclass ConnectorCreator:\n \"\"\"\n Factory class for creating external knowledge base connectors and auth providers.\n\n These are different from remote loaders as they typically require\n authentication and connect to external document storage systems.\n \"\"\"\n\n connectors = {\n \"google_drive\": GoogleDriveLoader,\n \"share_point\": SharePointLoader,\n }\n\n auth_providers = {\n \"google_drive\": GoogleDriveAuth,\n \"share_point\": SharePointAuth,\n }\n\n @classmethod\n def create_connector(cls, connector_type, *args, **kwargs):\n \"\"\"\n Create a connector instance for the specified type.\n\n Args:\n connector_type: Type of connector to create (e.g., 'google_drive')\n *args, **kwargs: Arguments to pass to the connector constructor\n\n Returns:\n Connector instance\n\n Raises:\n ValueError: If connector type is not supported\n \"\"\"\n connector_class = cls.connectors.get(connector_type.lower())\n if not connector_class:\n raise ValueError(f\"No connector class found for type {connector_type}\")\n return connector_class(*args, **kwargs)\n\n @classmethod\n def create_auth(cls, connector_type):\n \"\"\"\n Create an auth provider instance for the specified connector type.\n\n Args:\n connector_type: Type of connector auth to create (e.g., 'google_drive')\n\n Returns:\n Auth provider instance\n\n Raises:\n ValueError: If connector type is not supported for auth\n \"\"\"\n auth_class = cls.auth_providers.get(connector_type.lower())\n if not auth_class:\n raise ValueError(f\"No auth class found for type {connector_type}\")\n return auth_class()\n\n @classmethod\n def get_supported_connectors(cls):\n \"\"\"\n Get list of supported connector types.\n\n Returns:\n List of supported connector type strings\n \"\"\"\n return list(cls.connectors.keys())\n\n @classmethod\n def is_supported(cls, connector_type):\n \"\"\"\n Check if a connector type is supported.\n\n Args:\n connector_type: Type of connector to check\n\n Returns:\n True if supported, False otherwise\n \"\"\"\n return connector_type.lower() in cls.connectors\n" }, { "path": "application/parser/connectors/google_drive/__init__.py", "content": "\"\"\"\nGoogle Drive connector for DocsGPT.\n\nThis module provides authentication and document loading capabilities for Google Drive.\n\"\"\"\n\nfrom .auth import GoogleDriveAuth\nfrom .loader import GoogleDriveLoader\n\n__all__ = ['GoogleDriveAuth', 'GoogleDriveLoader']\n" }, { "path": "application/parser/connectors/google_drive/auth.py", "content": "import logging\nimport datetime\nfrom typing import Optional, Dict, Any\n\nfrom google.oauth2.credentials import Credentials\nfrom google_auth_oauthlib.flow import Flow\nfrom googleapiclient.discovery import build\nfrom googleapiclient.errors import HttpError\n\nfrom application.core.settings import settings\nfrom application.parser.connectors.base import BaseConnectorAuth\n\n\nclass GoogleDriveAuth(BaseConnectorAuth):\n \"\"\"\n Handles Google OAuth 2.0 authentication for Google Drive access.\n \"\"\"\n \n SCOPES = [\n 'https://www.googleapis.com/auth/drive.file'\n ]\n \n def __init__(self):\n self.client_id = settings.GOOGLE_CLIENT_ID\n self.client_secret = settings.GOOGLE_CLIENT_SECRET\n self.redirect_uri = f\"{settings.CONNECTOR_REDIRECT_BASE_URI}\"\n \n if not self.client_id or not self.client_secret:\n raise ValueError(\"Google OAuth credentials not configured. Please set GOOGLE_CLIENT_ID and GOOGLE_CLIENT_SECRET in settings.\")\n\n\n\n def get_authorization_url(self, state: Optional[str] = None) -> str:\n try:\n flow = Flow.from_client_config(\n {\n \"web\": {\n \"client_id\": self.client_id,\n \"client_secret\": self.client_secret,\n \"auth_uri\": \"https://accounts.google.com/o/oauth2/auth\",\n \"token_uri\": \"https://oauth2.googleapis.com/token\",\n \"redirect_uris\": [self.redirect_uri]\n }\n },\n scopes=self.SCOPES\n )\n flow.redirect_uri = self.redirect_uri\n \n authorization_url, _ = flow.authorization_url(\n access_type='offline',\n prompt='consent',\n include_granted_scopes='false',\n state=state\n )\n \n return authorization_url\n \n except Exception as e:\n logging.error(f\"Error generating authorization URL: {e}\")\n raise\n \n def exchange_code_for_tokens(self, authorization_code: str) -> Dict[str, Any]:\n try:\n if not authorization_code:\n raise ValueError(\"Authorization code is required\")\n\n flow = Flow.from_client_config(\n {\n \"web\": {\n \"client_id\": self.client_id,\n \"client_secret\": self.client_secret,\n \"auth_uri\": \"https://accounts.google.com/o/oauth2/auth\",\n \"token_uri\": \"https://oauth2.googleapis.com/token\",\n \"redirect_uris\": [self.redirect_uri]\n }\n },\n scopes=self.SCOPES\n )\n flow.redirect_uri = self.redirect_uri\n\n flow.fetch_token(code=authorization_code)\n\n credentials = flow.credentials\n\n if not credentials.refresh_token:\n logging.warning(\"OAuth flow did not return a refresh_token.\")\n if not credentials.token:\n raise ValueError(\"OAuth flow did not return an access token\")\n\n if not credentials.token_uri:\n credentials.token_uri = \"https://oauth2.googleapis.com/token\"\n\n if not credentials.client_id:\n credentials.client_id = self.client_id\n\n if not credentials.client_secret:\n credentials.client_secret = self.client_secret\n\n if not credentials.refresh_token:\n raise ValueError(\n \"No refresh token received. This typically happens when offline access wasn't granted. \"\n )\n\n return {\n 'access_token': credentials.token,\n 'refresh_token': credentials.refresh_token,\n 'token_uri': credentials.token_uri,\n 'client_id': credentials.client_id,\n 'client_secret': credentials.client_secret,\n 'scopes': credentials.scopes,\n 'expiry': credentials.expiry.isoformat() if credentials.expiry else None\n }\n\n except Exception as e:\n logging.error(f\"Error exchanging code for tokens: {e}\")\n raise\n \n def refresh_access_token(self, refresh_token: str) -> Dict[str, Any]:\n try:\n if not refresh_token:\n raise ValueError(\"Refresh token is required\")\n\n credentials = Credentials(\n token=None,\n refresh_token=refresh_token,\n token_uri=\"https://oauth2.googleapis.com/token\",\n client_id=self.client_id,\n client_secret=self.client_secret\n )\n\n from google.auth.transport.requests import Request\n credentials.refresh(Request())\n\n return {\n 'access_token': credentials.token,\n 'refresh_token': refresh_token,\n 'token_uri': credentials.token_uri,\n 'client_id': credentials.client_id,\n 'client_secret': credentials.client_secret,\n 'scopes': credentials.scopes,\n 'expiry': credentials.expiry.isoformat() if credentials.expiry else None\n }\n except Exception as e:\n logging.error(f\"Error refreshing access token: {e}\", exc_info=True)\n raise\n \n def create_credentials_from_token_info(self, token_info: Dict[str, Any]) -> Credentials:\n from application.core.settings import settings\n\n access_token = token_info.get('access_token')\n if not access_token:\n raise ValueError(\"No access token found in token_info\")\n\n credentials = Credentials(\n token=access_token,\n refresh_token=token_info.get('refresh_token'),\n token_uri= 'https://oauth2.googleapis.com/token',\n client_id=settings.GOOGLE_CLIENT_ID,\n client_secret=settings.GOOGLE_CLIENT_SECRET,\n scopes=token_info.get('scopes', ['https://www.googleapis.com/auth/drive.readonly'])\n )\n\n if not credentials.token:\n raise ValueError(\"Credentials created without valid access token\")\n\n return credentials\n \n def build_drive_service(self, credentials: Credentials):\n try:\n if not credentials:\n raise ValueError(\"No credentials provided\")\n\n if not credentials.token and not credentials.refresh_token:\n raise ValueError(\"No access token or refresh token available. User must re-authorize with offline access.\")\n\n needs_refresh = credentials.expired or not credentials.token\n if needs_refresh:\n if credentials.refresh_token:\n try:\n from google.auth.transport.requests import Request\n credentials.refresh(Request())\n except Exception as refresh_error:\n raise ValueError(f\"Failed to refresh credentials: {refresh_error}\")\n else:\n raise ValueError(\"No access token or refresh token available. User must re-authorize with offline access.\")\n\n return build('drive', 'v3', credentials=credentials)\n\n except HttpError as e:\n raise ValueError(f\"Failed to build Google Drive service: HTTP {e.resp.status}\")\n except Exception as e:\n raise ValueError(f\"Failed to build Google Drive service: {str(e)}\")\n \n def is_token_expired(self, token_info):\n if 'expiry' in token_info and token_info['expiry']:\n try:\n from dateutil import parser\n # Google Drive provides timezone-aware ISO8601 dates\n expiry_dt = parser.parse(token_info['expiry'])\n current_time = datetime.datetime.now(datetime.timezone.utc)\n return current_time >= expiry_dt - datetime.timedelta(seconds=60)\n except Exception:\n return True\n\n if 'access_token' in token_info and token_info['access_token']:\n return False\n\n return True\n \n def get_token_info_from_session(self, session_token: str) -> Dict[str, Any]:\n try:\n from application.core.mongo_db import MongoDB\n from application.core.settings import settings\n\n mongo = MongoDB.get_client()\n db = mongo[settings.MONGO_DB_NAME]\n \n sessions_collection = db[\"connector_sessions\"]\n session = sessions_collection.find_one({\"session_token\": session_token}) \n if not session:\n raise ValueError(f\"Invalid session token: {session_token}\")\n\n if \"token_info\" not in session:\n raise ValueError(\"Session missing token information\")\n\n token_info = session[\"token_info\"]\n if not token_info:\n raise ValueError(\"Invalid token information\")\n\n required_fields = [\"access_token\", \"refresh_token\"]\n missing_fields = [field for field in required_fields if field not in token_info or not token_info.get(field)]\n if missing_fields:\n raise ValueError(f\"Missing required token fields: {missing_fields}\")\n\n if 'token_uri' not in token_info:\n token_info['token_uri'] = 'https://oauth2.googleapis.com/token'\n\n return token_info\n\n except Exception as e:\n raise ValueError(f\"Failed to retrieve Google Drive token information: {str(e)}\")\n\n def validate_credentials(self, credentials: Credentials) -> bool:\n \"\"\"\n Validate Google Drive credentials by making a test API call.\n\n Args:\n credentials: Google credentials object\n\n Returns:\n True if credentials are valid, False otherwise\n \"\"\"\n try:\n service = self.build_drive_service(credentials)\n service.about().get(fields=\"user\").execute()\n return True\n\n except HttpError as e:\n logging.error(f\"HTTP error validating credentials: {e}\")\n return False\n except Exception as e:\n logging.error(f\"Error validating credentials: {e}\")\n return False\n" }, { "path": "application/parser/connectors/google_drive/loader.py", "content": "\"\"\"\nGoogle Drive loader for DocsGPT.\nLoads documents from Google Drive using Google Drive API.\n\"\"\"\n\nimport io\nimport logging\nimport os\nfrom typing import List, Dict, Any, Optional\n\nfrom googleapiclient.http import MediaIoBaseDownload\nfrom googleapiclient.errors import HttpError\n\nfrom application.parser.connectors.base import BaseConnectorLoader\nfrom application.parser.connectors.google_drive.auth import GoogleDriveAuth\nfrom application.parser.schema.base import Document\n\n\nclass GoogleDriveLoader(BaseConnectorLoader):\n\n SUPPORTED_MIME_TYPES = {\n 'application/pdf': '.pdf',\n 'application/vnd.google-apps.document': '.docx',\n 'application/vnd.google-apps.presentation': '.pptx',\n 'application/vnd.google-apps.spreadsheet': '.xlsx',\n 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': '.docx',\n 'application/vnd.openxmlformats-officedocument.presentationml.presentation': '.pptx',\n 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': '.xlsx',\n 'application/msword': '.doc',\n 'application/vnd.ms-powerpoint': '.ppt',\n 'application/vnd.ms-excel': '.xls',\n 'text/plain': '.txt',\n 'text/csv': '.csv',\n 'text/html': '.html',\n 'text/markdown': '.md',\n 'text/x-rst': '.rst',\n 'application/json': '.json',\n 'application/epub+zip': '.epub',\n 'application/rtf': '.rtf',\n 'image/jpeg': '.jpg',\n 'image/jpg': '.jpg',\n 'image/png': '.png',\n }\n\n EXPORT_FORMATS = {\n 'application/vnd.google-apps.document': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',\n 'application/vnd.google-apps.presentation': 'application/vnd.openxmlformats-officedocument.presentationml.presentation',\n 'application/vnd.google-apps.spreadsheet': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'\n }\n\n def __init__(self, session_token: str):\n self.auth = GoogleDriveAuth()\n self.session_token = session_token\n\n token_info = self.auth.get_token_info_from_session(session_token)\n self.credentials = self.auth.create_credentials_from_token_info(token_info)\n\n try:\n self.service = self.auth.build_drive_service(self.credentials)\n except Exception as e:\n logging.warning(f\"Could not build Google Drive service: {e}\")\n self.service = None\n\n self.next_page_token = None\n\n\n\n def _process_file(self, file_metadata: Dict[str, Any], load_content: bool = True) -> Optional[Document]:\n try:\n file_id = file_metadata.get('id')\n file_name = file_metadata.get('name', 'Unknown')\n mime_type = file_metadata.get('mimeType', 'application/octet-stream')\n\n if mime_type not in self.SUPPORTED_MIME_TYPES and not mime_type.startswith('application/vnd.google-apps.'):\n return None\n if mime_type not in self.SUPPORTED_MIME_TYPES and not mime_type.startswith('application/vnd.google-apps.'):\n logging.info(f\"Skipping unsupported file type: {mime_type} for file {file_name}\")\n return None\n # Google Drive provides timezone-aware ISO8601 dates\n doc_metadata = {\n 'file_name': file_name,\n 'mime_type': mime_type,\n 'size': file_metadata.get('size', None),\n 'created_time': file_metadata.get('createdTime'),\n 'modified_time': file_metadata.get('modifiedTime'),\n 'parents': file_metadata.get('parents', []),\n 'source': 'google_drive'\n }\n\n if not load_content:\n return Document(\n text=\"\",\n doc_id=file_id,\n extra_info=doc_metadata\n )\n\n content = self._download_file_content(file_id, mime_type)\n if content is None:\n logging.warning(f\"Could not load content for file {file_name} ({file_id})\")\n return None\n\n return Document(\n text=content,\n doc_id=file_id,\n extra_info=doc_metadata\n )\n\n except Exception as e:\n logging.error(f\"Error processing file: {e}\")\n return None\n\n def load_data(self, inputs: Dict[str, Any]) -> List[Document]:\n session_token = inputs.get('session_token')\n if session_token and session_token != self.session_token:\n logging.warning(\"Session token in inputs differs from loader's session token. Using loader's session token.\")\n self.config = inputs\n\n try:\n documents: List[Document] = []\n\n folder_id = inputs.get('folder_id')\n file_ids = inputs.get('file_ids', [])\n limit = inputs.get('limit', 100)\n list_only = inputs.get('list_only', False)\n load_content = not list_only\n page_token = inputs.get('page_token')\n search_query = inputs.get('search_query')\n self.next_page_token = None\n\n if file_ids:\n # Specific files requested: load them\n for file_id in file_ids:\n try:\n doc = self._load_file_by_id(file_id, load_content=load_content)\n if doc:\n if not search_query or (\n search_query.lower() in doc.extra_info.get('file_name', '').lower()\n ):\n documents.append(doc)\n elif hasattr(self, '_credential_refreshed') and self._credential_refreshed:\n self._credential_refreshed = False\n logging.info(f\"Retrying load of file {file_id} after credential refresh\")\n doc = self._load_file_by_id(file_id, load_content=load_content)\n if doc and (\n not search_query or \n search_query.lower() in doc.extra_info.get('file_name', '').lower()\n ):\n documents.append(doc)\n except Exception as e:\n logging.error(f\"Error loading file {file_id}: {e}\")\n continue\n else:\n # Browsing mode: list immediate children of provided folder or root\n parent_id = folder_id if folder_id else 'root'\n documents = self._list_items_in_parent(\n parent_id, \n limit=limit, \n load_content=load_content, \n page_token=page_token,\n search_query=search_query\n )\n\n logging.info(f\"Loaded {len(documents)} documents from Google Drive\")\n return documents\n\n except Exception as e:\n logging.error(f\"Error loading data from Google Drive: {e}\", exc_info=True)\n raise\n\n\n\n def _load_file_by_id(self, file_id: str, load_content: bool = True) -> Optional[Document]:\n self._ensure_service()\n\n try:\n file_metadata = self.service.files().get(\n fileId=file_id,\n fields='id,name,mimeType,size,createdTime,modifiedTime,parents'\n ).execute()\n\n return self._process_file(file_metadata, load_content=load_content)\n\n except HttpError as e:\n logging.error(f\"HTTP error loading file {file_id}: {e.resp.status} - {e.content}\")\n\n if e.resp.status in [401, 403]:\n if hasattr(self.credentials, 'refresh_token') and self.credentials.refresh_token:\n try:\n from google.auth.transport.requests import Request\n self.credentials.refresh(Request())\n self._ensure_service()\n return None\n except Exception as refresh_error:\n raise ValueError(f\"Authentication failed and could not be refreshed: {refresh_error}\")\n else:\n raise ValueError(\"Authentication failed and cannot be refreshed: missing refresh_token\")\n\n return None\n except Exception as e:\n logging.error(f\"Error loading file {file_id}: {e}\")\n return None\n\n\n def _list_items_in_parent(self, parent_id: str, limit: int = 100, load_content: bool = False, page_token: Optional[str] = None, search_query: Optional[str] = None) -> List[Document]:\n self._ensure_service()\n\n documents: List[Document] = []\n\n try:\n query = f\"'{parent_id}' in parents and trashed=false\"\n\n if search_query:\n safe_search = search_query.replace(\"'\", \"\\\\'\")\n query += f\" and name contains '{safe_search}'\"\n\n next_token_out: Optional[str] = None\n\n while True:\n page_size = 100\n if limit:\n remaining = max(0, limit - len(documents))\n if remaining == 0:\n break\n page_size = min(100, remaining)\n\n results = self.service.files().list(\n q=query,\n fields='nextPageToken,files(id,name,mimeType,size,createdTime,modifiedTime,parents)',\n pageToken=page_token,\n pageSize=page_size,\n orderBy='name'\n ).execute()\n\n items = results.get('files', [])\n for item in items:\n mime_type = item.get('mimeType')\n if mime_type == 'application/vnd.google-apps.folder':\n doc_metadata = {\n 'file_name': item.get('name', 'Unknown'),\n 'mime_type': mime_type,\n 'size': item.get('size', None),\n 'created_time': item.get('createdTime'),\n 'modified_time': item.get('modifiedTime'),\n 'parents': item.get('parents', []),\n 'source': 'google_drive',\n 'is_folder': True\n }\n documents.append(Document(text=\"\", doc_id=item.get('id'), extra_info=doc_metadata))\n else:\n doc = self._process_file(item, load_content=load_content)\n if doc:\n documents.append(doc)\n\n if limit and len(documents) >= limit:\n self.next_page_token = results.get('nextPageToken')\n return documents\n\n page_token = results.get('nextPageToken')\n next_token_out = page_token\n if not page_token:\n break\n\n self.next_page_token = next_token_out\n return documents\n except Exception as e:\n logging.error(f\"Error listing items under parent {parent_id}: {e}\")\n return documents\n\n\n\n\n def _download_file_content(self, file_id: str, mime_type: str) -> Optional[str]:\n if not self.credentials.token:\n logging.warning(\"No access token in credentials, attempting to refresh\")\n if hasattr(self.credentials, 'refresh_token') and self.credentials.refresh_token:\n try:\n from google.auth.transport.requests import Request\n self.credentials.refresh(Request())\n logging.info(\"Credentials refreshed successfully\")\n self._ensure_service()\n except Exception as e:\n logging.error(f\"Failed to refresh credentials: {e}\")\n raise ValueError(\"Authentication failed and cannot be refreshed: missing or invalid refresh_token\")\n else:\n logging.error(\"No access token and no refresh_token available\")\n raise ValueError(\"Authentication failed and cannot be refreshed: missing refresh_token\")\n\n if self.credentials.expired:\n logging.warning(\"Credentials are expired, attempting to refresh\")\n if hasattr(self.credentials, 'refresh_token') and self.credentials.refresh_token:\n try:\n from google.auth.transport.requests import Request\n self.credentials.refresh(Request())\n logging.info(\"Credentials refreshed successfully\")\n self._ensure_service()\n except Exception as e:\n logging.error(f\"Failed to refresh expired credentials: {e}\")\n raise ValueError(\"Authentication failed and cannot be refreshed: expired credentials\")\n else:\n logging.error(\"Credentials expired and no refresh_token available\")\n raise ValueError(\"Authentication failed and cannot be refreshed: missing refresh_token\")\n\n try:\n if mime_type in self.EXPORT_FORMATS:\n export_mime_type = self.EXPORT_FORMATS[mime_type]\n request = self.service.files().export_media(\n fileId=file_id,\n mimeType=export_mime_type\n )\n else:\n request = self.service.files().get_media(fileId=file_id)\n\n file_io = io.BytesIO()\n downloader = MediaIoBaseDownload(file_io, request)\n\n done = False\n while done is False:\n try:\n _, done = downloader.next_chunk()\n except HttpError as e:\n logging.error(f\"HTTP error downloading file {file_id}: {e.resp.status} - {e.content}\")\n return None\n except Exception as e:\n logging.error(f\"Error during download of file {file_id}: {e}\")\n return None\n\n content_bytes = file_io.getvalue()\n\n try:\n return content_bytes.decode('utf-8')\n except UnicodeDecodeError:\n logging.error(f\"Could not decode file {file_id} as text\")\n return None\n\n except HttpError as e:\n logging.error(f\"HTTP error downloading file {file_id}: {e.resp.status} - {e.content}\")\n\n if e.resp.status in [401, 403]:\n logging.error(f\"Authentication error downloading file {file_id}\")\n\n if hasattr(self.credentials, 'refresh_token') and self.credentials.refresh_token:\n logging.info(f\"Attempting to refresh credentials for file {file_id}\")\n try:\n from google.auth.transport.requests import Request\n self.credentials.refresh(Request())\n logging.info(\"Credentials refreshed successfully\")\n self._credential_refreshed = True\n self._ensure_service()\n return None\n except Exception as refresh_error:\n logging.error(f\"Error refreshing credentials: {refresh_error}\")\n raise ValueError(f\"Authentication failed and could not be refreshed: {refresh_error}\")\n else:\n logging.error(\"Cannot refresh credentials: missing refresh_token\")\n raise ValueError(\"Authentication failed and cannot be refreshed: missing refresh_token\")\n\n return None\n except Exception as e:\n logging.error(f\"Error downloading file {file_id}: {e}\")\n return None\n\n\n def _download_file_to_directory(self, file_id: str, local_dir: str) -> bool:\n try:\n self._ensure_service()\n return self._download_single_file(file_id, local_dir)\n except Exception as e:\n logging.error(f\"Error downloading file {file_id}: {e}\", exc_info=True)\n return False\n\n def _ensure_service(self):\n if not self.service:\n try:\n self.service = self.auth.build_drive_service(self.credentials)\n except Exception as e:\n raise ValueError(f\"Cannot access Google Drive: {e}\")\n\n def _download_single_file(self, file_id: str, local_dir: str) -> bool:\n file_metadata = self.service.files().get(\n fileId=file_id,\n fields='name,mimeType'\n ).execute()\n\n file_name = file_metadata['name']\n mime_type = file_metadata['mimeType']\n\n if mime_type not in self.SUPPORTED_MIME_TYPES and not mime_type.startswith('application/vnd.google-apps.'):\n return False\n\n os.makedirs(local_dir, exist_ok=True)\n full_path = os.path.join(local_dir, file_name)\n\n if mime_type in self.EXPORT_FORMATS:\n export_mime_type = self.EXPORT_FORMATS[mime_type]\n request = self.service.files().export_media(\n fileId=file_id,\n mimeType=export_mime_type\n )\n extension = self._get_extension_for_mime_type(export_mime_type)\n if not full_path.endswith(extension):\n full_path += extension\n else:\n request = self.service.files().get_media(fileId=file_id)\n\n with open(full_path, 'wb') as f:\n downloader = MediaIoBaseDownload(f, request)\n done = False\n while not done:\n _, done = downloader.next_chunk()\n\n return True\n\n def _download_folder_recursive(self, folder_id: str, local_dir: str, recursive: bool = True) -> int:\n files_downloaded = 0\n try:\n os.makedirs(local_dir, exist_ok=True)\n\n query = f\"'{folder_id}' in parents and trashed=false\"\n page_token = None\n\n while True:\n results = self.service.files().list(\n q=query,\n fields='nextPageToken, files(id, name, mimeType)',\n pageToken=page_token,\n pageSize=1000\n ).execute()\n\n items = results.get('files', [])\n logging.info(f\"Found {len(items)} items in folder {folder_id}\")\n\n for item in items:\n item_name = item['name']\n item_id = item['id']\n mime_type = item['mimeType']\n\n if mime_type == 'application/vnd.google-apps.folder':\n if recursive:\n # Create subfolder and recurse\n subfolder_path = os.path.join(local_dir, item_name)\n os.makedirs(subfolder_path, exist_ok=True)\n subfolder_files = self._download_folder_recursive(\n item_id,\n subfolder_path,\n recursive\n )\n files_downloaded += subfolder_files\n logging.info(f\"Downloaded {subfolder_files} files from subfolder {item_name}\")\n else:\n # Download file\n success = self._download_single_file(item_id, local_dir)\n if success:\n files_downloaded += 1\n logging.info(f\"Downloaded file: {item_name}\")\n else:\n logging.warning(f\"Failed to download file: {item_name}\")\n\n page_token = results.get('nextPageToken')\n if not page_token:\n break\n\n return files_downloaded\n\n except Exception as e:\n logging.error(f\"Error in _download_folder_recursive for folder {folder_id}: {e}\", exc_info=True)\n return files_downloaded\n\n def _get_extension_for_mime_type(self, mime_type: str) -> str:\n extensions = {\n 'application/pdf': '.pdf',\n 'text/plain': '.txt',\n 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': '.docx',\n 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': '.xlsx',\n 'application/vnd.openxmlformats-officedocument.presentationml.presentation': '.pptx',\n 'text/html': '.html',\n 'text/markdown': '.md',\n }\n return extensions.get(mime_type, '.bin')\n\n def _download_folder_contents(self, folder_id: str, local_dir: str, recursive: bool = True) -> int:\n try:\n self._ensure_service()\n return self._download_folder_recursive(folder_id, local_dir, recursive)\n except Exception as e:\n logging.error(f\"Error downloading folder {folder_id}: {e}\", exc_info=True)\n return 0\n\n def download_to_directory(self, local_dir: str, source_config: dict = None) -> dict:\n if source_config is None:\n source_config = {}\n\n config = source_config if source_config else getattr(self, 'config', {})\n files_downloaded = 0\n\n try:\n folder_ids = config.get('folder_ids', [])\n file_ids = config.get('file_ids', [])\n recursive = config.get('recursive', True)\n\n self._ensure_service()\n\n if file_ids:\n if isinstance(file_ids, str):\n file_ids = [file_ids]\n\n for file_id in file_ids:\n if self._download_file_to_directory(file_id, local_dir):\n files_downloaded += 1\n\n # Process folders\n if folder_ids:\n if isinstance(folder_ids, str):\n folder_ids = [folder_ids]\n\n for folder_id in folder_ids:\n try:\n folder_metadata = self.service.files().get(\n fileId=folder_id,\n fields='name'\n ).execute()\n folder_name = folder_metadata.get('name', '')\n folder_path = os.path.join(local_dir, folder_name)\n os.makedirs(folder_path, exist_ok=True)\n\n folder_files = self._download_folder_recursive(\n folder_id,\n folder_path,\n recursive\n )\n files_downloaded += folder_files\n logging.info(f\"Downloaded {folder_files} files from folder {folder_name}\")\n except Exception as e:\n logging.error(f\"Error downloading folder {folder_id}: {e}\", exc_info=True)\n\n if not file_ids and not folder_ids:\n raise ValueError(\"No folder_ids or file_ids provided for download\")\n\n return {\n \"files_downloaded\": files_downloaded,\n \"directory_path\": local_dir,\n \"empty_result\": files_downloaded == 0,\n \"source_type\": \"google_drive\",\n \"config_used\": config\n }\n\n except Exception as e:\n return {\n \"files_downloaded\": files_downloaded,\n \"directory_path\": local_dir,\n \"empty_result\": True,\n \"source_type\": \"google_drive\",\n \"config_used\": config,\n \"error\": str(e)\n }\n" }, { "path": "application/parser/connectors/share_point/__init__.py", "content": "\"\"\"\nShare Point connector package for DocsGPT.\n\nThis module provides authentication and document loading capabilities for Share Point.\n\"\"\"\n\nfrom .auth import SharePointAuth\nfrom .loader import SharePointLoader\n\n__all__ = ['SharePointAuth', 'SharePointLoader']" }, { "path": "application/parser/connectors/share_point/auth.py", "content": "import datetime\nimport logging\nfrom typing import Optional, Dict, Any\n\nfrom msal import ConfidentialClientApplication\n\nfrom application.core.settings import settings\nfrom application.parser.connectors.base import BaseConnectorAuth\n\nlogger = logging.getLogger(__name__)\n\n\nclass SharePointAuth(BaseConnectorAuth):\n \"\"\"\n Handles Microsoft OAuth 2.0 authentication for SharePoint/OneDrive.\n\n Note: Files.Read scope allows access to files the user has granted access to,\n similar to Google Drive's drive.file scope.\n \"\"\"\n\n SCOPES = [\n \"Files.Read\",\n \"Sites.Read.All\",\n \"User.Read\",\n ]\n\n def __init__(self):\n self.client_id = settings.MICROSOFT_CLIENT_ID\n self.client_secret = settings.MICROSOFT_CLIENT_SECRET\n\n if not self.client_id:\n raise ValueError(\n \"Microsoft OAuth credentials not configured. Please set MICROSOFT_CLIENT_ID in settings.\"\n )\n \n if not self.client_secret:\n raise ValueError(\n \"Microsoft OAuth credentials not configured. Please set MICROSOFT_CLIENT_SECRET in settings.\"\n )\n\n self.redirect_uri = settings.CONNECTOR_REDIRECT_BASE_URI\n self.tenant_id = settings.MICROSOFT_TENANT_ID\n self.authority = getattr(settings, \"MICROSOFT_AUTHORITY\", f\"https://login.microsoftonline.com/{self.tenant_id}\")\n\n self.auth_app = ConfidentialClientApplication(\n client_id=self.client_id,\n client_credential=self.client_secret,\n authority=self.authority\n )\n\n def get_authorization_url(self, state: Optional[str] = None) -> str:\n return self.auth_app.get_authorization_request_url(\n scopes=self.SCOPES, state=state, redirect_uri=self.redirect_uri\n )\n\n def exchange_code_for_tokens(self, authorization_code: str) -> Dict[str, Any]:\n result = self.auth_app.acquire_token_by_authorization_code(\n code=authorization_code,\n scopes=self.SCOPES,\n redirect_uri=self.redirect_uri\n )\n\n if \"error\" in result:\n logger.error(\"Token exchange failed: %s\", result.get(\"error_description\"))\n raise ValueError(f\"Error acquiring token: {result.get('error_description')}\")\n\n return self.map_token_response(result)\n\n def refresh_access_token(self, refresh_token: str) -> Dict[str, Any]:\n result = self.auth_app.acquire_token_by_refresh_token(refresh_token=refresh_token, scopes=self.SCOPES)\n\n if \"error\" in result:\n logger.error(\"Token refresh failed: %s\", result.get(\"error_description\"))\n raise ValueError(f\"Error refreshing token: {result.get('error_description')}\")\n\n return self.map_token_response(result)\n\n def get_token_info_from_session(self, session_token: str) -> Dict[str, Any]:\n try:\n from application.core.mongo_db import MongoDB\n from application.core.settings import settings\n\n mongo = MongoDB.get_client()\n db = mongo[settings.MONGO_DB_NAME]\n\n sessions_collection = db[\"connector_sessions\"]\n session = sessions_collection.find_one({\"session_token\": session_token})\n\n if not session:\n raise ValueError(f\"Invalid session token: {session_token}\")\n\n if \"token_info\" not in session:\n raise ValueError(\"Session missing token information\")\n\n token_info = session[\"token_info\"]\n if not token_info:\n raise ValueError(\"Invalid token information\")\n\n required_fields = [\"access_token\", \"refresh_token\"]\n missing_fields = [field for field in required_fields if field not in token_info or not token_info.get(field)]\n if missing_fields:\n raise ValueError(f\"Missing required token fields: {missing_fields}\")\n\n if 'token_uri' not in token_info:\n token_info['token_uri'] = f\"https://login.microsoftonline.com/{settings.MICROSOFT_TENANT_ID}/oauth2/v2.0/token\"\n\n return token_info\n\n except Exception as e:\n logger.error(\"Failed to retrieve token from session: %s\", e)\n raise ValueError(f\"Failed to retrieve SharePoint token information: {str(e)}\")\n\n def is_token_expired(self, token_info: Dict[str, Any]) -> bool:\n if not token_info:\n return True\n\n expiry_timestamp = token_info.get(\"expiry\")\n\n if expiry_timestamp is None:\n return True\n\n current_timestamp = int(datetime.datetime.now().timestamp())\n return (expiry_timestamp - current_timestamp) < 60\n\n def sanitize_token_info(self, token_info: Dict[str, Any], **extra_fields) -> Dict[str, Any]:\n return super().sanitize_token_info(\n token_info,\n allows_shared_content=token_info.get(\"allows_shared_content\", False),\n **extra_fields,\n )\n\n PERSONAL_ACCOUNT_TENANT_ID = \"9188040d-6c67-4c5b-b112-36a304b66dad\"\n\n def _allows_shared_content(self, id_token_claims: Dict[str, Any]) -> bool:\n \"\"\"Return True when the account is a work/school tenant that can access SharePoint shared content.\"\"\"\n tid = id_token_claims.get(\"tid\", \"\")\n return bool(tid) and tid != self.PERSONAL_ACCOUNT_TENANT_ID\n\n def map_token_response(self, result) -> Dict[str, Any]:\n claims = result.get(\"id_token_claims\", {})\n return {\n \"access_token\": result.get(\"access_token\"),\n \"refresh_token\": result.get(\"refresh_token\"),\n \"token_uri\": claims.get(\"iss\"),\n \"scopes\": result.get(\"scope\"),\n \"expiry\": claims.get(\"exp\"),\n \"allows_shared_content\": self._allows_shared_content(claims),\n \"user_info\": {\n \"name\": claims.get(\"name\"),\n \"email\": claims.get(\"preferred_username\"),\n },\n }\n" }, { "path": "application/parser/connectors/share_point/loader.py", "content": "\"\"\"\nSharePoint/OneDrive loader for DocsGPT.\nLoads documents from SharePoint/OneDrive using Microsoft Graph API.\n\"\"\"\n\nimport functools\nimport logging\nimport os\nfrom typing import List, Dict, Any, Optional, Tuple\nfrom urllib.parse import quote\n\nimport requests\n\nfrom application.parser.connectors.base import BaseConnectorLoader\nfrom application.parser.connectors.share_point.auth import SharePointAuth\nfrom application.parser.schema.base import Document\n\n\ndef _retry_on_auth_failure(func):\n \"\"\"Retry once after refreshing the access token on 401/403 responses.\"\"\"\n @functools.wraps(func)\n def wrapper(self, *args, **kwargs):\n try:\n return func(self, *args, **kwargs)\n except requests.exceptions.HTTPError as e:\n if e.response is not None and e.response.status_code in (401, 403):\n logging.info(f\"Auth failure in {func.__name__}, refreshing token and retrying\")\n try:\n new_token_info = self.auth.refresh_access_token(self.refresh_token)\n self.access_token = new_token_info.get('access_token')\n except Exception as refresh_error:\n raise ValueError(\n f\"Authentication failed and could not be refreshed: {refresh_error}\"\n ) from e\n return func(self, *args, **kwargs)\n raise\n return wrapper\n\n\nclass SharePointLoader(BaseConnectorLoader):\n\n SUPPORTED_MIME_TYPES = {\n 'application/pdf': '.pdf',\n 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': '.docx',\n 'application/vnd.openxmlformats-officedocument.presentationml.presentation': '.pptx',\n 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': '.xlsx',\n 'application/msword': '.doc',\n 'application/vnd.ms-powerpoint': '.ppt',\n 'application/vnd.ms-excel': '.xls',\n 'text/plain': '.txt',\n 'text/csv': '.csv',\n 'text/html': '.html',\n 'text/markdown': '.md',\n 'text/x-rst': '.rst',\n 'application/json': '.json',\n 'application/epub+zip': '.epub',\n 'application/rtf': '.rtf',\n 'image/jpeg': '.jpg',\n 'image/png': '.png',\n }\n\n EXTENSION_TO_MIME = {v: k for k, v in SUPPORTED_MIME_TYPES.items()}\n\n GRAPH_API_BASE = \"https://graph.microsoft.com/v1.0\"\n\n def __init__(self, session_token: str):\n self.auth = SharePointAuth()\n self.session_token = session_token\n\n token_info = self.auth.get_token_info_from_session(session_token)\n self.access_token = token_info.get('access_token')\n self.refresh_token = token_info.get('refresh_token')\n self.allows_shared_content = token_info.get('allows_shared_content', False)\n\n if not self.access_token:\n raise ValueError(\"No access token found in session\")\n\n self.next_page_token = None\n\n def _get_headers(self) -> Dict[str, str]:\n return {\n 'Authorization': f'Bearer {self.access_token}',\n 'Accept': 'application/json'\n }\n\n def _ensure_valid_token(self):\n if not self.access_token:\n raise ValueError(\"No access token available\")\n\n token_info = {'access_token': self.access_token, 'expiry': None}\n if self.auth.is_token_expired(token_info):\n logging.info(\"Token expired, attempting refresh\")\n try:\n new_token_info = self.auth.refresh_access_token(self.refresh_token)\n self.access_token = new_token_info.get('access_token')\n except Exception:\n raise ValueError(\"Failed to refresh access token\")\n\n def _get_item_url(self, item_ref: str) -> str:\n if ':' in item_ref:\n drive_id, item_id = item_ref.split(':', 1)\n return f\"{self.GRAPH_API_BASE}/drives/{drive_id}/items/{item_id}\"\n return f\"{self.GRAPH_API_BASE}/me/drive/items/{item_ref}\"\n\n def _process_file(self, file_metadata: Dict[str, Any], load_content: bool = True) -> Optional[Document]:\n try:\n drive_item_id = file_metadata.get('id')\n file_name = file_metadata.get('name', 'Unknown')\n file_data = file_metadata.get('file', {})\n mime_type = file_data.get('mimeType', 'application/octet-stream')\n\n if mime_type not in self.SUPPORTED_MIME_TYPES:\n logging.info(f\"Skipping unsupported file type: {mime_type} for file {file_name}\")\n return None\n\n doc_metadata = {\n 'file_name': file_name,\n 'mime_type': mime_type,\n 'size': file_metadata.get('size'),\n 'created_time': file_metadata.get('createdDateTime'),\n 'modified_time': file_metadata.get('lastModifiedDateTime'),\n 'source': 'share_point'\n }\n\n if not load_content:\n return Document(\n text=\"\",\n doc_id=drive_item_id,\n extra_info=doc_metadata\n )\n\n content = self._download_file_content(drive_item_id)\n if content is None:\n logging.warning(f\"Could not load content for file {file_name} ({drive_item_id})\")\n return None\n\n return Document(\n text=content,\n doc_id=drive_item_id,\n extra_info=doc_metadata\n )\n\n except Exception as e:\n logging.error(f\"Error processing file: {e}\")\n return None\n\n def load_data(self, inputs: Dict[str, Any]) -> List[Document]:\n try:\n documents: List[Document] = []\n\n folder_id = inputs.get('folder_id')\n file_ids = inputs.get('file_ids', [])\n limit = inputs.get('limit', 100)\n list_only = inputs.get('list_only', False)\n load_content = not list_only\n page_token = inputs.get('page_token')\n search_query = inputs.get('search_query')\n self.next_page_token = None\n\n shared = inputs.get('shared', False)\n\n if file_ids:\n for file_id in file_ids:\n try:\n doc = self._load_file_by_id(file_id, load_content=load_content)\n if doc:\n if not search_query or (\n search_query.lower() in doc.extra_info.get('file_name', '').lower()\n ):\n documents.append(doc)\n except Exception as e:\n logging.error(f\"Error loading file {file_id}: {e}\")\n continue\n elif shared:\n if not self.allows_shared_content:\n logging.warning(\"Shared content is only available for work/school Microsoft accounts\")\n return []\n documents = self._list_shared_items(\n limit=limit,\n load_content=load_content,\n page_token=page_token,\n search_query=search_query\n )\n else:\n parent_id = folder_id if folder_id else 'root'\n documents = self._list_items_in_parent(\n parent_id,\n limit=limit,\n load_content=load_content,\n page_token=page_token,\n search_query=search_query\n )\n\n logging.info(f\"Loaded {len(documents)} documents from SharePoint/OneDrive\")\n return documents\n\n except Exception as e:\n logging.error(f\"Error loading data from SharePoint/OneDrive: {e}\", exc_info=True)\n raise\n\n @_retry_on_auth_failure\n def _load_file_by_id(self, file_id: str, load_content: bool = True) -> Optional[Document]:\n self._ensure_valid_token()\n\n try:\n url = self._get_item_url(file_id)\n params = {'$select': 'id,name,file,createdDateTime,lastModifiedDateTime,size'}\n response = requests.get(url, headers=self._get_headers(), params=params)\n response.raise_for_status()\n\n file_metadata = response.json()\n return self._process_file(file_metadata, load_content=load_content)\n\n except requests.exceptions.HTTPError:\n raise\n except Exception as e:\n logging.error(f\"Error loading file {file_id}: {e}\")\n return None\n\n @_retry_on_auth_failure\n def _list_items_in_parent(self, parent_id: str, limit: int = 100, load_content: bool = False, page_token: Optional[str] = None, search_query: Optional[str] = None) -> List[Document]:\n self._ensure_valid_token()\n\n documents: List[Document] = []\n\n try:\n url = f\"{self._get_item_url(parent_id)}/children\"\n params = {'$top': min(100, limit) if limit else 100, '$select': 'id,name,file,folder,createdDateTime,lastModifiedDateTime,size'}\n if page_token:\n params['$skipToken'] = page_token\n\n if search_query:\n encoded_query = quote(search_query, safe='')\n if ':' in parent_id:\n drive_id = parent_id.split(':', 1)[0]\n search_url = f\"{self.GRAPH_API_BASE}/drives/{drive_id}/root/search(q='{encoded_query}')\"\n else:\n search_url = f\"{self.GRAPH_API_BASE}/me/drive/search(q='{encoded_query}')\"\n response = requests.get(search_url, headers=self._get_headers(), params=params)\n else:\n response = requests.get(url, headers=self._get_headers(), params=params)\n\n response.raise_for_status()\n\n results = response.json()\n\n items = results.get('value', [])\n for item in items:\n if 'folder' in item:\n doc_metadata = {\n 'file_name': item.get('name', 'Unknown'),\n 'mime_type': 'folder',\n 'size': item.get('size'),\n 'created_time': item.get('createdDateTime'),\n 'modified_time': item.get('lastModifiedDateTime'),\n 'source': 'share_point',\n 'is_folder': True\n }\n documents.append(Document(text=\"\", doc_id=item.get('id'), extra_info=doc_metadata))\n else:\n doc = self._process_file(item, load_content=load_content)\n if doc:\n documents.append(doc)\n\n if limit and len(documents) >= limit:\n break\n\n next_link = results.get('@odata.nextLink')\n if next_link:\n from urllib.parse import urlparse, parse_qs\n parsed = urlparse(next_link)\n query_params = parse_qs(parsed.query)\n skiptoken_list = query_params.get('$skiptoken')\n if skiptoken_list:\n self.next_page_token = skiptoken_list[0]\n else:\n self.next_page_token = None\n else:\n self.next_page_token = None\n return documents\n\n except Exception as e:\n logging.error(f\"Error listing items under parent {parent_id}: {e}\")\n return documents\n\n\n\n\n def _resolve_mime_type(self, resource: Dict[str, Any]) -> Tuple[str, bool]:\n \"\"\"Resolve mime type from resource, falling back to file extension.\"\"\"\n file_data = resource.get('file', {})\n mime_type = file_data.get('mimeType') if file_data else None\n\n if mime_type and mime_type in self.SUPPORTED_MIME_TYPES:\n return mime_type, True\n\n name = resource.get('name', '')\n ext = os.path.splitext(name)[1].lower()\n if ext in self.EXTENSION_TO_MIME:\n return self.EXTENSION_TO_MIME[ext], True\n\n return mime_type or 'application/octet-stream', False\n\n def _get_user_drive_web_url(self) -> Optional[str]:\n \"\"\"Fetch the current user's OneDrive web URL for KQL path exclusion.\"\"\"\n try:\n response = requests.get(\n f\"{self.GRAPH_API_BASE}/me/drive\",\n headers=self._get_headers(),\n params={'$select': 'webUrl'}\n )\n response.raise_for_status()\n return response.json().get('webUrl')\n except Exception as e:\n logging.warning(f\"Could not fetch user drive web URL: {e}\")\n return None\n\n def _build_shared_kql_query(self, search_query: Optional[str], user_drive_url: Optional[str]) -> str:\n \"\"\"Build KQL query string that excludes the user's own drive items.\"\"\"\n base_query = search_query if search_query else \"*\"\n if user_drive_url:\n return f'{base_query} AND -path:\"{user_drive_url}\"'\n return base_query\n\n def _list_shared_items(self, limit: int = 100, load_content: bool = False, page_token: Optional[str] = None, search_query: Optional[str] = None) -> List[Document]:\n \"\"\"Fetch shared drive items using Microsoft Graph Search API with local offset paging.\n\n We always fetch up to a fixed maximum number of hits from Graph (single request),\n then page through that array locally using `page_token` as a simple integer offset.\n This avoids relying on buggy or inconsistent remote `from`/`size` semantics.\n \"\"\"\n self._ensure_valid_token()\n documents: List[Document] = []\n\n try:\n user_drive_url = self._get_user_drive_web_url()\n query_text = self._build_shared_kql_query(search_query, user_drive_url)\n\n url = f\"{self.GRAPH_API_BASE}/search/query\"\n page_size = 500 # maximum number of hits we care about for selection\n\n body = {\n \"requests\": [\n {\n \"entityTypes\": [\"driveItem\"],\n \"query\": {\"queryString\": query_text},\n \"from\": 0,\n \"size\": page_size,\n }\n ]\n }\n\n headers = self._get_headers()\n headers[\"Content-Type\"] = \"application/json\"\n response = requests.post(url, headers=headers, json=body)\n response.raise_for_status()\n results = response.json()\n\n search_response = results.get(\"value\", [])\n if not search_response:\n logging.warning(\"Search API returned empty value array\")\n self.next_page_token = None\n return documents\n\n hits_containers = search_response[0].get(\"hitsContainers\", [])\n if not hits_containers:\n logging.warning(\"Search API returned no hitsContainers\")\n self.next_page_token = None\n return documents\n\n container = hits_containers[0]\n total = container.get(\"total\", 0)\n raw_hits = container.get(\"hits\", [])\n\n # Deduplicate by effective item ID (driveId:itemId) to avoid the same\n # resource appearing multiple times across the result set.\n deduped_hits = []\n seen_ids = set()\n for hit in raw_hits:\n resource = hit.get(\"resource\", {})\n item_id = resource.get(\"id\")\n drive_id = resource.get(\"parentReference\", {}).get(\"driveId\")\n effective_id = f\"{drive_id}:{item_id}\" if drive_id and item_id else item_id\n if not effective_id or effective_id in seen_ids:\n continue\n seen_ids.add(effective_id)\n deduped_hits.append(hit)\n\n hits = deduped_hits\n logging.info(\n f\"Search API returned {total} total results, {len(raw_hits)} raw hits, {len(hits)} unique hits in this batch\"\n )\n try:\n offset = int(page_token) if page_token is not None else 0\n except (TypeError, ValueError):\n logging.warning(\n f\"Invalid page_token '{page_token}' for shared items search, defaulting to 0\"\n )\n offset = 0\n\n if offset < 0:\n offset = 0\n if offset >= len(hits):\n self.next_page_token = None\n return documents\n\n end_index = offset + limit if limit else len(hits)\n end_index = min(end_index, len(hits))\n\n for hit in hits[offset:end_index]:\n resource = hit.get(\"resource\", {})\n item_name = resource.get(\"name\", \"Unknown\")\n item_id = resource.get(\"id\")\n drive_id = resource.get(\"parentReference\", {}).get(\"driveId\")\n\n effective_id = f\"{drive_id}:{item_id}\" if drive_id and item_id else item_id\n\n is_folder = \"folder\" in resource\n\n if is_folder:\n doc_metadata = {\n \"file_name\": item_name,\n \"mime_type\": \"folder\",\n \"size\": resource.get(\"size\"),\n \"created_time\": resource.get(\"createdDateTime\"),\n \"modified_time\": resource.get(\"lastModifiedDateTime\"),\n \"source\": \"share_point\",\n \"is_folder\": True,\n }\n documents.append(\n Document(text=\"\", doc_id=effective_id, extra_info=doc_metadata)\n )\n else:\n mime_type, supported = self._resolve_mime_type(resource)\n if not supported:\n logging.info(\n f\"Skipping unsupported shared file: {item_name} (mime: {mime_type})\"\n )\n continue\n\n doc_metadata = {\n \"file_name\": item_name,\n \"mime_type\": mime_type,\n \"size\": resource.get(\"size\"),\n \"created_time\": resource.get(\"createdDateTime\"),\n \"modified_time\": resource.get(\"lastModifiedDateTime\"),\n \"source\": \"share_point\",\n }\n\n content = \"\"\n if load_content:\n content = self._download_file_content(effective_id) or \"\"\n\n documents.append(\n Document(text=content, doc_id=effective_id, extra_info=doc_metadata)\n )\n\n if limit and end_index < len(hits):\n self.next_page_token = str(end_index)\n else:\n self.next_page_token = None\n\n return documents\n\n except Exception as e:\n logging.error(f\"Error listing shared items via search API: {e}\", exc_info=True)\n return documents\n\n @_retry_on_auth_failure\n def _download_file_content(self, file_id: str) -> Optional[str]:\n self._ensure_valid_token()\n\n try:\n url = f\"{self._get_item_url(file_id)}/content\"\n response = requests.get(url, headers=self._get_headers())\n response.raise_for_status()\n\n try:\n return response.content.decode('utf-8')\n except UnicodeDecodeError:\n logging.error(f\"Could not decode file {file_id} as text\")\n return None\n\n except requests.exceptions.HTTPError:\n raise\n except Exception as e:\n logging.error(f\"Error downloading file {file_id}: {e}\")\n return None\n\n def _download_single_file(self, file_id: str, local_dir: str) -> bool:\n try:\n url = self._get_item_url(file_id)\n params = {'$select': 'id,name,file'}\n response = requests.get(url, headers=self._get_headers(), params=params)\n response.raise_for_status()\n\n metadata = response.json()\n file_name = metadata.get('name', 'unknown')\n file_data = metadata.get('file', {})\n mime_type = file_data.get('mimeType', 'application/octet-stream')\n\n if mime_type not in self.SUPPORTED_MIME_TYPES:\n logging.info(f\"Skipping unsupported file type: {mime_type}\")\n return False\n\n os.makedirs(local_dir, exist_ok=True)\n full_path = os.path.join(local_dir, file_name)\n\n download_url = f\"{self._get_item_url(file_id)}/content\"\n download_response = requests.get(download_url, headers=self._get_headers())\n download_response.raise_for_status()\n\n with open(full_path, 'wb') as f:\n f.write(download_response.content)\n\n return True\n except Exception as e:\n logging.error(f\"Error in _download_single_file: {e}\")\n return False\n\n def _download_folder_recursive(self, folder_id: str, local_dir: str, recursive: bool = True) -> int:\n files_downloaded = 0\n try:\n os.makedirs(local_dir, exist_ok=True)\n\n url = f\"{self._get_item_url(folder_id)}/children\"\n params = {'$top': 1000}\n\n while url:\n response = requests.get(url, headers=self._get_headers(), params=params)\n response.raise_for_status()\n\n results = response.json()\n items = results.get('value', [])\n logging.info(f\"Found {len(items)} items in folder {folder_id}\")\n\n for item in items:\n item_name = item.get('name', 'unknown')\n item_id = item.get('id')\n\n if 'folder' in item:\n if recursive:\n subfolder_path = os.path.join(local_dir, item_name)\n os.makedirs(subfolder_path, exist_ok=True)\n subfolder_files = self._download_folder_recursive(\n item_id,\n subfolder_path,\n recursive\n )\n files_downloaded += subfolder_files\n logging.info(f\"Downloaded {subfolder_files} files from subfolder {item_name}\")\n else:\n success = self._download_single_file(item_id, local_dir)\n if success:\n files_downloaded += 1\n logging.info(f\"Downloaded file: {item_name}\")\n else:\n logging.warning(f\"Failed to download file: {item_name}\")\n\n url = results.get('@odata.nextLink')\n\n return files_downloaded\n\n except Exception as e:\n logging.error(f\"Error in _download_folder_recursive for folder {folder_id}: {e}\", exc_info=True)\n return files_downloaded\n\n def _download_folder_contents(self, folder_id: str, local_dir: str, recursive: bool = True) -> int:\n try:\n self._ensure_valid_token()\n return self._download_folder_recursive(folder_id, local_dir, recursive)\n except Exception as e:\n logging.error(f\"Error downloading folder {folder_id}: {e}\", exc_info=True)\n return 0\n\n def _download_file_to_directory(self, file_id: str, local_dir: str) -> bool:\n try:\n self._ensure_valid_token()\n return self._download_single_file(file_id, local_dir)\n except Exception as e:\n logging.error(f\"Error downloading file {file_id}: {e}\", exc_info=True)\n return False\n\n def download_to_directory(self, local_dir: str, source_config: Dict[str, Any] = None) -> Dict[str, Any]:\n if source_config is None:\n source_config = {}\n\n config = source_config if source_config else getattr(self, 'config', {})\n files_downloaded = 0\n\n try:\n folder_ids = config.get('folder_ids', [])\n file_ids = config.get('file_ids', [])\n recursive = config.get('recursive', True)\n\n if file_ids:\n if isinstance(file_ids, str):\n file_ids = [file_ids]\n\n for file_id in file_ids:\n if self._download_file_to_directory(file_id, local_dir):\n files_downloaded += 1\n\n if folder_ids:\n if isinstance(folder_ids, str):\n folder_ids = [folder_ids]\n\n for folder_id in folder_ids:\n try:\n url = self._get_item_url(folder_id)\n params = {'$select': 'id,name'}\n response = requests.get(url, headers=self._get_headers(), params=params)\n response.raise_for_status()\n\n folder_metadata = response.json()\n folder_name = folder_metadata.get('name', '')\n folder_path = os.path.join(local_dir, folder_name)\n os.makedirs(folder_path, exist_ok=True)\n\n folder_files = self._download_folder_recursive(\n folder_id,\n folder_path,\n recursive\n )\n files_downloaded += folder_files\n logging.info(f\"Downloaded {folder_files} files from folder {folder_name}\")\n except Exception as e:\n logging.error(f\"Error downloading folder {folder_id}: {e}\", exc_info=True)\n\n if not file_ids and not folder_ids:\n raise ValueError(\"No folder_ids or file_ids provided for download\")\n\n return {\n \"files_downloaded\": files_downloaded,\n \"directory_path\": local_dir,\n \"empty_result\": files_downloaded == 0,\n \"source_type\": \"share_point\",\n \"config_used\": config\n }\n\n except Exception as e:\n return {\n \"files_downloaded\": files_downloaded,\n \"directory_path\": local_dir,\n \"empty_result\": True,\n \"source_type\": \"share_point\",\n \"config_used\": config,\n \"error\": str(e)\n }\n" }, { "path": "application/parser/embedding_pipeline.py", "content": "import os\nimport logging\nfrom typing import List, Any\nfrom retry import retry\nfrom tqdm import tqdm\nfrom application.core.settings import settings\nfrom application.vectorstore.vector_creator import VectorCreator\n\n\ndef sanitize_content(content: str) -> str:\n \"\"\"\n Remove NUL characters that can cause vector store ingestion to fail.\n \n Args:\n content (str): Raw content that may contain NUL characters\n \n Returns:\n str: Sanitized content with NUL characters removed\n \"\"\"\n if not content:\n return content\n return content.replace('\\x00', '')\n\n\n@retry(tries=10, delay=60)\ndef add_text_to_store_with_retry(store: Any, doc: Any, source_id: str) -> None:\n \"\"\"Add a document's text and metadata to the vector store with retry logic.\n \n Args:\n store: The vector store object.\n doc: The document to be added.\n source_id: Unique identifier for the source.\n \n Raises:\n Exception: If document addition fails after all retry attempts.\n \"\"\"\n try:\n # Sanitize content to remove NUL characters that cause ingestion failures\n doc.page_content = sanitize_content(doc.page_content)\n \n doc.metadata[\"source_id\"] = str(source_id)\n store.add_texts([doc.page_content], metadatas=[doc.metadata])\n except Exception as e:\n logging.error(f\"Failed to add document with retry: {e}\", exc_info=True)\n raise\n\n\ndef embed_and_store_documents(docs: List[Any], folder_name: str, source_id: str, task_status: Any) -> None:\n \"\"\"Embeds documents and stores them in a vector store.\n\n Args:\n docs: List of documents to be embedded and stored.\n folder_name: Directory to save the vector store.\n source_id: Unique identifier for the source.\n task_status: Task state manager for progress updates.\n\n Returns:\n None\n \n Raises:\n OSError: If unable to create folder or save vector store.\n Exception: If vector store creation or document embedding fails.\n \"\"\"\n # Ensure the folder exists\n if not os.path.exists(folder_name):\n os.makedirs(folder_name)\n\n # Validate docs is not empty\n if not docs:\n raise ValueError(\"No documents to embed - check file format and extension\")\n\n # Initialize vector store\n if settings.VECTOR_STORE == \"faiss\":\n docs_init = [docs.pop(0)]\n store = VectorCreator.create_vectorstore(\n settings.VECTOR_STORE,\n docs_init=docs_init,\n source_id=source_id,\n embeddings_key=os.getenv(\"EMBEDDINGS_KEY\"),\n )\n else:\n store = VectorCreator.create_vectorstore(\n settings.VECTOR_STORE,\n source_id=source_id,\n embeddings_key=os.getenv(\"EMBEDDINGS_KEY\"),\n )\n store.delete_index()\n\n total_docs = len(docs)\n\n # Process and embed documents\n for idx, doc in tqdm(\n enumerate(docs),\n desc=\"Embedding 🦖\",\n unit=\"docs\",\n total=total_docs,\n bar_format=\"{l_bar}{bar}| Time Left: {remaining}\",\n ):\n try:\n # Update task status for progress tracking\n progress = int(((idx + 1) / total_docs) * 100)\n task_status.update_state(state=\"PROGRESS\", meta={\"current\": progress})\n\n # Add document to vector store\n add_text_to_store_with_retry(store, doc, source_id)\n except Exception as e:\n logging.error(f\"Error embedding document {idx}: {e}\", exc_info=True)\n logging.info(f\"Saving progress at document {idx} out of {total_docs}\")\n try:\n store.save_local(folder_name)\n logging.info(\"Progress saved successfully\")\n except Exception as save_error:\n logging.error(f\"CRITICAL: Failed to save progress: {save_error}\", exc_info=True)\n # Continue without breaking to attempt final save\n break\n\n # Save the vector store\n if settings.VECTOR_STORE == \"faiss\":\n try:\n store.save_local(folder_name)\n logging.info(\"Vector store saved successfully.\")\n except Exception as e:\n logging.error(f\"CRITICAL: Failed to save final vector store: {e}\", exc_info=True)\n raise OSError(f\"Unable to save vector store to {folder_name}: {e}\") from e\n else:\n logging.info(\"Vector store saved successfully.\")\n" }, { "path": "application/parser/file/__init__.py", "content": "\n" }, { "path": "application/parser/file/audio_parser.py", "content": "from pathlib import Path\nfrom typing import Dict, Union\n\nfrom application.core.settings import settings\nfrom application.parser.file.base_parser import BaseParser\nfrom application.stt.stt_creator import STTCreator\nfrom application.stt.upload_limits import enforce_audio_file_size_limit\n\n\nclass AudioParser(BaseParser):\n def __init__(self, parser_config=None):\n super().__init__(parser_config=parser_config)\n self._transcript_metadata: Dict[str, Dict] = {}\n\n def _init_parser(self) -> Dict:\n return {}\n\n def parse_file(self, file: Path, errors: str = \"ignore\") -> Union[str, list[str]]:\n _ = errors\n try:\n enforce_audio_file_size_limit(file.stat().st_size)\n except OSError:\n pass\n stt = STTCreator.create_stt(settings.STT_PROVIDER)\n result = stt.transcribe(\n file,\n language=settings.STT_LANGUAGE,\n timestamps=settings.STT_ENABLE_TIMESTAMPS,\n diarize=settings.STT_ENABLE_DIARIZATION,\n )\n\n transcript_metadata = {\n \"transcript_duration_s\": result.get(\"duration_s\"),\n \"transcript_language\": result.get(\"language\"),\n \"transcript_provider\": result.get(\"provider\"),\n }\n if result.get(\"segments\"):\n transcript_metadata[\"transcript_segments\"] = result[\"segments\"]\n\n self._transcript_metadata[str(file)] = {\n key: value\n for key, value in transcript_metadata.items()\n if value not in (None, [], {})\n }\n return result.get(\"text\", \"\")\n\n def get_file_metadata(self, file: Path) -> Dict:\n return self._transcript_metadata.get(str(file), {})\n" }, { "path": "application/parser/file/base.py", "content": "\"\"\"Base reader class.\"\"\"\nfrom abc import abstractmethod\nfrom typing import Any, List\n\nfrom langchain_core.documents import Document as LCDocument\nfrom application.parser.schema.base import Document\n\n\nclass BaseReader:\n \"\"\"Utilities for loading data from a directory.\"\"\"\n\n @abstractmethod\n def load_data(self, *args: Any, **load_kwargs: Any) -> List[Document]:\n \"\"\"Load data from the input directory.\"\"\"\n\n def load_langchain_documents(self, **load_kwargs: Any) -> List[LCDocument]:\n \"\"\"Load data in LangChain document format.\"\"\"\n docs = self.load_data(**load_kwargs)\n return [d.to_langchain_format() for d in docs]\n" }, { "path": "application/parser/file/base_parser.py", "content": "\"\"\"Base parser and config class.\"\"\"\n\nfrom abc import abstractmethod\nfrom pathlib import Path\nfrom typing import Dict, List, Optional, Union\n\n\nclass BaseParser:\n \"\"\"Base class for all parsers.\"\"\"\n\n def __init__(self, parser_config: Optional[Dict] = None):\n \"\"\"Init params.\"\"\"\n self._parser_config = parser_config\n\n def init_parser(self) -> None:\n \"\"\"Init parser and store it.\"\"\"\n parser_config = self._init_parser()\n self._parser_config = parser_config\n\n @property\n def parser_config_set(self) -> bool:\n \"\"\"Check if parser config is set.\"\"\"\n return self._parser_config is not None\n\n @property\n def parser_config(self) -> Dict:\n \"\"\"Check if parser config is set.\"\"\"\n if self._parser_config is None:\n raise ValueError(\"Parser config not set.\")\n return self._parser_config\n\n @abstractmethod\n def _init_parser(self) -> Dict:\n \"\"\"Initialize the parser with the config.\"\"\"\n\n @abstractmethod\n def parse_file(self, file: Path, errors: str = \"ignore\") -> Union[str, List[str]]:\n \"\"\"Parse file.\"\"\"\n\n def get_file_metadata(self, file: Path) -> Dict:\n \"\"\"Return parser-specific metadata for the most recently parsed file.\"\"\"\n _ = file\n return {}\n" }, { "path": "application/parser/file/bulk.py", "content": "\"\"\"Simple reader that reads files of different formats from a directory.\"\"\"\nimport logging\nfrom pathlib import Path\nfrom typing import Callable, Dict, List, Optional, Union\n\nfrom application.parser.file.base import BaseReader\nfrom application.parser.file.base_parser import BaseParser\nfrom application.parser.file.docs_parser import DocxParser, PDFParser\nfrom application.parser.file.epub_parser import EpubParser\nfrom application.parser.file.html_parser import HTMLParser\nfrom application.parser.file.markdown_parser import MarkdownParser\nfrom application.parser.file.rst_parser import RstParser\nfrom application.parser.file.tabular_parser import PandasCSVParser, ExcelParser\nfrom application.parser.file.json_parser import JSONParser\nfrom application.parser.file.pptx_parser import PPTXParser\nfrom application.parser.file.image_parser import ImageParser\nfrom application.parser.file.audio_parser import AudioParser\nfrom application.parser.schema.base import Document\nfrom application.stt.constants import SUPPORTED_AUDIO_EXTENSIONS\nfrom application.utils import num_tokens_from_string\nfrom application.core.settings import settings\n\n\ndef _build_audio_parser_mapping() -> Dict[str, BaseParser]:\n return {extension: AudioParser() for extension in SUPPORTED_AUDIO_EXTENSIONS}\n\n\ndef get_default_file_extractor(\n ocr_enabled: Optional[bool] = None,\n) -> Dict[str, BaseParser]:\n \"\"\"Get the default file extractor.\n\n Uses docling parsers by default for advanced document processing.\n Falls back to standard parsers if docling is not installed.\n \"\"\"\n try:\n from application.parser.file.docling_parser import (\n DoclingPDFParser,\n DoclingDocxParser,\n DoclingPPTXParser,\n DoclingXLSXParser,\n DoclingHTMLParser,\n DoclingImageParser,\n DoclingCSVParser,\n DoclingAsciiDocParser,\n DoclingVTTParser,\n DoclingXMLParser,\n )\n if ocr_enabled is None:\n ocr_enabled = settings.DOCLING_OCR_ENABLED\n return {\n # Documents\n \".pdf\": DoclingPDFParser(ocr_enabled=ocr_enabled),\n \".docx\": DoclingDocxParser(),\n \".pptx\": DoclingPPTXParser(),\n \".xlsx\": DoclingXLSXParser(),\n # Web formats\n \".html\": DoclingHTMLParser(),\n \".xhtml\": DoclingHTMLParser(),\n # Data formats\n \".csv\": DoclingCSVParser(),\n \".json\": JSONParser(), # Keep JSON parser (specialized handling)\n # Text/markup formats\n \".md\": MarkdownParser(), # Keep markdown parser (specialized handling)\n \".mdx\": MarkdownParser(),\n \".rst\": RstParser(),\n \".adoc\": DoclingAsciiDocParser(),\n \".asciidoc\": DoclingAsciiDocParser(),\n # Images (with OCR) - only use Docling when OCR is enabled\n \".png\": DoclingImageParser(ocr_enabled=ocr_enabled) if ocr_enabled else ImageParser(),\n \".jpg\": DoclingImageParser(ocr_enabled=ocr_enabled) if ocr_enabled else ImageParser(),\n \".jpeg\": DoclingImageParser(ocr_enabled=ocr_enabled) if ocr_enabled else ImageParser(),\n \".tiff\": DoclingImageParser(ocr_enabled=ocr_enabled) if ocr_enabled else ImageParser(),\n \".tif\": DoclingImageParser(ocr_enabled=ocr_enabled) if ocr_enabled else ImageParser(),\n \".bmp\": DoclingImageParser(ocr_enabled=ocr_enabled) if ocr_enabled else ImageParser(),\n \".webp\": DoclingImageParser(ocr_enabled=ocr_enabled) if ocr_enabled else ImageParser(),\n # Media/subtitles\n \".vtt\": DoclingVTTParser(),\n **_build_audio_parser_mapping(),\n # Specialized XML formats\n \".xml\": DoclingXMLParser(),\n # Formats docling doesn't support - use standard parsers\n \".epub\": EpubParser(),\n }\n except ImportError:\n logging.warning(\n \"docling is not installed. Using standard parsers. \"\n \"For advanced document parsing, install with: pip install docling\"\n )\n # Fallback to standard parsers\n return {\n \".pdf\": PDFParser(),\n \".docx\": DocxParser(),\n \".csv\": PandasCSVParser(),\n \".xlsx\": ExcelParser(),\n \".epub\": EpubParser(),\n \".md\": MarkdownParser(),\n \".rst\": RstParser(),\n \".html\": HTMLParser(),\n \".mdx\": MarkdownParser(),\n \".json\": JSONParser(),\n \".pptx\": PPTXParser(),\n \".png\": ImageParser(),\n \".jpg\": ImageParser(),\n \".jpeg\": ImageParser(),\n **_build_audio_parser_mapping(),\n }\n\n\n# For backwards compatibility\nDEFAULT_FILE_EXTRACTOR: Dict[str, BaseParser] = get_default_file_extractor()\n\n\nclass SimpleDirectoryReader(BaseReader):\n \"\"\"Simple directory reader.\n\n Can read files into separate documents, or concatenates\n files into one document text.\n\n Args:\n input_dir (str): Path to the directory.\n input_files (List): List of file paths to read (Optional; overrides input_dir)\n exclude_hidden (bool): Whether to exclude hidden files (dotfiles).\n errors (str): how encoding and decoding errors are to be handled,\n see https://docs.python.org/3/library/functions.html#open\n recursive (bool): Whether to recursively search in subdirectories.\n False by default.\n required_exts (Optional[List[str]]): List of required extensions.\n Default is None.\n file_extractor (Optional[Dict[str, BaseParser]]): A mapping of file\n extension to a BaseParser class that specifies how to convert that file\n to text. See DEFAULT_FILE_EXTRACTOR.\n num_files_limit (Optional[int]): Maximum number of files to read.\n Default is None.\n file_metadata (Optional[Callable[str, Dict]]): A function that takes\n in a filename and returns a Dict of metadata for the Document.\n Default is None.\n \"\"\"\n\n def __init__(\n self,\n input_dir: Optional[str] = None,\n input_files: Optional[List] = None,\n exclude_hidden: bool = True,\n errors: str = \"ignore\",\n recursive: bool = True,\n required_exts: Optional[List[str]] = None,\n file_extractor: Optional[Dict[str, BaseParser]] = None,\n num_files_limit: Optional[int] = None,\n file_metadata: Optional[Callable[[str], Dict]] = None,\n ) -> None:\n \"\"\"Initialize with parameters.\"\"\"\n super().__init__()\n\n if not input_dir and not input_files:\n raise ValueError(\"Must provide either `input_dir` or `input_files`.\")\n\n self.errors = errors\n\n self.recursive = recursive\n self.exclude_hidden = exclude_hidden\n # Normalize extensions to lowercase for case-insensitive matching\n self.required_exts = (\n [ext.lower() for ext in required_exts] if required_exts else None\n )\n self.num_files_limit = num_files_limit\n\n if input_files:\n self.input_files = []\n for path in input_files:\n print(path)\n input_file = Path(path)\n self.input_files.append(input_file)\n elif input_dir:\n self.input_dir = Path(input_dir)\n self.input_files = self._add_files(self.input_dir)\n\n self.file_extractor = file_extractor or DEFAULT_FILE_EXTRACTOR\n self.file_metadata = file_metadata\n\n def _add_files(self, input_dir: Path) -> List[Path]:\n \"\"\"Add files.\"\"\"\n input_files = sorted(input_dir.iterdir())\n new_input_files = []\n dirs_to_explore = []\n for input_file in input_files:\n if input_file.is_dir():\n if self.recursive:\n dirs_to_explore.append(input_file)\n elif self.exclude_hidden and input_file.name.startswith(\".\"):\n continue\n elif (\n self.required_exts is not None\n and input_file.suffix.lower() not in self.required_exts\n ):\n continue\n else:\n new_input_files.append(input_file)\n\n for dir_to_explore in dirs_to_explore:\n sub_input_files = self._add_files(dir_to_explore)\n new_input_files.extend(sub_input_files)\n\n if self.num_files_limit is not None and self.num_files_limit > 0:\n new_input_files = new_input_files[0: self.num_files_limit]\n\n # print total number of files added\n logging.debug(\n f\"> [SimpleDirectoryReader] Total files added: {len(new_input_files)}\"\n )\n\n return new_input_files\n\n def load_data(self, concatenate: bool = False) -> List[Document]:\n \"\"\"Load data from the input directory.\n\n Args:\n concatenate (bool): whether to concatenate all files into one document.\n If set to True, file metadata is ignored.\n False by default.\n\n Returns:\n List[Document]: A list of documents.\n \"\"\"\n data: Union[str, List[str]] = \"\"\n data_list: List[str] = []\n metadata_list = []\n self.file_token_counts = {}\n \n for input_file in self.input_files:\n suffix_lower = input_file.suffix.lower()\n parser_metadata = {}\n if suffix_lower in self.file_extractor:\n parser = self.file_extractor[suffix_lower]\n if not parser.parser_config_set:\n parser.init_parser()\n data = parser.parse_file(input_file, errors=self.errors)\n parser_metadata = parser.get_file_metadata(input_file)\n else:\n # do standard read\n with open(input_file, \"r\", errors=self.errors) as f:\n data = f.read()\n \n # Calculate token count for this file\n if isinstance(data, List):\n file_tokens = sum(num_tokens_from_string(str(d)) for d in data)\n else:\n file_tokens = num_tokens_from_string(str(data))\n \n full_path = str(input_file.resolve())\n self.file_token_counts[full_path] = file_tokens\n \n base_metadata = {\n 'title': input_file.name,\n 'token_count': file_tokens,\n }\n if parser_metadata:\n base_metadata.update(parser_metadata)\n \n if hasattr(self, 'input_dir'):\n try:\n relative_path = str(input_file.relative_to(self.input_dir))\n base_metadata['source'] = relative_path\n except ValueError:\n base_metadata['source'] = str(input_file)\n else:\n base_metadata['source'] = str(input_file)\n\n if self.file_metadata is not None:\n custom_metadata = self.file_metadata(input_file.name)\n base_metadata.update(custom_metadata)\n\n if isinstance(data, List):\n # Extend data_list with each item in the data list\n data_list.extend([str(d) for d in data])\n metadata_list.extend([base_metadata for _ in data])\n else:\n data_list.append(str(data))\n metadata_list.append(base_metadata)\n \n # Build directory structure if input_dir is provided\n if hasattr(self, 'input_dir'):\n self.directory_structure = self.build_directory_structure(self.input_dir)\n logging.info(\"Directory structure built successfully\")\n else:\n self.directory_structure = {}\n\n if concatenate:\n return [Document(\"\\n\".join(data_list))]\n elif self.file_metadata is not None:\n return [Document(d, extra_info=m) for d, m in zip(data_list, metadata_list)]\n else:\n return [Document(d) for d in data_list]\n\n def build_directory_structure(self, base_path):\n \"\"\"Build a dictionary representing the directory structure.\n\n Args:\n base_path: The base path to start building the structure from.\n\n Returns:\n dict: A nested dictionary representing the directory structure.\n \"\"\"\n import mimetypes\n \n def build_tree(path):\n \"\"\"Helper function to recursively build the directory tree.\"\"\"\n result = {}\n \n for item in path.iterdir():\n if self.exclude_hidden and item.name.startswith('.'):\n continue\n \n if item.is_dir():\n subtree = build_tree(item)\n if subtree:\n result[item.name] = subtree\n else:\n if self.required_exts is not None and item.suffix.lower() not in self.required_exts:\n continue\n \n full_path = str(item.resolve())\n file_size_bytes = item.stat().st_size\n mime_type = mimetypes.guess_type(item.name)[0] or \"application/octet-stream\"\n \n file_info = {\n \"type\": mime_type,\n \"size_bytes\": file_size_bytes\n }\n \n if hasattr(self, 'file_token_counts') and full_path in self.file_token_counts:\n file_info[\"token_count\"] = self.file_token_counts[full_path]\n \n result[item.name] = file_info\n \n return result\n \n return build_tree(Path(base_path))\n" }, { "path": "application/parser/file/constants.py", "content": "\"\"\"Shared file-extension constants for parsing and ingestion flows.\"\"\"\n\nfrom application.stt.constants import SUPPORTED_AUDIO_EXTENSIONS\n\n\nSUPPORTED_SOURCE_DOCUMENT_EXTENSIONS = (\n \".rst\",\n \".md\",\n \".pdf\",\n \".txt\",\n \".docx\",\n \".csv\",\n \".epub\",\n \".html\",\n \".mdx\",\n \".json\",\n \".xlsx\",\n \".pptx\",\n)\n\nSUPPORTED_SOURCE_IMAGE_EXTENSIONS = (\".png\", \".jpg\", \".jpeg\")\n\nSUPPORTED_SOURCE_EXTENSIONS = (\n *SUPPORTED_SOURCE_DOCUMENT_EXTENSIONS,\n *SUPPORTED_SOURCE_IMAGE_EXTENSIONS,\n *SUPPORTED_AUDIO_EXTENSIONS,\n)\n" }, { "path": "application/parser/file/docling_parser.py", "content": "\"\"\"Docling parser.\n\nUses docling library for advanced document parsing with layout detection,\ntable structure recognition, and unified document representation.\n\nSupports: PDF, DOCX, PPTX, XLSX, HTML, XHTML, CSV, Markdown, AsciiDoc,\nimages (PNG, JPEG, TIFF, BMP, WEBP), WebVTT, and specialized XML formats.\n\"\"\"\nimport importlib.util\nimport logging\nfrom pathlib import Path\nfrom typing import Dict, List, Optional, Union\n\nfrom application.parser.file.base_parser import BaseParser\n\nlogger = logging.getLogger(__name__)\n\n\nclass DoclingParser(BaseParser):\n \"\"\"Parser using docling for advanced document processing.\n\n Docling provides:\n - Advanced PDF layout analysis\n - Table structure recognition\n - Reading order detection\n - OCR for scanned documents (supports RapidOCR)\n - Unified DoclingDocument format\n - Export to Markdown\n\n Uses hybrid OCR approach by default:\n - Text regions: Direct PDF text extraction (fast)\n - Bitmap/image regions: OCR only these areas (smart)\n \"\"\"\n\n def __init__(\n self,\n ocr_enabled: bool = True,\n table_structure: bool = True,\n export_format: str = \"markdown\",\n use_rapidocr: bool = True,\n ocr_languages: Optional[List[str]] = None,\n force_full_page_ocr: bool = False,\n ):\n \"\"\"Initialize DoclingParser.\n\n Args:\n ocr_enabled: Enable OCR for bitmap/image regions in documents\n table_structure: Enable table structure recognition\n export_format: Output format ('markdown', 'text', 'html')\n use_rapidocr: Use RapidOCR engine (default True, works well in Docker)\n ocr_languages: List of OCR languages (default: ['english'])\n force_full_page_ocr: Force OCR on entire page (False = smart hybrid OCR)\n \"\"\"\n super().__init__()\n self.ocr_enabled = ocr_enabled\n self.table_structure = table_structure\n self.export_format = export_format\n self.use_rapidocr = use_rapidocr\n self.ocr_languages = ocr_languages or [\"english\"]\n self.force_full_page_ocr = force_full_page_ocr\n self._converter = None\n\n def _create_converter(self):\n \"\"\"Create a docling converter with hybrid OCR configuration.\n\n Uses smart OCR approach:\n - When ocr_enabled=True and force_full_page_ocr=False (default):\n Layout model detects text vs bitmap regions, OCR only runs on bitmaps\n - When ocr_enabled=True and force_full_page_ocr=True:\n OCR runs on entire page (for scanned documents/images)\n - When ocr_enabled=False:\n No OCR, only native text extraction\n\n Returns:\n DocumentConverter instance\n \"\"\"\n from docling.document_converter import (\n DocumentConverter,\n ImageFormatOption,\n InputFormat,\n PdfFormatOption,\n )\n from docling.datamodel.pipeline_options import PdfPipelineOptions\n\n pipeline_options = PdfPipelineOptions(\n do_ocr=self.ocr_enabled,\n do_table_structure=self.table_structure,\n )\n\n if self.ocr_enabled:\n ocr_options = self._get_ocr_options()\n if ocr_options is not None:\n pipeline_options.ocr_options = ocr_options\n\n return DocumentConverter(\n format_options={\n InputFormat.PDF: PdfFormatOption(\n pipeline_options=pipeline_options,\n ),\n InputFormat.IMAGE: ImageFormatOption(\n pipeline_options=pipeline_options,\n ),\n }\n )\n\n def _init_parser(self) -> Dict:\n \"\"\"Initialize the docling converter with hybrid OCR.\"\"\"\n logger.info(\"Initializing DoclingParser...\")\n logger.info(f\" ocr_enabled={self.ocr_enabled}\")\n logger.info(f\" force_full_page_ocr={self.force_full_page_ocr}\")\n logger.info(f\" use_rapidocr={self.use_rapidocr}\")\n\n if importlib.util.find_spec(\"docling.document_converter\") is None:\n raise ImportError(\n \"docling is required for DoclingParser. \"\n \"Install it with: pip install docling\"\n )\n\n # Create converter with hybrid OCR (smart: text direct, bitmaps OCR'd)\n self._converter = self._create_converter()\n\n logger.info(\"DoclingParser initialized successfully\")\n return {\n \"ocr_enabled\": self.ocr_enabled,\n \"table_structure\": self.table_structure,\n \"export_format\": self.export_format,\n \"use_rapidocr\": self.use_rapidocr,\n \"ocr_languages\": self.ocr_languages,\n \"force_full_page_ocr\": self.force_full_page_ocr,\n }\n\n def _get_ocr_options(self):\n \"\"\"Get OCR options based on configuration.\n\n Returns RapidOcrOptions if use_rapidocr is True and available,\n otherwise returns None to use docling defaults.\n \"\"\"\n if not self.use_rapidocr:\n return None\n\n try:\n from docling.datamodel.pipeline_options import RapidOcrOptions\n\n return RapidOcrOptions(\n lang=self.ocr_languages,\n force_full_page_ocr=self.force_full_page_ocr,\n )\n except ImportError as e:\n logger.warning(f\"Failed to import RapidOcrOptions: {e}\")\n return None\n except Exception as e:\n logger.error(f\"Error creating RapidOcrOptions: {e}\")\n return None\n\n def _export_content(self, document) -> str:\n \"\"\"Export document content in the configured format.\n\n Handles edge case where text is nested under picture elements (e.g., OCR'd\n images). If the standard export returns minimal content but document.texts\n contains extracted text, falls back to direct text extraction.\n \"\"\"\n if self.export_format == \"markdown\":\n content = document.export_to_markdown()\n elif self.export_format == \"html\":\n content = document.export_to_html()\n else:\n content = document.export_to_text()\n\n # Handle case where text is nested under pictures (common with OCR'd images)\n # Standard exports may return just \"\" while actual text exists\n stripped_content = content.strip()\n is_minimal = len(stripped_content) < 50 or stripped_content == \"\"\n\n if is_minimal and hasattr(document, \"texts\") and document.texts:\n # Extract text directly from document.texts\n extracted_texts = [t.text for t in document.texts if t.text]\n if extracted_texts:\n logger.info(\n f\"Standard export minimal ({len(stripped_content)} chars), \"\n f\"extracting {len(extracted_texts)} texts directly\"\n )\n return \"\\n\\n\".join(extracted_texts)\n\n return content\n\n def parse_file(self, file: Path, errors: str = \"ignore\") -> Union[str, List[str]]:\n \"\"\"Parse file using docling with hybrid OCR.\n\n Uses smart OCR approach where the layout model detects text vs bitmap\n regions. Text is extracted directly, bitmaps are OCR'd only when needed.\n\n Args:\n file: Path to the file to parse\n errors: Error handling mode (ignored, docling handles internally)\n\n Returns:\n Parsed document content as markdown string\n \"\"\"\n logger.info(f\"parse_file called for: {file}\")\n\n if self._converter is None:\n self._init_parser()\n\n try:\n logger.info(f\"Converting file with hybrid OCR: {file}\")\n result = self._converter.convert(str(file))\n content = self._export_content(result.document)\n logger.info(f\"Parse complete, content length: {len(content)} chars\")\n\n return content\n\n except Exception as e:\n logger.error(f\"Error parsing file with docling: {e}\", exc_info=True)\n if errors == \"ignore\":\n return f\"[Error parsing file with docling: {str(e)}]\"\n raise\n\n\nclass DoclingPDFParser(DoclingParser):\n \"\"\"Docling-based PDF parser with advanced features and RapidOCR support.\n\n Uses hybrid OCR approach by default:\n - Text regions: Direct PDF text extraction (fast)\n - Bitmap/image regions: OCR only these areas (smart)\n\n Set force_full_page_ocr=True only for fully scanned documents.\n \"\"\"\n\n def __init__(\n self,\n ocr_enabled: bool = True,\n table_structure: bool = True,\n use_rapidocr: bool = True,\n ocr_languages: Optional[List[str]] = None,\n force_full_page_ocr: bool = False,\n ):\n super().__init__(\n ocr_enabled=ocr_enabled,\n table_structure=table_structure,\n export_format=\"markdown\",\n use_rapidocr=use_rapidocr,\n ocr_languages=ocr_languages,\n force_full_page_ocr=force_full_page_ocr,\n )\n\n\nclass DoclingDocxParser(DoclingParser):\n \"\"\"Docling-based DOCX parser.\"\"\"\n\n def __init__(self):\n super().__init__(export_format=\"markdown\")\n\n\nclass DoclingPPTXParser(DoclingParser):\n \"\"\"Docling-based PPTX parser.\"\"\"\n\n def __init__(self):\n super().__init__(export_format=\"markdown\")\n\n\nclass DoclingXLSXParser(DoclingParser):\n \"\"\"Docling-based XLSX parser with table structure.\"\"\"\n\n def __init__(self):\n super().__init__(table_structure=True, export_format=\"markdown\")\n\n\nclass DoclingHTMLParser(DoclingParser):\n \"\"\"Docling-based HTML parser.\"\"\"\n\n def __init__(self):\n super().__init__(export_format=\"markdown\")\n\n\nclass DoclingImageParser(DoclingParser):\n \"\"\"Docling-based image parser with OCR and RapidOCR support.\n\n For images, force_full_page_ocr=True is used since images are entirely\n visual and require full OCR to extract any text.\n \"\"\"\n\n def __init__(\n self,\n ocr_enabled: bool = True,\n use_rapidocr: bool = True,\n ocr_languages: Optional[List[str]] = None,\n force_full_page_ocr: bool = True,\n ):\n super().__init__(\n ocr_enabled=ocr_enabled,\n export_format=\"markdown\",\n use_rapidocr=use_rapidocr,\n ocr_languages=ocr_languages,\n force_full_page_ocr=force_full_page_ocr,\n )\n\n\nclass DoclingCSVParser(DoclingParser):\n \"\"\"Docling-based CSV parser.\"\"\"\n\n def __init__(self):\n super().__init__(table_structure=True, export_format=\"markdown\")\n\n\nclass DoclingMarkdownParser(DoclingParser):\n \"\"\"Docling-based Markdown parser.\"\"\"\n\n def __init__(self):\n super().__init__(export_format=\"markdown\")\n\n\nclass DoclingAsciiDocParser(DoclingParser):\n \"\"\"Docling-based AsciiDoc parser.\"\"\"\n\n def __init__(self):\n super().__init__(export_format=\"markdown\")\n\n\nclass DoclingVTTParser(DoclingParser):\n \"\"\"Docling-based WebVTT (video text tracks) parser.\"\"\"\n\n def __init__(self):\n super().__init__(export_format=\"markdown\")\n\n\nclass DoclingXMLParser(DoclingParser):\n \"\"\"Docling-based XML parser (USPTO, JATS).\"\"\"\n\n def __init__(self):\n super().__init__(export_format=\"markdown\")\n" }, { "path": "application/parser/file/docs_parser.py", "content": "\"\"\"Docs parser.\n\nContains parsers for docx, pdf files.\n\n\"\"\"\nfrom pathlib import Path\nfrom typing import Dict\n\nfrom application.parser.file.base_parser import BaseParser\nfrom application.core.settings import settings\nimport requests\n\nclass PDFParser(BaseParser):\n \"\"\"PDF parser.\"\"\"\n\n def _init_parser(self) -> Dict:\n \"\"\"Init parser.\"\"\"\n return {}\n\n def parse_file(self, file: Path, errors: str = \"ignore\") -> str:\n \"\"\"Parse file.\"\"\"\n if settings.PARSE_PDF_AS_IMAGE:\n doc2md_service = \"https://llm.arc53.com/doc2md\"\n # alternatively you can use local vision capable LLM\n with open(file, \"rb\") as file_loaded:\n files = {'file': file_loaded}\n response = requests.post(doc2md_service, files=files)\n data = response.json()[\"markdown\"]\n return data\n\n try:\n from pypdf import PdfReader\n except ImportError:\n raise ValueError(\"pypdf is required to read PDF files.\")\n text_list = []\n with open(file, \"rb\") as fp:\n # Create a PDF object\n pdf = PdfReader(fp)\n\n # Get the number of pages in the PDF document\n num_pages = len(pdf.pages)\n\n # Iterate over every page\n for page_index in range(num_pages):\n # Extract the text from the page\n page = pdf.pages[page_index]\n page_text = page.extract_text()\n text_list.append(page_text)\n text = \"\\n\".join(text_list)\n\n return text\n\n\nclass DocxParser(BaseParser):\n \"\"\"Docx parser.\"\"\"\n\n def _init_parser(self) -> Dict:\n \"\"\"Init parser.\"\"\"\n return {}\n\n def parse_file(self, file: Path, errors: str = \"ignore\") -> str:\n \"\"\"Parse file.\"\"\"\n try:\n import docx2txt\n except ImportError:\n raise ValueError(\"docx2txt is required to read Microsoft Word files.\")\n\n text = docx2txt.process(file)\n\n return text" }, { "path": "application/parser/file/epub_parser.py", "content": "\"\"\"Epub parser.\n\nContains parsers for epub files.\n\"\"\"\n\nfrom pathlib import Path\nfrom typing import Dict\n\nfrom application.parser.file.base_parser import BaseParser\n\n\nclass EpubParser(BaseParser):\n \"\"\"Epub Parser.\"\"\"\n\n def _init_parser(self) -> Dict:\n \"\"\"Init parser.\"\"\"\n return {}\n\n def parse_file(self, file: Path, errors: str = \"ignore\") -> str:\n \"\"\"Parse file.\"\"\"\n try:\n import ebooklib\n from ebooklib import epub\n except ImportError:\n raise ValueError(\"`EbookLib` is required to read Epub files.\")\n try:\n import html2text\n except ImportError:\n raise ValueError(\"`html2text` is required to parse Epub files.\")\n\n text_list = []\n book = epub.read_epub(file, options={\"ignore_ncx\": True})\n\n # Iterate through all chapters.\n for item in book.get_items():\n # Chapters are typically located in epub documents items.\n if item.get_type() == ebooklib.ITEM_DOCUMENT:\n text_list.append(\n html2text.html2text(item.get_content().decode(\"utf-8\"))\n )\n\n text = \"\\n\".join(text_list)\n return text\n" }, { "path": "application/parser/file/html_parser.py", "content": "\"\"\"HTML parser.\n\nContains parser for html files.\n\n\"\"\"\nfrom pathlib import Path\nfrom typing import Dict, Union\n\nfrom application.parser.file.base_parser import BaseParser\n\n\nclass HTMLParser(BaseParser):\n \"\"\"HTML parser.\"\"\"\n\n def _init_parser(self) -> Dict:\n \"\"\"Init parser.\"\"\"\n return {}\n\n def parse_file(self, file: Path, errors: str = \"ignore\") -> Union[str, list[str]]:\n from langchain_community.document_loaders import BSHTMLLoader\n\n loader = BSHTMLLoader(file)\n data = loader.load() \n return data\n" }, { "path": "application/parser/file/image_parser.py", "content": "\"\"\"Image parser.\n\nContains parser for .png, .jpg, .jpeg files.\n\n\"\"\"\nfrom pathlib import Path\nimport requests\nfrom typing import Dict, Union\n\nfrom application.parser.file.base_parser import BaseParser\nfrom application.core.settings import settings\n\n\nclass ImageParser(BaseParser):\n \"\"\"Image parser.\"\"\"\n\n def _init_parser(self) -> Dict:\n \"\"\"Init parser.\"\"\"\n return {}\n\n def parse_file(self, file: Path, errors: str = \"ignore\") -> Union[str, list[str]]:\n if settings.PARSE_IMAGE_REMOTE:\n doc2md_service = \"https://llm.arc53.com/doc2md\"\n # alternatively you can use local vision capable LLM\n with open(file, \"rb\") as file_loaded:\n files = {'file': file_loaded}\n response = requests.post(doc2md_service, files=files) \n data = response.json()[\"markdown\"] \n else:\n data = \"\"\n return data\n" }, { "path": "application/parser/file/json_parser.py", "content": "import json\nfrom typing import Any, Dict, List, Union\nfrom pathlib import Path\n\nfrom application.parser.file.base_parser import BaseParser\n\nclass JSONParser(BaseParser):\n r\"\"\"JSON (.json) parser.\n\n Parses JSON files into a list of strings or a concatenated document.\n It handles both JSON objects (dictionaries) and arrays (lists).\n\n Args:\n concat_rows (bool): Whether to concatenate all rows into one document.\n If set to False, a Document will be created for each item in the JSON.\n True by default.\n\n row_joiner (str): Separator to use for joining each row.\n Only used when `concat_rows=True`.\n Set to \"\\n\" by default.\n\n json_config (dict): Options for parsing JSON. Can be used to specify options like\n custom decoding or formatting. Set to empty dict by default.\n\n \"\"\"\n\n def __init__(\n self,\n *args: Any,\n concat_rows: bool = True,\n row_joiner: str = \"\\n\",\n json_config: dict = {},\n **kwargs: Any\n ) -> None:\n \"\"\"Init params.\"\"\"\n super().__init__(*args, **kwargs)\n self._concat_rows = concat_rows\n self._row_joiner = row_joiner\n self._json_config = json_config\n\n def _init_parser(self) -> Dict:\n \"\"\"Init parser.\"\"\"\n return {}\n\n def parse_file(self, file: Path, errors: str = \"ignore\") -> Union[str, List[str]]:\n \"\"\"Parse JSON file.\"\"\"\n \n with open(file, 'r', encoding='utf-8') as f:\n data = json.load(f, **self._json_config)\n\n if isinstance(data, dict):\n data = [data]\n\n if self._concat_rows:\n return self._row_joiner.join([str(item) for item in data])\n else:\n return data\n" }, { "path": "application/parser/file/markdown_parser.py", "content": "\"\"\"Markdown parser.\n\nContains parser for md files.\n\n\"\"\"\nimport re\nfrom pathlib import Path\nfrom typing import Any, Dict, List, Optional, Tuple, Union, cast\n\nfrom application.parser.file.base_parser import BaseParser\nfrom application.utils import num_tokens_from_string\n\n\nclass MarkdownParser(BaseParser):\n \"\"\"Markdown parser.\n\n Extract text from markdown files.\n Returns dictionary with keys as headers and values as the text between headers.\n\n \"\"\"\n\n def __init__(\n self,\n *args: Any,\n remove_hyperlinks: bool = True,\n remove_images: bool = True,\n max_tokens: int = 2048,\n # remove_tables: bool = True,\n **kwargs: Any,\n ) -> None:\n \"\"\"Init params.\"\"\"\n super().__init__(*args, **kwargs)\n self._remove_hyperlinks = remove_hyperlinks\n self._remove_images = remove_images\n self._max_tokens = max_tokens\n # self._remove_tables = remove_tables\n\n def tups_chunk_append(self, tups: List[Tuple[Optional[str], str]], current_header: Optional[str],\n current_text: str):\n \"\"\"Append to tups chunk.\"\"\"\n num_tokens = num_tokens_from_string(current_text)\n if num_tokens > self._max_tokens:\n chunks = [current_text[i:i + self._max_tokens] for i in range(0, len(current_text), self._max_tokens)]\n for chunk in chunks:\n tups.append((current_header, chunk))\n else:\n tups.append((current_header, current_text))\n return tups\n\n def markdown_to_tups(self, markdown_text: str) -> List[Tuple[Optional[str], str]]:\n \"\"\"Convert a markdown file to a dictionary.\n\n The keys are the headers and the values are the text under each header.\n\n \"\"\"\n markdown_tups: List[Tuple[Optional[str], str]] = []\n lines = markdown_text.split(\"\\n\")\n\n current_header = None\n current_text = \"\"\n\n for line in lines:\n header_match = re.match(r\"^#+\\s\", line)\n if header_match:\n if current_header is not None:\n if current_text == \"\" or None:\n continue\n markdown_tups = self.tups_chunk_append(markdown_tups, current_header, current_text)\n\n current_header = line\n current_text = \"\"\n else:\n current_text += line + \"\\n\"\n markdown_tups = self.tups_chunk_append(markdown_tups, current_header, current_text)\n\n if current_header is not None:\n # pass linting, assert keys are defined\n markdown_tups = [\n (re.sub(r\"#\", \"\", cast(str, key)).strip(), re.sub(r\"<.*?>\", \"\", value))\n for key, value in markdown_tups\n ]\n else:\n markdown_tups = [\n (key, re.sub(\"\\n\", \"\", value)) for key, value in markdown_tups\n ]\n\n return markdown_tups\n\n def remove_images(self, content: str) -> str:\n \"\"\"Get a dictionary of a markdown file from its path.\"\"\"\n pattern = r\"!{1}\\[\\[(.*)\\]\\]\"\n content = re.sub(pattern, \"\", content)\n return content\n\n # def remove_tables(self, content: str) -> List[List[str]]:\n # \"\"\"Convert markdown tables to nested lists.\"\"\"\n # table_rows_pattern = r\"((\\r?\\n){2}|^)([^\\r\\n]*\\|[^\\r\\n]*(\\r?\\n)?)+(?=(\\r?\\n){2}|$)\"\n # table_cells_pattern = r\"([^\\|\\r\\n]*)\\|\"\n #\n # table_rows = re.findall(table_rows_pattern, content, re.MULTILINE)\n # table_lists = []\n # for row in table_rows:\n # cells = re.findall(table_cells_pattern, row[2])\n # cells = [cell.strip() for cell in cells if cell.strip()]\n # table_lists.append(cells)\n # return str(table_lists)\n\n def remove_hyperlinks(self, content: str) -> str:\n \"\"\"Get a dictionary of a markdown file from its path.\"\"\"\n pattern = r\"\\[(.*?)\\]\\((.*?)\\)\"\n content = re.sub(pattern, r\"\\1\", content)\n return content\n\n def _init_parser(self) -> Dict:\n \"\"\"Initialize the parser with the config.\"\"\"\n return {}\n\n def parse_tups(\n self, filepath: Path, errors: str = \"ignore\"\n ) -> List[Tuple[Optional[str], str]]:\n \"\"\"Parse file into tuples.\"\"\"\n with open(filepath, \"r\") as f:\n content = f.read()\n if self._remove_hyperlinks:\n content = self.remove_hyperlinks(content)\n if self._remove_images:\n content = self.remove_images(content)\n # if self._remove_tables:\n # content = self.remove_tables(content)\n markdown_tups = self.markdown_to_tups(content)\n return markdown_tups\n\n def parse_file(\n self, filepath: Path, errors: str = \"ignore\"\n ) -> Union[str, List[str]]:\n \"\"\"Parse file into string.\"\"\"\n tups = self.parse_tups(filepath, errors=errors)\n results = []\n # TODO: don't include headers right now\n for header, value in tups:\n if header is None:\n results.append(value)\n else:\n results.append(f\"\\n\\n{header}\\n{value}\")\n return results\n" }, { "path": "application/parser/file/openapi3_parser.py", "content": "from urllib.parse import urlparse\n\nfrom openapi_parser import parse\n\ntry:\n from application.parser.file.base_parser import BaseParser\nexcept ModuleNotFoundError:\n from base_parser import BaseParser\n\n\nclass OpenAPI3Parser(BaseParser):\n def init_parser(self) -> None:\n return super().init_parser()\n\n def get_base_urls(self, urls):\n base_urls = []\n for i in urls:\n parsed_url = urlparse(i)\n base_url = parsed_url.scheme + \"://\" + parsed_url.netloc\n if base_url not in base_urls:\n base_urls.append(base_url)\n return base_urls\n\n def get_info_from_paths(self, path):\n info = \"\"\n if path.operations:\n for operation in path.operations:\n info += (\n f\"\\n{operation.method.value}=\"\n f\"{operation.responses[0].description}\"\n )\n return info\n\n def parse_file(self, file_path):\n data = parse(file_path)\n results = \"\"\n base_urls = self.get_base_urls(link.url for link in data.servers)\n base_urls = \",\".join([base_url for base_url in base_urls])\n results += f\"Base URL:{base_urls}\\n\"\n i = 1\n for path in data.paths:\n info = self.get_info_from_paths(path)\n results += (\n f\"Path{i}: {path.url}\\n\"\n f\"description: {path.description}\\n\"\n f\"parameters: {path.parameters}\\nmethods: {info}\\n\"\n )\n i += 1\n with open(\"results.txt\", \"w\") as f:\n f.write(results)\n return results\n" }, { "path": "application/parser/file/pptx_parser.py", "content": "\"\"\"PPT parser.\nContains parsers for presentation (.pptx) files to extract slide text.\n\"\"\"\nfrom pathlib import Path\nfrom typing import Any, Dict, List, Union\n\nfrom application.parser.file.base_parser import BaseParser\n\nclass PPTXParser(BaseParser):\n r\"\"\"PPTX (.pptx) parser for extracting text from PowerPoint slides.\n Args:\n concat_slides (bool): Specifies whether to concatenate all slide text into one document.\n - If True, slide texts will be joined together as a single string.\n - If False, each slide's text will be stored as a separate entry in a list.\n Set to True by default.\n slide_separator (str): Separator used to join slides' text content.\n Only used when `concat_slides=True`. Default is \"\\n\".\n Refer to https://python-pptx.readthedocs.io/en/latest/ for more information.\n \"\"\"\n\n def __init__(\n self,\n *args: Any,\n concat_slides: bool = True,\n slide_separator: str = \"\\n\",\n **kwargs: Any\n ) -> None:\n \"\"\"Init params.\"\"\"\n super().__init__(*args, **kwargs)\n self._concat_slides = concat_slides\n self._slide_separator = slide_separator\n\n def _init_parser(self) -> Dict:\n \"\"\"Init parser.\"\"\"\n return {}\n\n def parse_file(self, file: Path, errors: str = \"ignore\") -> Union[str, List[str]]:\n r\"\"\"\n Parse a .pptx file and extract text from each slide.\n Args:\n file (Path): Path to the .pptx file.\n errors (str): Error handling policy ('ignore' by default).\n Returns:\n Union[str, List[str]]: Concatenated text if concat_slides is True,\n otherwise a list of slide texts.\n \"\"\"\n\n try:\n from pptx import Presentation\n except ImportError:\n raise ImportError(\"pptx module is required to read .PPTX files.\")\n\n try:\n presentation = Presentation(file)\n slide_texts=[]\n\n # Iterate over each slide in the presentation\n for slide in presentation.slides:\n slide_text=\"\"\n\n # Iterate over each shape in the slide\n for shape in slide.shapes:\n # Check if the shape has a 'text' attribute and append that to the slide_text\n if hasattr(shape,\"text\"):\n slide_text+=shape.text\n\n slide_texts.append(slide_text.strip())\n\n if self._concat_slides:\n return self._slide_separator.join(slide_texts)\n else:\n return slide_texts\n\n except Exception as e:\n raise e" }, { "path": "application/parser/file/rst_parser.py", "content": "\"\"\"reStructuredText parser.\n\nContains parser for md files.\n\n\"\"\"\nimport re\nfrom pathlib import Path\nfrom typing import Any, Dict, List, Optional, Tuple, Union\n\nfrom application.parser.file.base_parser import BaseParser\n\n\nclass RstParser(BaseParser):\n \"\"\"reStructuredText parser.\n\n Extract text from .rst files.\n Returns dictionary with keys as headers and values as the text between headers.\n\n \"\"\"\n\n def __init__(\n self,\n *args: Any,\n remove_hyperlinks: bool = True,\n remove_images: bool = True,\n remove_table_excess: bool = True,\n remove_interpreters: bool = True,\n remove_directives: bool = True,\n remove_whitespaces_excess: bool = True,\n # Be careful with remove_characters_excess, might cause data loss\n remove_characters_excess: bool = True,\n **kwargs: Any,\n ) -> None:\n \"\"\"Init params.\"\"\"\n super().__init__(*args, **kwargs)\n self._remove_hyperlinks = remove_hyperlinks\n self._remove_images = remove_images\n self._remove_table_excess = remove_table_excess\n self._remove_interpreters = remove_interpreters\n self._remove_directives = remove_directives\n self._remove_whitespaces_excess = remove_whitespaces_excess\n self._remove_characters_excess = remove_characters_excess\n\n def rst_to_tups(self, rst_text: str) -> List[Tuple[Optional[str], str]]:\n \"\"\"Convert a reStructuredText file to a dictionary.\n\n The keys are the headers and the values are the text under each header.\n\n \"\"\"\n rst_tups: List[Tuple[Optional[str], str]] = []\n lines = rst_text.split(\"\\n\")\n\n current_header = None\n current_text = \"\"\n\n for i, line in enumerate(lines):\n header_match = re.match(r\"^[^\\S\\n]*[-=]+[^\\S\\n]*$\", line)\n if header_match and i > 0 and (\n len(lines[i - 1].strip()) == len(header_match.group().strip()) or lines[i - 2] == lines[i - 2]):\n if current_header is not None:\n if current_text == \"\" or None:\n continue\n # removes the next heading from current Document\n if current_text.endswith(lines[i - 1] + \"\\n\"):\n current_text = current_text[:len(current_text) - len(lines[i - 1] + \"\\n\")]\n rst_tups.append((current_header, current_text))\n\n current_header = lines[i - 1]\n current_text = \"\"\n else:\n current_text += line + \"\\n\"\n\n rst_tups.append((current_header, current_text))\n\n # TODO: Format for rst\n #\n # if current_header is not None:\n # # pass linting, assert keys are defined\n # rst_tups = [\n # (re.sub(r\"#\", \"\", cast(str, key)).strip(), re.sub(r\"<.*?>\", \"\", value))\n # for key, value in rst_tups\n # ]\n # else:\n # rst_tups = [\n # (key, re.sub(\"\\n\", \"\", value)) for key, value in rst_tups\n # ]\n\n if current_header is None:\n rst_tups = [\n (key, re.sub(\"\\n\", \"\", value)) for key, value in rst_tups\n ]\n return rst_tups\n\n def chunk_by_token_count(self, text: str, max_tokens: int = 100) -> List[str]:\n \"\"\"Chunk text by token count.\"\"\"\n \n avg_token_length = 5\n \n chunk_size = max_tokens * avg_token_length\n\n chunks = []\n for i in range(0, len(text), chunk_size):\n chunk = text[i:i+chunk_size]\n if i + chunk_size < len(text):\n last_space = chunk.rfind(' ')\n if last_space != -1:\n chunk = chunk[:last_space]\n \n chunks.append(chunk.strip())\n \n return chunks\n \n def remove_images(self, content: str) -> str:\n pattern = r\"\\.\\. image:: (.*)\"\n content = re.sub(pattern, \"\", content)\n return content\n\n def remove_hyperlinks(self, content: str) -> str:\n pattern = r\"`(.*?) <(.*?)>`_\"\n content = re.sub(pattern, r\"\\1\", content)\n return content\n\n def remove_directives(self, content: str) -> str:\n \"\"\"Removes reStructuredText Directives\"\"\"\n pattern = r\"`\\.\\.([^:]+)::\"\n content = re.sub(pattern, \"\", content)\n return content\n\n def remove_interpreters(self, content: str) -> str:\n \"\"\"Removes reStructuredText Interpreted Text Roles\"\"\"\n pattern = r\":(\\w+):\"\n content = re.sub(pattern, \"\", content)\n return content\n\n def remove_table_excess(self, content: str) -> str:\n \"\"\"Pattern to remove grid table separators\"\"\"\n pattern = r\"^\\+[-]+\\+[-]+\\+$\"\n content = re.sub(pattern, \"\", content, flags=re.MULTILINE)\n return content\n\n def remove_whitespaces_excess(self, content: List[Tuple[str, Any]]) -> List[Tuple[str, Any]]:\n \"\"\"Pattern to match 2 or more consecutive whitespaces\"\"\"\n pattern = r\"\\s{2,}\"\n content = [(key, re.sub(pattern, \" \", value)) for key, value in content]\n return content\n\n def remove_characters_excess(self, content: List[Tuple[str, Any]]) -> List[Tuple[str, Any]]:\n \"\"\"Pattern to match 2 or more consecutive characters\"\"\"\n pattern = r\"(\\S)\\1{2,}\"\n content = [(key, re.sub(pattern, r\"\\1\\1\\1\", value, flags=re.MULTILINE)) for key, value in content]\n return content\n\n def _init_parser(self) -> Dict:\n \"\"\"Initialize the parser with the config.\"\"\"\n return {}\n\n def parse_tups(\n self, filepath: Path, errors: str = \"ignore\",max_tokens: Optional[int] = 1000\n ) -> List[Tuple[Optional[str], str]]:\n \"\"\"Parse file into tuples.\"\"\"\n with open(filepath, \"r\") as f:\n content = f.read()\n if self._remove_hyperlinks:\n content = self.remove_hyperlinks(content)\n if self._remove_images:\n content = self.remove_images(content)\n if self._remove_table_excess:\n content = self.remove_table_excess(content)\n if self._remove_directives:\n content = self.remove_directives(content)\n if self._remove_interpreters:\n content = self.remove_interpreters(content)\n rst_tups = self.rst_to_tups(content)\n if self._remove_whitespaces_excess:\n rst_tups = self.remove_whitespaces_excess(rst_tups)\n if self._remove_characters_excess:\n rst_tups = self.remove_characters_excess(rst_tups)\n\n # Apply chunking if max_tokens is provided\n if max_tokens is not None:\n chunked_tups = []\n for header, text in rst_tups:\n chunks = self.chunk_by_token_count(text, max_tokens)\n for idx, chunk in enumerate(chunks):\n chunked_tups.append((f\"{header} - Chunk {idx + 1}\", chunk))\n return chunked_tups \n return rst_tups\n\n def parse_file(\n self, filepath: Path, errors: str = \"ignore\"\n ) -> Union[str, List[str]]:\n \"\"\"Parse file into string.\"\"\"\n tups = self.parse_tups(filepath, errors=errors)\n results = []\n # TODO: don't include headers right now\n for header, value in tups:\n if header is None:\n results.append(value)\n else:\n results.append(f\"\\n\\n{header}\\n{value}\")\n return results\n" }, { "path": "application/parser/file/tabular_parser.py", "content": "\"\"\"Tabular parser.\n\nContains parsers for tabular data files.\n\n\"\"\"\nfrom pathlib import Path\nfrom typing import Any, Dict, List, Union\n\nfrom application.parser.file.base_parser import BaseParser\n\n\nclass CSVParser(BaseParser):\n \"\"\"CSV parser.\n\n Args:\n concat_rows (bool): whether to concatenate all rows into one document.\n If set to False, a Document will be created for each row.\n True by default.\n\n \"\"\"\n\n def __init__(self, *args: Any, concat_rows: bool = True, **kwargs: Any) -> None:\n \"\"\"Init params.\"\"\"\n super().__init__(*args, **kwargs)\n self._concat_rows = concat_rows\n\n def _init_parser(self) -> Dict:\n \"\"\"Init parser.\"\"\"\n return {}\n\n def parse_file(self, file: Path, errors: str = \"ignore\") -> Union[str, List[str]]:\n \"\"\"Parse file.\n\n Returns:\n Union[str, List[str]]: a string or a List of strings.\n\n \"\"\"\n try:\n import csv\n except ImportError:\n raise ValueError(\"csv module is required to read CSV files.\")\n text_list = []\n with open(file, \"r\") as fp:\n csv_reader = csv.reader(fp)\n for row in csv_reader:\n text_list.append(\", \".join(row))\n if self._concat_rows:\n return \"\\n\".join(text_list)\n else:\n return text_list\n\n\nclass PandasCSVParser(BaseParser):\n r\"\"\"Pandas-based CSV parser.\n\n Parses CSVs using the separator detection from Pandas `read_csv`function.\n If special parameters are required, use the `pandas_config` dict.\n\n Args:\n concat_rows (bool): whether to concatenate all rows into one document.\n If set to False, a Document will be created for each row.\n True by default.\n\n col_joiner (str): Separator to use for joining cols per row.\n Set to \", \" by default.\n\n row_joiner (str): Separator to use for joining each row.\n Only used when `concat_rows=True`.\n Set to \"\\n\" by default.\n\n pandas_config (dict): Options for the `pandas.read_csv` function call.\n Refer to https://pandas.pydata.org/docs/reference/api/pandas.read_csv.html\n for more information.\n Set to empty dict by default, this means pandas will try to figure\n out the separators, table head, etc. on its own.\n \n header_period (int): Controls how headers are included in output:\n - 0: Headers only at the beginning\n - 1: Headers in every row\n - N > 1: Headers every N rows\n \n header_prefix (str): Prefix for header rows. Default is \"HEADERS: \".\n \"\"\"\n\n def __init__(\n self,\n *args: Any,\n concat_rows: bool = True,\n col_joiner: str = \", \",\n row_joiner: str = \"\\n\",\n pandas_config: dict = {},\n header_period: int = 20,\n header_prefix: str = \"HEADERS: \",\n **kwargs: Any\n ) -> None:\n \"\"\"Init params.\"\"\"\n super().__init__(*args, **kwargs)\n self._concat_rows = concat_rows\n self._col_joiner = col_joiner\n self._row_joiner = row_joiner\n self._pandas_config = pandas_config\n self._header_period = header_period\n self._header_prefix = header_prefix\n\n def _init_parser(self) -> Dict:\n \"\"\"Init parser.\"\"\"\n return {}\n\n def parse_file(self, file: Path, errors: str = \"ignore\") -> Union[str, List[str]]:\n \"\"\"Parse file.\"\"\"\n try:\n import pandas as pd\n except ImportError:\n raise ValueError(\"pandas module is required to read CSV files.\")\n\n df = pd.read_csv(file, **self._pandas_config)\n headers = df.columns.tolist()\n header_row = f\"{self._header_prefix}{self._col_joiner.join(headers)}\"\n\n if not self._concat_rows:\n return df.apply(\n lambda row: (self._col_joiner).join(row.astype(str).tolist()), axis=1\n ).tolist()\n \n text_list = []\n if self._header_period != 1:\n text_list.append(header_row)\n \n for i, row in df.iterrows():\n if (self._header_period > 1 and i > 0 and i % self._header_period == 0):\n text_list.append(header_row)\n text_list.append(self._col_joiner.join(row.astype(str).tolist()))\n if self._header_period == 1 and i < len(df) - 1:\n text_list.append(header_row)\n\n return self._row_joiner.join(text_list)\n\n\nclass ExcelParser(BaseParser):\n r\"\"\"Excel (.xlsx) parser.\n\n Parses Excel files using Pandas `read_excel` function.\n If special parameters are required, use the `pandas_config` dict.\n\n Args:\n concat_rows (bool): whether to concatenate all rows into one document.\n If set to False, a Document will be created for each row.\n True by default.\n\n col_joiner (str): Separator to use for joining cols per row.\n Set to \", \" by default.\n\n row_joiner (str): Separator to use for joining each row.\n Only used when `concat_rows=True`.\n Set to \"\\n\" by default.\n\n pandas_config (dict): Options for the `pandas.read_excel` function call.\n Refer to https://pandas.pydata.org/docs/reference/api/pandas.read_excel.html\n for more information.\n Set to empty dict by default, this means pandas will try to figure\n out the table structure on its own.\n \n header_period (int): Controls how headers are included in output:\n - 0: Headers only at the beginning (default)\n - 1: Headers in every row\n - N > 1: Headers every N rows\n \n header_prefix (str): Prefix for header rows. Default is \"HEADERS: \".\n \"\"\"\n\n def __init__(\n self,\n *args: Any,\n concat_rows: bool = True,\n col_joiner: str = \", \",\n row_joiner: str = \"\\n\",\n pandas_config: dict = {},\n header_period: int = 20,\n header_prefix: str = \"HEADERS: \",\n **kwargs: Any\n ) -> None:\n \"\"\"Init params.\"\"\"\n super().__init__(*args, **kwargs)\n self._concat_rows = concat_rows\n self._col_joiner = col_joiner\n self._row_joiner = row_joiner\n self._pandas_config = pandas_config\n self._header_period = header_period\n self._header_prefix = header_prefix\n\n def _init_parser(self) -> Dict:\n \"\"\"Init parser.\"\"\"\n return {}\n\n def parse_file(self, file: Path, errors: str = \"ignore\") -> Union[str, List[str]]:\n \"\"\"Parse file.\"\"\"\n try:\n import pandas as pd\n except ImportError:\n raise ValueError(\"pandas module is required to read Excel files.\")\n\n df = pd.read_excel(file, **self._pandas_config)\n headers = df.columns.tolist()\n header_row = f\"{self._header_prefix}{self._col_joiner.join(headers)}\"\n \n if not self._concat_rows:\n return df.apply(\n lambda row: (self._col_joiner).join(row.astype(str).tolist()), axis=1\n ).tolist()\n \n text_list = []\n if self._header_period != 1:\n text_list.append(header_row)\n\n for i, row in df.iterrows():\n if (self._header_period > 1 and i > 0 and i % self._header_period == 0):\n text_list.append(header_row)\n text_list.append(self._col_joiner.join(row.astype(str).tolist()))\n if self._header_period == 1 and i < len(df) - 1:\n text_list.append(header_row)\n return self._row_joiner.join(text_list)" }, { "path": "application/parser/remote/base.py", "content": "\"\"\"Base reader class.\"\"\"\nfrom abc import abstractmethod\nfrom typing import Any, List\n\nfrom langchain_core.documents import Document as LCDocument\nfrom application.parser.schema.base import Document\n\n\nclass BaseRemote:\n \"\"\"Utilities for loading data from a directory.\"\"\"\n\n @abstractmethod\n def load_data(self, *args: Any, **load_kwargs: Any) -> List[Document]:\n \"\"\"Load data from the input directory.\"\"\"\n\n def load_langchain_documents(self, **load_kwargs: Any) -> List[LCDocument]:\n \"\"\"Load data in LangChain document format.\"\"\"\n docs = self.load_data(**load_kwargs)\n return [d.to_langchain_format() for d in docs]\n" }, { "path": "application/parser/remote/crawler_loader.py", "content": "import logging\nimport os\nimport requests\nfrom urllib.parse import urlparse, urljoin\nfrom bs4 import BeautifulSoup\nfrom application.parser.remote.base import BaseRemote\nfrom application.parser.schema.base import Document\nfrom application.core.url_validation import validate_url, SSRFError\nfrom langchain_community.document_loaders import WebBaseLoader\n\nclass CrawlerLoader(BaseRemote):\n def __init__(self, limit=10):\n self.loader = WebBaseLoader # Initialize the document loader\n self.limit = limit # Set the limit for the number of pages to scrape\n\n def load_data(self, inputs):\n url = inputs\n if isinstance(url, list) and url:\n url = url[0]\n\n # Validate URL to prevent SSRF attacks\n try:\n url = validate_url(url)\n except SSRFError as e:\n logging.error(f\"URL validation failed: {e}\")\n return []\n\n visited_urls = set()\n base_url = urlparse(url).scheme + \"://\" + urlparse(url).hostname\n urls_to_visit = [url]\n loaded_content = []\n\n while urls_to_visit:\n current_url = urls_to_visit.pop(0)\n visited_urls.add(current_url)\n\n try:\n # Validate each URL before making requests\n try:\n validate_url(current_url)\n except SSRFError as e:\n logging.warning(f\"Skipping URL due to validation failure: {current_url} - {e}\")\n continue\n\n response = requests.get(current_url, timeout=30)\n response.raise_for_status()\n loader = self.loader([current_url])\n docs = loader.load()\n # Convert the loaded documents to your Document schema\n for doc in docs:\n metadata = dict(doc.metadata or {})\n source_url = metadata.get(\"source\") or current_url\n metadata[\"file_path\"] = self._url_to_virtual_path(source_url)\n loaded_content.append(\n Document(\n doc.page_content,\n extra_info=metadata\n )\n )\n except Exception as e:\n logging.error(f\"Error processing URL {current_url}: {e}\", exc_info=True)\n continue\n\n # Parse the HTML content to extract all links\n soup = BeautifulSoup(response.text, 'html.parser')\n all_links = [\n urljoin(current_url, a['href'])\n for a in soup.find_all('a', href=True)\n if base_url in urljoin(current_url, a['href'])\n ]\n\n # Add new links to the list of URLs to visit if they haven't been visited yet\n urls_to_visit.extend([link for link in all_links if link not in visited_urls])\n urls_to_visit = list(set(urls_to_visit))\n\n # Stop crawling if the limit of pages to scrape is reached\n if self.limit is not None and len(visited_urls) >= self.limit:\n break\n\n return loaded_content\n\n def _url_to_virtual_path(self, url):\n \"\"\"\n Convert a URL to a virtual file path ending with .md.\n\n Examples:\n https://docs.docsgpt.cloud/ -> index.md\n https://docs.docsgpt.cloud/guides/setup -> guides/setup.md\n https://docs.docsgpt.cloud/guides/setup/ -> guides/setup.md\n https://example.com/page.html -> page.md\n \"\"\"\n parsed = urlparse(url)\n path = parsed.path.strip(\"/\")\n\n if not path:\n return \"index.md\"\n\n # Remove common file extensions and add .md\n base, ext = os.path.splitext(path)\n if ext.lower() in [\".html\", \".htm\", \".php\", \".asp\", \".aspx\", \".jsp\"]:\n path = base\n\n if not path.endswith(\".md\"):\n path = f\"{path}.md\"\n\n return path\n" }, { "path": "application/parser/remote/crawler_markdown.py", "content": "import requests\nfrom urllib.parse import urlparse, urljoin\nfrom bs4 import BeautifulSoup\nfrom application.parser.remote.base import BaseRemote\nfrom application.core.url_validation import validate_url, SSRFError\nimport re\nfrom markdownify import markdownify\nfrom application.parser.schema.base import Document\nimport tldextract\nimport os\n\nclass CrawlerLoader(BaseRemote):\n def __init__(self, limit=10, allow_subdomains=False):\n \"\"\"\n Given a URL crawl web pages up to `self.limit`,\n convert HTML content to Markdown, and returning a list of Document objects.\n\n :param limit: The maximum number of pages to crawl.\n :param allow_subdomains: If True, crawl pages on subdomains of the base domain.\n \"\"\"\n self.limit = limit\n self.allow_subdomains = allow_subdomains\n self.session = requests.Session()\n\n def load_data(self, inputs):\n url = inputs\n if isinstance(url, list) and url:\n url = url[0]\n\n # Validate URL to prevent SSRF attacks\n try:\n url = validate_url(url)\n except SSRFError as e:\n print(f\"URL validation failed: {e}\")\n return []\n\n # Keep track of visited URLs to avoid revisiting the same page\n visited_urls = set()\n\n # Determine the base domain for link filtering using tldextract\n base_domain = self._get_base_domain(url)\n urls_to_visit = {url}\n documents = []\n\n while urls_to_visit:\n current_url = urls_to_visit.pop()\n\n # Skip if already visited\n if current_url in visited_urls:\n continue\n visited_urls.add(current_url)\n\n # Fetch the page content\n html_content = self._fetch_page(current_url)\n if html_content is None:\n continue\n\n # Convert the HTML to Markdown for cleaner text formatting\n title, language, processed_markdown = self._process_html_to_markdown(html_content, current_url)\n if processed_markdown:\n # Generate virtual file path from URL for consistent file-like matching\n virtual_path = self._url_to_virtual_path(current_url)\n \n # Create a Document for each visited page\n documents.append(\n Document(\n processed_markdown, # content\n None, # doc_id\n None, # embedding\n {\n \"source\": current_url,\n \"title\": title,\n \"language\": language,\n \"file_path\": virtual_path,\n }, # extra_info\n )\n )\n\n # Extract links and filter them according to domain rules\n new_links = self._extract_links(html_content, current_url)\n filtered_links = self._filter_links(new_links, base_domain)\n\n # Add any new, not-yet-visited links to the queue\n urls_to_visit.update(link for link in filtered_links if link not in visited_urls)\n\n # If we've reached the limit, stop crawling\n if self.limit is not None and len(visited_urls) >= self.limit:\n break\n\n return documents\n\n def _fetch_page(self, url):\n try:\n # Validate URL before fetching to prevent SSRF\n validate_url(url)\n response = self.session.get(url, timeout=10)\n response.raise_for_status()\n return response.text\n except SSRFError as e:\n print(f\"URL validation failed for {url}: {e}\")\n return None\n except requests.exceptions.RequestException as e:\n print(f\"Error fetching URL {url}: {e}\")\n return None\n\n def _process_html_to_markdown(self, html_content, current_url):\n soup = BeautifulSoup(html_content, 'html.parser')\n title_tag = soup.find('title')\n title = title_tag.text.strip() if title_tag else \"No Title\"\n\n # Extract language\n language_tag = soup.find('html')\n language = language_tag.get('lang', 'en') if language_tag else \"en\"\n\n markdownified = markdownify(html_content, heading_style=\"ATX\", newline_style=\"BACKSLASH\")\n # Reduce sequences of more than two newlines to exactly three\n markdownified = re.sub(r'\\n{3,}', '\\n\\n\\n', markdownified)\n return title, language, markdownified\n\n def _extract_links(self, html_content, current_url):\n soup = BeautifulSoup(html_content, 'html.parser')\n links = []\n for a in soup.find_all('a', href=True):\n full_url = urljoin(current_url, a['href'])\n links.append((full_url, a.text.strip()))\n return links\n\n def _get_base_domain(self, url):\n extracted = tldextract.extract(url)\n # Reconstruct the domain as domain.suffix\n base_domain = f\"{extracted.domain}.{extracted.suffix}\"\n return base_domain\n\n def _filter_links(self, links, base_domain):\n \"\"\"\n Filter the extracted links to only include those that match the crawling criteria:\n - If allow_subdomains is True, allow any link whose domain ends with the base_domain.\n - If allow_subdomains is False, only allow exact matches of the base_domain.\n \"\"\"\n filtered = []\n for link, _ in links:\n parsed_link = urlparse(link)\n if not parsed_link.netloc:\n continue\n\n extracted = tldextract.extract(parsed_link.netloc)\n link_base = f\"{extracted.domain}.{extracted.suffix}\"\n\n if self.allow_subdomains:\n # For subdomains: sub.example.com ends with example.com\n if link_base == base_domain or link_base.endswith(\".\" + base_domain):\n filtered.append(link)\n else:\n # Exact domain match\n if link_base == base_domain:\n filtered.append(link)\n return filtered\n\n def _url_to_virtual_path(self, url):\n \"\"\"\n Convert a URL to a virtual file path ending with .md.\n\n Examples:\n https://docs.docsgpt.cloud/ -> index.md\n https://docs.docsgpt.cloud/guides/setup -> guides/setup.md\n https://docs.docsgpt.cloud/guides/setup/ -> guides/setup.md\n https://example.com/page.html -> page.md\n \"\"\"\n parsed = urlparse(url)\n path = parsed.path.strip(\"/\")\n\n if not path:\n return \"index.md\"\n\n # Remove common file extensions and add .md\n base, ext = os.path.splitext(path)\n if ext.lower() in [\".html\", \".htm\", \".php\", \".asp\", \".aspx\", \".jsp\"]:\n path = base\n\n # Ensure path ends with .md\n if not path.endswith(\".md\"):\n path = path + \".md\"\n\n return path" }, { "path": "application/parser/remote/github_loader.py", "content": "import base64\nimport requests\nimport time\nfrom typing import List, Optional\nfrom application.parser.remote.base import BaseRemote\nfrom application.parser.schema.base import Document\nimport mimetypes\nfrom application.core.settings import settings\n\nclass GitHubLoader(BaseRemote):\n def __init__(self):\n self.access_token = settings.GITHUB_ACCESS_TOKEN\n self.headers = {\n \"Authorization\": f\"token {self.access_token}\",\n \"Accept\": \"application/vnd.github.v3+json\"\n } if self.access_token else {\n \"Accept\": \"application/vnd.github.v3+json\"\n }\n return\n\n def is_text_file(self, file_path: str) -> bool:\n \"\"\"Determine if a file is a text file based on extension.\"\"\"\n # Common text file extensions\n text_extensions = {\n '.txt', '.md', '.markdown', '.rst', '.json', '.xml', '.yaml', '.yml',\n '.py', '.js', '.ts', '.jsx', '.tsx', '.java', '.c', '.cpp', '.h', '.hpp',\n '.cs', '.go', '.rs', '.rb', '.php', '.swift', '.kt', '.scala',\n '.html', '.css', '.scss', '.sass', '.less',\n '.sh', '.bash', '.zsh', '.fish',\n '.sql', '.r', '.m', '.mat',\n '.ini', '.cfg', '.conf', '.config', '.env',\n '.gitignore', '.dockerignore', '.editorconfig',\n '.log', '.csv', '.tsv'\n }\n\n # Get file extension\n file_lower = file_path.lower()\n for ext in text_extensions:\n if file_lower.endswith(ext):\n return True\n\n # Also check MIME type\n mime_type, _ = mimetypes.guess_type(file_path)\n if mime_type and (mime_type.startswith(\"text\") or mime_type in [\"application/json\", \"application/xml\"]):\n return True\n\n return False\n\n def fetch_file_content(self, repo_url: str, file_path: str) -> Optional[str]:\n \"\"\"Fetch file content. Returns None if file should be skipped (binary files or empty files).\"\"\"\n url = f\"https://api.github.com/repos/{repo_url}/contents/{file_path}\"\n response = self._make_request(url)\n\n content = response.json()\n\n if content.get(\"encoding\") == \"base64\":\n if self.is_text_file(file_path): # Handle only text files\n try:\n decoded_content = base64.b64decode(content[\"content\"]).decode(\"utf-8\").strip()\n # Skip empty files\n if not decoded_content:\n return None\n return decoded_content\n except Exception:\n # If decoding fails, it's probably a binary file\n return None\n else:\n # Skip binary files by returning None\n return None\n else:\n file_content = content['content'].strip()\n # Skip empty files\n if not file_content:\n return None\n return file_content\n\n def _make_request(self, url: str, max_retries: int = 3) -> requests.Response:\n \"\"\"Make a request with retry logic for rate limiting\"\"\"\n for attempt in range(max_retries):\n response = requests.get(url, headers=self.headers)\n\n if response.status_code == 200:\n return response\n elif response.status_code == 403:\n # Check if it's a rate limit issue\n try:\n error_data = response.json()\n error_msg = error_data.get(\"message\", \"\")\n\n # Check rate limit headers\n remaining = response.headers.get(\"X-RateLimit-Remaining\", \"unknown\")\n reset_time = response.headers.get(\"X-RateLimit-Reset\", \"unknown\")\n\n print(f\"GitHub API 403 Error: {error_msg}\")\n print(f\"Rate limit remaining: {remaining}, Reset time: {reset_time}\")\n\n if \"rate limit\" in error_msg.lower():\n if attempt < max_retries - 1:\n wait_time = 2 ** attempt # Exponential backoff\n print(f\"Rate limit hit, waiting {wait_time} seconds before retry...\")\n time.sleep(wait_time)\n continue\n\n # Provide helpful error message\n if remaining == \"0\":\n raise Exception(f\"GitHub API rate limit exceeded. Please set GITHUB_ACCESS_TOKEN environment variable. Reset time: {reset_time}\")\n else:\n raise Exception(f\"GitHub API error: {error_msg}. This may require authentication - set GITHUB_ACCESS_TOKEN environment variable.\")\n except Exception as e:\n if isinstance(e, Exception) and \"GitHub API\" in str(e):\n raise\n # If we can't parse the response, raise the original error\n response.raise_for_status()\n else:\n response.raise_for_status()\n\n return response\n\n def fetch_repo_files(self, repo_url: str, path: str = \"\") -> List[str]:\n url = f\"https://api.github.com/repos/{repo_url}/contents/{path}\"\n response = self._make_request(url)\n\n contents = response.json()\n\n # Handle error responses from GitHub API\n if isinstance(contents, dict) and \"message\" in contents:\n raise Exception(f\"GitHub API error: {contents.get('message')}\")\n\n # Ensure contents is a list\n if not isinstance(contents, list):\n raise TypeError(f\"Expected list from GitHub API, got {type(contents).__name__}: {contents}\")\n\n files = []\n for item in contents:\n if item[\"type\"] == \"file\":\n files.append(item[\"path\"])\n elif item[\"type\"] == \"dir\":\n files.extend(self.fetch_repo_files(repo_url, item[\"path\"]))\n return files\n\n def load_data(self, repo_url: str) -> List[Document]:\n repo_name = repo_url.split(\"github.com/\")[-1]\n files = self.fetch_repo_files(repo_name)\n documents = []\n for file_path in files:\n content = self.fetch_file_content(repo_name, file_path)\n # Skip binary files (content is None)\n if content is None:\n continue\n documents.append(Document(\n text=content,\n doc_id=file_path,\n extra_info={\n \"title\": file_path,\n \"source\": f\"https://github.com/{repo_name}/blob/main/{file_path}\"\n }\n ))\n return documents\n" }, { "path": "application/parser/remote/reddit_loader.py", "content": "from application.parser.remote.base import BaseRemote\nfrom langchain_community.document_loaders import RedditPostsLoader\nimport json\n\n\nclass RedditPostsLoaderRemote(BaseRemote):\n def load_data(self, inputs):\n try:\n data = json.loads(inputs)\n except json.JSONDecodeError as e:\n raise ValueError(f\"Invalid JSON input: {e}\")\n\n required_fields = [\"client_id\", \"client_secret\", \"user_agent\", \"search_queries\"]\n missing_fields = [field for field in required_fields if field not in data]\n if missing_fields:\n raise ValueError(f\"Missing required fields: {', '.join(missing_fields)}\")\n client_id = data.get(\"client_id\")\n client_secret = data.get(\"client_secret\")\n user_agent = data.get(\"user_agent\")\n categories = data.get(\"categories\", [\"new\", \"hot\"])\n mode = data.get(\"mode\", \"subreddit\")\n search_queries = data.get(\"search_queries\")\n number_posts = data.get(\"number_posts\", 10)\n self.loader = RedditPostsLoader(\n client_id=client_id,\n client_secret=client_secret,\n user_agent=user_agent,\n categories=categories,\n mode=mode,\n search_queries=search_queries,\n number_posts=number_posts,\n )\n documents = self.loader.load()\n print(f\"Loaded {len(documents)} documents from Reddit\")\n return documents\n" }, { "path": "application/parser/remote/remote_creator.py", "content": "from application.parser.remote.sitemap_loader import SitemapLoader\nfrom application.parser.remote.crawler_loader import CrawlerLoader\nfrom application.parser.remote.web_loader import WebLoader\nfrom application.parser.remote.reddit_loader import RedditPostsLoaderRemote\nfrom application.parser.remote.github_loader import GitHubLoader\nfrom application.parser.remote.s3_loader import S3Loader\n\n\nclass RemoteCreator:\n \"\"\"\n Factory class for creating remote content loaders.\n\n These loaders fetch content from remote web sources like URLs,\n sitemaps, web crawlers, social media platforms, etc.\n\n For external knowledge base connectors (like Google Drive),\n use ConnectorCreator instead.\n \"\"\"\n\n loaders = {\n \"url\": WebLoader,\n \"sitemap\": SitemapLoader,\n \"crawler\": CrawlerLoader,\n \"reddit\": RedditPostsLoaderRemote,\n \"github\": GitHubLoader,\n \"s3\": S3Loader,\n }\n\n @classmethod\n def create_loader(cls, type, *args, **kwargs):\n loader_class = cls.loaders.get(type.lower())\n if not loader_class:\n raise ValueError(f\"No loader class found for type {type}\")\n return loader_class(*args, **kwargs)\n" }, { "path": "application/parser/remote/s3_loader.py", "content": "import json\nimport logging\nimport os\nimport tempfile\nimport mimetypes\nfrom typing import List, Optional\nfrom application.parser.remote.base import BaseRemote\nfrom application.parser.schema.base import Document\n\ntry:\n import boto3\n from botocore.exceptions import ClientError, NoCredentialsError\nexcept ImportError:\n boto3 = None\n\nlogger = logging.getLogger(__name__)\n\n\nclass S3Loader(BaseRemote):\n \"\"\"Load documents from an AWS S3 bucket.\"\"\"\n\n def __init__(self):\n if boto3 is None:\n raise ImportError(\n \"boto3 is required for S3Loader. Install it with: pip install boto3\"\n )\n self.s3_client = None\n\n def _normalize_endpoint_url(self, endpoint_url: str, bucket: str) -> tuple[str, str]:\n \"\"\"\n Normalize endpoint URL for S3-compatible services.\n\n Detects common mistakes like using bucket-prefixed URLs and extracts\n the correct endpoint and bucket name.\n\n Args:\n endpoint_url: The provided endpoint URL\n bucket: The provided bucket name\n\n Returns:\n Tuple of (normalized_endpoint_url, bucket_name)\n \"\"\"\n import re\n from urllib.parse import urlparse\n\n if not endpoint_url:\n return endpoint_url, bucket\n\n parsed = urlparse(endpoint_url)\n host = parsed.netloc or parsed.path\n\n # Check for DigitalOcean Spaces bucket-prefixed URL pattern\n # e.g., https://mybucket.nyc3.digitaloceanspaces.com\n do_match = re.match(r\"^([^.]+)\\.([a-z0-9]+)\\.digitaloceanspaces\\.com$\", host)\n if do_match:\n extracted_bucket = do_match.group(1)\n region = do_match.group(2)\n correct_endpoint = f\"https://{region}.digitaloceanspaces.com\"\n logger.warning(\n f\"Detected bucket-prefixed DigitalOcean Spaces URL. \"\n f\"Extracted bucket '{extracted_bucket}' from endpoint. \"\n f\"Using endpoint: {correct_endpoint}\"\n )\n # If bucket wasn't provided or differs, use extracted one\n if not bucket or bucket != extracted_bucket:\n logger.info(f\"Using extracted bucket name: '{extracted_bucket}' (was: '{bucket}')\")\n bucket = extracted_bucket\n return correct_endpoint, bucket\n\n # Check for just \"digitaloceanspaces.com\" without region\n if host == \"digitaloceanspaces.com\":\n logger.error(\n \"Invalid DigitalOcean Spaces endpoint: missing region. \"\n \"Use format: https://.digitaloceanspaces.com (e.g., https://lon1.digitaloceanspaces.com)\"\n )\n\n return endpoint_url, bucket\n\n def _init_client(\n self,\n aws_access_key_id: str,\n aws_secret_access_key: str,\n region_name: str = \"us-east-1\",\n endpoint_url: Optional[str] = None,\n bucket: Optional[str] = None,\n ) -> Optional[str]:\n \"\"\"\n Initialize the S3 client with credentials.\n\n Returns:\n The potentially corrected bucket name if endpoint URL was normalized\n \"\"\"\n from botocore.config import Config\n\n client_kwargs = {\n \"aws_access_key_id\": aws_access_key_id,\n \"aws_secret_access_key\": aws_secret_access_key,\n \"region_name\": region_name,\n }\n\n logger.info(f\"Initializing S3 client with region: {region_name}\")\n\n corrected_bucket = bucket\n if endpoint_url:\n # Normalize the endpoint URL and potentially extract bucket name\n normalized_endpoint, corrected_bucket = self._normalize_endpoint_url(endpoint_url, bucket)\n logger.info(f\"Original endpoint URL: {endpoint_url}\")\n logger.info(f\"Normalized endpoint URL: {normalized_endpoint}\")\n logger.info(f\"Bucket name: '{corrected_bucket}'\")\n\n client_kwargs[\"endpoint_url\"] = normalized_endpoint\n # Use path-style addressing for S3-compatible services\n # (DigitalOcean Spaces, MinIO, etc.)\n client_kwargs[\"config\"] = Config(s3={\"addressing_style\": \"path\"})\n else:\n logger.info(\"Using default AWS S3 endpoint\")\n\n self.s3_client = boto3.client(\"s3\", **client_kwargs)\n logger.info(\"S3 client initialized successfully\")\n\n return corrected_bucket\n\n def is_text_file(self, file_path: str) -> bool:\n \"\"\"Determine if a file is a text file based on extension.\"\"\"\n text_extensions = {\n \".txt\",\n \".md\",\n \".markdown\",\n \".rst\",\n \".json\",\n \".xml\",\n \".yaml\",\n \".yml\",\n \".py\",\n \".js\",\n \".ts\",\n \".jsx\",\n \".tsx\",\n \".java\",\n \".c\",\n \".cpp\",\n \".h\",\n \".hpp\",\n \".cs\",\n \".go\",\n \".rs\",\n \".rb\",\n \".php\",\n \".swift\",\n \".kt\",\n \".scala\",\n \".html\",\n \".css\",\n \".scss\",\n \".sass\",\n \".less\",\n \".sh\",\n \".bash\",\n \".zsh\",\n \".fish\",\n \".sql\",\n \".r\",\n \".m\",\n \".mat\",\n \".ini\",\n \".cfg\",\n \".conf\",\n \".config\",\n \".env\",\n \".gitignore\",\n \".dockerignore\",\n \".editorconfig\",\n \".log\",\n \".csv\",\n \".tsv\",\n }\n\n file_lower = file_path.lower()\n for ext in text_extensions:\n if file_lower.endswith(ext):\n return True\n\n mime_type, _ = mimetypes.guess_type(file_path)\n if mime_type and (\n mime_type.startswith(\"text\")\n or mime_type in [\"application/json\", \"application/xml\"]\n ):\n return True\n\n return False\n\n def is_supported_document(self, file_path: str) -> bool:\n \"\"\"Check if file is a supported document type for parsing.\"\"\"\n document_extensions = {\n \".pdf\",\n \".docx\",\n \".doc\",\n \".xlsx\",\n \".xls\",\n \".pptx\",\n \".ppt\",\n \".epub\",\n \".odt\",\n \".rtf\",\n }\n\n file_lower = file_path.lower()\n for ext in document_extensions:\n if file_lower.endswith(ext):\n return True\n\n return False\n\n def list_objects(self, bucket: str, prefix: str = \"\") -> List[str]:\n \"\"\"\n List all objects in the bucket with the given prefix.\n\n Args:\n bucket: S3 bucket name\n prefix: Optional path prefix to filter objects\n\n Returns:\n List of object keys\n \"\"\"\n objects = []\n paginator = self.s3_client.get_paginator(\"list_objects_v2\")\n\n logger.info(f\"Listing objects in bucket: '{bucket}' with prefix: '{prefix}'\")\n logger.debug(f\"S3 client endpoint: {self.s3_client.meta.endpoint_url}\")\n\n try:\n page_count = 0\n for page in paginator.paginate(Bucket=bucket, Prefix=prefix):\n page_count += 1\n logger.debug(f\"Processing page {page_count}, keys in response: {list(page.keys())}\")\n if \"Contents\" in page:\n for obj in page[\"Contents\"]:\n key = obj[\"Key\"]\n if not key.endswith(\"/\"):\n objects.append(key)\n logger.debug(f\"Found object: {key}\")\n else:\n logger.info(f\"Page {page_count} has no 'Contents' key - bucket may be empty or prefix not found\")\n\n logger.info(f\"Found {len(objects)} objects in bucket '{bucket}'\")\n\n except ClientError as e:\n error_code = e.response.get(\"Error\", {}).get(\"Code\", \"\")\n error_message = e.response.get(\"Error\", {}).get(\"Message\", \"\")\n logger.error(f\"ClientError listing objects - Code: {error_code}, Message: {error_message}\")\n logger.error(f\"Full error response: {e.response}\")\n logger.error(f\"Bucket: '{bucket}', Prefix: '{prefix}', Endpoint: {self.s3_client.meta.endpoint_url}\")\n\n if error_code == \"NoSuchBucket\":\n raise Exception(f\"S3 bucket '{bucket}' does not exist\")\n elif error_code == \"AccessDenied\":\n raise Exception(\n f\"Access denied to S3 bucket '{bucket}'. Check your credentials and permissions.\"\n )\n elif error_code == \"NoSuchKey\":\n # This is unusual for ListObjectsV2 - may indicate endpoint/bucket configuration issue\n logger.error(\n \"NoSuchKey error on ListObjectsV2 - this may indicate the bucket name \"\n \"is incorrect or the endpoint URL format is wrong. \"\n \"For DigitalOcean Spaces, the endpoint should be like: \"\n \"https://.digitaloceanspaces.com and bucket should be just the space name.\"\n )\n raise Exception(\n f\"S3 error: {e}. For S3-compatible services, verify: \"\n f\"1) Endpoint URL format (e.g., https://nyc3.digitaloceanspaces.com), \"\n f\"2) Bucket name is just the space/bucket name without region prefix\"\n )\n else:\n raise Exception(f\"S3 error: {e}\")\n except NoCredentialsError:\n raise Exception(\n \"AWS credentials not found. Please provide valid credentials.\"\n )\n\n return objects\n\n def get_object_content(self, bucket: str, key: str) -> Optional[str]:\n \"\"\"\n Get the content of an S3 object as text.\n\n Args:\n bucket: S3 bucket name\n key: Object key\n\n Returns:\n File content as string, or None if file should be skipped\n \"\"\"\n if not self.is_text_file(key) and not self.is_supported_document(key):\n return None\n\n try:\n response = self.s3_client.get_object(Bucket=bucket, Key=key)\n content = response[\"Body\"].read()\n\n if self.is_text_file(key):\n try:\n decoded_content = content.decode(\"utf-8\").strip()\n if not decoded_content:\n return None\n return decoded_content\n except UnicodeDecodeError:\n return None\n elif self.is_supported_document(key):\n return self._process_document(content, key)\n\n except ClientError as e:\n error_code = e.response.get(\"Error\", {}).get(\"Code\", \"\")\n if error_code == \"NoSuchKey\":\n return None\n elif error_code == \"AccessDenied\":\n print(f\"Access denied to object: {key}\")\n return None\n else:\n print(f\"Error fetching object {key}: {e}\")\n return None\n\n return None\n\n def _process_document(self, content: bytes, key: str) -> Optional[str]:\n \"\"\"\n Process a document file (PDF, DOCX, etc.) and extract text.\n\n Args:\n content: File content as bytes\n key: Object key (filename)\n\n Returns:\n Extracted text content\n \"\"\"\n ext = os.path.splitext(key)[1].lower()\n\n with tempfile.NamedTemporaryFile(suffix=ext, delete=False) as tmp_file:\n tmp_file.write(content)\n tmp_path = tmp_file.name\n\n try:\n from application.parser.file.bulk import SimpleDirectoryReader\n\n reader = SimpleDirectoryReader(input_files=[tmp_path])\n documents = reader.load_data()\n if documents:\n return \"\\n\\n\".join(doc.text for doc in documents if doc.text)\n return None\n except Exception as e:\n print(f\"Error processing document {key}: {e}\")\n return None\n finally:\n if os.path.exists(tmp_path):\n os.unlink(tmp_path)\n\n def load_data(self, inputs) -> List[Document]:\n \"\"\"\n Load documents from an S3 bucket.\n\n Args:\n inputs: JSON string or dict containing:\n - aws_access_key_id: AWS access key ID\n - aws_secret_access_key: AWS secret access key\n - bucket: S3 bucket name\n - prefix: Optional path prefix to filter objects\n - region: AWS region (default: us-east-1)\n - endpoint_url: Custom S3 endpoint URL (for MinIO, R2, etc.)\n\n Returns:\n List of Document objects\n \"\"\"\n if isinstance(inputs, str):\n try:\n data = json.loads(inputs)\n except json.JSONDecodeError as e:\n raise ValueError(f\"Invalid JSON input: {e}\")\n else:\n data = inputs\n\n required_fields = [\"aws_access_key_id\", \"aws_secret_access_key\", \"bucket\"]\n missing_fields = [field for field in required_fields if not data.get(field)]\n if missing_fields:\n raise ValueError(f\"Missing required fields: {', '.join(missing_fields)}\")\n\n aws_access_key_id = data[\"aws_access_key_id\"]\n aws_secret_access_key = data[\"aws_secret_access_key\"]\n bucket = data[\"bucket\"]\n prefix = data.get(\"prefix\", \"\")\n region = data.get(\"region\", \"us-east-1\")\n endpoint_url = data.get(\"endpoint_url\", \"\")\n\n logger.info(f\"Loading data from S3 - Bucket: '{bucket}', Prefix: '{prefix}', Region: '{region}'\")\n if endpoint_url:\n logger.info(f\"Custom endpoint URL provided: '{endpoint_url}'\")\n\n corrected_bucket = self._init_client(\n aws_access_key_id, aws_secret_access_key, region, endpoint_url or None, bucket\n )\n\n # Use the corrected bucket name if endpoint URL normalization extracted one\n if corrected_bucket and corrected_bucket != bucket:\n logger.info(f\"Using corrected bucket name: '{corrected_bucket}' (original: '{bucket}')\")\n bucket = corrected_bucket\n\n objects = self.list_objects(bucket, prefix)\n documents = []\n\n for key in objects:\n content = self.get_object_content(bucket, key)\n if content is None:\n continue\n\n documents.append(\n Document(\n text=content,\n doc_id=key,\n extra_info={\n \"title\": os.path.basename(key),\n \"source\": f\"s3://{bucket}/{key}\",\n \"bucket\": bucket,\n \"key\": key,\n },\n )\n )\n\n logger.info(f\"Loaded {len(documents)} documents from S3 bucket '{bucket}'\")\n return documents\n" }, { "path": "application/parser/remote/sitemap_loader.py", "content": "import logging\nimport requests\nimport re # Import regular expression library\nimport defusedxml.ElementTree as ET\nfrom application.parser.remote.base import BaseRemote\nfrom application.core.url_validation import validate_url, SSRFError\n\nclass SitemapLoader(BaseRemote):\n def __init__(self, limit=20):\n from langchain_community.document_loaders import WebBaseLoader\n self.loader = WebBaseLoader\n self.limit = limit # Adding limit to control the number of URLs to process\n\n def load_data(self, inputs):\n sitemap_url= inputs\n # Check if the input is a list and if it is, use the first element\n if isinstance(sitemap_url, list) and sitemap_url:\n sitemap_url = sitemap_url[0]\n\n # Validate URL to prevent SSRF attacks\n try:\n sitemap_url = validate_url(sitemap_url)\n except SSRFError as e:\n logging.error(f\"URL validation failed: {e}\")\n return []\n\n urls = self._extract_urls(sitemap_url)\n if not urls:\n print(f\"No URLs found in the sitemap: {sitemap_url}\")\n return []\n\n # Load content of extracted URLs\n documents = []\n processed_urls = 0 # Counter for processed URLs\n for url in urls:\n if self.limit is not None and processed_urls >= self.limit:\n break # Stop processing if the limit is reached\n\n try:\n loader = self.loader([url])\n documents.extend(loader.load())\n processed_urls += 1 # Increment the counter after processing each URL\n except Exception as e:\n logging.error(f\"Error processing URL {url}: {e}\", exc_info=True)\n continue\n\n return documents\n\n def _extract_urls(self, sitemap_url):\n try:\n # Validate URL before fetching to prevent SSRF\n validate_url(sitemap_url)\n response = requests.get(sitemap_url, timeout=30)\n response.raise_for_status() # Raise an exception for HTTP errors\n except SSRFError as e:\n print(f\"URL validation failed for sitemap: {sitemap_url}. Error: {e}\")\n return []\n except (requests.exceptions.HTTPError, requests.exceptions.ConnectionError) as e:\n print(f\"Failed to fetch sitemap: {sitemap_url}. Error: {e}\")\n return []\n\n # Determine if this is a sitemap or a URL\n if self._is_sitemap(response):\n # It's a sitemap, so parse it and extract URLs\n return self._parse_sitemap(response.content)\n else:\n # It's not a sitemap, return the URL itself\n return [sitemap_url]\n\n def _is_sitemap(self, response):\n content_type = response.headers.get('Content-Type', '')\n if 'xml' in content_type or response.url.endswith('.xml'):\n return True\n\n if ' None:\n \"\"\"Post init.\"\"\"\n if self.text is None:\n raise ValueError(\"text field not set.\")\n\n @classmethod\n def get_type(cls) -> str:\n \"\"\"Get Document type.\"\"\"\n return \"Document\"\n\n def to_langchain_format(self) -> LCDocument:\n \"\"\"Convert struct to LangChain document format.\"\"\"\n metadata = self.extra_info or {}\n return LCDocument(page_content=self.text, metadata=metadata)\n\n @classmethod\n def from_langchain_format(cls, doc: LCDocument) -> \"Document\":\n \"\"\"Convert struct from LangChain document format.\"\"\"\n return cls(text=doc.page_content, extra_info=doc.metadata)\n" }, { "path": "application/parser/schema/schema.py", "content": "\"\"\"Base schema for data structures.\"\"\"\nfrom abc import abstractmethod\nfrom dataclasses import dataclass\nfrom typing import Any, Dict, List, Optional\n\nfrom dataclasses_json import DataClassJsonMixin\n\n\n@dataclass\nclass BaseDocument(DataClassJsonMixin):\n \"\"\"Base document.\n\n Generic abstract interfaces that captures both index structs\n as well as documents.\n\n \"\"\"\n\n # TODO: consolidate fields from Document/IndexStruct into base class\n text: Optional[str] = None\n doc_id: Optional[str] = None\n embedding: Optional[List[float]] = None\n\n # extra fields\n extra_info: Optional[Dict[str, Any]] = None\n\n @classmethod\n @abstractmethod\n def get_type(cls) -> str:\n \"\"\"Get Document type.\"\"\"\n\n def get_text(self) -> str:\n \"\"\"Get text.\"\"\"\n if self.text is None:\n raise ValueError(\"text field not set.\")\n return self.text\n\n def get_doc_id(self) -> str:\n \"\"\"Get doc_id.\"\"\"\n if self.doc_id is None:\n raise ValueError(\"doc_id not set.\")\n return self.doc_id\n\n @property\n def is_doc_id_none(self) -> bool:\n \"\"\"Check if doc_id is None.\"\"\"\n return self.doc_id is None\n\n def get_embedding(self) -> List[float]:\n \"\"\"Get embedding.\n\n Errors if embedding is None.\n\n \"\"\"\n if self.embedding is None:\n raise ValueError(\"embedding not set.\")\n return self.embedding\n\n @property\n def extra_info_str(self) -> Optional[str]:\n \"\"\"Extra info string.\"\"\"\n if self.extra_info is None:\n return None\n\n return \"\\n\".join([f\"{k}: {str(v)}\" for k, v in self.extra_info.items()])\n" }, { "path": "application/prompts/chat_combine_creative.txt", "content": "You are a helpful AI assistant, DocsGPT. You are proactive and helpful. Try to use tools, if they are available to you, \nbe proactive and fill in missing information.\nUsers can Upload documents for your context as attachments or sources via UI using the Conversation input box.\nIf appropriate, your answers can include code examples, formatted as follows:\n```(language)\n(code)\n```\nUsers are also able to see charts and diagrams if you use them with valid mermaid syntax in your responses.\nTry to respond with mermaid charts if visualization helps with users queries.\nYou effectively utilize chat history, ensuring relevant and tailored responses. \nTry to use additional provided context if it's available, otherwise use your knowledge and tool capabilities.\nAllow yourself to be very creative and use your imagination.\n----------------\nPossible additional context from uploaded sources:\n{summaries}" }, { "path": "application/prompts/chat_combine_default.txt", "content": "You are a helpful AI assistant, DocsGPT. You are proactive and helpful. Try to use tools, if they are available to you, \nbe proactive and fill in missing information.\nUsers can Upload documents for your context as attachments or sources via UI using the Conversation input box.\nIf appropriate, your answers can include code examples, formatted as follows:\n```(language)\n(code)\n```\nUsers are also able to see charts and diagrams if you use them with valid mermaid syntax in your responses.\nTry to respond with mermaid charts if visualization helps with users queries.\nYou effectively utilize chat history, ensuring relevant and tailored responses. \nTry to use additional provided context if it's available, otherwise use your knowledge and tool capabilities.\n----------------\nPossible additional context from uploaded sources:\n{summaries}" }, { "path": "application/prompts/chat_combine_strict.txt", "content": "You are a helpful AI assistant, DocsGPT. You are proactive and helpful. Try to use tools, if they are available to you, \nbe proactive and fill in missing information.\nUsers can Upload documents for your context as attachments or sources via UI using the Conversation input box.\nIf appropriate, your answers can include code examples, formatted as follows:\n```(language)\n(code)\n```\nUsers are also able to see charts and diagrams if you use them with valid mermaid syntax in your responses.\nTry to respond with mermaid charts if visualization helps with users queries.\nYou effectively utilize chat history, ensuring relevant and tailored responses. \nUse context provided below or use available tools tool capabilities to answer user queries.\nIf you dont have enough information from the context or tools, answer \"I don't know\" or \"I don't have enough information\".\nNever make up information or provide false information!\nAllow yourself to be very creative and use your imagination.\n----------------\nContext from uploaded sources:\n{summaries}" }, { "path": "application/prompts/chat_reduce_prompt.txt", "content": "Use the following pieces of context to help answer the users question. If its not relevant to the question, respond with \"-\"\n----------------\n{context}" }, { "path": "application/prompts/compression/v1.0.txt", "content": "Your task is to create a detailed summary of the conversation so far, paying close attention to the user's explicit requests and your previous actions.\n\nThis summary should be thorough in capturing technical details, code patterns, and architectural decisions that would be essential for continuing work without losing context.\n\nBefore providing your final summary, wrap your analysis in tags to organize your thoughts and ensure you've covered all necessary points. In your analysis process:\n\n1. Chronologically analyze each message, tool call and section of the conversation. For each section thoroughly identify:\n - The user's explicit requests and intents\n - Your approach to addressing the user's requests\n - Key decisions, concepts and patterns\n - Specific details like if applicable:\n - file names\n - full code snippets\n - function signatures\n - file edits\n - Errors that you ran into and how you fixed them\n - Pay special attention to specific user feedback that you received, especially if the user told you to do something differently.\n\n2. Double-check for accuracy and completeness, addressing each required element thoroughly.\n\nYour summary should include the following sections:\n\n1. Primary Request and Intent: Capture all of the user's explicit requests and intents in detail\n2. Key Concepts: List all important concepts discussed.\n3. Files and Code Sections: Enumerate specific files and code sections examined, modified, or created. Pay special attention to the most recent messages and include full code snippets where applicable and include a summary of why this file read or edit is important.\n4. Errors and fixes: List all errors that you ran into, and how you fixed them. Pay special attention to specific user feedback that you received, especially if the user told you to do something differently.\n5. Problem Solving: Document problems solved and any ongoing troubleshooting efforts.\n6. All user messages: List ALL user messages that are not tool results. These are critical for understanding the users' feedback and changing intent.\n7. Tool Calls: List ALL tool calls made, including their inputs relevant parts of the outputs.\n8. Pending Tasks: Outline any pending tasks that you have explicitly been asked to work on.\n9. Current Work: Describe in detail precisely what was being worked on immediately before this summary request, paying special attention to the most recent messages from both user and assistant. Include file names and code snippets where applicable.\n10. Optional Next Step: List the next step that you will take that is related to the most recent work you were doing. IMPORTANT: ensure that this step is DIRECTLY in line with the user's most recent explicit requests, and the task you were working on immediately before this summary request. If your last task was concluded, then only list next steps if they are explicitly in line with the users request. Do not start on tangential requests or really old requests that were already completed without confirming with the user first.\nIf there is a next step, include direct quotes from the most recent conversation showing exactly what task you were working on and where you left off. This should be verbatim to ensure there's no drift in task interpretation.\n\nPlease provide your summary based on the conversation and tools used so far, following this structure and ensuring precision and thoroughness in your response.\n" }, { "path": "application/prompts/react_final_prompt.txt", "content": "Query: {query}\nObservations: {observations}\nNow, using the insights from the observations, formulate a well-structured and precise final answer." }, { "path": "application/prompts/react_planning_prompt.txt", "content": "You are an AI assistant and talk like you're thinking out loud. Given the following query, outline a concise thought process that includes key steps and considerations necessary for effective analysis and response. Avoid pointwise formatting. The goal is to break down the query into manageable components without excessive detail, focusing on clarity and logical progression.\n\nInclude the following elements in your thought and execution process:\n1. Identify the main objective of the query.\n2. Determine any relevant context or background information needed to understand the query.\n3. List potential approaches or methods to address the query.\n4. Highlight any critical factors or constraints that may influence the outcome.\n5. Plan with available tools to help you with the analysis but dont execute them. Tools will be executed by another AI.\n\nQuery: {query} \nSummaries: {summaries}\nPrompt: {prompt}\nObservations(potentially previous tool calls): {observations}\n" }, { "path": "application/requirements.txt", "content": "anthropic==0.75.0\nboto3==1.42.17\nbeautifulsoup4==4.14.3\ncel-python==0.5.0\ncelery==5.6.0\ncryptography==46.0.3\ndataclasses-json==0.6.7\ndefusedxml==0.7.1\ndocling>=2.16.0\nrapidocr>=1.4.0\nonnxruntime>=1.19.0\ndocx2txt==0.9\nddgs>=8.0.0\nebooklib==0.20\nescodegen==1.0.11\nesprima==4.0.1\nesutils==1.0.1\nelevenlabs==2.27.0\nFlask==3.1.2\nfaiss-cpu==1.13.2\nfastmcp==2.14.1\nflask-restx==1.3.2\ngoogle-genai==1.54.0\ngoogle-api-python-client==2.187.0\ngoogle-auth-httplib2==0.3.0\ngoogle-auth-oauthlib==1.2.3\ngTTS==2.5.4\ngunicorn==23.0.0\nhtml2text==2025.4.15\njavalang==0.13.0\njinja2==3.1.6\njiter==0.12.0\njmespath==1.0.1\njoblib==1.5.3\njsonpatch==1.33\njsonpointer==3.0.0\nkombu==5.6.1\nlangchain==1.2.0\nlangchain-community==0.4.1\nlangchain-core==1.2.5\nlangchain-openai==1.1.6\nlangchain-text-splitters==1.1.0\nlangsmith==0.5.1\nlazy-object-proxy==1.12.0\nlxml==6.0.2\nmarkupsafe==3.0.3\nmarshmallow>=3.18.0,<5.0.0\nmpmath==1.3.0\nmultidict==6.7.0\nmsal==1.34.0\nmypy-extensions==1.1.0\nnetworkx==3.6.1\nnumpy==2.4.0\nopenai==2.14.0\nopenapi3-parser==1.1.22\norjson==3.11.5\npackaging==24.2\npandas==2.3.3\nopenpyxl==3.1.5\npathable==0.4.4\npdf2image>=1.17.0\npillow\nportalocker>=2.7.0,<3.0.0\nprance==25.4.8.0\nprompt-toolkit==3.0.52\nprotobuf==6.33.2\npsycopg2-binary==2.9.11\npy==1.11.0\npydantic\npydantic-core\npydantic-settings\npymongo==4.15.5\npypdf==6.5.0\npython-dateutil==2.9.0.post0\npython-dotenv\npython-jose==3.5.0\npython-pptx==1.0.2\nredis==7.1.0\nreferencing>=0.28.0,<0.38.0\nregex==2025.11.3\nrequests==2.32.5\nretry==0.9.2\nsentence-transformers==5.2.0\ntiktoken==0.12.0\ntokenizers==0.22.1\ntorch==2.9.1\ntqdm==4.67.1\ntransformers==4.57.3\ntyping-extensions==4.15.0\ntyping-inspect==0.9.0\ntzdata==2025.3\nurllib3==2.6.3\nvine==5.1.0\nwcwidth==0.2.14\nwerkzeug>=3.1.0\nyarl==1.22.0\nmarkdownify==1.2.2\ntldextract==5.3.0\nwebsockets==15.0.1" }, { "path": "application/retriever/__init__.py", "content": "" }, { "path": "application/retriever/base.py", "content": "from abc import ABC, abstractmethod\n\n\nclass BaseRetriever(ABC):\n def __init__(self):\n pass\n\n @abstractmethod\n def search(self, *args, **kwargs):\n pass\n" }, { "path": "application/retriever/classic_rag.py", "content": "import logging\nimport os\n\nfrom application.core.settings import settings\nfrom application.llm.llm_creator import LLMCreator\nfrom application.retriever.base import BaseRetriever\nfrom application.utils import num_tokens_from_string\nfrom application.vectorstore.vector_creator import VectorCreator\n\n\nclass ClassicRAG(BaseRetriever):\n def __init__(\n self,\n source,\n chat_history=None,\n prompt=\"\",\n chunks=2,\n doc_token_limit=50000,\n model_id=\"docsgpt-local\",\n user_api_key=None,\n agent_id=None,\n llm_name=settings.LLM_PROVIDER,\n api_key=settings.API_KEY,\n decoded_token=None,\n ):\n self.original_question = source.get(\"question\", \"\")\n self.chat_history = chat_history if chat_history is not None else []\n self.prompt = prompt\n if isinstance(chunks, str):\n try:\n self.chunks = int(chunks)\n except ValueError:\n logging.warning(\n f\"Invalid chunks value '{chunks}', using default value 2\"\n )\n self.chunks = 2\n else:\n self.chunks = chunks\n user_id = decoded_token.get(\"sub\") if decoded_token else \"default\"\n logging.info(\n f\"ClassicRAG initialized with chunks={self.chunks}, user_id={user_id}, \"\n f\"sources={'active_docs' in source and source['active_docs'] is not None}\"\n )\n self.model_id = model_id\n self.doc_token_limit = doc_token_limit\n self.user_api_key = user_api_key\n self.agent_id = agent_id\n self.llm_name = llm_name\n self.api_key = api_key\n self.llm = LLMCreator.create_llm(\n self.llm_name,\n api_key=self.api_key,\n user_api_key=self.user_api_key,\n decoded_token=decoded_token,\n agent_id=self.agent_id,\n )\n\n if \"active_docs\" in source and source[\"active_docs\"] is not None:\n if isinstance(source[\"active_docs\"], list):\n self.vectorstores = source[\"active_docs\"]\n else:\n self.vectorstores = [source[\"active_docs\"]]\n else:\n self.vectorstores = []\n self.question = self._rephrase_query()\n self.decoded_token = decoded_token\n self._validate_vectorstore_config()\n\n def _validate_vectorstore_config(self):\n \"\"\"Validate vectorstore IDs and remove any empty/invalid entries\"\"\"\n if not self.vectorstores:\n logging.warning(\"No vectorstores configured for retrieval\")\n return\n invalid_ids = [\n vs_id for vs_id in self.vectorstores if not vs_id or not vs_id.strip()\n ]\n if invalid_ids:\n logging.warning(f\"Found invalid vectorstore IDs: {invalid_ids}\")\n self.vectorstores = [\n vs_id for vs_id in self.vectorstores if vs_id and vs_id.strip()\n ]\n\n def _rephrase_query(self):\n \"\"\"Rephrase user query with chat history context for better retrieval\"\"\"\n if (\n not self.original_question\n or not self.chat_history\n or self.chat_history == []\n or self.chunks == 0\n or not self.vectorstores\n ):\n return self.original_question\n prompt = (\n \"Given the following conversation history:\\n\"\n f\"{self.chat_history}\\n\\n\"\n \"Rephrase the following user question to be a standalone search query \"\n \"that captures all relevant context from the conversation:\\n\"\n )\n\n messages = [\n {\"role\": \"system\", \"content\": prompt},\n {\"role\": \"user\", \"content\": self.original_question},\n ]\n\n try:\n rephrased_query = self.llm.gen(model=self.model_id, messages=messages)\n print(f\"Rephrased query: {rephrased_query}\")\n return rephrased_query if rephrased_query else self.original_question\n except Exception as e:\n logging.error(f\"Error rephrasing query: {e}\", exc_info=True)\n return self.original_question\n\n def _get_data(self):\n if self.chunks == 0 or not self.vectorstores:\n logging.info(\n f\"ClassicRAG._get_data: Skipping retrieval - chunks={self.chunks}, \"\n f\"vectorstores_count={len(self.vectorstores) if self.vectorstores else 0}\"\n )\n return []\n\n all_docs = []\n chunks_per_source = max(1, self.chunks // len(self.vectorstores))\n token_budget = max(int(self.doc_token_limit * 0.9), 100)\n cumulative_tokens = 0\n\n for vectorstore_id in self.vectorstores:\n if vectorstore_id:\n try:\n docsearch = VectorCreator.create_vectorstore(\n settings.VECTOR_STORE, vectorstore_id, settings.EMBEDDINGS_KEY\n )\n docs_temp = docsearch.search(\n self.question, k=max(chunks_per_source * 2, 20)\n )\n\n for doc in docs_temp:\n if cumulative_tokens >= token_budget:\n break\n\n if hasattr(doc, \"page_content\") and hasattr(doc, \"metadata\"):\n page_content = doc.page_content\n metadata = doc.metadata\n else:\n page_content = doc.get(\"text\", doc.get(\"page_content\", \"\"))\n metadata = doc.get(\"metadata\", {})\n\n title = metadata.get(\n \"title\", metadata.get(\"post_title\", page_content)\n )\n if not isinstance(title, str):\n title = str(title)\n title = title.split(\"/\")[-1]\n\n filename = (\n metadata.get(\"filename\")\n or metadata.get(\"file_name\")\n or metadata.get(\"source\")\n )\n if isinstance(filename, str):\n filename = os.path.basename(filename) or filename\n else:\n filename = title\n if not filename:\n filename = title\n source_path = metadata.get(\"source\") or vectorstore_id\n\n doc_text_with_header = f\"{filename}\\n{page_content}\"\n doc_tokens = num_tokens_from_string(doc_text_with_header)\n\n if cumulative_tokens + doc_tokens < token_budget:\n all_docs.append(\n {\n \"title\": title,\n \"text\": page_content,\n \"source\": source_path,\n \"filename\": filename,\n }\n )\n cumulative_tokens += doc_tokens\n\n if cumulative_tokens >= token_budget:\n break\n\n except Exception as e:\n logging.error(\n f\"Error searching vectorstore {vectorstore_id}: {e}\",\n exc_info=True,\n )\n continue\n\n logging.info(\n f\"ClassicRAG._get_data: Retrieval complete - retrieved {len(all_docs)} documents \"\n f\"(requested chunks={self.chunks}, chunks_per_source={chunks_per_source}, \"\n f\"cumulative_tokens={cumulative_tokens}/{token_budget})\"\n )\n return all_docs\n\n def search(self, query: str = \"\"):\n \"\"\"Search for documents using optional query override\"\"\"\n if query:\n self.original_question = query\n self.question = self._rephrase_query()\n return self._get_data()\n" }, { "path": "application/retriever/retriever_creator.py", "content": "from application.retriever.classic_rag import ClassicRAG\n\n\nclass RetrieverCreator:\n retrievers = {\n \"classic\": ClassicRAG,\n \"default\": ClassicRAG,\n }\n\n @classmethod\n def create_retriever(cls, type, *args, **kwargs):\n retriever_type = (type or \"default\").lower()\n retiever_class = cls.retrievers.get(retriever_type)\n if not retiever_class:\n raise ValueError(f\"No retievers class found for type {type}\")\n return retiever_class(*args, **kwargs)\n" }, { "path": "application/security/__init__.py", "content": "" }, { "path": "application/security/encryption.py", "content": "import base64\nimport json\nimport os\n\nfrom cryptography.hazmat.backends import default_backend\nfrom cryptography.hazmat.primitives import hashes\nfrom cryptography.hazmat.primitives.ciphers import algorithms, Cipher, modes\nfrom cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC\n\nfrom application.core.settings import settings\n\n\ndef _derive_key(user_id: str, salt: bytes) -> bytes:\n app_secret = settings.ENCRYPTION_SECRET_KEY\n\n password = f\"{app_secret}#{user_id}\".encode()\n\n kdf = PBKDF2HMAC(\n algorithm=hashes.SHA256(),\n length=32,\n salt=salt,\n iterations=100000,\n backend=default_backend(),\n )\n\n return kdf.derive(password)\n\n\ndef encrypt_credentials(credentials: dict, user_id: str) -> str:\n if not credentials:\n return \"\"\n try:\n salt = os.urandom(16)\n iv = os.urandom(16)\n key = _derive_key(user_id, salt)\n\n json_str = json.dumps(credentials)\n\n cipher = Cipher(algorithms.AES(key), modes.CBC(iv), backend=default_backend())\n encryptor = cipher.encryptor()\n\n padded_data = _pad_data(json_str.encode())\n encrypted_data = encryptor.update(padded_data) + encryptor.finalize()\n\n result = salt + iv + encrypted_data\n return base64.b64encode(result).decode()\n except Exception as e:\n print(f\"Warning: Failed to encrypt credentials: {e}\")\n return \"\"\n\n\ndef decrypt_credentials(encrypted_data: str, user_id: str) -> dict:\n if not encrypted_data:\n return {}\n try:\n data = base64.b64decode(encrypted_data.encode())\n\n salt = data[:16]\n iv = data[16:32]\n encrypted_content = data[32:]\n\n key = _derive_key(user_id, salt)\n\n cipher = Cipher(algorithms.AES(key), modes.CBC(iv), backend=default_backend())\n decryptor = cipher.decryptor()\n\n decrypted_padded = decryptor.update(encrypted_content) + decryptor.finalize()\n decrypted_data = _unpad_data(decrypted_padded)\n\n return json.loads(decrypted_data.decode())\n except Exception as e:\n print(f\"Warning: Failed to decrypt credentials: {e}\")\n return {}\n\n\ndef _pad_data(data: bytes) -> bytes:\n block_size = 16\n padding_len = block_size - (len(data) % block_size)\n padding = bytes([padding_len]) * padding_len\n return data + padding\n\n\ndef _unpad_data(data: bytes) -> bytes:\n padding_len = data[-1]\n return data[:-padding_len]\n" }, { "path": "application/seed/__init__.py", "content": "" }, { "path": "application/seed/commands.py", "content": "import click\n\nfrom application.core.mongo_db import MongoDB\nfrom application.core.settings import settings\nfrom application.seed.seeder import DatabaseSeeder\n\n\n@click.group()\ndef seed():\n \"\"\"Database seeding commands\"\"\"\n pass\n\n\n@seed.command()\n@click.option(\"--force\", is_flag=True, help=\"Force reseeding even if data exists\")\ndef init(force):\n \"\"\"Initialize database with seed data\"\"\"\n mongo = MongoDB.get_client()\n db = mongo[settings.MONGO_DB_NAME]\n\n seeder = DatabaseSeeder(db)\n seeder.seed_initial_data(force=force)\n\n\nif __name__ == \"__main__\":\n seed()\n" }, { "path": "application/seed/config/agents_template.yaml", "content": "# Configuration for Premade Agents\n# This file contains template agents that will be seeded into the database\n\nagents:\n # Basic Agent Template\n - name: \"Agent Name\" # Required: Unique name for the agent\n description: \"What this agent does\" # Required: Brief description of the agent's purpose\n image: \"URL_TO_IMAGE\" # Optional: URL to agent's avatar/image\n agent_type: \"classic\" # Required: Type of agent (e.g., classic, react, etc.)\n prompt_id: \"default\" # Optional: Reference to prompt template\n prompt: # Optional: Define new prompt\n name: \"New Prompt\"\n content: \"You are new agent with cool new prompt.\"\n chunks: \"0\" # Optional: Chunking strategy for documents\n retriever: \"\" # Optional: Retriever type for document search\n\n # Source Configuration (where the agent gets its knowledge)\n source: # Optional: Select a source to link with agent\n name: \"Source Display Name\" # Human-readable name for the source\n url: \"https://example.com/data-source\" # URL or path to knowledge source\n loader: \"url\" # Type of loader (url, pdf, txt, etc.)\n\n # Tools Configuration (what capabilities the agent has)\n tools: # Optional: Remove if agent doesn't need tools\n - name: \"tool_name\" # Must match a supported tool name\n display_name: \"Tool Display Name\" # Optional: Human-readable name for the tool\n config:\n # Tool-specific configuration\n # Example for DuckDuckGo:\n # token: \"${DDG_API_KEY}\" # ${} denotes environment variable\n\n # Add more tools as needed\n # - name: \"another_tool\"\n # config:\n # param1: \"value1\"\n # param2: \"${ENV_VAR}\"" }, { "path": "application/seed/config/premade_agents.yaml", "content": "# Configuration for Premade Agents\n\nagents:\n - name: \"Assistant\"\n description: \"Your general-purpose AI assistant. Ready to help with a wide range of tasks.\"\n image: \"https://d3dg1063dc54p9.cloudfront.net/imgs/agents/agent-logo.svg\"\n agent_type: \"classic\"\n prompt_id: \"default\"\n chunks: \"0\"\n retriever: \"\"\n\n # Tools Configuration\n tools:\n - name: \"tool_name\"\n display_name: \"read_webpage\" \n config:\n\n - name: \"Researcher\"\n description: \"A specialized research agent that performs deep dives into subjects.\"\n image: \"https://d3dg1063dc54p9.cloudfront.net/imgs/agents/agent-researcher.svg\"\n agent_type: \"react\"\n prompt:\n name: \"Researcher-Agent\"\n content: |\n You are a specialized AI research assistant, DocsGPT. Your primary function is to conduct in-depth research on a given subject or question. You are methodical, thorough, and analytical. You should perform multiple iterations of thinking to gather and synthesize information before providing a final, comprehensive answer.\n\n You have access to the 'Read Webpage' tool. Use this tool to explore sources, gather data, and deepen your understanding. Be proactive in using the tool to fill in knowledge gaps and validate information.\n\n Users can Upload documents for your context as attachments or sources via UI using the Conversation input box.\n If appropriate, your answers can include code examples, formatted as follows:\n ```(language)\n (code)\n ```\n Users are also able to see charts and diagrams if you use them with valid mermaid syntax in your responses. Try to respond with mermaid charts if visualization helps with users queries. You effectively utilize chat history, ensuring relevant and tailored responses. Try to use additional provided context if it's available, otherwise use your knowledge and tool capabilities.\n ----------------\n Possible additional context from uploaded sources:\n {summaries}\n\n chunks: \"0\"\n retriever: \"\"\n\n # Tools Configuration\n tools:\n - name: \"tool_name\"\n display_name: \"read_webpage\" \n config:\n\n - name: \"Search Widget\"\n description: \"A powerful search widget agent. Ask it anything about DocsGPT\"\n image: \"https://d3dg1063dc54p9.cloudfront.net/imgs/agents/agent-search.svg\"\n agent_type: \"classic\"\n prompt:\n name: \"Search-Agent\"\n content: |\n You are a website search assistant, DocsGPT. Your sole purpose is to help users find information within the provided context of the DocsGPT documentation. Act as a specialized search engine.\n\n Your answers must be based *only* on the provided context. Do not use any external knowledge. If the answer is not in the context, inform the user that you could not find the information within the documentation.\n\n Keep your responses concise and directly related to the user's query, pointing them to the most relevant information.\n ----------------\n Possible additional context from uploaded sources:\n {summaries}\n\n chunks: \"8\"\n retriever: \"\"\n\n source:\n name: \"DocsGPT-Docs\"\n url: \"https://d3dg1063dc54p9.cloudfront.net/agent-source/docsgpt-documentation.md\" # URL to DocsGPT documentation\n loader: \"url\" \n\n - name: \"Support Widget\"\n description: \"A friendly support widget agent to help you with any questions.\"\n image: \"https://d3dg1063dc54p9.cloudfront.net/imgs/agents/agent-support.svg\"\n agent_type: \"classic\"\n prompt:\n name: \"Support-Agent\"\n content: |\n You are a helpful AI support widget agent, DocsGPT. Your goal is to assist users by answering their questions about our website, product and its features. Provide friendly, clear, and direct support.\n\n Your knowledge is strictly limited to the provided context from the DocsGPT documentation. You must not answer questions outside of this scope. If a user asks something you cannot answer from the context, politely state that you can only help with questions about this website.\n\n Effectively utilize chat history to understand the user's issue fully. Guide users to the information they need in a helpful and conversational manner.\n ----------------\n Possible additional context from uploaded sources:\n {summaries}\n\n chunks: \"8\"\n retriever: \"\"\n\n source:\n name: \"DocsGPT-Docs\"\n url: \"https://d3dg1063dc54p9.cloudfront.net/agent-source/docsgpt-documentation.md\" # URL to DocsGPT documentation\n loader: \"url\" " }, { "path": "application/seed/seeder.py", "content": "import logging\nimport os\nfrom datetime import datetime, timezone\nfrom typing import Dict, List, Optional, Union\n\nimport yaml\nfrom bson import ObjectId\nfrom bson.dbref import DBRef\n\nfrom dotenv import load_dotenv\nfrom pymongo import MongoClient\n\nfrom application.agents.tools.tool_manager import ToolManager\nfrom application.api.user.tasks import ingest_remote\n\nload_dotenv()\ntool_config = {}\ntool_manager = ToolManager(config=tool_config)\n\n\nclass DatabaseSeeder:\n def __init__(self, db):\n self.db = db\n self.tools_collection = self.db[\"user_tools\"]\n self.sources_collection = self.db[\"sources\"]\n self.agents_collection = self.db[\"agents\"]\n self.prompts_collection = self.db[\"prompts\"]\n self.system_user_id = \"system\"\n self.logger = logging.getLogger(__name__)\n\n def seed_initial_data(self, config_path: str = None, force=False):\n \"\"\"Main entry point for seeding all initial data\"\"\"\n if not force and self._is_already_seeded():\n self.logger.info(\"Database already seeded. Use force=True to reseed.\")\n return\n config_path = config_path or os.path.join(\n os.path.dirname(__file__), \"config\", \"premade_agents.yaml\"\n )\n\n try:\n with open(config_path, \"r\") as f:\n config = yaml.safe_load(f)\n self._seed_from_config(config)\n except Exception as e:\n self.logger.error(f\"Failed to load seeding config: {str(e)}\")\n raise\n\n def _seed_from_config(self, config: Dict):\n \"\"\"Seed all data from configuration\"\"\"\n self.logger.info(\"🌱 Starting seeding...\")\n\n if not config.get(\"agents\"):\n self.logger.warning(\"No agents found in config\")\n return\n used_tool_ids = set()\n\n for agent_config in config[\"agents\"]:\n try:\n self.logger.info(f\"Processing agent: {agent_config['name']}\")\n\n # 1. Handle Source\n\n source_result = self._handle_source(agent_config)\n if source_result is False:\n self.logger.error(\n f\"Skipping agent {agent_config['name']} due to source ingestion failure\"\n )\n continue\n source_id = source_result\n # 2. Handle Tools\n\n tool_ids = self._handle_tools(agent_config)\n if len(tool_ids) == 0:\n self.logger.warning(\n f\"No valid tools for agent {agent_config['name']}\"\n )\n used_tool_ids.update(tool_ids)\n\n # 3. Handle Prompt\n\n prompt_id = self._handle_prompt(agent_config)\n\n # 4. Create Agent\n\n agent_data = {\n \"user\": self.system_user_id,\n \"name\": agent_config[\"name\"],\n \"description\": agent_config[\"description\"],\n \"image\": agent_config.get(\"image\", \"\"),\n \"source\": (\n DBRef(\"sources\", ObjectId(source_id)) if source_id else \"\"\n ),\n \"tools\": [str(tid) for tid in tool_ids],\n \"agent_type\": agent_config[\"agent_type\"],\n \"prompt_id\": prompt_id or agent_config.get(\"prompt_id\", \"default\"),\n \"chunks\": agent_config.get(\"chunks\", \"0\"),\n \"retriever\": agent_config.get(\"retriever\", \"\"),\n \"status\": \"template\",\n \"createdAt\": datetime.now(timezone.utc),\n \"updatedAt\": datetime.now(timezone.utc),\n }\n\n existing = self.agents_collection.find_one(\n {\"user\": self.system_user_id, \"name\": agent_config[\"name\"]}\n )\n if existing:\n self.logger.info(f\"Updating existing agent: {agent_config['name']}\")\n self.agents_collection.update_one(\n {\"_id\": existing[\"_id\"]}, {\"$set\": agent_data}\n )\n agent_id = existing[\"_id\"]\n else:\n self.logger.info(f\"Creating new agent: {agent_config['name']}\")\n result = self.agents_collection.insert_one(agent_data)\n agent_id = result.inserted_id\n self.logger.info(\n f\"Successfully processed agent: {agent_config['name']} (ID: {agent_id})\"\n )\n except Exception as e:\n self.logger.error(\n f\"Error processing agent {agent_config['name']}: {str(e)}\"\n )\n continue\n self.logger.info(\"✅ Database seeding completed\")\n\n def _handle_source(self, agent_config: Dict) -> Union[ObjectId, None, bool]:\n \"\"\"Handle source ingestion and return source ID\"\"\"\n if not agent_config.get(\"source\"):\n self.logger.info(\n \"No source provided for agent - will create agent without source\"\n )\n return None\n source_config = agent_config[\"source\"]\n self.logger.info(f\"Ingesting source: {source_config['url']}\")\n\n try:\n existing = self.sources_collection.find_one(\n {\"user\": self.system_user_id, \"remote_data\": source_config[\"url\"]}\n )\n if existing:\n self.logger.info(f\"Source already exists: {existing['_id']}\")\n return existing[\"_id\"]\n # Ingest new source using worker\n\n task = ingest_remote.delay(\n source_data=source_config[\"url\"],\n job_name=source_config[\"name\"],\n user=self.system_user_id,\n loader=source_config.get(\"loader\", \"url\"),\n )\n\n result = task.get(timeout=300)\n\n if not task.successful():\n raise Exception(f\"Source ingestion failed: {result}\")\n source_id = None\n if isinstance(result, dict) and \"id\" in result:\n source_id = result[\"id\"]\n else:\n raise Exception(f\"Source ingestion result missing 'id': {result}\")\n self.logger.info(f\"Source ingested successfully: {source_id}\")\n return source_id\n except Exception as e:\n self.logger.error(f\"Failed to ingest source: {str(e)}\")\n return False\n\n def _handle_tools(self, agent_config: Dict) -> List[ObjectId]:\n \"\"\"Handle tool creation and return list of tool IDs\"\"\"\n tool_ids = []\n if not agent_config.get(\"tools\"):\n return tool_ids\n for tool_config in agent_config[\"tools\"]:\n try:\n tool_name = tool_config[\"name\"]\n processed_config = self._process_config(tool_config.get(\"config\", {}))\n self.logger.info(f\"Processing tool: {tool_name}\")\n\n existing = self.tools_collection.find_one(\n {\n \"user\": self.system_user_id,\n \"name\": tool_name,\n \"config\": processed_config,\n }\n )\n if existing:\n self.logger.info(f\"Tool already exists: {existing['_id']}\")\n tool_ids.append(existing[\"_id\"])\n continue\n tool_data = {\n \"user\": self.system_user_id,\n \"name\": tool_name,\n \"displayName\": tool_config.get(\"display_name\", tool_name),\n \"description\": tool_config.get(\"description\", \"\"),\n \"actions\": tool_manager.tools[tool_name].get_actions_metadata(),\n \"config\": processed_config,\n \"status\": True,\n }\n\n result = self.tools_collection.insert_one(tool_data)\n tool_ids.append(result.inserted_id)\n self.logger.info(f\"Created new tool: {result.inserted_id}\")\n except Exception as e:\n self.logger.error(f\"Failed to process tool {tool_name}: {str(e)}\")\n continue\n return tool_ids\n\n def _handle_prompt(self, agent_config: Dict) -> Optional[str]:\n \"\"\"Handle prompt creation and return prompt ID\"\"\"\n if not agent_config.get(\"prompt\"):\n return None\n\n prompt_config = agent_config[\"prompt\"]\n prompt_name = prompt_config.get(\"name\", f\"{agent_config['name']} Prompt\")\n prompt_content = prompt_config.get(\"content\", \"\")\n\n if not prompt_content:\n self.logger.warning(\n f\"No prompt content provided for agent {agent_config['name']}\"\n )\n return None\n\n self.logger.info(f\"Processing prompt: {prompt_name}\")\n\n try:\n existing = self.prompts_collection.find_one(\n {\n \"user\": self.system_user_id,\n \"name\": prompt_name,\n \"content\": prompt_content,\n }\n )\n if existing:\n self.logger.info(f\"Prompt already exists: {existing['_id']}\")\n return str(existing[\"_id\"])\n\n prompt_data = {\n \"name\": prompt_name,\n \"content\": prompt_content,\n \"user\": self.system_user_id,\n }\n\n result = self.prompts_collection.insert_one(prompt_data)\n prompt_id = str(result.inserted_id)\n self.logger.info(f\"Created new prompt: {prompt_id}\")\n return prompt_id\n\n except Exception as e:\n self.logger.error(f\"Failed to process prompt {prompt_name}: {str(e)}\")\n return None\n\n def _process_config(self, config: Dict) -> Dict:\n \"\"\"Process config values to replace environment variables\"\"\"\n processed = {}\n for key, value in config.items():\n if (\n isinstance(value, str)\n and value.startswith(\"${\")\n and value.endswith(\"}\")\n ):\n env_var = value[2:-1]\n processed[key] = os.getenv(env_var, \"\")\n else:\n processed[key] = value\n return processed\n\n def _is_already_seeded(self) -> bool:\n \"\"\"Check if premade agents already exist\"\"\"\n return self.agents_collection.count_documents({\"user\": self.system_user_id}) > 0\n\n @classmethod\n def initialize_from_env(cls, worker=None):\n \"\"\"Factory method to create seeder from environment\"\"\"\n mongo_uri = os.getenv(\"MONGO_URI\", \"mongodb://localhost:27017\")\n db_name = os.getenv(\"MONGO_DB_NAME\", \"docsgpt\")\n client = MongoClient(mongo_uri)\n db = client[db_name]\n return cls(db)\n" }, { "path": "application/storage/__init__.py", "content": "" }, { "path": "application/storage/base.py", "content": "\"\"\"Base storage class for file system abstraction.\"\"\"\n\nfrom abc import ABC, abstractmethod\nfrom typing import BinaryIO, List, Callable\n\n\nclass BaseStorage(ABC):\n \"\"\"Abstract base class for storage implementations.\"\"\"\n\n @abstractmethod\n def save_file(self, file_data: BinaryIO, path: str, **kwargs) -> dict:\n \"\"\"\n Save a file to storage.\n\n Args:\n file_data: File-like object containing the data\n path: Path where the file should be stored\n\n Returns:\n dict: A dictionary containing metadata about the saved file, including:\n - 'path': The path where the file was saved\n - 'storage_type': The type of storage (e.g., 'local', 's3')\n - Other storage-specific metadata (e.g., 'uri', 'bucket_name', etc.)\n \"\"\"\n pass\n\n @abstractmethod\n def get_file(self, path: str) -> BinaryIO:\n \"\"\"\n Retrieve a file from storage.\n\n Args:\n path: Path to the file\n\n Returns:\n BinaryIO: File-like object containing the file data\n \"\"\"\n pass\n\n @abstractmethod\n def process_file(self, path: str, processor_func: Callable, **kwargs):\n \"\"\"\n Process a file using the provided processor function.\n\n This method handles the details of retrieving the file and providing\n it to the processor function in an appropriate way based on the storage type.\n\n Args:\n path: Path to the file\n processor_func: Function that processes the file\n **kwargs: Additional arguments to pass to the processor function\n\n Returns:\n The result of the processor function\n \"\"\"\n pass\n\n @abstractmethod\n def delete_file(self, path: str) -> bool:\n \"\"\"\n Delete a file from storage.\n\n Args:\n path: Path to the file\n\n Returns:\n bool: True if deletion was successful\n \"\"\"\n pass\n\n @abstractmethod\n def file_exists(self, path: str) -> bool:\n \"\"\"\n Check if a file exists.\n\n Args:\n path: Path to the file\n\n Returns:\n bool: True if the file exists\n \"\"\"\n pass\n\n @abstractmethod\n def list_files(self, directory: str) -> List[str]:\n \"\"\"\n List all files in a directory.\n\n Args:\n directory: Directory path to list\n\n Returns:\n List[str]: List of file paths\n \"\"\"\n pass\n \n @abstractmethod\n def is_directory(self, path: str) -> bool:\n \"\"\"\n Check if a path is a directory.\n\n Args:\n path: Path to check\n\n Returns:\n bool: True if the path is a directory\n \"\"\"\n pass\n\n @abstractmethod\n def remove_directory(self, directory: str) -> bool:\n \"\"\"\n Remove a directory and all its contents.\n\n For local storage, this removes the directory and all files/subdirectories within it.\n For S3 storage, this removes all objects with the directory path as a prefix.\n\n Args:\n directory: Directory path to remove\n\n Returns:\n bool: True if removal was successful, False otherwise\n \"\"\"\n pass\n" }, { "path": "application/storage/local.py", "content": "\"\"\"Local file system implementation.\"\"\"\nimport os\nimport shutil\nfrom typing import BinaryIO, List, Callable\n\nfrom application.storage.base import BaseStorage\n\n\nclass LocalStorage(BaseStorage):\n \"\"\"Local file system storage implementation.\"\"\"\n\n def __init__(self, base_dir: str = None):\n \"\"\"\n Initialize local storage.\n\n Args:\n base_dir: Base directory for all operations. If None, uses current directory.\n \"\"\"\n self.base_dir = base_dir or os.path.dirname(\n os.path.dirname(os.path.dirname(os.path.abspath(__file__)))\n )\n\n def _get_full_path(self, path: str) -> str:\n \"\"\"Get absolute path by combining base_dir and path.\"\"\"\n if os.path.isabs(path):\n return path\n return os.path.join(self.base_dir, path)\n\n def save_file(self, file_data: BinaryIO, path: str, **kwargs) -> dict:\n \"\"\"Save a file to local storage.\"\"\"\n full_path = self._get_full_path(path)\n\n os.makedirs(os.path.dirname(full_path), exist_ok=True)\n\n if hasattr(file_data, 'save'):\n file_data.save(full_path)\n else:\n with open(full_path, 'wb') as f:\n shutil.copyfileobj(file_data, f)\n\n return {\n 'storage_type': 'local'\n }\n\n def get_file(self, path: str) -> BinaryIO:\n \"\"\"Get a file from local storage.\"\"\"\n full_path = self._get_full_path(path)\n\n if not os.path.exists(full_path):\n raise FileNotFoundError(f\"File not found: {full_path}\")\n\n return open(full_path, 'rb')\n\n def delete_file(self, path: str) -> bool:\n \"\"\"Delete a file from local storage.\"\"\"\n full_path = self._get_full_path(path)\n\n if not os.path.exists(full_path):\n return False\n\n os.remove(full_path)\n return True\n\n def file_exists(self, path: str) -> bool:\n \"\"\"Check if a file exists in local storage.\"\"\"\n full_path = self._get_full_path(path)\n return os.path.exists(full_path)\n\n def list_files(self, directory: str) -> List[str]:\n \"\"\"List all files in a directory in local storage.\"\"\"\n full_path = self._get_full_path(directory)\n\n if not os.path.exists(full_path):\n return []\n\n result = []\n for root, _, files in os.walk(full_path):\n for file in files:\n rel_path = os.path.relpath(os.path.join(root, file), self.base_dir)\n result.append(rel_path)\n\n return result\n\n def process_file(self, path: str, processor_func: Callable, **kwargs):\n \"\"\"\n Process a file using the provided processor function.\n\n For local storage, we can directly pass the full path to the processor.\n\n Args:\n path: Path to the file\n processor_func: Function that processes the file\n **kwargs: Additional arguments to pass to the processor function\n\n Returns:\n The result of the processor function\n \"\"\"\n full_path = self._get_full_path(path)\n\n if not os.path.exists(full_path):\n raise FileNotFoundError(f\"File not found: {full_path}\")\n\n return processor_func(local_path=full_path, **kwargs)\n\n def is_directory(self, path: str) -> bool:\n \"\"\"\n Check if a path is a directory in local storage.\n \n Args:\n path: Path to check\n \n Returns:\n bool: True if the path is a directory, False otherwise\n \"\"\"\n full_path = self._get_full_path(path)\n return os.path.isdir(full_path)\n\n def remove_directory(self, directory: str) -> bool:\n \"\"\"\n Remove a directory and all its contents from local storage.\n\n Args:\n directory: Directory path to remove\n\n Returns:\n bool: True if removal was successful, False otherwise\n \"\"\"\n full_path = self._get_full_path(directory)\n\n if not os.path.exists(full_path):\n return False\n\n if not os.path.isdir(full_path):\n return False\n\n try:\n shutil.rmtree(full_path)\n return True\n except (OSError, PermissionError):\n return False\n" }, { "path": "application/storage/s3.py", "content": "\"\"\"S3 storage implementation.\"\"\"\n\nimport io\nimport os\nfrom typing import BinaryIO, Callable, List\n\nimport boto3\nfrom application.core.settings import settings\n\nfrom application.storage.base import BaseStorage\nfrom botocore.exceptions import ClientError\n\n\nclass S3Storage(BaseStorage):\n \"\"\"AWS S3 storage implementation.\"\"\"\n\n def __init__(self, bucket_name=None):\n \"\"\"\n Initialize S3 storage.\n\n Args:\n bucket_name: S3 bucket name (optional, defaults to settings)\n \"\"\"\n self.bucket_name = bucket_name or getattr(\n settings, \"S3_BUCKET_NAME\", \"docsgpt-test-bucket\"\n )\n\n # Get credentials from settings\n\n aws_access_key_id = getattr(settings, \"SAGEMAKER_ACCESS_KEY\", None)\n aws_secret_access_key = getattr(settings, \"SAGEMAKER_SECRET_KEY\", None)\n region_name = getattr(settings, \"SAGEMAKER_REGION\", None)\n\n self.s3 = boto3.client(\n \"s3\",\n aws_access_key_id=aws_access_key_id,\n aws_secret_access_key=aws_secret_access_key,\n region_name=region_name,\n )\n\n def save_file(\n self,\n file_data: BinaryIO,\n path: str,\n storage_class: str = \"INTELLIGENT_TIERING\",\n **kwargs,\n ) -> dict:\n \"\"\"Save a file to S3 storage.\"\"\"\n self.s3.upload_fileobj(\n file_data, self.bucket_name, path, ExtraArgs={\"StorageClass\": storage_class}\n )\n\n region = getattr(settings, \"SAGEMAKER_REGION\", None)\n\n return {\n \"storage_type\": \"s3\",\n \"bucket_name\": self.bucket_name,\n \"uri\": f\"s3://{self.bucket_name}/{path}\",\n \"region\": region,\n }\n\n def get_file(self, path: str) -> BinaryIO:\n \"\"\"Get a file from S3 storage.\"\"\"\n if not self.file_exists(path):\n raise FileNotFoundError(f\"File not found: {path}\")\n file_obj = io.BytesIO()\n self.s3.download_fileobj(self.bucket_name, path, file_obj)\n file_obj.seek(0)\n return file_obj\n\n def delete_file(self, path: str) -> bool:\n \"\"\"Delete a file from S3 storage.\"\"\"\n try:\n self.s3.delete_object(Bucket=self.bucket_name, Key=path)\n return True\n except ClientError:\n return False\n\n def file_exists(self, path: str) -> bool:\n \"\"\"Check if a file exists in S3 storage.\"\"\"\n try:\n self.s3.head_object(Bucket=self.bucket_name, Key=path)\n return True\n except ClientError:\n return False\n\n def list_files(self, directory: str) -> List[str]:\n \"\"\"List all files in a directory in S3 storage.\"\"\"\n # Ensure directory ends with a slash if it's not empty\n\n if directory and not directory.endswith(\"/\"):\n directory += \"/\"\n result = []\n paginator = self.s3.get_paginator(\"list_objects_v2\")\n pages = paginator.paginate(Bucket=self.bucket_name, Prefix=directory)\n\n for page in pages:\n if \"Contents\" in page:\n for obj in page[\"Contents\"]:\n result.append(obj[\"Key\"])\n return result\n\n def process_file(self, path: str, processor_func: Callable, **kwargs):\n \"\"\"\n Process a file using the provided processor function.\n\n Args:\n path: Path to the file\n processor_func: Function that processes the file\n **kwargs: Additional arguments to pass to the processor function\n\n Returns:\n The result of the processor function\n \"\"\"\n import logging\n import tempfile\n\n if not self.file_exists(path):\n raise FileNotFoundError(f\"File not found in S3: {path}\")\n with tempfile.NamedTemporaryFile(\n suffix=os.path.splitext(path)[1], delete=True\n ) as temp_file:\n try:\n # Download the file from S3 to the temporary file\n\n self.s3.download_fileobj(self.bucket_name, path, temp_file)\n temp_file.flush()\n\n return processor_func(local_path=temp_file.name, **kwargs)\n except Exception as e:\n logging.error(f\"Error processing S3 file {path}: {e}\", exc_info=True)\n raise\n\n def is_directory(self, path: str) -> bool:\n \"\"\"\n Check if a path is a directory in S3 storage.\n\n In S3, directories are virtual concepts. A path is considered a directory\n if there are objects with the path as a prefix.\n\n Args:\n path: Path to check\n\n Returns:\n bool: True if the path is a directory, False otherwise\n \"\"\"\n # Ensure path ends with a slash if not empty\n if path and not path.endswith('/'):\n path += '/'\n\n response = self.s3.list_objects_v2(\n Bucket=self.bucket_name,\n Prefix=path,\n MaxKeys=1\n )\n\n return 'Contents' in response\n\n def remove_directory(self, directory: str) -> bool:\n \"\"\"\n Remove a directory and all its contents from S3 storage.\n\n In S3, this removes all objects with the directory path as a prefix.\n Since S3 doesn't have actual directories, this effectively removes\n all files within the virtual directory structure.\n\n Args:\n directory: Directory path to remove\n\n Returns:\n bool: True if removal was successful, False otherwise\n \"\"\"\n # Ensure directory ends with a slash if not empty\n if directory and not directory.endswith('/'):\n directory += '/'\n\n try:\n # Get all objects with the directory prefix\n objects_to_delete = []\n paginator = self.s3.get_paginator('list_objects_v2')\n pages = paginator.paginate(Bucket=self.bucket_name, Prefix=directory)\n\n for page in pages:\n if 'Contents' in page:\n for obj in page['Contents']:\n objects_to_delete.append({'Key': obj['Key']})\n\n if not objects_to_delete:\n return False\n\n batch_size = 1000\n for i in range(0, len(objects_to_delete), batch_size):\n batch = objects_to_delete[i:i + batch_size]\n\n response = self.s3.delete_objects(\n Bucket=self.bucket_name,\n Delete={'Objects': batch}\n )\n\n if 'Errors' in response and response['Errors']:\n return False\n\n return True\n\n except ClientError:\n return False\n" }, { "path": "application/storage/storage_creator.py", "content": "\"\"\"Storage factory for creating different storage implementations.\"\"\"\nfrom typing import Dict, Type\n\nfrom application.storage.base import BaseStorage\nfrom application.storage.local import LocalStorage\nfrom application.storage.s3 import S3Storage\nfrom application.core.settings import settings\n\n\nclass StorageCreator:\n storages: Dict[str, Type[BaseStorage]] = {\n \"local\": LocalStorage,\n \"s3\": S3Storage,\n }\n \n _instance = None\n \n @classmethod\n def get_storage(cls) -> BaseStorage:\n if cls._instance is None:\n storage_type = getattr(settings, \"STORAGE_TYPE\", \"local\")\n cls._instance = cls.create_storage(storage_type)\n \n return cls._instance\n \n @classmethod\n def create_storage(cls, type_name: str, *args, **kwargs) -> BaseStorage:\n storage_class = cls.storages.get(type_name.lower())\n if not storage_class:\n raise ValueError(f\"No storage implementation found for type {type_name}\")\n \n return storage_class(*args, **kwargs)\n" }, { "path": "application/stt/__init__.py", "content": "\"\"\"Speech-to-text providers.\"\"\"\n" }, { "path": "application/stt/base.py", "content": "from abc import ABC, abstractmethod\nfrom pathlib import Path\nfrom typing import Any, Dict, Optional\n\n\nclass BaseSTT(ABC):\n @abstractmethod\n def transcribe(\n self,\n file_path: Path,\n language: Optional[str] = None,\n timestamps: bool = False,\n diarize: bool = False,\n ) -> Dict[str, Any]:\n pass\n" }, { "path": "application/stt/constants.py", "content": "SUPPORTED_AUDIO_EXTENSIONS = (\".wav\", \".mp3\", \".m4a\", \".ogg\", \".webm\")\n\nSUPPORTED_AUDIO_MIME_TYPES = {\n \"application/ogg\",\n \"audio/aac\",\n \"audio/mp3\",\n \"audio/mp4\",\n \"audio/mpeg\",\n \"audio/ogg\",\n \"audio/wav\",\n \"audio/webm\",\n \"audio/x-m4a\",\n \"audio/x-wav\",\n \"video/webm\",\n}\n" }, { "path": "application/stt/faster_whisper_stt.py", "content": "from pathlib import Path\nfrom typing import Dict, Optional\n\nfrom application.stt.base import BaseSTT\n\n\nclass FasterWhisperSTT(BaseSTT):\n def __init__(\n self,\n model_size: str = \"base\",\n device: str = \"auto\",\n compute_type: str = \"int8\",\n ):\n self.model_size = model_size\n self.device = device\n self.compute_type = compute_type\n self._model = None\n\n def _get_model(self):\n if self._model is None:\n try:\n from faster_whisper import WhisperModel\n except ImportError as exc:\n raise ImportError(\n \"faster-whisper is required to use the faster_whisper STT provider.\"\n ) from exc\n\n self._model = WhisperModel(\n self.model_size,\n device=self.device,\n compute_type=self.compute_type,\n )\n return self._model\n\n def transcribe(\n self,\n file_path: Path,\n language: Optional[str] = None,\n timestamps: bool = False,\n diarize: bool = False,\n ) -> Dict[str, object]:\n _ = diarize\n model = self._get_model()\n segments_iter, info = model.transcribe(\n str(file_path),\n language=language,\n word_timestamps=timestamps,\n )\n\n segments = []\n text_parts = []\n for segment in segments_iter:\n segment_text = getattr(segment, \"text\", \"\").strip()\n if segment_text:\n text_parts.append(segment_text)\n segments.append(\n {\n \"start\": getattr(segment, \"start\", None),\n \"end\": getattr(segment, \"end\", None),\n \"text\": segment_text,\n }\n )\n\n return {\n \"text\": \" \".join(text_parts).strip(),\n \"language\": getattr(info, \"language\", language),\n \"duration_s\": getattr(info, \"duration\", None),\n \"segments\": segments if timestamps else [],\n \"provider\": \"faster_whisper\",\n }\n" }, { "path": "application/stt/live_session.py", "content": "import json\nimport re\nimport uuid\nfrom typing import Dict, Optional\n\n\nLIVE_STT_SESSION_PREFIX = \"stt_live_session:\"\nLIVE_STT_SESSION_TTL_SECONDS = 15 * 60\nLIVE_STT_MUTABLE_TAIL_WORDS = 8\nLIVE_STT_SILENCE_MUTABLE_TAIL_WORDS = 2\nLIVE_STT_MIN_COMMITTED_OVERLAP_WORDS = 2\n\n\ndef normalize_transcript_text(text: str) -> str:\n return \" \".join((text or \"\").split()).strip()\n\n\ndef join_transcript_parts(*parts: str) -> str:\n return \" \".join(part for part in map(normalize_transcript_text, parts) if part)\n\n\ndef _normalize_word(word: str) -> str:\n normalized = re.sub(r\"[^\\w]+\", \"\", word.casefold(), flags=re.UNICODE)\n return normalized or word.casefold()\n\n\ndef _split_words(text: str) -> list[str]:\n normalized = normalize_transcript_text(text)\n return normalized.split() if normalized else []\n\n\ndef _common_prefix_length(left_words: list[str], right_words: list[str]) -> int:\n max_index = min(len(left_words), len(right_words))\n prefix_length = 0\n for index in range(max_index):\n if _normalize_word(left_words[index]) != _normalize_word(right_words[index]):\n break\n prefix_length += 1\n return prefix_length\n\n\ndef _find_suffix_prefix_overlap(\n left_words: list[str], right_words: list[str], min_overlap: int\n) -> int:\n max_overlap = min(len(left_words), len(right_words))\n if max_overlap < min_overlap:\n return 0\n\n left_keys = [_normalize_word(word) for word in left_words]\n right_keys = [_normalize_word(word) for word in right_words]\n\n for overlap_size in range(max_overlap, min_overlap - 1, -1):\n if left_keys[-overlap_size:] == right_keys[:overlap_size]:\n return overlap_size\n return 0\n\n\ndef strip_committed_prefix(committed_text: str, hypothesis_text: str) -> str:\n committed_words = _split_words(committed_text)\n hypothesis_words = _split_words(hypothesis_text)\n if not committed_words or not hypothesis_words:\n return normalize_transcript_text(hypothesis_text)\n\n full_prefix_length = _common_prefix_length(committed_words, hypothesis_words)\n if full_prefix_length == len(committed_words):\n return \" \".join(hypothesis_words[full_prefix_length:])\n\n overlap_size = _find_suffix_prefix_overlap(\n committed_words,\n hypothesis_words,\n LIVE_STT_MIN_COMMITTED_OVERLAP_WORDS,\n )\n if overlap_size:\n return \" \".join(hypothesis_words[overlap_size:])\n return \" \".join(hypothesis_words)\n\n\ndef _calculate_commit_count(\n previous_hypothesis: str, current_hypothesis: str, is_silence: bool\n) -> int:\n previous_words = _split_words(previous_hypothesis)\n current_words = _split_words(current_hypothesis)\n if not current_words:\n return 0\n\n if not previous_words:\n if is_silence:\n return max(0, len(current_words) - LIVE_STT_SILENCE_MUTABLE_TAIL_WORDS)\n return 0\n\n stable_prefix_length = _common_prefix_length(previous_words, current_words)\n if not stable_prefix_length:\n return 0\n\n mutable_tail_words = (\n LIVE_STT_SILENCE_MUTABLE_TAIL_WORDS\n if is_silence\n else LIVE_STT_MUTABLE_TAIL_WORDS\n )\n max_committable_by_tail = max(0, len(current_words) - mutable_tail_words)\n return min(stable_prefix_length, max_committable_by_tail)\n\n\ndef create_live_stt_session(\n user: str, language: Optional[str] = None\n) -> Dict[str, object]:\n return {\n \"session_id\": str(uuid.uuid4()),\n \"user\": user,\n \"language\": language,\n \"committed_text\": \"\",\n \"mutable_text\": \"\",\n \"previous_hypothesis\": \"\",\n \"latest_hypothesis\": \"\",\n \"last_chunk_index\": -1,\n }\n\n\ndef get_live_stt_session_key(session_id: str) -> str:\n return f\"{LIVE_STT_SESSION_PREFIX}{session_id}\"\n\n\ndef save_live_stt_session(redis_client, session_state: Dict[str, object]) -> None:\n redis_client.setex(\n get_live_stt_session_key(str(session_state[\"session_id\"])),\n LIVE_STT_SESSION_TTL_SECONDS,\n json.dumps(session_state),\n )\n\n\ndef load_live_stt_session(redis_client, session_id: str) -> Optional[Dict[str, object]]:\n raw_session = redis_client.get(get_live_stt_session_key(session_id))\n if not raw_session:\n return None\n if isinstance(raw_session, bytes):\n raw_session = raw_session.decode(\"utf-8\")\n return json.loads(raw_session)\n\n\ndef delete_live_stt_session(redis_client, session_id: str) -> None:\n redis_client.delete(get_live_stt_session_key(session_id))\n\n\ndef apply_live_stt_hypothesis(\n session_state: Dict[str, object],\n hypothesis_text: str,\n chunk_index: int,\n is_silence: bool = False,\n) -> Dict[str, object]:\n last_chunk_index = int(session_state.get(\"last_chunk_index\", -1))\n if chunk_index < 0:\n raise ValueError(\"chunk_index must be non-negative\")\n if chunk_index < last_chunk_index:\n raise ValueError(\"chunk_index is older than the last processed chunk\")\n if chunk_index == last_chunk_index:\n return session_state\n\n committed_text = normalize_transcript_text(str(session_state.get(\"committed_text\", \"\")))\n previous_hypothesis = normalize_transcript_text(\n str(session_state.get(\"latest_hypothesis\", \"\"))\n )\n current_hypothesis = strip_committed_prefix(committed_text, hypothesis_text)\n\n if not current_hypothesis and is_silence and previous_hypothesis:\n committed_text = join_transcript_parts(committed_text, previous_hypothesis)\n previous_hypothesis = \"\"\n\n commit_count = _calculate_commit_count(\n previous_hypothesis,\n current_hypothesis,\n is_silence=is_silence,\n )\n current_words = _split_words(current_hypothesis)\n\n if commit_count:\n committed_text = join_transcript_parts(\n committed_text,\n \" \".join(current_words[:commit_count]),\n )\n current_hypothesis = \" \".join(current_words[commit_count:])\n\n session_state[\"committed_text\"] = committed_text\n session_state[\"mutable_text\"] = normalize_transcript_text(current_hypothesis)\n session_state[\"previous_hypothesis\"] = previous_hypothesis\n session_state[\"latest_hypothesis\"] = normalize_transcript_text(current_hypothesis)\n session_state[\"last_chunk_index\"] = chunk_index\n return session_state\n\n\ndef get_live_stt_transcript_text(session_state: Dict[str, object]) -> str:\n return join_transcript_parts(\n str(session_state.get(\"committed_text\", \"\")),\n str(session_state.get(\"mutable_text\", \"\")),\n )\n\n\ndef finalize_live_stt_session(session_state: Dict[str, object]) -> str:\n return join_transcript_parts(\n str(session_state.get(\"committed_text\", \"\")),\n str(session_state.get(\"latest_hypothesis\", \"\")),\n )\n" }, { "path": "application/stt/openai_stt.py", "content": "from pathlib import Path\nfrom typing import Any, Dict, Optional\n\nfrom openai import OpenAI\n\nfrom application.core.settings import settings\nfrom application.stt.base import BaseSTT\n\n\nclass OpenAISTT(BaseSTT):\n def __init__(\n self,\n api_key: Optional[str] = None,\n base_url: Optional[str] = None,\n model: Optional[str] = None,\n ):\n self.api_key = api_key or settings.OPENAI_API_KEY or settings.API_KEY\n self.base_url = base_url or settings.OPENAI_BASE_URL or \"https://api.openai.com/v1\"\n self.model = model or settings.OPENAI_STT_MODEL\n self.client = OpenAI(api_key=self.api_key, base_url=self.base_url)\n\n def transcribe(\n self,\n file_path: Path,\n language: Optional[str] = None,\n timestamps: bool = False,\n diarize: bool = False,\n ) -> Dict[str, Any]:\n _ = diarize\n request: Dict[str, Any] = {\n \"file\": file_path,\n \"model\": self.model,\n \"response_format\": \"verbose_json\",\n }\n if language:\n request[\"language\"] = language\n if timestamps:\n request[\"timestamp_granularities\"] = [\"segment\"]\n\n with open(file_path, \"rb\") as audio_file:\n request[\"file\"] = audio_file\n response = self.client.audio.transcriptions.create(**request)\n response_dict = self._to_dict(response)\n segments = response_dict.get(\"segments\") or []\n\n return {\n \"text\": response_dict.get(\"text\", \"\"),\n \"language\": response_dict.get(\"language\") or language,\n \"duration_s\": response_dict.get(\"duration\"),\n \"segments\": [self._to_dict(segment) for segment in segments],\n \"provider\": \"openai\",\n }\n\n @staticmethod\n def _to_dict(value: Any) -> Dict[str, Any]:\n if hasattr(value, \"model_dump\"):\n return value.model_dump()\n if isinstance(value, dict):\n return value\n return {}\n" }, { "path": "application/stt/stt_creator.py", "content": "from application.stt.base import BaseSTT\nfrom application.stt.faster_whisper_stt import FasterWhisperSTT\nfrom application.stt.openai_stt import OpenAISTT\n\n\nclass STTCreator:\n stt_providers = {\n \"openai\": OpenAISTT,\n \"faster_whisper\": FasterWhisperSTT,\n }\n\n @classmethod\n def create_stt(cls, stt_type, *args, **kwargs) -> BaseSTT:\n stt_class = cls.stt_providers.get(stt_type.lower())\n if not stt_class:\n raise ValueError(f\"No stt class found for type {stt_type}\")\n return stt_class(*args, **kwargs)\n" }, { "path": "application/stt/upload_limits.py", "content": "from pathlib import Path\n\nfrom application.core.settings import settings\nfrom application.stt.constants import SUPPORTED_AUDIO_EXTENSIONS\nfrom application.utils import safe_filename\n\n\nSTT_REQUEST_SIZE_OVERHEAD_BYTES = 1024 * 1024\nSTT_SIZE_LIMITED_PATHS = frozenset((\"/api/stt\", \"/api/stt/live/chunk\"))\n\n\nclass AudioFileTooLargeError(ValueError):\n pass\n\n\ndef get_stt_max_file_size_bytes() -> int:\n return max(0, settings.STT_MAX_FILE_SIZE_MB) * 1024 * 1024\n\n\ndef build_stt_file_size_limit_message() -> str:\n return f\"Audio file exceeds {settings.STT_MAX_FILE_SIZE_MB}MB limit\"\n\n\ndef is_audio_filename(filename: str | Path | None) -> bool:\n if not filename:\n return False\n safe_name = safe_filename(Path(str(filename)).name)\n return Path(safe_name).suffix.lower() in SUPPORTED_AUDIO_EXTENSIONS\n\n\ndef enforce_audio_file_size_limit(size_bytes: int) -> None:\n max_size_bytes = get_stt_max_file_size_bytes()\n if max_size_bytes and size_bytes > max_size_bytes:\n raise AudioFileTooLargeError(build_stt_file_size_limit_message())\n\n\ndef should_reject_stt_request(path: str, content_length: int | None) -> bool:\n if path not in STT_SIZE_LIMITED_PATHS or content_length is None:\n return False\n max_request_size_bytes = (\n get_stt_max_file_size_bytes() + STT_REQUEST_SIZE_OVERHEAD_BYTES\n )\n return content_length > max_request_size_bytes\n" }, { "path": "application/templates/__init__.py", "content": "" }, { "path": "application/templates/namespaces.py", "content": "import logging\nimport uuid\nfrom abc import ABC, abstractmethod\nfrom datetime import datetime, timezone\nfrom typing import Any, Dict, Optional\n\nlogger = logging.getLogger(__name__)\n\n\nclass NamespaceBuilder(ABC):\n \"\"\"Base class for building template context namespaces\"\"\"\n\n @abstractmethod\n def build(self, **kwargs) -> Dict[str, Any]:\n \"\"\"Build namespace context dictionary\"\"\"\n pass\n\n @property\n @abstractmethod\n def namespace_name(self) -> str:\n \"\"\"Name of this namespace for template access\"\"\"\n pass\n\n\nclass SystemNamespace(NamespaceBuilder):\n \"\"\"System metadata namespace: {{ system.* }}\"\"\"\n\n @property\n def namespace_name(self) -> str:\n return \"system\"\n\n def build(\n self, request_id: Optional[str] = None, user_id: Optional[str] = None, **kwargs\n ) -> Dict[str, Any]:\n \"\"\"\n Build system context with metadata.\n\n Args:\n request_id: Unique request identifier\n user_id: Current user identifier\n\n Returns:\n Dictionary with system variables\n \"\"\"\n now = datetime.now(timezone.utc)\n\n return {\n \"date\": now.strftime(\"%Y-%m-%d\"),\n \"time\": now.strftime(\"%H:%M:%S\"),\n \"timestamp\": now.isoformat(),\n \"request_id\": request_id or str(uuid.uuid4()),\n \"user_id\": user_id,\n }\n\n\nclass PassthroughNamespace(NamespaceBuilder):\n \"\"\"Request parameters namespace: {{ passthrough.* }}\"\"\"\n\n @property\n def namespace_name(self) -> str:\n return \"passthrough\"\n\n def build(\n self, passthrough_data: Optional[Dict[str, Any]] = None, **kwargs\n ) -> Dict[str, Any]:\n \"\"\"\n Build passthrough context from request parameters.\n\n Args:\n passthrough_data: Dictionary of parameters from web request\n\n Returns:\n Dictionary with passthrough variables\n \"\"\"\n if not passthrough_data:\n return {}\n safe_data = {}\n for key, value in passthrough_data.items():\n if isinstance(value, (str, int, float, bool, type(None))):\n safe_data[key] = value\n else:\n logger.warning(\n f\"Skipping non-serializable passthrough value for key '{key}': {type(value)}\"\n )\n return safe_data\n\n\nclass SourceNamespace(NamespaceBuilder):\n \"\"\"RAG source documents namespace: {{ source.* }}\"\"\"\n\n @property\n def namespace_name(self) -> str:\n return \"source\"\n\n def build(\n self, docs: Optional[list] = None, docs_together: Optional[str] = None, **kwargs\n ) -> Dict[str, Any]:\n \"\"\"\n Build source context from RAG retrieval results.\n\n Args:\n docs: List of retrieved documents\n docs_together: Concatenated document content (for backward compatibility)\n\n Returns:\n Dictionary with source variables\n \"\"\"\n context = {}\n\n if docs:\n context[\"documents\"] = docs\n context[\"count\"] = len(docs)\n if docs_together:\n context[\"docs_together\"] = docs_together # Add docs_together for custom templates\n context[\"content\"] = docs_together\n context[\"summaries\"] = docs_together\n return context\n\n\nclass ToolsNamespace(NamespaceBuilder):\n \"\"\"Pre-executed tools namespace: {{ tools.* }}\"\"\"\n\n @property\n def namespace_name(self) -> str:\n return \"tools\"\n\n def build(\n self, tools_data: Optional[Dict[str, Any]] = None, **kwargs\n ) -> Dict[str, Any]:\n \"\"\"\n Build tools context with pre-executed tool results.\n\n Args:\n tools_data: Dictionary of pre-fetched tool results organized by tool name\n e.g., {\"memory\": {\"notes\": \"content\", \"tasks\": \"list\"}}\n\n Returns:\n Dictionary with tool results organized by tool name\n \"\"\"\n if not tools_data:\n return {}\n\n safe_data = {}\n for tool_name, tool_result in tools_data.items():\n if isinstance(tool_result, (str, dict, list, int, float, bool, type(None))):\n safe_data[tool_name] = tool_result\n else:\n logger.warning(\n f\"Skipping non-serializable tool result for '{tool_name}': {type(tool_result)}\"\n )\n return safe_data\n\n\nclass NamespaceManager:\n \"\"\"Manages all namespace builders and context assembly\"\"\"\n\n def __init__(self):\n self._builders = {\n \"system\": SystemNamespace(),\n \"passthrough\": PassthroughNamespace(),\n \"source\": SourceNamespace(),\n \"tools\": ToolsNamespace(),\n }\n\n def build_context(self, **kwargs) -> Dict[str, Any]:\n \"\"\"\n Build complete template context from all namespaces.\n\n Args:\n **kwargs: Parameters to pass to namespace builders\n\n Returns:\n Complete context dictionary for template rendering\n \"\"\"\n context = {}\n\n for namespace_name, builder in self._builders.items():\n try:\n namespace_context = builder.build(**kwargs)\n # Always include namespace, even if empty, to prevent undefined errors\n context[namespace_name] = namespace_context if namespace_context else {}\n except Exception as e:\n logger.error(f\"Failed to build {namespace_name} namespace: {str(e)}\")\n # Include empty namespace on error to prevent template failures\n context[namespace_name] = {}\n return context\n\n def get_builder(self, namespace_name: str) -> Optional[NamespaceBuilder]:\n \"\"\"Get specific namespace builder\"\"\"\n return self._builders.get(namespace_name)\n" }, { "path": "application/templates/template_engine.py", "content": "import logging\nfrom typing import Any, Dict, List, Optional, Set\n\nfrom jinja2 import (\n ChainableUndefined,\n Environment,\n nodes,\n select_autoescape,\n TemplateSyntaxError,\n)\nfrom jinja2.exceptions import UndefinedError\n\nlogger = logging.getLogger(__name__)\n\n\nclass TemplateRenderError(Exception):\n \"\"\"Raised when template rendering fails\"\"\"\n\n pass\n\n\nclass TemplateEngine:\n \"\"\"Jinja2-based template engine for dynamic prompt rendering\"\"\"\n\n def __init__(self):\n self._env = Environment(\n undefined=ChainableUndefined,\n trim_blocks=True,\n lstrip_blocks=True,\n autoescape=select_autoescape(default_for_string=True, default=True),\n )\n\n def render(self, template_content: str, context: Dict[str, Any]) -> str:\n \"\"\"\n Render template with provided context.\n\n Args:\n template_content: Raw template string with Jinja2 syntax\n context: Dictionary of variables to inject into template\n\n Returns:\n Rendered template string\n\n Raises:\n TemplateRenderError: If template syntax is invalid or variables undefined\n \"\"\"\n if not template_content:\n return \"\"\n try:\n template = self._env.from_string(template_content)\n return template.render(**context)\n except TemplateSyntaxError as e:\n error_msg = f\"Template syntax error at line {e.lineno}: {e.message}\"\n logger.error(error_msg)\n raise TemplateRenderError(error_msg) from e\n except UndefinedError as e:\n error_msg = f\"Undefined variable in template: {e.message}\"\n logger.error(error_msg)\n raise TemplateRenderError(error_msg) from e\n except Exception as e:\n error_msg = f\"Template rendering failed: {str(e)}\"\n logger.error(error_msg)\n raise TemplateRenderError(error_msg) from e\n\n def validate_template(self, template_content: str) -> bool:\n \"\"\"\n Validate template syntax without rendering.\n\n Args:\n template_content: Template string to validate\n\n Returns:\n True if template is syntactically valid\n \"\"\"\n if not template_content:\n return True\n try:\n self._env.from_string(template_content)\n return True\n except TemplateSyntaxError as e:\n logger.debug(f\"Template syntax invalid at line {e.lineno}: {e.message}\")\n return False\n except Exception as e:\n logger.debug(f\"Template validation error: {type(e).__name__}: {str(e)}\")\n return False\n\n def extract_variables(self, template_content: str) -> Set[str]:\n \"\"\"\n Extract all variable names from template.\n\n Args:\n template_content: Template string to analyze\n\n Returns:\n Set of variable names found in template\n \"\"\"\n if not template_content:\n return set()\n try:\n ast = self._env.parse(template_content)\n return set(self._env.get_template_module(ast).make_module().keys())\n except TemplateSyntaxError as e:\n logger.debug(f\"Cannot extract variables - syntax error at line {e.lineno}\")\n return set()\n except Exception as e:\n logger.debug(f\"Cannot extract variables: {type(e).__name__}\")\n return set()\n\n def extract_tool_usages(\n self, template_content: str\n ) -> Dict[str, Set[Optional[str]]]:\n \"\"\"Extract tool and action references from a template\"\"\"\n if not template_content:\n return {}\n try:\n ast = self._env.parse(template_content)\n except TemplateSyntaxError as e:\n logger.debug(f\"extract_tool_usages - syntax error at line {e.lineno}\")\n return {}\n except Exception as e:\n logger.debug(f\"extract_tool_usages - parse error: {type(e).__name__}\")\n return {}\n\n usages: Dict[str, Set[Optional[str]]] = {}\n\n def record(path: List[str]) -> None:\n if not path:\n return\n tool_name = path[0]\n action_name = path[1] if len(path) > 1 else None\n if not tool_name:\n return\n tool_entry = usages.setdefault(tool_name, set())\n tool_entry.add(action_name)\n\n for node in ast.find_all(nodes.Getattr):\n path = []\n current = node\n while isinstance(current, nodes.Getattr):\n path.append(current.attr)\n current = current.node\n if isinstance(current, nodes.Name) and current.name == \"tools\":\n path.reverse()\n record(path)\n\n for node in ast.find_all(nodes.Getitem):\n path = []\n current = node\n while isinstance(current, nodes.Getitem):\n key = current.arg\n if isinstance(key, nodes.Const) and isinstance(key.value, str):\n path.append(key.value)\n else:\n path = []\n break\n current = current.node\n if path and isinstance(current, nodes.Name) and current.name == \"tools\":\n path.reverse()\n record(path)\n\n return usages\n" }, { "path": "application/tts/base.py", "content": "from abc import ABC, abstractmethod\n\n\nclass BaseTTS(ABC):\n def __init__(self):\n pass\n\n @abstractmethod\n def text_to_speech(self, *args, **kwargs):\n pass" }, { "path": "application/tts/elevenlabs.py", "content": "from io import BytesIO\nimport base64\nfrom application.tts.base import BaseTTS\nfrom application.core.settings import settings\n\n\nclass ElevenlabsTTS(BaseTTS):\n def __init__(self):\n from elevenlabs.client import ElevenLabs\n\n self.client = ElevenLabs(\n api_key=settings.ELEVENLABS_API_KEY,\n )\n \n\n def text_to_speech(self, text):\n lang = \"en\"\n audio = self.client.text_to_speech.convert(\n voice_id=\"nPczCjzI2devNBz1zQrb\", \n model_id=\"eleven_multilingual_v2\",\n text=text,\n output_format=\"mp3_44100_128\"\n )\n audio_data = BytesIO()\n for chunk in audio:\n audio_data.write(chunk)\n audio_bytes = audio_data.getvalue()\n\n # Encode to base64\n audio_base64 = base64.b64encode(audio_bytes).decode(\"utf-8\")\n return audio_base64, lang" }, { "path": "application/tts/google_tts.py", "content": "import io\nimport base64\nfrom gtts import gTTS\nfrom application.tts.base import BaseTTS\n\n\nclass GoogleTTS(BaseTTS):\n def __init__(self):\n pass\n\n\n def text_to_speech(self, text):\n lang = \"en\"\n audio_fp = io.BytesIO()\n tts = gTTS(text=text, lang=lang, slow=False)\n tts.write_to_fp(audio_fp)\n audio_fp.seek(0)\n audio_base64 = base64.b64encode(audio_fp.read()).decode(\"utf-8\")\n return audio_base64, lang\n" }, { "path": "application/tts/tts_creator.py", "content": "from application.tts.google_tts import GoogleTTS\nfrom application.tts.elevenlabs import ElevenlabsTTS\nfrom application.tts.base import BaseTTS\n\n\n\nclass TTSCreator:\n tts_providers = {\n \"google_tts\": GoogleTTS,\n \"elevenlabs\": ElevenlabsTTS,\n }\n\n @classmethod\n def create_tts(cls, tts_type, *args, **kwargs)-> BaseTTS:\n tts_class = cls.tts_providers.get(tts_type.lower())\n if not tts_class:\n raise ValueError(f\"No tts class found for type {tts_type}\")\n return tts_class(*args, **kwargs)" }, { "path": "application/usage.py", "content": "import sys\nimport logging\nfrom datetime import datetime\n\nfrom application.core.mongo_db import MongoDB\nfrom application.core.settings import settings\nfrom application.utils import num_tokens_from_object_or_list, num_tokens_from_string\n\nlogger = logging.getLogger(__name__)\n\nmongo = MongoDB.get_client()\ndb = mongo[settings.MONGO_DB_NAME]\nusage_collection = db[\"token_usage\"]\n\n\ndef _serialize_for_token_count(value):\n \"\"\"Normalize payloads into token-countable primitives.\"\"\"\n if isinstance(value, str):\n # Avoid counting large binary payloads in data URLs as text tokens.\n if value.startswith(\"data:\") and \";base64,\" in value:\n return \"\"\n return value\n\n if value is None:\n return \"\"\n\n if isinstance(value, list):\n return [_serialize_for_token_count(item) for item in value]\n\n if isinstance(value, dict):\n serialized = {}\n for key, raw in value.items():\n key_lower = str(key).lower()\n\n # Skip raw binary-like fields; keep textual tool-call fields.\n if key_lower in {\"data\", \"base64\", \"image_data\"} and isinstance(raw, str):\n continue\n if key_lower == \"url\" and isinstance(raw, str) and \";base64,\" in raw:\n continue\n\n serialized[key] = _serialize_for_token_count(raw)\n return serialized\n\n if hasattr(value, \"model_dump\") and callable(getattr(value, \"model_dump\")):\n return _serialize_for_token_count(value.model_dump())\n if hasattr(value, \"to_dict\") and callable(getattr(value, \"to_dict\")):\n return _serialize_for_token_count(value.to_dict())\n if hasattr(value, \"__dict__\"):\n return _serialize_for_token_count(vars(value))\n\n return str(value)\n\n\ndef _count_tokens(value):\n serialized = _serialize_for_token_count(value)\n if isinstance(serialized, str):\n return num_tokens_from_string(serialized)\n return num_tokens_from_object_or_list(serialized)\n\n\ndef _count_prompt_tokens(messages, tools=None, usage_attachments=None, **kwargs):\n prompt_tokens = 0\n\n for message in messages or []:\n if not isinstance(message, dict):\n prompt_tokens += _count_tokens(message)\n continue\n\n prompt_tokens += _count_tokens(message.get(\"content\"))\n\n # Include tool-related message fields for providers that use OpenAI-native format.\n prompt_tokens += _count_tokens(message.get(\"tool_calls\"))\n prompt_tokens += _count_tokens(message.get(\"tool_call_id\"))\n prompt_tokens += _count_tokens(message.get(\"function_call\"))\n prompt_tokens += _count_tokens(message.get(\"function_response\"))\n\n # Count tool schema payload passed to the model.\n prompt_tokens += _count_tokens(tools)\n\n # Count structured-output/schema payloads when provided.\n prompt_tokens += _count_tokens(kwargs.get(\"response_format\"))\n prompt_tokens += _count_tokens(kwargs.get(\"response_schema\"))\n\n # Optional usage-only attachment context (not forwarded to provider).\n prompt_tokens += _count_tokens(usage_attachments)\n\n return prompt_tokens\n\n\ndef update_token_usage(decoded_token, user_api_key, token_usage, agent_id=None):\n if \"pytest\" in sys.modules:\n return\n user_id = decoded_token.get(\"sub\") if isinstance(decoded_token, dict) else None\n normalized_agent_id = str(agent_id) if agent_id else None\n\n if not user_id and not user_api_key and not normalized_agent_id:\n logger.warning(\n \"Skipping token usage insert: missing user_id, api_key, and agent_id\"\n )\n return\n\n usage_data = {\n \"user_id\": user_id,\n \"api_key\": user_api_key,\n \"prompt_tokens\": token_usage[\"prompt_tokens\"],\n \"generated_tokens\": token_usage[\"generated_tokens\"],\n \"timestamp\": datetime.now(),\n }\n if normalized_agent_id:\n usage_data[\"agent_id\"] = normalized_agent_id\n usage_collection.insert_one(usage_data)\n\n\ndef gen_token_usage(func):\n def wrapper(self, model, messages, stream, tools, **kwargs):\n usage_attachments = kwargs.pop(\"_usage_attachments\", None)\n call_usage = {\"prompt_tokens\": 0, \"generated_tokens\": 0}\n call_usage[\"prompt_tokens\"] += _count_prompt_tokens(\n messages,\n tools=tools,\n usage_attachments=usage_attachments,\n **kwargs,\n )\n result = func(self, model, messages, stream, tools, **kwargs)\n call_usage[\"generated_tokens\"] += _count_tokens(result)\n self.token_usage[\"prompt_tokens\"] += call_usage[\"prompt_tokens\"]\n self.token_usage[\"generated_tokens\"] += call_usage[\"generated_tokens\"]\n update_token_usage(\n self.decoded_token,\n self.user_api_key,\n call_usage,\n getattr(self, \"agent_id\", None),\n )\n return result\n\n return wrapper\n\n\ndef stream_token_usage(func):\n def wrapper(self, model, messages, stream, tools, **kwargs):\n usage_attachments = kwargs.pop(\"_usage_attachments\", None)\n call_usage = {\"prompt_tokens\": 0, \"generated_tokens\": 0}\n call_usage[\"prompt_tokens\"] += _count_prompt_tokens(\n messages,\n tools=tools,\n usage_attachments=usage_attachments,\n **kwargs,\n )\n batch = []\n result = func(self, model, messages, stream, tools, **kwargs)\n for r in result:\n batch.append(r)\n yield r\n for line in batch:\n call_usage[\"generated_tokens\"] += _count_tokens(line)\n self.token_usage[\"prompt_tokens\"] += call_usage[\"prompt_tokens\"]\n self.token_usage[\"generated_tokens\"] += call_usage[\"generated_tokens\"]\n update_token_usage(\n self.decoded_token,\n self.user_api_key,\n call_usage,\n getattr(self, \"agent_id\", None),\n )\n\n return wrapper\n" }, { "path": "application/utils.py", "content": "import base64\nimport hashlib\nimport io\nimport logging\nimport os\nimport re\nimport uuid\nfrom typing import List\n\nimport tiktoken\nfrom flask import jsonify, make_response\nfrom werkzeug.utils import secure_filename\n\nfrom application.core.model_utils import get_token_limit\n\nfrom application.core.settings import settings\n\nlogger = logging.getLogger(__name__)\n\n\n_encoding = None\n\n\ndef get_encoding():\n global _encoding\n if _encoding is None:\n _encoding = tiktoken.get_encoding(\"cl100k_base\")\n return _encoding\n\n\ndef get_gpt_model() -> str:\n \"\"\"Get GPT model based on provider\"\"\"\n model_map = {\n \"openai\": \"gpt-4o-mini\",\n \"anthropic\": \"claude-2\",\n \"groq\": \"llama3-8b-8192\",\n \"novita\": \"deepseek/deepseek-r1\",\n }\n return settings.LLM_NAME or model_map.get(settings.LLM_PROVIDER, \"\")\n\n\ndef safe_filename(filename):\n \"\"\"Create safe filename, preserving extension. Handles non-Latin characters.\"\"\"\n if not filename:\n return str(uuid.uuid4())\n _, extension = os.path.splitext(filename)\n\n safe_name = secure_filename(filename)\n\n # If secure_filename returns just the extension or an empty string\n\n if not safe_name or safe_name == extension.lstrip(\".\"):\n return f\"{str(uuid.uuid4())}{extension}\"\n return safe_name\n\n\ndef num_tokens_from_string(string: str) -> int:\n encoding = get_encoding()\n if isinstance(string, str):\n num_tokens = len(encoding.encode(string))\n return num_tokens\n else:\n return 0\n\n\ndef num_tokens_from_object_or_list(thing):\n if isinstance(thing, list):\n return sum([num_tokens_from_object_or_list(x) for x in thing])\n elif isinstance(thing, dict):\n return sum([num_tokens_from_object_or_list(x) for x in thing.values()])\n elif isinstance(thing, str):\n return num_tokens_from_string(thing)\n else:\n return 0\n\n\ndef count_tokens_docs(docs):\n docs_content = \"\"\n for doc in docs:\n docs_content += doc.page_content\n tokens = num_tokens_from_string(docs_content)\n return tokens\n\n\ndef calculate_doc_token_budget(\n model_id: str = \"gpt-4o\"\n) -> int:\n total_context = get_token_limit(model_id)\n reserved = sum(settings.RESERVED_TOKENS.values())\n doc_budget = total_context - reserved\n return max(doc_budget, 1000)\n\n\ndef get_missing_fields(data, required_fields):\n \"\"\"Check for missing required fields. Returns list of missing field names.\"\"\"\n return [field for field in required_fields if field not in data]\n\n\ndef check_required_fields(data, required_fields):\n \"\"\"Validate required fields. Returns Flask 400 response if validation fails, None otherwise.\"\"\"\n missing_fields = get_missing_fields(data, required_fields)\n if missing_fields:\n return make_response(\n jsonify(\n {\n \"success\": False,\n \"message\": f\"Missing required fields: {', '.join(missing_fields)}\",\n }\n ),\n 400,\n )\n return None\n\n\ndef get_field_validation_errors(data, required_fields):\n \"\"\"Check for missing and empty fields. Returns dict with 'missing_fields' and 'empty_fields', or None.\"\"\"\n missing_fields = []\n empty_fields = []\n\n for field in required_fields:\n if field not in data:\n missing_fields.append(field)\n elif not data[field]:\n empty_fields.append(field)\n if missing_fields or empty_fields:\n return {\"missing_fields\": missing_fields, \"empty_fields\": empty_fields}\n return None\n\n\ndef validate_required_fields(data, required_fields):\n \"\"\"Validate required fields (must exist and be non-empty). Returns Flask 400 response if validation fails, None otherwise.\"\"\"\n errors_dict = get_field_validation_errors(data, required_fields)\n if errors_dict:\n errors = []\n if errors_dict[\"missing_fields\"]:\n errors.append(\n f\"Missing required fields: {', '.join(errors_dict['missing_fields'])}\"\n )\n if errors_dict[\"empty_fields\"]:\n errors.append(\n f\"Empty values in required fields: {', '.join(errors_dict['empty_fields'])}\"\n )\n return make_response(\n jsonify({\"success\": False, \"message\": \" | \".join(errors)}), 400\n )\n return None\n\n\ndef get_hash(data):\n return hashlib.md5(data.encode(), usedforsecurity=False).hexdigest()\n\n\ndef limit_chat_history(history, max_token_limit=None, model_id=\"docsgpt-local\"):\n \"\"\"Limit chat history to fit within token limit.\"\"\"\n model_token_limit = get_token_limit(model_id)\n max_token_limit = (\n max_token_limit\n if max_token_limit and max_token_limit < model_token_limit\n else model_token_limit\n )\n\n if not history:\n return []\n trimmed_history = []\n tokens_current_history = 0\n\n for message in reversed(history):\n tokens_batch = 0\n if \"prompt\" in message and \"response\" in message:\n tokens_batch += num_tokens_from_string(message[\"prompt\"])\n tokens_batch += num_tokens_from_string(message[\"response\"])\n if \"tool_calls\" in message:\n for tool_call in message[\"tool_calls\"]:\n tool_call_string = f\"Tool: {tool_call.get('tool_name')} | Action: {tool_call.get('action_name')} | Args: {tool_call.get('arguments')} | Response: {tool_call.get('result')}\"\n tokens_batch += num_tokens_from_string(tool_call_string)\n if tokens_current_history + tokens_batch < max_token_limit:\n tokens_current_history += tokens_batch\n trimmed_history.insert(0, message)\n else:\n break\n return trimmed_history\n\n\ndef validate_function_name(function_name):\n \"\"\"Validate function name matches allowed pattern (alphanumeric, underscore, hyphen).\"\"\"\n if not re.match(r\"^[a-zA-Z0-9_-]+$\", function_name):\n return False\n return True\n\n\ndef generate_image_url(image_path):\n if isinstance(image_path, str) and (\n image_path.startswith(\"http://\") or image_path.startswith(\"https://\")\n ):\n return image_path\n strategy = getattr(settings, \"URL_STRATEGY\", \"backend\")\n if strategy == \"s3\":\n bucket_name = getattr(settings, \"S3_BUCKET_NAME\", \"docsgpt-test-bucket\")\n region_name = getattr(settings, \"SAGEMAKER_REGION\", \"eu-central-1\")\n return f\"https://{bucket_name}.s3.{region_name}.amazonaws.com/{image_path}\"\n else:\n base_url = getattr(settings, \"API_URL\", \"http://localhost:7091\")\n return f\"{base_url}/api/images/{image_path}\"\n\n\ndef calculate_compression_threshold(\n model_id: str, threshold_percentage: float = 0.8\n) -> int:\n \"\"\"\n Calculate token threshold for triggering compression.\n\n Args:\n model_id: Model identifier\n threshold_percentage: Percentage of context window (default 80%)\n\n Returns:\n Token count threshold\n \"\"\"\n total_context = get_token_limit(model_id)\n threshold = int(total_context * threshold_percentage)\n return threshold\n\n\ndef convert_pdf_to_images(\n file_path: str,\n storage=None,\n max_pages: int = 20,\n dpi: int = 150,\n image_format: str = \"PNG\",\n) -> List[dict]:\n \"\"\"\n Convert PDF pages to images for LLMs that support images but not PDFs.\n\n This enables \"synthetic PDF support\" by converting each PDF page to an image\n that can be sent to vision-capable LLMs like Claude.\n\n Args:\n file_path: Path to the PDF file (can be storage path)\n storage: Optional storage instance for retrieving files\n max_pages: Maximum number of pages to convert (default 20 to avoid context overflow)\n dpi: Resolution for rendering (default 150 for balance of quality/size)\n image_format: Output format (PNG recommended for quality)\n\n Returns:\n List of dicts with keys:\n - 'data': base64-encoded image data\n - 'mime_type': MIME type (e.g., 'image/png')\n - 'page': Page number (1-indexed)\n\n Raises:\n ImportError: If pdf2image is not installed\n FileNotFoundError: If file doesn't exist\n Exception: If conversion fails\n \"\"\"\n try:\n from pdf2image import convert_from_path, convert_from_bytes\n except ImportError:\n raise ImportError(\n \"pdf2image is required for PDF-to-image conversion. \"\n \"Install it with: pip install pdf2image\\n\"\n \"Also ensure poppler-utils is installed on your system.\"\n )\n\n images_data = []\n mime_type = f\"image/{image_format.lower()}\"\n\n try:\n # Get PDF content either from storage or direct file path\n if storage and hasattr(storage, \"get_file\"):\n with storage.get_file(file_path) as pdf_file:\n pdf_bytes = pdf_file.read()\n pil_images = convert_from_bytes(\n pdf_bytes,\n dpi=dpi,\n fmt=image_format.lower(),\n first_page=1,\n last_page=max_pages,\n )\n else:\n pil_images = convert_from_path(\n file_path,\n dpi=dpi,\n fmt=image_format.lower(),\n first_page=1,\n last_page=max_pages,\n )\n\n for page_num, pil_image in enumerate(pil_images, start=1):\n # Convert PIL image to base64\n buffer = io.BytesIO()\n pil_image.save(buffer, format=image_format)\n buffer.seek(0)\n base64_data = base64.b64encode(buffer.read()).decode(\"utf-8\")\n\n images_data.append({\n \"data\": base64_data,\n \"mime_type\": mime_type,\n \"page\": page_num,\n })\n\n return images_data\n\n except FileNotFoundError:\n logger.error(f\"PDF file not found: {file_path}\")\n raise\n except Exception as e:\n logger.error(f\"Error converting PDF to images: {e}\", exc_info=True)\n raise\n\n\ndef clean_text_for_tts(text: str) -> str:\n \"\"\"\n clean text for Text-to-Speech processing.\n \"\"\"\n # Handle code blocks and links\n\n text = re.sub(r\"```mermaid[\\s\\S]*?```\", \" flowchart, \", text) ## ```mermaid...```\n text = re.sub(r\"```[\\s\\S]*?```\", \" code block, \", text) ## ```code```\n text = re.sub(r\"\\[([^\\]]+)\\]\\([^\\)]+\\)\", r\"\\1\", text) ## [text](url)\n text = re.sub(r\"!\\[([^\\]]*)\\]\\([^\\)]+\\)\", \"\", text) ## ![alt](url)\n\n # Remove markdown formatting\n\n text = re.sub(r\"`([^`]+)`\", r\"\\1\", text) ## `code`\n text = re.sub(r\"\\{([^}]*)\\}\", r\" \\1 \", text) ## {text}\n text = re.sub(r\"[{}]\", \" \", text) ## unmatched {}\n text = re.sub(r\"\\[([^\\]]+)\\]\", r\" \\1 \", text) ## [text]\n text = re.sub(r\"[\\[\\]]\", \" \", text) ## unmatched []\n text = re.sub(r\"(\\*\\*|__)(.*?)\\1\", r\"\\2\", text) ## **bold** __bold__\n text = re.sub(r\"(\\*|_)(.*?)\\1\", r\"\\2\", text) ## *italic* _italic_\n text = re.sub(r\"^#{1,6}\\s+\", \"\", text, flags=re.MULTILINE) ## # headers\n text = re.sub(r\"^>\\s+\", \"\", text, flags=re.MULTILINE) ## > blockquotes\n text = re.sub(r\"^[\\s]*[-\\*\\+]\\s+\", \"\", text, flags=re.MULTILINE) ## - * + lists\n text = re.sub(r\"^[\\s]*\\d+\\.\\s+\", \"\", text, flags=re.MULTILINE) ## 1. numbered lists\n text = re.sub(\n r\"^[\\*\\-_]{3,}\\s*$\", \"\", text, flags=re.MULTILINE\n ) ## --- *** ___ rules\n text = re.sub(r\"<[^>]*>\", \"\", text) ## tags\n\n # Remove non-ASCII (emojis, special Unicode)\n\n text = re.sub(r\"[^\\x20-\\x7E\\n\\r\\t]\", \"\", text)\n\n # Replace special sequences\n\n text = re.sub(r\"-->\", \", \", text) ## -->\n text = re.sub(r\"<--\", \", \", text) ## <--\n text = re.sub(r\"=>\", \", \", text) ## =>\n text = re.sub(r\"::\", \" \", text) ## ::\n\n # Normalize whitespace\n\n text = re.sub(r\"\\s+\", \" \", text)\n text = text.strip()\n\n return text\n" }, { "path": "application/vectorstore/__init__.py", "content": "" }, { "path": "application/vectorstore/base.py", "content": "import logging\nimport os\nfrom abc import ABC, abstractmethod\n\nimport requests\nfrom langchain_openai import OpenAIEmbeddings\n\nfrom application.core.settings import settings\n\n\nclass RemoteEmbeddings:\n \"\"\"\n Wrapper for remote embeddings API (OpenAI-compatible).\n Used when EMBEDDINGS_BASE_URL is configured.\n Sends requests to {base_url}/v1/embeddings in OpenAI format.\n \"\"\"\n\n def __init__(self, api_url: str, model_name: str, api_key: str = None):\n self.api_url = api_url.rstrip(\"/\")\n self.model_name = model_name\n self.headers = {\"Content-Type\": \"application/json\"}\n if api_key:\n self.headers[\"Authorization\"] = f\"Bearer {api_key}\"\n self.dimension = 768\n\n def _embed(self, inputs):\n \"\"\"Send embedding request to remote API in OpenAI-compatible format.\"\"\"\n payload = {\"input\": inputs}\n if self.model_name:\n payload[\"model\"] = self.model_name\n\n url = f\"{self.api_url}/v1/embeddings\"\n response = requests.post(url, headers=self.headers, json=payload, timeout=180)\n response.raise_for_status()\n result = response.json()\n\n # Handle OpenAI-compatible response format\n if isinstance(result, dict):\n if \"error\" in result:\n raise ValueError(f\"Remote embeddings API error: {result['error']}\")\n if \"data\" in result:\n # Sort by index to ensure correct order\n data = sorted(result[\"data\"], key=lambda x: x.get(\"index\", 0))\n return [item[\"embedding\"] for item in data]\n raise ValueError(\n f\"Unexpected response format from remote embeddings API: {result}\"\n )\n else:\n raise ValueError(\n f\"Unexpected response format from remote embeddings API: {result}\"\n )\n\n def embed_query(self, query: str):\n \"\"\"Embed a single query string.\"\"\"\n embeddings_list = self._embed(query)\n if (\n isinstance(embeddings_list, list)\n and len(embeddings_list) == 1\n and isinstance(embeddings_list[0], list)\n ):\n if self.dimension is None:\n self.dimension = len(embeddings_list[0])\n return embeddings_list[0]\n raise ValueError(\n f\"Unexpected result structure after embedding query: {embeddings_list}\"\n )\n\n def embed_documents(self, documents: list):\n \"\"\"Embed a list of documents.\"\"\"\n if not documents:\n return []\n embeddings_list = self._embed(documents)\n if self.dimension is None and embeddings_list:\n self.dimension = len(embeddings_list[0])\n return embeddings_list\n\n def __call__(self, text):\n if isinstance(text, str):\n return self.embed_query(text)\n elif isinstance(text, list):\n return self.embed_documents(text)\n else:\n raise ValueError(\"Input must be a string or a list of strings\")\n\n\ndef _get_embeddings_wrapper():\n \"\"\"Lazy import of EmbeddingsWrapper to avoid loading SentenceTransformer when using remote embeddings.\"\"\"\n from application.vectorstore.embeddings_local import EmbeddingsWrapper\n\n return EmbeddingsWrapper\n\n\nclass EmbeddingsSingleton:\n _instances = {}\n\n @staticmethod\n def get_instance(embeddings_name, *args, **kwargs):\n if embeddings_name not in EmbeddingsSingleton._instances:\n EmbeddingsSingleton._instances[embeddings_name] = (\n EmbeddingsSingleton._create_instance(embeddings_name, *args, **kwargs)\n )\n return EmbeddingsSingleton._instances[embeddings_name]\n\n @staticmethod\n def _create_instance(embeddings_name, *args, **kwargs):\n if embeddings_name == \"openai_text-embedding-ada-002\":\n return OpenAIEmbeddings(*args, **kwargs)\n\n # Lazy import EmbeddingsWrapper only when needed (avoids loading SentenceTransformer)\n EmbeddingsWrapper = _get_embeddings_wrapper()\n\n embeddings_factory = {\n \"huggingface_sentence-transformers/all-mpnet-base-v2\": lambda: EmbeddingsWrapper(\n \"sentence-transformers/all-mpnet-base-v2\"\n ),\n \"huggingface_sentence-transformers-all-mpnet-base-v2\": lambda: EmbeddingsWrapper(\n \"sentence-transformers/all-mpnet-base-v2\"\n ),\n \"huggingface_hkunlp/instructor-large\": lambda: EmbeddingsWrapper(\n \"hkunlp/instructor-large\"\n ),\n }\n\n if embeddings_name in embeddings_factory:\n return embeddings_factory[embeddings_name](*args, **kwargs)\n else:\n return EmbeddingsWrapper(embeddings_name, *args, **kwargs)\n\n\nclass BaseVectorStore(ABC):\n def __init__(self):\n pass\n\n @abstractmethod\n def search(self, *args, **kwargs):\n \"\"\"Search for similar documents/chunks in the vectorstore\"\"\"\n pass\n\n @abstractmethod\n def add_texts(self, texts, metadatas=None, *args, **kwargs):\n \"\"\"Add texts with their embeddings to the vectorstore\"\"\"\n pass\n\n def delete_index(self, *args, **kwargs):\n \"\"\"Delete the entire index/collection\"\"\"\n pass\n\n def save_local(self, *args, **kwargs):\n \"\"\"Save vectorstore to local storage\"\"\"\n pass\n\n def get_chunks(self, *args, **kwargs):\n \"\"\"Get all chunks from the vectorstore\"\"\"\n pass\n\n def add_chunk(self, text, metadata=None, *args, **kwargs):\n \"\"\"Add a single chunk to the vectorstore\"\"\"\n pass\n\n def delete_chunk(self, chunk_id, *args, **kwargs):\n \"\"\"Delete a specific chunk from the vectorstore\"\"\"\n pass\n\n def is_azure_configured(self):\n return (\n settings.OPENAI_API_BASE\n and settings.OPENAI_API_VERSION\n and settings.AZURE_DEPLOYMENT_NAME\n )\n\n def _get_embeddings(self, embeddings_name, embeddings_key=None):\n # Check for remote embeddings first\n if settings.EMBEDDINGS_BASE_URL:\n logging.info(\n f\"Using remote embeddings API at: {settings.EMBEDDINGS_BASE_URL}\"\n )\n cache_key = f\"remote_{settings.EMBEDDINGS_BASE_URL}_{embeddings_name}\"\n if cache_key not in EmbeddingsSingleton._instances:\n EmbeddingsSingleton._instances[cache_key] = RemoteEmbeddings(\n api_url=settings.EMBEDDINGS_BASE_URL,\n model_name=embeddings_name,\n api_key=embeddings_key,\n )\n return EmbeddingsSingleton._instances[cache_key]\n\n if embeddings_name == \"openai_text-embedding-ada-002\":\n if self.is_azure_configured():\n os.environ[\"OPENAI_API_TYPE\"] = \"azure\"\n embedding_instance = EmbeddingsSingleton.get_instance(\n embeddings_name, model=settings.AZURE_EMBEDDINGS_DEPLOYMENT_NAME\n )\n else:\n embedding_instance = EmbeddingsSingleton.get_instance(\n embeddings_name, openai_api_key=embeddings_key\n )\n elif embeddings_name == \"huggingface_sentence-transformers/all-mpnet-base-v2\":\n possible_paths = [\n \"/app/models/all-mpnet-base-v2\", # Docker absolute path\n \"./models/all-mpnet-base-v2\", # Relative path\n ]\n local_model_path = None\n for path in possible_paths:\n if os.path.exists(path):\n local_model_path = path\n logging.info(f\"Found local model at path: {path}\")\n break\n else:\n logging.info(f\"Path does not exist: {path}\")\n if local_model_path:\n embedding_instance = EmbeddingsSingleton.get_instance(\n local_model_path,\n )\n else:\n logging.warning(\n f\"Local model not found in any of the paths: {possible_paths}. Falling back to HuggingFace download.\"\n )\n embedding_instance = EmbeddingsSingleton.get_instance(\n embeddings_name,\n )\n else:\n embedding_instance = EmbeddingsSingleton.get_instance(embeddings_name)\n return embedding_instance\n" }, { "path": "application/vectorstore/document_class.py", "content": "class Document(str):\n \"\"\"Class for storing a piece of text and associated metadata.\"\"\"\n\n def __new__(cls, page_content: str, metadata: dict):\n instance = super().__new__(cls, page_content)\n instance.page_content = page_content\n instance.metadata = metadata\n return instance\n" }, { "path": "application/vectorstore/elasticsearch.py", "content": "from application.vectorstore.base import BaseVectorStore\nfrom application.core.settings import settings\nfrom application.vectorstore.document_class import Document\n\n\nclass ElasticsearchStore(BaseVectorStore):\n _es_connection = None # Class attribute to hold the Elasticsearch connection\n\n def __init__(self, source_id, embeddings_key, index_name=settings.ELASTIC_INDEX):\n super().__init__()\n self.source_id = source_id.replace(\"application/indexes/\", \"\").rstrip(\"/\")\n self.embeddings_key = embeddings_key\n self.index_name = index_name\n \n if ElasticsearchStore._es_connection is None:\n connection_params = {}\n if settings.ELASTIC_URL:\n connection_params[\"hosts\"] = [settings.ELASTIC_URL]\n connection_params[\"http_auth\"] = (settings.ELASTIC_USERNAME, settings.ELASTIC_PASSWORD)\n elif settings.ELASTIC_CLOUD_ID:\n connection_params[\"cloud_id\"] = settings.ELASTIC_CLOUD_ID\n connection_params[\"basic_auth\"] = (settings.ELASTIC_USERNAME, settings.ELASTIC_PASSWORD)\n else:\n raise ValueError(\"Please provide either elasticsearch_url or cloud_id.\")\n\n import elasticsearch\n ElasticsearchStore._es_connection = elasticsearch.Elasticsearch(**connection_params)\n \n self.docsearch = ElasticsearchStore._es_connection\n\n def connect_to_elasticsearch(\n *,\n es_url = None,\n cloud_id = None,\n api_key = None,\n username = None,\n password = None,\n ):\n try:\n import elasticsearch\n except ImportError:\n raise ImportError(\n \"Could not import elasticsearch python package. \"\n \"Please install it with `pip install elasticsearch`.\"\n )\n\n if es_url and cloud_id:\n raise ValueError(\n \"Both es_url and cloud_id are defined. Please provide only one.\"\n )\n\n connection_params = {}\n\n if es_url:\n connection_params[\"hosts\"] = [es_url]\n elif cloud_id:\n connection_params[\"cloud_id\"] = cloud_id\n else:\n raise ValueError(\"Please provide either elasticsearch_url or cloud_id.\")\n\n if api_key:\n connection_params[\"api_key\"] = api_key\n elif username and password:\n connection_params[\"basic_auth\"] = (username, password)\n\n es_client = elasticsearch.Elasticsearch(\n **connection_params,\n )\n try:\n es_client.info()\n except Exception as e:\n raise e\n\n return es_client\n\n def search(self, question, k=2, index_name=settings.ELASTIC_INDEX, *args, **kwargs):\n embeddings = self._get_embeddings(settings.EMBEDDINGS_NAME, self.embeddings_key)\n vector = embeddings.embed_query(question)\n knn = {\n \"filter\": [{\"match\": {\"metadata.source_id.keyword\": self.source_id}}],\n \"field\": \"vector\",\n \"k\": k,\n \"num_candidates\": 100,\n \"query_vector\": vector,\n }\n full_query = {\n \"knn\": knn,\n \"query\": {\n \"bool\": {\n \"must\": [\n {\n \"match\": {\n \"text\": {\n \"query\": question,\n }\n }\n }\n ],\n \"filter\": [{\"match\": {\"metadata.source_id.keyword\": self.source_id}}],\n }\n },\n \"rank\": {\"rrf\": {}},\n }\n resp = self.docsearch.search(index=self.index_name, query=full_query['query'], size=k, knn=full_query['knn'])\n # create Documents objects from the results page_content ['_source']['text'], metadata ['_source']['metadata']\n doc_list = []\n for hit in resp['hits']['hits']:\n \n doc_list.append(Document(page_content = hit['_source']['text'], metadata = hit['_source']['metadata']))\n return doc_list\n\n def _create_index_if_not_exists(\n self, index_name, dims_length\n ):\n\n if self._es_connection.indices.exists(index=index_name):\n print(f\"Index {index_name} already exists.\")\n\n else:\n\n indexSettings = self.index(\n dims_length=dims_length,\n )\n self._es_connection.indices.create(index=index_name, **indexSettings)\n\n def index(\n self,\n dims_length,\n ):\n return {\n \"mappings\": {\n \"properties\": {\n \"vector\": {\n \"type\": \"dense_vector\",\n \"dims\": dims_length,\n \"index\": True,\n \"similarity\": \"cosine\",\n },\n }\n }\n }\n\n def add_texts(\n self,\n texts,\n metadatas = None,\n ids = None,\n refresh_indices = True,\n create_index_if_not_exists = True,\n bulk_kwargs = None,\n **kwargs,\n ):\n \n bulk_kwargs = bulk_kwargs or {}\n import uuid\n embeddings = []\n ids = ids or [str(uuid.uuid4()) for _ in texts]\n requests = []\n embeddings = self._get_embeddings(settings.EMBEDDINGS_NAME, self.embeddings_key)\n\n vectors = embeddings.embed_documents(list(texts))\n\n dims_length = len(vectors[0])\n\n if create_index_if_not_exists:\n self._create_index_if_not_exists(\n index_name=self.index_name, dims_length=dims_length\n )\n\n for i, (text, vector) in enumerate(zip(texts, vectors)):\n metadata = metadatas[i] if metadatas else {}\n\n requests.append(\n {\n \"_op_type\": \"index\",\n \"_index\": self.index_name,\n \"text\": text,\n \"vector\": vector,\n \"metadata\": metadata,\n \"_id\": ids[i],\n }\n )\n\n\n if len(requests) > 0:\n from elasticsearch.helpers import BulkIndexError, bulk\n try:\n success, failed = bulk(\n self._es_connection,\n requests,\n stats_only=True,\n refresh=refresh_indices,\n **bulk_kwargs,\n )\n return ids\n except BulkIndexError as e:\n print(f\"Error adding texts: {e}\")\n firstError = e.errors[0].get(\"index\", {}).get(\"error\", {})\n print(f\"First error reason: {firstError.get('reason')}\")\n raise e\n\n else:\n return []\n\n def delete_index(self):\n self._es_connection.delete_by_query(index=self.index_name, query={\"match\": {\n \"metadata.source_id.keyword\": self.source_id}},)\n" }, { "path": "application/vectorstore/embeddings_local.py", "content": "\"\"\"\nLocal embeddings using SentenceTransformer.\nThis module is only imported when EMBEDDINGS_BASE_URL is not set,\nto avoid loading SentenceTransformer into memory when using remote embeddings.\n\"\"\"\n\nimport logging\n\nfrom sentence_transformers import SentenceTransformer\n\n\nclass EmbeddingsWrapper:\n def __init__(self, model_name, *args, **kwargs):\n logging.info(f\"Initializing EmbeddingsWrapper with model: {model_name}\")\n try:\n kwargs.setdefault(\"trust_remote_code\", True)\n self.model = SentenceTransformer(\n model_name,\n config_kwargs={\"allow_dangerous_deserialization\": True},\n *args,\n **kwargs,\n )\n if self.model is None or self.model._first_module() is None:\n raise ValueError(\n f\"SentenceTransformer model failed to load properly for: {model_name}\"\n )\n self.dimension = self.model.get_sentence_embedding_dimension()\n logging.info(f\"Successfully loaded model with dimension: {self.dimension}\")\n except Exception as e:\n logging.error(\n f\"Failed to initialize SentenceTransformer with model {model_name}: {str(e)}\",\n exc_info=True,\n )\n raise\n\n def embed_query(self, query: str):\n return self.model.encode(query).tolist()\n\n def embed_documents(self, documents: list):\n return self.model.encode(documents).tolist()\n\n def __call__(self, text):\n if isinstance(text, str):\n return self.embed_query(text)\n elif isinstance(text, list):\n return self.embed_documents(text)\n else:\n raise ValueError(\"Input must be a string or a list of strings\")\n" }, { "path": "application/vectorstore/faiss.py", "content": "import os\nimport tempfile\nimport io\n\nfrom langchain_community.vectorstores import FAISS\n\nfrom application.core.settings import settings\nfrom application.parser.schema.base import Document\nfrom application.vectorstore.base import BaseVectorStore\nfrom application.storage.storage_creator import StorageCreator\n\n\ndef get_vectorstore(path: str) -> str:\n if path:\n vectorstore = f\"indexes/{path}\"\n else:\n vectorstore = \"indexes\"\n return vectorstore\n\n\nclass FaissStore(BaseVectorStore):\n def __init__(self, source_id: str, embeddings_key: str, docs_init=None):\n super().__init__()\n self.source_id = source_id\n self.path = get_vectorstore(source_id)\n self.embeddings = self._get_embeddings(settings.EMBEDDINGS_NAME, embeddings_key)\n self.storage = StorageCreator.get_storage()\n\n try:\n if docs_init:\n self.docsearch = FAISS.from_documents(docs_init, self.embeddings)\n else:\n with tempfile.TemporaryDirectory() as temp_dir:\n faiss_path = f\"{self.path}/index.faiss\"\n pkl_path = f\"{self.path}/index.pkl\"\n\n if not self.storage.file_exists(\n faiss_path\n ) or not self.storage.file_exists(pkl_path):\n raise FileNotFoundError(\n f\"Index files not found in storage at {self.path}\"\n )\n\n faiss_file = self.storage.get_file(faiss_path)\n pkl_file = self.storage.get_file(pkl_path)\n\n local_faiss_path = os.path.join(temp_dir, \"index.faiss\")\n local_pkl_path = os.path.join(temp_dir, \"index.pkl\")\n\n with open(local_faiss_path, \"wb\") as f:\n f.write(faiss_file.read())\n\n with open(local_pkl_path, \"wb\") as f:\n f.write(pkl_file.read())\n\n self.docsearch = FAISS.load_local(\n temp_dir, self.embeddings, allow_dangerous_deserialization=True\n )\n except Exception as e:\n raise Exception(f\"Error loading FAISS index: {str(e)}\")\n\n self.assert_embedding_dimensions(self.embeddings)\n\n def search(self, *args, **kwargs):\n return self.docsearch.similarity_search(*args, **kwargs)\n\n def add_texts(self, *args, **kwargs):\n return self.docsearch.add_texts(*args, **kwargs)\n\n def _save_to_storage(self):\n \"\"\"\n Save the FAISS index to storage using temporary directory pattern.\n Works consistently for both local and S3 storage.\n \"\"\"\n with tempfile.TemporaryDirectory() as temp_dir:\n self.docsearch.save_local(temp_dir)\n\n faiss_path = os.path.join(temp_dir, \"index.faiss\")\n pkl_path = os.path.join(temp_dir, \"index.pkl\")\n\n with open(faiss_path, \"rb\") as f_faiss:\n faiss_data = f_faiss.read()\n\n with open(pkl_path, \"rb\") as f_pkl:\n pkl_data = f_pkl.read()\n\n storage_path = get_vectorstore(self.source_id)\n self.storage.save_file(io.BytesIO(faiss_data), f\"{storage_path}/index.faiss\")\n self.storage.save_file(io.BytesIO(pkl_data), f\"{storage_path}/index.pkl\")\n\n return True\n\n def save_local(self, path=None):\n if path:\n os.makedirs(path, exist_ok=True)\n self.docsearch.save_local(path)\n\n self._save_to_storage()\n\n return True\n\n def delete_index(self, *args, **kwargs):\n return self.docsearch.delete(*args, **kwargs)\n\n def assert_embedding_dimensions(self, embeddings):\n \"\"\"Check that the word embedding dimension of the docsearch index matches the dimension of the word embeddings used.\"\"\"\n if (\n settings.EMBEDDINGS_NAME\n == \"huggingface_sentence-transformers/all-mpnet-base-v2\"\n ):\n word_embedding_dimension = getattr(embeddings, \"dimension\", None)\n if word_embedding_dimension is None:\n raise AttributeError(\n \"'dimension' attribute not found in embeddings instance.\"\n )\n\n docsearch_index_dimension = self.docsearch.index.d\n if word_embedding_dimension != docsearch_index_dimension:\n raise ValueError(\n f\"Embedding dimension mismatch: embeddings.dimension ({word_embedding_dimension}) != docsearch index dimension ({docsearch_index_dimension})\"\n )\n\n def get_chunks(self):\n chunks = []\n if self.docsearch:\n for doc_id, doc in self.docsearch.docstore._dict.items():\n chunk_data = {\n \"doc_id\": doc_id,\n \"text\": doc.page_content,\n \"metadata\": doc.metadata,\n }\n chunks.append(chunk_data)\n return chunks\n\n def add_chunk(self, text, metadata=None):\n \"\"\"Add a new chunk and save to storage.\"\"\"\n metadata = metadata or {}\n doc = Document(text=text, extra_info=metadata).to_langchain_format()\n doc_id = self.docsearch.add_documents([doc])\n self._save_to_storage()\n return doc_id\n\n\n\n def delete_chunk(self, chunk_id):\n \"\"\"Delete a chunk and save to storage.\"\"\"\n self.delete_index([chunk_id])\n self._save_to_storage()\n return True\n" }, { "path": "application/vectorstore/lancedb.py", "content": "from typing import List, Optional\nimport importlib\nfrom application.vectorstore.base import BaseVectorStore\nfrom application.core.settings import settings\n\nclass LanceDBVectorStore(BaseVectorStore):\n \"\"\"Class for LanceDB Vector Store integration.\"\"\"\n\n def __init__(self, path: str = settings.LANCEDB_PATH,\n table_name_prefix: str = settings.LANCEDB_TABLE_NAME,\n source_id: str = None,\n embeddings_key: str = \"embeddings\"):\n \"\"\"Initialize the LanceDB vector store.\"\"\"\n super().__init__()\n self.path = path\n self.table_name = f\"{table_name_prefix}_{source_id}\" if source_id else table_name_prefix\n self.embeddings_key = embeddings_key\n self._lance_db = None\n self.docsearch = None\n self._pa = None # PyArrow (pa) will be lazy loaded\n\n @property\n def pa(self):\n \"\"\"Lazy load pyarrow module.\"\"\"\n if self._pa is None:\n self._pa = importlib.import_module(\"pyarrow\")\n return self._pa\n\n @property\n def lancedb(self):\n \"\"\"Lazy load lancedb module.\"\"\"\n if not hasattr(self, \"_lancedb_module\"):\n self._lancedb_module = importlib.import_module(\"lancedb\")\n return self._lancedb_module\n\n @property\n def lance_db(self):\n \"\"\"Lazy load the LanceDB connection.\"\"\"\n if self._lance_db is None:\n self._lance_db = self.lancedb.connect(self.path)\n return self._lance_db\n\n @property\n def table(self):\n \"\"\"Lazy load the LanceDB table.\"\"\"\n if self.docsearch is None:\n if self.table_name in self.lance_db.table_names():\n self.docsearch = self.lance_db.open_table(self.table_name)\n else:\n self.docsearch = None\n return self.docsearch\n\n def ensure_table_exists(self):\n \"\"\"Ensure the table exists before performing operations.\"\"\"\n if self.table is None:\n embeddings = self._get_embeddings(settings.EMBEDDINGS_NAME, self.embeddings_key)\n schema = self.pa.schema([\n self.pa.field(\"vector\", self.pa.list_(self.pa.float32(), list_size=embeddings.dimension)),\n self.pa.field(\"text\", self.pa.string()),\n self.pa.field(\"metadata\", self.pa.struct([\n self.pa.field(\"key\", self.pa.string()),\n self.pa.field(\"value\", self.pa.string())\n ]))\n ])\n self.docsearch = self.lance_db.create_table(self.table_name, schema=schema)\n\n def add_texts(self, texts: List[str], metadatas: Optional[List[dict]] = None, source_id: str = None):\n \"\"\"Add texts with metadata and their embeddings to the LanceDB table.\"\"\"\n embeddings = self._get_embeddings(settings.EMBEDDINGS_NAME, self.embeddings_key).embed_documents(texts)\n vectors = []\n for embedding, text, metadata in zip(embeddings, texts, metadatas or [{}] * len(texts)):\n if source_id:\n metadata[\"source_id\"] = source_id\n metadata_struct = [{\"key\": k, \"value\": str(v)} for k, v in metadata.items()]\n vectors.append({\n \"vector\": embedding,\n \"text\": text,\n \"metadata\": metadata_struct\n })\n self.ensure_table_exists()\n self.docsearch.add(vectors)\n\n def search(self, query: str, k: int = 2, *args, **kwargs):\n \"\"\"Search LanceDB for the top k most similar vectors.\"\"\"\n self.ensure_table_exists()\n query_embedding = self._get_embeddings(settings.EMBEDDINGS_NAME, self.embeddings_key).embed_query(query)\n results = self.docsearch.search(query_embedding).limit(k).to_list()\n return [(result[\"_distance\"], result[\"text\"], result[\"metadata\"]) for result in results]\n\n def delete_index(self):\n \"\"\"Delete the entire LanceDB index (table).\"\"\"\n if self.table:\n self.lance_db.drop_table(self.table_name)\n\n def assert_embedding_dimensions(self, embeddings):\n \"\"\"Ensure that embedding dimensions match the table index dimensions.\"\"\"\n word_embedding_dimension = embeddings.dimension\n if self.table:\n table_index_dimension = len(self.docsearch.schema[\"vector\"].type.value_type)\n if word_embedding_dimension != table_index_dimension:\n raise ValueError(\n f\"Embedding dimension mismatch: embeddings.dimension ({word_embedding_dimension}) \"\n f\"!= table index dimension ({table_index_dimension})\"\n )\n\n def filter_documents(self, filter_condition: dict) -> List[dict]:\n \"\"\"Filter documents based on certain conditions.\"\"\"\n self.ensure_table_exists()\n\n # Ensure source_id exists in the filter condition\n if 'source_id' not in filter_condition:\n raise ValueError(\"filter_condition must contain 'source_id'\")\n\n source_id = filter_condition[\"source_id\"]\n\n # Use LanceDB's native filtering if supported, otherwise filter manually\n filtered_data = self.docsearch.filter(lambda x: x.metadata and x.metadata.get(\"source_id\") == source_id).to_list()\n\n return filtered_data" }, { "path": "application/vectorstore/milvus.py", "content": "from typing import List, Optional\nfrom uuid import uuid4\n\n\nfrom application.core.settings import settings\nfrom application.vectorstore.base import BaseVectorStore\n\n\nclass MilvusStore(BaseVectorStore):\n def __init__(self, source_id: str = \"\", embeddings_key: str = \"embeddings\"):\n super().__init__()\n from langchain_milvus import Milvus\n\n connection_args = {\n \"uri\": settings.MILVUS_URI,\n \"token\": settings.MILVUS_TOKEN,\n }\n self._docsearch = Milvus(\n embedding_function=self._get_embeddings(settings.EMBEDDINGS_NAME, embeddings_key),\n collection_name=settings.MILVUS_COLLECTION_NAME,\n connection_args=connection_args,\n )\n self._source_id = source_id\n\n def search(self, question, k=2, *args, **kwargs):\n expr = f\"source_id == '{self._source_id}'\"\n return self._docsearch.similarity_search(query=question, k=k, expr=expr, *args, **kwargs)\n\n def add_texts(self, texts: List[str], metadatas: Optional[List[dict]], *args, **kwargs):\n ids = [str(uuid4()) for _ in range(len(texts))]\n\n return self._docsearch.add_texts(texts=texts, metadatas=metadatas, ids=ids, *args, **kwargs)\n\n def save_local(self, *args, **kwargs):\n pass\n\n def delete_index(self, *args, **kwargs):\n pass\n" }, { "path": "application/vectorstore/mongodb.py", "content": "import logging\nfrom application.core.settings import settings\nfrom application.vectorstore.base import BaseVectorStore\nfrom application.vectorstore.document_class import Document\n\n\nclass MongoDBVectorStore(BaseVectorStore):\n def __init__(\n self,\n source_id: str = \"\",\n embeddings_key: str = \"embeddings\",\n collection: str = \"documents\",\n index_name: str = \"vector_search_index\",\n text_key: str = \"text\",\n embedding_key: str = \"embedding\",\n database: str = \"docsgpt\",\n ):\n self._index_name = index_name\n self._text_key = text_key\n self._embedding_key = embedding_key\n self._embeddings_key = embeddings_key\n self._mongo_uri = settings.MONGO_URI\n self._source_id = source_id.replace(\"application/indexes/\", \"\").rstrip(\"/\")\n self._embedding = self._get_embeddings(settings.EMBEDDINGS_NAME, embeddings_key)\n\n try:\n import pymongo\n except ImportError:\n raise ImportError(\n \"Could not import pymongo python package. \"\n \"Please install it with `pip install pymongo`.\"\n )\n\n self._client = pymongo.MongoClient(self._mongo_uri)\n self._database = self._client[database]\n self._collection = self._database[collection]\n\n def search(self, question, k=2, *args, **kwargs):\n query_vector = self._embedding.embed_query(question)\n\n pipeline = [\n {\n \"$vectorSearch\": {\n \"queryVector\": query_vector,\n \"path\": self._embedding_key,\n \"limit\": k,\n \"numCandidates\": k * 10,\n \"index\": self._index_name,\n \"filter\": {\"source_id\": {\"$eq\": self._source_id}},\n }\n }\n ]\n\n cursor = self._collection.aggregate(pipeline)\n\n results = []\n for doc in cursor:\n text = doc[self._text_key]\n doc.pop(\"_id\")\n doc.pop(self._text_key)\n doc.pop(self._embedding_key)\n metadata = doc\n results.append(Document(text, metadata))\n return results\n\n def _insert_texts(self, texts, metadatas):\n if not texts:\n return []\n embeddings = self._embedding.embed_documents(texts)\n\n to_insert = [\n {self._text_key: t, self._embedding_key: embedding, **m}\n for t, m, embedding in zip(texts, metadatas, embeddings)\n ]\n\n insert_result = self._collection.insert_many(to_insert)\n return insert_result.inserted_ids\n\n def add_texts(\n self,\n texts,\n metadatas=None,\n ids=None,\n refresh_indices=True,\n create_index_if_not_exists=True,\n bulk_kwargs=None,\n **kwargs,\n ):\n\n # dims = self._embedding.client[1].word_embedding_dimension\n # # check if index exists\n # if create_index_if_not_exists:\n # # check if index exists\n # info = self._collection.index_information()\n # if self._index_name not in info:\n # index_mongo = {\n # \"fields\": [{\n # \"type\": \"vector\",\n # \"path\": self._embedding_key,\n # \"numDimensions\": dims,\n # \"similarity\": \"cosine\",\n # },\n # {\n # \"type\": \"filter\",\n # \"path\": \"store\"\n # }]\n # }\n # self._collection.create_index(self._index_name, index_mongo)\n\n batch_size = 100\n _metadatas = metadatas or ({} for _ in texts)\n texts_batch = []\n metadatas_batch = []\n result_ids = []\n for i, (text, metadata) in enumerate(zip(texts, _metadatas)):\n texts_batch.append(text)\n metadatas_batch.append(metadata)\n if (i + 1) % batch_size == 0:\n result_ids.extend(self._insert_texts(texts_batch, metadatas_batch))\n texts_batch = []\n metadatas_batch = []\n if texts_batch:\n result_ids.extend(self._insert_texts(texts_batch, metadatas_batch))\n return result_ids\n\n def delete_index(self, *args, **kwargs):\n self._collection.delete_many({\"source_id\": self._source_id})\n\n def get_chunks(self):\n try:\n chunks = []\n cursor = self._collection.find({\"source_id\": self._source_id})\n for doc in cursor:\n doc_id = str(doc.get(\"_id\"))\n text = doc.get(self._text_key)\n metadata = {\n k: v\n for k, v in doc.items()\n if k\n not in [\"_id\", self._text_key, self._embedding_key, \"source_id\"]\n }\n\n if text:\n chunks.append(\n {\"doc_id\": doc_id, \"text\": text, \"metadata\": metadata}\n )\n\n return chunks\n except Exception as e:\n logging.error(f\"Error getting chunks: {e}\", exc_info=True)\n return []\n\n def add_chunk(self, text, metadata=None):\n metadata = metadata or {}\n embeddings = self._embedding.embed_documents([text])\n if not embeddings:\n raise ValueError(\"Could not generate embedding for chunk\")\n\n chunk_data = {\n self._text_key: text,\n self._embedding_key: embeddings[0],\n \"source_id\": self._source_id,\n **metadata,\n }\n result = self._collection.insert_one(chunk_data)\n return str(result.inserted_id)\n\n def delete_chunk(self, chunk_id):\n try:\n from bson.objectid import ObjectId\n\n object_id = ObjectId(chunk_id)\n result = self._collection.delete_one({\"_id\": object_id})\n return result.deleted_count > 0\n except Exception as e:\n logging.error(f\"Error deleting chunk: {e}\", exc_info=True)\n return False\n" }, { "path": "application/vectorstore/pgvector.py", "content": "import logging\nfrom typing import List, Optional, Any, Dict\nfrom application.core.settings import settings\nfrom application.vectorstore.base import BaseVectorStore\nfrom application.vectorstore.document_class import Document\n\n\nclass PGVectorStore(BaseVectorStore):\n def __init__(\n self,\n source_id: str = \"\",\n embeddings_key: str = \"embeddings\",\n table_name: str = \"documents\",\n decoded_token: Optional[str] = None,\n vector_column: str = \"embedding\",\n text_column: str = \"text\",\n metadata_column: str = \"metadata\",\n connection_string: str = None,\n ):\n super().__init__()\n # Store the source_id for use in add_chunk\n self._source_id = str(source_id).replace(\"application/indexes/\", \"\").rstrip(\"/\")\n self._embeddings_key = embeddings_key\n self._table_name = table_name\n self._vector_column = vector_column\n self._text_column = text_column\n self._metadata_column = metadata_column\n self._embedding = self._get_embeddings(settings.EMBEDDINGS_NAME, embeddings_key)\n \n # Use provided connection string or fall back to settings\n self._connection_string = connection_string or getattr(settings, 'PGVECTOR_CONNECTION_STRING', None)\n \n if not self._connection_string:\n raise ValueError(\n \"PostgreSQL connection string is required. \"\n \"Set PGVECTOR_CONNECTION_STRING in settings or pass connection_string parameter.\"\n )\n\n try:\n import psycopg2\n from psycopg2.extras import Json\n import pgvector.psycopg2\n except ImportError:\n raise ImportError(\n \"Could not import required packages. \"\n \"Please install with `pip install psycopg2-binary pgvector`.\"\n )\n\n self._psycopg2 = psycopg2\n self._Json = Json\n self._pgvector = pgvector.psycopg2\n self._connection = None\n self._ensure_table_exists()\n\n def _get_connection(self):\n \"\"\"Get or create database connection\"\"\"\n if self._connection is None or self._connection.closed:\n self._connection = self._psycopg2.connect(self._connection_string)\n # Register pgvector types\n self._pgvector.register_vector(self._connection)\n return self._connection\n\n def _ensure_table_exists(self):\n \"\"\"Create table and enable pgvector extension if they don't exist\"\"\"\n conn = self._get_connection()\n cursor = conn.cursor()\n \n try:\n # Enable pgvector extension\n cursor.execute(\"CREATE EXTENSION IF NOT EXISTS vector;\")\n \n embedding_dim = getattr(self._embedding, 'dimension', 768)\n \n # Create table with vector column\n create_table_query = f\"\"\"\n CREATE TABLE IF NOT EXISTS {self._table_name} (\n id SERIAL PRIMARY KEY,\n {self._text_column} TEXT NOT NULL,\n {self._vector_column} vector({embedding_dim}),\n {self._metadata_column} JSONB,\n source_id TEXT NOT NULL,\n created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP\n );\n \"\"\"\n cursor.execute(create_table_query)\n \n # Create index for vector similarity search\n index_query = f\"\"\"\n CREATE INDEX IF NOT EXISTS {self._table_name}_{self._vector_column}_idx \n ON {self._table_name} USING ivfflat ({self._vector_column} vector_cosine_ops)\n WITH (lists = 100);\n \"\"\"\n cursor.execute(index_query)\n \n # Create index for source_id filtering\n source_index_query = f\"\"\"\n CREATE INDEX IF NOT EXISTS {self._table_name}_source_id_idx \n ON {self._table_name} (source_id);\n \"\"\"\n cursor.execute(source_index_query)\n \n conn.commit()\n except Exception as e:\n conn.rollback()\n logging.error(f\"Error creating table: {e}\")\n raise\n finally:\n cursor.close()\n\n def search(self, question: str, k: int = 2, *args, **kwargs) -> List[Document]:\n \"\"\"Search for similar documents using vector similarity\"\"\"\n query_vector = self._embedding.embed_query(question)\n \n conn = self._get_connection()\n cursor = conn.cursor()\n \n try:\n # Use cosine distance for similarity search with proper vector formatting\n search_query = f\"\"\"\n SELECT {self._text_column}, {self._metadata_column}, \n ({self._vector_column} <=> %s::vector) as distance\n FROM {self._table_name}\n WHERE source_id = %s\n ORDER BY {self._vector_column} <=> %s::vector\n LIMIT %s;\n \"\"\"\n \n cursor.execute(search_query, (query_vector, self._source_id, query_vector, k))\n results = cursor.fetchall()\n \n \n documents = []\n for text, metadata, distance in results:\n metadata = metadata or {}\n documents.append(Document(page_content=text, metadata=metadata))\n \n return documents\n \n except Exception as e:\n logging.error(f\"Error searching documents: {e}\", exc_info=True)\n return []\n finally:\n cursor.close()\n\n def add_texts(\n self,\n texts: List[str],\n metadatas: Optional[List[Dict[str, Any]]] = None,\n *args,\n **kwargs,\n ) -> List[str]:\n \"\"\"Add texts with their embeddings to the vector store\"\"\"\n if not texts:\n return []\n\n embeddings = self._embedding.embed_documents(texts)\n metadatas = metadatas or [{}] * len(texts)\n \n conn = self._get_connection()\n cursor = conn.cursor()\n \n try:\n insert_query = f\"\"\"\n INSERT INTO {self._table_name} ({self._text_column}, {self._vector_column}, {self._metadata_column}, source_id)\n VALUES (%s, %s, %s, %s)\n RETURNING id;\n \"\"\"\n \n inserted_ids = []\n for text, embedding, metadata in zip(texts, embeddings, metadatas):\n cursor.execute(\n insert_query,\n (text, embedding, self._Json(metadata), self._source_id)\n )\n inserted_id = cursor.fetchone()[0]\n inserted_ids.append(str(inserted_id))\n \n conn.commit()\n return inserted_ids\n \n except Exception as e:\n conn.rollback()\n logging.error(f\"Error adding texts: {e}\")\n raise\n finally:\n cursor.close()\n\n def delete_index(self, *args, **kwargs):\n \"\"\"Delete all documents for this source_id\"\"\"\n conn = self._get_connection()\n cursor = conn.cursor()\n \n try:\n delete_query = f\"DELETE FROM {self._table_name} WHERE source_id = %s;\"\n cursor.execute(delete_query, (self._source_id,))\n conn.commit()\n \n except Exception as e:\n conn.rollback()\n logging.error(f\"Error deleting index: {e}\")\n raise\n finally:\n cursor.close()\n\n def save_local(self, *args, **kwargs):\n \"\"\"No-op for PostgreSQL - data is already persisted\"\"\"\n pass\n\n def get_chunks(self) -> List[Dict[str, Any]]:\n \"\"\"Get all chunks for this source_id\"\"\"\n conn = self._get_connection()\n cursor = conn.cursor()\n \n try:\n select_query = f\"\"\"\n SELECT id, {self._text_column}, {self._metadata_column}\n FROM {self._table_name}\n WHERE source_id = %s;\n \"\"\"\n cursor.execute(select_query, (self._source_id,))\n results = cursor.fetchall()\n \n chunks = []\n for doc_id, text, metadata in results:\n chunks.append({\n \"doc_id\": str(doc_id),\n \"text\": text,\n \"metadata\": metadata or {}\n })\n \n return chunks\n \n except Exception as e:\n logging.error(f\"Error getting chunks: {e}\")\n return []\n finally:\n cursor.close()\n\n def add_chunk(self, text: str, metadata: Optional[Dict[str, Any]] = None) -> str:\n \"\"\"Add a single chunk to the vector store\"\"\"\n metadata = metadata or {}\n\n final_metadata = metadata.copy()\n\n final_metadata[\"source_id\"] = self._source_id\n\n embeddings = self._embedding.embed_documents([text])\n\n if not embeddings:\n raise ValueError(\"Could not generate embedding for chunk\")\n \n conn = self._get_connection()\n cursor = conn.cursor()\n \n try:\n insert_query = f\"\"\"\n INSERT INTO {self._table_name} ({self._text_column}, {self._vector_column}, {self._metadata_column}, source_id)\n VALUES (%s, %s, %s, %s)\n RETURNING id;\n \"\"\"\n \n cursor.execute(\n insert_query,\n (text, embeddings[0], self._Json(final_metadata), self._source_id)\n )\n inserted_id = cursor.fetchone()[0]\n conn.commit()\n \n return str(inserted_id)\n \n except Exception as e:\n conn.rollback()\n logging.error(f\"Error adding chunk: {e}\")\n raise\n finally:\n cursor.close()\n\n def delete_chunk(self, chunk_id: str) -> bool:\n \"\"\"Delete a specific chunk by its ID\"\"\"\n conn = self._get_connection()\n cursor = conn.cursor()\n \n try:\n delete_query = f\"DELETE FROM {self._table_name} WHERE id = %s AND source_id = %s;\"\n cursor.execute(delete_query, (int(chunk_id), self._source_id))\n deleted_count = cursor.rowcount\n conn.commit()\n \n return deleted_count > 0\n \n except Exception as e:\n conn.rollback()\n logging.error(f\"Error deleting chunk: {e}\")\n return False\n finally:\n cursor.close()\n\n def __del__(self):\n \"\"\"Close database connection when object is destroyed\"\"\"\n if hasattr(self, '_connection') and self._connection and not self._connection.closed:\n self._connection.close()" }, { "path": "application/vectorstore/qdrant.py", "content": "import logging\nfrom application.vectorstore.base import BaseVectorStore\nfrom application.core.settings import settings\nfrom application.vectorstore.document_class import Document\n\n\nclass QdrantStore(BaseVectorStore):\n def __init__(self, source_id: str = \"\", embeddings_key: str = \"embeddings\"):\n from qdrant_client import models\n from langchain_community.vectorstores.qdrant import Qdrant\n\n # Store the source_id for use in add_chunk\n self._source_id = str(source_id).replace(\"application/indexes/\", \"\").rstrip(\"/\")\n \n self._filter = models.Filter(\n must=[\n models.FieldCondition(\n key=\"metadata.source_id\",\n match=models.MatchValue(value=self._source_id),\n )\n ]\n )\n\n embedding=self._get_embeddings(settings.EMBEDDINGS_NAME, embeddings_key)\n self._docsearch = Qdrant.construct_instance(\n [\"TEXT_TO_OBTAIN_EMBEDDINGS_DIMENSION\"],\n embedding=embedding,\n collection_name=settings.QDRANT_COLLECTION_NAME,\n location=settings.QDRANT_LOCATION,\n url=settings.QDRANT_URL,\n port=settings.QDRANT_PORT,\n grpc_port=settings.QDRANT_GRPC_PORT,\n https=settings.QDRANT_HTTPS,\n prefer_grpc=settings.QDRANT_PREFER_GRPC,\n api_key=settings.QDRANT_API_KEY,\n prefix=settings.QDRANT_PREFIX,\n timeout=settings.QDRANT_TIMEOUT,\n path=settings.QDRANT_PATH,\n distance_func=settings.QDRANT_DISTANCE_FUNC,\n )\n try:\n collections = self._docsearch.client.get_collections()\n collection_exists = settings.QDRANT_COLLECTION_NAME in [\n collection.name for collection in collections.collections\n ]\n \n if not collection_exists:\n self._docsearch.client.recreate_collection(\n collection_name=settings.QDRANT_COLLECTION_NAME,\n vectors_config=models.VectorParams(size=embedding.client[1].word_embedding_dimension, distance=models.Distance.COSINE),\n )\n \n # Ensure the required index exists for metadata.source_id\n try:\n self._docsearch.client.create_payload_index(\n collection_name=settings.QDRANT_COLLECTION_NAME,\n field_name=\"metadata.source_id\",\n field_schema=models.PayloadSchemaType.KEYWORD,\n )\n except Exception as index_error:\n # Index might already exist, which is fine\n if \"already exists\" not in str(index_error).lower():\n logging.warning(f\"Could not create index for metadata.source_id: {index_error}\")\n \n except Exception as e:\n logging.warning(f\"Could not check for collection: {e}\")\n\n def search(self, *args, **kwargs):\n return self._docsearch.similarity_search(filter=self._filter, *args, **kwargs)\n\n def add_texts(self, *args, **kwargs):\n return self._docsearch.add_texts(*args, **kwargs)\n\n def save_local(self, *args, **kwargs):\n pass\n\n def delete_index(self, *args, **kwargs):\n return self._docsearch.client.delete(\n collection_name=settings.QDRANT_COLLECTION_NAME, points_selector=self._filter\n )\n\n def get_chunks(self):\n try:\n\n chunks = []\n offset = None\n while True:\n records, offset = self._docsearch.client.scroll(\n collection_name=settings.QDRANT_COLLECTION_NAME,\n scroll_filter=self._filter,\n limit=10,\n with_payload=True,\n with_vectors=False,\n offset=offset,\n )\n for record in records:\n doc_id = record.id\n text = record.payload.get(\"page_content\")\n metadata = record.payload.get(\"metadata\")\n chunks.append(\n {\"doc_id\": doc_id, \"text\": text, \"metadata\": metadata}\n )\n if offset is None:\n break\n return chunks\n except Exception as e:\n logging.error(f\"Error getting chunks: {e}\", exc_info=True)\n return []\n\n def add_chunk(self, text, metadata=None):\n import uuid\n metadata = metadata or {}\n \n # Create a copy to avoid modifying the original metadata\n final_metadata = metadata.copy()\n \n # Ensure the source_id is in the metadata so the chunk can be found by filters\n final_metadata[\"source_id\"] = self._source_id\n \n doc = Document(page_content=text, metadata=final_metadata)\n # Generate a unique ID for the document\n doc_id = str(uuid.uuid4())\n doc.id = doc_id\n doc_ids = self._docsearch.add_documents([doc])\n return doc_ids[0] if doc_ids else doc_id\n\n def delete_chunk(self, chunk_id):\n try:\n self._docsearch.client.delete(\n collection_name=settings.QDRANT_COLLECTION_NAME,\n points_selector=[chunk_id],\n )\n return True\n except Exception as e:\n logging.error(f\"Error deleting chunk: {e}\", exc_info=True)\n return False\n" }, { "path": "application/vectorstore/vector_creator.py", "content": "from application.vectorstore.faiss import FaissStore\nfrom application.vectorstore.elasticsearch import ElasticsearchStore\nfrom application.vectorstore.milvus import MilvusStore\nfrom application.vectorstore.mongodb import MongoDBVectorStore\nfrom application.vectorstore.qdrant import QdrantStore\nfrom application.vectorstore.pgvector import PGVectorStore\n\n\nclass VectorCreator:\n vectorstores = {\n \"faiss\": FaissStore,\n \"elasticsearch\": ElasticsearchStore,\n \"mongodb\": MongoDBVectorStore,\n \"qdrant\": QdrantStore,\n \"milvus\": MilvusStore,\n \"pgvector\": PGVectorStore\n }\n\n @classmethod\n def create_vectorstore(cls, type, *args, **kwargs):\n vectorstore_class = cls.vectorstores.get(type.lower())\n if not vectorstore_class:\n raise ValueError(f\"No vectorstore class found for type {type}\")\n return vectorstore_class(*args, **kwargs)\n" }, { "path": "application/worker.py", "content": "import datetime\nimport json\nimport logging\nimport mimetypes\nimport os\nimport shutil\nimport string\nimport tempfile\nfrom typing import Any, Dict\nimport zipfile\n\nfrom collections import Counter\nfrom urllib.parse import urljoin\n\nimport requests\nfrom bson.dbref import DBRef\nfrom bson.objectid import ObjectId\n\nfrom application.agents.agent_creator import AgentCreator\nfrom application.api.answer.services.stream_processor import get_prompt\n\nfrom application.cache import get_redis_instance\nfrom application.core.mongo_db import MongoDB\nfrom application.core.settings import settings\nfrom application.parser.chunking import Chunker\nfrom application.parser.connectors.connector_creator import ConnectorCreator\nfrom application.parser.embedding_pipeline import embed_and_store_documents\nfrom application.parser.file.bulk import SimpleDirectoryReader, get_default_file_extractor\nfrom application.parser.file.constants import SUPPORTED_SOURCE_EXTENSIONS\nfrom application.parser.remote.remote_creator import RemoteCreator\nfrom application.parser.schema.base import Document\nfrom application.retriever.retriever_creator import RetrieverCreator\n\nfrom application.storage.storage_creator import StorageCreator\nfrom application.utils import count_tokens_docs, num_tokens_from_string\n\nmongo = MongoDB.get_client()\ndb = mongo[settings.MONGO_DB_NAME]\nsources_collection = db[\"sources\"]\n\n# Constants\n\n\nMIN_TOKENS = 150\nMAX_TOKENS = 1250\nRECURSION_DEPTH = 2\n\n\n# Define a function to extract metadata from a given filename.\n\n\ndef metadata_from_filename(title):\n return {\"title\": title}\n\n\ndef _normalize_file_name_map(file_name_map):\n if not file_name_map:\n return {}\n if isinstance(file_name_map, str):\n try:\n file_name_map = json.loads(file_name_map)\n except Exception:\n return {}\n return file_name_map if isinstance(file_name_map, dict) else {}\n\n\ndef _get_display_name(file_name_map, rel_path):\n if not file_name_map or not rel_path:\n return None\n if rel_path in file_name_map:\n return file_name_map[rel_path]\n base_name = os.path.basename(rel_path)\n return file_name_map.get(base_name)\n\n\ndef _apply_display_names_to_structure(structure, file_name_map, prefix=\"\"):\n if not isinstance(structure, dict) or not file_name_map:\n return structure\n for name, node in structure.items():\n if isinstance(node, dict) and \"type\" in node and \"size_bytes\" in node:\n rel_path = f\"{prefix}/{name}\" if prefix else name\n display_name = _get_display_name(file_name_map, rel_path)\n if display_name:\n node[\"display_name\"] = display_name\n elif isinstance(node, dict):\n next_prefix = f\"{prefix}/{name}\" if prefix else name\n _apply_display_names_to_structure(node, file_name_map, next_prefix)\n return structure\n\n\n# Define a function to generate a random string of a given length.\n\n\ndef generate_random_string(length):\n return \"\".join([string.ascii_letters[i % 52] for i in range(length)])\n\n\ncurrent_dir = os.path.dirname(\n os.path.dirname(os.path.dirname(os.path.abspath(__file__)))\n)\n\n# Zip extraction security limits\nMAX_UNCOMPRESSED_SIZE = 500 * 1024 * 1024 # 500 MB max uncompressed size\nMAX_FILE_COUNT = 10000 # Maximum number of files to extract\nMAX_COMPRESSION_RATIO = 100 # Maximum compression ratio (to detect zip bombs)\n\n\nclass ZipExtractionError(Exception):\n \"\"\"Raised when zip extraction fails due to security constraints.\"\"\"\n pass\n\n\ndef _is_path_safe(base_path: str, target_path: str) -> bool:\n \"\"\"\n Check if target_path is safely within base_path (prevents zip slip attacks).\n\n Args:\n base_path: The base directory where extraction should occur.\n target_path: The full path where a file would be extracted.\n\n Returns:\n True if the path is safe, False otherwise.\n \"\"\"\n # Resolve to absolute paths and check containment\n base_resolved = os.path.realpath(base_path)\n target_resolved = os.path.realpath(target_path)\n return target_resolved.startswith(base_resolved + os.sep) or target_resolved == base_resolved\n\n\ndef _validate_zip_safety(zip_path: str, extract_to: str) -> None:\n \"\"\"\n Validate a zip file for security issues before extraction.\n\n Checks for:\n - Zip bombs (excessive compression ratio or uncompressed size)\n - Too many files\n - Path traversal attacks (zip slip)\n\n Args:\n zip_path: Path to the zip file.\n extract_to: Destination directory.\n\n Raises:\n ZipExtractionError: If the zip file fails security validation.\n \"\"\"\n try:\n with zipfile.ZipFile(zip_path, \"r\") as zip_ref:\n # Get compressed size\n compressed_size = os.path.getsize(zip_path)\n\n # Calculate total uncompressed size and file count\n total_uncompressed = 0\n file_count = 0\n\n for info in zip_ref.infolist():\n file_count += 1\n\n # Check file count limit\n if file_count > MAX_FILE_COUNT:\n raise ZipExtractionError(\n f\"Zip file contains too many files (>{MAX_FILE_COUNT}). \"\n \"This may be a zip bomb attack.\"\n )\n\n # Accumulate uncompressed size\n total_uncompressed += info.file_size\n\n # Check total uncompressed size\n if total_uncompressed > MAX_UNCOMPRESSED_SIZE:\n raise ZipExtractionError(\n f\"Zip file uncompressed size exceeds limit \"\n f\"({total_uncompressed / (1024*1024):.1f} MB > \"\n f\"{MAX_UNCOMPRESSED_SIZE / (1024*1024):.1f} MB). \"\n \"This may be a zip bomb attack.\"\n )\n\n # Check for path traversal (zip slip)\n target_path = os.path.join(extract_to, info.filename)\n if not _is_path_safe(extract_to, target_path):\n raise ZipExtractionError(\n f\"Zip file contains path traversal attempt: {info.filename}\"\n )\n\n # Check compression ratio (only if compressed size is meaningful)\n if compressed_size > 0 and total_uncompressed > 0:\n compression_ratio = total_uncompressed / compressed_size\n if compression_ratio > MAX_COMPRESSION_RATIO:\n raise ZipExtractionError(\n f\"Zip file has suspicious compression ratio ({compression_ratio:.1f}:1 > \"\n f\"{MAX_COMPRESSION_RATIO}:1). This may be a zip bomb attack.\"\n )\n\n except zipfile.BadZipFile as e:\n raise ZipExtractionError(f\"Invalid or corrupted zip file: {e}\")\n\n\ndef extract_zip_recursive(zip_path, extract_to, current_depth=0, max_depth=5):\n \"\"\"\n Recursively extract zip files with security protections.\n\n Security measures:\n - Limits recursion depth to prevent infinite loops\n - Validates uncompressed size to prevent zip bombs\n - Limits number of files to prevent resource exhaustion\n - Checks compression ratio to detect zip bombs\n - Validates paths to prevent zip slip attacks\n\n Args:\n zip_path (str): Path to the zip file to be extracted.\n extract_to (str): Destination path for extracted files.\n current_depth (int): Current depth of recursion.\n max_depth (int): Maximum allowed depth of recursion to prevent infinite loops.\n \"\"\"\n if current_depth > max_depth:\n logging.warning(f\"Reached maximum recursion depth of {max_depth}\")\n return\n\n try:\n # Validate zip file safety before extraction\n _validate_zip_safety(zip_path, extract_to)\n\n # Safe to extract\n with zipfile.ZipFile(zip_path, \"r\") as zip_ref:\n zip_ref.extractall(extract_to)\n os.remove(zip_path) # Remove the zip file after extracting\n\n except ZipExtractionError as e:\n logging.error(f\"Zip security validation failed for {zip_path}: {e}\")\n # Remove the potentially malicious zip file\n try:\n os.remove(zip_path)\n except OSError:\n pass\n return\n except Exception as e:\n logging.error(f\"Error extracting zip file {zip_path}: {e}\", exc_info=True)\n return\n\n # Check for nested zip files and extract them\n for root, dirs, files in os.walk(extract_to):\n for file in files:\n if file.endswith(\".zip\"):\n # If a nested zip file is found, extract it recursively\n file_path = os.path.join(root, file)\n extract_zip_recursive(file_path, root, current_depth + 1, max_depth)\n\n\ndef download_file(url, params, dest_path):\n try:\n response = requests.get(url, params=params)\n response.raise_for_status()\n with open(dest_path, \"wb\") as f:\n f.write(response.content)\n except requests.RequestException as e:\n logging.error(f\"Error downloading file: {e}\")\n raise\n\n\ndef upload_index(full_path, file_data):\n files = None\n try:\n headers = {}\n if settings.INTERNAL_KEY:\n headers[\"X-Internal-Key\"] = settings.INTERNAL_KEY\n\n if settings.VECTOR_STORE == \"faiss\":\n faiss_path = full_path + \"/index.faiss\"\n pkl_path = full_path + \"/index.pkl\"\n\n if not os.path.exists(faiss_path):\n logging.error(f\"FAISS index file not found: {faiss_path}\")\n raise FileNotFoundError(f\"FAISS index file not found: {faiss_path}\")\n\n if not os.path.exists(pkl_path):\n logging.error(f\"FAISS pickle file not found: {pkl_path}\")\n raise FileNotFoundError(f\"FAISS pickle file not found: {pkl_path}\")\n\n files = {\n \"file_faiss\": open(faiss_path, \"rb\"),\n \"file_pkl\": open(pkl_path, \"rb\"),\n }\n response = requests.post(\n urljoin(settings.API_URL, \"/api/upload_index\"),\n files=files,\n data=file_data,\n headers=headers,\n )\n else:\n response = requests.post(\n urljoin(settings.API_URL, \"/api/upload_index\"),\n data=file_data,\n headers=headers,\n )\n response.raise_for_status()\n except (requests.RequestException, FileNotFoundError) as e:\n logging.error(f\"Error uploading index: {e}\")\n raise\n finally:\n if settings.VECTOR_STORE == \"faiss\" and files is not None:\n for file in files.values():\n file.close()\n\n\ndef run_agent_logic(agent_config, input_data):\n try:\n from application.core.model_utils import (\n get_api_key_for_provider,\n get_default_model_id,\n get_provider_from_model_id,\n validate_model_id,\n )\n from application.utils import calculate_doc_token_budget\n\n source = agent_config.get(\"source\")\n retriever = agent_config.get(\"retriever\", \"classic\")\n if isinstance(source, DBRef):\n source_doc = db.dereference(source)\n source = str(source_doc[\"_id\"])\n retriever = source_doc.get(\"retriever\", agent_config.get(\"retriever\"))\n else:\n source = {}\n source = {\"active_docs\": source}\n chunks = int(agent_config.get(\"chunks\", 2))\n prompt_id = agent_config.get(\"prompt_id\", \"default\")\n user_api_key = agent_config[\"key\"]\n agent_id = str(agent_config.get(\"_id\")) if agent_config.get(\"_id\") else None\n agent_type = agent_config.get(\"agent_type\", \"classic\")\n decoded_token = {\"sub\": agent_config.get(\"user\")}\n json_schema = agent_config.get(\"json_schema\")\n prompt = get_prompt(prompt_id, db[\"prompts\"])\n\n # Determine model_id: check agent's default_model_id, fallback to system default\n agent_default_model = agent_config.get(\"default_model_id\", \"\")\n if agent_default_model and validate_model_id(agent_default_model):\n model_id = agent_default_model\n else:\n model_id = get_default_model_id()\n\n # Get provider and API key for the selected model\n provider = get_provider_from_model_id(model_id) if model_id else settings.LLM_PROVIDER\n system_api_key = get_api_key_for_provider(provider or settings.LLM_PROVIDER)\n\n # Calculate proper doc_token_limit based on model's context window\n doc_token_limit = calculate_doc_token_budget(\n model_id=model_id\n )\n\n retriever = RetrieverCreator.create_retriever(\n retriever,\n source=source,\n chat_history=[],\n prompt=prompt,\n chunks=chunks,\n doc_token_limit=doc_token_limit,\n model_id=model_id,\n user_api_key=user_api_key,\n agent_id=agent_id,\n decoded_token=decoded_token,\n )\n\n # Pre-fetch documents using the retriever\n retrieved_docs = []\n try:\n docs = retriever.search(input_data)\n if docs:\n retrieved_docs = docs\n except Exception as e:\n logging.warning(f\"Failed to retrieve documents: {e}\")\n\n agent = AgentCreator.create_agent(\n agent_type,\n endpoint=\"webhook\",\n llm_name=provider or settings.LLM_PROVIDER,\n model_id=model_id,\n api_key=system_api_key,\n agent_id=agent_id,\n user_api_key=user_api_key,\n prompt=prompt,\n chat_history=[],\n retrieved_docs=retrieved_docs,\n decoded_token=decoded_token,\n attachments=[],\n json_schema=json_schema,\n )\n answer = agent.gen(query=input_data)\n response_full = \"\"\n thought = \"\"\n source_log_docs = []\n tool_calls = []\n\n for line in answer:\n if \"answer\" in line:\n response_full += str(line[\"answer\"])\n elif \"sources\" in line:\n source_log_docs.extend(line[\"sources\"])\n elif \"tool_calls\" in line:\n tool_calls.extend(line[\"tool_calls\"])\n elif \"thought\" in line:\n thought += line[\"thought\"]\n result = {\n \"answer\": response_full,\n \"sources\": source_log_docs,\n \"tool_calls\": tool_calls,\n \"thought\": thought,\n }\n logging.info(f\"Agent response: {result}\")\n return result\n except Exception as e:\n logging.error(f\"Error in run_agent_logic: {e}\", exc_info=True)\n raise\n\n\n# Define the main function for ingesting and processing documents.\n\n\ndef ingest_worker(\n self,\n directory,\n formats,\n job_name,\n file_path,\n filename,\n user,\n retriever=\"classic\",\n file_name_map=None,\n):\n \"\"\"\n Ingest and process documents.\n\n Args:\n self: Reference to the instance of the task.\n directory (str): Specifies the directory for ingesting ('inputs' or 'temp').\n formats (list of str): List of file extensions to consider for ingestion (e.g., [\".rst\", \".md\"]).\n job_name (str): Name of the job for this ingestion task (original, unsanitized).\n file_path (str): Complete file path to use consistently throughout the pipeline.\n filename (str): Original unsanitized filename provided by the user.\n user (str): Identifier for the user initiating the ingestion (original, unsanitized).\n retriever (str): Type of retriever to use for processing the documents.\n file_name_map (dict|str|None): Optional mapping of safe relative paths to original filenames.\n\n Returns:\n dict: Information about the completed ingestion task, including input parameters and a \"limited\" flag.\n \"\"\"\n input_files = None\n recursive = True\n limit = None\n exclude = True\n sample = False\n\n storage = StorageCreator.get_storage()\n\n logging.info(f\"Ingest path: {file_path}\", extra={\"user\": user, \"job\": job_name})\n\n # Create temporary working directory\n\n with tempfile.TemporaryDirectory() as temp_dir:\n try:\n os.makedirs(temp_dir, exist_ok=True)\n\n if storage.is_directory(file_path):\n # Handle directory case\n logging.info(f\"Processing directory: {file_path}\")\n files_list = storage.list_files(file_path)\n\n for storage_file_path in files_list:\n if storage.is_directory(storage_file_path):\n continue\n\n # Create relative path structure in temp directory\n rel_path = os.path.relpath(storage_file_path, file_path)\n local_file_path = os.path.join(temp_dir, rel_path)\n\n os.makedirs(os.path.dirname(local_file_path), exist_ok=True)\n\n # Download file\n try:\n file_data = storage.get_file(storage_file_path)\n with open(local_file_path, \"wb\") as f:\n f.write(file_data.read())\n except Exception as e:\n logging.error(\n f\"Error downloading file {storage_file_path}: {e}\"\n )\n continue\n else:\n # Handle single file case\n temp_filename = os.path.basename(file_path)\n temp_file_path = os.path.join(temp_dir, temp_filename)\n\n file_data = storage.get_file(file_path)\n with open(temp_file_path, \"wb\") as f:\n f.write(file_data.read())\n\n # Handle zip files\n if temp_filename.endswith(\".zip\"):\n logging.info(f\"Extracting zip file: {temp_filename}\")\n extract_zip_recursive(\n temp_file_path,\n temp_dir,\n current_depth=0,\n max_depth=RECURSION_DEPTH,\n )\n\n self.update_state(state=\"PROGRESS\", meta={\"current\": 1})\n if sample:\n logging.info(f\"Sample mode enabled. Using {limit} documents.\")\n reader = SimpleDirectoryReader(\n input_dir=temp_dir,\n input_files=input_files,\n recursive=recursive,\n required_exts=formats,\n exclude_hidden=exclude,\n file_metadata=metadata_from_filename,\n )\n raw_docs = reader.load_data()\n\n directory_structure = getattr(reader, \"directory_structure\", {})\n logging.info(f\"Directory structure from reader: {directory_structure}\")\n file_name_map = _normalize_file_name_map(file_name_map)\n if file_name_map:\n for doc in raw_docs:\n extra_info = getattr(doc, \"extra_info\", None)\n if not isinstance(extra_info, dict):\n continue\n rel_path = extra_info.get(\"source\") or extra_info.get(\"file_path\")\n display_name = _get_display_name(file_name_map, rel_path)\n if display_name:\n display_name = str(display_name)\n extra_info[\"filename\"] = display_name\n extra_info[\"file_name\"] = display_name\n extra_info[\"title\"] = display_name\n directory_structure = _apply_display_names_to_structure(\n directory_structure, file_name_map\n )\n\n chunker = Chunker(\n chunking_strategy=\"classic_chunk\",\n max_tokens=MAX_TOKENS,\n min_tokens=MIN_TOKENS,\n duplicate_headers=False,\n )\n raw_docs = chunker.chunk(documents=raw_docs)\n\n docs = [Document.to_langchain_format(raw_doc) for raw_doc in raw_docs]\n\n id = ObjectId()\n\n vector_store_path = os.path.join(temp_dir, \"vector_store\")\n os.makedirs(vector_store_path, exist_ok=True)\n\n embed_and_store_documents(docs, vector_store_path, id, self)\n\n tokens = count_tokens_docs(docs)\n\n self.update_state(state=\"PROGRESS\", meta={\"current\": 100})\n\n if sample:\n for i in range(min(5, len(raw_docs))):\n logging.info(f\"Sample document {i}: {raw_docs[i]}\")\n file_data = {\n \"name\": job_name,\n \"file\": filename,\n \"user\": user,\n \"tokens\": tokens,\n \"retriever\": retriever,\n \"id\": str(id),\n \"type\": \"local\",\n \"file_path\": file_path,\n \"directory_structure\": json.dumps(directory_structure),\n }\n if file_name_map:\n file_data[\"file_name_map\"] = json.dumps(file_name_map)\n\n upload_index(vector_store_path, file_data)\n except Exception as e:\n logging.error(f\"Error in ingest_worker: {e}\", exc_info=True)\n raise\n return {\n \"directory\": directory,\n \"formats\": formats,\n \"name_job\": job_name, # Use original job_name\n \"filename\": filename,\n \"user\": user, # Use original user\n \"limited\": False,\n }\n\n\ndef reingest_source_worker(self, source_id, user):\n \"\"\"\n Re-ingestion worker that handles incremental updates by:\n 1. Adding chunks from newly added files\n 2. Removing chunks from deleted files\n\n Args:\n self: Task instance\n source_id: ID of the source to re-ingest\n user: User identifier\n\n Returns:\n dict: Information about the re-ingestion task\n \"\"\"\n try:\n from application.vectorstore.vector_creator import VectorCreator\n\n self.update_state(\n state=\"PROGRESS\",\n meta={\"current\": 10, \"status\": \"Initializing re-ingestion scan\"},\n )\n\n source = sources_collection.find_one({\"_id\": ObjectId(source_id), \"user\": user})\n if not source:\n raise ValueError(f\"Source {source_id} not found or access denied\")\n\n storage = StorageCreator.get_storage()\n source_file_path = source.get(\"file_path\", \"\")\n file_name_map = _normalize_file_name_map(source.get(\"file_name_map\"))\n\n self.update_state(\n state=\"PROGRESS\", meta={\"current\": 20, \"status\": \"Scanning current files\"}\n )\n\n with tempfile.TemporaryDirectory() as temp_dir:\n # Download all files from storage to temp directory, preserving directory structure\n if storage.is_directory(source_file_path):\n files_list = storage.list_files(source_file_path)\n\n for storage_file_path in files_list:\n if storage.is_directory(storage_file_path):\n continue\n\n rel_path = os.path.relpath(storage_file_path, source_file_path)\n local_file_path = os.path.join(temp_dir, rel_path)\n\n os.makedirs(os.path.dirname(local_file_path), exist_ok=True)\n\n # Download file\n try:\n file_data = storage.get_file(storage_file_path)\n with open(local_file_path, \"wb\") as f:\n f.write(file_data.read())\n except Exception as e:\n logging.error(\n f\"Error downloading file {storage_file_path}: {e}\"\n )\n continue\n\n reader = SimpleDirectoryReader(\n input_dir=temp_dir,\n recursive=True,\n required_exts=list(SUPPORTED_SOURCE_EXTENSIONS),\n exclude_hidden=True,\n file_metadata=metadata_from_filename,\n )\n reader.load_data()\n directory_structure = reader.directory_structure\n logging.info(\n f\"Directory structure built with token counts: {directory_structure}\"\n )\n\n try:\n old_directory_structure = source.get(\"directory_structure\") or {}\n if isinstance(old_directory_structure, str):\n try:\n old_directory_structure = json.loads(old_directory_structure)\n except Exception:\n old_directory_structure = {}\n\n def _flatten_directory_structure(struct, prefix=\"\"):\n files = set()\n if isinstance(struct, dict):\n for name, meta in struct.items():\n current_path = (\n os.path.join(prefix, name) if prefix else name\n )\n if isinstance(meta, dict) and (\n \"type\" in meta and \"size_bytes\" in meta\n ):\n files.add(current_path)\n elif isinstance(meta, dict):\n files |= _flatten_directory_structure(\n meta, current_path\n )\n return files\n\n old_files = _flatten_directory_structure(old_directory_structure)\n new_files = _flatten_directory_structure(directory_structure)\n\n added_files = sorted(new_files - old_files)\n removed_files = sorted(old_files - new_files)\n\n if added_files:\n logging.info(f\"Files added since last ingest: {added_files}\")\n else:\n logging.info(\"No files added since last ingest.\")\n\n if removed_files:\n logging.info(f\"Files removed since last ingest: {removed_files}\")\n else:\n logging.info(\"No files removed since last ingest.\")\n\n except Exception as e:\n logging.error(\n f\"Error comparing directory structures: {e}\", exc_info=True\n )\n added_files = []\n removed_files = []\n try:\n if not added_files and not removed_files:\n logging.info(\"No changes detected.\")\n return {\n \"source_id\": source_id,\n \"user\": user,\n \"status\": \"no_changes\",\n \"added_files\": [],\n \"removed_files\": [],\n }\n\n vector_store = VectorCreator.create_vectorstore(\n settings.VECTOR_STORE,\n source_id,\n settings.EMBEDDINGS_KEY,\n )\n\n self.update_state(\n state=\"PROGRESS\",\n meta={\"current\": 40, \"status\": \"Processing file changes\"},\n )\n\n # 1) Delete chunks from removed files\n deleted = 0\n if removed_files:\n try:\n for ch in vector_store.get_chunks() or []:\n metadata = (\n ch.get(\"metadata\", {})\n if isinstance(ch, dict)\n else getattr(ch, \"metadata\", {})\n )\n raw_source = metadata.get(\"source\")\n\n source_file = str(raw_source) if raw_source else \"\"\n\n if source_file in removed_files:\n cid = ch.get(\"doc_id\")\n if cid:\n try:\n vector_store.delete_chunk(cid)\n deleted += 1\n except Exception as de:\n logging.error(\n f\"Failed deleting chunk {cid}: {de}\"\n )\n logging.info(\n f\"Deleted {deleted} chunks from {len(removed_files)} removed files\"\n )\n except Exception as e:\n logging.error(\n f\"Error during deletion of removed file chunks: {e}\",\n exc_info=True,\n )\n\n # 2) Add chunks from new files\n added = 0\n if added_files:\n try:\n # Build list of local files for added files only\n added_local_files = []\n for rel_path in added_files:\n local_path = os.path.join(temp_dir, rel_path)\n if os.path.isfile(local_path):\n added_local_files.append(local_path)\n\n if added_local_files:\n reader_new = SimpleDirectoryReader(\n input_files=added_local_files,\n exclude_hidden=True,\n errors=\"ignore\",\n file_metadata=metadata_from_filename,\n )\n raw_docs_new = reader_new.load_data()\n chunker_new = Chunker(\n chunking_strategy=\"classic_chunk\",\n max_tokens=MAX_TOKENS,\n min_tokens=MIN_TOKENS,\n duplicate_headers=False,\n )\n chunked_new = chunker_new.chunk(documents=raw_docs_new)\n\n for (\n file_path,\n token_count,\n ) in reader_new.file_token_counts.items():\n try:\n rel_path = os.path.relpath(\n file_path, start=temp_dir\n )\n path_parts = rel_path.split(os.sep)\n current_dir = directory_structure\n\n for part in path_parts[:-1]:\n if part in current_dir and isinstance(\n current_dir[part], dict\n ):\n current_dir = current_dir[part]\n else:\n break\n\n filename = path_parts[-1]\n if filename in current_dir and isinstance(\n current_dir[filename], dict\n ):\n current_dir[filename][\n \"token_count\"\n ] = token_count\n logging.info(\n f\"Updated token count for {rel_path}: {token_count}\"\n )\n except Exception as e:\n logging.warning(\n f\"Could not update token count for {file_path}: {e}\"\n )\n\n for d in chunked_new:\n meta = dict(d.extra_info or {})\n try:\n raw_src = meta.get(\"source\")\n if isinstance(raw_src, str) and os.path.isabs(\n raw_src\n ):\n meta[\"source\"] = os.path.relpath(\n raw_src, start=temp_dir\n )\n except Exception:\n pass\n display_name = _get_display_name(\n file_name_map, meta.get(\"source\")\n )\n if display_name:\n display_name = str(display_name)\n meta[\"filename\"] = display_name\n meta[\"file_name\"] = display_name\n meta[\"title\"] = display_name\n\n vector_store.add_chunk(d.text, metadata=meta)\n added += 1\n logging.info(\n f\"Added {added} chunks from {len(added_files)} new files\"\n )\n except Exception as e:\n logging.error(\n f\"Error during ingestion of new files: {e}\", exc_info=True\n )\n\n # 3) Update source directory structure timestamp\n try:\n total_tokens = sum(reader.file_token_counts.values())\n directory_structure = _apply_display_names_to_structure(\n directory_structure, file_name_map\n )\n\n sources_collection.update_one(\n {\"_id\": ObjectId(source_id)},\n {\n \"$set\": {\n \"directory_structure\": directory_structure,\n \"date\": datetime.datetime.now(),\n \"tokens\": total_tokens,\n }\n },\n )\n except Exception as e:\n logging.error(\n f\"Error updating directory_structure in DB: {e}\", exc_info=True\n )\n\n self.update_state(\n state=\"PROGRESS\",\n meta={\"current\": 100, \"status\": \"Re-ingestion completed\"},\n )\n\n return {\n \"source_id\": source_id,\n \"user\": user,\n \"status\": \"completed\",\n \"added_files\": added_files,\n \"removed_files\": removed_files,\n \"chunks_added\": added,\n \"chunks_deleted\": deleted,\n }\n except Exception as e:\n logging.error(\n f\"Error while processing file changes: {e}\", exc_info=True\n )\n raise\n\n except Exception as e:\n logging.error(f\"Error in reingest_source_worker: {e}\", exc_info=True)\n raise\n\n\ndef remote_worker(\n self,\n source_data,\n name_job,\n user,\n loader,\n directory=\"temp\",\n retriever=\"classic\",\n sync_frequency=\"never\",\n operation_mode=\"upload\",\n doc_id=None,\n):\n full_path = os.path.join(directory, user, name_job)\n if not os.path.exists(full_path):\n os.makedirs(full_path)\n self.update_state(state=\"PROGRESS\", meta={\"current\": 1})\n try:\n logging.info(\"Initializing remote loader with type: %s\", loader)\n remote_loader = RemoteCreator.create_loader(loader)\n raw_docs = remote_loader.load_data(source_data)\n\n chunker = Chunker(\n chunking_strategy=\"classic_chunk\",\n max_tokens=MAX_TOKENS,\n min_tokens=MIN_TOKENS,\n duplicate_headers=False,\n )\n docs = chunker.chunk(documents=raw_docs)\n docs = [Document.to_langchain_format(raw_doc) for raw_doc in raw_docs]\n tokens = count_tokens_docs(docs)\n logging.info(\"Total tokens calculated: %d\", tokens)\n\n # Build directory structure from loaded documents\n # Format matches local file uploads: nested structure with type, size_bytes, token_count\n directory_structure = {}\n for doc in raw_docs:\n # Get the file path from extra_info\n # For crawlers: file_path is a virtual path like \"guides/setup.md\"\n # For other remotes: use key or title as fallback\n file_path = \"\"\n if doc.extra_info:\n file_path = (\n doc.extra_info.get(\"file_path\", \"\")\n or doc.extra_info.get(\"key\", \"\")\n or doc.extra_info.get(\"title\", \"\")\n )\n if not file_path:\n file_path = doc.doc_id or \"\"\n\n if file_path:\n # Calculate token count\n token_count = num_tokens_from_string(doc.text) if doc.text else 0\n\n # Estimate size in bytes from text content\n size_bytes = len(doc.text.encode(\"utf-8\")) if doc.text else 0\n\n # Guess mime type from extension\n file_name = (\n file_path.split(\"/\")[-1] if \"/\" in file_path else file_path\n )\n ext = os.path.splitext(file_name)[1].lower()\n mime_types = {\n \".txt\": \"text/plain\",\n \".md\": \"text/markdown\",\n \".pdf\": \"application/pdf\",\n \".docx\": \"application/vnd.openxmlformats-officedocument.wordprocessingml.document\",\n \".doc\": \"application/msword\",\n \".html\": \"text/html\",\n \".json\": \"application/json\",\n \".csv\": \"text/csv\",\n \".xml\": \"application/xml\",\n \".py\": \"text/x-python\",\n \".js\": \"text/javascript\",\n \".ts\": \"text/typescript\",\n \".jsx\": \"text/jsx\",\n \".tsx\": \"text/tsx\",\n }\n file_type = mime_types.get(ext, \"application/octet-stream\")\n\n # Build nested directory structure from path\n # e.g., \"guides/setup.md\" -> {\"guides\": {\"setup.md\": {...}}}\n path_parts = file_path.split(\"/\")\n current_level = directory_structure\n for i, part in enumerate(path_parts):\n if i == len(path_parts) - 1:\n # Last part is the file\n current_level[part] = {\n \"type\": file_type,\n \"size_bytes\": size_bytes,\n \"token_count\": token_count,\n }\n else:\n # Intermediate parts are directories\n if part not in current_level:\n current_level[part] = {}\n current_level = current_level[part]\n\n logging.info(\n f\"Built directory structure with {len(directory_structure)} files: \"\n f\"{list(directory_structure.keys())}\"\n )\n\n if operation_mode == \"upload\":\n id = ObjectId()\n embed_and_store_documents(docs, full_path, id, self)\n elif operation_mode == \"sync\":\n if not doc_id or not ObjectId.is_valid(doc_id):\n logging.error(\"Invalid doc_id provided for sync operation: %s\", doc_id)\n raise ValueError(\"doc_id must be provided for sync operation.\")\n id = ObjectId(doc_id)\n embed_and_store_documents(docs, full_path, id, self)\n self.update_state(state=\"PROGRESS\", meta={\"current\": 100})\n\n # Serialize remote_data as JSON if it's a dict (for S3, Reddit, etc.)\n remote_data_serialized = (\n json.dumps(source_data) if isinstance(source_data, dict) else source_data\n )\n file_data = {\n \"name\": name_job,\n \"user\": user,\n \"tokens\": tokens,\n \"retriever\": retriever,\n \"id\": str(id),\n \"type\": loader,\n \"remote_data\": remote_data_serialized,\n \"sync_frequency\": sync_frequency,\n \"directory_structure\": json.dumps(directory_structure),\n }\n\n if operation_mode == \"sync\":\n file_data[\"last_sync\"] = datetime.datetime.now()\n upload_index(full_path, file_data)\n except Exception as e:\n logging.error(\"Error in remote_worker task: %s\", str(e), exc_info=True)\n raise\n finally:\n if os.path.exists(full_path):\n shutil.rmtree(full_path)\n logging.info(\"remote_worker task completed successfully\")\n return {\n \"id\": str(id),\n \"urls\": source_data,\n \"name_job\": name_job,\n \"user\": user,\n \"limited\": False,\n }\n\n\ndef sync(\n self,\n source_data,\n name_job,\n user,\n loader,\n sync_frequency,\n retriever,\n doc_id=None,\n directory=\"temp\",\n):\n try:\n remote_worker(\n self,\n source_data,\n name_job,\n user,\n loader,\n directory,\n retriever,\n sync_frequency,\n \"sync\",\n doc_id,\n )\n except Exception as e:\n logging.error(f\"Error during sync: {e}\", exc_info=True)\n return {\"status\": \"error\", \"error\": str(e)}\n return {\"status\": \"success\"}\n\n\ndef sync_worker(self, frequency):\n sync_counts = Counter()\n sources = sources_collection.find()\n for doc in sources:\n if doc.get(\"sync_frequency\") == frequency:\n name = doc.get(\"name\")\n user = doc.get(\"user\")\n source_type = doc.get(\"type\")\n source_data = doc.get(\"remote_data\")\n retriever = doc.get(\"retriever\")\n doc_id = str(doc.get(\"_id\"))\n resp = sync(\n self, source_data, name, user, source_type, frequency, retriever, doc_id\n )\n sync_counts[\"total_sync_count\"] += 1\n sync_counts[\n \"sync_success\" if resp[\"status\"] == \"success\" else \"sync_failure\"\n ] += 1\n return {\n key: sync_counts[key]\n for key in [\"total_sync_count\", \"sync_success\", \"sync_failure\"]\n }\n\n\ndef attachment_worker(self, file_info, user):\n \"\"\"\n Process and store a single attachment without vectorization.\n \"\"\"\n\n mongo = MongoDB.get_client()\n db = mongo[settings.MONGO_DB_NAME]\n attachments_collection = db[\"attachments\"]\n\n filename = file_info[\"filename\"]\n attachment_id = file_info[\"attachment_id\"]\n relative_path = file_info[\"path\"]\n metadata = file_info.get(\"metadata\", {})\n\n try:\n self.update_state(state=\"PROGRESS\", meta={\"current\": 10})\n storage = StorageCreator.get_storage()\n\n self.update_state(\n state=\"PROGRESS\", meta={\"current\": 30, \"status\": \"Processing content\"}\n )\n\n file_extractor = get_default_file_extractor(\n ocr_enabled=settings.DOCLING_OCR_ATTACHMENTS_ENABLED\n )\n attachment_document = storage.process_file(\n relative_path,\n lambda local_path, **kwargs: SimpleDirectoryReader(\n input_files=[local_path],\n exclude_hidden=True,\n errors=\"ignore\",\n file_extractor=file_extractor,\n file_metadata=metadata_from_filename,\n )\n .load_data()[0],\n )\n content = attachment_document.text\n parser_metadata = {\n key: value\n for key, value in (attachment_document.extra_info or {}).items()\n if key.startswith(\"transcript_\")\n }\n if parser_metadata:\n metadata = {**metadata, **parser_metadata}\n\n token_count = num_tokens_from_string(content)\n if token_count > 100000:\n content = content[:250000]\n token_count = num_tokens_from_string(content)\n\n self.update_state(\n state=\"PROGRESS\", meta={\"current\": 80, \"status\": \"Storing in database\"}\n )\n\n mime_type = mimetypes.guess_type(filename)[0] or \"application/octet-stream\"\n\n doc_id = ObjectId(attachment_id)\n attachments_collection.insert_one(\n {\n \"_id\": doc_id,\n \"user\": user,\n \"path\": relative_path,\n \"filename\": filename,\n \"content\": content,\n \"token_count\": token_count,\n \"mime_type\": mime_type,\n \"date\": datetime.datetime.now(),\n \"metadata\": metadata,\n }\n )\n\n logging.info(\n f\"Stored attachment with ID: {attachment_id}\", extra={\"user\": user}\n )\n\n self.update_state(state=\"PROGRESS\", meta={\"current\": 100, \"status\": \"Complete\"})\n\n return {\n \"filename\": filename,\n \"path\": relative_path,\n \"token_count\": token_count,\n \"attachment_id\": attachment_id,\n \"mime_type\": mime_type,\n \"metadata\": metadata,\n }\n except Exception as e:\n logging.error(\n f\"Error processing file {filename}: {e}\",\n extra={\"user\": user},\n exc_info=True,\n )\n raise\n\n\ndef agent_webhook_worker(self, agent_id, payload):\n \"\"\"\n Process the webhook payload for an agent.\n\n Args:\n self: Reference to the instance of the task.\n agent_id (str): Unique identifier for the agent.\n payload (dict): The payload data from the webhook.\n\n Returns:\n dict: Information about the processed webhook.\n \"\"\"\n mongo = MongoDB.get_client()\n db = mongo[\"docsgpt\"]\n agents_collection = db[\"agents\"]\n\n self.update_state(state=\"PROGRESS\", meta={\"current\": 1})\n try:\n agent_oid = ObjectId(agent_id)\n agent_config = agents_collection.find_one({\"_id\": agent_oid})\n if not agent_config:\n raise ValueError(f\"Agent with ID {agent_id} not found.\")\n input_data = json.dumps(payload)\n except Exception as e:\n logging.error(f\"Error processing agent webhook: {e}\", exc_info=True)\n return {\"status\": \"error\", \"error\": str(e)}\n self.update_state(state=\"PROGRESS\", meta={\"current\": 50})\n try:\n result = run_agent_logic(agent_config, input_data)\n except Exception as e:\n logging.error(f\"Error running agent logic: {e}\", exc_info=True)\n return {\"status\": \"error\"}\n else:\n logging.info(\n f\"Webhook processed for agent {agent_id}\", extra={\"agent_id\": agent_id}\n )\n return {\"status\": \"success\", \"result\": result}\n finally:\n self.update_state(state=\"PROGRESS\", meta={\"current\": 100})\n\n\ndef ingest_connector(\n self,\n job_name: str,\n user: str,\n source_type: str,\n session_token=None,\n file_ids=None,\n folder_ids=None,\n recursive=True,\n retriever: str = \"classic\",\n operation_mode: str = \"upload\",\n doc_id=None,\n sync_frequency: str = \"never\",\n) -> Dict[str, Any]:\n \"\"\"\n Ingestion for internal knowledge bases (GoogleDrive, etc.).\n\n Args:\n job_name: Name of the ingestion job\n user: User identifier\n source_type: Type of remote source (\"google_drive\", \"dropbox\", etc.)\n session_token: Authentication token for the service\n file_ids: List of file IDs to download\n folder_ids: List of folder IDs to download\n recursive: Whether to recursively download folders\n retriever: Type of retriever to use\n operation_mode: \"upload\" for initial ingestion, \"sync\" for incremental sync\n doc_id: Document ID for sync operations (required when operation_mode=\"sync\")\n sync_frequency: How often to sync (\"never\", \"daily\", \"weekly\", \"monthly\")\n \"\"\"\n logging.info(\n f\"Starting remote ingestion from {source_type} for user: {user}, job: {job_name}\"\n )\n self.update_state(state=\"PROGRESS\", meta={\"current\": 1})\n\n with tempfile.TemporaryDirectory() as temp_dir:\n try:\n # Step 1: Initialize the appropriate loader\n self.update_state(\n state=\"PROGRESS\",\n meta={\"current\": 10, \"status\": \"Initializing connector\"},\n )\n\n if not session_token:\n raise ValueError(f\"{source_type} connector requires session_token\")\n\n if not ConnectorCreator.is_supported(source_type):\n raise ValueError(\n f\"Unsupported connector type: {source_type}. Supported types: {ConnectorCreator.get_supported_connectors()}\"\n )\n\n remote_loader = ConnectorCreator.create_connector(\n source_type, session_token\n )\n\n # Create a clean config for storage\n api_source_config = {\n \"file_ids\": file_ids or [],\n \"folder_ids\": folder_ids or [],\n \"recursive\": recursive,\n }\n\n # Step 2: Download files to temp directory\n self.update_state(\n state=\"PROGRESS\", meta={\"current\": 20, \"status\": \"Downloading files\"}\n )\n download_info = remote_loader.download_to_directory(\n temp_dir, api_source_config\n )\n\n if download_info.get(\"empty_result\", False) or not download_info.get(\n \"files_downloaded\", 0\n ):\n logging.warning(f\"No files were downloaded from {source_type}\")\n # Create empty result directly instead of calling a separate method\n return {\n \"name\": job_name,\n \"user\": user,\n \"tokens\": 0,\n \"type\": source_type,\n \"source_config\": api_source_config,\n \"directory_structure\": \"{}\",\n }\n\n # Step 3: Use SimpleDirectoryReader to process downloaded files\n self.update_state(\n state=\"PROGRESS\", meta={\"current\": 40, \"status\": \"Processing files\"}\n )\n reader = SimpleDirectoryReader(\n input_dir=temp_dir,\n recursive=True,\n required_exts=list(SUPPORTED_SOURCE_EXTENSIONS),\n exclude_hidden=True,\n file_metadata=metadata_from_filename,\n )\n raw_docs = reader.load_data()\n directory_structure = getattr(reader, \"directory_structure\", {})\n\n # Step 4: Process documents (chunking, embedding, etc.)\n self.update_state(\n state=\"PROGRESS\", meta={\"current\": 60, \"status\": \"Processing documents\"}\n )\n\n chunker = Chunker(\n chunking_strategy=\"classic_chunk\",\n max_tokens=MAX_TOKENS,\n min_tokens=MIN_TOKENS,\n duplicate_headers=False,\n )\n raw_docs = chunker.chunk(documents=raw_docs)\n\n # Preserve source information in document metadata\n for doc in raw_docs:\n if hasattr(doc, \"extra_info\") and doc.extra_info:\n source = doc.extra_info.get(\"source\")\n if source and os.path.isabs(source):\n # Convert absolute path to relative path\n doc.extra_info[\"source\"] = os.path.relpath(\n source, start=temp_dir\n )\n\n docs = [Document.to_langchain_format(raw_doc) for raw_doc in raw_docs]\n\n if operation_mode == \"upload\":\n id = ObjectId()\n elif operation_mode == \"sync\":\n if not doc_id or not ObjectId.is_valid(doc_id):\n logging.error(\n \"Invalid doc_id provided for sync operation: %s\", doc_id\n )\n raise ValueError(\"doc_id must be provided for sync operation.\")\n id = ObjectId(doc_id)\n else:\n raise ValueError(f\"Invalid operation_mode: {operation_mode}\")\n\n vector_store_path = os.path.join(temp_dir, \"vector_store\")\n os.makedirs(vector_store_path, exist_ok=True)\n\n self.update_state(\n state=\"PROGRESS\", meta={\"current\": 80, \"status\": \"Storing documents\"}\n )\n embed_and_store_documents(docs, vector_store_path, id, self)\n\n tokens = count_tokens_docs(docs)\n\n # Step 6: Upload index files\n file_data = {\n \"user\": user,\n \"name\": job_name,\n \"tokens\": tokens,\n \"retriever\": retriever,\n \"id\": str(id),\n \"type\": \"connector:file\",\n \"remote_data\": json.dumps(\n {\"provider\": source_type, **api_source_config}\n ),\n \"directory_structure\": json.dumps(directory_structure),\n \"sync_frequency\": sync_frequency,\n }\n\n if operation_mode == \"sync\":\n file_data[\"last_sync\"] = datetime.datetime.now()\n else:\n file_data[\"last_sync\"] = datetime.datetime.now()\n\n upload_index(vector_store_path, file_data)\n\n # Ensure we mark the task as complete\n self.update_state(\n state=\"PROGRESS\", meta={\"current\": 100, \"status\": \"Complete\"}\n )\n\n logging.info(f\"Remote ingestion completed: {job_name}\")\n\n return {\n \"user\": user,\n \"name\": job_name,\n \"tokens\": tokens,\n \"type\": source_type,\n \"id\": str(id),\n \"status\": \"complete\",\n }\n\n except Exception as e:\n logging.error(f\"Error during remote ingestion: {e}\", exc_info=True)\n raise\n\n\ndef mcp_oauth(self, config: Dict[str, Any], user_id: str = None) -> Dict[str, Any]:\n \"\"\"Worker to handle MCP OAuth flow asynchronously.\"\"\"\n\n try:\n import asyncio\n\n from application.agents.tools.mcp_tool import MCPTool\n\n task_id = self.request.id\n redis_client = get_redis_instance()\n\n def update_status(status_data: Dict[str, Any]):\n status_key = f\"mcp_oauth_status:{task_id}\"\n redis_client.setex(status_key, 600, json.dumps(status_data))\n\n update_status(\n {\n \"status\": \"in_progress\",\n \"message\": \"Starting OAuth...\",\n \"task_id\": task_id,\n }\n )\n\n tool_config = config.copy()\n tool_config[\"oauth_task_id\"] = task_id\n mcp_tool = MCPTool(tool_config, user_id)\n\n async def run_oauth_discovery():\n if not mcp_tool._client:\n mcp_tool._setup_client()\n return await mcp_tool._execute_with_client(\"list_tools\")\n\n update_status(\n {\n \"status\": \"awaiting_redirect\",\n \"message\": \"Awaiting OAuth redirect...\",\n \"task_id\": task_id,\n }\n )\n\n loop = asyncio.new_event_loop()\n asyncio.set_event_loop(loop)\n\n try:\n loop.run_until_complete(run_oauth_discovery())\n tools = mcp_tool.get_actions_metadata()\n\n update_status(\n {\n \"status\": \"completed\",\n \"message\": f\"Connected \\u2014 found {len(tools)} tool{'s' if len(tools) != 1 else ''}.\",\n \"tools\": tools,\n \"tools_count\": len(tools),\n \"task_id\": task_id,\n }\n )\n\n return {\"success\": True, \"tools\": tools, \"tools_count\": len(tools)}\n except Exception as e:\n error_msg = f\"OAuth failed: {str(e)}\"\n logging.error(\"MCP OAuth discovery failed: %s\", error_msg, exc_info=True)\n update_status(\n {\n \"status\": \"error\",\n \"message\": error_msg,\n \"task_id\": task_id,\n }\n )\n return {\"success\": False, \"error\": error_msg}\n finally:\n loop.close()\n except Exception as e:\n error_msg = f\"OAuth init failed: {str(e)}\"\n logging.error(\"MCP OAuth init failed: %s\", error_msg, exc_info=True)\n update_status(\n {\n \"status\": \"error\",\n \"message\": error_msg,\n \"task_id\": task_id,\n }\n )\n return {\"success\": False, \"error\": error_msg}\n\n\ndef mcp_oauth_status(self, task_id: str) -> Dict[str, Any]:\n \"\"\"Check the status of an MCP OAuth flow.\"\"\"\n redis_client = get_redis_instance()\n status_key = f\"mcp_oauth_status:{task_id}\"\n\n status_data = redis_client.get(status_key)\n if status_data:\n return json.loads(status_data)\n return {\"status\": \"not_found\", \"message\": \"Status not found\"}\n" }, { "path": "application/wsgi.py", "content": "from application.app import app\nfrom application.core.settings import settings\n\nif __name__ == \"__main__\":\n app.run(debug=settings.FLASK_DEBUG_MODE, port=7091)\n" }, { "path": "codecov.yml", "content": "ignore:\n - \"*/tests/*\"" }, { "path": "deployment/docker-compose-azure.yaml", "content": "services:\n frontend:\n build: ../frontend\n environment:\n - VITE_API_HOST=http://localhost:7091\n - VITE_API_STREAMING=$VITE_API_STREAMING\n ports:\n - \"5173:5173\"\n depends_on:\n - backend\n\n backend:\n build: ../application\n env_file:\n - ../.env\n environment:\n # Override URLs to use docker service names\n - CELERY_BROKER_URL=redis://redis:6379/0\n - CELERY_RESULT_BACKEND=redis://redis:6379/1\n - MONGO_URI=mongodb://mongo:27017/docsgpt\n ports:\n - \"7091:7091\"\n volumes:\n - ../application/indexes:/app/application/indexes\n - ../application/inputs:/app/application/inputs\n - ../application/vectors:/app/application/vectors\n depends_on:\n - redis\n - mongo\n\n worker:\n build: ../application\n command: celery -A application.app.celery worker -l INFO\n env_file:\n - ../.env\n environment:\n # Override URLs to use docker service names\n - CELERY_BROKER_URL=redis://redis:6379/0\n - CELERY_RESULT_BACKEND=redis://redis:6379/1\n - MONGO_URI=mongodb://mongo:27017/docsgpt\n - API_URL=http://backend:7091\n depends_on:\n - redis\n - mongo\n\n redis:\n image: redis:6-alpine\n ports:\n - 6379:6379\n\n mongo:\n image: mongo:6\n ports:\n - 27017:27017\n volumes:\n - mongodb_data_container:/data/db\n\n\n\nvolumes:\n mongodb_data_container:" }, { "path": "deployment/docker-compose-dev.yaml", "content": "name: docsgpt-oss\nservices:\n\n redis:\n image: redis:6-alpine\n ports:\n - 6379:6379\n\n mongo:\n image: mongo:6\n ports:\n - 27017:27017\n volumes:\n - mongodb_data_container:/data/db\n\n\n\nvolumes:\n mongodb_data_container:" }, { "path": "deployment/docker-compose-hub.yaml", "content": "name: docsgpt-oss\nservices:\n\n frontend:\n image: arc53/docsgpt-fe:develop\n environment:\n - VITE_API_HOST=http://localhost:7091\n - VITE_API_STREAMING=${VITE_API_STREAMING:-true}\n - VITE_GOOGLE_CLIENT_ID=${VITE_GOOGLE_CLIENT_ID:-}\n ports:\n - \"5173:5173\"\n depends_on:\n - backend\n\n\n backend:\n user: root\n image: arc53/docsgpt:develop\n env_file:\n - ../.env\n environment:\n - CELERY_BROKER_URL=redis://redis:6379/0\n - CELERY_RESULT_BACKEND=redis://redis:6379/1\n - MONGO_URI=mongodb://mongo:27017/docsgpt\n - CACHE_REDIS_URL=redis://redis:6379/2\n ports:\n - \"7091:7091\"\n volumes:\n - ../application/indexes:/app/indexes\n - ../application/inputs:/app/inputs\n - ../application/vectors:/app/vectors\n depends_on:\n - redis\n - mongo\n\n\n worker:\n user: root\n image: arc53/docsgpt:develop\n command: celery -A application.app.celery worker -l INFO -B\n env_file:\n - ../.env\n environment:\n - CELERY_BROKER_URL=redis://redis:6379/0\n - CELERY_RESULT_BACKEND=redis://redis:6379/1\n - MONGO_URI=mongodb://mongo:27017/docsgpt\n - API_URL=http://backend:7091\n - CACHE_REDIS_URL=redis://redis:6379/2\n volumes:\n - ../application/indexes:/app/indexes\n - ../application/inputs:/app/inputs\n - ../application/vectors:/app/vectors\n depends_on:\n - redis\n - mongo\n\n redis:\n image: redis:6-alpine\n ports:\n - 6379:6379\n\n mongo:\n image: mongo:6\n ports:\n - 27017:27017\n volumes:\n - mongodb_data_container:/data/db\n\nvolumes:\n mongodb_data_container:" }, { "path": "deployment/docker-compose-local.yaml", "content": "services:\n frontend:\n build: ../frontend\n volumes:\n - ../frontend/src:/app/src\n environment:\n - VITE_API_HOST=http://localhost:7091\n - VITE_API_STREAMING=$VITE_API_STREAMING\n - VITE_EMBEDDINGS_NAME=$EMBEDDINGS_NAME\n ports:\n - \"5173:5173\"\n\n redis:\n image: redis:6-alpine\n ports:\n - 6379:6379\n\n mongo:\n image: mongo:6\n ports:\n - 27017:27017\n volumes:\n - mongodb_data_container:/data/db\n\nvolumes:\n mongodb_data_container:\n" }, { "path": "deployment/docker-compose.yaml", "content": "name: docsgpt-oss\nservices:\n frontend:\n build: ../frontend\n volumes:\n - ../frontend/src:/app/src\n environment:\n - VITE_API_HOST=http://localhost:7091\n - VITE_API_STREAMING=$VITE_API_STREAMING\n - VITE_GOOGLE_CLIENT_ID=$VITE_GOOGLE_CLIENT_ID\n ports:\n - \"5173:5173\"\n depends_on:\n - backend\n\n backend:\n user: root\n build: ../application\n env_file:\n - ../.env\n environment:\n # Override URLs to use docker service names\n - CELERY_BROKER_URL=redis://redis:6379/0\n - CELERY_RESULT_BACKEND=redis://redis:6379/1\n - MONGO_URI=mongodb://mongo:27017/docsgpt\n - CACHE_REDIS_URL=redis://redis:6379/2\n ports:\n - \"7091:7091\"\n volumes:\n - ../application/indexes:/app/indexes\n - ../application/inputs:/app/inputs\n - ../application/vectors:/app/vectors\n depends_on:\n - redis\n - mongo\n\n worker:\n user: root\n build: ../application\n command: celery -A application.app.celery worker -l INFO -B\n env_file:\n - ../.env\n environment:\n # Override URLs to use docker service names\n - CELERY_BROKER_URL=redis://redis:6379/0\n - CELERY_RESULT_BACKEND=redis://redis:6379/1\n - MONGO_URI=mongodb://mongo:27017/docsgpt\n - API_URL=http://backend:7091\n - CACHE_REDIS_URL=redis://redis:6379/2\n volumes:\n - ../application/indexes:/app/indexes\n - ../application/inputs:/app/inputs\n - ../application/vectors:/app/vectors\n depends_on:\n - redis\n - mongo\n\n redis:\n image: redis:6-alpine\n ports:\n - 6379:6379\n\n mongo:\n image: mongo:6\n ports:\n - 27017:27017\n volumes:\n - mongodb_data_container:/data/db\n\nvolumes:\n mongodb_data_container:\n" }, { "path": "deployment/k8s/deployments/docsgpt-deploy.yaml", "content": "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: docsgpt-api\nspec:\n replicas: 1\n selector:\n matchLabels:\n app: docsgpt-api\n template:\n metadata:\n labels:\n app: docsgpt-api\n spec:\n containers:\n - name: docsgpt-api\n image: arc53/docsgpt\n ports:\n - containerPort: 7091\n resources:\n limits:\n memory: \"4Gi\"\n cpu: \"2\"\n requests:\n memory: \"2Gi\"\n cpu: \"1\"\n envFrom:\n - secretRef:\n name: docsgpt-secrets\n env:\n - name: FLASK_APP\n value: \"application/app.py\"\n - name: DEPLOYMENT_TYPE\n value: \"cloud\"\n---\napiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: docsgpt-worker\nspec:\n replicas: 1\n selector:\n matchLabels:\n app: docsgpt-worker\n template:\n metadata:\n labels:\n app: docsgpt-worker\n spec:\n containers:\n - name: docsgpt-worker\n image: arc53/docsgpt\n command: [\"celery\", \"-A\", \"application.app.celery\", \"worker\", \"-l\", \"INFO\", \"-n\", \"worker.%h\"]\n resources:\n limits:\n memory: \"4Gi\"\n cpu: \"2\"\n requests:\n memory: \"2Gi\"\n cpu: \"1\"\n envFrom:\n - secretRef:\n name: docsgpt-secrets\n env:\n - name: API_URL\n value: \"http://\"\n---\napiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: docsgpt-frontend\nspec:\n replicas: 1\n selector:\n matchLabels:\n app: docsgpt-frontend\n template:\n metadata:\n labels:\n app: docsgpt-frontend\n spec:\n containers:\n - name: docsgpt-frontend\n image: arc53/docsgpt-fe\n ports:\n - containerPort: 5173\n resources:\n limits:\n memory: \"1Gi\"\n cpu: \"1\"\n requests:\n memory: \"256Mi\"\n cpu: \"100m\"\n env:\n - name: VITE_API_HOST\n value: \"http://\"\n - name: VITE_API_STREAMING\n value: \"true\"" }, { "path": "deployment/k8s/deployments/mongo-deploy.yaml", "content": "apiVersion: v1\nkind: PersistentVolumeClaim\nmetadata:\n name: mongodb-pvc\nspec:\n accessModes:\n - ReadWriteOnce\n resources:\n requests:\n storage: 10Gi # Adjust size as needed\n\n---\n\napiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: mongodb\nspec:\n replicas: 1\n selector:\n matchLabels:\n app: mongodb\n template:\n metadata:\n labels:\n app: mongodb\n spec:\n containers:\n - name: mongodb\n image: mongo:latest\n ports:\n - containerPort: 27017\n resources:\n limits:\n memory: \"1Gi\"\n cpu: \"0.5\"\n requests:\n memory: \"512Mi\"\n cpu: \"250m\"\n volumeMounts:\n - name: mongodb-data\n mountPath: /data/db\n volumes:\n - name: mongodb-data\n persistentVolumeClaim:\n claimName: mongodb-pvc" }, { "path": "deployment/k8s/deployments/qdrant-deploy.yaml", "content": "apiVersion: v1\nkind: PersistentVolumeClaim\nmetadata:\n name: qdrant-pvc\nspec:\n accessModes:\n - ReadWriteOnce\n resources:\n requests:\n storage: 10Gi\n\n---\n\napiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: qdrant\nspec:\n replicas: 1\n selector:\n matchLabels:\n app: qdrant\n template:\n metadata:\n labels:\n app: qdrant\n spec:\n containers:\n - name: qdrant\n image: qdrant/qdrant:latest\n ports:\n - containerPort: 6333\n resources:\n limits:\n memory: \"2Gi\" # Adjust based on your needs\n cpu: \"1\" # Adjust based on your needs\n requests:\n memory: \"1Gi\" # Adjust based on your needs\n cpu: \"500m\" # Adjust based on your needs\n volumeMounts:\n - name: qdrant-data\n mountPath: /qdrant/storage\n volumes:\n - name: qdrant-data\n persistentVolumeClaim:\n claimName: qdrant-pvc" }, { "path": "deployment/k8s/deployments/redis-deploy.yaml", "content": "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: redis\nspec:\n replicas: 1\n selector:\n matchLabels:\n app: redis\n template:\n metadata:\n labels:\n app: redis\n spec:\n containers:\n - name: redis\n image: redis:latest\n ports:\n - containerPort: 6379\n resources:\n limits:\n memory: \"1Gi\"\n cpu: \"0.5\"\n requests:\n memory: \"512Mi\"\n cpu: \"250m\"" }, { "path": "deployment/k8s/docsgpt-secrets.yaml", "content": "apiVersion: v1\nkind: Secret\nmetadata:\n name: docsgpt-secrets\ntype: Opaque\ndata:\n LLM_PROVIDER: ZG9jc2dwdA==\n INTERNAL_KEY: aW50ZXJuYWw=\n CELERY_BROKER_URL: cmVkaXM6Ly9yZWRpcy1zZXJ2aWNlOjYzNzkvMA==\n CELERY_RESULT_BACKEND: cmVkaXM6Ly9yZWRpcy1zZXJ2aWNlOjYzNzkvMA==\n QDRANT_URL: cmVkaXM6Ly9yZWRpcy1zZXJ2aWNlOjYzNzkvMA==\n QDRANT_PORT: NjM3OQ==\n MONGO_URI: bW9uZ29kYjovL21vbmdvZGItc2VydmljZToyNzAxNy9kb2NzZ3B0P3JldHJ5V3JpdGVzPXRydWUmdz1tYWpvcml0eQ==\n mongo-user: bW9uZ28tdXNlcg==\n mongo-password: bW9uZ28tcGFzc3dvcmQ=" }, { "path": "deployment/k8s/services/docsgpt-service.yaml", "content": "apiVersion: v1\nkind: Service\nmetadata:\n name: docsgpt-api-service\nspec:\n selector:\n app: docsgpt-api\n ports:\n - protocol: TCP\n port: 80\n targetPort: 7091\n type: LoadBalancer\n---\napiVersion: v1\nkind: Service\nmetadata:\n name: docsgpt-frontend-service\nspec:\n selector:\n app: docsgpt-frontend\n ports:\n - protocol: TCP\n port: 80\n targetPort: 5173\n type: LoadBalancer" }, { "path": "deployment/k8s/services/mongo-service.yaml", "content": "apiVersion: v1\nkind: Service\nmetadata:\n name: mongodb-service\nspec:\n selector:\n app: mongodb\n ports:\n - protocol: TCP\n port: 27017\n targetPort: 27017\n type: ClusterIP" }, { "path": "deployment/k8s/services/qdrant-service.yaml", "content": "apiVersion: v1\nkind: Service\nmetadata:\n name: qdrant\nspec:\n selector:\n app: qdrant\n ports:\n - protocol: TCP\n port: 6333\n targetPort: 6333\n type: ClusterIP" }, { "path": "deployment/k8s/services/redis-service.yaml", "content": "apiVersion: v1\nkind: Service\nmetadata:\n name: redis-service\nspec:\n selector:\n app: redis\n ports:\n - protocol: TCP\n port: 6379\n targetPort: 6379\n type: ClusterIP" }, { "path": "deployment/optional/docker-compose.optional.ollama-cpu.yaml", "content": "version: \"3.8\"\nservices:\n ollama:\n image: ollama/ollama\n ports:\n - \"11434:11434\"\n volumes:\n - ollama_data:/root/.ollama\n\nvolumes:\n ollama_data:" }, { "path": "deployment/optional/docker-compose.optional.ollama-gpu.yaml", "content": "version: \"3.8\"\nservices:\n ollama:\n image: ollama/ollama\n ports:\n - \"11434:11434\"\n volumes:\n - ollama_data:/root/.ollama\n deploy:\n resources:\n reservations:\n devices:\n - capabilities: [gpu]\n\nvolumes:\n ollama_data:" }, { "path": "docs/README.md", "content": "# nextra-docsgpt\n\n## Setting Up Docs Folder of DocsGPT Locally\n\n### 1. Clone the DocsGPT repository:\n\n```bash\ngit clone https://github.com/arc53/DocsGPT.git\n```\n### 2. Navigate to the docs folder:\n\n```bash\ncd DocsGPT/docs\n```\n\nThe docs folder contains the markdown files that make up the documentation. The majority of the files are in the pages directory. Some notable files in this folder include:\n\n`index.mdx`: The main documentation file.\n`_app.js`: This file is used to customize the default Next.js application shell.\n`theme.config.jsx`: This file is for configuring the Nextra theme for the documentation.\n\n### 3. Verify that you have Node.js and npm installed in your system. You can check by running:\n\n```bash\nnode --version\nnpm --version\n```\n\n### 4. If not installed, download Node.js and npm from the respective official websites.\n\n### 5. Once you have Node.js and npm running, proceed to install yarn - another package manager that helps to manage project dependencies:\n\n```bash\nnpm install --global yarn\n```\n\n### 6. Install the project dependencies using yarn:\n\n```bash\nyarn install\n```\n\n### 7. After the successful installation of the project dependencies, start the local server:\n\n```bash\nyarn dev\n```\n\n- Now, you should be able to view the docs on your local environment by visiting `http://localhost:3000`. You can explore the different markdown files and make changes as you see fit.\n\n- **Footnotes:** This guide assumes you have Node.js and npm installed. The guide involves running a local server using yarn, and viewing the documentation offline. If you encounter any issues, it may be worth verifying your Node.js and npm installations and whether you have installed yarn correctly.\n" }, { "path": "docs/app/[[...mdxPath]]/page.jsx", "content": "import { generateStaticParamsFor, importPage } from 'nextra/pages';\n\nimport { useMDXComponents } from '../../mdx-components';\n\nexport const generateStaticParams = generateStaticParamsFor('mdxPath');\n\nexport async function generateMetadata(props) {\n const params = await props.params;\n const { metadata } = await importPage(params?.mdxPath);\n return metadata;\n}\n\nconst Wrapper = useMDXComponents().wrapper;\n\nexport default async function Page(props) {\n const params = await props.params;\n const result = await importPage(params?.mdxPath);\n const { default: MDXContent, metadata, sourceCode, toc } = result;\n\n return (\n \n \n \n );\n}\n" }, { "path": "docs/app/layout.jsx", "content": "import Image from 'next/image';\nimport { Analytics } from '@vercel/analytics/react';\nimport { Banner, Head } from 'nextra/components';\nimport { getPageMap } from 'nextra/page-map';\nimport { Footer, Layout, Navbar } from 'nextra-theme-docs';\nimport 'nextra-theme-docs/style.css';\n\nimport CuteLogo from '../public/cute-docsgpt.png';\nimport themeConfig from '../theme.config';\n\nconst github = 'https://github.com/arc53/DocsGPT';\n\nexport const metadata = {\n title: {\n default: 'DocsGPT Documentation',\n template: '%s - DocsGPT Documentation',\n },\n description:\n 'Use DocsGPT to chat with your data. DocsGPT is a GPT-powered chatbot that can answer questions about your data.',\n};\n\nconst navbar = (\n \n \"DocsGPT\n DocsGPT Docs\n \n }\n projectLink={github}\n chatLink=\"https://discord.com/invite/n5BX8dh8rU\"\n />\n);\n\nconst footer = (\n \n);\n\nexport default async function RootLayout({ children }) {\n return (\n \n \n \n \n \n \n \n \n \n \n
\n Welcome to the new DocsGPT docs!\n
\n \n }\n navbar={navbar}\n footer={footer}\n pageMap={await getPageMap()}\n {...themeConfig}\n >\n {children}\n \n \n \n \n );\n}\n" }, { "path": "docs/components/DeploymentCards.jsx", "content": "'use client';\n\nimport Image from 'next/image';\n\nconst iconMap = {\n 'Amazon Lightsail': '/lightsail.png',\n 'Railway': '/railway.png',\n 'Civo Compute Cloud': '/civo.png',\n 'DigitalOcean Droplet': '/digitalocean.png',\n 'Kamatera Cloud': '/kamatera.png',\n};\n\n\nexport function DeploymentCards({ items }) {\n return (\n <>\n
\n {items.map(({ title, link, description }) => {\n const isExternal = link.startsWith('https://');\n const iconSrc = iconMap[title] || '/default-icon.png'; // Default icon if not found\n\n return (\n \n \n
\n {iconSrc &&
{title}
} {/* Reduced icon size */}\n
\n

{title}

\n {description &&

{description}

}\n

{new URL(link).hostname.replace('www.', '')}

\n \n
\n );\n })}\n \n\n \n \n );\n}\n" }, { "path": "docs/components/ToolCards.jsx", "content": "'use client';\n\nimport Image from 'next/image';\n\nconst iconMap = {\n 'API Tool': '/toolIcons/tool_api_tool.svg',\n 'Brave Search Tool': '/toolIcons/tool_brave.svg',\n 'Cryptoprice Tool': '/toolIcons/tool_cryptoprice.svg',\n 'Ntfy Tool': '/toolIcons/tool_ntfy.svg',\n 'PostgreSQL Tool': '/toolIcons/tool_postgres.svg',\n 'Read Webpage Tool': '/toolIcons/tool_read_webpage.svg',\n 'Telegram Tool': '/toolIcons/tool_telegram.svg'\n};\n\n\nexport function ToolCards({ items }) {\n return (\n <>\n
\n {items.map(({ title, link, description }) => {\n const isExternal = link.startsWith('https://');\n const iconSrc = iconMap[title] || '/default-icon.png'; // Default icon if not found\n\n return (\n \n \n
\n {iconSrc &&
{title}
} {/* Reduced icon size */}\n
\n

{title}

\n {description &&

{description}

}\n {/* Card URL element removed from here */}\n \n
\n );\n })}\n \n\n \n \n );\n}\n" }, { "path": "docs/content/Agents/_meta.js", "content": "export default {\n \"basics\": {\n \"title\": \"🤖 Agent Basics\",\n \"href\": \"/Agents/basics\"\n },\n \"api\": {\n \"title\": \"🔌 Agent API\",\n \"href\": \"/Agents/api\"\n },\n \"webhooks\": {\n \"title\": \"🪝 Agent Webhooks\",\n \"href\": \"/Agents/webhooks\"\n },\n \"nodes\": {\n \"title\": \"🧩 Workflow Nodes\",\n \"href\": \"/Agents/nodes\"\n }\n}\n" }, { "path": "docs/content/Agents/api.mdx", "content": "---\ntitle: Interacting with Agents via API\ndescription: Learn how to programmatically interact with DocsGPT Agents using the streaming and non-streaming API endpoints.\n---\n\nimport { Callout, Tabs } from 'nextra/components';\n\n# Interacting with Agents via API\n\nDocsGPT Agents can be accessed programmatically through API endpoints. This page covers:\n\n- Non-streaming answers (`/api/answer`)\n- Streaming answers over SSE (`/stream`)\n- File/image attachments (`/api/store_attachment` + `/api/task_status` + `/stream`)\n\nWhen you use an agent `api_key`, DocsGPT loads that agent's configuration automatically (prompt, tools, sources, default model). You usually only need to send `question` and `api_key`.\n\n## Base URL\n\n\nFor DocsGPT Cloud, use `https://gptcloud.arc53.com` as the base URL.\n\n\n- Local: `http://localhost:7091`\n- Cloud: `https://gptcloud.arc53.com`\n\n## How Request Resolution Works\n\nDocsGPT resolves your request in this order:\n\n1. If `api_key` is provided, DocsGPT loads the mapped agent and executes with that config.\n2. If `agent_id` is provided (typically with JWT auth), DocsGPT loads that agent if allowed.\n3. If neither is provided, DocsGPT uses request-level fields (`prompt_id`, `active_docs`, `retriever`, etc.).\n\nAuthentication:\n\n- Agent API-key flow: include `api_key` in JSON/form payload.\n- JWT flow (if auth enabled): include `Authorization: Bearer `.\n\n## Endpoints\n\n- `POST /api/answer` (non-streaming)\n- `POST /stream` (SSE streaming)\n- `POST /api/store_attachment` (multipart upload)\n- `GET /api/task_status?task_id=...` (Celery task polling)\n\n## Request Parameters\n\nCommon request body fields:\n\n| Field | Type | Required | Applies to | Notes |\n| --- | --- | --- | --- | --- |\n| `question` | `string` | Yes | `/api/answer`, `/stream` | User query. |\n| `api_key` | `string` | Usually | `/api/answer`, `/stream` | Recommended for agent API use. Loads agent config from key. |\n| `conversation_id` | `string` | No | `/api/answer`, `/stream` | Continue an existing conversation. |\n| `history` | `string` (JSON-encoded array) | No | `/api/answer`, `/stream` | Used for new conversations. Format: `[{\\\"prompt\\\":\\\"...\\\",\\\"response\\\":\\\"...\\\"}]`. |\n| `model_id` | `string` | No | `/api/answer`, `/stream` | Override model for this request. |\n| `save_conversation` | `boolean` | No | `/api/answer`, `/stream` | Default `true`. If `false`, no conversation is persisted. |\n| `passthrough` | `object` | No | `/api/answer`, `/stream` | Dynamic values injected into prompt templates. |\n| `prompt_id` | `string` | No | `/api/answer`, `/stream` | Ignored when `api_key` already defines prompt. |\n| `active_docs` | `string` or `string[]` | No | `/api/answer`, `/stream` | Overrides active docs when not using key-owned source config. |\n| `retriever` | `string` | No | `/api/answer`, `/stream` | Retriever type (for example `classic`). |\n| `chunks` | `number` | No | `/api/answer`, `/stream` | Retrieval chunk count, default `2`. |\n| `isNoneDoc` | `boolean` | No | `/api/answer`, `/stream` | Skip document retrieval. |\n| `agent_id` | `string` | No | `/api/answer`, `/stream` | Alternative to `api_key` when using authenticated user context. |\n\nStreaming-only fields:\n\n| Field | Type | Required | Notes |\n| --- | --- | --- | --- |\n| `attachments` | `string[]` | No | List of attachment IDs from `/api/task_status` success result. |\n| `index` | `number` | No | Update an existing query index. If provided, `conversation_id` is required. |\n\n## Non-Streaming API (`/api/answer`)\n\n`/api/answer` waits for completion and returns one JSON response.\n\n\n`attachments` are currently handled through `/stream`. For file/image-attached queries, use the streaming endpoint.\n\n\nResponse fields:\n\n- `conversation_id`\n- `answer`\n- `sources`\n- `tool_calls`\n- `thought`\n- Optional structured output metadata (`structured`, `schema`) when enabled\n\n### Examples\n\n\n \n ```bash\n curl -X POST http://localhost:7091/api/answer \\\n -H \"Content-Type: application/json\" \\\n -d '{\"question\":\"your question here\",\"api_key\":\"your_agent_api_key\"}'\n ```\n \n \n ```python\n import requests\n\n API_URL = \"http://localhost:7091/api/answer\"\n API_KEY = \"your_agent_api_key\"\n QUESTION = \"your question here\"\n\n response = requests.post(\n API_URL,\n json={\"question\": QUESTION, \"api_key\": API_KEY}\n )\n\n if response.status_code == 200:\n print(response.json())\n else:\n print(f\"Error: {response.status_code}\")\n print(response.text)\n ```\n \n \n ```javascript\n const apiUrl = 'http://localhost:7091/api/answer';\n const apiKey = 'your_agent_api_key';\n const question = 'your question here';\n\n async function getAnswer() {\n try {\n const response = await fetch(apiUrl, {\n method: 'POST',\n headers: {\n 'Content-Type': 'application/json',\n },\n body: JSON.stringify({ question, api_key: apiKey }),\n });\n\n if (!response.ok) {\n throw new Error(`HTTP error! Status: ${response.status}`);\n }\n\n const data = await response.json();\n console.log(data);\n } catch (error) {\n console.error(\"Failed to fetch answer:\", error);\n }\n }\n\n getAnswer();\n ```\n \n\n\n---\n\n## Streaming API (`/stream`)\n\n`/stream` returns a Server-Sent Events (SSE) stream so you can render output token-by-token.\n\n### SSE Event Types\n\nEach `data:` frame is JSON with `type`:\n\n- `answer`: incremental answer chunk\n- `source`: source list/chunks\n- `tool_calls`: tool invocation results/metadata\n- `thought`: reasoning/thought chunk (agent dependent)\n- `structured_answer`: final structured payload (when schema mode is active)\n- `id`: final conversation ID\n- `error`: error message\n- `end`: stream is complete\n\n### Examples\n\n\n \n ```bash\n curl -X POST http://localhost:7091/stream \\\n -H \"Content-Type: application/json\" \\\n -H \"Accept: text/event-stream\" \\\n -d '{\"question\":\"your question here\",\"api_key\":\"your_agent_api_key\"}'\n ```\n \n \n ```python\n import requests\n import json\n\n API_URL = \"http://localhost:7091/stream\"\n payload = {\n \"question\": \"your question here\",\n \"api_key\": \"your_agent_api_key\"\n }\n\n with requests.post(API_URL, json=payload, stream=True) as r:\n for line in r.iter_lines():\n if line:\n decoded_line = line.decode('utf-8')\n if decoded_line.startswith('data: '):\n try:\n data = json.loads(decoded_line[6:])\n print(data)\n except json.JSONDecodeError:\n pass\n ```\n \n \n ```javascript\n const apiUrl = 'http://localhost:7091/stream';\n const apiKey = 'your_agent_api_key';\n const question = 'your question here';\n\n async function getStream() {\n try {\n const response = await fetch(apiUrl, {\n method: 'POST',\n headers: {\n 'Content-Type': 'application/json',\n 'Accept': 'text/event-stream'\n },\n body: JSON.stringify({ question, api_key: apiKey }),\n });\n\n if (!response.ok) {\n throw new Error(`HTTP error! Status: ${response.status}`);\n }\n\n const reader = response.body.getReader();\n const decoder = new TextDecoder();\n\n while (true) {\n const { done, value } = await reader.read();\n if (done) break;\n\n const chunk = decoder.decode(value, { stream: true });\n // Note: This parsing method assumes each chunk contains whole lines.\n // For a more robust production implementation, buffer the chunks\n // and process them line by line.\n const lines = chunk.split('\\n');\n \n for (const line of lines) {\n if (line.startsWith('data: ')) {\n try {\n const data = JSON.parse(line.substring(6));\n console.log(data);\n } catch (e) {\n console.error(\"Failed to parse JSON from SSE event:\", e);\n }\n }\n }\n }\n } catch (error) {\n console.error(\"Failed to fetch stream:\", error);\n }\n }\n\n getStream();\n ```\n \n\n\n---\n\n## Attachments API (Including Images)\n\nTo attach an image (or other file) to a query:\n\n1. Upload file(s) to `/api/store_attachment` (multipart/form-data).\n2. Poll `/api/task_status` until `status=SUCCESS`.\n3. Read `result.attachment_id` from task result.\n4. Send that ID in `/stream` as `attachments: [\"...\"]`.\n\n\nAttachments are processed asynchronously. Do not call `/stream` with an attachment until its task has finished with `SUCCESS`.\n\n\n### Step 1: Upload Attachment\n\n`POST /api/store_attachment`\n\n- Content type: `multipart/form-data`\n- Form fields:\n - `file` (required, can be repeated for multi-file upload)\n - `api_key` (optional if JWT is present; useful for API-key-only flows)\n\nExample upload (single image):\n\n```bash\ncurl -X POST http://localhost:7091/api/store_attachment \\\n -F \"file=@/absolute/path/to/image.png\" \\\n -F \"api_key=your_agent_api_key\"\n```\n\nPossible response (single-file upload):\n\n```json\n{\n \"success\": true,\n \"task_id\": \"34f1cb56-7c7f-4d5f-a973-4ea7e65f7a10\",\n \"message\": \"File uploaded successfully. Processing started.\"\n}\n```\n\n### Step 2: Poll Task Status\n\n```bash\ncurl \"http://localhost:7091/api/task_status?task_id=34f1cb56-7c7f-4d5f-a973-4ea7e65f7a10\"\n```\n\nWhen complete:\n\n```json\n{\n \"status\": \"SUCCESS\",\n \"result\": {\n \"attachment_id\": \"67b4f8f2618dc9f19384a9e1\",\n \"filename\": \"image.png\",\n \"mime_type\": \"image/png\"\n }\n}\n```\n\n### Step 3: Attach to `/stream` Request\n\nUse the `attachment_id` in `attachments`.\n\n```bash\ncurl -X POST http://localhost:7091/stream \\\n -H \"Content-Type: application/json\" \\\n -H \"Accept: text/event-stream\" \\\n -d '{\n \"question\": \"Describe this image\",\n \"api_key\": \"your_agent_api_key\",\n \"attachments\": [\"67b4f8f2618dc9f19384a9e1\"]\n }'\n```\n\n### Image/Attachment Behavior Notes\n\n- Typical image MIME types supported for native vision flows: `image/png`, `image/jpeg`, `image/jpg`, `image/webp`, `image/gif`.\n- If the selected model/provider does not support a file type natively, DocsGPT falls back to parsed text content.\n- For providers that support images but not native PDF file attachments, DocsGPT can convert PDF pages to images (synthetic PDF support).\n- Attachments are user-scoped. Upload and query must be done under the same user context (same API key owner or same JWT user).\n" }, { "path": "docs/content/Agents/basics.mdx", "content": "---\ntitle: Understanding DocsGPT Agents\ndescription: Learn about DocsGPT Agents, their types, how to create and manage them, and how they can enhance your interaction with documents and tools.\n---\n\nimport { Callout } from 'nextra/components';\nimport Image from 'next/image'; // Assuming you might want to embed images later, like the ones you uploaded.\n\n# Understanding DocsGPT Agents 🤖\n\nDocsGPT Agents are advanced, configurable AI entities designed to go beyond simple question-answering. They act as specialized assistants or workers that combine instructions (prompts), knowledge (document sources), and capabilities (tools) to perform a wide range of tasks, automate workflows, and provide tailored interactions.\n\nThink of an Agent as a pre-configured version of DocsGPT, fine-tuned for a specific purpose, such as classifying documents, responding to new form submissions, or validating emails.\n\n## Why Use Agents?\n\n* **Personalization:** Create AI assistants that behave and respond according to specific roles or personas.\n* **Task Specialization:** Design agents focused on particular tasks, like customer support, data extraction, or content generation.\n* **Knowledge Integration:** Equip agents with specific document sources, making them experts in particular domains.\n* **Tool Utilization:** Grant agents access to various tools, allowing them to interact with external services, fetch live data, or perform actions.\n* **Automation:** Automate repetitive tasks by defining an agent's behavior and integrating it via webhooks or other means.\n* **Shareability:** Share your custom-configured agents with others or use agents shared with you.\n\nAgents provide a more structured and powerful way to leverage LLMs compared to a standard chat interface, as they come with a pre-defined context, instruction set, and set of capabilities.\n\n## Core Components of an Agent\n\nWhen you create or configure an agent, you'll work with these key components: \n\n**Meta:**\n * **Agent Name:** A user-friendly name to identify the agent (e.g., \"Support Ticket Classifier,\" \"Product Spec Expert\").\n * **Describe your agent:** A brief description for you or users to understand the agent's purpose.\n\n**Source:**\n * **Select source:** The knowledge base for the agent. You can select from previously uploaded documents or data sources. This is what the agent will \"know.\"\n * **Chunks per query:** A numerical value determining how many relevant text chunks from the selected source are sent to the LLM with each query. This helps manage context length and relevance.\n\n**Prompt:**\nThe main set of instructions or system [prompt](/Guides/Customising-prompts) that defines the agent's persona, objectives, constraints, and how it should behave or respond.\n\n**Tools:** A selection of available [DocsGPT Tools](/Tools/basics) that the agent can use to perform actions or access external information.\n\n**Agent type:** The underlying operational logic or architecture the agent uses. DocsGPT supports different types of agents, each suited for different kinds of tasks.\n\n## Understanding Agent Types\n\nDocsGPT allows for different \"types\" of agents, each with a distinct way of processing information and generating responses. The code for these agent types can be found in the `application/agents/` directory.\n\n### 1. Classic Agent (`classic_agent.py`)\n\n**How it works:** The Classic Agent follows a traditional Retrieval Augmented Generation (RAG) approach.\n 1. **Retrieve:** When a query is made, it first searches the selected Source documents for relevant information.\n 2. **Augment:** This retrieved data is then added to the context, along with the main Prompt and the user's query.\n 3. **Generate:** The LLM generates a response based on this augmented context. It can also utilize any configured tools if the LLM decides they are necessary.\n\n**Best for:**\n * Direct question-answering over a specific set of documents.\n * Tasks where the primary goal is to extract and synthesize information from the provided sources.\n * Simpler tool integrations where the decision to use a tool is straightforward.\n\n### 2. ReAct Agent (`react_agent.py`)\n\n**How it works:** The ReAct Agent employs a more sophisticated \"Reason and Act\" framework. This involves a multi-step process:\n 1. **Plan (Thought):** Based on the query, its prompt, and available tools/sources, the LLM first generates a plan or a sequence of thoughts on how to approach the problem. You might see this output as a \"thought\" process during generation.\n 2. **Act:** The agent then executes actions based on this plan. This might involve querying its sources, using a tool, or performing internal reasoning.\n 3. **Observe:** It gathers observations from the results of its actions (e.g., data from a tool, snippets from documents).\n 4. **Repeat (if necessary):** Steps 2 and 3 can be repeated as the agent refines its approach or gathers more information.\n 5. **Conclude:** Finally, it generates the final answer based on the initial query and all accumulated observations.\n\n**Best for:**\n * More complex tasks that require multi-step reasoning or problem-solving.\n * Scenarios where the agent needs to dynamically decide which tools to use and in what order, based on intermediate results.\n * Interactive tasks where the agent needs to \"think\" through a problem.\n\n\nDevelopers looking to introduce new agent architectures can explore the `application/agents/` directory. `classic_agent.py` and `react_agent.py` serve as excellent starting points, demonstrating how to inherit from `BaseAgent` and structure agent logic.\n\n\n## Navigating and Managing Agents in DocsGPT\n\nYou can easily access and manage your agents through the DocsGPT user interface. Recently used agents appear at the top of the left sidebar for quick access. Below these, the \"Manage Agents\" button will take you to the main Agents page.\n\n### Creating a New Agent\n\n1. Navigate to the \"Agents\" page.\n2. Click the **\"New Agent\"** button.\n3. You will be presented with the \"New Agent\" configuration screen:\n\n\n\n4. Fill in the fields as described in the \"Core Components of an Agent\" section.\n5. Once configured, you can **\"Save Draft\"** to continue editing later or **\"Publish\"** to make the agent active.\n\n## Interacting with and Editing Agents\n\nOnce an agent is created, you can:\n\n* **Chat with it:** Select the agent to start an interaction.\n* **View Logs:** Access usage statistics, monitor token consumption per interaction, and review user message feedbacks. This is crucial for understanding how your agent is being used and performing.\n* **Edit an Agent:**\n * Modify any of its configuration settings (name, description, source, prompt, tools, type).\n * **Generate a Public Link:** From the edit screen, you can create a shareable public link that allows others to import and use your agent.\n * **Get a Webhook URL:** You can also obtain a Webhook URL for the agent. This allows external applications or services to trigger the agent and receive responses programmatically, enabling powerful integrations and automations.\n\n## Seeding Premade Agents from YAML\n\nYou can bootstrap a fresh DocsGPT deployment with a curated set of agents by seeding them directly into MongoDB.\n\n1. **Customize the configuration** – edit `application/seed/config/premade_agents.yaml` (or copy from `application/seed/config/agents_template.yaml`) to describe the agents you want to provision. Each entry lets you define prompts, tools, and optional data sources.\n2. **Ensure dependencies are running** – MongoDB must be reachable using the credentials in `.env`, and a Celery worker should be available if any agent sources need to be ingested via `ingest_remote`.\n3. **Execute the seeder** – run `python -m application.seed.commands init`. Add `--force` when you need to reseed an existing environment.\n\nThe seeder keeps templates under the `system` user so they appear in the UI for anyone to clone or customize. Environment variable placeholders such as `${MY_TOKEN}` inside tool configs are resolved during the seeding process.\n" }, { "path": "docs/content/Agents/nodes.mdx", "content": "# Workflow Nodes\n\nDocsGPT workflows are composed of **Nodes** that are connected to form a processing graph. These nodes interact with a **Shared State**—a global dictionary of variables that persists throughout the execution of the workflow.\n\n## The Shared State\n\nEvery workflow run maintains a state object (a JSON-like dictionary).\n- **Initial State**: Contains the user's input query (`{{query}}`) and chat history (`{{chat_history}}`).\n- **Accessing Variables**: You can access any variable in the state using the double-curly braces syntax: `{{variable_name}}`.\n- **Modifying State**: Nodes read from this state and write their outputs back to it.\n\n---\n\n## AI Agent Node\n\nThe **AI Agent Node** is the core processing unit. It uses a Large Language Model (LLM) to generate text, answer questions, or perform tasks using tools.\n\n### Inputs (Template Variables)\n\nThe primary input is the **Prompt Template**. This field supports variable substitution.\n\n- **Prompt Template**: The text sent to the model.\n - *Example*: `\"Summarize the following text: {{user_input_text}}\"`\n - If left empty, it defaults to the initial user query (`{{query}}`).\n- **System Prompt**: Instructions that define the agent's persona and constraints.\n- **Tools**: A list of tools the agent can use (e.g., search, calculator).\n- **LLM Settings**: Specific provider, model name, and parameters.\n\n### Outputs (Emissions)\n\nWhen the agent completes its task, it stores the result in the shared state.\n\n- **Output Variable**: The name of the variable where the result will be saved.\n - *Default*: If not specified, it is saved as `node_{node_id}_output`.\n - *Custom*: You can set this to something meaningful, like `summary` or `translated_text`.\n- **Streaming**: If \"Stream to user\" is enabled, the output is sent to the user in real-time as it is generated, in addition to being saved to the state.\n\n---\n\n## Set State Node\n\nThe **Set State Node** allows you to manipulate variables within the shared state directly without calling an LLM. This is useful for initialization, formatting, or control flow logic.\n\n### Operations\n\nYou can define multiple operations in a single node. Each operation targets a specific **Key** (variable name).\n\n1. **Set**: Assigns a specific value to a variable.\n - *Value*: Can be a static string or a template using variables.\n - *Example*: Set `current_step` to `1`.\n - *Example*: Set `formatted_response` to `Analysis: {{analysis_result}}`.\n\n2. **Increment**: Increases the value of a numeric variable.\n - *Value*: The amount to add (default is 1).\n - *Example*: Increment `retry_count` by `1`.\n\n3. **Append**: Adds a value to a list variable.\n - *Value*: The item to add to the list.\n - *Example*: Append `{{last_result}}` to `history_list`.\n\n### Usage Examples\n\n- **Loop Counters**: Use a *Set State* node to initialize a counter (`i = 0`) before a loop, and another to increment it inside the loop.\n- **Accumulators**: Use *Append* to collect results from multiple parallel branches into a single list.\n- **Renaming**: Copy the output of a previous node to a more generic name (e.g., set `context` to `{{search_results}}`) so subsequent nodes can use a standard variable name.\n" }, { "path": "docs/content/Agents/webhooks.mdx", "content": "---\ntitle: Triggering Agents with Webhooks\ndescription: Learn how to automate and integrate DocsGPT Agents using webhooks for asynchronous task execution.\n---\n\nimport { Callout, Tabs } from 'nextra/components';\n\n# Triggering Agents with Webhooks\n\nAgent Webhooks provide a powerful mechanism to trigger an agent's execution from external systems. Unlike the direct API which provides an immediate response, webhooks are designed for **asynchronous** operations. When you call a webhook, DocsGPT enqueues the agent's task for background processing and immediately returns a `task_id`. You then use this ID to poll for the result.\n\nThis workflow is ideal for integrating with services that expect a quick initial response (e.g., form submissions) or for triggering long-running tasks without tying up a client connection.\n\nEach agent has its own unique webhook URL, which can be generated from the agent's edit page in the DocsGPT UI. This URL includes a secure token for authentication.\n\n### API Endpoints\n\n- **Webhook URL:** `http://localhost:7091/api/webhooks/agents/{AGENT_WEBHOOK_TOKEN}`\n- **Task Status URL:** `http://localhost:7091/api/task_status`\n\n\nFor DocsGPT Cloud, use `https://gptcloud.arc53.com/` as the base URL.\n\n\nFor more technical details, you can explore the API swagger documentation available for the cloud version or your local instance.\n\n---\n\n## The Webhook Workflow\n\nThe process involves two main steps: triggering the task and polling for the result.\n\n### Step 1: Trigger the Webhook\n\nSend an HTTP `POST` request to the agent's unique webhook URL with the required payload. The structure of this payload should match what the agent's prompt and tools are designed to handle.\n\n- **Method:** `POST`\n- **Response:** A JSON object with a `task_id`. `{\"task_id\": \"a1b2c3d4-e5f6-...\"}`\n\n\n \n ```bash\n curl -X POST \\\n http://localhost:7091/api/webhooks/agents/your_webhook_token \\\n -H \"Content-Type: application/json\" \\\n -d '{\"question\": \"Your message to agent\"}'\n ```\n \n \n ```python\n import requests\n\n WEBHOOK_URL = \"http://localhost:7091/api/webhooks/agents/your_webhook_token\"\n payload = {\"question\": \"Your message to agent\"}\n\n try:\n response = requests.post(WEBHOOK_URL, json=payload)\n response.raise_for_status()\n task_id = response.json().get(\"task_id\")\n print(f\"Task successfully created with ID: {task_id}\")\n except requests.exceptions.RequestException as e:\n print(f\"Error triggering webhook: {e}\")\n ```\n \n \n ```javascript\n const webhookUrl = 'http://localhost:7091/api/webhooks/agents/your_webhook_token';\n const payload = { question: 'Your message to agent' };\n\n async function triggerWebhook() {\n try {\n const response = await fetch(webhookUrl, {\n method: 'POST',\n headers: { 'Content-Type': 'application/json' },\n body: JSON.stringify(payload)\n });\n if (!response.ok) throw new Error(`HTTP error! ${response.status}`);\n const data = await response.json();\n console.log(`Task successfully created with ID: ${data.task_id}`);\n return data.task_id;\n } catch (error) {\n console.error('Error triggering webhook:', error);\n }\n }\n \n triggerWebhook();\n ```\n \n\n\n### Step 2: Poll for the Result\n\nOnce you have the `task_id`, periodically send a `GET` request to the `/api/task_status` endpoint until the task `status` is `SUCCESS` or `FAILURE`.\n\n- **`status`**: The current state of the task (`PENDING`, `STARTED`, `SUCCESS`, `FAILURE`).\n- **`result`**: The final output from the agent, available when the status is `SUCCESS` or `FAILURE`.\n\n\n \n ```bash\n # Replace the task_id with the one you received\n curl http://localhost:7091/api/task_status?task_id=YOUR_TASK_ID\n ```\n \n \n ```python\n import requests\n import time\n\n STATUS_URL = \"http://localhost:7091/api/task_status\"\n task_id = \"YOUR_TASK_ID\"\n\n while True:\n response = requests.get(STATUS_URL, params={\"task_id\": task_id})\n data = response.json()\n status = data.get(\"status\")\n print(f\"Current task status: {status}\")\n\n if status in [\"SUCCESS\", \"FAILURE\"]:\n print(\"Final Result:\")\n print(data.get(\"result\"))\n break\n \n time.sleep(2)\n ```\n \n \n ```javascript\n const statusUrl = 'http://localhost:7091/api/task_status';\n const taskId = 'YOUR_TASK_ID';\n\n const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));\n\n async function pollForResult() {\n while (true) {\n const response = await fetch(`${statusUrl}?task_id=${taskId}`);\n const data = await response.json();\n const status = data.status;\n console.log(`Current task status: ${status}`);\n\n if (status === 'SUCCESS' || status === 'FAILURE') {\n console.log('Final Result:', data.result);\n break;\n }\n await sleep(2000);\n }\n }\n\n pollForResult();\n ```\n \n\n" }, { "path": "docs/content/Deploying/Amazon-Lightsail.mdx", "content": "---\ntitle: Hosting DocsGPT on Amazon Lightsail\ndescription:\ndisplay: hidden\n---\n\n# Self-hosting DocsGPT on Amazon Lightsail\n\nHere's a step-by-step guide on how to set up an Amazon Lightsail instance to host DocsGPT.\n\n## Configuring your instance\n\n(If you know how to create a Lightsail instance, you can skip to the recommended configuration part by clicking [here](#connecting-to-your-newly-created-instance)).\n\n### 1. Create an AWS Account: \nIf you haven't already, create or log in to your AWS account at https://lightsail.aws.amazon.com.\n\n### 2. Create an Instance: \n\na. Click \"Create Instance.\"\n\nb. Select the \"Instance location.\" In most cases, the default location works fine.\n\nc. Choose \"Linux/Unix\" as the image and \"Ubuntu 20.04 LTS\" as the Operating System.\n\nd. Configure the instance plan based on your requirements. A \"1 GB, 1vCPU, 40GB SSD, and 2TB transfer\" setup is recommended for most scenarios.\n\ne. Give your instance a unique name and click \"Create Instance.\"\n\nPS: It may take a few minutes for the instance setup to complete.\n\n### Connecting to Your newly created Instance\n\nYour instance will be ready a few minutes after creation. To access it, open the instance and click \"Connect using SSH.\"\n\n#### Clone the DocsGPT Repository\n\nA terminal window will pop up, and the first step will be to clone the DocsGPT Git repository:\n\n`git clone https://github.com/arc53/DocsGPT.git`\n\n#### Download the package information\n\nOnce it has finished cloning the repository, it is time to download the package information from all sources. To do so, simply enter the following command:\n\n`sudo apt update`\n\n#### Install Docker and Docker Compose\n\nDocsGPT backend and worker use Python, Frontend is written on React and the whole application is containerized using Docker. To install Docker and Docker Compose, enter the following commands:\n\n`sudo apt install docker.io`\n\nAnd now install docker-compose:\n\n`sudo apt install docker-compose`\n\n#### Access the DocsGPT Folder\n\nEnter the following command to access the folder in which the DocsGPT docker-compose file is present.\n\n`cd DocsGPT/`\n\n#### Prepare the Environment\n\nInside the DocsGPT folder create a `.env` file and copy the contents of `.env_sample` into it.\n\n`nano .env`\n\nMake sure your `.env` file looks like this:\n\n```\nOPENAI_API_KEY=(Your OpenAI API key)\nVITE_API_STREAMING=true\nSELF_HOSTED_MODEL=false\n```\n\nTo save the file, press CTRL+X, then Y, and then ENTER.\n\nNext, set the correct IP for the Backend by opening the docker-compose.yml file:\n\n`nano deployment/docker-compose.yaml`\n\nAnd Change line 7 to: `VITE_API_HOST=http://localhost:7091`\nto this `VITE_API_HOST=http://:7091`\n\nThis will allow the frontend to connect to the backend.\n\n#### Running the Application\n\nYou're almost there! Now that all the necessary bits and pieces have been installed, it is time to run the application. To do so, use the following command:\n\n`sudo docker compose -f deployment/docker-compose.yaml up -d`\n\nLaunching it for the first time will take a few minutes to download all the necessary dependencies and build.\n\nOnce this is done you can go ahead and close the terminal window.\n\n#### Enabling Ports \n\na. Before you are able to access your live instance, you must first enable the port that it is using.\n\nb. Open your Lightsail instance and head to \"Networking\".\n\nc. Then click on \"Add rule\" under \"IPv4 Firewall\", enter `5173` as your port, and hit \"Create\". \nRepeat the process for port `7091`.\n\n#### Access your instance\n\nYour instance is now available at your Public IP Address on port 5173. Enjoy using DocsGPT!" }, { "path": "docs/content/Deploying/Development-Environment.mdx", "content": "---\ntitle: Setting Up a Development Environment\ndescription: Guide to setting up a development environment for DocsGPT, including backend and frontend setup.\n---\n\n# Setting Up a Development Environment\n\nThis guide will walk you through setting up a development environment for DocsGPT. This setup allows you to modify and test the application's backend and frontend components.\n\n## 1. Spin Up MongoDB and Redis\n\nFor development purposes, you can quickly start MongoDB and Redis containers, which are the primary database and caching systems used by DocsGPT. We provide a dedicated Docker Compose file, `docker-compose-dev.yaml`, located in the `deployment` directory, that includes only these essential services.\n\nYou can find the `docker-compose-dev.yaml` file [here](https://github.com/arc53/DocsGPT/blob/main/deployment/docker-compose-dev.yaml).\n\n**Steps to start MongoDB and Redis:**\n\n1. Navigate to the root directory of your DocsGPT repository in your terminal.\n\n2. Run the following commands to build and start the containers defined in `docker-compose-dev.yaml`:\n\n ```bash\n docker compose -f deployment/docker-compose-dev.yaml build\n docker compose -f deployment/docker-compose-dev.yaml up -d\n ```\n\n These commands will start MongoDB and Redis in detached mode, running in the background.\n\n## 2. Run the Backend\n\nTo run the DocsGPT backend locally, you'll need to set up a Python environment and install the necessary dependencies.\n\n**Prerequisites:**\n\n* **Python 3.12:** Ensure you have Python 3.12 installed on your system. You can check your Python version by running `python --version` or `python3 --version` in your terminal.\n\n**Steps to run the backend:**\n\n1. **Configure Environment Variables:**\n\n DocsGPT backend settings are configured using environment variables. You can set these either in a `.env` file or directly in the `settings.py` file. For a comprehensive overview of all settings, please refer to the [DocsGPT Settings Guide](/Deploying/DocsGPT-Settings).\n\n * **Option 1: Using a `.env` file (Recommended):**\n * If you haven't already, create a file named `.env` in the **root directory** of your DocsGPT project.\n * Modify the `.env` file to adjust settings as needed. You can find a comprehensive list of configurable options in [`application/core/settings.py`](https://github.com/arc53/DocsGPT/blob/main/application/core/settings.py).\n\n * **Option 2: Exporting Environment Variables:**\n * Alternatively, you can export environment variables directly in your terminal. However, using a `.env` file is generally more organized for development.\n\n2. **Create a Python Virtual Environment (Optional but Recommended):**\n\n Using a virtual environment isolates project dependencies and avoids conflicts with system-wide Python packages.\n\n * **macOS and Linux:**\n\n ```bash\n python -m venv venv\n . venv/bin/activate\n ```\n\n * **Windows:**\n\n ```bash\n python -m venv venv\n venv/Scripts/activate\n ```\n\n3. **Download Embedding Model:**\n\n The backend requires an embedding model. Download the `mpnet-base-v2` model and place it in the `models/` directory within the project root. You can use the following script:\n\n ```bash\n wget https://d3dg1063dc54p9.cloudfront.net/models/embeddings/mpnet-base-v2.zip\n unzip mpnet-base-v2.zip -d model\n rm mpnet-base-v2.zip\n ```\n\n Alternatively, you can manually download the zip file from [here](https://d3dg1063dc54p9.cloudfront.net/models/embeddings/mpnet-base-v2.zip), unzip it, and place the extracted folder in `models/`.\n\n4. **Install Backend Dependencies:**\n\n Navigate to the root of your DocsGPT repository and install the required Python packages:\n\n ```bash\n pip install -r application/requirements.txt\n ```\n\n5. **Run the Flask App:**\n\n Start the Flask backend application:\n\n ```bash\n flask --app application/app.py run --host=0.0.0.0 --port=7091\n ```\n\n This command will launch the backend server, making it accessible on `http://localhost:7091`.\n\n6. **Start the Celery Worker:**\n\n Open a new terminal window (and activate your virtual environment if you used one). Start the Celery worker to handle background tasks:\n\n ```bash\n celery -A application.app.celery worker -l INFO\n ```\n\n This command will start the Celery worker, which processes tasks such as document parsing and vector embedding.\n\n **macOS note:** Due to a threading issue, start Celery with the solo pool:\n\n ```bash\n python -m celery -A application.app.celery worker -l INFO --pool=solo\n ```\n\n**Running in Debugger (VSCode):**\n\nFor easier debugging, you can launch the Flask app and Celery worker directly from VSCode's debugger.\n\n* Press Shift + Cmd + D (macOS) or Shift + Windows + D (Windows) to open the Run and Debug view.\n* You should see configurations named \"Flask\" and \"Celery\". Select the desired configuration and click the \"Start Debugging\" button (green play icon).\n\n## 3. Start the Frontend\n\nTo run the DocsGPT frontend locally, you'll need Node.js and npm (Node Package Manager).\n\n**Prerequisites:**\n\n* **Node.js version 16 or higher:** Ensure you have Node.js version 16 or greater installed. You can check your Node.js version by running `node -v` in your terminal. npm is usually bundled with Node.js.\n\n**Steps to start the frontend:**\n\n1. **Navigate to the Frontend Directory:**\n\n In your terminal, change the current directory to the `frontend` folder within your DocsGPT repository:\n\n ```bash\n cd frontend\n ```\n\n2. **Install Global Packages (If Needed):**\n\n If you don't have `husky` and `vite` installed globally, you can install them:\n\n ```bash\n npm install husky -g\n npm install vite -g\n ```\n You can skip this step if you already have these packages installed or prefer to use local installations (though global installation simplifies running the commands in this guide).\n\n3. **Install Frontend Dependencies:**\n\n Install the project's frontend dependencies using npm:\n\n ```bash\n npm install --include=dev\n ```\n\n This command reads the `package.json` file in the `frontend` directory and installs all listed dependencies, including development dependencies.\n\n4. **Run the Frontend App:**\n\n Start the frontend development server:\n\n ```bash\n npm run dev\n ```\n\n This command will start the Vite development server. The frontend application will typically be accessible at [http://localhost:5173/](http://localhost:5173/). The terminal will display the exact URL where the frontend is running.\n\nWith both the backend and frontend running, you should now have a fully functional DocsGPT development environment. You can access the application in your browser at [http://localhost:5173/](http://localhost:5173/) and start developing!\n" }, { "path": "docs/content/Deploying/Docker-Deploying.mdx", "content": "---\ntitle: Docker Deployment of DocsGPT\ndescription: Deploy DocsGPT using Docker and Docker Compose for easy setup and management.\n---\n\n# Docker Deployment of DocsGPT\n\nDocker is the recommended method for deploying DocsGPT, providing a consistent and isolated environment for the application to run. This guide will walk you through deploying DocsGPT using Docker and Docker Compose.\n\n## Prerequisites\n\n* **Docker Engine:** You need to have Docker Engine installed on your system.\n * **macOS:** [Docker Desktop for Mac](https://docs.docker.com/desktop/install/mac-install/)\n * **Linux:** [Docker Engine Installation Guide](https://docs.docker.com/engine/install/) (follow instructions for your specific distribution)\n * **Windows:** [Docker Desktop for Windows](https://docs.docker.com/desktop/install/windows-install/) (requires WSL 2 backend, see notes below)\n* **Docker Compose:** Docker Compose is usually included with Docker Desktop. If you are using Docker Engine separately, ensure you have Docker Compose V2 installed.\n\n**Important Note for Windows Users:** Docker Desktop on Windows generally requires the WSL 2 backend to function correctly, especially when using features like host networking which are utilized in DocsGPT's Docker Compose setup. Ensure WSL 2 is enabled and configured in Docker Desktop settings.\n\n## Quickest Setup: Using DocsGPT Public API\n\nThe fastest way to try out DocsGPT is by using the public API endpoint. This requires minimal configuration and no local LLM setup.\n\n1. **Clone the DocsGPT Repository (if you haven't already):**\n\n ```bash\n git clone https://github.com/arc53/DocsGPT.git\n cd DocsGPT\n ```\n\n2. **Create a `.env` file:**\n\n In the root directory of your DocsGPT repository, create a file named `.env`.\n\n3. **Add Public API Configuration to `.env`:**\n\n Open the `.env` file and add the following lines:\n\n ```\n LLM_PROVIDER=docsgpt\n VITE_API_STREAMING=true\n ```\n\n This minimal configuration tells DocsGPT to use the public API. For more advanced settings and other LLM options, refer to the [DocsGPT Settings Guide](/Deploying/DocsGPT-Settings).\n\n4. **Launch DocsGPT with Docker Compose:**\n\n Navigate to the root directory of the DocsGPT repository in your terminal and run:\n\n ```bash\n docker compose -f deployment/docker-compose.yaml up -d\n ```\n\n The `-d` flag runs Docker Compose in detached mode (in the background).\n\n5. **Access DocsGPT in your browser:**\n\n Once the containers are running, open your web browser and go to [http://localhost:5173/](http://localhost:5173/).\n\n6. **Stopping DocsGPT:**\n\n To stop the application, navigate to the same directory in your terminal and run:\n\n ```bash\n docker compose -f deployment/docker-compose.yaml down\n ```\n\n## Optional Ollama Setup (Local Models)\n\nDocsGPT provides optional Docker Compose files to easily integrate with [Ollama](https://ollama.com/) for running local models. These files add an official Ollama container to your Docker Compose setup. These files are located in the `deployment/optional/` directory.\n\nThere are two Ollama optional files:\n\n* **`docker-compose.optional.ollama-cpu.yaml`**: For running Ollama on CPU.\n* **`docker-compose.optional.ollama-gpu.yaml`**: For running Ollama on GPU (requires Docker to be configured for GPU usage).\n\n### Launching with Ollama and Pulling a Model\n\n1. **Clone the DocsGPT Repository and Create `.env` (as described above).**\n\n2. **Launch DocsGPT with Ollama Docker Compose:**\n\n Choose the appropriate Ollama Compose file (CPU or GPU) and launch DocsGPT:\n\n **CPU:**\n ```bash\n docker compose --env-file .env -f deployment/docker-compose.yaml -f deployment/optional/docker-compose.optional.ollama-cpu.yaml up -d\n ```\n **GPU:**\n ```bash\n docker compose --env-file .env -f deployment/docker-compose.yaml -f deployment/optional/docker-compose.optional.ollama-gpu.yaml up -d\n ```\n\n3. **Pull the Ollama Model:**\n\n **Crucially, after launching with Ollama, you need to pull the desired model into the Ollama container.** Find the `LLM_NAME` you configured in your `.env` file (e.g., `llama3.2:1b`). Then execute the following command to pull the model *inside* the running Ollama container:\n\n ```bash\n docker compose -f deployment/docker-compose.yaml -f deployment/optional/docker-compose.optional.ollama-cpu.yaml exec -it ollama ollama pull \n ```\n or (for GPU):\n ```bash\n docker compose -f deployment/docker-compose.yaml -f deployment/optional/docker-compose.optional.ollama-gpu.yaml exec -it ollama ollama pull \n ```\n Replace `` with the actual model name from your `.env` file.\n\n4. **Access DocsGPT in your browser:**\n\n Once the model is pulled and containers are running, open your web browser and go to [http://localhost:5173/](http://localhost:5173/).\n\n5. **Stopping Ollama Setup:**\n\n To stop a DocsGPT setup launched with Ollama optional files, use `docker compose down` and include all the compose files used during the `up` command:\n\n ```bash\n docker compose -f deployment/docker-compose.yaml -f deployment/optional/docker-compose.optional.ollama-cpu.yaml down\n ```\n or\n\n ```bash\n docker compose -f deployment/docker-compose.yaml -f deployment/optional/docker-compose.optional.ollama-gpu.yaml down\n ```\n\n**Important for GPU Usage:**\n\n* **NVIDIA Container Toolkit (for NVIDIA GPUs):** If you are using NVIDIA GPUs, you need to have the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html) installed and configured on your system for Docker to access your GPU.\n* **Docker GPU Configuration:** Ensure Docker is configured to utilize your GPU. Refer to the [Ollama Docker Hub page](https://hub.docker.com/r/ollama/ollama) and Docker documentation for GPU setup instructions specific to your GPU type (NVIDIA, AMD, Intel).\n\n## Restarting After Configuration Changes\n\nWhenever you modify the `.env` file or any Docker Compose files, you need to restart the Docker containers for the changes to be applied. Use the same `docker compose down` and `docker compose up -d` commands you used to launch DocsGPT, ensuring you include all relevant `-f` flags for optional files if you are using them.\n\n## Further Configuration\n\nThis guide covers the basic Docker deployment of DocsGPT. For detailed information on configuring various aspects of DocsGPT, such as LLM providers, models, vector stores, and more, please refer to the comprehensive [DocsGPT Settings Guide](/Deploying/DocsGPT-Settings).\n" }, { "path": "docs/content/Deploying/DocsGPT-Settings.mdx", "content": "---\ntitle: DocsGPT Settings\ndescription: Configure your DocsGPT application by understanding the basic settings.\n---\n\n# DocsGPT Settings\n\nDocsGPT is highly configurable, allowing you to tailor it to your specific needs and preferences. You can control various aspects of the application, from choosing the Large Language Model (LLM) provider to selecting embedding models and vector stores.\n\nThis document will guide you through the basic settings you can configure in DocsGPT. These settings determine how DocsGPT interacts with LLMs and processes your data.\n\n## Configuration Methods\n\nThere are two primary ways to configure DocsGPT settings:\n\n### 1. Configuration via `.env` file (Recommended)\n\nThe easiest and recommended way to configure basic settings is by using a `.env` file. This file should be located in the **root directory** of your DocsGPT project (the same directory where `setup.sh` is located).\n\n**Example `.env` file structure:**\n\n```\nLLM_PROVIDER=openai\nAPI_KEY=YOUR_OPENAI_API_KEY\nLLM_NAME=gpt-4o\n```\n\n### 2. Configuration via `settings.py` file (Advanced)\n\nFor more advanced configurations or if you prefer to manage settings directly in code, you can modify the `settings.py` file. This file is located in the `application/core` directory of your DocsGPT project.\n\nWhile modifying `settings.py` offers more flexibility, it's generally recommended to use the `.env` file for basic settings and reserve `settings.py` for more complex adjustments or when you need to configure settings programmatically.\n\n**Location of `settings.py`:** `application/core/settings.py`\n\n## Basic Settings Explained\n\nHere are some of the most fundamental settings you'll likely want to configure:\n\n- **`LLM_PROVIDER`**: This setting determines which Large Language Model (LLM) provider DocsGPT will use. It tells DocsGPT which API to interact with.\n\n - **Common values:**\n - `docsgpt`: Use the DocsGPT Public API Endpoint (simple and free, as offered in `setup.sh` option 1).\n - `openai`: Use OpenAI's API (requires an API key).\n - `google`: Use Google's Vertex AI or Gemini models.\n - `anthropic`: Use Anthropic's Claude models.\n - `groq`: Use Groq's models.\n - `huggingface`: Use HuggingFace Inference API.\n - `azure_openai`: Use Azure OpenAI Service.\n - `openai` (when using local inference engines like Ollama, Llama.cpp, TGI, etc.): This signals DocsGPT to use an OpenAI-compatible API format, even if the actual LLM is running locally.\n\n- **`LLM_NAME`**: Specifies the specific model to use from the chosen LLM provider. The available models depend on the `LLM_PROVIDER` you've selected.\n\n - **Examples:**\n - For `LLM_PROVIDER=openai`: `gpt-4o`\n - For `LLM_PROVIDER=google`: `gemini-2.0-flash`\n - For local models (e.g., Ollama): `llama3.2:1b` (or any model name available in your setup).\n\n- **`EMBEDDINGS_NAME`**: This setting defines which embedding model DocsGPT will use to generate vector embeddings for your documents. Embeddings are numerical representations of text that allow DocsGPT to understand the semantic meaning of your documents for efficient search and retrieval.\n\n - **Default value:** `huggingface_sentence-transformers/all-mpnet-base-v2` (a good general-purpose embedding model).\n - **Other options:** You can explore other embedding models from Hugging Face Sentence Transformers or other providers if needed.\n\n- **`API_KEY`**: Required for most cloud-based LLM providers. This is your authentication key to access the LLM provider's API. You'll need to obtain this key from your chosen provider's platform.\n\n- **`OPENAI_BASE_URL`**: Specifically used when `LLM_PROVIDER` is set to `openai` but you are connecting to a local inference engine (like Ollama, Llama.cpp, etc.) that exposes an OpenAI-compatible API. This setting tells DocsGPT where to find your local LLM server.\n\n- **`STT_PROVIDER`**: Selects the speech-to-text provider used for microphone transcription in chat and for audio file ingestion through the parser pipeline.\n\n## Configuration Examples\n\nLet's look at some concrete examples of how to configure these settings in your `.env` file.\n\n### Example for Cloud API Provider (OpenAI)\n\nTo use OpenAI's `gpt-4o` model, you would configure your `.env` file like this:\n\n```\nLLM_PROVIDER=openai\nAPI_KEY=YOUR_OPENAI_API_KEY # Replace with your actual OpenAI API key\nLLM_NAME=gpt-4o\n```\n\nMake sure to replace `YOUR_OPENAI_API_KEY` with your actual OpenAI API key.\n\n### Example for Local Deployment\n\nTo use a local Ollama server with the `llama3.2:1b` model, you would configure your `.env` file like this:\n\n```\nLLM_PROVIDER=openai # Using OpenAI compatible API format for local models\nAPI_KEY=None # API Key is not needed for local Ollama\nLLM_NAME=llama3.2:1b\nOPENAI_BASE_URL=http://host.docker.internal:11434/v1 # Default Ollama API URL within Docker\nEMBEDDINGS_NAME=huggingface_sentence-transformers/all-mpnet-base-v2 # You can also run embeddings locally if needed\n```\n\nIn this case, even though you are using Ollama locally, `LLM_PROVIDER` is set to `openai` because Ollama (and many other local inference engines) are designed to be API-compatible with OpenAI. `OPENAI_BASE_URL` points DocsGPT to the local Ollama server.\n\n## Speech-to-Text Settings\n\nDocsGPT can transcribe audio in two places:\n\n- Voice input in the chat.\n- Audio file ingestion. Uploaded `.wav`, `.mp3`, `.m4a`, `.ogg`, and `.webm` files are transcribed first and then passed through the normal parser, chunking, embedding, and indexing pipeline.\n\nFor an end-to-end walkthrough, see the [Speech and Audio Guide](/Guides/speech-and-audio).\n\n| Setting | Purpose | Typical values |\n| --- | --- | --- |\n| `STT_PROVIDER` | Speech-to-text backend provider. | `openai`, `faster_whisper` |\n| `OPENAI_STT_MODEL` | OpenAI transcription model used when `STT_PROVIDER=openai`. | `gpt-4o-mini-transcribe` |\n| `STT_LANGUAGE` | Optional language hint passed to the provider. Leave unset for auto-detection when supported. | `en`, `es`, unset |\n| `STT_MAX_FILE_SIZE_MB` | Maximum file size accepted by the synchronous `/api/stt` endpoint. | `50` |\n| `STT_ENABLE_TIMESTAMPS` | Include timestamp segments in the normalized transcript response and stored parser metadata. | `true`, `false` |\n| `STT_ENABLE_DIARIZATION` | Reserved provider option for speaker diarization. Some providers may ignore it. | `true`, `false` |\n\n### Example: OpenAI Speech-to-Text\n\n```env\nSTT_PROVIDER=openai\nOPENAI_API_KEY=YOUR_OPENAI_API_KEY\nOPENAI_STT_MODEL=gpt-4o-mini-transcribe\nSTT_LANGUAGE=\nSTT_MAX_FILE_SIZE_MB=50\nSTT_ENABLE_TIMESTAMPS=false\nSTT_ENABLE_DIARIZATION=false\n```\n\nIf you already use `API_KEY` for OpenAI, DocsGPT can reuse that key for transcription. Set `OPENAI_API_KEY` only when you want a dedicated key.\n\n### Example: Local `faster_whisper`\n\n```env\nSTT_PROVIDER=faster_whisper\nSTT_LANGUAGE=en\nSTT_ENABLE_TIMESTAMPS=true\nSTT_ENABLE_DIARIZATION=false\n```\n\n`faster_whisper` is an optional backend dependency. Install it in the Python environment used by the DocsGPT API and worker before selecting this provider.\n\n## Authentication Settings\n\nDocsGPT includes a JWT (JSON Web Token) based authentication feature for managing sessions or securing local deployments while allowing access.\n\n### `AUTH_TYPE` Overview\n\nThe `AUTH_TYPE` setting in your `.env` file or `settings.py` determines the authentication method used by DocsGPT. This allows you to control how users authenticate with your DocsGPT instance.\n\n| Value | Description |\n| ------------- | ------------------------------------------------------------------------------------------- |\n| `None` | No authentication is used. Anyone can access the app. |\n| `simple_jwt` | A single, long-lived JWT token is generated at startup. All requests use this shared token. |\n| `session_jwt` | Unique JWT tokens are generated for each session/user. |\n\n#### How to Configure\n\nAdd the following to your `.env` file (or set in `settings.py`):\n\n```env\n# No authentication (default)\nAUTH_TYPE=None\n\n# OR: Simple JWT (shared token)\nAUTH_TYPE=simple_jwt\nJWT_SECRET_KEY=your_secret_key_here\n\n# OR: Session JWT (per-user/session tokens)\nAUTH_TYPE=session_jwt\nJWT_SECRET_KEY=your_secret_key_here\n```\n\n- If `AUTH_TYPE` is set to `simple_jwt` or `session_jwt`, a `JWT_SECRET_KEY` is required.\n- If `JWT_SECRET_KEY` is not set, DocsGPT will generate one and store it in `.jwt_secret_key` in the project root.\n\n#### How Each Method Works\n\n- **None**: No authentication. All API and UI access is open.\n- **simple_jwt**:\n - A single JWT token is generated at startup and printed to the console.\n - Use this token in the `Authorization` header for all API requests:\n ```http\n Authorization: Bearer \n ```\n - The frontend will prompt for this token if not already set.\n- **session_jwt**:\n - Clients can request a new token from `/api/generate_token`.\n - Use the received token in the `Authorization` header for subsequent requests.\n - Each user/session gets a unique token.\n\n#### Security Notes\n\n- Always keep your `JWT_SECRET_KEY` secure and private.\n- If you set it manually, use a strong, random string.\n- If not set, DocsGPT will generate a secure key and persist it in `.jwt_secret_key`.\n\n#### Checking Current Auth Type\n\n- Use the `/api/config` endpoint to check the current `auth_type` and whether authentication is required.\n\n#### Frontend Token Input for `simple_jwt`\n\nIf you have configured `AUTH_TYPE=simple_jwt`, the DocsGPT frontend will prompt you to enter the JWT token if it's not already set or is invalid. Paste the `SIMPLE_JWT_TOKEN` (printed to your console when the backend starts) into this field to access the application.\n\n\n\n## Exploring More Settings\n\nThese are just the basic settings to get you started. The `settings.py` file contains many more advanced options that you can explore to further customize DocsGPT, such as:\n\n- Vector store configuration (`VECTOR_STORE`, Qdrant, Milvus, LanceDB settings) If you're looking for an easy way to set up a vector store with pgvector, try [Neon](https://get.neon.com/docsgpt).\n- Retriever settings (`RETRIEVERS_ENABLED`)\n- Cache settings (`CACHE_REDIS_URL`)\n- And many more!\n\nFor a complete list of available settings and their descriptions, refer to the `settings.py` file in `application/core`. Remember to restart your Docker containers after making changes to your `.env` file or `settings.py` for the changes to take effect.\n" }, { "path": "docs/content/Deploying/Hosting-the-app.mdx", "content": "import { DeploymentCards } from '../../components/DeploymentCards';\n\n# Deployment Guides\n\n\n" }, { "path": "docs/content/Deploying/Kubernetes-Deploying.mdx", "content": "---\ntitle: Deploying DocsGPT on Kubernetes\ndescription: Learn how to self-host DocsGPT on a Kubernetes cluster for scalable and robust deployments.\n---\n\n# Self-hosting DocsGPT\n on Kubernetes\n\nThis guide will walk you through deploying DocsGPT on Kubernetes.\n\n## Prerequisites\n\nEnsure you have the following installed before proceeding:\n\n- [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/)\n- Access to a Kubernetes cluster.\n- [Neon](https://get.neon.com/docsgpt) (optional) for a quick and easy vector store setup with pgvector.\n\n## Folder Structure\n\nThe `deployment/k8s` folder contains the necessary deployment and service configuration files:\n\n- `deployments/`\n- `services/`\n- `docsgpt-secrets.yaml`\n\n## Deployment Instructions\n\n1. **Clone the Repository**\n\n ```sh\n git clone https://github.com/arc53/DocsGPT.git\n cd docsgpt/deployment/k8s\n ```\n\n2. **Configure Secrets (optional)**\n\n Ensure that you have all the necessary secrets in `docsgpt-secrets.yaml`. Update it with your secrets before applying if you want. By default we will use qdrant as a vectorstore and public docsgpt llm as llm for inference.\n \n Alternatively, you can use [Neon](https://get.neon.com/docsgpt) as an easy way to set up your vector store with pgvector, which is highly recommended for quick deployments.\n\n3. **Apply Kubernetes Deployments**\n\n Deploy your DocsGPT resources using the following commands:\n\n ```sh\n kubectl apply -f deployments/\n ```\n\n4. **Apply Kubernetes Services**\n\n Set up your services using the following commands:\n\n ```sh\n kubectl apply -f services/\n ```\n\n5. **Apply Secrets**\n\n Apply the secret configurations:\n\n ```sh\n kubectl apply -f docsgpt-secrets.yaml\n ```\n\n6. **Substitute API URL**\n\n After deploying the services, you need to update the environment variable `VITE_API_HOST` in your deployment file `deployments/docsgpt-deploy.yaml` with the actual endpoint URL created by your `docsgpt-api-service`.\n\n ```sh\n kubectl get services/docsgpt-api-service -o jsonpath='{.status.loadBalancer.ingress[0].ip}' | xargs -I {} sed -i \"s||{}|g\" deployments/docsgpt-deploy.yaml\n ```\n\n7. **Rerun Deployment**\n\n After making the changes, reapply the deployment configuration to update the environment variables:\n\n ```sh\n kubectl apply -f deployments/\n ```\n\n## Verifying the Deployment\n\nTo verify if everything is set up correctly, you can run the following:\n\n```sh\nkubectl get pods\nkubectl get services\n```\n\nEnsure that the pods are running and the services are available.\n\n## Accessing DocsGPT\n\nTo access DocsGPT, you need to find the external IP address of the frontend service. You can do this by running:\n\n```sh\nkubectl get services/docsgpt-frontend-service | awk 'NR>1 {print \"http://\" $4}'\n```\n\n## Troubleshooting\n\nIf you encounter any issues, you can check the logs of the pods for more details:\n\n```sh\nkubectl logs \n```\n\nReplace `` with the actual name of your DocsGPT pod." }, { "path": "docs/content/Deploying/Railway.mdx", "content": "---\ntitle: Hosting DocsGPT on Railway\ndescription: Learn how to deploy your own DocsGPT instance on Railway with this step-by-step tutorial\n---\n\n# Self-hosting DocsGPT on Railway\n\n \n\nHere's a step-by-step guide on how to host DocsGPT on Railway App.\n\n \n\nAt first Clone and set up the project locally to run , test and Modify.\n\n \n\n### 1. Clone and GitHub SetUp\n\na. Open Terminal (Windows Shell or Git bash(recommended)).\n\n \n\nb. Type `git clone https://github.com/arc53/DocsGPT.git`\n\n \n\n#### Download the package information\n\n \n\nOnce it has finished cloning the repository, it is time to download the package information from all sources. To do so, simply enter the following command:\n\n \n\n`sudo apt update`\n\n \n\n#### Install Docker and Docker Compose\n\n \n\nDocsGPT backend and worker use Python, Frontend is written on React and the whole application is containerized using Docker. To install Docker and Docker Compose, enter the following commands:\n\n \n\n`sudo apt install docker.io`\n\n \n\nAnd now install docker-compose:\n\n \n\n`sudo apt install docker-compose`\n\n \n\n#### Access the DocsGPT Folder\n\n \n\nEnter the following command to access the folder in which the DocsGPT docker-compose file is present.\n\n \n\n`cd DocsGPT/`\n\n \n\n#### Prepare the Environment\n\n \n\nInside the DocsGPT folder create a `.env` file and copy the contents of `.env_sample` into it.\n\n \n\n`nano .env`\n\n \n\nMake sure your `.env` file looks like this:\n\n \n\n```\n\nOPENAI_API_KEY=(Your OpenAI API key)\n\nVITE_API_STREAMING=true\n\nSELF_HOSTED_MODEL=false\n\n```\n\n \n\nTo save the file, press CTRL+X, then Y, and then ENTER.\n\n \n\nNext, set the correct IP for the Backend by opening the docker-compose.yaml file:\n\n \n\n`nano deployment/docker-compose.yaml`\n\n \n\nAnd Change line 7 to: `VITE_API_HOST=http://localhost:7091`\n\nto this `VITE_API_HOST=http://:7091`\n\n \n\nThis will allow the frontend to connect to the backend.\n\n \n\n#### Running the Application\n\n \n\nYou're almost there! Now that all the necessary bits and pieces have been installed, it is time to run the application. To do so, use the following command:\n\n \n\n`sudo docker compose -f deployment/docker-compose.yaml up -d`\n\n \n\nLaunching it for the first time will take a few minutes to download all the necessary dependencies and build.\n\n \n\nOnce this is done you can go ahead and close the terminal window.\n\n \n\n### 2. Pushing it to your own Repository\n\n \n\na. Create a Repository on your GitHub.\n\n \n\nb. Open Terminal in the same directory of the Cloned project.\n\n \n\nc. Type `git init`\n\n \n\nd. `git add .`\n\n \n\ne. `git commit -m \"first-commit\"`\n\n \n\nf. `git remote add origin `\n\n \n\ng. `git push git push --set-upstream origin master`\n\nYour local files will now be pushed to your GitHub Account. :)\n \n\n### 3. Create a Railway Account:\n\n \n\nIf you haven't already, create or log in to your railway account do it by visiting [Railway](https://railway.app/)\n\n \n\nSignup via **GitHub** [Recommended].\n\n \n\n### 4. Start New Project:\n\n \n\na. Open Railway app and Click on \"Start New Project.\"\n\n \n\nb. Choose any from the list of options available (Recommended \"**Deploy from GitHub Repo**\")\n\n \n\nc. Choose the required Repository from your GitHub.\n\n \n\nd. Configure and allow access to modify your GitHub content from the pop-up window.\n\n \n\ne. Agree to all the terms and conditions.\n\n \n\nPS: It may take a few minutes for the account setup to complete.\n\n \n\n#### You will get A free trial of $5 (use it for trial and then purchase if satisfied and needed)\n\n \n\n### 5. Connecting to Your newly Railway app with GitHub\n\n \n\na. Choose DocsGPT repo from the list of your GitHub repository that you want to deploy now.\n\n \n\nb. Click on Deploy now.\n\n \n\n![Three Tabs will be there](/Railway-selection.png)\n\n \n\nc. Select Variables Tab.\n\n \n\nd. Upload the env file here that you used for local setup.\n\n \n\ne. Go to Settings Tab now.\n\n \n\nf. Go to \"Networking\" and click on Generate Domain Name, to get the URL of your hosted project.\n\n \n\ng. You can update the Root directory, build command, installation command as per need.\n\n*[However recommended not the disturb these options and leave them as default if not that needed.]*\n\n \n \n\nYour own DocsGPT is now available at the Generated domain URl. :)\n" }, { "path": "docs/content/Deploying/_meta.js", "content": "export default {\n \"DocsGPT-Settings\": {\n \"title\": \"⚙️ App Configuration\",\n \"href\": \"/Deploying/DocsGPT-Settings\"\n },\n \"Docker-Deploying\": {\n \"title\": \"🛳️ Docker Setup\",\n \"href\": \"/Deploying/Docker-Deploying\"\n },\n \"Development-Environment\": {\n \"title\": \"🛠️Development Environment\",\n \"href\": \"/Deploying/Development-Environment\"\n },\n \"Kubernetes-Deploying\": {\n \"title\": \"☸️ Deploying on Kubernetes\",\n \"href\": \"/Deploying/Kubernetes-Deploying\"\n },\n \"Hosting-the-app\": {\n \"title\": \"☁️ Hosting DocsGPT\",\n \"href\": \"/Deploying/Hosting-the-app\"\n },\n \"Amazon-Lightsail\": {\n \"title\": \"Hosting DocsGPT on Amazon Lightsail\",\n \"href\": \"/Deploying/Amazon-Lightsail\",\n \"display\": \"hidden\"\n },\n \"Railway\": {\n \"title\": \"Hosting DocsGPT on Railway\",\n \"href\": \"/Deploying/Railway\",\n \"display\": \"hidden\"\n }\n}\n" }, { "path": "docs/content/Extensions/Chatwoot-extension.mdx", "content": "---\ntitle: Comprehensive Guide to Setting Up the Chatwoot Extension with DocsGPT\ndescription: This step-by-step guide walks you through the process of setting up the Chatwoot extension with DocsGPT, enabling seamless integration for automated responses and enhanced customer support. Learn how to launch DocsGPT, retrieve your Chatwoot access token, configure the .env file, and start the extension.\n---\n## Chatwoot Extension Setup Guide\n\n### Step 1: Prepare and Start DocsGPT\n\n- **Launch DocsGPT**: Follow the instructions in our [Quickstart](/quickstart) to start DocsGPT. Make sure to load your documentation.\n\n### Step 2: Get Access Token from Chatwoot\n\n- Go to Chatwoot.\n- In your profile settings (located at the bottom left), scroll down and copy the **Access Token**.\n\n### Step 3: Set Up Chatwoot Extension\n\n- Navigate to `/extensions/chatwoot`.\n- Copy the `.env_sample` file and create a new file named `.env`.\n- Fill in the values in the `.env` file as follows:\n\n```env\ndocsgpt_url=\nchatwoot_url=\ndocsgpt_key=\nchatwoot_token=\n```\n\n### Step 4: Start the Extension\n\n- Use the command `flask run` to start the extension.\n\n### Step 5: Optional - Extra Validation\n\n- In app.py, uncomment lines 12-13 and 71-75.\n- Add the following lines to your .env file:\n```account_id=(optional) 1\nassignee_id=(optional) 1\n```\nThese Chatwoot values help ensure you respond to the correct widget and handle questions assigned to a specific user.\n\n### Stopping Bot Responses for Specific User or Session\n\n- If you want the bot to stop responding to questions for a specific user or session, add a label `human-requested` in your conversation.\n\n### Additional Notes\n\n- For further details on training on other documentation, refer to our [wiki](https://github.com/arc53/DocsGPT/wiki/How-to-train-on-other-documentation)." }, { "path": "docs/content/Extensions/Chrome-extension.mdx", "content": "---\ntitle: Add DocsGPT Chrome Extension to Your Browser\ndescription: Install the DocsGPT Chrome extension to access AI-powered document assistance directly from your browser for enhanced productivity.\n---\n\nimport {Steps} from 'nextra/components'\nimport { Callout } from 'nextra/components'\n\n\n## Chrome Extension Setup Guide\n\nTo enhance your DocsGPT experience, you can install the DocsGPT Chrome extension. Here's how:\n\n### Step 1\n\n\n\nIn the DocsGPT GitHub repository, click on the **Code** button and select **Download ZIP**.\n### Step 2\nUnzip the downloaded file to a location you can easily access.\n### Step 3\nOpen the Google Chrome browser and click on the three dots menu (upper right corner).\n### Step 4\nSelect **More Tools** and then **Extensions**.\n### Step 5\nTurn on the **Developer mode** switch in the top right corner of the **Extensions page**.\n### Step 6\nClick on the **Load unpacked** button.\n### Step 7\n7. Select the **Chrome** folder where the DocsGPT files have been unzipped (docsgpt-main > extensions > chrome).\n### Step 8\nThe extension should now be added to Google Chrome and can be managed on the Extensions page.\n### Step 9\n To disable or remove the extension, simply turn off the toggle switch on the extension card or click the **Remove** button.\n\n\n\n\n" }, { "path": "docs/content/Extensions/_meta.js", "content": "export default {\n \"api-key-guide\": {\n \"title\": \"🔑 Getting API key\",\n \"href\": \"/Extensions/api-key-guide\"\n },\n \"chat-widget\": {\n \"title\": \"💬️ Chat Widget\",\n \"href\": \"/Extensions/chat-widget\"\n },\n \"search-widget\": {\n \"title\": \"🔎 Search Widget\",\n \"href\": \"/Extensions/search-widget\"\n },\n \"Chrome-extension\": {\n \"title\": \"🌐 Chrome Extension\",\n \"href\": \"/Extensions/Chrome-extension\"\n },\n \"Chatwoot-extension\": {\n \"title\": \"🗣️ Chatwoot Extension\",\n \"href\": \"/Extensions/Chatwoot-extension\"\n }\n}\n" }, { "path": "docs/content/Extensions/api-key-guide.mdx", "content": "---\ntitle: API Keys for DocsGPT Integrations\ndescription: Learn how to obtain, understand, and use DocsGPT API keys to integrate DocsGPT into your external applications and widgets.\n---\n\n# Guide to DocsGPT API Keys\n\nDocsGPT API keys are essential for developers and users who wish to integrate the DocsGPT models into external applications, such as [our widget](/Extensions/chat-widget). This guide will walk you through the steps of obtaining an API key, starting from uploading your document to understanding the key variables associated with API keys.\n\n## Obtaining Your API Key\n\nAfter uploading your document, you can obtain an API key either through the graphical user interface or via an API call:\n\n- **Graphical User Interface:** Navigate to the Settings section of the DocsGPT web app, find the API Keys option, and press 'Create New' to generate your key.\n- **API Call:** Alternatively, you can use the `/api/create_api_key` endpoint to create a new API key. For detailed instructions, visit [DocsGPT API Documentation](https://gptcloud.arc53.com/).\n\n## Understanding Key Variables\n\nUpon creating your API key, you will encounter several key variables. Each serves a specific purpose:\n\n- **Name:** Assign a name to your API key for easy identification.\n- **Source:** Indicates the source document(s) linked to your API key, which DocsGPT will use to generate responses.\n- **ID:** A unique identifier for your API key. You can view this by making a call to `/api/get_api_keys`.\n- **Key:** The API key itself, which will be used in your application to authenticate API requests.\n\nWith your API key ready, you can now integrate DocsGPT into your application, such as the DocsGPT Widget or any other software, via `/api/answer` or `/stream` endpoints. The source document is preset with the API key, allowing you to bypass fields like `selectDocs` and `active_docs` during implementation.\n\nCongratulations on taking the first step towards enhancing your applications with DocsGPT!\n" }, { "path": "docs/content/Extensions/chat-widget.mdx", "content": "---\ntitle: Integrate DocsGPT Chat Widget into Your Web Application\ndescription: Embed the DocsGPT Widget in your React, HTML, or Nextra projects to provide AI-powered chat functionality to your users.\n---\nimport { Tabs } from 'nextra/components'\n\n# Integrating DocsGPT Chat Widget\n\n## Introduction\n\nThe DocsGPT Widget is a powerful tool that allows you to integrate AI-driven document assistance directly into your web applications. This guide will walk you through embedding the DocsGPT Widget into your projects, whether you're using React, plain HTML, or Nextra. Enhance your user experience by providing seamless access to intelligent document search and chatbot capabilities.\n\nTry out the interactive widget showcase and customize its parameters at the [DocsGPT Widget Demo](https://widget.docsgpt.cloud/).\n\n## Setup\n\n \n\n### Installation\n\nMake sure you have Node.js and npm (or yarn, pnpm) installed in your project. Navigate to your project directory in the terminal and install the `docsgpt` package:\n\n```bash npm\nnpm install docsgpt\n```\n\n### Usage\n\nIn your React component file, import the `DocsGPTWidget` component:\n\n```js\nimport { DocsGPTWidget } from \"docsgpt\";\n```\n\nNow, you can embed the widget within your React component's JSX:\n\n```jsx\n\n```\n \n \n\n### Installation\n\nTo use the DocsGPT Widget directly in HTML, include the widget script from a CDN in your HTML file:\n\n```html filename=\"html\"\n\n```\n\n### Usage\n\nIn your HTML ``, add a `
` element where you want to render the widget. Set an `id` for easy targeting.\n\n```html filename=\"html\"\n
\n```\n\nThen, in a `\n```\n\n \n \n\n### Installation\n\nMake sure you have Node.js and npm (or yarn, pnpm) installed in your project. Navigate to your project directory in the terminal and install the `docsgpt` package:\n\n```bash npm\nnpm install docsgpt\n```\n\n### Usage with Nextra (Next.js + MDX)\n\nTo integrate the DocsGPT Widget into a [Nextra](https://nextra.site/) documentation site (built with Next.js and MDX), create or modify your `pages/_app.js` file as follows:\n\n```js filename=\"pages/_app.js\"\nimport { DocsGPTWidget } from \"docsgpt\";\n\nexport default function MyApp({ Component, pageProps }) {\n return (\n <>\n \n \n \n )\n}\n```\n \n\n\n---\n\n## Properties Table\n\nThe DocsGPT Widget offers a range of customizable properties that allow you to tailor its appearance and behavior to perfectly match your web application. These parameters can be modified directly when embedding the widget in your React components or HTML code. Below is a detailed overview of each available prop:\n\n| **Prop** | **Type** | **Default Value** | **Description** |\n|--------------------|------------------|-------------------------------------------------------------|-----------------------------------------------------------------------------------------------------|\n| **`apiHost`** | `string` | `\"https://gptcloud.arc53.com\"` | **Required.** The URL of your DocsGPT API backend. This endpoint handles vector search and chatbot queries. |\n| **`apiKey`** | `string` | `\"your-api-key\"` | API key for authentication with your DocsGPT API. Leave empty if no authentication is required. |\n| **`avatar`** | `string` | [`dino-icon-link`](https://d3dg1063dc54p9.cloudfront.net/cute-docsgpt.png) | URL for the avatar image displayed in the chatbot interface. |\n| **`title`** | `string` | `\"Get AI assistance\"` | Title text shown in the chatbot header. |\n| **`description`** | `string` | `\"DocsGPT's AI Chatbot is here to help\"` | Sub-title or descriptive text displayed below the title in the chatbot header. |\n| **`heroTitle`** | `string` | `\"Welcome to DocsGPT !\"` | Welcome message displayed when the chatbot is initially opened. |\n| **`heroDescription`** | `string` | `\"This chatbot is built with DocsGPT and utilises GenAI, please review important information using sources.\"` | Introductory text providing context or disclaimers about the chatbot. |\n| **`theme`** | `\"dark\" \\| \"light\"` | `\"dark\"` | Color theme of the widget interface. Options: `\"dark\"` or `\"light\"`. Defaults to `\"dark\"`. |\n| **`buttonIcon`** | `string` | `\"https://your-icon\"` | URL for the icon image used in the widget's launch button. |\n| **`buttonBg`** | `string` | `\"#222327\"` | Background color of the widget's launch button. |\n| **`size`** | `\"small\" \\| \"medium\"` | `\"medium\"` | Size of the widget. Options: `\"small\"` or `\"medium\"`. Defaults to `\"medium\"`. |\n| **`showSources`** | `boolean` | `false` | Enables displaying source URLs for data fetched within the widget. When set to `true`, the widget will show the original sources of the fetched data. |\n\n---\n\n## Notes on Widget Properties\n\n* **Full Customization:** Every property listed in the table can be customized. Override the defaults to create a widget that perfectly matches your branding and application context. From avatars and titles to color schemes, you have fine-grained control over the widget's presentation.\n* **API Key Handling:** The `apiKey` prop is optional. Only include it if your DocsGPT backend API is configured to require API key authentication. `apiHost` for DocsGPT Cloud is `https://gptcloud.arc53.com/`\n\n## Explore and Customize Further\n\nThe DocsGPT Widget is fully open-source, allowing for deep customization and extension beyond the readily available props.\n\nThe complete source code for the React-based widget is available in the `extensions/react-widget` directory within the main [DocsGPT GitHub Repository](https://github.com/arc53/DocsGPT). Feel free to explore the code, fork the repository, and tailor the widget to your exact requirements." }, { "path": "docs/content/Extensions/search-widget.mdx", "content": "---\ntitle: Integrate DocsGPT Search Bar into Your Web Application\ndescription: Embed the DocsGPT Search Bar Widget in your React or HTML projects to provide AI-powered document search functionality to your users.\n---\nimport { Tabs } from 'nextra/components'\n\n# Integrating DocsGPT Search Bar Widget\n\n## Introduction\n\nThe DocsGPT Search Bar Widget offers a simple yet powerful way to embed AI-powered document search directly into your web applications. This widget allows users to perform searches across your documents or pages, enabling them to quickly find the information they need. This guide will walk you through embedding the Search Bar Widget into your projects, whether you're using React or plain HTML.\n\nTry out the interactive widget showcase and customize its parameters at the [DocsGPT Widget Demo](https://widget.docsgpt.cloud/).\n\n## Setup\n\n\n \n## React Setup\n\n### Installation\n\nMake sure you have Node.js and npm (or yarn, pnpm) installed in your project. Navigate to your project directory in the terminal and install the `docsgpt` package:\n\n```bash npm\nnpm install docsgpt\n```\n\n### Usage\n\nIn your React component file, import the `SearchBar` component:\n\n```js\nimport { SearchBar } from \"docsgpt\";\n```\n\nNow, you can embed the widget within your React component's JSX:\n\n```jsx\n\n```\n \n \n\n### Installation\n\nTo use the DocsGPT Search Bar Widget directly in HTML, include the widget script from a CDN in your HTML file:\n\n```html filename=\"html\"\n\n```\n\n### Usage\n\nIn your HTML ``, add a `
` element where you want to render the Search Bar Widget. Set an `id` for easy targeting.\n\n```html filename=\"html\"\n
\n```\n\nThen, in a `\n```\n\n \n\n\n---\n\n## Properties Table\n\nThe DocsGPT Search Bar Widget offers a range of customizable properties that allow you to tailor its appearance and behavior to perfectly match your web application. These parameters can be modified directly when embedding the widget in your React components or HTML code. Below is a detailed overview of each available prop:\n\n| **Prop** | **Type** | **Default Value** | **Description** |\n|-----------------|-----------|-------------------------------------|--------------------------------------------------------------------------------------------------|\n| **`apiKey`** | `string` | `\"your-api-key\"` | API key for authentication with your DocsGPT API. Leave empty if no authentication is required. |\n| **`apiHost`** | `string` | `\"https://gptcloud.arc53.com\"` | **Required.** The URL of your DocsGPT API backend. This endpoint handles vector similarity search queries. |\n| **`theme`** | `\"dark\" \\| \"light\"` | `\"dark\"` | Color theme of the search bar. Options: `\"dark\"` or `\"light\"`. Defaults to `\"dark\"`. |\n| **`placeholder`** | `string` | `\"Search or Ask AI...\"` | Placeholder text displayed in the search input field. |\n| **`width`** | `string` | `\"256px\"` | Width of the search bar. Accepts any valid CSS width value (e.g., `\"300px\"`, `\"100%\"`, `\"20rem\"`). |\n\n---\n\n## Notes on Widget Properties\n\n* **Full Customization:** Every property listed in the table can be customized. Override the defaults to create a Search Bar Widget that perfectly matches your branding and application context.\n* **API Key Handling:** The `apiKey` prop is optional. Only include it if your DocsGPT backend API is configured to require API key authentication. `apiHost` for DocsGPT Cloud is `https://gptcloud.arc53.com/`\n\n## Explore and Customize Further\n\nThe DocsGPT Search Bar Widget is fully open-source, allowing for deep customization and extension beyond the readily available props.\n\nThe complete source code for the React-based widget is available in the `extensions/react-widget` directory within the main [DocsGPT GitHub Repository](https://github.com/arc53/DocsGPT). Feel free to explore the code, fork the repository, and tailor the widget to your exact requirements." }, { "path": "docs/content/Guides/Architecture.mdx", "content": "---\ntitle: Architecture\ndescription: High-level architecture of DocsGPT\n---\n\n## Introduction\n\nDocsGPT is designed as a modular and scalable application for knowledge based GenAI system. This document outlines the high-level architecture of DocsGPT, highlighting its key components.\n\n## High-Level Architecture\n\nThis diagram provides a bird's-eye view of the DocsGPT architecture, illustrating the main components and their interactions.\n\n```mermaid\nflowchart LR\n User[\"User\"] --> Frontend[\"Frontend (React/Vite)\"]\n Frontend --> Backend[\"Backend API (Flask)\"]\n Backend --> LLM[\"LLM Integration Layer\"] & VectorStore[\"Vector Stores\"] & TaskQueue[\"Task Queue (Celery)\"] & Databases[\"Databases (MongoDB, Redis)\"]\n LLM -- Cloud APIs / Local Engines --> InferenceEngine[\"Inference Engine\"]\n VectorStore -- Document Embeddings --> Indexes[(\"Indexes\")]\n TaskQueue -- Asynchronous Tasks --> DocumentIngestion[\"Document Ingestion\"]\n\n style Frontend fill:#AA00FF,color:#FFFFFF\n style Backend fill:#AA00FF,color:#FFFFFF\n style LLM fill:#AA00FF,color:#FFFFFF\n style TaskQueue fill:#AA00FF,color:#FFFFFF,stroke:#AA00FF\n style DocumentIngestion fill:#AA00FF,color:#FFFFFF,stroke:none\n```\n\n## Component Descriptions\n\n### 1. Frontend (React/Vite)\n\n* **Technology:** Built using React and Vite.\n* **Responsibility:** This is the user interface of DocsGPT, providing users with an UI to ask questions and receive answers, configure prompts, tools and other settings. It handles user input, displays conversation history, shows sources, and manages settings.\n* **Key Features:**\n * Clean and responsive UI.\n * Simple static client-side rendering.\n * Manages conversation state and settings.\n * Communicates with the Backend API for data retrieval and processing.\n\n### 2. Backend API (Flask)\n\n* **Technology:** Implemented using Flask (Python).\n* **Responsibility:** The Backend API serves as the core logic and orchestration layer of DocsGPT. It receives requests from the Frontend, Extensions or API clients, processes them, and coordinates interactions between different components.\n* **Key Features:**\n * API endpoints for handling user queries, document uploads, and settings configurations.\n * Manages the overall application flow and logic.\n * Integrates with the LLM Integration Layer, Vector Stores, Task Queue, Tools, Agents and Databases.\n * Provides Swagger documentation for API endpoints.\n\n### 3. LLM Integration Layer (Part of backend)\n\n* **Technology:** Supports multiple LLM APIs and local engines.\n* **Responsibility:** This layer provides an abstraction for interacting with Large Language Models (LLMs).\n* **Key Features:**\n * Supports LLMs from OpenAI, Google, Anthropic, Groq, HuggingFace Inference API, Azure OpenAI, also compatable with local models like Ollama, LLaMa.cpp, Text Generation Inference (TGI), SGLang, vLLM, Aphrodite, FriendliAI, and LMDeploy.\n * Manages API key handling and request formatting and Tool fromatting.\n * Offers caching mechanisms to improve response times and reduce API usage.\n * Handles streaming responses for a more interactive user experience.\n\n### 4. Vector Stores (Part of backend)\n\n* **Technology:** Supports multiple vector databases.\n* **Responsibility:** Vector Stores are used to store and retrieve vector embeddings of document chunks. This enables semantic search and retrieval of relevant document snippets in response to user queries.\n* **Key Features:**\n * Supports vector databases including FAISS, Elasticsearch, Qdrant, Milvus, and LanceDB.\n * Provides storage and indexing of high-dimensional vector embeddings.\n * Enables editing and updating of vector indexes including specific chunks.\n\n### 5. Parser Integration Layer (Part of backend)\n\n* **Technology:** Supports multiple formats for file processing and remote source uploading.\n* **Responsibility:** Parser Integration Layer handles uploading, parsing, chunking, embedding, and indexing documents.\n* **Key Features:**\n * Supports various document formats (PDF, DOCX, TXT, etc.) and remote sources (web URLs, sitemaps).\n * Handles document parsing, text chunking, and embedding generation.\n * Utilizes Celery for asynchronous processing, ensuring efficient handling of large documents.\n\n### 6. Task Queue (Celery)\n\n* **Technology:** Celery with Redis as broker and backend.\n* **Responsibility:** Celery handles asynchronous task processing, for long-running operations such as document ingestion and indexing. This ensures that the main application remains responsive and efficient.\n* **Key Features:**\n * Manages background tasks for document processing and indexing.\n * Improves application responsiveness by offloading heavy tasks.\n * Enhances scalability and reliability through distributed task processing.\n\n### 7. Databases (MongoDB, Redis)\n\n* **Technology:** MongoDB and Redis.\n* **Responsibility:** Databases are used for persistent data storage and caching. MongoDB stores structured data such as conversations, documents, user settings, and API keys. Redis is used as a cache, as well as a message broker for Celery.\n\n## Request Flow Diagram\n\nThis diagram illustrates the sequence of steps involved when a user submits a question to DocsGPT.\n\n```mermaid\nsequenceDiagram\n participant User\n participant Frontend\n participant BackendAPI\n participant LLMIntegrationLayer\n participant VectorStores\n participant InferenceEngine\n\n User->>Frontend: User asks a question\n Frontend->>BackendAPI: API Request (Question)\n BackendAPI->>VectorStores: Fetch relevant document chunks (Similarity Search)\n VectorStores-->>BackendAPI: Return document chunks\n BackendAPI->>LLMIntegrationLayer: Send question and document chunks\n LLMIntegrationLayer->>InferenceEngine: LLM API Request (Prompt + Context)\n InferenceEngine-->>LLMIntegrationLayer: LLM API Response (Answer)\n LLMIntegrationLayer-->>BackendAPI: Return Answer\n BackendAPI->>Frontend: API Response (Answer)\n Frontend->>User: Display Answer\n\n Note over Frontend,BackendAPI: Data flow is simplified for clarity\n```\n\n## Deployment Architecture\n\nDocsGPT is designed to be deployed using Docker and Kubernetes, here is a qucik overview of a simple k8s deployment.\n\n```mermaid\ngraph LR\n subgraph Kubernetes Cluster\n subgraph Nodes\n subgraph Node 1\n FrontendPod[Frontend Pod]\n BackendAPIPod[Backend API Pod]\n end\n subgraph Node 2\n CeleryWorkerPod[Celery Worker Pod]\n RedisPod[Redis Pod]\n end\n subgraph Node 3\n MongoDBPod[MongoDB Pod]\n VectorStorePod[Vector Store Pod]\n end\n end\n LoadBalancer[Load Balancer] --> docsgpt-frontend-service[docsgpt-frontend-service]\n LoadBalancer --> docsgpt-api-service[docsgpt-api-service]\n docsgpt-frontend-service --> FrontendPod\n docsgpt-api-service --> BackendAPIPod\n BackendAPIPod --> CeleryWorkerPod\n BackendAPIPod --> RedisPod\n BackendAPIPod --> MongoDBPod\n BackendAPIPod --> VectorStorePod\n CeleryWorkerPod --> RedisPod\n BackendAPIPod --> InferenceEngine[(Inference Engine)]\n VectorStorePod --> Indexes[(Indexes)]\n MongoDBPod --> Data[(Data)]\n RedisPod --> Cache[(Cache)]\n end\n User[User] --> LoadBalancer\n```\n" }, { "path": "docs/content/Guides/Customising-prompts.mdx", "content": "---\ntitle: Customizing Prompts\ndescription: This guide explains how to customize prompts in DocsGPT using the new template-based system with dynamic variable injection.\n---\n\nimport Image from 'next/image'\n\n# Customizing Prompts in DocsGPT\n\nCustomizing prompts for DocsGPT gives you powerful control over the AI's behavior and responses. With the new template-based system, you can inject dynamic context through organized namespaces, making prompts flexible and maintainable without hardcoding values.\n\n## Quick Start\n\n1. Navigate to `SideBar -> Settings`.\n2. In Settings, select the `Active Prompt` to see various prompt styles.\n3. Click on the `edit icon` on your chosen prompt to customize it.\n\n### Video Demo\n\"prompts\"\n\n---\n\n## Template-Based Prompt System\n\nDocsGPT now uses **Jinja2 templating** with four organized namespaces for dynamic variable injection:\n\n### Available Namespaces\n\n#### 1. **`system`** - System Metadata\nAccess system-level information:\n\n```jinja\n{{ system.date }} # Current date (YYYY-MM-DD)\n{{ system.time }} # Current time (HH:MM:SS)\n{{ system.timestamp }} # ISO 8601 timestamp\n{{ system.request_id }} # Unique request identifier\n{{ system.user_id }} # Current user ID\n```\n\n#### 2. **`source`** - Retrieved Documents\nAccess RAG (Retrieval-Augmented Generation) document context:\n\n```jinja\n{{ source.content }} # Concatenated document content\n{{ source.summaries }} # Alias for content (backward compatible)\n{{ source.documents }} # List of document objects\n{{ source.count }} # Number of retrieved documents\n```\n\n#### 3. **`passthrough`** - Request Parameters\nAccess custom parameters passed in the API request:\n\n```jinja\n{{ passthrough.company }} # Custom field from request\n{{ passthrough.user_name }} # User-provided data\n{{ passthrough.context }} # Any custom parameter\n```\n\nTo use passthrough data, send it in your API request:\n```json\n{\n \"question\": \"What is the pricing?\",\n \"passthrough\": {\n \"company\": \"Acme Corp\",\n \"user_name\": \"Alice\",\n \"plan_type\": \"enterprise\"\n }\n}\n```\n\n#### 4. **`tools`** - Pre-fetched Tool Data\nAccess results from tools that run before the agent (like memory tool):\n\n```jinja\n{{ tools.memory.root }} # Memory tool directory listing\n{{ tools.memory.available }} # Boolean: is memory available\n```\n\n---\n\n## Example Prompts\n\n### Basic Prompt with Documents\n```jinja\nYou are a helpful AI assistant for DocsGPT.\n\nCurrent date: {{ system.date }}\n\nUse the following documents to answer the question:\n\n{{ source.content }}\n\nProvide accurate, helpful answers with code examples when relevant.\n```\n\n### Advanced Prompt with All Namespaces\n```jinja\nYou are an AI assistant for {{ passthrough.company }}.\n\n**System Info:**\n- Date: {{ system.date }}\n- Request ID: {{ system.request_id }}\n\n**User Context:**\n- User: {{ passthrough.user_name }}\n- Role: {{ passthrough.role }}\n\n**Available Documents ({{ source.count }}):**\n{{ source.content }}\n\n**Memory Context:**\n{% if tools.memory.available %}\n{{ tools.memory.root }}\n{% else %}\nNo saved context available.\n{% endif %}\n\nPlease provide detailed, accurate answers based on the documents above.\n```\n\n### Conditional Logic Example\n```jinja\nYou are a DocsGPT assistant.\n\n{% if source.count > 0 %}\nI found {{ source.count }} relevant document(s):\n\n{{ source.content }}\n\nBase your answer on these documents.\n{% else %}\nNo documents were found. Please answer based on your general knowledge.\n{% endif %}\n```\n\n---\n\n## Migration Guide\n\n### Legacy Format (Still Supported)\nThe old `{summaries}` format continues to work for backward compatibility:\n\n```markdown\nYou are a helpful assistant.\n\nDocuments:\n{summaries}\n```\n\nThis will automatically substitute `{summaries}` with document content.\n\n### New Template Format (Recommended)\nMigrate to the new template syntax for more flexibility:\n\n```jinja\nYou are a helpful assistant.\n\nDocuments:\n{{ source.content }}\n```\n\n**Migration mapping:**\n- `{summaries}` → `{{ source.content }}` or `{{ source.summaries }}`\n\n---\n\n## Best Practices\n\n### 1. **Use Descriptive Context**\n```jinja\n**Retrieved Documents:**\n{{ source.content }}\n\n**User Query Context:**\n- Company: {{ passthrough.company }}\n- Department: {{ passthrough.department }}\n```\n\n### 2. **Handle Missing Data Gracefully**\n```jinja\n{% if passthrough.user_name %}\nHello {{ passthrough.user_name }}!\n{% endif %}\n```\n\n### 3. **Leverage Memory for Continuity**\n```jinja\n{% if tools.memory.available %}\n**Previous Context:**\n{{ tools.memory.root }}\n{% endif %}\n\n**Current Question:**\nPlease consider the above context when answering.\n```\n\n### 4. **Add Clear Instructions**\n```jinja\nYou are a technical support assistant.\n\n**Guidelines:**\n1. Always reference the documents below\n2. Provide step-by-step instructions\n3. Include code examples when relevant\n\n**Reference Documents:**\n{{ source.content }}\n```\n\n---\n\n## Advanced Features\n\n### Looping Over Documents\n```jinja\n{% for doc in source.documents %}\n**Source {{ loop.index }}:** {{ doc.filename }}\n{{ doc.text }}\n\n{% endfor %}\n```\n\n### Date-Based Behavior\n```jinja\n{% if system.date > \"2025-01-01\" %}\nNote: This is information from 2025 or later.\n{% endif %}\n```\n\n### Custom Formatting\n```jinja\n**Request Information**\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n• Request ID: {{ system.request_id }}\n• User: {{ passthrough.user_name | default(\"Guest\") }}\n• Time: {{ system.time }}\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n```\n\n---\n\n## Tool Pre-Fetching\n\n### Memory Tool Configuration\nEnable memory tool pre-fetching to inject saved context into prompts:\n\n```python\n# In your tool configuration\n{\n \"name\": \"memory\",\n \"config\": {\n \"pre_fetch_enabled\": true # Default: true\n }\n}\n```\n\nControl pre-fetching globally:\n```bash\n# .env file\nENABLE_TOOL_PREFETCH=true\n```\n\nOr per-request:\n```json\n{\n \"question\": \"What are the requirements?\",\n \"disable_tool_prefetch\": false\n}\n```\n\n---\n\n## Debugging Prompts\n\n### View Rendered Prompts in Logs\nSet log level to `INFO` to see the final rendered prompt sent to the LLM:\n\n```bash\nexport LOG_LEVEL=INFO\n```\n\nYou'll see output like:\n```\nINFO - Rendered system prompt for agent (length: 1234 chars):\n================================================================================\nYou are a helpful assistant for Acme Corp.\n\nCurrent date: 2025-10-30\nRequest ID: req_abc123\n\nDocuments:\nTechnical documentation about...\n================================================================================\n```\n\n### Template Validation\nTest your template syntax before saving:\n```python\nfrom application.api.answer.services.prompt_renderer import PromptRenderer\n\nrenderer = PromptRenderer()\nis_valid = renderer.validate_template(\"Your prompt with {{ variables }}\")\n```\n\n---\n\n## Common Use Cases\n\n### 1. Customer Support Bot\n```jinja\nYou are a customer support assistant for {{ passthrough.company }}.\n\n**Customer:** {{ passthrough.customer_name }}\n**Ticket ID:** {{ system.request_id }}\n**Date:** {{ system.date }}\n\n**Knowledge Base:**\n{{ source.content }}\n\n**Previous Interactions:**\n{{ tools.memory.root }}\n\nPlease provide helpful, friendly support based on the knowledge base above.\n```\n\n### 2. Technical Documentation Assistant\n```jinja\nYou are a technical documentation expert.\n\n**Available Documentation ({{ source.count }} documents):**\n{{ source.content }}\n\n**Requirements:**\n- Provide code examples in {{ passthrough.language }}\n- Focus on {{ passthrough.framework }} best practices\n- Include relevant links when possible\n```\n\n### 3. Internal Knowledge Base\n```jinja\nYou are an internal AI assistant for {{ passthrough.department }}.\n\n**Employee:** {{ passthrough.employee_name }}\n**Access Level:** {{ passthrough.access_level }}\n\n**Relevant Documents:**\n{{ source.content }}\n\nProvide detailed answers appropriate for {{ passthrough.access_level }} access level.\n```\n\n---\n\n## Template Syntax Reference\n\n### Variables\n```jinja\n{{ variable_name }} # Output variable\n{{ namespace.field }} # Access nested field\n{{ variable | default(\"N/A\") }} # Default value\n```\n\n### Conditionals\n```jinja\n{% if condition %}\n Content\n{% elif other_condition %}\n Other content\n{% else %}\n Default content\n{% endif %}\n```\n\n### Loops\n```jinja\n{% for item in list %}\n {{ item.field }}\n{% endfor %}\n```\n\n### Comments\n```jinja\n{# This is a comment and won't appear in output #}\n```\n\n---\n\n## Security Considerations\n\n1. **Input Sanitization**: Passthrough data is automatically sanitized to prevent injection attacks\n2. **Type Filtering**: Only primitive types (string, int, float, bool, None) are allowed in passthrough\n3. **Autoescaping**: Jinja2 autoescaping is enabled by default\n4. **Size Limits**: Consider the token budget when including large documents\n\n---\n\n## Troubleshooting\n\n### Problem: Variables Not Rendering\n**Solution:** Ensure you're using the correct namespace:\n```jinja\n❌ {{ company }}\n✅ {{ passthrough.company }}\n```\n\n### Problem: Empty Output for Tool Data\n**Solution:** Check that tool pre-fetching is enabled and the tool is configured correctly.\n\n### Problem: Syntax Errors\n**Solution:** Validate template syntax. Common issues:\n```jinja\n❌ {{ variable } # Missing closing brace\n❌ {% if x % # Missing closing %}\n✅ {{ variable }}\n✅ {% if x %}...{% endif %}\n```\n\n### Problem: Legacy Prompts Not Working\n**Solution:** The system auto-detects template syntax. If your prompt uses `{summaries}`, it will work in legacy mode. To use new features, add `{{ }}` syntax.\n\n---\n\n## API Reference\n\n### Render Prompt via API\n```python\nfrom application.api.answer.services.prompt_renderer import PromptRenderer\n\nrenderer = PromptRenderer()\nrendered = renderer.render_prompt(\n prompt_content=\"Your template with {{ passthrough.name }}\",\n user_id=\"user_123\",\n request_id=\"req_456\",\n passthrough_data={\"name\": \"Alice\"},\n docs_together=\"Document content here\",\n tools_data={\"memory\": {\"root\": \"Files: notes.txt\"}}\n)\n```\n\n---\n\n## Conclusion\n\nThe new template-based prompt system provides powerful flexibility while maintaining backward compatibility. By leveraging namespaces, you can create dynamic, context-aware prompts that adapt to your specific use case.\n\n**Key Benefits:**\n- ✅ Dynamic variable injection\n- ✅ Organized namespaces\n- ✅ Backward compatible\n- ✅ Security built-in\n- ✅ Easy to debug\n\nStart with simple templates and gradually add complexity as needed. Happy prompting! 🚀\n" }, { "path": "docs/content/Guides/How-to-train-on-other-documentation.mdx", "content": "---\ntitle: How to Train on Other Documentation\ndescription: A step-by-step guide on how to effectively train DocsGPT on additional documentation sources.\n---\n\nimport { Callout } from 'nextra/components'\nimport Image from 'next/image'\nimport { Steps } from 'nextra/components'\n\n## How to train on other documentation\n\nTraining on other documentation sources can greatly enhance the versatility and depth of DocsGPT's knowledge. By incorporating diverse materials, you can broaden the AI's understanding and improve its ability to generate insightful responses across a range of topics. Here's a step-by-step guide on how to effectively train DocsGPT on additional documentation sources:\n\n**Get your document ready**:\n\nMake sure you have the document on which you want to train on ready with you on the device which you are using .You can also use links to the documentation to train on.\n\n\n Note: The document should be either of the given file formats .pdf, .txt, .rst, .docx, .md, .zip and limited to 25mb.You can also train using the link of the documentation.\n\n\n\n### Video Demo\n\n\"prompts\"\n\n\n\n\n### Step1\nNavigate to the sidebar where you will find `Source Docs` option,here you will find 3 options built in which are default,Web Search and None.\n\n\n### Step 2\nClick on the `Upload icon` just beside the source docs options,now browse and upload the document which you want to train on or select the `remote` option if you have to insert the link of the documentation.\n\n\n### Step 3\nNow you will be able to see the name of the file uploaded under the Uploaded Files ,now click on `Train`,once you click on train it might take some time to train on the document. You will be able to see the `Training progress` and once the training is completed you can click the `finish` button and there you go your document is uploaded.\n\n\n### Step 4\nGo to `New chat` and from the side bar select the document you uploaded under the `Source Docs` and go ahead with your chat, now you can ask questions regarding the document you uploaded and you will get the effective answer based on it.\n\n\n\n\n\n" }, { "path": "docs/content/Guides/How-to-use-different-LLM.mdx", "content": "---\ntitle:\ndescription:\n---\n\nimport { Callout } from 'nextra/components'\nimport Image from 'next/image'\nimport { Steps } from 'nextra/components'\n\n# Setting Up Local Language Models for Your App\n\nSetting up local language models for your app can significantly enhance its capabilities, enabling it to understand and generate text in multiple languages without relying on external APIs. By integrating local language models, you can improve privacy, reduce latency, and ensure continuous functionality even in offline environments. Here's a comprehensive guide on how to set up local language models for your application:\n\n## Steps:\n### For cloud version LLM change:\n\n### Step 1\nVisit the chat screen and you will be to see the default LLM selected.\n### Step 2\nClick on it and you will get a drop down of various LLM's available to choose.\n### Step 3\nChoose the LLM of your choice.\n\n\n\n\n\n\n### Video Demo\n\"prompts\"\n\n### For Open source llm change:\n\n### Step 1\nFor open source version please edit `LLM_PROVIDER`, `LLM_NAME` and others in the .env file. Refer to [⚙️ App Configuration](/Deploying/DocsGPT-Settings) for more information.\n### Step 2\nVisit [☁️ Cloud Providers](/Models/cloud-providers) for the updated list of online models. Make sure you have the right API_KEY and correct LLM_PROVIDER.\nFor self-hosted please visit [🖥️ Local Inference](/Models/local-inference). \n\n\n\n\n" }, { "path": "docs/content/Guides/Integrations/_meta.js", "content": "export default {\n \"google-drive-connector\": {\n \"title\": \"🔗 Google Drive\",\n \"href\": \"/Guides/Integrations/google-drive-connector\"\n }\n}\n" }, { "path": "docs/content/Guides/Integrations/google-drive-connector.mdx", "content": "---\ntitle: Google Drive Connector\ndescription: Connect your Google Drive as an external knowledge base to upload and process files directly from your Google Drive account.\n---\n\nimport { Callout } from 'nextra/components'\nimport { Steps } from 'nextra/components'\n\n# Google Drive Connector\n\nThe Google Drive Connector allows you to seamlessly connect your Google Drive account as an external knowledge base. This integration enables you to upload and process files directly from your Google Drive without manually downloading and uploading them to DocsGPT.\n\n## Features\n\n- **Direct File Access**: Browse and select files directly from your Google Drive\n- **Comprehensive File Support**: Supports all major document formats including:\n - Google Workspace files (Docs, Sheets, Slides)\n - Microsoft Office files (.docx, .xlsx, .pptx, .doc, .ppt, .xls)\n - PDF documents\n - Text files (.txt, .md, .rst, .html, .rtf)\n - Data files (.csv, .json)\n - Image files (.png, .jpg, .jpeg)\n - E-books (.epub)\n- **Secure Authentication**: Uses OAuth 2.0 for secure access to your Google Drive\n- **Real-time Sync**: Process files directly from Google Drive without local downloads\n\n\nThe Google Drive Connector requires proper configuration of Google API credentials. Follow the setup instructions below to enable this feature.\n\n\n## Prerequisites\n\nBefore setting up the Google Drive Connector, you'll need:\n\n1. A Google Cloud Platform (GCP) project\n2. Google Drive API enabled\n3. OAuth 2.0 credentials configured\n4. DocsGPT instance with proper environment variables\n\n## Setup Instructions\n\n\n\n### Step 1: Create a Google Cloud Project\n\n1. Go to the [Google Cloud Console](https://console.cloud.google.com/)\n2. Create a new project or select an existing one\n3. Note down your Project ID for later use\n\n### Step 2: Enable Google Drive API\n\n1. In the Google Cloud Console, navigate to **APIs & Services** > **Library**\n2. Search for \"Google Drive API\"\n3. Click on \"Google Drive API\" and click **Enable**\n\n### Step 3: Create OAuth 2.0 Credentials\n\n1. Go to **APIs & Services** > **Credentials**\n2. Click **Create Credentials** > **OAuth client ID**\n3. If prompted, configure the OAuth consent screen:\n - Choose **External** user type (unless you're using Google Workspace)\n - Fill in the required fields (App name, User support email, Developer contact)\n - Add your domain to **Authorized domains** if deploying publicly\n4. For Application type, select **Web application**\n5. Add your DocsGPT frontend URL to **Authorized JavaScript origins**:\n - For local development: `http://localhost:3000`\n - For production: `https://yourdomain.com`\n6. Add your DocsGPT callback URL to **Authorized redirect URIs**:\n - For local development: `http://localhost:7091/api/connectors/callback?provider=google_drive`\n - For production: `https://yourdomain.com/api/connectors/callback?provider=google_drive`\n7. Click **Create** and note down the **Client ID** and **Client Secret**\n\n\n\n### Step 4: Configure Backend Environment Variables\n\nAdd the following environment variables to your backend configuration:\n\n**For Docker deployment**, add to your `.env` file in the root directory:\n\n```env\n# Google Drive Connector Configuration\nGOOGLE_CLIENT_ID=your_google_client_id_here\nGOOGLE_CLIENT_SECRET=your_google_client_secret_here\n```\n\n**For manual deployment**, set these environment variables in your system or application configuration.\n\n### Step 5: Configure Frontend Environment Variables\n\nAdd the following environment variables to your frontend `.env` file:\n\n```env\n# Google Drive Frontend Configuration\nVITE_GOOGLE_CLIENT_ID=your_google_client_id_here\n```\n\n\nMake sure to use the same Google Client ID in both backend and frontend configurations.\n\n\n### Step 6: Restart Your Application\n\nAfter configuring the environment variables:\n\n1. **For Docker**: Restart your Docker containers\n ```bash\n docker-compose down\n docker-compose up -d\n ```\n\n2. **For manual deployment**: Restart both backend and frontend services\n\n\n\n## Using the Google Drive Connector\n\nOnce configured, you can use the Google Drive Connector to upload files:\n\n\n\n### Step 1: Access the Upload Interface\n\n1. Navigate to the DocsGPT interface\n2. Go to the upload/training section\n3. You should now see \"Google Drive\" as an available upload option\n\n### Step 2: Connect Your Google Account\n\n1. Select \"Google Drive\" as your upload method\n2. Click \"Connect to Google Drive\"\n3. You'll be redirected to Google's OAuth consent screen\n4. Grant the necessary permissions to DocsGPT\n5. You'll be redirected back to DocsGPT with a successful connection\n\n### Step 3: Select Files\n\n1. Once connected, click \"Select Files\"\n2. The Google Drive picker will open\n3. Browse your Google Drive and select the files you want to process\n4. Click \"Select\" to confirm your choices\n\n### Step 4: Process Files\n\n1. Review your selected files\n2. Click \"Train\" or \"Upload\" to process the files\n3. DocsGPT will download and process the files from your Google Drive\n4. Once processing is complete, the files will be available in your knowledge base\n\n\n\n## Supported File Types\n\nThe Google Drive Connector supports the following file types:\n\n| File Type | Extensions | Description |\n|-----------|------------|-------------|\n| **Google Workspace** | - | Google Docs, Sheets, Slides (automatically converted) |\n| **Microsoft Office** | .docx, .xlsx, .pptx | Modern Office formats |\n| **Legacy Office** | .doc, .ppt, .xls | Older Office formats |\n| **PDF Documents** | .pdf | Portable Document Format |\n| **Text Files** | .txt, .md, .rst, .html, .rtf | Various text formats |\n| **Data Files** | .csv, .json | Structured data formats |\n| **Images** | .png, .jpg, .jpeg | Image files (with OCR if enabled) |\n| **E-books** | .epub | Electronic publication format |\n\n## Troubleshooting\n\n### Common Issues\n\n**\"Google Drive option not appearing\"**\n- Verify that `VITE_GOOGLE_CLIENT_ID` is set in frontend environment\n- Check that `VITE_GOOGLE_CLIENT_ID` environment variable is present in your frontend configuration\n- Check browser console for any JavaScript errors\n- Ensure the frontend has been restarted after adding environment variables\n\n**\"Authentication failed\"**\n- Verify that your OAuth 2.0 credentials are correctly configured\n- Check that the redirect URI `http:///api/connectors/callback?provider=google_drive` is correctly added in GCP console\n- Ensure the Google Drive API is enabled in your GCP project\n\n**\"Permission denied\" errors**\n- Verify that the OAuth consent screen is properly configured\n- Check that your Google account has access to the files you're trying to select\n- Ensure the required scopes are granted during authentication\n\n**\"Files not processing\"**\n- Check that the backend environment variables are correctly set\n- Verify that the OAuth credentials have the necessary permissions\n- Check the backend logs for any error messages\n\n### Environment Variable Checklist\n\n**Backend (.env in root directory):**\n- ✅ `GOOGLE_CLIENT_ID`\n- ✅ `GOOGLE_CLIENT_SECRET`\n\n**Frontend (.env in frontend directory):**\n- ✅ `VITE_GOOGLE_CLIENT_ID`\n\n### Security Considerations\n\n- Keep your Google Client Secret secure and never expose it in frontend code\n- Regularly rotate your OAuth credentials\n- Use HTTPS in production to protect authentication tokens\n- Ensure proper OAuth consent screen configuration for production use\n\n\nFor production deployments, make sure to add your actual domain to the OAuth consent screen and authorized origins/redirect URIs.\n\n\n\n" }, { "path": "docs/content/Guides/My-AI-answers-questions-using-external-knowledge.mdx", "content": "---\ntitle:\ndescription:\n---\n\n# Avoiding hallucinations\n\nIf your AI uses external knowledge and is not explicit enough, it is ok, because we try to make DocsGPT friendly.\n\nBut if you want to adjust it, here is a simple way:-\n\n- Got to `application/prompts/chat_combine_prompt.txt`\n\n- And change it to\n\n\n```\n\nYou are a DocsGPT, friendly and helpful AI assistant by Arc53 that provides help with documents. You give thorough answers with code examples, if possible.\nWrite an answer for the question below based on the provided context.\nIf the context provides insufficient information, reply \"I cannot answer\".\nYou have access to chat history and can use it to help answer the question.\n----------------\n{summaries}\n\n```\n" }, { "path": "docs/content/Guides/_meta.js", "content": "export default {\n \"Customising-prompts\": {\n \"title\": \"️💻 Customising Prompts\",\n \"href\": \"/Guides/Customising-prompts\"\n },\n \"How-to-train-on-other-documentation\": {\n \"title\": \"📥 Training on docs\",\n \"href\": \"/Guides/How-to-train-on-other-documentation\"\n },\n \"How-to-use-different-LLM\": {\n \"title\": \"️🤖 How to use different LLM's\",\n \"href\": \"/Guides/How-to-use-different-LLM\",\n \"display\": \"hidden\"\n },\n \"My-AI-answers-questions-using-external-knowledge\": {\n \"title\": \"💭️ Avoiding hallucinations\",\n \"href\": \"/Guides/My-AI-answers-questions-using-external-knowledge\",\n \"display\": \"hidden\"\n },\n \"Architecture\": {\n \"title\": \"🏗️ Architecture\",\n \"href\": \"/Guides/Architecture\"\n },\n \"compression\": {\n \"title\": \"🗜️ Context Compression\",\n \"href\": \"/Guides/compression\"\n },\n \"ocr\": {\n \"title\": \"OCR\",\n \"href\": \"/Guides/ocr\"\n },\n \"Integrations\": {\n \"title\": \"🔗 Integrations\"\n }\n}\n" }, { "path": "docs/content/Guides/compression.md", "content": "# Context Compression\n\nDocsGPT implements a smart context compression system to manage long conversations effectively. This feature prevents conversations from hitting the LLM's context window limit while preserving critical information and continuity.\n\n## How It Works\n\nThe compression system operates on a \"summarize and truncate\" principle:\n\n1. **Threshold Check**: Before each request, the system calculates the total token count of the conversation history.\n2. **Trigger**: If the token count exceeds a configured threshold (default: 80% of the model's context limit), compression is triggered.\n3. **Summarization**: An LLM (potentially a different, cheaper/faster one) processes the older part of the conversation—including previous summaries, user messages, agent responses, and tool outputs.\n4. **Context Replacement**: The system generates a comprehensive summary of the older history. For subsequent requests, the LLM receives this **Summary + Recent Messages** instead of the full raw history.\n\n### Key Features\n\n* **Recursive Summarization**: New summaries incorporate previous summaries, ensuring that information from the very beginning of a long chat is not lost.\n* **Tool Call Support**: The compression logic explicitly handles tool calls and their outputs (e.g., file readings, search results), summarizing their results so the agent retains knowledge of what it has already done.\n* **\"Needle in a Haystack\" Preservation**: The prompts are designed to identify and preserve specific, critical details (like passwords, keys, or specific user instructions) even when compressing large amounts of text.\n\n## Configuration\n\nYou can configure the compression behavior in your `.env` file or `application/core/settings.py`:\n\n| Setting | Default | Description |\n| :--- | :--- | :--- |\n| `ENABLE_CONVERSATION_COMPRESSION` | `True` | Master switch to enable/disable the feature. |\n| `COMPRESSION_THRESHOLD_PERCENTAGE` | `0.8` | The fraction of the context window (0.0 to 1.0) that triggers compression. |\n| `COMPRESSION_MODEL_OVERRIDE` | `None` | (Optional) Specify a different model ID to use specifically for the summarization task (e.g., using `gpt-3.5-turbo` to compress for `gpt-4`). |\n| `COMPRESSION_MAX_HISTORY_POINTS` | `3` | The number of past compression points to keep in the database (older ones are discarded as they are incorporated into newer summaries). |\n\n## Architecture\n\nThe system is modularized into several components:\n\n* **`CompressionThresholdChecker`**: Calculates token usage and decides when to compress.\n* **`CompressionService`**: Orchestrates the compression process, manages DB updates, and reconstructs the context (Summary + Recent Messages) for the LLM.\n* **`CompressionPromptBuilder`**: Constructs the specific prompts used to instruct the LLM to summarize the conversation effectively.\n" }, { "path": "docs/content/Guides/ocr.mdx", "content": "---\ntitle: OCR for Sources and Attachments\ndescription: How OCR works in DocsGPT, how to configure it, and what changes for source ingestion vs chat attachments.\n---\n\nimport { Callout } from 'nextra/components'\n\n# Docling OCR for Sources and Attachments\n\nDocsGPT uses Docling as the default parser layer for many document formats. OCR is optional and controlled by two settings:\n\n```env\nDOCLING_OCR_ENABLED=false\nDOCLING_OCR_ATTACHMENTS_ENABLED=false\n```\n\n- `DOCLING_OCR_ENABLED`: OCR behavior for Source Docs ingestion.\n- `DOCLING_OCR_ATTACHMENTS_ENABLED`: OCR behavior for chat attachments uploaded from the message box.\n\n## Processing Flow\n\n### Source Docs flow (Upload and Train)\n\n1. Files are uploaded through `/api/upload`.\n2. Ingestion runs asynchronously in Celery (`ingest_worker`).\n3. `SimpleDirectoryReader` parses files with `get_default_file_extractor`.\n4. For PDFs and image formats, Docling parsers are used. OCR in this path is controlled by `DOCLING_OCR_ENABLED`.\n5. Parsed text is chunked, embedded, and stored in the vector store.\n6. Retrieval during chat uses this indexed text and returns source citations.\n\n### Attachment flow (Chat-only file context)\n\n1. Files are uploaded through `/api/store_attachment`.\n2. Celery task `attachment_worker` parses and stores the attachment in MongoDB (`attachments` collection).\n3. OCR in this path is controlled by `DOCLING_OCR_ATTACHMENTS_ENABLED`.\n4. Attachments are not vectorized and are not added to the source index.\n5. During answer generation, selected attachment IDs are loaded and passed directly to the LLM pipeline.\n\n## How Docling OCR Works\n\nDocling OCR behavior is different for PDFs vs images:\n\n- PDF parser defaults to hybrid OCR:\n - text regions: extracted directly\n - bitmap/image regions: OCR only where needed\n- Image parser defaults to full-page OCR (the whole image is visual content).\n\nBy default, Docling parser classes use RapidOCR options (language default: `english`).\n\n\nParser internals like OCR language and force-full-page OCR are currently set by code defaults, not separate `.env` settings.\n\n\n## Attachment Behavior by Model Support\n\nWhen attachments are used in chat, behavior depends on the selected model/provider:\n\n- If a MIME type is supported, DocsGPT sends files/images through provider-native attachment APIs.\n- If unsupported, DocsGPT falls back to the parsed text content stored for the attachment.\n- For providers that support images but not native PDF attachments, PDF files are converted to images (synthetic PDF support).\n\nThis means OCR quality is especially important for text fallback paths and for models without native attachment support.\n\n## Recommended Configuration\n\nFor most OCR-enabled use cases, enable both flags:\n\n```env\nDOCLING_OCR_ENABLED=true\nDOCLING_OCR_ATTACHMENTS_ENABLED=true\n```\n\nAfter changing these settings, restart the API and Celery worker.\n\n## Legacy Fallback Notes\n\n- If Docling is unavailable, DocsGPT falls back to legacy parsers.\n- With OCR disabled, text-based PDFs can still parse, but scanned/image-heavy content may produce little text.\n- For image parsing without Docling OCR, the legacy image parser only extracts text when `PARSE_IMAGE_REMOTE=true`.\n\n" }, { "path": "docs/content/Models/_meta.js", "content": "export default {\n \"cloud-providers\": {\n \"title\": \"☁️ Cloud Providers\",\n \"href\": \"/Models/cloud-providers\"\n },\n \"local-inference\": {\n \"title\": \"🖥️ Local Inference\",\n \"href\": \"/Models/local-inference\"\n },\n \"embeddings\": {\n \"title\": \"📝 Embeddings\",\n \"href\": \"/Models/embeddings\"\n }\n}\n" }, { "path": "docs/content/Models/cloud-providers.mdx", "content": "---\ntitle: Connecting DocsGPT to Cloud LLM Providers\ndescription: Connect DocsGPT to various Cloud Large Language Model (LLM) providers to power your document Q&A.\n---\n\n# Connecting DocsGPT to Cloud LLM Providers\n\nDocsGPT is designed to seamlessly integrate with a variety of Cloud Large Language Model (LLM) providers, giving you access to state-of-the-art AI models for document question answering.\n\n## Configuration via `.env` file\n\nThe primary method for configuring your LLM provider in DocsGPT is through the `.env` file. For a comprehensive understanding of all available settings, please refer to the detailed [DocsGPT Settings Guide](/Deploying/DocsGPT-Settings).\n\nTo connect to a cloud LLM provider, you will typically need to configure the following basic settings in your `.env` file:\n\n* **`LLM_PROVIDER`**: This setting is essential and identifies the specific cloud provider you wish to use (e.g., `openai`, `google`, `anthropic`).\n* **`LLM_NAME`**: Specifies the exact model you want to utilize from your chosen provider (e.g., `gpt-5.1`, `gemini-flash-latest`, `claude-3-5-sonnet-20241022`). Refer to your provider's documentation for a list of available models.\n* **`API_KEY`**: Almost all cloud LLM providers require an API key for authentication. Obtain your API key from your chosen provider's platform and securely store it in your `.env` file.\n\n## Explicitly Supported Cloud Providers\n\nDocsGPT offers direct, streamlined support for the following cloud LLM providers, making configuration straightforward. The table below outlines the `LLM_PROVIDER` and example `LLM_NAME` values to use for each provider in your `.env` file.\n\n| Provider | `LLM_PROVIDER` | Example `LLM_NAME` |\n| :--------------------------- | :------------- | :-------------------------- |\n| DocsGPT Public API | `docsgpt` | `None` |\n| OpenAI | `openai` | `gpt-5.1` |\n| Google (Vertex AI, Gemini) | `google` | `gemini-flash-latest` |\n| Anthropic (Claude) | `anthropic` | `claude-3-5-sonnet-20241022`|\n| Groq | `groq` | `llama-3.3-70b-versatile` |\n| HuggingFace Inference API | `huggingface` | `meta-llama/Llama-3.1-8B-Instruct` |\n| Azure OpenAI | `azure_openai` | `azure-gpt-4` |\n| Prem AI | `premai` | (See Prem AI docs) |\n| AWS SageMaker | `sagemaker` | (See SageMaker docs) |\n| Novita AI | `novita` | (See Novita docs) |\n\n## Connecting to OpenAI-Compatible Cloud APIs\n\nDocsGPT's flexible architecture allows you to connect to any cloud provider that offers an API compatible with the OpenAI API standard. This opens up a vast ecosystem of LLM services.\n\nTo connect to an OpenAI-compatible cloud provider, you will still use `LLM_PROVIDER=openai` in your `.env` file. However, you will also need to specify the API endpoint of your chosen provider using the `OPENAI_BASE_URL` setting. You will also likely need to provide an `API_KEY` and `LLM_NAME` as required by that provider.\n\n**Example for DeepSeek (OpenAI-Compatible API):**\n\nTo connect to DeepSeek, which offers an OpenAI-compatible API, your `.env` file could be configured as follows:\n\n```\nLLM_PROVIDER=openai\nAPI_KEY=YOUR_API_KEY # Your DeepSeek API key\nLLM_NAME=deepseek-chat # Or your desired DeepSeek model name\nOPENAI_BASE_URL=https://api.deepseek.com/v1 # DeepSeek's OpenAI API URL\n```\n\nRemember to consult the documentation of your chosen OpenAI-compatible cloud provider for their specific API endpoint, required model names, and authentication methods.\n\n## Adding Support for Other Cloud Providers\n\nIf you wish to connect to a cloud provider that is not explicitly listed above or doesn't offer OpenAI API compatibility, you can extend DocsGPT to support it. Within the DocsGPT repository, navigate to the `application/llm` directory. Here, you will find Python files defining the existing LLM integrations. You can use these files as examples to create a new module for your desired cloud provider. After creating your new LLM module, you will need to register it within the `llm_creator.py` file. This process involves some coding, but it allows for virtually unlimited extensibility to connect to any cloud-based LLM service with an accessible API." }, { "path": "docs/content/Models/embeddings.md", "content": "---\ntitle: Understanding and Configuring Embedding Models in DocsGPT\ndescription: Learn about embedding models, their importance in DocsGPT, and how to configure them for optimal performance.\n---\n\n# Understanding and Configuring Embedding Models in DocsGPT\n\nEmbedding models are a crucial component of DocsGPT, enabling its powerful document understanding and question-answering capabilities. This guide will explain what embedding models are, why they are essential for DocsGPT, and how to configure them.\n\n## What are Embedding Models?\n\nIn simple terms, an embedding model is a type of language model that converts text into numerical vectors. These vectors, known as embeddings, capture the semantic meaning of the text. Think of it as translating words and sentences into a language that computers can understand mathematically, where similar meanings are represented by vectors that are close to each other in vector space.\n\n**Why are embedding models important for DocsGPT?**\n\nDocsGPT uses embedding models for several key tasks:\n\n* **Semantic Search:** When you upload documents to DocsGPT, the application uses an embedding model to generate embeddings for each document chunk. These embeddings are stored in a vector store. When you ask a question, your query is also converted into an embedding. DocsGPT then performs a semantic search in the vector store, finding document chunks whose embeddings are most similar to your query embedding. This allows DocsGPT to retrieve relevant information based on the *meaning* of your question and documents, not just keyword matching.\n* **Document Understanding:** Embeddings help DocsGPT understand the underlying meaning of your documents, enabling it to answer questions accurately and contextually, even if the exact keywords from your question are not present in the retrieved document chunks.\n\nIn essence, embedding models are the bridge that allows DocsGPT to understand the nuances of human language and connect your questions to the relevant information within your documents.\n\n## Out-of-the-Box Embedding Model Support in DocsGPT\n\nDocsGPT is designed to be flexible and supports a wide range of embedding models right out of the box. Currently, DocsGPT provides native support for models from two major sources:\n\n* **Sentence Transformers:** DocsGPT supports all models available through the [Sentence Transformers library](https://www.sbert.net/). This library offers a vast selection of pre-trained embedding models, known for their quality and efficiency in various semantic tasks.\n* **OpenAI Embeddings:** DocsGPT also supports using embedding models from OpenAI, specifically the `text-embedding-ada-002` model, which is a powerful and widely used embedding model from OpenAI's API.\n\n## Configuring Sentence Transformer Models\n\nTo utilize Sentence Transformer models within DocsGPT, you need to follow these steps:\n\n1. **Download the Model:** Sentence Transformer models are typically hosted on Hugging Face Model Hub. You need to download your chosen model and place it in the `model/` folder in the root directory of your DocsGPT project.\n\n For example, to use the `all-mpnet-base-v2` model, you would set `EMBEDDINGS_NAME` as described below, and ensure that the model files are available locally (DocsGPT will attempt to download it if it's not found, but local download is recommended for development and offline use).\n\n2. **Set `EMBEDDINGS_NAME` in `.env` (or `settings.py`):** You need to configure the `EMBEDDINGS_NAME` setting in your `.env` file (or `settings.py`) to point to the desired Sentence Transformer model.\n\n * **Using a pre-downloaded model from `model/` folder:** You can specify a path to the downloaded model within the `model/` directory. For instance, if you downloaded `all-mpnet-base-v2` and it's in `model/all-mpnet-base-v2`, you could potentially use a relative path like (though direct path to the model name is usually sufficient):\n\n ```\n EMBEDDINGS_NAME=huggingface_sentence-transformers/all-mpnet-base-v2\n ```\n or simply use the model identifier:\n ```\n EMBEDDINGS_NAME=sentence-transformers/all-mpnet-base-v2\n ```\n\n * **Using a model directly from Hugging Face Model Hub:** You can directly specify the model identifier from Hugging Face Model Hub:\n\n ```\n EMBEDDINGS_NAME=huggingface_sentence-transformers/all-mpnet-base-v2\n ```\n\n## Using OpenAI Embeddings\n\nTo use OpenAI's `text-embedding-ada-002` embedding model, you need to set `EMBEDDINGS_NAME` to `openai_text-embedding-ada-002` and ensure you have your OpenAI API key configured correctly via `API_KEY` in your `.env` file (if you are not using Azure OpenAI).\n\n**Example `.env` configuration for OpenAI Embeddings:**\n\n```\nLLM_PROVIDER=openai\nAPI_KEY=YOUR_OPENAI_API_KEY # Your OpenAI API Key\nEMBEDDINGS_NAME=openai_text-embedding-ada-002\n```\n\n## Adding Support for Other Embedding Models\n\nIf you wish to use an embedding model that is not supported out-of-the-box, a good starting point for adding custom embedding model support is to examine the `base.py` file located in the `application/vectorstore` directory.\n\nSpecifically, pay attention to the `EmbeddingsWrapper` and `EmbeddingsSingleton` classes. `EmbeddingsWrapper` provides a way to wrap different embedding model libraries into a consistent interface for DocsGPT. `EmbeddingsSingleton` manages the instantiation and retrieval of embedding model instances. By understanding these classes and the existing embedding model implementations, you can create your own custom integration for virtually any embedding model library you desire." }, { "path": "docs/content/Models/local-inference.mdx", "content": "---\ntitle: Connecting DocsGPT to Local Inference Engines\ndescription: Connect DocsGPT to local inference engines for running LLMs directly on your hardware.\n---\n\n# Connecting DocsGPT to Local Inference Engines\n\nDocsGPT can be configured to leverage local inference engines, allowing you to run Large Language Models directly on your own infrastructure. This approach offers enhanced privacy and control over your LLM processing.\n\nCurrently, DocsGPT primarily supports local inference engines that are compatible with the OpenAI API format. This means you can connect DocsGPT to various local LLM servers that mimic the OpenAI API structure.\n\n## Configuration via `.env` file\n\nSetting up a local inference engine with DocsGPT is configured through environment variables in the `.env` file. For a detailed explanation of all settings, please consult the [DocsGPT Settings Guide](/Deploying/DocsGPT-Settings).\n\nTo connect to a local inference engine, you will generally need to configure these settings in your `.env` file:\n\n* **`LLM_PROVIDER`**: Crucially set this to `openai`. This tells DocsGPT to use the OpenAI-compatible API format for communication, even though the LLM is local.\n* **`LLM_NAME`**: Specify the model name as recognized by your local inference engine. This might be a model identifier or left as `None` if the engine doesn't require explicit model naming in the API request.\n* **`OPENAI_BASE_URL`**: This is essential. Set this to the base URL of your local inference engine's API endpoint. This tells DocsGPT where to find your local LLM server.\n* **`API_KEY`**: Generally, for local inference engines, you can set `API_KEY=None` as authentication is usually not required in local setups.\n\n## Native llama.cpp Support\n\nDocsGPT includes native support for llama.cpp without requiring an OpenAI-compatible server. To use this:\n\n```\nLLM_PROVIDER=llama.cpp\nLLM_NAME=your-model-name\n```\n\nThis provider integrates directly with llama.cpp Python bindings.\n\n## Supported Local Inference Engines (OpenAI API Compatible)\n\nDocsGPT is also readily configurable to work with the following local inference engines, all communicating via the OpenAI API format. Here are example `OPENAI_BASE_URL` values for each, based on default setups:\n\n| Inference Engine | `LLM_PROVIDER` | `OPENAI_BASE_URL` |\n| :---------------------------- | :------------- | :------------------------- |\n| LLaMa.cpp (server mode) | `openai` | `http://localhost:8000/v1` |\n| Ollama | `openai` | `http://localhost:11434/v1` |\n| Text Generation Inference (TGI)| `openai` | `http://localhost:8080/v1` |\n| SGLang | `openai` | `http://localhost:30000/v1` |\n| vLLM | `openai` | `http://localhost:8000/v1` |\n| Aphrodite | `openai` | `http://localhost:2242/v1` |\n| FriendliAI | `openai` | `http://localhost:8997/v1` |\n| LMDeploy | `openai` | `http://localhost:23333/v1` |\n\n**Important Note on `localhost` vs `host.docker.internal`:**\n\nThe `OPENAI_BASE_URL` examples above use `http://localhost`. If you are running DocsGPT within Docker and your local inference engine is running on your host machine (outside of Docker), you will likely need to replace `localhost` with `http://host.docker.internal` to ensure Docker can correctly access your host's services. For example, `http://host.docker.internal:11434/v1` for Ollama.\n\n## How the Model Registry Works\n\nDocsGPT uses a **Model Registry** to automatically detect and register available models based on your environment configuration. Understanding this system helps you configure models correctly.\n\n### Automatic Model Detection\n\nWhen DocsGPT starts, the Model Registry scans your environment variables and automatically registers models from providers that have valid API keys configured:\n\n| Environment Variable | Provider Models Registered |\n| :--------------------- | :------------------------- |\n| `OPENAI_API_KEY` | OpenAI models (gpt-5.1, gpt-5-mini, etc.) |\n| `ANTHROPIC_API_KEY` | Anthropic models (Claude family) |\n| `GOOGLE_API_KEY` | Google models (Gemini family) |\n| `GROQ_API_KEY` | Groq models (Llama, Mixtral) |\n| `HUGGINGFACE_API_KEY` | HuggingFace models |\n\nYou can also use the generic `API_KEY` variable with `LLM_PROVIDER` to configure a single provider.\n\n### Custom OpenAI-Compatible Models\n\nWhen you set `OPENAI_BASE_URL` along with `LLM_PROVIDER=openai` and `LLM_NAME`, the registry automatically creates a custom model entry pointing to your local inference server. This is how local engines like Ollama, vLLM, and others get registered.\n\n### Default Model Selection\n\nThe registry determines the default model in this priority order:\n\n1. If `LLM_NAME` is set and matches a registered model, that model becomes the default\n2. Otherwise, the first model from the configured `LLM_PROVIDER` is selected\n3. If neither is set, the first available model in the registry is used\n\n### Multiple Providers\n\nYou can configure multiple API keys simultaneously (e.g., both `OPENAI_API_KEY` and `ANTHROPIC_API_KEY`). The registry will load models from all configured providers, giving users the ability to switch between them in the UI.\n\n## Adding Support for Other Local Engines\n\nWhile DocsGPT currently focuses on OpenAI API compatible local engines, you can extend its capabilities to support other local inference solutions. To do this, navigate to the `application/llm` directory in the DocsGPT repository. Examine the existing Python files for examples of LLM integrations. You can create a new module for your desired local engine, and then register it in the `llm_creator.py` file within the same directory. This allows for custom integration with a wide range of local LLM servers beyond those listed above." }, { "path": "docs/content/Tools/_meta.js", "content": "export default {\n \"basics\": {\n \"title\": \"🔧 Tools Basics\",\n \"href\": \"/Tools/basics\"\n },\n \"api-tool\": {\n \"title\": \"🗝️ API Tool\",\n \"href\": \"/Tools/api-tool\"\n },\n \"creating-a-tool\": {\n \"title\": \"🛠️ Creating a Custom Tool\",\n \"href\": \"/Tools/creating-a-tool\"\n }\n}\n" }, { "path": "docs/content/Tools/api-tool.mdx", "content": "---\ntitle: 🗝️ Generic API Tool\ndescription: Learn how to configure and use the API Tool in DocsGPT to connect with any RESTful API without writing custom code.\n---\n\nimport { Callout } from 'nextra/components';\nimport Image from 'next/image';\n\n# Using the Generic API Tool\n\nThe API Tool provides a no-code/low-code solution to make DocsGPT interact with third-party or internal RESTful APIs. It acts as a bridge, allowing the Large Language Model (LLM) to leverage external services based on your chat interactions.\n This guide will walk you through its capabilities, configuration, and best practices.\n\n## Introduction to the Generic API Tool\n\n**When to Use It:**\n * Ideal for quickly integrating existing APIs where the interaction involves standard HTTP requests (GET, POST, PUT, DELETE).\n * Suitable for fetching data to enrich answers (e.g., current weather, stock prices, product details).\n * Useful for triggering simple actions in other systems (e.g., sending a notification, creating a basic task).\n\n**Contrast with Custom Python Tools:**\n * **API Tool:** Best for straightforward API calls. Configuration is done through the DocsGPT UI.\n * **Custom Python Tools:** Preferable when you need complex logic before or after the API call, handle non-standard authentication (like complex OAuth flows), manage multi-step API interactions, or require intricate data processing not easily managed by the LLM alone. See [Creating a Custom Tool](/Tools/creating-a-tool) for more.\n\n## Capabilities of the API Tool\n\n**Supported HTTP Methods:** You can configure actions using standard HTTP methods such as:\n * `GET`: To retrieve data.\n * `POST`: To submit data to create a new resource.\n * `PUT`: To update an existing resource.\n * `DELETE`: To remove a resource.\n\n**Request Configuration:**\n * **Headers:** Define static or dynamic HTTP headers for authentication (e.g., API keys), content type specification, etc.\n * **Query Parameters:** Specify URL query parameters, which can be static or dynamically filled by the LLM based on user input.\n * **Request Body:** Define the structure of the request body (e.g., JSON), with fields that can be static or dynamically populated by the LLM.\n\n**Response Handling:**\n * The API Tool executes the request and receives the raw response from the API (typically JSON or plain text).\n * This raw response is then passed back to the LLM.\n * The LLM uses this response, along with the context of your query and the description of the API tool action, to formulate an answer or decide on follow-up actions. The API tool itself doesn't deeply parse or transform the response beyond basic content type detection (e.g., loading JSON into a parsable object).\n\n## Configuring an API as a Tool\n\nYou can configure the API Tool through the DocsGPT user interface, found in **Settings -> Tools**. When you add or modify an API Tool, you'll define specific actions that DocsGPT can perform.\n\n\n The configuration involves defining how DocsGPT should call an API endpoint. Each configured API call essentially becomes a distinct \"action\" the LLM can choose to use.\n\n\nBelow is an example of how you might configure an API action, inspired by setting up a phone number validation service:\n\n\n_Figure 1: Example configuration for an API Tool action to validate phone numbers._\n\n**Defining an API Endpoint/Action:**\n\nWhen you configure a new API action, you'll fill in the following fields:\n\n- **`Name`:** A user-friendly name for this specific API action (e.g., \"Phone-check\" as in the image, or more specific like \"ValidateUSPhoneNumber\"). This helps in managing your tools.\n- **`Description`:** This is a **critical field**. Provide a clear and concise description of what the API action does, what kind of input it expects (implicitly), and what kind of output it provides. The LLM uses this description to understand when and how to use this action.\n- **`URL`:** The full endpoint URL for the API request.\n- **`HTTP Method`:** Select the appropriate HTTP method (e.g., GET, POST) from a dropdown.\n- **`Headers`:** You can add custom HTTP headers as key-value pairs (Name, Value). Indicate if the value should be `Filled by LLM` or is static. If filled by LLM, provide a `Description` for the LLM.\n\n- **`Query Parameters`:** For `GET` requests or when parameters are sent in the URL.\n * **`Name`:** The name of the query parameter (e.g., `api_key`, `phone`).\n * **`Type`:** The data type of the parameter (e.g., `string`).\n * **`Filled by LLM` (Checkbox):**\n - **Unchecked (Static):** The `Value` you provide will be used for every call (e.g., for an `api_key` that doesn't change).\n - **Checked (Dynamic):** The LLM will extract the appropriate value from the user's chat query based on the `Description` you provide for this parameter. The `Value` field is typically left empty or contains a placeholder if `Filled by LLM` is checked.\n * `Description`: Context for the LLM if the parameter is to be filled dynamically, or for your own reference if static.\n * `Value`: The static value if not filled by LLM.\n\n- **`Request Body`:** Used to send data (commonly JSON) to the API. Similar to Query Parameters, you define fields with `Name`, `Type`, whether it's `Filled by LLM`, a `Description` for dynamic fields, and a static `Value` if applicable.\n\n**Response Handling Guidance for the LLM:**\n\nWhile the API Tool configuration UI doesn't have explicit fields for defining response parsing rules (like JSONPath extractors), you significantly influence how the LLM handles the response through:\n * **Tool Action `Description`:** Clearly state what kind of information the API returns (e.g., \"This API returns a JSON object with 'status' and 'location' fields for the phone number.\"). This helps the LLM know what to look for in the API's output.\n * **Prompt Engineering:** For more complex scenarios, you might need to adjust your global or agent-specific prompts to guide DocsGPT on how to interpret and present information from API tool responses. See [Customising Prompts](/Guides/Customising-prompts).\n\n## Using the Configured API Tool in Chat\n\nOnce an API action is configured and enabled, DocsGPT's LLM can decide to use it based on your natural language queries.\n\n**Example (based on the phone validation tool in Figure 1):**\n\n1. **User Query:** \"Hey DocsGPT, can you check if +14155555555 is a valid phone number?\"\n\n2. **DocsGPT (LLM Orchestration):**\n * The LLM analyzes the query.\n * It matches the intent (\"check if ... is a valid phone number\") with the description of the \"Phone-check\" API action.\n * It identifies `+14155555555` as the value for the `phone` parameter (which was marked as `Filled by LLM` with the description \"Phone number to check\").\n * DocsGPT constructs the GET API request.\n3. **API Tool Execution:**\n * The API Tool makes the HTTP GET request.\n * The external API (AbstractAPI) processes the request and returns a JSON response, e.g.:\n ```json\n {\n \"phone\": \"+14155555555\",\n \"valid\": true,\n \"format\": {\n \"international\": \"+1 415-555-5555\",\n \"national\": \"(415) 555-5555\"\n },\n \"country\": {\n \"code\": \"US\",\n \"name\": \"United States\",\n \"prefix\": \"+1\"\n },\n \"location\": \"California\",\n \"type\": \"Landline\"\n }\n ```\n\n4. **DocsGPT Response Formulation:**\n * The API Tool passes this JSON response back to the LLM.\n * The LLM, guided by the tool's description and the user's original query, extracts relevant information and formulates a user-friendly answer.\n * **DocsGPT Chat Response:** \"Yes, +14155555555 appears to be a valid landline phone number in California, United States.\"\n\n## Advanced Tips and Best Practices\n\n**Clear Description is the Key:** The LLM relies heavily on the `Description` field of the API action and its parameters. Make them unambiguous and action-oriented. Clearly state what the tool does and what kind of input it expects (even if implicitly through parameter descriptions). \n\n**Iterative Testing:** After configuring an API tool, test it with various phrasings of user queries to ensure the LLM triggers it correctly and interprets the response as expected.\n \n**Error Handling:**\n * If an API call fails, the API Tool will return an error message and status code from the `requests` library or the API itself. The LLM may relay this error or try to explain it.\n * Check DocsGPT's backend logs for more detailed error information if you encounter issues.\n \n**Security Considerations:**\n * **API Keys:** Be mindful of API keys and other sensitive credentials. The example image shows an API key directly in the configuration. For production or shared environments avoid exposing configurations with sensitive keys.\n * **Rate Limits:** Be aware of the rate limits of the APIs you are integrating. Frequent calls from DocsGPT could exceed these limits.\n * **Data Privacy:** Consider the data privacy implications of sending user query data to third-party APIs.\n- **Idempotency:** For tools that modify data (POST, PUT, DELETE), be aware of whether the API operations are idempotent to avoid unintended consequences from repeated calls if the LLM retries an action.\n\n## Limitations\n\nWhile powerful, the Generic API Tool has some limitations:\n\n- **Complex Authentication:** Advanced authentication flows like OAuth 2.0 (especially 3-legged OAuth requiring user redirection) or custom signature-based authentication often require custom Python tools.\n- **Multi-Step API Interactions:** If a task requires multiple API calls that depend on each other (e.g., fetch a list, then for each item, fetch details), this kind of complex chaining and logic is better handled by a custom Python tool.\n- **Complex Data Transformations:** If the API response needs significant transformation or processing before being useful to the LLM, a custom Python tool offers more flexibility.\n- **Real-time Streaming (SSE, WebSockets):** The tool is designed for request-response interactions, not for maintaining persistent streaming connections.\n\nFor scenarios that exceed these limitations, developing a [Custom Python Tool](/Tools/creating-a-tool) is the recommended approach." }, { "path": "docs/content/Tools/basics.mdx", "content": "---\ntitle: Tools Basics - Enhancing DocsGPT Capabilities\ndescription: Understand what DocsGPT Tools are, how they work, and explore the built-in tools available to extend DocsGPT's functionality.\n---\n\nimport { Callout } from 'nextra/components';\nimport Image from 'next/image';\nimport { ToolCards } from '../../components/ToolCards';\n\n# Understanding DocsGPT Tools\n\nDocsGPT Tools are powerful extensions that significantly enhance the capabilities of your DocsGPT application. \nThey allow DocsGPT to move beyond its core function of retrieving information from your documents and enable it to perform actions, \ninteract with external data sources, and integrate with other services. You can find and configure available tools within \nthe \"Tools\" section of the DocsGPT application settings in the user interface.\n\n## What are Tools?\n\n- **Purpose:** The primary purpose of Tools is to bridge the gap between understanding a user's request (natural language processing by the LLM) and executing a tangible action. This could involve fetching live data from the web, sending notifications, running code snippets, querying databases, or interacting with third-party APIs.\n\n- **LLM as an Orchestrator:** The Large Language Model (LLM) at the heart of DocsGPT is designed to act as an intelligent orchestrator. Based on your query and the declared capabilities of the available tools (defined in their metadata), the LLM decides if a tool is needed, which tool to use, and what parameters to pass to it.\n\n- **Action-Oriented Interactions:** Tools enable more dynamic and action-oriented interactions. For example:\n * *\"What's the latest news on renewable energy?\"* - This might trigger a web search tool to fetch current articles.\n * *\"Fetch the order status for customer ID 12345 from our database.\"* - This could use a database tool.\n * *\"Summarize the content of this webpage and send the summary to the #general channel on Telegram.\"* - This might involve a web scraping tool followed by a Telegram notification tool.\n\n## Overview of Built-in Tools\n\nDocsGPT includes a suite of pre-built tools designed to expand its capabilities out-of-the-box. Below is an overview of the currently available tools. \n\n\n\n## Using Tools in DocsGPT (User Perspective)\n\nInteracting with tools in DocsGPT is designed to be intuitive:\n\n1. **Natural Language Interaction:** As a user, you typically interact with DocsGPT using natural language queries or commands. The LLM within DocsGPT analyzes your input to determine if a specific task can or should be handled by one of the available and configured tools.\n\n2. **Configuration in UI:**\n * Tools are generally managed and configured within the DocsGPT application's settings, found under a \"Tools\" section in the GUI.\n * For tools that interact with external services (like Brave Search, Telegram, or any service via the API Tool), you might need to provide authentication credentials (e.g., API keys, tokens) or specific endpoint information during the tool's setup in the UI.\n\n3. **Prompt Engineering for Tools:** While the LLM aims to intelligently use tools, for more complex or reliable agent-like behaviors, you might need to customize the system prompts. Modifying the prompt can guide the LLM on when and how to prioritize or chain tools to achieve specific outcomes, especially if you're building an agent designed to perform a certain sequence of actions every time. For more on this, see [Customising Prompts](/Guides/Customising-prompts).\n\n## Advancing with Tools\n\nUnderstanding the basics of DocsGPT Tools opens up many possibilities:\n\n* **Leverage the API Tool:** For quick integrations with numerous external services, explore the [API Tool Detailed Guide](/Tools/api-tool).\n* **Develop Custom Tools:** If you have specific needs not covered by built-in tools or the generic API tool, you can develop your own. See our guide on `[Developing Custom Tools](/Tools/creating-a-tool)` (placeholder for now).\n* **Build AI Agents:** Tools are the fundamental building blocks for creating sophisticated AI agents within DocsGPT. Explore how these can be combined by looking into the `[Agents section/tab concept - link to be added once available]`.\n\nBy harnessing the power of Tools, you can transform DocsGPT into a more versatile and proactive assistant tailored to your unique workflows." }, { "path": "docs/content/Tools/creating-a-tool.mdx", "content": "---\ntitle: 🛠️ Creating a Custom Tool\ndescription: Learn how to create custom Python tools to extend DocsGPT's functionality and integrate with various services or perform specific actions.\n---\n\nimport { Callout } from 'nextra/components';\nimport { Steps } from 'nextra/components';\n\n# 🛠️ Creating a Custom Python Tool\n\nThis guide provides developers with a comprehensive, step-by-step approach to creating their own custom tools for DocsGPT. By developing custom tools, you can significantly extend DocsGPT's capabilities, enabling it to interact with new data sources, services, and perform specialized actions tailored to your unique needs.\n\n## Introduction to Custom Tool Development\n\n### Why Create Custom Tools?\n\nWhile DocsGPT offers a range of built-in tools and a versatile API Tool, there are many scenarios where a custom Python tool is the best solution:\n\n* **Integrating with Proprietary Systems:** Connect to internal APIs, databases, or services that are not publicly accessible or require complex authentication.\n* **Adding Domain-Specific Functionalities:** Implement logic specific to your industry or use case that isn't covered by general-purpose tools.\n* **Automating Unique Workflows:** Create tools that orchestrate multiple steps or interact with systems in a way unique to your operational needs.\n* **Connecting to Any System with an Accessible Interface:** If you can interact with a system programmatically using Python (e.g., through libraries, SDKs, or direct HTTP requests), you can likely build a DocsGPT tool for it.\n* **Complex Logic or Data Transformation:** When API interactions require intricate logic before sending a request or after receiving a response, or when data needs significant transformation that is difficult for an LLM to handle directly.\n\n### Prerequisites\n\nBefore you begin, ensure you have:\n\n* A solid understanding of Python programming.\n* Familiarity with the DocsGPT project structure, particularly the `application/agents/tools/` directory where custom tools reside.\n* Basic knowledge of how APIs work, as many tools involve interacting with external or internal APIs.\n* Your DocsGPT development environment set up. If not, please refer to the [Setting Up a Development Environment](/Deploying/Development-Environment) guide.\n\n## The Anatomy of a DocsGPT Tool\n\nCustom tools in DocsGPT are Python classes that inherit from a base `Tool` class and implement specific methods to define their behavior, capabilities, and configuration needs.\n\nThe **foundation** for all custom tools is the abstract base class, located in `application/agents/tools/base.py`. Your custom tool class **must** inherit from this class.\n\n### Essential Methods to Implement\n\nYour custom tool class needs to implement the following methods:\n\n1. **`__init__(self, config: dict)`**\n \n - **Purpose:** The constructor for your tool. It's called when DocsGPT initializes the tool.\n - **Usage:** This method is typically used to receive and store tool-specific configurations passed via the `config` dictionary. This dictionary is populated based on the tool's settings, often configured through the DocsGPT UI or environment variables. For example, you would store API keys, base URLs, or database connection strings here.\n - **Example** (`brave.py`)**:**\n ``` python\n class BraveSearchTool(Tool):\n def __init__(self, config):\n self.config = config\n self.token = config.get(\"token\", \"\") # API Key for Brave Search\n self.base_url = \"https://api.search.brave.com/res/v1\"\n ```\n\n2. **`execute_action(self, action_name: str, **kwargs) -> dict`**\n \n - **Purpose:** This is the workhorse of your tool. The LLM, acting as an agent, calls this method when it decides to use one of the actions your tool provides.\n - **Parameters:**\n - `action_name` (str): A string specifying which of the tool's actions to run (e.g., \"brave_web_search\").\n - `**kwargs` (dict): A dictionary containing the parameters for that specific action. These parameters are defined in the tool's metadata (`get_actions_metadata()`) and are extracted or inferred by the LLM from the user's query.\n - **Return Value:** A dictionary containing the result of the action. It's good practice to include keys like:\n - `status_code` (int): An HTTP-like status code (e.g., 200 for success, 500 for error).\n - `message` (str): A human-readable message describing the outcome.\n - `data` (any): The actual data payload returned by the action (if applicable).\n - `error` (str): An error message if the action failed.\n - **Example (`read_webpage.py`):**\n \n ``` python\n def execute_action(self, action_name: str, **kwargs) -> str:\n if action_name != \"read_webpage\":\n return f\"Error: Unknown action '{action_name}'. This tool only supports 'read_webpage'.\"\n \n url = kwargs.get(\"url\")\n if not url:\n return \"Error: URL parameter is missing.\"\n # ... (logic to fetch and parse webpage) ...\n try:\n # ...\n return markdown_content \n except Exception as e:\n return f\"Error processing URL {url}: {e}\"\n ```\n \n A more structured return:\n \n ``` python\n # ... inside execute_action\n try:\n # ... logic ...\n return {\"status_code\": 200, \"message\": \"Webpage read successfully\", \"data\": markdown_content}\n except Exception as e:\n return {\"status_code\": 500, \"message\": f\"Error processing URL {url}\", \"error\": str(e)}\n ```\n \n3. **`get_actions_metadata(self) -> list`**\n \n - **Purpose:** This method is **critical** for the LLM to understand what your tool can do, when to use it, and what parameters it needs. It effectively advertises your tool's capabilities.\n - **Return Value:** A list of dictionaries. Each dictionary describes one distinct action the tool can perform and must follow a specific JSON schema structure.\n - `name` (str): A unique and descriptive name for the action (e.g., `mytool_get_user_details`). It's a common convention to prefix with the tool name to avoid collisions.\n - `description` (str): A clear, concise, and unambiguous description of what the action does. **Write this for the LLM.** The LLM uses this description to decide if this action is appropriate for a given user query.\n - `parameters` (dict): A JSON Schema object defining the parameters that the action expects. This schema tells the LLM what arguments are needed, their types, and which are required.\n - `type`: Should always be `\"object\"`.\n - `properties`: A dictionary where each key is a parameter name, and the value is an object defining its `type` (e.g., \"string\", \"integer\", \"boolean\") and `description`.\n - `required`: A list of strings, where each string is the name of a parameter that is mandatory for the action.\n - **Example (`postgres.py` - partial):**\n \n ``` python\n def get_actions_metadata(self):\n return [\n {\n \"name\": \"postgres_execute_sql\",\n \"description\": \"Execute an SQL query against the PostgreSQL database...\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"sql_query\": {\n \"type\": \"string\",\n \"description\": \"The SQL query to execute.\",\n },\n },\n \"required\": [\"sql_query\"],\n \"additionalProperties\": False, # Good practice to prevent unexpected params\n },\n },\n # ... other actions like postgres_get_schema\n ]\n ```\n \n4. **`get_config_requirements(self) -> dict`**\n \n - **Purpose:** Defines the configuration parameters that your tool needs to function (e.g., API keys, specific base URLs, connection strings, default settings). This information can be used by the DocsGPT UI to dynamically render configuration fields for your tool or for validation.\n - **Return Value:** A dictionary where keys are the configuration item names (which will be keys in the `config` dict passed to `__init__`) and values are dictionaries describing each requirement:\n - `type` (str): The expected data type of the config value (e.g., \"string\", \"boolean\", \"integer\").\n - `description` (str): A human-readable description of what this configuration item is for.\n - `secret` (bool, optional): Set to `True` if the value is sensitive (e.g., an API key) and should be masked or handled specially in UIs. Defaults to `False`.\n - **Example (`brave.py`):**\n \n ``` python\n def get_config_requirements(self):\n return {\n \"token\": { # This 'token' will be a key in the config dict for __init__\n \"type\": \"string\",\n \"description\": \"Brave Search API key for authentication\",\n \"secret\": True\n },\n }\n ``` \n\n## Tool Registration and Discovery\n\nDocsGPT's ToolManager (located in application/agents/tools/tool_manager.py) automatically discovers and loads tools.\n\nAs long as your custom tool:\n\n1. Is placed in a Python file within the `application/agents/tools/` directory (and the filename is not `base.py` or starts with `__`).\n2. Correctly inherits from the `Tool` base class.\n3. Implements all the abstract methods (`execute_action`, `get_actions_metadata`, `get_config_requirements`).\n\nThe `ToolManager` should be able to load it when DocsGPT starts. \n\n## Configuration & Secrets Management\n\n- **Configuration Source:** The `config` dictionary passed to your tool's `__init__` method is typically populated from settings defined in the DocsGPT UI (if available for the tool) or from environment variables/configuration files that DocsGPT loads (see [⚙️ App Configuration](/Deploying/DocsGPT-Settings)). The keys in this dictionary should match the names you define in `get_config_requirements()`.\n- **Secrets:** Never hardcode secrets (like API keys or passwords) directly into your tool's Python code. Instead, define them as configuration requirements (using `secret: True` in `get_config_requirements()`) and let DocsGPT's configuration system inject them via the `config` dictionary at runtime. This ensures that secrets are managed securely and are not exposed in your codebase.\n\n## Best Practices for Tool Development\n\n- **Atomicity:** Design tool actions to be as atomic (single, well-defined purpose) as possible. This makes them easier for the LLM to understand and combine.\n- **Clarity in Metadata:** Ensure action names and descriptions in `get_actions_metadata()` are extremely clear, specific, and unambiguous. This is the primary way the LLM understands your tool.\n- **Robust Error Handling:** Implement comprehensive error handling within your `execute_action` logic (and the private methods it calls). Return informative error messages in the result dictionary so the LLM or user can understand what went wrong.\n- **Security:**\n - Be mindful of the security implications of your tool, especially if it interacts with sensitive systems or can execute arbitrary code/queries.\n - Validate and sanitize any inputs, especially if they are used to construct database queries or shell commands, to prevent injection attacks.\n- **Performance:** Consider the performance implications of your tool's actions. If an action is slow, it will impact the user experience. Optimize where possible.\n\n## (Optional) Contributing Your Tool\n\nIf you develop a custom tool that you believe could be valuable to the broader DocsGPT community and is general-purpose:\n\n1. Ensure it's well-documented (both in code and with clear metadata).\n2. Make sure it adheres to the best practices outlined above.\n3. Consider opening a Pull Request to the [DocsGPT GitHub repository](https://github.com/arc53/DocsGPT) with your new tool, including any necessary documentation updates.\n\nBy following this guide, you can create powerful custom tools that extend DocsGPT's capabilities to your specific operational environment." }, { "path": "docs/content/_meta.js", "content": "export default {\n \"index\": \"Home\",\n \"quickstart\": \"Quickstart\",\n \"Deploying\": \"Deploying\",\n \"Models\": \"Models\",\n \"Tools\": \"Tools\",\n \"Agents\": \"Agents\",\n \"Extensions\": \"Extensions\",\n \"https://gptcloud.arc53.com/\": {\n \"title\": \"API\",\n \"href\": \"https://gptcloud.arc53.com/\"\n },\n \"Guides\": \"Guides\",\n \"changelog\": {\n \"title\": \"Changelog\",\n \"display\": \"hidden\"\n }\n}\n" }, { "path": "docs/content/changelog.mdx", "content": "---\ntitle: 'Changelog'\n---" }, { "path": "docs/content/index.mdx", "content": "---\ntitle: 'Home'\ndescription: Documentation of DocsGPT - quickstart, deployment guides, model configuration, and widget integration documentation.\n---\nimport { Cards } from 'nextra/components'\nimport Image from 'next/image'\n\nexport const allGuides = {\n \"quickstart\": {\n \"title\": \"⚡️ Quickstart\",\n \"href\": \"/quickstart\"\n },\n \"DocsGPT-Settings\": {\n \"title\": \"⚙️ App Configuration\",\n \"href\": \"/Deploying/DocsGPT-Settings\"\n },\n \"Docker-Deploying\": {\n \"title\": \"🛳️ Docker Setup\",\n \"href\": \"/Deploying/Docker-Deploying\"\n },\n \"Development-Environment\": {\n \"title\": \"🛠️Development Environment\",\n \"href\": \"/Deploying/Development-Environment\"\n },\n \"https://gptcloud.arc53.com/\": {\n \"title\": \"🧑‍💻️ API\",\n \"href\": \"https://gptcloud.arc53.com/\",\n \"newWindow\": true\n },\n \"cloud-providers\": {\n \"title\": \"☁️ Cloud Providers\",\n \"href\": \"/Models/cloud-providers\"\n },\n \"local-inference\": {\n \"title\": \"🖥️ Local Inference\",\n \"href\": \"/Models/local-inference\"\n },\n \"embeddings\": {\n \"title\": \"📝 Embeddings\",\n \"href\": \"/Models/embeddings\"\n },\n \"api-key-guide\": {\n \"title\": \"🔑 Getting API key\",\n \"href\": \"/Extensions/api-key-guide\"\n },\n \"chat-widget\": {\n \"title\": \"💬️ Chat Widget\",\n \"href\": \"/Extensions/chat-widget\"\n },\n \"search-widget\": {\n \"title\": \"🔎 Search Widget\",\n \"href\": \"/Extensions/search-widget\"\n },\n \"Customising-prompts\": {\n \"title\": \"️💻 Customising Prompts\",\n \"href\": \"/Guides/Customising-prompts\"\n }\n};\n\n# **DocsGPT 🦖**\n\nDocsGPT is an open-source genAI tool that helps users get reliable answers from any knowledge source, while avoiding hallucinations. It enables quick and reliable information retrieval, with tooling and agentic system capability built in, including speech-to-text workflows for chat and audio knowledge ingestion.\n\n\n\n\n\nTry it yourself: [https://www.docsgpt.cloud/](https://www.docsgpt.cloud/)\n\n### Features:\n- **🗂️ Wide Format Support:** Reads PDF, DOCX, CSV, XLSX, EPUB, MD, RST, HTML, MDX, JSON, PPTX, images, and audio files such as MP3, WAV, M4A, OGG, and WebM.\n- **🎙️ Speech Workflows:** Record voice input into chat, transcribe on the backend, and index uploaded audio files as searchable source material.\n- **🌐 Web & Data Integration:** Ingests from URLs, sitemaps, Reddit, GitHub and web crawlers.\n- **✅ Reliable Answers:** Get accurate, hallucination-free responses with source citations viewable in a clean UI.\n- **🔑 Streamlined API Keys:** Generate keys linked to your settings, documents, and models, simplifying chatbot and integration setup.\n- **🔗 Actionable Tooling:** Connect to APIs, tools, and other services to enable LLM actions.\n- **🧩 Pre-built Integrations:** Use readily available HTML/React chat widgets, search tools, Discord/Telegram bots, and more.\n- **🔌 Flexible Deployment:** Works with major LLMs (OpenAI, Google, Anthropic) and local models (Ollama, llama_cpp).\n- **🏢 Secure & Scalable:** Run privately and securely with Kubernetes support, designed for enterprise-grade reliability.\n\n**Contribute and Extend:** As an open-source project, community contributions are highly encouraged! If you develop valuable customizations or enhancements, consider contributing them back to the main repository to benefit other DocsGPT users.\n\n (\n \n ))}\n />\n" }, { "path": "docs/content/quickstart.mdx", "content": "---\ntitle: Quickstart - Launching DocsGPT Web App\ndescription: Get started with DocsGPT quickly by launching the web application using the setup script.\n---\n\n# Quickstart\n\n**Prerequisites:**\n\n* **Docker:** Ensure you have Docker installed and running on your system.\n\n## Launching DocsGPT (macOS and Linux)\n\nThe easiest way to launch DocsGPT is using the provided `setup.sh` script. This script automates the configuration process and offers several setup options.\n\n**Steps:**\n\n1. **Download the DocsGPT Repository:**\n\n First, you need to download the DocsGPT repository to your local machine. You can do this using Git:\n\n ```bash\n git clone https://github.com/arc53/DocsGPT.git\n cd DocsGPT\n ```\n\n2. **Run the `setup.sh` script:**\n\n Navigate to the DocsGPT directory in your terminal and execute the `setup.sh` script:\n\n ```bash\n ./setup.sh\n ```\n\n3. **Follow the interactive setup:**\n\n The `setup.sh` script will guide you through an interactive menu with the following options:\n\n ```\n Welcome to DocsGPT Setup!\n How would you like to proceed?\n 1) Use DocsGPT Public API Endpoint (simple and free)\n 2) Serve Local (with Ollama)\n 3) Connect Local Inference Engine\n 4) Connect Cloud API Provider\n 5) Advanced: Build images locally (for developers)\n Choose option (1-5):\n ```\n\n Let's break down each option:\n\n * **1) Use DocsGPT Public API Endpoint (simple and free):** This is the simplest option to get started. It utilizes the DocsGPT public API, requiring no API keys or local model downloads. Choose this for a quick and easy setup.\n\n * **2) Serve Local (with Ollama):** This option allows you to run a Large Language Model locally using [Ollama](https://ollama.com/). You'll be prompted to choose between CPU or GPU for Ollama and select a model to download. This is a good option for local processing and experimentation.\n\n * **3) Connect Local Inference Engine:** If you are already running a local inference engine like Llama.cpp, Text Generation Inference (TGI), vLLM, or others, choose this option. You'll be asked to select your engine and provide the necessary connection details. This is for users with existing local LLM infrastructure.\n\n * **4) Connect Cloud API Provider:** This option lets you connect DocsGPT to a commercial Cloud API provider such as OpenAI, Google (Vertex AI/Gemini), Anthropic (Claude), Groq, HuggingFace Inference API, or Azure OpenAI. You will need an API key from your chosen provider. Select this if you prefer to use a powerful cloud-based LLM.\n\n * **5) Modify DocsGPT's source code and rebuild the Docker images locally.** Instead of pulling prebuilt images from Docker Hub or using the hosted/public API, you build the entire backend and frontend from source, customizing how DocsGPT works internally, or run it in an environment without internet access.\n\n After selecting an option and providing any required information (like API keys or model names), the script will configure your `.env` file and start DocsGPT using Docker Compose.\n\n4. **Access DocsGPT in your browser:**\n\n Once the setup is complete and Docker containers are running, navigate to [http://localhost:5173/](http://localhost:5173/) in your web browser to access the DocsGPT web application.\n\n5. **Stopping DocsGPT:**\n\n To stop DocsGPT, simply open a new terminal in the `DocsGPT` directory and run:\n\n ```bash\n docker compose -f deployment/docker-compose.yaml down\n ```\n (or the specific `docker compose` command shown at the end of the `setup.sh` execution, which may include optional compose files depending on your choices).\n\n## Launching DocsGPT (Windows)\n\nFor Windows users, we provide a PowerShell script that offers the same functionality as the macOS/Linux setup script.\n\n**Steps:**\n\n1. **Download the DocsGPT Repository:**\n\n First, you need to download the DocsGPT repository to your local machine. You can do this using Git:\n\n ```powershell\n git clone https://github.com/arc53/DocsGPT.git\n cd DocsGPT\n ```\n\n2. **Run the `setup.ps1` script:**\n\n Execute the PowerShell setup script:\n\n ```powershell\n PowerShell -ExecutionPolicy Bypass -File .\\setup.ps1\n ```\n\n3. **Follow the interactive setup:**\n\n Just like the Linux/macOS script, the PowerShell script will guide you through setting DocsGPT.\n The script will handle environment configuration and start DocsGPT based on your selections.\n\n4. **Access DocsGPT in your browser:**\n\n Once the setup is complete and Docker containers are running, navigate to [http://localhost:5173/](http://localhost:5173/) in your web browser to access the DocsGPT web application.\n\n5. **Stopping DocsGPT:**\n\n To stop DocsGPT run the Docker Compose down command displayed at the end of the setup script's execution.\n\n**Important for Windows:** Ensure Docker Desktop is installed and running correctly on your Windows system before proceeding. The script will attempt to start Docker if it's not running, but you may need to start it manually if there are issues.\n\n**Alternative Method:**\nIf you prefer a more manual approach, you can follow our [Docker Deployment documentation](/Deploying/Docker-Deploying) for detailed instructions on setting up DocsGPT on Windows using Docker commands directly.\n\n## Advanced Configuration\n\nFor more advanced customization of DocsGPT settings, such as configuring vector stores, embedding models, and other parameters, please refer to the [DocsGPT Settings documentation](/Deploying/DocsGPT-Settings). This guide explains how to modify the `.env` file or `settings.py` for deeper configuration.\n\nEnjoy using DocsGPT!\n" }, { "path": "docs/mdx-components.jsx", "content": "import { useMDXComponents as getThemeComponents } from 'nextra-theme-docs';\n\nexport function useMDXComponents(components) {\n return {\n ...getThemeComponents(),\n ...components,\n };\n}\n" }, { "path": "docs/next.config.js", "content": "const nextra = require('nextra').default;\n\nconst withNextra = nextra({\n defaultShowCopyCode: true,\n});\n\nmodule.exports = withNextra({\n reactStrictMode: true,\n});\n" }, { "path": "docs/package.json", "content": "{\n \"scripts\": {\n \"dev\": \"next dev\",\n \"build\": \"next build\",\n \"postbuild\": \"pagefind --site .next/server/app --output-path public/_pagefind\",\n \"start\": \"next start\"\n },\n \"license\": \"MIT\",\n \"dependencies\": {\n \"@vercel/analytics\": \"^1.1.1\",\n \"docsgpt-react\": \"^0.5.1\",\n \"next\": \"^15.5.9\",\n \"nextra\": \"^4.6.1\",\n \"nextra-theme-docs\": \"^4.6.1\",\n \"react\": \"^18.2.0\",\n \"react-dom\": \"^18.2.0\"\n },\n \"devDependencies\": {\n \"pagefind\": \"^1.3.0\",\n \"typescript\": \"^5.9.3\"\n }\n}\n" }, { "path": "docs/public/favicons/site.webmanifest", "content": "{\n \"name\": \"\",\n \"short_name\": \"\",\n \"icons\": [\n {\n \"src\": \"/android-chrome-192x192.png\",\n \"sizes\": \"192x192\",\n \"type\": \"image/png\"\n },\n {\n \"src\": \"/android-chrome-512x512.png\",\n \"sizes\": \"512x512\",\n \"type\": \"image/png\"\n }\n ],\n \"theme_color\": \"#ffffff\",\n \"background_color\": \"#ffffff\",\n \"display\": \"standalone\"\n}\n" }, { "path": "docs/public/llms.txt", "content": "# DocsGPT\n\n> DocsGPT is an open-source platform for building AI agents and assistants with document retrieval, tools, and multi-model support.\n\nThis file is a curated map of DocsGPT documentation for LLM and agent use.\nPrioritize Core, Deploying, and Agents for implementation tasks.\n\n## Core\n\n- [Docs Home](https://docs.docsgpt.cloud/): Main documentation landing page.\n- [Quickstart](https://docs.docsgpt.cloud/quickstart): Fastest path to run DocsGPT locally.\n- [Architecture](https://docs.docsgpt.cloud/Guides/Architecture): High-level system architecture.\n- [Development Environment](https://docs.docsgpt.cloud/Deploying/Development-Environment): Backend and frontend local setup.\n- [DocsGPT Settings](https://docs.docsgpt.cloud/Deploying/DocsGPT-Settings): Environment variables and core app configuration.\n\n## Deploying\n\n- [Docker Deployment](https://docs.docsgpt.cloud/Deploying/Docker-Deploying): Run DocsGPT with Docker and Docker Compose.\n- [Kubernetes Deployment](https://docs.docsgpt.cloud/Deploying/Kubernetes-Deploying): Deploy DocsGPT on Kubernetes clusters.\n- [Hosting DocsGPT](https://docs.docsgpt.cloud/Deploying/Hosting-the-app): Hosting overview with cloud options.\n\n## Agents\n\n- [Agent Basics](https://docs.docsgpt.cloud/Agents/basics): Core concepts for building and managing agents.\n- [Workflow Nodes](https://docs.docsgpt.cloud/Agents/nodes): Node types and behavior in agent workflows.\n- [Agent API](https://docs.docsgpt.cloud/Agents/api): Programmatic agent interaction (streaming and non-streaming).\n- [Agent Webhooks](https://docs.docsgpt.cloud/Agents/webhooks): Trigger and automate agents with webhooks.\n\n## Tools\n\n- [Tools Basics](https://docs.docsgpt.cloud/Tools/basics): How tools extend agent capabilities.\n- [Generic API Tool](https://docs.docsgpt.cloud/Tools/api-tool): Configure API calls without custom code.\n- [Creating a Custom Tool](https://docs.docsgpt.cloud/Tools/creating-a-tool): Build custom Python tools for DocsGPT.\n\n## Models\n\n- [Cloud LLM Providers](https://docs.docsgpt.cloud/Models/cloud-providers): Configure hosted model providers.\n- [Local Inference](https://docs.docsgpt.cloud/Models/local-inference): Connect DocsGPT to local inference backends.\n- [Embeddings](https://docs.docsgpt.cloud/Models/embeddings): Select and configure embedding models.\n\n## Extensions\n\n- [API Keys for Integrations](https://docs.docsgpt.cloud/Extensions/api-key-guide): Generate and use DocsGPT API keys.\n- [Chat Widget](https://docs.docsgpt.cloud/Extensions/chat-widget): Embed the DocsGPT chat widget.\n- [Search Widget](https://docs.docsgpt.cloud/Extensions/search-widget): Embed the DocsGPT search widget.\n- [Chrome Extension](https://docs.docsgpt.cloud/Extensions/Chrome-extension): Install and use the browser extension.\n- [Chatwoot Extension](https://docs.docsgpt.cloud/Extensions/Chatwoot-extension): Integrate DocsGPT with Chatwoot.\n\n## Integrations\n\n- [Google Drive Connector](https://docs.docsgpt.cloud/Guides/Integrations/google-drive-connector): Ingest and sync files from Google Drive.\n\n## Optional\n\n- [Customizing Prompts](https://docs.docsgpt.cloud/Guides/Customising-prompts): Template-based prompt customization.\n- [How to Train on Other Documentation](https://docs.docsgpt.cloud/Guides/How-to-train-on-other-documentation): Add additional documentation sources.\n- [Context Compression](https://docs.docsgpt.cloud/Guides/compression): Reduce context while preserving key information.\n- [OCR for Sources and Attachments](https://docs.docsgpt.cloud/Guides/ocr): OCR behavior for ingestion and chat uploads.\n- [How to Use Different LLMs](https://docs.docsgpt.cloud/Guides/How-to-use-different-LLM): Additional model-selection guidance.\n- [Avoiding Hallucinations](https://docs.docsgpt.cloud/Guides/My-AI-answers-questions-using-external-knowledge): Improve answer grounding with external knowledge.\n- [Amazon Lightsail Deployment](https://docs.docsgpt.cloud/Deploying/Amazon-Lightsail): Deploy DocsGPT on AWS Lightsail.\n- [Railway Deployment](https://docs.docsgpt.cloud/Deploying/Railway): Deploy DocsGPT on Railway.\n- [Changelog](https://docs.docsgpt.cloud/changelog): Project release history.\n" }, { "path": "docs/theme.config.jsx", "content": "const github = 'https://github.com/arc53/DocsGPT';\nconst isDevelopment = process.env.NODE_ENV === 'development';\n\nconst config = {\n docsRepositoryBase: `${github}/blob/main/docs`,\n darkMode: true,\n search: isDevelopment ? null : undefined,\n nextThemes: {\n defaultTheme: 'dark',\n },\n sidebar: {\n defaultMenuCollapseLevel: 1,\n },\n toc: {\n float: true,\n },\n editLink: 'Edit this page on GitHub',\n};\n\nexport default config;\n" }, { "path": "extensions/chatwoot/.env_sample", "content": "docsgpt_url=\nchatwoot_url=\ndocsgpt_key=\nchatwoot_token=xxxxx\naccount_id=(optional) 1\nassignee_id=(optional) 1" }, { "path": "extensions/chatwoot/__init__.py", "content": "" }, { "path": "extensions/chatwoot/app.py", "content": "import os\nimport pprint\n\nimport dotenv\nimport requests\nfrom flask import Flask, request\n\ndotenv.load_dotenv()\ndocsgpt_url = os.getenv(\"docsgpt_url\")\nchatwoot_url = os.getenv(\"chatwoot_url\")\ndocsgpt_key = os.getenv(\"docsgpt_key\")\nchatwoot_token = os.getenv(\"chatwoot_token\")\n# account_id = os.getenv(\"account_id\")\n# assignee_id = os.getenv(\"assignee_id\")\nlabel_stop = \"human-requested\"\n\n\ndef send_to_bot(sender, message):\n data = {\n 'sender': sender,\n 'question': message,\n 'api_key': docsgpt_key,\n 'embeddings_key': docsgpt_key,\n 'history': ''\n }\n headers = {\"Content-Type\": \"application/json\",\n \"Accept\": \"application/json\"}\n\n r = requests.post(f'{docsgpt_url}/api/answer',\n json=data, headers=headers)\n return r.json()['answer']\n\n\ndef send_to_chatwoot(account, conversation, message):\n data = {\n 'content': message\n }\n url = f\"{chatwoot_url}/api/v1/accounts/{account}/conversations/{conversation}/messages\"\n headers = {\"Content-Type\": \"application/json\",\n \"Accept\": \"application/json\",\n \"api_access_token\": f\"{chatwoot_token}\"}\n\n r = requests.post(url,\n json=data, headers=headers)\n return r.json()\n\n\napp = Flask(__name__)\n\n\n@app.route('/docsgpt', methods=['POST'])\ndef docsgpt():\n data = request.get_json()\n pp = pprint.PrettyPrinter(indent=4)\n pp.pprint(data)\n try:\n message_type = data['message_type']\n except KeyError:\n return \"Not a message\"\n message = data['content']\n conversation = data['conversation']['id']\n contact = data['sender']['id']\n account = data['account']['id']\n assignee = data['conversation']['meta']['assignee']['id']\n print(account)\n print(label_stop)\n print(data['conversation']['labels'])\n print(assignee)\n\n if label_stop in data['conversation']['labels']:\n return \"Label stop\"\n # elif str(account) != str(account_id):\n # return \"Not the right account\"\n\n # elif str(assignee) != str(assignee_id):\n # return \"Not the right assignee\"\n\n if (message_type == \"incoming\"):\n bot_response = send_to_bot(contact, message)\n create_message = send_to_chatwoot(\n account, conversation, bot_response)\n else:\n return \"Not an incoming message\"\n\n return create_message\n\n\nif __name__ == '__main__':\n app.run(host='0.0.0.0', port=80)\n" }, { "path": "extensions/chrome/_locales/en/messages.json", "content": "{\n \"l10nTabName\": {\n \"message\":\"Localization\"\n ,\"description\":\"name of the localization tab\"\n }\n ,\"l10nHeader\": {\n \"message\":\"It does localization too! (this whole tab is, actually)\"\n ,\"description\":\"Header text for the localization section\"\n }\n ,\"l10nIntro\": {\n \"message\":\"'L10n' refers to 'Localization' - 'L' an 'n' are obvious, and 10 comes from the number of letters between those two. It is the process/whatever of displaying something in the language of choice. It uses 'I18n', 'Internationalization', which refers to the tools / framework supporting L10n. I.e., something is internationalized if it has I18n support, and can be localized. Something is localized for you if it is in your language / dialect.\"\n ,\"description\":\"introduce the basic idea.\"\n }\n ,\"l10nProd\": {\n \"message\":\"You are planning to allow localization, right? You have no idea who will be using your extension! You have no idea who will be translating it! At least support the basics, it's not hard, and having the framework in place will let you transition much more easily later on.\"\n ,\"description\":\"drive the point home. It's good for you.\"\n }\n ,\"l10nFirstParagraph\": {\n \"message\":\"When the options page loads, elements decorated with data-l10n will automatically be localized!\"\n ,\"description\":\"inform that elements will be localized on load\"\n }\n ,\"l10nSecondParagraph\": {\n \"message\":\"If you need more complex localization, you can also define data-l10n-args. This should contain $containerType$ filled with $dataType$, which will be passed into Chrome's i18n API as $functionArgs$. In fact, this paragraph does just that, and wraps the args in mono-space font. Easy!\"\n ,\"description\":\"introduce the data-l10n-args attribute. End on a lame note.\"\n ,\"placeholders\": {\n \"containerType\": {\n \"content\":\"$1\"\n ,\"example\":\"'array', 'list', or something similar\"\n ,\"description\":\"type of the args container\"\n }\n ,\"dataType\": {\n \"content\":\"$2\"\n ,\"example\":\"string\"\n ,\"description\":\"type of data in each array index\"\n }\n ,\"functionArgs\": {\n \"content\":\"$3\"\n ,\"example\":\"arguments\"\n ,\"description\":\"whatever you call what you pass into a function/method. args, params, etc.\"\n }\n }\n }\n ,\"l10nThirdParagraph\": {\n \"message\":\"Message contents are passed right into innerHTML without processing - include any tags (or even scripts) that you feel like. If you have an input field, the placeholder will be set instead, and buttons will have the value attribute set.\"\n ,\"description\":\"inform that we handle placeholders, buttons, and direct HTML input\"\n }\n ,\"l10nButtonsBefore\": {\n \"message\":\"Different types of buttons are handled as well. <button> elements have their html set:\"\n }\n ,\"l10nButton\": {\n \"message\":\"in a button\"\n }\n ,\"l10nButtonsBetween\": {\n \"message\":\"while <input type='submit'> and <input type='button'> get their 'value' set (note: no HTML):\"\n }\n ,\"l10nSubmit\": {\n \"message\":\"a submit value\"\n }\n ,\"l10nButtonsAfter\": {\n \"message\":\"Awesome, no?\"\n }\n ,\"l10nExtras\": {\n \"message\":\"You can even set data-l10n on things like the <title> tag, which lets you have translatable page titles, or fieldset <legend> tags, or anywhere else - the default Boil.localize() behavior will check every tag in the document, not just the body.\"\n ,\"description\":\"inform about places which may not be obvious, like , etc\"\n }\n}\n" }, { "path": "extensions/chrome/dist/output.css", "content": "/*\n! tailwindcss v3.2.7 | MIT License | https://tailwindcss.com\n*/\n\n/*\n1. Prevent padding and border from affecting element width. (https://github.com/mozdevs/cssremedy/issues/4)\n2. Allow adding a border to an element by just adding a border-width. (https://github.com/tailwindcss/tailwindcss/pull/116)\n*/\n\n*,\n::before,\n::after {\n box-sizing: border-box;\n /* 1 */\n border-width: 0;\n /* 2 */\n border-style: solid;\n /* 2 */\n border-color: #e5e7eb;\n /* 2 */\n}\n\n::before,\n::after {\n --tw-content: '';\n}\n\n/*\n1. Use a consistent sensible line-height in all browsers.\n2. Prevent adjustments of font size after orientation changes in iOS.\n3. Use a more readable tab size.\n4. Use the user's configured `sans` font-family by default.\n5. Use the user's configured `sans` font-feature-settings by default.\n*/\n\nhtml {\n line-height: 1.5;\n /* 1 */\n -webkit-text-size-adjust: 100%;\n /* 2 */\n -moz-tab-size: 4;\n /* 3 */\n -o-tab-size: 4;\n tab-size: 4;\n /* 3 */\n font-family: ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, \"Segoe UI\", Roboto, \"Helvetica Neue\", Arial, \"Noto Sans\", sans-serif, \"Apple Color Emoji\", \"Segoe UI Emoji\", \"Segoe UI Symbol\", \"Noto Color Emoji\";\n /* 4 */\n font-feature-settings: normal;\n /* 5 */\n}\n\n/*\n1. Remove the margin in all browsers.\n2. Inherit line-height from `html` so users can set them as a class directly on the `html` element.\n*/\n\nbody {\n margin: 0;\n /* 1 */\n line-height: inherit;\n /* 2 */\n}\n\n/*\n1. Add the correct height in Firefox.\n2. Correct the inheritance of border color in Firefox. (https://bugzilla.mozilla.org/show_bug.cgi?id=190655)\n3. Ensure horizontal rules are visible by default.\n*/\n\nhr {\n height: 0;\n /* 1 */\n color: inherit;\n /* 2 */\n border-top-width: 1px;\n /* 3 */\n}\n\n/*\nAdd the correct text decoration in Chrome, Edge, and Safari.\n*/\n\nabbr:where([title]) {\n -webkit-text-decoration: underline dotted;\n text-decoration: underline dotted;\n}\n\n/*\nRemove the default font size and weight for headings.\n*/\n\nh1,\nh2,\nh3,\nh4,\nh5,\nh6 {\n font-size: inherit;\n font-weight: inherit;\n}\n\n/*\nReset links to optimize for opt-in styling instead of opt-out.\n*/\n\na {\n color: inherit;\n text-decoration: inherit;\n}\n\n/*\nAdd the correct font weight in Edge and Safari.\n*/\n\nb,\nstrong {\n font-weight: bolder;\n}\n\n/*\n1. Use the user's configured `mono` font family by default.\n2. Correct the odd `em` font sizing in all browsers.\n*/\n\ncode,\nkbd,\nsamp,\npre {\n font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, \"Liberation Mono\", \"Courier New\", monospace;\n /* 1 */\n font-size: 1em;\n /* 2 */\n}\n\n/*\nAdd the correct font size in all browsers.\n*/\n\nsmall {\n font-size: 80%;\n}\n\n/*\nPrevent `sub` and `sup` elements from affecting the line height in all browsers.\n*/\n\nsub,\nsup {\n font-size: 75%;\n line-height: 0;\n position: relative;\n vertical-align: baseline;\n}\n\nsub {\n bottom: -0.25em;\n}\n\nsup {\n top: -0.5em;\n}\n\n/*\n1. Remove text indentation from table contents in Chrome and Safari. (https://bugs.chromium.org/p/chromium/issues/detail?id=999088, https://bugs.webkit.org/show_bug.cgi?id=201297)\n2. Correct table border color inheritance in all Chrome and Safari. (https://bugs.chromium.org/p/chromium/issues/detail?id=935729, https://bugs.webkit.org/show_bug.cgi?id=195016)\n3. Remove gaps between table borders by default.\n*/\n\ntable {\n text-indent: 0;\n /* 1 */\n border-color: inherit;\n /* 2 */\n border-collapse: collapse;\n /* 3 */\n}\n\n/*\n1. Change the font styles in all browsers.\n2. Remove the margin in Firefox and Safari.\n3. Remove default padding in all browsers.\n*/\n\nbutton,\ninput,\noptgroup,\nselect,\ntextarea {\n font-family: inherit;\n /* 1 */\n font-size: 100%;\n /* 1 */\n font-weight: inherit;\n /* 1 */\n line-height: inherit;\n /* 1 */\n color: inherit;\n /* 1 */\n margin: 0;\n /* 2 */\n padding: 0;\n /* 3 */\n}\n\n/*\nRemove the inheritance of text transform in Edge and Firefox.\n*/\n\nbutton,\nselect {\n text-transform: none;\n}\n\n/*\n1. Correct the inability to style clickable types in iOS and Safari.\n2. Remove default button styles.\n*/\n\nbutton,\n[type='button'],\n[type='reset'],\n[type='submit'] {\n -webkit-appearance: button;\n /* 1 */\n background-color: transparent;\n /* 2 */\n background-image: none;\n /* 2 */\n}\n\n/*\nUse the modern Firefox focus style for all focusable elements.\n*/\n\n:-moz-focusring {\n outline: auto;\n}\n\n/*\nRemove the additional `:invalid` styles in Firefox. (https://github.com/mozilla/gecko-dev/blob/2f9eacd9d3d995c937b4251a5557d95d494c9be1/layout/style/res/forms.css#L728-L737)\n*/\n\n:-moz-ui-invalid {\n box-shadow: none;\n}\n\n/*\nAdd the correct vertical alignment in Chrome and Firefox.\n*/\n\nprogress {\n vertical-align: baseline;\n}\n\n/*\nCorrect the cursor style of increment and decrement buttons in Safari.\n*/\n\n::-webkit-inner-spin-button,\n::-webkit-outer-spin-button {\n height: auto;\n}\n\n/*\n1. Correct the odd appearance in Chrome and Safari.\n2. Correct the outline style in Safari.\n*/\n\n[type='search'] {\n -webkit-appearance: textfield;\n /* 1 */\n outline-offset: -2px;\n /* 2 */\n}\n\n/*\nRemove the inner padding in Chrome and Safari on macOS.\n*/\n\n::-webkit-search-decoration {\n -webkit-appearance: none;\n}\n\n/*\n1. Correct the inability to style clickable types in iOS and Safari.\n2. Change font properties to `inherit` in Safari.\n*/\n\n::-webkit-file-upload-button {\n -webkit-appearance: button;\n /* 1 */\n font: inherit;\n /* 2 */\n}\n\n/*\nAdd the correct display in Chrome and Safari.\n*/\n\nsummary {\n display: list-item;\n}\n\n/*\nRemoves the default spacing and border for appropriate elements.\n*/\n\nblockquote,\ndl,\ndd,\nh1,\nh2,\nh3,\nh4,\nh5,\nh6,\nhr,\nfigure,\np,\npre {\n margin: 0;\n}\n\nfieldset {\n margin: 0;\n padding: 0;\n}\n\nlegend {\n padding: 0;\n}\n\nol,\nul,\nmenu {\n list-style: none;\n margin: 0;\n padding: 0;\n}\n\n/*\nPrevent resizing textareas horizontally by default.\n*/\n\ntextarea {\n resize: vertical;\n}\n\n/*\n1. Reset the default placeholder opacity in Firefox. (https://github.com/tailwindlabs/tailwindcss/issues/3300)\n2. Set the default placeholder color to the user's configured gray 400 color.\n*/\n\ninput::-moz-placeholder, textarea::-moz-placeholder {\n opacity: 1;\n /* 1 */\n color: #9ca3af;\n /* 2 */\n}\n\ninput::placeholder,\ntextarea::placeholder {\n opacity: 1;\n /* 1 */\n color: #9ca3af;\n /* 2 */\n}\n\n/*\nSet the default cursor for buttons.\n*/\n\nbutton,\n[role=\"button\"] {\n cursor: pointer;\n}\n\n/*\nMake sure disabled buttons don't get the pointer cursor.\n*/\n\n:disabled {\n cursor: default;\n}\n\n/*\n1. Make replaced elements `display: block` by default. (https://github.com/mozdevs/cssremedy/issues/14)\n2. Add `vertical-align: middle` to align replaced elements more sensibly by default. (https://github.com/jensimmons/cssremedy/issues/14#issuecomment-634934210)\n This can trigger a poorly considered lint error in some tools but is included by design.\n*/\n\nimg,\nsvg,\nvideo,\ncanvas,\naudio,\niframe,\nembed,\nobject {\n display: block;\n /* 1 */\n vertical-align: middle;\n /* 2 */\n}\n\n/*\nConstrain images and videos to the parent width and preserve their intrinsic aspect ratio. (https://github.com/mozdevs/cssremedy/issues/14)\n*/\n\nimg,\nvideo {\n max-width: 100%;\n height: auto;\n}\n\n/* Make elements with the HTML hidden attribute stay hidden by default */\n\n[hidden] {\n display: none;\n}\n\n*, ::before, ::after {\n --tw-border-spacing-x: 0;\n --tw-border-spacing-y: 0;\n --tw-translate-x: 0;\n --tw-translate-y: 0;\n --tw-rotate: 0;\n --tw-skew-x: 0;\n --tw-skew-y: 0;\n --tw-scale-x: 1;\n --tw-scale-y: 1;\n --tw-pan-x: ;\n --tw-pan-y: ;\n --tw-pinch-zoom: ;\n --tw-scroll-snap-strictness: proximity;\n --tw-ordinal: ;\n --tw-slashed-zero: ;\n --tw-numeric-figure: ;\n --tw-numeric-spacing: ;\n --tw-numeric-fraction: ;\n --tw-ring-inset: ;\n --tw-ring-offset-width: 0px;\n --tw-ring-offset-color: #fff;\n --tw-ring-color: rgb(59 130 246 / 0.5);\n --tw-ring-offset-shadow: 0 0 #0000;\n --tw-ring-shadow: 0 0 #0000;\n --tw-shadow: 0 0 #0000;\n --tw-shadow-colored: 0 0 #0000;\n --tw-blur: ;\n --tw-brightness: ;\n --tw-contrast: ;\n --tw-grayscale: ;\n --tw-hue-rotate: ;\n --tw-invert: ;\n --tw-saturate: ;\n --tw-sepia: ;\n --tw-drop-shadow: ;\n --tw-backdrop-blur: ;\n --tw-backdrop-brightness: ;\n --tw-backdrop-contrast: ;\n --tw-backdrop-grayscale: ;\n --tw-backdrop-hue-rotate: ;\n --tw-backdrop-invert: ;\n --tw-backdrop-opacity: ;\n --tw-backdrop-saturate: ;\n --tw-backdrop-sepia: ;\n}\n\n::backdrop {\n --tw-border-spacing-x: 0;\n --tw-border-spacing-y: 0;\n --tw-translate-x: 0;\n --tw-translate-y: 0;\n --tw-rotate: 0;\n --tw-skew-x: 0;\n --tw-skew-y: 0;\n --tw-scale-x: 1;\n --tw-scale-y: 1;\n --tw-pan-x: ;\n --tw-pan-y: ;\n --tw-pinch-zoom: ;\n --tw-scroll-snap-strictness: proximity;\n --tw-ordinal: ;\n --tw-slashed-zero: ;\n --tw-numeric-figure: ;\n --tw-numeric-spacing: ;\n --tw-numeric-fraction: ;\n --tw-ring-inset: ;\n --tw-ring-offset-width: 0px;\n --tw-ring-offset-color: #fff;\n --tw-ring-color: rgb(59 130 246 / 0.5);\n --tw-ring-offset-shadow: 0 0 #0000;\n --tw-ring-shadow: 0 0 #0000;\n --tw-shadow: 0 0 #0000;\n --tw-shadow-colored: 0 0 #0000;\n --tw-blur: ;\n --tw-brightness: ;\n --tw-contrast: ;\n --tw-grayscale: ;\n --tw-hue-rotate: ;\n --tw-invert: ;\n --tw-saturate: ;\n --tw-sepia: ;\n --tw-drop-shadow: ;\n --tw-backdrop-blur: ;\n --tw-backdrop-brightness: ;\n --tw-backdrop-contrast: ;\n --tw-backdrop-grayscale: ;\n --tw-backdrop-hue-rotate: ;\n --tw-backdrop-invert: ;\n --tw-backdrop-opacity: ;\n --tw-backdrop-saturate: ;\n --tw-backdrop-sepia: ;\n}\n\n.mb-2 {\n margin-bottom: 0.5rem;\n}\n\n.ml-2 {\n margin-left: 0.5rem;\n}\n\n.mr-2 {\n margin-right: 0.5rem;\n}\n\n.mt-4 {\n margin-top: 1rem;\n}\n\n.flex {\n display: flex;\n}\n\n.w-\\[26rem\\] {\n width: 26rem;\n}\n\n.w-full {\n width: 100%;\n}\n\n.flex-col {\n flex-direction: column;\n}\n\n.items-center {\n align-items: center;\n}\n\n.justify-between {\n justify-content: space-between;\n}\n\n.self-start {\n align-self: flex-start;\n}\n\n.self-end {\n align-self: flex-end;\n}\n\n.rounded-lg {\n border-radius: 0.5rem;\n}\n\n.bg-blue-500 {\n --tw-bg-opacity: 1;\n background-color: rgb(59 130 246 / var(--tw-bg-opacity));\n}\n\n.bg-gray-200 {\n --tw-bg-opacity: 1;\n background-color: rgb(229 231 235 / var(--tw-bg-opacity));\n}\n\n.bg-gray-900 {\n --tw-bg-opacity: 1;\n background-color: rgb(17 24 39 / var(--tw-bg-opacity));\n}\n\n.bg-indigo-500 {\n --tw-bg-opacity: 1;\n background-color: rgb(99 102 241 / var(--tw-bg-opacity));\n}\n\n.bg-white {\n --tw-bg-opacity: 1;\n background-color: rgb(255 255 255 / var(--tw-bg-opacity));\n}\n\n.p-2 {\n padding: 0.5rem;\n}\n\n.p-4 {\n padding: 1rem;\n}\n\n.text-lg {\n font-size: 1.125rem;\n line-height: 1.75rem;\n}\n\n.text-sm {\n font-size: 0.875rem;\n line-height: 1.25rem;\n}\n\n.font-medium {\n font-weight: 500;\n}\n\n.text-blue-500 {\n --tw-text-opacity: 1;\n color: rgb(59 130 246 / var(--tw-text-opacity));\n}\n\n.text-gray-700 {\n --tw-text-opacity: 1;\n color: rgb(55 65 81 / var(--tw-text-opacity));\n}\n\n.text-white {\n --tw-text-opacity: 1;\n color: rgb(255 255 255 / var(--tw-text-opacity));\n}\n\n.shadow {\n --tw-shadow: 0 1px 3px 0 rgb(0 0 0 / 0.1), 0 1px 2px -1px rgb(0 0 0 / 0.1);\n --tw-shadow-colored: 0 1px 3px 0 var(--tw-shadow-color), 0 1px 2px -1px var(--tw-shadow-color);\n box-shadow: var(--tw-ring-offset-shadow, 0 0 #0000), var(--tw-ring-shadow, 0 0 #0000), var(--tw-shadow);\n}\n\n#chat-container {\n width: 500px;\n height: 450px;\n background-color: white;\n padding: 10px;\n overflow: auto;\n}\n\n.bg-gray-200 {\n background-color: #edf2f7;\n}\n\n.bg-gray-900 {\n background-color: #1a202c;\n}\n\n.rounded-lg {\n border-radius: 0.5rem;\n}\n\n.shadow {\n box-shadow: 0 1px 3px rgba(0, 0, 0, 0.12), 0 1px 2px rgba(0, 0, 0, 0.24);\n}\n\n.text-gray-700 {\n color: #4a5568;\n}\n\n.text-sm {\n font-size: 0.875rem;\n}\n\n.p-4 {\n padding: 1.5rem;\n}\n\n.hover\\:text-blue-800:hover {\n --tw-text-opacity: 1;\n color: rgb(30 64 175 / var(--tw-text-opacity));\n}\n \n\n\n" }, { "path": "extensions/chrome/js/jquery/.gitignore", "content": "build\njquery-migrate.js\njquery-migrate.min.js\n" }, { "path": "extensions/chrome/js/jquery/README.md", "content": "jQuery Component\n================\n\nShim repository for jQuery.\n" }, { "path": "extensions/chrome/js/jquery/bower.json", "content": "{\n \"name\": \"jquery\",\n \"version\": \"2.0.0\",\n \"description\": \"jQuery component\",\n \"keywords\": [\n \"jquery\",\n \"component\"\n ],\n \"scripts\": [\n \"jquery.js\"\n ],\n \"license\": \"MIT\"\n}\n" }, { "path": "extensions/chrome/js/jquery/component.json", "content": "{\n \"name\": \"jquery\",\n \"version\": \"2.0.0\",\n \"description\": \"jQuery component\",\n \"keywords\": [\n \"jquery\",\n \"component\"\n ],\n \"scripts\": [\n \"jquery.js\"\n ],\n \"license\": \"MIT\",\n \"gitHead\": \"46f8412bd1bb9b1b30b5b0eb88560e2d4196509c\",\n \"readme\": \"jQuery Component\\n================\\n\\nShim repository for jQuery.\\n\",\n \"readmeFilename\": \"README.md\",\n \"_id\": \"jquery@2.0.0\",\n \"repository\": {\n \"type\": \"git\",\n \"url\": \"git://github.com/components/jquery.git\"\n }\n}" }, { "path": "extensions/chrome/js/jquery/composer.json", "content": "{\n \"name\": \"components/jquery\",\n \"description\": \"jQuery JavaScript Library\",\n \"type\": \"component\",\n \"homepage\": \"http://jquery.com\",\n \"license\": \"MIT\",\n \"support\": {\n \"irc\": \"irc://irc.freenode.org/jquery\",\n \"issues\": \"http://bugs.jquery.com\",\n \"forum\": \"http://forum.jquery.com\",\n \"wiki\": \"http://docs.jquery.com/\",\n \"source\": \"https://github.com/jquery/jquery\"\n },\n \"authors\": [\n {\n \"name\": \"John Resig\",\n \"email\": \"jeresig@gmail.com\"\n }\n ],\n \"require\": {\n \"robloach/component-installer\": \"*\"\n },\n \"extra\": {\n \"component\": {\n \"scripts\": [\n \"jquery.js\"\n ]\n }\n }\n}\n" }, { "path": "extensions/chrome/js/jquery/jquery.js", "content": "/*!\n * jQuery JavaScript Library v2.0.0\n * http://jquery.com/\n *\n * Includes Sizzle.js\n * http://sizzlejs.com/\n *\n * Copyright 2005, 2013 jQuery Foundation, Inc. and other contributors\n * Released under the MIT license\n * http://jquery.org/license\n *\n * Date: 2013-04-18\n */\n(function( window, undefined ) {\n\n// Can't do this because several apps including ASP.NET trace\n// the stack via arguments.caller.callee and Firefox dies if\n// you try to trace through \"use strict\" call chains. (#13335)\n// Support: Firefox 18+\n//\"use strict\";\nvar\n\t// A central reference to the root jQuery(document)\n\trootjQuery,\n\n\t// The deferred used on DOM ready\n\treadyList,\n\n\t// Support: IE9\n\t// For `typeof xmlNode.method` instead of `xmlNode.method !== undefined`\n\tcore_strundefined = typeof undefined,\n\n\t// Use the correct document accordingly with window argument (sandbox)\n\tlocation = window.location,\n\tdocument = window.document,\n\tdocElem = document.documentElement,\n\n\t// Map over jQuery in case of overwrite\n\t_jQuery = window.jQuery,\n\n\t// Map over the $ in case of overwrite\n\t_$ = window.$,\n\n\t// [[Class]] -> type pairs\n\tclass2type = {},\n\n\t// List of deleted data cache ids, so we can reuse them\n\tcore_deletedIds = [],\n\n\tcore_version = \"2.0.0\",\n\n\t// Save a reference to some core methods\n\tcore_concat = core_deletedIds.concat,\n\tcore_push = core_deletedIds.push,\n\tcore_slice = core_deletedIds.slice,\n\tcore_indexOf = core_deletedIds.indexOf,\n\tcore_toString = class2type.toString,\n\tcore_hasOwn = class2type.hasOwnProperty,\n\tcore_trim = core_version.trim,\n\n\t// Define a local copy of jQuery\n\tjQuery = function( selector, context ) {\n\t\t// The jQuery object is actually just the init constructor 'enhanced'\n\t\treturn new jQuery.fn.init( selector, context, rootjQuery );\n\t},\n\n\t// Used for matching numbers\n\tcore_pnum = /[+-]?(?:\\d*\\.|)\\d+(?:[eE][+-]?\\d+|)/.source,\n\n\t// Used for splitting on whitespace\n\tcore_rnotwhite = /\\S+/g,\n\n\t// A simple way to check for HTML strings\n\t// Prioritize #id over <tag> to avoid XSS via location.hash (#9521)\n\t// Strict HTML recognition (#11290: must start with <)\n\trquickExpr = /^(?:(<[\\w\\W]+>)[^>]*|#([\\w-]*))$/,\n\n\t// Match a standalone tag\n\trsingleTag = /^<(\\w+)\\s*\\/?>(?:<\\/\\1>|)$/,\n\n\t// Matches dashed string for camelizing\n\trmsPrefix = /^-ms-/,\n\trdashAlpha = /-([\\da-z])/gi,\n\n\t// Used by jQuery.camelCase as callback to replace()\n\tfcamelCase = function( all, letter ) {\n\t\treturn letter.toUpperCase();\n\t},\n\n\t// The ready event handler and self cleanup method\n\tcompleted = function() {\n\t\tdocument.removeEventListener( \"DOMContentLoaded\", completed, false );\n\t\twindow.removeEventListener( \"load\", completed, false );\n\t\tjQuery.ready();\n\t};\n\njQuery.fn = jQuery.prototype = {\n\t// The current version of jQuery being used\n\tjquery: core_version,\n\n\tconstructor: jQuery,\n\tinit: function( selector, context, rootjQuery ) {\n\t\tvar match, elem;\n\n\t\t// HANDLE: $(\"\"), $(null), $(undefined), $(false)\n\t\tif ( !selector ) {\n\t\t\treturn this;\n\t\t}\n\n\t\t// Handle HTML strings\n\t\tif ( typeof selector === \"string\" ) {\n\t\t\tif ( selector.charAt(0) === \"<\" && selector.charAt( selector.length - 1 ) === \">\" && selector.length >= 3 ) {\n\t\t\t\t// Assume that strings that start and end with <> are HTML and skip the regex check\n\t\t\t\tmatch = [ null, selector, null ];\n\n\t\t\t} else {\n\t\t\t\tmatch = rquickExpr.exec( selector );\n\t\t\t}\n\n\t\t\t// Match html or make sure no context is specified for #id\n\t\t\tif ( match && (match[1] || !context) ) {\n\n\t\t\t\t// HANDLE: $(html) -> $(array)\n\t\t\t\tif ( match[1] ) {\n\t\t\t\t\tcontext = context instanceof jQuery ? context[0] : context;\n\n\t\t\t\t\t// scripts is true for back-compat\n\t\t\t\t\tjQuery.merge( this, jQuery.parseHTML(\n\t\t\t\t\t\tmatch[1],\n\t\t\t\t\t\tcontext && context.nodeType ? context.ownerDocument || context : document,\n\t\t\t\t\t\ttrue\n\t\t\t\t\t) );\n\n\t\t\t\t\t// HANDLE: $(html, props)\n\t\t\t\t\tif ( rsingleTag.test( match[1] ) && jQuery.isPlainObject( context ) ) {\n\t\t\t\t\t\tfor ( match in context ) {\n\t\t\t\t\t\t\t// Properties of context are called as methods if possible\n\t\t\t\t\t\t\tif ( jQuery.isFunction( this[ match ] ) ) {\n\t\t\t\t\t\t\t\tthis[ match ]( context[ match ] );\n\n\t\t\t\t\t\t\t// ...and otherwise set as attributes\n\t\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\t\tthis.attr( match, context[ match ] );\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\n\t\t\t\t\treturn this;\n\n\t\t\t\t// HANDLE: $(#id)\n\t\t\t\t} else {\n\t\t\t\t\telem = document.getElementById( match[2] );\n\n\t\t\t\t\t// Check parentNode to catch when Blackberry 4.6 returns\n\t\t\t\t\t// nodes that are no longer in the document #6963\n\t\t\t\t\tif ( elem && elem.parentNode ) {\n\t\t\t\t\t\t// Inject the element directly into the jQuery object\n\t\t\t\t\t\tthis.length = 1;\n\t\t\t\t\t\tthis[0] = elem;\n\t\t\t\t\t}\n\n\t\t\t\t\tthis.context = document;\n\t\t\t\t\tthis.selector = selector;\n\t\t\t\t\treturn this;\n\t\t\t\t}\n\n\t\t\t// HANDLE: $(expr, $(...))\n\t\t\t} else if ( !context || context.jquery ) {\n\t\t\t\treturn ( context || rootjQuery ).find( selector );\n\n\t\t\t// HANDLE: $(expr, context)\n\t\t\t// (which is just equivalent to: $(context).find(expr)\n\t\t\t} else {\n\t\t\t\treturn this.constructor( context ).find( selector );\n\t\t\t}\n\n\t\t// HANDLE: $(DOMElement)\n\t\t} else if ( selector.nodeType ) {\n\t\t\tthis.context = this[0] = selector;\n\t\t\tthis.length = 1;\n\t\t\treturn this;\n\n\t\t// HANDLE: $(function)\n\t\t// Shortcut for document ready\n\t\t} else if ( jQuery.isFunction( selector ) ) {\n\t\t\treturn rootjQuery.ready( selector );\n\t\t}\n\n\t\tif ( selector.selector !== undefined ) {\n\t\t\tthis.selector = selector.selector;\n\t\t\tthis.context = selector.context;\n\t\t}\n\n\t\treturn jQuery.makeArray( selector, this );\n\t},\n\n\t// Start with an empty selector\n\tselector: \"\",\n\n\t// The default length of a jQuery object is 0\n\tlength: 0,\n\n\ttoArray: function() {\n\t\treturn core_slice.call( this );\n\t},\n\n\t// Get the Nth element in the matched element set OR\n\t// Get the whole matched element set as a clean array\n\tget: function( num ) {\n\t\treturn num == null ?\n\n\t\t\t// Return a 'clean' array\n\t\t\tthis.toArray() :\n\n\t\t\t// Return just the object\n\t\t\t( num < 0 ? this[ this.length + num ] : this[ num ] );\n\t},\n\n\t// Take an array of elements and push it onto the stack\n\t// (returning the new matched element set)\n\tpushStack: function( elems ) {\n\n\t\t// Build a new jQuery matched element set\n\t\tvar ret = jQuery.merge( this.constructor(), elems );\n\n\t\t// Add the old object onto the stack (as a reference)\n\t\tret.prevObject = this;\n\t\tret.context = this.context;\n\n\t\t// Return the newly-formed element set\n\t\treturn ret;\n\t},\n\n\t// Execute a callback for every element in the matched set.\n\t// (You can seed the arguments with an array of args, but this is\n\t// only used internally.)\n\teach: function( callback, args ) {\n\t\treturn jQuery.each( this, callback, args );\n\t},\n\n\tready: function( fn ) {\n\t\t// Add the callback\n\t\tjQuery.ready.promise().done( fn );\n\n\t\treturn this;\n\t},\n\n\tslice: function() {\n\t\treturn this.pushStack( core_slice.apply( this, arguments ) );\n\t},\n\n\tfirst: function() {\n\t\treturn this.eq( 0 );\n\t},\n\n\tlast: function() {\n\t\treturn this.eq( -1 );\n\t},\n\n\teq: function( i ) {\n\t\tvar len = this.length,\n\t\t\tj = +i + ( i < 0 ? len : 0 );\n\t\treturn this.pushStack( j >= 0 && j < len ? [ this[j] ] : [] );\n\t},\n\n\tmap: function( callback ) {\n\t\treturn this.pushStack( jQuery.map(this, function( elem, i ) {\n\t\t\treturn callback.call( elem, i, elem );\n\t\t}));\n\t},\n\n\tend: function() {\n\t\treturn this.prevObject || this.constructor(null);\n\t},\n\n\t// For internal use only.\n\t// Behaves like an Array's method, not like a jQuery method.\n\tpush: core_push,\n\tsort: [].sort,\n\tsplice: [].splice\n};\n\n// Give the init function the jQuery prototype for later instantiation\njQuery.fn.init.prototype = jQuery.fn;\n\njQuery.extend = jQuery.fn.extend = function() {\n\tvar options, name, src, copy, copyIsArray, clone,\n\t\ttarget = arguments[0] || {},\n\t\ti = 1,\n\t\tlength = arguments.length,\n\t\tdeep = false;\n\n\t// Handle a deep copy situation\n\tif ( typeof target === \"boolean\" ) {\n\t\tdeep = target;\n\t\ttarget = arguments[1] || {};\n\t\t// skip the boolean and the target\n\t\ti = 2;\n\t}\n\n\t// Handle case when target is a string or something (possible in deep copy)\n\tif ( typeof target !== \"object\" && !jQuery.isFunction(target) ) {\n\t\ttarget = {};\n\t}\n\n\t// extend jQuery itself if only one argument is passed\n\tif ( length === i ) {\n\t\ttarget = this;\n\t\t--i;\n\t}\n\n\tfor ( ; i < length; i++ ) {\n\t\t// Only deal with non-null/undefined values\n\t\tif ( (options = arguments[ i ]) != null ) {\n\t\t\t// Extend the base object\n\t\t\tfor ( name in options ) {\n\t\t\t\tsrc = target[ name ];\n\t\t\t\tcopy = options[ name ];\n\n\t\t\t\t// Prevent never-ending loop\n\t\t\t\tif ( target === copy ) {\n\t\t\t\t\tcontinue;\n\t\t\t\t}\n\n\t\t\t\t// Recurse if we're merging plain objects or arrays\n\t\t\t\tif ( deep && copy && ( jQuery.isPlainObject(copy) || (copyIsArray = jQuery.isArray(copy)) ) ) {\n\t\t\t\t\tif ( copyIsArray ) {\n\t\t\t\t\t\tcopyIsArray = false;\n\t\t\t\t\t\tclone = src && jQuery.isArray(src) ? src : [];\n\n\t\t\t\t\t} else {\n\t\t\t\t\t\tclone = src && jQuery.isPlainObject(src) ? src : {};\n\t\t\t\t\t}\n\n\t\t\t\t\t// Never move original objects, clone them\n\t\t\t\t\ttarget[ name ] = jQuery.extend( deep, clone, copy );\n\n\t\t\t\t// Don't bring in undefined values\n\t\t\t\t} else if ( copy !== undefined ) {\n\t\t\t\t\ttarget[ name ] = copy;\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\t// Return the modified object\n\treturn target;\n};\n\njQuery.extend({\n\t// Unique for each copy of jQuery on the page\n\texpando: \"jQuery\" + ( core_version + Math.random() ).replace( /\\D/g, \"\" ),\n\n\tnoConflict: function( deep ) {\n\t\tif ( window.$ === jQuery ) {\n\t\t\twindow.$ = _$;\n\t\t}\n\n\t\tif ( deep && window.jQuery === jQuery ) {\n\t\t\twindow.jQuery = _jQuery;\n\t\t}\n\n\t\treturn jQuery;\n\t},\n\n\t// Is the DOM ready to be used? Set to true once it occurs.\n\tisReady: false,\n\n\t// A counter to track how many items to wait for before\n\t// the ready event fires. See #6781\n\treadyWait: 1,\n\n\t// Hold (or release) the ready event\n\tholdReady: function( hold ) {\n\t\tif ( hold ) {\n\t\t\tjQuery.readyWait++;\n\t\t} else {\n\t\t\tjQuery.ready( true );\n\t\t}\n\t},\n\n\t// Handle when the DOM is ready\n\tready: function( wait ) {\n\n\t\t// Abort if there are pending holds or we're already ready\n\t\tif ( wait === true ? --jQuery.readyWait : jQuery.isReady ) {\n\t\t\treturn;\n\t\t}\n\n\t\t// Remember that the DOM is ready\n\t\tjQuery.isReady = true;\n\n\t\t// If a normal DOM Ready event fired, decrement, and wait if need be\n\t\tif ( wait !== true && --jQuery.readyWait > 0 ) {\n\t\t\treturn;\n\t\t}\n\n\t\t// If there are functions bound, to execute\n\t\treadyList.resolveWith( document, [ jQuery ] );\n\n\t\t// Trigger any bound ready events\n\t\tif ( jQuery.fn.trigger ) {\n\t\t\tjQuery( document ).trigger(\"ready\").off(\"ready\");\n\t\t}\n\t},\n\n\t// See test/unit/core.js for details concerning isFunction.\n\t// Since version 1.3, DOM methods and functions like alert\n\t// aren't supported. They return false on IE (#2968).\n\tisFunction: function( obj ) {\n\t\treturn jQuery.type(obj) === \"function\";\n\t},\n\n\tisArray: Array.isArray,\n\n\tisWindow: function( obj ) {\n\t\treturn obj != null && obj === obj.window;\n\t},\n\n\tisNumeric: function( obj ) {\n\t\treturn !isNaN( parseFloat(obj) ) && isFinite( obj );\n\t},\n\n\ttype: function( obj ) {\n\t\tif ( obj == null ) {\n\t\t\treturn String( obj );\n\t\t}\n\t\t// Support: Safari <= 5.1 (functionish RegExp)\n\t\treturn typeof obj === \"object\" || typeof obj === \"function\" ?\n\t\t\tclass2type[ core_toString.call(obj) ] || \"object\" :\n\t\t\ttypeof obj;\n\t},\n\n\tisPlainObject: function( obj ) {\n\t\t// Not plain objects:\n\t\t// - Any object or value whose internal [[Class]] property is not \"[object Object]\"\n\t\t// - DOM nodes\n\t\t// - window\n\t\tif ( jQuery.type( obj ) !== \"object\" || obj.nodeType || jQuery.isWindow( obj ) ) {\n\t\t\treturn false;\n\t\t}\n\n\t\t// Support: Firefox <20\n\t\t// The try/catch suppresses exceptions thrown when attempting to access\n\t\t// the \"constructor\" property of certain host objects, ie. |window.location|\n\t\t// https://bugzilla.mozilla.org/show_bug.cgi?id=814622\n\t\ttry {\n\t\t\tif ( obj.constructor &&\n\t\t\t\t\t!core_hasOwn.call( obj.constructor.prototype, \"isPrototypeOf\" ) ) {\n\t\t\t\treturn false;\n\t\t\t}\n\t\t} catch ( e ) {\n\t\t\treturn false;\n\t\t}\n\n\t\t// If the function hasn't returned already, we're confident that\n\t\t// |obj| is a plain object, created by {} or constructed with new Object\n\t\treturn true;\n\t},\n\n\tisEmptyObject: function( obj ) {\n\t\tvar name;\n\t\tfor ( name in obj ) {\n\t\t\treturn false;\n\t\t}\n\t\treturn true;\n\t},\n\n\terror: function( msg ) {\n\t\tthrow new Error( msg );\n\t},\n\n\t// data: string of html\n\t// context (optional): If specified, the fragment will be created in this context, defaults to document\n\t// keepScripts (optional): If true, will include scripts passed in the html string\n\tparseHTML: function( data, context, keepScripts ) {\n\t\tif ( !data || typeof data !== \"string\" ) {\n\t\t\treturn null;\n\t\t}\n\t\tif ( typeof context === \"boolean\" ) {\n\t\t\tkeepScripts = context;\n\t\t\tcontext = false;\n\t\t}\n\t\tcontext = context || document;\n\n\t\tvar parsed = rsingleTag.exec( data ),\n\t\t\tscripts = !keepScripts && [];\n\n\t\t// Single tag\n\t\tif ( parsed ) {\n\t\t\treturn [ context.createElement( parsed[1] ) ];\n\t\t}\n\n\t\tparsed = jQuery.buildFragment( [ data ], context, scripts );\n\n\t\tif ( scripts ) {\n\t\t\tjQuery( scripts ).remove();\n\t\t}\n\n\t\treturn jQuery.merge( [], parsed.childNodes );\n\t},\n\n\tparseJSON: JSON.parse,\n\n\t// Cross-browser xml parsing\n\tparseXML: function( data ) {\n\t\tvar xml, tmp;\n\t\tif ( !data || typeof data !== \"string\" ) {\n\t\t\treturn null;\n\t\t}\n\n\t\t// Support: IE9\n\t\ttry {\n\t\t\ttmp = new DOMParser();\n\t\t\txml = tmp.parseFromString( data , \"text/xml\" );\n\t\t} catch ( e ) {\n\t\t\txml = undefined;\n\t\t}\n\n\t\tif ( !xml || xml.getElementsByTagName( \"parsererror\" ).length ) {\n\t\t\tjQuery.error( \"Invalid XML: \" + data );\n\t\t}\n\t\treturn xml;\n\t},\n\n\tnoop: function() {},\n\n\t// Evaluates a script in a global context\n\tglobalEval: function( code ) {\n\t\tvar script,\n\t\t\t\tindirect = eval;\n\n\t\tcode = jQuery.trim( code );\n\n\t\tif ( code ) {\n\t\t\t// If the code includes a valid, prologue position\n\t\t\t// strict mode pragma, execute code by injecting a\n\t\t\t// script tag into the document.\n\t\t\tif ( code.indexOf(\"use strict\") === 1 ) {\n\t\t\t\tscript = document.createElement(\"script\");\n\t\t\t\tscript.text = code;\n\t\t\t\tdocument.head.appendChild( script ).parentNode.removeChild( script );\n\t\t\t} else {\n\t\t\t// Otherwise, avoid the DOM node creation, insertion\n\t\t\t// and removal by using an indirect global eval\n\t\t\t\tindirect( code );\n\t\t\t}\n\t\t}\n\t},\n\n\t// Convert dashed to camelCase; used by the css and data modules\n\t// Microsoft forgot to hump their vendor prefix (#9572)\n\tcamelCase: function( string ) {\n\t\treturn string.replace( rmsPrefix, \"ms-\" ).replace( rdashAlpha, fcamelCase );\n\t},\n\n\tnodeName: function( elem, name ) {\n\t\treturn elem.nodeName && elem.nodeName.toLowerCase() === name.toLowerCase();\n\t},\n\n\t// args is for internal usage only\n\teach: function( obj, callback, args ) {\n\t\tvar value,\n\t\t\ti = 0,\n\t\t\tlength = obj.length,\n\t\t\tisArray = isArraylike( obj );\n\n\t\tif ( args ) {\n\t\t\tif ( isArray ) {\n\t\t\t\tfor ( ; i < length; i++ ) {\n\t\t\t\t\tvalue = callback.apply( obj[ i ], args );\n\n\t\t\t\t\tif ( value === false ) {\n\t\t\t\t\t\tbreak;\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tfor ( i in obj ) {\n\t\t\t\t\tvalue = callback.apply( obj[ i ], args );\n\n\t\t\t\t\tif ( value === false ) {\n\t\t\t\t\t\tbreak;\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\n\t\t// A special, fast, case for the most common use of each\n\t\t} else {\n\t\t\tif ( isArray ) {\n\t\t\t\tfor ( ; i < length; i++ ) {\n\t\t\t\t\tvalue = callback.call( obj[ i ], i, obj[ i ] );\n\n\t\t\t\t\tif ( value === false ) {\n\t\t\t\t\t\tbreak;\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tfor ( i in obj ) {\n\t\t\t\t\tvalue = callback.call( obj[ i ], i, obj[ i ] );\n\n\t\t\t\t\tif ( value === false ) {\n\t\t\t\t\t\tbreak;\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\treturn obj;\n\t},\n\n\ttrim: function( text ) {\n\t\treturn text == null ? \"\" : core_trim.call( text );\n\t},\n\n\t// results is for internal usage only\n\tmakeArray: function( arr, results ) {\n\t\tvar ret = results || [];\n\n\t\tif ( arr != null ) {\n\t\t\tif ( isArraylike( Object(arr) ) ) {\n\t\t\t\tjQuery.merge( ret,\n\t\t\t\t\ttypeof arr === \"string\" ?\n\t\t\t\t\t[ arr ] : arr\n\t\t\t\t);\n\t\t\t} else {\n\t\t\t\tcore_push.call( ret, arr );\n\t\t\t}\n\t\t}\n\n\t\treturn ret;\n\t},\n\n\tinArray: function( elem, arr, i ) {\n\t\treturn arr == null ? -1 : core_indexOf.call( arr, elem, i );\n\t},\n\n\tmerge: function( first, second ) {\n\t\tvar l = second.length,\n\t\t\ti = first.length,\n\t\t\tj = 0;\n\n\t\tif ( typeof l === \"number\" ) {\n\t\t\tfor ( ; j < l; j++ ) {\n\t\t\t\tfirst[ i++ ] = second[ j ];\n\t\t\t}\n\t\t} else {\n\t\t\twhile ( second[j] !== undefined ) {\n\t\t\t\tfirst[ i++ ] = second[ j++ ];\n\t\t\t}\n\t\t}\n\n\t\tfirst.length = i;\n\n\t\treturn first;\n\t},\n\n\tgrep: function( elems, callback, inv ) {\n\t\tvar retVal,\n\t\t\tret = [],\n\t\t\ti = 0,\n\t\t\tlength = elems.length;\n\t\tinv = !!inv;\n\n\t\t// Go through the array, only saving the items\n\t\t// that pass the validator function\n\t\tfor ( ; i < length; i++ ) {\n\t\t\tretVal = !!callback( elems[ i ], i );\n\t\t\tif ( inv !== retVal ) {\n\t\t\t\tret.push( elems[ i ] );\n\t\t\t}\n\t\t}\n\n\t\treturn ret;\n\t},\n\n\t// arg is for internal usage only\n\tmap: function( elems, callback, arg ) {\n\t\tvar value,\n\t\t\ti = 0,\n\t\t\tlength = elems.length,\n\t\t\tisArray = isArraylike( elems ),\n\t\t\tret = [];\n\n\t\t// Go through the array, translating each of the items to their\n\t\tif ( isArray ) {\n\t\t\tfor ( ; i < length; i++ ) {\n\t\t\t\tvalue = callback( elems[ i ], i, arg );\n\n\t\t\t\tif ( value != null ) {\n\t\t\t\t\tret[ ret.length ] = value;\n\t\t\t\t}\n\t\t\t}\n\n\t\t// Go through every key on the object,\n\t\t} else {\n\t\t\tfor ( i in elems ) {\n\t\t\t\tvalue = callback( elems[ i ], i, arg );\n\n\t\t\t\tif ( value != null ) {\n\t\t\t\t\tret[ ret.length ] = value;\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\t// Flatten any nested arrays\n\t\treturn core_concat.apply( [], ret );\n\t},\n\n\t// A global GUID counter for objects\n\tguid: 1,\n\n\t// Bind a function to a context, optionally partially applying any\n\t// arguments.\n\tproxy: function( fn, context ) {\n\t\tvar tmp, args, proxy;\n\n\t\tif ( typeof context === \"string\" ) {\n\t\t\ttmp = fn[ context ];\n\t\t\tcontext = fn;\n\t\t\tfn = tmp;\n\t\t}\n\n\t\t// Quick check to determine if target is callable, in the spec\n\t\t// this throws a TypeError, but we will just return undefined.\n\t\tif ( !jQuery.isFunction( fn ) ) {\n\t\t\treturn undefined;\n\t\t}\n\n\t\t// Simulated bind\n\t\targs = core_slice.call( arguments, 2 );\n\t\tproxy = function() {\n\t\t\treturn fn.apply( context || this, args.concat( core_slice.call( arguments ) ) );\n\t\t};\n\n\t\t// Set the guid of unique handler to the same of original handler, so it can be removed\n\t\tproxy.guid = fn.guid = fn.guid || jQuery.guid++;\n\n\t\treturn proxy;\n\t},\n\n\t// Multifunctional method to get and set values of a collection\n\t// The value/s can optionally be executed if it's a function\n\taccess: function( elems, fn, key, value, chainable, emptyGet, raw ) {\n\t\tvar i = 0,\n\t\t\tlength = elems.length,\n\t\t\tbulk = key == null;\n\n\t\t// Sets many values\n\t\tif ( jQuery.type( key ) === \"object\" ) {\n\t\t\tchainable = true;\n\t\t\tfor ( i in key ) {\n\t\t\t\tjQuery.access( elems, fn, i, key[i], true, emptyGet, raw );\n\t\t\t}\n\n\t\t// Sets one value\n\t\t} else if ( value !== undefined ) {\n\t\t\tchainable = true;\n\n\t\t\tif ( !jQuery.isFunction( value ) ) {\n\t\t\t\traw = true;\n\t\t\t}\n\n\t\t\tif ( bulk ) {\n\t\t\t\t// Bulk operations run against the entire set\n\t\t\t\tif ( raw ) {\n\t\t\t\t\tfn.call( elems, value );\n\t\t\t\t\tfn = null;\n\n\t\t\t\t// ...except when executing function values\n\t\t\t\t} else {\n\t\t\t\t\tbulk = fn;\n\t\t\t\t\tfn = function( elem, key, value ) {\n\t\t\t\t\t\treturn bulk.call( jQuery( elem ), value );\n\t\t\t\t\t};\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif ( fn ) {\n\t\t\t\tfor ( ; i < length; i++ ) {\n\t\t\t\t\tfn( elems[i], key, raw ? value : value.call( elems[i], i, fn( elems[i], key ) ) );\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\treturn chainable ?\n\t\t\telems :\n\n\t\t\t// Gets\n\t\t\tbulk ?\n\t\t\t\tfn.call( elems ) :\n\t\t\t\tlength ? fn( elems[0], key ) : emptyGet;\n\t},\n\n\tnow: Date.now,\n\n\t// A method for quickly swapping in/out CSS properties to get correct calculations.\n\t// Note: this method belongs to the css module but it's needed here for the support module.\n\t// If support gets modularized, this method should be moved back to the css module.\n\tswap: function( elem, options, callback, args ) {\n\t\tvar ret, name,\n\t\t\told = {};\n\n\t\t// Remember the old values, and insert the new ones\n\t\tfor ( name in options ) {\n\t\t\told[ name ] = elem.style[ name ];\n\t\t\telem.style[ name ] = options[ name ];\n\t\t}\n\n\t\tret = callback.apply( elem, args || [] );\n\n\t\t// Revert the old values\n\t\tfor ( name in options ) {\n\t\t\telem.style[ name ] = old[ name ];\n\t\t}\n\n\t\treturn ret;\n\t}\n});\n\njQuery.ready.promise = function( obj ) {\n\tif ( !readyList ) {\n\n\t\treadyList = jQuery.Deferred();\n\n\t\t// Catch cases where $(document).ready() is called after the browser event has already occurred.\n\t\t// we once tried to use readyState \"interactive\" here, but it caused issues like the one\n\t\t// discovered by ChrisS here: http://bugs.jquery.com/ticket/12282#comment:15\n\t\tif ( document.readyState === \"complete\" ) {\n\t\t\t// Handle it asynchronously to allow scripts the opportunity to delay ready\n\t\t\tsetTimeout( jQuery.ready );\n\n\t\t} else {\n\n\t\t\t// Use the handy event callback\n\t\t\tdocument.addEventListener( \"DOMContentLoaded\", completed, false );\n\n\t\t\t// A fallback to window.onload, that will always work\n\t\t\twindow.addEventListener( \"load\", completed, false );\n\t\t}\n\t}\n\treturn readyList.promise( obj );\n};\n\n// Populate the class2type map\njQuery.each(\"Boolean Number String Function Array Date RegExp Object Error\".split(\" \"), function(i, name) {\n\tclass2type[ \"[object \" + name + \"]\" ] = name.toLowerCase();\n});\n\nfunction isArraylike( obj ) {\n\tvar length = obj.length,\n\t\ttype = jQuery.type( obj );\n\n\tif ( jQuery.isWindow( obj ) ) {\n\t\treturn false;\n\t}\n\n\tif ( obj.nodeType === 1 && length ) {\n\t\treturn true;\n\t}\n\n\treturn type === \"array\" || type !== \"function\" &&\n\t\t( length === 0 ||\n\t\ttypeof length === \"number\" && length > 0 && ( length - 1 ) in obj );\n}\n\n// All jQuery objects should point back to these\nrootjQuery = jQuery(document);\n/*!\n * Sizzle CSS Selector Engine v1.9.2-pre\n * http://sizzlejs.com/\n *\n * Copyright 2013 jQuery Foundation, Inc. and other contributors\n * Released under the MIT license\n * http://jquery.org/license\n *\n * Date: 2013-04-16\n */\n(function( window, undefined ) {\n\nvar i,\n\tcachedruns,\n\tExpr,\n\tgetText,\n\tisXML,\n\tcompile,\n\toutermostContext,\n\tsortInput,\n\n\t// Local document vars\n\tsetDocument,\n\tdocument,\n\tdocElem,\n\tdocumentIsHTML,\n\trbuggyQSA,\n\trbuggyMatches,\n\tmatches,\n\tcontains,\n\n\t// Instance-specific data\n\texpando = \"sizzle\" + -(new Date()),\n\tpreferredDoc = window.document,\n\tsupport = {},\n\tdirruns = 0,\n\tdone = 0,\n\tclassCache = createCache(),\n\ttokenCache = createCache(),\n\tcompilerCache = createCache(),\n\thasDuplicate = false,\n\tsortOrder = function() { return 0; },\n\n\t// General-purpose constants\n\tstrundefined = typeof undefined,\n\tMAX_NEGATIVE = 1 << 31,\n\n\t// Array methods\n\tarr = [],\n\tpop = arr.pop,\n\tpush_native = arr.push,\n\tpush = arr.push,\n\tslice = arr.slice,\n\t// Use a stripped-down indexOf if we can't use a native one\n\tindexOf = arr.indexOf || function( elem ) {\n\t\tvar i = 0,\n\t\t\tlen = this.length;\n\t\tfor ( ; i < len; i++ ) {\n\t\t\tif ( this[i] === elem ) {\n\t\t\t\treturn i;\n\t\t\t}\n\t\t}\n\t\treturn -1;\n\t},\n\n\tbooleans = \"checked|selected|async|autofocus|autoplay|controls|defer|disabled|hidden|ismap|loop|multiple|open|readonly|required|scoped\",\n\n\t// Regular expressions\n\n\t// Whitespace characters http://www.w3.org/TR/css3-selectors/#whitespace\n\twhitespace = \"[\\\\x20\\\\t\\\\r\\\\n\\\\f]\",\n\t// http://www.w3.org/TR/css3-syntax/#characters\n\tcharacterEncoding = \"(?:\\\\\\\\.|[\\\\w-]|[^\\\\x00-\\\\xa0])+\",\n\n\t// Loosely modeled on CSS identifier characters\n\t// An unquoted value should be a CSS identifier http://www.w3.org/TR/css3-selectors/#attribute-selectors\n\t// Proper syntax: http://www.w3.org/TR/CSS21/syndata.html#value-def-identifier\n\tidentifier = characterEncoding.replace( \"w\", \"w#\" ),\n\n\t// Acceptable operators http://www.w3.org/TR/selectors/#attribute-selectors\n\tattributes = \"\\\\[\" + whitespace + \"*(\" + characterEncoding + \")\" + whitespace +\n\t\t\"*(?:([*^$|!~]?=)\" + whitespace + \"*(?:(['\\\"])((?:\\\\\\\\.|[^\\\\\\\\])*?)\\\\3|(\" + identifier + \")|)|)\" + whitespace + \"*\\\\]\",\n\n\t// Prefer arguments quoted,\n\t// then not containing pseudos/brackets,\n\t// then attribute selectors/non-parenthetical expressions,\n\t// then anything else\n\t// These preferences are here to reduce the number of selectors\n\t// needing tokenize in the PSEUDO preFilter\n\tpseudos = \":(\" + characterEncoding + \")(?:\\\\(((['\\\"])((?:\\\\\\\\.|[^\\\\\\\\])*?)\\\\3|((?:\\\\\\\\.|[^\\\\\\\\()[\\\\]]|\" + attributes.replace( 3, 8 ) + \")*)|.*)\\\\)|)\",\n\n\t// Leading and non-escaped trailing whitespace, capturing some non-whitespace characters preceding the latter\n\trtrim = new RegExp( \"^\" + whitespace + \"+|((?:^|[^\\\\\\\\])(?:\\\\\\\\.)*)\" + whitespace + \"+$\", \"g\" ),\n\n\trcomma = new RegExp( \"^\" + whitespace + \"*,\" + whitespace + \"*\" ),\n\trcombinators = new RegExp( \"^\" + whitespace + \"*([>+~]|\" + whitespace + \")\" + whitespace + \"*\" ),\n\n\trsibling = new RegExp( whitespace + \"*[+~]\" ),\n\trattributeQuotes = new RegExp( \"=\" + whitespace + \"*([^\\\\]'\\\"]*)\" + whitespace + \"*\\\\]\", \"g\" ),\n\n\trpseudo = new RegExp( pseudos ),\n\tridentifier = new RegExp( \"^\" + identifier + \"$\" ),\n\n\tmatchExpr = {\n\t\t\"ID\": new RegExp( \"^#(\" + characterEncoding + \")\" ),\n\t\t\"CLASS\": new RegExp( \"^\\\\.(\" + characterEncoding + \")\" ),\n\t\t\"TAG\": new RegExp( \"^(\" + characterEncoding.replace( \"w\", \"w*\" ) + \")\" ),\n\t\t\"ATTR\": new RegExp( \"^\" + attributes ),\n\t\t\"PSEUDO\": new RegExp( \"^\" + pseudos ),\n\t\t\"CHILD\": new RegExp( \"^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\\\(\" + whitespace +\n\t\t\t\"*(even|odd|(([+-]|)(\\\\d*)n|)\" + whitespace + \"*(?:([+-]|)\" + whitespace +\n\t\t\t\"*(\\\\d+)|))\" + whitespace + \"*\\\\)|)\", \"i\" ),\n\t\t\"boolean\": new RegExp( \"^(?:\" + booleans + \")$\", \"i\" ),\n\t\t// For use in libraries implementing .is()\n\t\t// We use this for POS matching in `select`\n\t\t\"needsContext\": new RegExp( \"^\" + whitespace + \"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\\\(\" +\n\t\t\twhitespace + \"*((?:-\\\\d)?\\\\d*)\" + whitespace + \"*\\\\)|)(?=[^-]|$)\", \"i\" )\n\t},\n\n\trnative = /^[^{]+\\{\\s*\\[native \\w/,\n\n\t// Easily-parseable/retrievable ID or TAG or CLASS selectors\n\trquickExpr = /^(?:#([\\w-]+)|(\\w+)|\\.([\\w-]+))$/,\n\n\trinputs = /^(?:input|select|textarea|button)$/i,\n\trheader = /^h\\d$/i,\n\n\trescape = /'|\\\\/g,\n\n\t// CSS escapes http://www.w3.org/TR/CSS21/syndata.html#escaped-characters\n\trunescape = /\\\\([\\da-fA-F]{1,6}[\\x20\\t\\r\\n\\f]?|.)/g,\n\tfunescape = function( _, escaped ) {\n\t\tvar high = \"0x\" + escaped - 0x10000;\n\t\t// NaN means non-codepoint\n\t\treturn high !== high ?\n\t\t\tescaped :\n\t\t\t// BMP codepoint\n\t\t\thigh < 0 ?\n\t\t\t\tString.fromCharCode( high + 0x10000 ) :\n\t\t\t\t// Supplemental Plane codepoint (surrogate pair)\n\t\t\t\tString.fromCharCode( high >> 10 | 0xD800, high & 0x3FF | 0xDC00 );\n\t};\n\n// Optimize for push.apply( _, NodeList )\ntry {\n\tpush.apply(\n\t\t(arr = slice.call( preferredDoc.childNodes )),\n\t\tpreferredDoc.childNodes\n\t);\n\t// Support: Android<4.0\n\t// Detect silently failing push.apply\n\tarr[ preferredDoc.childNodes.length ].nodeType;\n} catch ( e ) {\n\tpush = { apply: arr.length ?\n\n\t\t// Leverage slice if possible\n\t\tfunction( target, els ) {\n\t\t\tpush_native.apply( target, slice.call(els) );\n\t\t} :\n\n\t\t// Support: IE<9\n\t\t// Otherwise append directly\n\t\tfunction( target, els ) {\n\t\t\tvar j = target.length,\n\t\t\t\ti = 0;\n\t\t\t// Can't trust NodeList.length\n\t\t\twhile ( (target[j++] = els[i++]) ) {}\n\t\t\ttarget.length = j - 1;\n\t\t}\n\t};\n}\n\n/**\n * For feature detection\n * @param {Function} fn The function to test for native support\n */\nfunction isNative( fn ) {\n\treturn rnative.test( fn + \"\" );\n}\n\n/**\n * Create key-value caches of limited size\n * @returns {Function(string, Object)} Returns the Object data after storing it on itself with\n *\tproperty name the (space-suffixed) string and (if the cache is larger than Expr.cacheLength)\n *\tdeleting the oldest entry\n */\nfunction createCache() {\n\tvar cache,\n\t\tkeys = [];\n\n\treturn (cache = function( key, value ) {\n\t\t// Use (key + \" \") to avoid collision with native prototype properties (see Issue #157)\n\t\tif ( keys.push( key += \" \" ) > Expr.cacheLength ) {\n\t\t\t// Only keep the most recent entries\n\t\t\tdelete cache[ keys.shift() ];\n\t\t}\n\t\treturn (cache[ key ] = value);\n\t});\n}\n\n/**\n * Mark a function for special use by Sizzle\n * @param {Function} fn The function to mark\n */\nfunction markFunction( fn ) {\n\tfn[ expando ] = true;\n\treturn fn;\n}\n\n/**\n * Support testing using an element\n * @param {Function} fn Passed the created div and expects a boolean result\n */\nfunction assert( fn ) {\n\tvar div = document.createElement(\"div\");\n\n\ttry {\n\t\treturn !!fn( div );\n\t} catch (e) {\n\t\treturn false;\n\t} finally {\n\t\tif ( div.parentNode ) {\n\t\t\tdiv.parentNode.removeChild( div );\n\t\t}\n\t\t// release memory in IE\n\t\tdiv = null;\n\t}\n}\n\nfunction Sizzle( selector, context, results, seed ) {\n\tvar match, elem, m, nodeType,\n\t\t// QSA vars\n\t\ti, groups, old, nid, newContext, newSelector;\n\n\tif ( ( context ? context.ownerDocument || context : preferredDoc ) !== document ) {\n\t\tsetDocument( context );\n\t}\n\n\tcontext = context || document;\n\tresults = results || [];\n\n\tif ( !selector || typeof selector !== \"string\" ) {\n\t\treturn results;\n\t}\n\n\tif ( (nodeType = context.nodeType) !== 1 && nodeType !== 9 ) {\n\t\treturn [];\n\t}\n\n\tif ( documentIsHTML && !seed ) {\n\n\t\t// Shortcuts\n\t\tif ( (match = rquickExpr.exec( selector )) ) {\n\t\t\t// Speed-up: Sizzle(\"#ID\")\n\t\t\tif ( (m = match[1]) ) {\n\t\t\t\tif ( nodeType === 9 ) {\n\t\t\t\t\telem = context.getElementById( m );\n\t\t\t\t\t// Check parentNode to catch when Blackberry 4.6 returns\n\t\t\t\t\t// nodes that are no longer in the document #6963\n\t\t\t\t\tif ( elem && elem.parentNode ) {\n\t\t\t\t\t\t// Handle the case where IE, Opera, and Webkit return items\n\t\t\t\t\t\t// by name instead of ID\n\t\t\t\t\t\tif ( elem.id === m ) {\n\t\t\t\t\t\t\tresults.push( elem );\n\t\t\t\t\t\t\treturn results;\n\t\t\t\t\t\t}\n\t\t\t\t\t} else {\n\t\t\t\t\t\treturn results;\n\t\t\t\t\t}\n\t\t\t\t} else {\n\t\t\t\t\t// Context is not a document\n\t\t\t\t\tif ( context.ownerDocument && (elem = context.ownerDocument.getElementById( m )) &&\n\t\t\t\t\t\tcontains( context, elem ) && elem.id === m ) {\n\t\t\t\t\t\tresults.push( elem );\n\t\t\t\t\t\treturn results;\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t// Speed-up: Sizzle(\"TAG\")\n\t\t\t} else if ( match[2] ) {\n\t\t\t\tpush.apply( results, context.getElementsByTagName( selector ) );\n\t\t\t\treturn results;\n\n\t\t\t// Speed-up: Sizzle(\".CLASS\")\n\t\t\t} else if ( (m = match[3]) && support.getElementsByClassName && context.getElementsByClassName ) {\n\t\t\t\tpush.apply( results, context.getElementsByClassName( m ) );\n\t\t\t\treturn results;\n\t\t\t}\n\t\t}\n\n\t\t// QSA path\n\t\tif ( support.qsa && (!rbuggyQSA || !rbuggyQSA.test( selector )) ) {\n\t\t\tnid = old = expando;\n\t\t\tnewContext = context;\n\t\t\tnewSelector = nodeType === 9 && selector;\n\n\t\t\t// qSA works strangely on Element-rooted queries\n\t\t\t// We can work around this by specifying an extra ID on the root\n\t\t\t// and working up from there (Thanks to Andrew Dupont for the technique)\n\t\t\t// IE 8 doesn't work on object elements\n\t\t\tif ( nodeType === 1 && context.nodeName.toLowerCase() !== \"object\" ) {\n\t\t\t\tgroups = tokenize( selector );\n\n\t\t\t\tif ( (old = context.getAttribute(\"id\")) ) {\n\t\t\t\t\tnid = old.replace( rescape, \"\\\\$&\" );\n\t\t\t\t} else {\n\t\t\t\t\tcontext.setAttribute( \"id\", nid );\n\t\t\t\t}\n\t\t\t\tnid = \"[id='\" + nid + \"'] \";\n\n\t\t\t\ti = groups.length;\n\t\t\t\twhile ( i-- ) {\n\t\t\t\t\tgroups[i] = nid + toSelector( groups[i] );\n\t\t\t\t}\n\t\t\t\tnewContext = rsibling.test( selector ) && context.parentNode || context;\n\t\t\t\tnewSelector = groups.join(\",\");\n\t\t\t}\n\n\t\t\tif ( newSelector ) {\n\t\t\t\ttry {\n\t\t\t\t\tpush.apply( results,\n\t\t\t\t\t\tnewContext.querySelectorAll( newSelector )\n\t\t\t\t\t);\n\t\t\t\t\treturn results;\n\t\t\t\t} catch(qsaError) {\n\t\t\t\t} finally {\n\t\t\t\t\tif ( !old ) {\n\t\t\t\t\t\tcontext.removeAttribute(\"id\");\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\t// All others\n\treturn select( selector.replace( rtrim, \"$1\" ), context, results, seed );\n}\n\n/**\n * Detect xml\n * @param {Element|Object} elem An element or a document\n */\nisXML = Sizzle.isXML = function( elem ) {\n\t// documentElement is verified for cases where it doesn't yet exist\n\t// (such as loading iframes in IE - #4833)\n\tvar documentElement = elem && (elem.ownerDocument || elem).documentElement;\n\treturn documentElement ? documentElement.nodeName !== \"HTML\" : false;\n};\n\n/**\n * Sets document-related variables once based on the current document\n * @param {Element|Object} [doc] An element or document object to use to set the document\n * @returns {Object} Returns the current document\n */\nsetDocument = Sizzle.setDocument = function( node ) {\n\tvar doc = node ? node.ownerDocument || node : preferredDoc;\n\n\t// If no document and documentElement is available, return\n\tif ( doc === document || doc.nodeType !== 9 || !doc.documentElement ) {\n\t\treturn document;\n\t}\n\n\t// Set our document\n\tdocument = doc;\n\tdocElem = doc.documentElement;\n\n\t// Support tests\n\tdocumentIsHTML = !isXML( doc );\n\n\t// Check if getElementsByTagName(\"*\") returns only elements\n\tsupport.getElementsByTagName = assert(function( div ) {\n\t\tdiv.appendChild( doc.createComment(\"\") );\n\t\treturn !div.getElementsByTagName(\"*\").length;\n\t});\n\n\t// Support: IE<8\n\t// Verify that getAttribute really returns attributes and not properties (excepting IE8 booleans)\n\tsupport.attributes = assert(function( div ) {\n\t\tdiv.className = \"i\";\n\t\treturn !div.getAttribute(\"className\");\n\t});\n\n\t// Check if getElementsByClassName can be trusted\n\tsupport.getElementsByClassName = assert(function( div ) {\n\t\tdiv.innerHTML = \"<div class='a'></div><div class='a i'></div>\";\n\n\t\t// Support: Safari<4\n\t\t// Catch class over-caching\n\t\tdiv.firstChild.className = \"i\";\n\t\t// Support: Opera<10\n\t\t// Catch gEBCN failure to find non-leading classes\n\t\treturn div.getElementsByClassName(\"i\").length === 2;\n\t});\n\n\t// Support: Webkit<537.32 - Safari 6.0.3/Chrome 25 (fixed in Chrome 27)\n\t// Detached nodes confoundingly follow *each other*\n\tsupport.sortDetached = assert(function( div1 ) {\n\t\t// Should return 1, but returns 4 (following)\n\t\treturn div1.compareDocumentPosition( document.createElement(\"div\") ) & 1;\n\t});\n\n\t// Support: IE<10\n\t// Check if getElementById returns elements by name\n\t// Support: Windows 8 Native Apps\n\t// Assigning innerHTML with \"name\" attributes throws uncatchable exceptions\n\t// (http://msdn.microsoft.com/en-us/library/ie/hh465388.aspx)\n\t// and the broken getElementById methods don't pick up programatically-set names,\n\t// so use a roundabout getElementsByName test\n\tsupport.getById = assert(function( div ) {\n\t\tdocElem.appendChild( div ).id = expando;\n\t\treturn !doc.getElementsByName || !doc.getElementsByName( expando ).length;\n\t});\n\n\t// ID find and filter\n\tif ( support.getById ) {\n\t\tExpr.find[\"ID\"] = function( id, context ) {\n\t\t\tif ( typeof context.getElementById !== strundefined && documentIsHTML ) {\n\t\t\t\tvar m = context.getElementById( id );\n\t\t\t\t// Check parentNode to catch when Blackberry 4.6 returns\n\t\t\t\t// nodes that are no longer in the document #6963\n\t\t\t\treturn m && m.parentNode ? [m] : [];\n\t\t\t}\n\t\t};\n\t\tExpr.filter[\"ID\"] = function( id ) {\n\t\t\tvar attrId = id.replace( runescape, funescape );\n\t\t\treturn function( elem ) {\n\t\t\t\treturn elem.getAttribute(\"id\") === attrId;\n\t\t\t};\n\t\t};\n\t} else {\n\t\tExpr.find[\"ID\"] = function( id, context ) {\n\t\t\tif ( typeof context.getElementById !== strundefined && documentIsHTML ) {\n\t\t\t\tvar m = context.getElementById( id );\n\n\t\t\t\treturn m ?\n\t\t\t\t\tm.id === id || typeof m.getAttributeNode !== strundefined && m.getAttributeNode(\"id\").value === id ?\n\t\t\t\t\t\t[m] :\n\t\t\t\t\t\tundefined :\n\t\t\t\t\t[];\n\t\t\t}\n\t\t};\n\t\tExpr.filter[\"ID\"] = function( id ) {\n\t\t\tvar attrId = id.replace( runescape, funescape );\n\t\t\treturn function( elem ) {\n\t\t\t\tvar node = typeof elem.getAttributeNode !== strundefined && elem.getAttributeNode(\"id\");\n\t\t\t\treturn node && node.value === attrId;\n\t\t\t};\n\t\t};\n\t}\n\n\t// Tag\n\tExpr.find[\"TAG\"] = support.getElementsByTagName ?\n\t\tfunction( tag, context ) {\n\t\t\tif ( typeof context.getElementsByTagName !== strundefined ) {\n\t\t\t\treturn context.getElementsByTagName( tag );\n\t\t\t}\n\t\t} :\n\t\tfunction( tag, context ) {\n\t\t\tvar elem,\n\t\t\t\ttmp = [],\n\t\t\t\ti = 0,\n\t\t\t\tresults = context.getElementsByTagName( tag );\n\n\t\t\t// Filter out possible comments\n\t\t\tif ( tag === \"*\" ) {\n\t\t\t\twhile ( (elem = results[i++]) ) {\n\t\t\t\t\tif ( elem.nodeType === 1 ) {\n\t\t\t\t\t\ttmp.push( elem );\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\treturn tmp;\n\t\t\t}\n\t\t\treturn results;\n\t\t};\n\n\t// Class\n\tExpr.find[\"CLASS\"] = support.getElementsByClassName && function( className, context ) {\n\t\tif ( typeof context.getElementsByClassName !== strundefined && documentIsHTML ) {\n\t\t\treturn context.getElementsByClassName( className );\n\t\t}\n\t};\n\n\t// QSA and matchesSelector support\n\n\t// matchesSelector(:active) reports false when true (IE9/Opera 11.5)\n\trbuggyMatches = [];\n\n\t// qSa(:focus) reports false when true (Chrome 21)\n\t// We allow this because of a bug in IE8/9 that throws an error\n\t// whenever `document.activeElement` is accessed on an iframe\n\t// So, we allow :focus to pass through QSA all the time to avoid the IE error\n\t// See http://bugs.jquery.com/ticket/13378\n\trbuggyQSA = [];\n\n\tif ( (support.qsa = isNative(doc.querySelectorAll)) ) {\n\t\t// Build QSA regex\n\t\t// Regex strategy adopted from Diego Perini\n\t\tassert(function( div ) {\n\t\t\t// Select is set to empty string on purpose\n\t\t\t// This is to test IE's treatment of not explicitly\n\t\t\t// setting a boolean content attribute,\n\t\t\t// since its presence should be enough\n\t\t\t// http://bugs.jquery.com/ticket/12359\n\t\t\tdiv.innerHTML = \"<select><option selected=''></option></select>\";\n\n\t\t\t// Support: IE8\n\t\t\t// Boolean attributes and \"value\" are not treated correctly\n\t\t\tif ( !div.querySelectorAll(\"[selected]\").length ) {\n\t\t\t\trbuggyQSA.push( \"\\\\[\" + whitespace + \"*(?:value|\" + booleans + \")\" );\n\t\t\t}\n\n\t\t\t// Webkit/Opera - :checked should return selected option elements\n\t\t\t// http://www.w3.org/TR/2011/REC-css3-selectors-20110929/#checked\n\t\t\t// IE8 throws error here and will not see later tests\n\t\t\tif ( !div.querySelectorAll(\":checked\").length ) {\n\t\t\t\trbuggyQSA.push(\":checked\");\n\t\t\t}\n\t\t});\n\n\t\tassert(function( div ) {\n\n\t\t\t// Support: Opera 10-12/IE8\n\t\t\t// ^= $= *= and empty values\n\t\t\t// Should not select anything\n\t\t\t// Support: Windows 8 Native Apps\n\t\t\t// The type attribute is restricted during .innerHTML assignment\n\t\t\tvar input = document.createElement(\"input\");\n\t\t\tinput.setAttribute( \"type\", \"hidden\" );\n\t\t\tdiv.appendChild( input ).setAttribute( \"t\", \"\" );\n\n\t\t\tif ( div.querySelectorAll(\"[t^='']\").length ) {\n\t\t\t\trbuggyQSA.push( \"[*^$]=\" + whitespace + \"*(?:''|\\\"\\\")\" );\n\t\t\t}\n\n\t\t\t// FF 3.5 - :enabled/:disabled and hidden elements (hidden elements are still enabled)\n\t\t\t// IE8 throws error here and will not see later tests\n\t\t\tif ( !div.querySelectorAll(\":enabled\").length ) {\n\t\t\t\trbuggyQSA.push( \":enabled\", \":disabled\" );\n\t\t\t}\n\n\t\t\t// Opera 10-11 does not throw on post-comma invalid pseudos\n\t\t\tdiv.querySelectorAll(\"*,:x\");\n\t\t\trbuggyQSA.push(\",.*:\");\n\t\t});\n\t}\n\n\tif ( (support.matchesSelector = isNative( (matches = docElem.webkitMatchesSelector ||\n\t\tdocElem.mozMatchesSelector ||\n\t\tdocElem.oMatchesSelector ||\n\t\tdocElem.msMatchesSelector) )) ) {\n\n\t\tassert(function( div ) {\n\t\t\t// Check to see if it's possible to do matchesSelector\n\t\t\t// on a disconnected node (IE 9)\n\t\t\tsupport.disconnectedMatch = matches.call( div, \"div\" );\n\n\t\t\t// This should fail with an exception\n\t\t\t// Gecko does not error, returns false instead\n\t\t\tmatches.call( div, \"[s!='']:x\" );\n\t\t\trbuggyMatches.push( \"!=\", pseudos );\n\t\t});\n\t}\n\n\trbuggyQSA = rbuggyQSA.length && new RegExp( rbuggyQSA.join(\"|\") );\n\trbuggyMatches = rbuggyMatches.length && new RegExp( rbuggyMatches.join(\"|\") );\n\n\t// Element contains another\n\t// Purposefully does not implement inclusive descendent\n\t// As in, an element does not contain itself\n\tcontains = isNative(docElem.contains) || docElem.compareDocumentPosition ?\n\t\tfunction( a, b ) {\n\t\t\tvar adown = a.nodeType === 9 ? a.documentElement : a,\n\t\t\t\tbup = b && b.parentNode;\n\t\t\treturn a === bup || !!( bup && bup.nodeType === 1 && (\n\t\t\t\tadown.contains ?\n\t\t\t\t\tadown.contains( bup ) :\n\t\t\t\t\ta.compareDocumentPosition && a.compareDocumentPosition( bup ) & 16\n\t\t\t));\n\t\t} :\n\t\tfunction( a, b ) {\n\t\t\tif ( b ) {\n\t\t\t\twhile ( (b = b.parentNode) ) {\n\t\t\t\t\tif ( b === a ) {\n\t\t\t\t\t\treturn true;\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn false;\n\t\t};\n\n\t// Document order sorting\n\tsortOrder = docElem.compareDocumentPosition ?\n\tfunction( a, b ) {\n\n\t\t// Flag for duplicate removal\n\t\tif ( a === b ) {\n\t\t\thasDuplicate = true;\n\t\t\treturn 0;\n\t\t}\n\n\t\tvar compare = b.compareDocumentPosition && a.compareDocumentPosition && a.compareDocumentPosition( b );\n\n\t\tif ( compare ) {\n\t\t\t// Disconnected nodes\n\t\t\tif ( compare & 1 ||\n\t\t\t\t(!support.sortDetached && b.compareDocumentPosition( a ) === compare) ) {\n\n\t\t\t\t// Choose the first element that is related to our preferred document\n\t\t\t\tif ( a === doc || contains(preferredDoc, a) ) {\n\t\t\t\t\treturn -1;\n\t\t\t\t}\n\t\t\t\tif ( b === doc || contains(preferredDoc, b) ) {\n\t\t\t\t\treturn 1;\n\t\t\t\t}\n\n\t\t\t\t// Maintain original order\n\t\t\t\treturn sortInput ?\n\t\t\t\t\t( indexOf.call( sortInput, a ) - indexOf.call( sortInput, b ) ) :\n\t\t\t\t\t0;\n\t\t\t}\n\n\t\t\treturn compare & 4 ? -1 : 1;\n\t\t}\n\n\t\t// Not directly comparable, sort on existence of method\n\t\treturn a.compareDocumentPosition ? -1 : 1;\n\t} :\n\tfunction( a, b ) {\n\t\tvar cur,\n\t\t\ti = 0,\n\t\t\taup = a.parentNode,\n\t\t\tbup = b.parentNode,\n\t\t\tap = [ a ],\n\t\t\tbp = [ b ];\n\n\t\t// Exit early if the nodes are identical\n\t\tif ( a === b ) {\n\t\t\thasDuplicate = true;\n\t\t\treturn 0;\n\n\t\t// Parentless nodes are either documents or disconnected\n\t\t} else if ( !aup || !bup ) {\n\t\t\treturn a === doc ? -1 :\n\t\t\t\tb === doc ? 1 :\n\t\t\t\taup ? -1 :\n\t\t\t\tbup ? 1 :\n\t\t\t\tsortInput ?\n\t\t\t\t( indexOf.call( sortInput, a ) - indexOf.call( sortInput, b ) ) :\n\t\t\t\t0;\n\n\t\t// If the nodes are siblings, we can do a quick check\n\t\t} else if ( aup === bup ) {\n\t\t\treturn siblingCheck( a, b );\n\t\t}\n\n\t\t// Otherwise we need full lists of their ancestors for comparison\n\t\tcur = a;\n\t\twhile ( (cur = cur.parentNode) ) {\n\t\t\tap.unshift( cur );\n\t\t}\n\t\tcur = b;\n\t\twhile ( (cur = cur.parentNode) ) {\n\t\t\tbp.unshift( cur );\n\t\t}\n\n\t\t// Walk down the tree looking for a discrepancy\n\t\twhile ( ap[i] === bp[i] ) {\n\t\t\ti++;\n\t\t}\n\n\t\treturn i ?\n\t\t\t// Do a sibling check if the nodes have a common ancestor\n\t\t\tsiblingCheck( ap[i], bp[i] ) :\n\n\t\t\t// Otherwise nodes in our document sort first\n\t\t\tap[i] === preferredDoc ? -1 :\n\t\t\tbp[i] === preferredDoc ? 1 :\n\t\t\t0;\n\t};\n\n\treturn document;\n};\n\nSizzle.matches = function( expr, elements ) {\n\treturn Sizzle( expr, null, null, elements );\n};\n\nSizzle.matchesSelector = function( elem, expr ) {\n\t// Set document vars if needed\n\tif ( ( elem.ownerDocument || elem ) !== document ) {\n\t\tsetDocument( elem );\n\t}\n\n\t// Make sure that attribute selectors are quoted\n\texpr = expr.replace( rattributeQuotes, \"='$1']\" );\n\n\t// rbuggyQSA always contains :focus, so no need for an existence check\n\tif ( support.matchesSelector && documentIsHTML &&\n\t\t(!rbuggyMatches || !rbuggyMatches.test(expr)) &&\n\t\t(!rbuggyQSA || !rbuggyQSA.test(expr)) ) {\n\n\t\ttry {\n\t\t\tvar ret = matches.call( elem, expr );\n\n\t\t\t// IE 9's matchesSelector returns false on disconnected nodes\n\t\t\tif ( ret || support.disconnectedMatch ||\n\t\t\t\t\t// As well, disconnected nodes are said to be in a document\n\t\t\t\t\t// fragment in IE 9\n\t\t\t\t\telem.document && elem.document.nodeType !== 11 ) {\n\t\t\t\treturn ret;\n\t\t\t}\n\t\t} catch(e) {}\n\t}\n\n\treturn Sizzle( expr, document, null, [elem] ).length > 0;\n};\n\nSizzle.contains = function( context, elem ) {\n\t// Set document vars if needed\n\tif ( ( context.ownerDocument || context ) !== document ) {\n\t\tsetDocument( context );\n\t}\n\treturn contains( context, elem );\n};\n\nSizzle.attr = function( elem, name ) {\n\t// Set document vars if needed\n\tif ( ( elem.ownerDocument || elem ) !== document ) {\n\t\tsetDocument( elem );\n\t}\n\n\tvar fn = Expr.attrHandle[ name.toLowerCase() ],\n\t\tval = fn && fn( elem, name, !documentIsHTML );\n\n\treturn val === undefined ?\n\t\tsupport.attributes || !documentIsHTML ?\n\t\t\telem.getAttribute( name ) :\n\t\t\t(val = elem.getAttributeNode(name)) && val.specified ?\n\t\t\t\tval.value :\n\t\t\t\tnull :\n\t\tval;\n};\n\nSizzle.error = function( msg ) {\n\tthrow new Error( \"Syntax error, unrecognized expression: \" + msg );\n};\n\n// Document sorting and removing duplicates\nSizzle.uniqueSort = function( results ) {\n\tvar elem,\n\t\tduplicates = [],\n\t\tj = 0,\n\t\ti = 0;\n\n\t// Unless we *know* we can detect duplicates, assume their presence\n\thasDuplicate = !support.detectDuplicates;\n\tsortInput = !support.sortStable && results.slice( 0 );\n\tresults.sort( sortOrder );\n\n\tif ( hasDuplicate ) {\n\t\twhile ( (elem = results[i++]) ) {\n\t\t\tif ( elem === results[ i ] ) {\n\t\t\t\tj = duplicates.push( i );\n\t\t\t}\n\t\t}\n\t\twhile ( j-- ) {\n\t\t\tresults.splice( duplicates[ j ], 1 );\n\t\t}\n\t}\n\n\treturn results;\n};\n\n/**\n * Checks document order of two siblings\n * @param {Element} a\n * @param {Element} b\n * @returns Returns -1 if a precedes b, 1 if a follows b\n */\nfunction siblingCheck( a, b ) {\n\tvar cur = b && a,\n\t\tdiff = cur && ( ~b.sourceIndex || MAX_NEGATIVE ) - ( ~a.sourceIndex || MAX_NEGATIVE );\n\n\t// Use IE sourceIndex if available on both nodes\n\tif ( diff ) {\n\t\treturn diff;\n\t}\n\n\t// Check if b follows a\n\tif ( cur ) {\n\t\twhile ( (cur = cur.nextSibling) ) {\n\t\t\tif ( cur === b ) {\n\t\t\t\treturn -1;\n\t\t\t}\n\t\t}\n\t}\n\n\treturn a ? 1 : -1;\n}\n\n// Fetches boolean attributes by node\nfunction boolHandler( elem, name, isXML ) {\n\tvar val;\n\treturn isXML ?\n\t\tundefined :\n\t\t(val = elem.getAttributeNode( name )) && val.specified ?\n\t\t\tval.value :\n\t\t\telem[ name ] === true ? name.toLowerCase() : null;\n}\n\n// Fetches attributes without interpolation\n// http://msdn.microsoft.com/en-us/library/ms536429%28VS.85%29.aspx\nfunction interpolationHandler( elem, name, isXML ) {\n\tvar val;\n\treturn isXML ?\n\t\tundefined :\n\t\t(val = elem.getAttribute( name, name.toLowerCase() === \"type\" ? 1 : 2 ));\n}\n\n// Returns a function to use in pseudos for input types\nfunction createInputPseudo( type ) {\n\treturn function( elem ) {\n\t\tvar name = elem.nodeName.toLowerCase();\n\t\treturn name === \"input\" && elem.type === type;\n\t};\n}\n\n// Returns a function to use in pseudos for buttons\nfunction createButtonPseudo( type ) {\n\treturn function( elem ) {\n\t\tvar name = elem.nodeName.toLowerCase();\n\t\treturn (name === \"input\" || name === \"button\") && elem.type === type;\n\t};\n}\n\n// Returns a function to use in pseudos for positionals\nfunction createPositionalPseudo( fn ) {\n\treturn markFunction(function( argument ) {\n\t\targument = +argument;\n\t\treturn markFunction(function( seed, matches ) {\n\t\t\tvar j,\n\t\t\t\tmatchIndexes = fn( [], seed.length, argument ),\n\t\t\t\ti = matchIndexes.length;\n\n\t\t\t// Match elements found at the specified indexes\n\t\t\twhile ( i-- ) {\n\t\t\t\tif ( seed[ (j = matchIndexes[i]) ] ) {\n\t\t\t\t\tseed[j] = !(matches[j] = seed[j]);\n\t\t\t\t}\n\t\t\t}\n\t\t});\n\t});\n}\n\n/**\n * Utility function for retrieving the text value of an array of DOM nodes\n * @param {Array|Element} elem\n */\ngetText = Sizzle.getText = function( elem ) {\n\tvar node,\n\t\tret = \"\",\n\t\ti = 0,\n\t\tnodeType = elem.nodeType;\n\n\tif ( !nodeType ) {\n\t\t// If no nodeType, this is expected to be an array\n\t\tfor ( ; (node = elem[i]); i++ ) {\n\t\t\t// Do not traverse comment nodes\n\t\t\tret += getText( node );\n\t\t}\n\t} else if ( nodeType === 1 || nodeType === 9 || nodeType === 11 ) {\n\t\t// Use textContent for elements\n\t\t// innerText usage removed for consistency of new lines (see #11153)\n\t\tif ( typeof elem.textContent === \"string\" ) {\n\t\t\treturn elem.textContent;\n\t\t} else {\n\t\t\t// Traverse its children\n\t\t\tfor ( elem = elem.firstChild; elem; elem = elem.nextSibling ) {\n\t\t\t\tret += getText( elem );\n\t\t\t}\n\t\t}\n\t} else if ( nodeType === 3 || nodeType === 4 ) {\n\t\treturn elem.nodeValue;\n\t}\n\t// Do not include comment or processing instruction nodes\n\n\treturn ret;\n};\n\nExpr = Sizzle.selectors = {\n\n\t// Can be adjusted by the user\n\tcacheLength: 50,\n\n\tcreatePseudo: markFunction,\n\n\tmatch: matchExpr,\n\n\tattrHandle: {},\n\n\tfind: {},\n\n\trelative: {\n\t\t\">\": { dir: \"parentNode\", first: true },\n\t\t\" \": { dir: \"parentNode\" },\n\t\t\"+\": { dir: \"previousSibling\", first: true },\n\t\t\"~\": { dir: \"previousSibling\" }\n\t},\n\n\tpreFilter: {\n\t\t\"ATTR\": function( match ) {\n\t\t\tmatch[1] = match[1].replace( runescape, funescape );\n\n\t\t\t// Move the given value to match[3] whether quoted or unquoted\n\t\t\tmatch[3] = ( match[4] || match[5] || \"\" ).replace( runescape, funescape );\n\n\t\t\tif ( match[2] === \"~=\" ) {\n\t\t\t\tmatch[3] = \" \" + match[3] + \" \";\n\t\t\t}\n\n\t\t\treturn match.slice( 0, 4 );\n\t\t},\n\n\t\t\"CHILD\": function( match ) {\n\t\t\t/* matches from matchExpr[\"CHILD\"]\n\t\t\t\t1 type (only|nth|...)\n\t\t\t\t2 what (child|of-type)\n\t\t\t\t3 argument (even|odd|\\d*|\\d*n([+-]\\d+)?|...)\n\t\t\t\t4 xn-component of xn+y argument ([+-]?\\d*n|)\n\t\t\t\t5 sign of xn-component\n\t\t\t\t6 x of xn-component\n\t\t\t\t7 sign of y-component\n\t\t\t\t8 y of y-component\n\t\t\t*/\n\t\t\tmatch[1] = match[1].toLowerCase();\n\n\t\t\tif ( match[1].slice( 0, 3 ) === \"nth\" ) {\n\t\t\t\t// nth-* requires argument\n\t\t\t\tif ( !match[3] ) {\n\t\t\t\t\tSizzle.error( match[0] );\n\t\t\t\t}\n\n\t\t\t\t// numeric x and y parameters for Expr.filter.CHILD\n\t\t\t\t// remember that false/true cast respectively to 0/1\n\t\t\t\tmatch[4] = +( match[4] ? match[5] + (match[6] || 1) : 2 * ( match[3] === \"even\" || match[3] === \"odd\" ) );\n\t\t\t\tmatch[5] = +( ( match[7] + match[8] ) || match[3] === \"odd\" );\n\n\t\t\t// other types prohibit arguments\n\t\t\t} else if ( match[3] ) {\n\t\t\t\tSizzle.error( match[0] );\n\t\t\t}\n\n\t\t\treturn match;\n\t\t},\n\n\t\t\"PSEUDO\": function( match ) {\n\t\t\tvar excess,\n\t\t\t\tunquoted = !match[5] && match[2];\n\n\t\t\tif ( matchExpr[\"CHILD\"].test( match[0] ) ) {\n\t\t\t\treturn null;\n\t\t\t}\n\n\t\t\t// Accept quoted arguments as-is\n\t\t\tif ( match[4] ) {\n\t\t\t\tmatch[2] = match[4];\n\n\t\t\t// Strip excess characters from unquoted arguments\n\t\t\t} else if ( unquoted && rpseudo.test( unquoted ) &&\n\t\t\t\t// Get excess from tokenize (recursively)\n\t\t\t\t(excess = tokenize( unquoted, true )) &&\n\t\t\t\t// advance to the next closing parenthesis\n\t\t\t\t(excess = unquoted.indexOf( \")\", unquoted.length - excess ) - unquoted.length) ) {\n\n\t\t\t\t// excess is a negative index\n\t\t\t\tmatch[0] = match[0].slice( 0, excess );\n\t\t\t\tmatch[2] = unquoted.slice( 0, excess );\n\t\t\t}\n\n\t\t\t// Return only captures needed by the pseudo filter method (type and argument)\n\t\t\treturn match.slice( 0, 3 );\n\t\t}\n\t},\n\n\tfilter: {\n\n\t\t\"TAG\": function( nodeNameSelector ) {\n\t\t\tvar nodeName = nodeNameSelector.replace( runescape, funescape ).toLowerCase();\n\t\t\treturn nodeNameSelector === \"*\" ?\n\t\t\t\tfunction() { return true; } :\n\t\t\t\tfunction( elem ) {\n\t\t\t\t\treturn elem.nodeName && elem.nodeName.toLowerCase() === nodeName;\n\t\t\t\t};\n\t\t},\n\n\t\t\"CLASS\": function( className ) {\n\t\t\tvar pattern = classCache[ className + \" \" ];\n\n\t\t\treturn pattern ||\n\t\t\t\t(pattern = new RegExp( \"(^|\" + whitespace + \")\" + className + \"(\" + whitespace + \"|$)\" )) &&\n\t\t\t\tclassCache( className, function( elem ) {\n\t\t\t\t\treturn pattern.test( typeof elem.className === \"string\" && elem.className || typeof elem.getAttribute !== strundefined && elem.getAttribute(\"class\") || \"\" );\n\t\t\t\t});\n\t\t},\n\n\t\t\"ATTR\": function( name, operator, check ) {\n\t\t\treturn function( elem ) {\n\t\t\t\tvar result = Sizzle.attr( elem, name );\n\n\t\t\t\tif ( result == null ) {\n\t\t\t\t\treturn operator === \"!=\";\n\t\t\t\t}\n\t\t\t\tif ( !operator ) {\n\t\t\t\t\treturn true;\n\t\t\t\t}\n\n\t\t\t\tresult += \"\";\n\n\t\t\t\treturn operator === \"=\" ? result === check :\n\t\t\t\t\toperator === \"!=\" ? result !== check :\n\t\t\t\t\toperator === \"^=\" ? check && result.indexOf( check ) === 0 :\n\t\t\t\t\toperator === \"*=\" ? check && result.indexOf( check ) > -1 :\n\t\t\t\t\toperator === \"$=\" ? check && result.slice( -check.length ) === check :\n\t\t\t\t\toperator === \"~=\" ? ( \" \" + result + \" \" ).indexOf( check ) > -1 :\n\t\t\t\t\toperator === \"|=\" ? result === check || result.slice( 0, check.length + 1 ) === check + \"-\" :\n\t\t\t\t\tfalse;\n\t\t\t};\n\t\t},\n\n\t\t\"CHILD\": function( type, what, argument, first, last ) {\n\t\t\tvar simple = type.slice( 0, 3 ) !== \"nth\",\n\t\t\t\tforward = type.slice( -4 ) !== \"last\",\n\t\t\t\tofType = what === \"of-type\";\n\n\t\t\treturn first === 1 && last === 0 ?\n\n\t\t\t\t// Shortcut for :nth-*(n)\n\t\t\t\tfunction( elem ) {\n\t\t\t\t\treturn !!elem.parentNode;\n\t\t\t\t} :\n\n\t\t\t\tfunction( elem, context, xml ) {\n\t\t\t\t\tvar cache, outerCache, node, diff, nodeIndex, start,\n\t\t\t\t\t\tdir = simple !== forward ? \"nextSibling\" : \"previousSibling\",\n\t\t\t\t\t\tparent = elem.parentNode,\n\t\t\t\t\t\tname = ofType && elem.nodeName.toLowerCase(),\n\t\t\t\t\t\tuseCache = !xml && !ofType;\n\n\t\t\t\t\tif ( parent ) {\n\n\t\t\t\t\t\t// :(first|last|only)-(child|of-type)\n\t\t\t\t\t\tif ( simple ) {\n\t\t\t\t\t\t\twhile ( dir ) {\n\t\t\t\t\t\t\t\tnode = elem;\n\t\t\t\t\t\t\t\twhile ( (node = node[ dir ]) ) {\n\t\t\t\t\t\t\t\t\tif ( ofType ? node.nodeName.toLowerCase() === name : node.nodeType === 1 ) {\n\t\t\t\t\t\t\t\t\t\treturn false;\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t// Reverse direction for :only-* (if we haven't yet done so)\n\t\t\t\t\t\t\t\tstart = dir = type === \"only\" && !start && \"nextSibling\";\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\treturn true;\n\t\t\t\t\t\t}\n\n\t\t\t\t\t\tstart = [ forward ? parent.firstChild : parent.lastChild ];\n\n\t\t\t\t\t\t// non-xml :nth-child(...) stores cache data on `parent`\n\t\t\t\t\t\tif ( forward && useCache ) {\n\t\t\t\t\t\t\t// Seek `elem` from a previously-cached index\n\t\t\t\t\t\t\touterCache = parent[ expando ] || (parent[ expando ] = {});\n\t\t\t\t\t\t\tcache = outerCache[ type ] || [];\n\t\t\t\t\t\t\tnodeIndex = cache[0] === dirruns && cache[1];\n\t\t\t\t\t\t\tdiff = cache[0] === dirruns && cache[2];\n\t\t\t\t\t\t\tnode = nodeIndex && parent.childNodes[ nodeIndex ];\n\n\t\t\t\t\t\t\twhile ( (node = ++nodeIndex && node && node[ dir ] ||\n\n\t\t\t\t\t\t\t\t// Fallback to seeking `elem` from the start\n\t\t\t\t\t\t\t\t(diff = nodeIndex = 0) || start.pop()) ) {\n\n\t\t\t\t\t\t\t\t// When found, cache indexes on `parent` and break\n\t\t\t\t\t\t\t\tif ( node.nodeType === 1 && ++diff && node === elem ) {\n\t\t\t\t\t\t\t\t\touterCache[ type ] = [ dirruns, nodeIndex, diff ];\n\t\t\t\t\t\t\t\t\tbreak;\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t// Use previously-cached element index if available\n\t\t\t\t\t\t} else if ( useCache && (cache = (elem[ expando ] || (elem[ expando ] = {}))[ type ]) && cache[0] === dirruns ) {\n\t\t\t\t\t\t\tdiff = cache[1];\n\n\t\t\t\t\t\t// xml :nth-child(...) or :nth-last-child(...) or :nth(-last)?-of-type(...)\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\t// Use the same loop as above to seek `elem` from the start\n\t\t\t\t\t\t\twhile ( (node = ++nodeIndex && node && node[ dir ] ||\n\t\t\t\t\t\t\t\t(diff = nodeIndex = 0) || start.pop()) ) {\n\n\t\t\t\t\t\t\t\tif ( ( ofType ? node.nodeName.toLowerCase() === name : node.nodeType === 1 ) && ++diff ) {\n\t\t\t\t\t\t\t\t\t// Cache the index of each encountered element\n\t\t\t\t\t\t\t\t\tif ( useCache ) {\n\t\t\t\t\t\t\t\t\t\t(node[ expando ] || (node[ expando ] = {}))[ type ] = [ dirruns, diff ];\n\t\t\t\t\t\t\t\t\t}\n\n\t\t\t\t\t\t\t\t\tif ( node === elem ) {\n\t\t\t\t\t\t\t\t\t\tbreak;\n\t\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\n\t\t\t\t\t\t// Incorporate the offset, then check against cycle size\n\t\t\t\t\t\tdiff -= last;\n\t\t\t\t\t\treturn diff === first || ( diff % first === 0 && diff / first >= 0 );\n\t\t\t\t\t}\n\t\t\t\t};\n\t\t},\n\n\t\t\"PSEUDO\": function( pseudo, argument ) {\n\t\t\t// pseudo-class names are case-insensitive\n\t\t\t// http://www.w3.org/TR/selectors/#pseudo-classes\n\t\t\t// Prioritize by case sensitivity in case custom pseudos are added with uppercase letters\n\t\t\t// Remember that setFilters inherits from pseudos\n\t\t\tvar args,\n\t\t\t\tfn = Expr.pseudos[ pseudo ] || Expr.setFilters[ pseudo.toLowerCase() ] ||\n\t\t\t\t\tSizzle.error( \"unsupported pseudo: \" + pseudo );\n\n\t\t\t// The user may use createPseudo to indicate that\n\t\t\t// arguments are needed to create the filter function\n\t\t\t// just as Sizzle does\n\t\t\tif ( fn[ expando ] ) {\n\t\t\t\treturn fn( argument );\n\t\t\t}\n\n\t\t\t// But maintain support for old signatures\n\t\t\tif ( fn.length > 1 ) {\n\t\t\t\targs = [ pseudo, pseudo, \"\", argument ];\n\t\t\t\treturn Expr.setFilters.hasOwnProperty( pseudo.toLowerCase() ) ?\n\t\t\t\t\tmarkFunction(function( seed, matches ) {\n\t\t\t\t\t\tvar idx,\n\t\t\t\t\t\t\tmatched = fn( seed, argument ),\n\t\t\t\t\t\t\ti = matched.length;\n\t\t\t\t\t\twhile ( i-- ) {\n\t\t\t\t\t\t\tidx = indexOf.call( seed, matched[i] );\n\t\t\t\t\t\t\tseed[ idx ] = !( matches[ idx ] = matched[i] );\n\t\t\t\t\t\t}\n\t\t\t\t\t}) :\n\t\t\t\t\tfunction( elem ) {\n\t\t\t\t\t\treturn fn( elem, 0, args );\n\t\t\t\t\t};\n\t\t\t}\n\n\t\t\treturn fn;\n\t\t}\n\t},\n\n\tpseudos: {\n\t\t// Potentially complex pseudos\n\t\t\"not\": markFunction(function( selector ) {\n\t\t\t// Trim the selector passed to compile\n\t\t\t// to avoid treating leading and trailing\n\t\t\t// spaces as combinators\n\t\t\tvar input = [],\n\t\t\t\tresults = [],\n\t\t\t\tmatcher = compile( selector.replace( rtrim, \"$1\" ) );\n\n\t\t\treturn matcher[ expando ] ?\n\t\t\t\tmarkFunction(function( seed, matches, context, xml ) {\n\t\t\t\t\tvar elem,\n\t\t\t\t\t\tunmatched = matcher( seed, null, xml, [] ),\n\t\t\t\t\t\ti = seed.length;\n\n\t\t\t\t\t// Match elements unmatched by `matcher`\n\t\t\t\t\twhile ( i-- ) {\n\t\t\t\t\t\tif ( (elem = unmatched[i]) ) {\n\t\t\t\t\t\t\tseed[i] = !(matches[i] = elem);\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}) :\n\t\t\t\tfunction( elem, context, xml ) {\n\t\t\t\t\tinput[0] = elem;\n\t\t\t\t\tmatcher( input, null, xml, results );\n\t\t\t\t\treturn !results.pop();\n\t\t\t\t};\n\t\t}),\n\n\t\t\"has\": markFunction(function( selector ) {\n\t\t\treturn function( elem ) {\n\t\t\t\treturn Sizzle( selector, elem ).length > 0;\n\t\t\t};\n\t\t}),\n\n\t\t\"contains\": markFunction(function( text ) {\n\t\t\treturn function( elem ) {\n\t\t\t\treturn ( elem.textContent || elem.innerText || getText( elem ) ).indexOf( text ) > -1;\n\t\t\t};\n\t\t}),\n\n\t\t// \"Whether an element is represented by a :lang() selector\n\t\t// is based solely on the element's language value\n\t\t// being equal to the identifier C,\n\t\t// or beginning with the identifier C immediately followed by \"-\".\n\t\t// The matching of C against the element's language value is performed case-insensitively.\n\t\t// The identifier C does not have to be a valid language name.\"\n\t\t// http://www.w3.org/TR/selectors/#lang-pseudo\n\t\t\"lang\": markFunction( function( lang ) {\n\t\t\t// lang value must be a valid identifier\n\t\t\tif ( !ridentifier.test(lang || \"\") ) {\n\t\t\t\tSizzle.error( \"unsupported lang: \" + lang );\n\t\t\t}\n\t\t\tlang = lang.replace( runescape, funescape ).toLowerCase();\n\t\t\treturn function( elem ) {\n\t\t\t\tvar elemLang;\n\t\t\t\tdo {\n\t\t\t\t\tif ( (elemLang = documentIsHTML ?\n\t\t\t\t\t\telem.lang :\n\t\t\t\t\t\telem.getAttribute(\"xml:lang\") || elem.getAttribute(\"lang\")) ) {\n\n\t\t\t\t\t\telemLang = elemLang.toLowerCase();\n\t\t\t\t\t\treturn elemLang === lang || elemLang.indexOf( lang + \"-\" ) === 0;\n\t\t\t\t\t}\n\t\t\t\t} while ( (elem = elem.parentNode) && elem.nodeType === 1 );\n\t\t\t\treturn false;\n\t\t\t};\n\t\t}),\n\n\t\t// Miscellaneous\n\t\t\"target\": function( elem ) {\n\t\t\tvar hash = window.location && window.location.hash;\n\t\t\treturn hash && hash.slice( 1 ) === elem.id;\n\t\t},\n\n\t\t\"root\": function( elem ) {\n\t\t\treturn elem === docElem;\n\t\t},\n\n\t\t\"focus\": function( elem ) {\n\t\t\treturn elem === document.activeElement && (!document.hasFocus || document.hasFocus()) && !!(elem.type || elem.href || ~elem.tabIndex);\n\t\t},\n\n\t\t// Boolean properties\n\t\t\"enabled\": function( elem ) {\n\t\t\treturn elem.disabled === false;\n\t\t},\n\n\t\t\"disabled\": function( elem ) {\n\t\t\treturn elem.disabled === true;\n\t\t},\n\n\t\t\"checked\": function( elem ) {\n\t\t\t// In CSS3, :checked should return both checked and selected elements\n\t\t\t// http://www.w3.org/TR/2011/REC-css3-selectors-20110929/#checked\n\t\t\tvar nodeName = elem.nodeName.toLowerCase();\n\t\t\treturn (nodeName === \"input\" && !!elem.checked) || (nodeName === \"option\" && !!elem.selected);\n\t\t},\n\n\t\t\"selected\": function( elem ) {\n\t\t\t// Accessing this property makes selected-by-default\n\t\t\t// options in Safari work properly\n\t\t\tif ( elem.parentNode ) {\n\t\t\t\telem.parentNode.selectedIndex;\n\t\t\t}\n\n\t\t\treturn elem.selected === true;\n\t\t},\n\n\t\t// Contents\n\t\t\"empty\": function( elem ) {\n\t\t\t// http://www.w3.org/TR/selectors/#empty-pseudo\n\t\t\t// :empty is only affected by element nodes and content nodes(including text(3), cdata(4)),\n\t\t\t// not comment, processing instructions, or others\n\t\t\t// Thanks to Diego Perini for the nodeName shortcut\n\t\t\t// Greater than \"@\" means alpha characters (specifically not starting with \"#\" or \"?\")\n\t\t\tfor ( elem = elem.firstChild; elem; elem = elem.nextSibling ) {\n\t\t\t\tif ( elem.nodeName > \"@\" || elem.nodeType === 3 || elem.nodeType === 4 ) {\n\t\t\t\t\treturn false;\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn true;\n\t\t},\n\n\t\t\"parent\": function( elem ) {\n\t\t\treturn !Expr.pseudos[\"empty\"]( elem );\n\t\t},\n\n\t\t// Element/input types\n\t\t\"header\": function( elem ) {\n\t\t\treturn rheader.test( elem.nodeName );\n\t\t},\n\n\t\t\"input\": function( elem ) {\n\t\t\treturn rinputs.test( elem.nodeName );\n\t\t},\n\n\t\t\"button\": function( elem ) {\n\t\t\tvar name = elem.nodeName.toLowerCase();\n\t\t\treturn name === \"input\" && elem.type === \"button\" || name === \"button\";\n\t\t},\n\n\t\t\"text\": function( elem ) {\n\t\t\tvar attr;\n\t\t\t// IE6 and 7 will map elem.type to 'text' for new HTML5 types (search, etc)\n\t\t\t// use getAttribute instead to test this case\n\t\t\treturn elem.nodeName.toLowerCase() === \"input\" &&\n\t\t\t\telem.type === \"text\" &&\n\t\t\t\t( (attr = elem.getAttribute(\"type\")) == null || attr.toLowerCase() === elem.type );\n\t\t},\n\n\t\t// Position-in-collection\n\t\t\"first\": createPositionalPseudo(function() {\n\t\t\treturn [ 0 ];\n\t\t}),\n\n\t\t\"last\": createPositionalPseudo(function( matchIndexes, length ) {\n\t\t\treturn [ length - 1 ];\n\t\t}),\n\n\t\t\"eq\": createPositionalPseudo(function( matchIndexes, length, argument ) {\n\t\t\treturn [ argument < 0 ? argument + length : argument ];\n\t\t}),\n\n\t\t\"even\": createPositionalPseudo(function( matchIndexes, length ) {\n\t\t\tvar i = 0;\n\t\t\tfor ( ; i < length; i += 2 ) {\n\t\t\t\tmatchIndexes.push( i );\n\t\t\t}\n\t\t\treturn matchIndexes;\n\t\t}),\n\n\t\t\"odd\": createPositionalPseudo(function( matchIndexes, length ) {\n\t\t\tvar i = 1;\n\t\t\tfor ( ; i < length; i += 2 ) {\n\t\t\t\tmatchIndexes.push( i );\n\t\t\t}\n\t\t\treturn matchIndexes;\n\t\t}),\n\n\t\t\"lt\": createPositionalPseudo(function( matchIndexes, length, argument ) {\n\t\t\tvar i = argument < 0 ? argument + length : argument;\n\t\t\tfor ( ; --i >= 0; ) {\n\t\t\t\tmatchIndexes.push( i );\n\t\t\t}\n\t\t\treturn matchIndexes;\n\t\t}),\n\n\t\t\"gt\": createPositionalPseudo(function( matchIndexes, length, argument ) {\n\t\t\tvar i = argument < 0 ? argument + length : argument;\n\t\t\tfor ( ; ++i < length; ) {\n\t\t\t\tmatchIndexes.push( i );\n\t\t\t}\n\t\t\treturn matchIndexes;\n\t\t})\n\t}\n};\n\n// Add button/input type pseudos\nfor ( i in { radio: true, checkbox: true, file: true, password: true, image: true } ) {\n\tExpr.pseudos[ i ] = createInputPseudo( i );\n}\nfor ( i in { submit: true, reset: true } ) {\n\tExpr.pseudos[ i ] = createButtonPseudo( i );\n}\n\nfunction tokenize( selector, parseOnly ) {\n\tvar matched, match, tokens, type,\n\t\tsoFar, groups, preFilters,\n\t\tcached = tokenCache[ selector + \" \" ];\n\n\tif ( cached ) {\n\t\treturn parseOnly ? 0 : cached.slice( 0 );\n\t}\n\n\tsoFar = selector;\n\tgroups = [];\n\tpreFilters = Expr.preFilter;\n\n\twhile ( soFar ) {\n\n\t\t// Comma and first run\n\t\tif ( !matched || (match = rcomma.exec( soFar )) ) {\n\t\t\tif ( match ) {\n\t\t\t\t// Don't consume trailing commas as valid\n\t\t\t\tsoFar = soFar.slice( match[0].length ) || soFar;\n\t\t\t}\n\t\t\tgroups.push( tokens = [] );\n\t\t}\n\n\t\tmatched = false;\n\n\t\t// Combinators\n\t\tif ( (match = rcombinators.exec( soFar )) ) {\n\t\t\tmatched = match.shift();\n\t\t\ttokens.push( {\n\t\t\t\tvalue: matched,\n\t\t\t\t// Cast descendant combinators to space\n\t\t\t\ttype: match[0].replace( rtrim, \" \" )\n\t\t\t} );\n\t\t\tsoFar = soFar.slice( matched.length );\n\t\t}\n\n\t\t// Filters\n\t\tfor ( type in Expr.filter ) {\n\t\t\tif ( (match = matchExpr[ type ].exec( soFar )) && (!preFilters[ type ] ||\n\t\t\t\t(match = preFilters[ type ]( match ))) ) {\n\t\t\t\tmatched = match.shift();\n\t\t\t\ttokens.push( {\n\t\t\t\t\tvalue: matched,\n\t\t\t\t\ttype: type,\n\t\t\t\t\tmatches: match\n\t\t\t\t} );\n\t\t\t\tsoFar = soFar.slice( matched.length );\n\t\t\t}\n\t\t}\n\n\t\tif ( !matched ) {\n\t\t\tbreak;\n\t\t}\n\t}\n\n\t// Return the length of the invalid excess\n\t// if we're just parsing\n\t// Otherwise, throw an error or return tokens\n\treturn parseOnly ?\n\t\tsoFar.length :\n\t\tsoFar ?\n\t\t\tSizzle.error( selector ) :\n\t\t\t// Cache the tokens\n\t\t\ttokenCache( selector, groups ).slice( 0 );\n}\n\nfunction toSelector( tokens ) {\n\tvar i = 0,\n\t\tlen = tokens.length,\n\t\tselector = \"\";\n\tfor ( ; i < len; i++ ) {\n\t\tselector += tokens[i].value;\n\t}\n\treturn selector;\n}\n\nfunction addCombinator( matcher, combinator, base ) {\n\tvar dir = combinator.dir,\n\t\tcheckNonElements = base && dir === \"parentNode\",\n\t\tdoneName = done++;\n\n\treturn combinator.first ?\n\t\t// Check against closest ancestor/preceding element\n\t\tfunction( elem, context, xml ) {\n\t\t\twhile ( (elem = elem[ dir ]) ) {\n\t\t\t\tif ( elem.nodeType === 1 || checkNonElements ) {\n\t\t\t\t\treturn matcher( elem, context, xml );\n\t\t\t\t}\n\t\t\t}\n\t\t} :\n\n\t\t// Check against all ancestor/preceding elements\n\t\tfunction( elem, context, xml ) {\n\t\t\tvar data, cache, outerCache,\n\t\t\t\tdirkey = dirruns + \" \" + doneName;\n\n\t\t\t// We can't set arbitrary data on XML nodes, so they don't benefit from dir caching\n\t\t\tif ( xml ) {\n\t\t\t\twhile ( (elem = elem[ dir ]) ) {\n\t\t\t\t\tif ( elem.nodeType === 1 || checkNonElements ) {\n\t\t\t\t\t\tif ( matcher( elem, context, xml ) ) {\n\t\t\t\t\t\t\treturn true;\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\twhile ( (elem = elem[ dir ]) ) {\n\t\t\t\t\tif ( elem.nodeType === 1 || checkNonElements ) {\n\t\t\t\t\t\touterCache = elem[ expando ] || (elem[ expando ] = {});\n\t\t\t\t\t\tif ( (cache = outerCache[ dir ]) && cache[0] === dirkey ) {\n\t\t\t\t\t\t\tif ( (data = cache[1]) === true || data === cachedruns ) {\n\t\t\t\t\t\t\t\treturn data === true;\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\tcache = outerCache[ dir ] = [ dirkey ];\n\t\t\t\t\t\t\tcache[1] = matcher( elem, context, xml ) || cachedruns;\n\t\t\t\t\t\t\tif ( cache[1] === true ) {\n\t\t\t\t\t\t\t\treturn true;\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t};\n}\n\nfunction elementMatcher( matchers ) {\n\treturn matchers.length > 1 ?\n\t\tfunction( elem, context, xml ) {\n\t\t\tvar i = matchers.length;\n\t\t\twhile ( i-- ) {\n\t\t\t\tif ( !matchers[i]( elem, context, xml ) ) {\n\t\t\t\t\treturn false;\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn true;\n\t\t} :\n\t\tmatchers[0];\n}\n\nfunction condense( unmatched, map, filter, context, xml ) {\n\tvar elem,\n\t\tnewUnmatched = [],\n\t\ti = 0,\n\t\tlen = unmatched.length,\n\t\tmapped = map != null;\n\n\tfor ( ; i < len; i++ ) {\n\t\tif ( (elem = unmatched[i]) ) {\n\t\t\tif ( !filter || filter( elem, context, xml ) ) {\n\t\t\t\tnewUnmatched.push( elem );\n\t\t\t\tif ( mapped ) {\n\t\t\t\t\tmap.push( i );\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\treturn newUnmatched;\n}\n\nfunction setMatcher( preFilter, selector, matcher, postFilter, postFinder, postSelector ) {\n\tif ( postFilter && !postFilter[ expando ] ) {\n\t\tpostFilter = setMatcher( postFilter );\n\t}\n\tif ( postFinder && !postFinder[ expando ] ) {\n\t\tpostFinder = setMatcher( postFinder, postSelector );\n\t}\n\treturn markFunction(function( seed, results, context, xml ) {\n\t\tvar temp, i, elem,\n\t\t\tpreMap = [],\n\t\t\tpostMap = [],\n\t\t\tpreexisting = results.length,\n\n\t\t\t// Get initial elements from seed or context\n\t\t\telems = seed || multipleContexts( selector || \"*\", context.nodeType ? [ context ] : context, [] ),\n\n\t\t\t// Prefilter to get matcher input, preserving a map for seed-results synchronization\n\t\t\tmatcherIn = preFilter && ( seed || !selector ) ?\n\t\t\t\tcondense( elems, preMap, preFilter, context, xml ) :\n\t\t\t\telems,\n\n\t\t\tmatcherOut = matcher ?\n\t\t\t\t// If we have a postFinder, or filtered seed, or non-seed postFilter or preexisting results,\n\t\t\t\tpostFinder || ( seed ? preFilter : preexisting || postFilter ) ?\n\n\t\t\t\t\t// ...intermediate processing is necessary\n\t\t\t\t\t[] :\n\n\t\t\t\t\t// ...otherwise use results directly\n\t\t\t\t\tresults :\n\t\t\t\tmatcherIn;\n\n\t\t// Find primary matches\n\t\tif ( matcher ) {\n\t\t\tmatcher( matcherIn, matcherOut, context, xml );\n\t\t}\n\n\t\t// Apply postFilter\n\t\tif ( postFilter ) {\n\t\t\ttemp = condense( matcherOut, postMap );\n\t\t\tpostFilter( temp, [], context, xml );\n\n\t\t\t// Un-match failing elements by moving them back to matcherIn\n\t\t\ti = temp.length;\n\t\t\twhile ( i-- ) {\n\t\t\t\tif ( (elem = temp[i]) ) {\n\t\t\t\t\tmatcherOut[ postMap[i] ] = !(matcherIn[ postMap[i] ] = elem);\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tif ( seed ) {\n\t\t\tif ( postFinder || preFilter ) {\n\t\t\t\tif ( postFinder ) {\n\t\t\t\t\t// Get the final matcherOut by condensing this intermediate into postFinder contexts\n\t\t\t\t\ttemp = [];\n\t\t\t\t\ti = matcherOut.length;\n\t\t\t\t\twhile ( i-- ) {\n\t\t\t\t\t\tif ( (elem = matcherOut[i]) ) {\n\t\t\t\t\t\t\t// Restore matcherIn since elem is not yet a final match\n\t\t\t\t\t\t\ttemp.push( (matcherIn[i] = elem) );\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tpostFinder( null, (matcherOut = []), temp, xml );\n\t\t\t\t}\n\n\t\t\t\t// Move matched elements from seed to results to keep them synchronized\n\t\t\t\ti = matcherOut.length;\n\t\t\t\twhile ( i-- ) {\n\t\t\t\t\tif ( (elem = matcherOut[i]) &&\n\t\t\t\t\t\t(temp = postFinder ? indexOf.call( seed, elem ) : preMap[i]) > -1 ) {\n\n\t\t\t\t\t\tseed[temp] = !(results[temp] = elem);\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\n\t\t// Add elements to results, through postFinder if defined\n\t\t} else {\n\t\t\tmatcherOut = condense(\n\t\t\t\tmatcherOut === results ?\n\t\t\t\t\tmatcherOut.splice( preexisting, matcherOut.length ) :\n\t\t\t\t\tmatcherOut\n\t\t\t);\n\t\t\tif ( postFinder ) {\n\t\t\t\tpostFinder( null, results, matcherOut, xml );\n\t\t\t} else {\n\t\t\t\tpush.apply( results, matcherOut );\n\t\t\t}\n\t\t}\n\t});\n}\n\nfunction matcherFromTokens( tokens ) {\n\tvar checkContext, matcher, j,\n\t\tlen = tokens.length,\n\t\tleadingRelative = Expr.relative[ tokens[0].type ],\n\t\timplicitRelative = leadingRelative || Expr.relative[\" \"],\n\t\ti = leadingRelative ? 1 : 0,\n\n\t\t// The foundational matcher ensures that elements are reachable from top-level context(s)\n\t\tmatchContext = addCombinator( function( elem ) {\n\t\t\treturn elem === checkContext;\n\t\t}, implicitRelative, true ),\n\t\tmatchAnyContext = addCombinator( function( elem ) {\n\t\t\treturn indexOf.call( checkContext, elem ) > -1;\n\t\t}, implicitRelative, true ),\n\t\tmatchers = [ function( elem, context, xml ) {\n\t\t\treturn ( !leadingRelative && ( xml || context !== outermostContext ) ) || (\n\t\t\t\t(checkContext = context).nodeType ?\n\t\t\t\t\tmatchContext( elem, context, xml ) :\n\t\t\t\t\tmatchAnyContext( elem, context, xml ) );\n\t\t} ];\n\n\tfor ( ; i < len; i++ ) {\n\t\tif ( (matcher = Expr.relative[ tokens[i].type ]) ) {\n\t\t\tmatchers = [ addCombinator(elementMatcher( matchers ), matcher) ];\n\t\t} else {\n\t\t\tmatcher = Expr.filter[ tokens[i].type ].apply( null, tokens[i].matches );\n\n\t\t\t// Return special upon seeing a positional matcher\n\t\t\tif ( matcher[ expando ] ) {\n\t\t\t\t// Find the next relative operator (if any) for proper handling\n\t\t\t\tj = ++i;\n\t\t\t\tfor ( ; j < len; j++ ) {\n\t\t\t\t\tif ( Expr.relative[ tokens[j].type ] ) {\n\t\t\t\t\t\tbreak;\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\treturn setMatcher(\n\t\t\t\t\ti > 1 && elementMatcher( matchers ),\n\t\t\t\t\ti > 1 && toSelector( tokens.slice( 0, i - 1 ) ).replace( rtrim, \"$1\" ),\n\t\t\t\t\tmatcher,\n\t\t\t\t\ti < j && matcherFromTokens( tokens.slice( i, j ) ),\n\t\t\t\t\tj < len && matcherFromTokens( (tokens = tokens.slice( j )) ),\n\t\t\t\t\tj < len && toSelector( tokens )\n\t\t\t\t);\n\t\t\t}\n\t\t\tmatchers.push( matcher );\n\t\t}\n\t}\n\n\treturn elementMatcher( matchers );\n}\n\nfunction matcherFromGroupMatchers( elementMatchers, setMatchers ) {\n\t// A counter to specify which element is currently being matched\n\tvar matcherCachedRuns = 0,\n\t\tbySet = setMatchers.length > 0,\n\t\tbyElement = elementMatchers.length > 0,\n\t\tsuperMatcher = function( seed, context, xml, results, expandContext ) {\n\t\t\tvar elem, j, matcher,\n\t\t\t\tsetMatched = [],\n\t\t\t\tmatchedCount = 0,\n\t\t\t\ti = \"0\",\n\t\t\t\tunmatched = seed && [],\n\t\t\t\toutermost = expandContext != null,\n\t\t\t\tcontextBackup = outermostContext,\n\t\t\t\t// We must always have either seed elements or context\n\t\t\t\telems = seed || byElement && Expr.find[\"TAG\"]( \"*\", expandContext && context.parentNode || context ),\n\t\t\t\t// Use integer dirruns iff this is the outermost matcher\n\t\t\t\tdirrunsUnique = (dirruns += contextBackup == null ? 1 : Math.random() || 0.1);\n\n\t\t\tif ( outermost ) {\n\t\t\t\toutermostContext = context !== document && context;\n\t\t\t\tcachedruns = matcherCachedRuns;\n\t\t\t}\n\n\t\t\t// Add elements passing elementMatchers directly to results\n\t\t\t// Keep `i` a string if there are no elements so `matchedCount` will be \"00\" below\n\t\t\tfor ( ; (elem = elems[i]) != null; i++ ) {\n\t\t\t\tif ( byElement && elem ) {\n\t\t\t\t\tj = 0;\n\t\t\t\t\twhile ( (matcher = elementMatchers[j++]) ) {\n\t\t\t\t\t\tif ( matcher( elem, context, xml ) ) {\n\t\t\t\t\t\t\tresults.push( elem );\n\t\t\t\t\t\t\tbreak;\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tif ( outermost ) {\n\t\t\t\t\t\tdirruns = dirrunsUnique;\n\t\t\t\t\t\tcachedruns = ++matcherCachedRuns;\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\t// Track unmatched elements for set filters\n\t\t\t\tif ( bySet ) {\n\t\t\t\t\t// They will have gone through all possible matchers\n\t\t\t\t\tif ( (elem = !matcher && elem) ) {\n\t\t\t\t\t\tmatchedCount--;\n\t\t\t\t\t}\n\n\t\t\t\t\t// Lengthen the array for every element, matched or not\n\t\t\t\t\tif ( seed ) {\n\t\t\t\t\t\tunmatched.push( elem );\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Apply set filters to unmatched elements\n\t\t\tmatchedCount += i;\n\t\t\tif ( bySet && i !== matchedCount ) {\n\t\t\t\tj = 0;\n\t\t\t\twhile ( (matcher = setMatchers[j++]) ) {\n\t\t\t\t\tmatcher( unmatched, setMatched, context, xml );\n\t\t\t\t}\n\n\t\t\t\tif ( seed ) {\n\t\t\t\t\t// Reintegrate element matches to eliminate the need for sorting\n\t\t\t\t\tif ( matchedCount > 0 ) {\n\t\t\t\t\t\twhile ( i-- ) {\n\t\t\t\t\t\t\tif ( !(unmatched[i] || setMatched[i]) ) {\n\t\t\t\t\t\t\t\tsetMatched[i] = pop.call( results );\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\n\t\t\t\t\t// Discard index placeholder values to get only actual matches\n\t\t\t\t\tsetMatched = condense( setMatched );\n\t\t\t\t}\n\n\t\t\t\t// Add matches to results\n\t\t\t\tpush.apply( results, setMatched );\n\n\t\t\t\t// Seedless set matches succeeding multiple successful matchers stipulate sorting\n\t\t\t\tif ( outermost && !seed && setMatched.length > 0 &&\n\t\t\t\t\t( matchedCount + setMatchers.length ) > 1 ) {\n\n\t\t\t\t\tSizzle.uniqueSort( results );\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Override manipulation of globals by nested matchers\n\t\t\tif ( outermost ) {\n\t\t\t\tdirruns = dirrunsUnique;\n\t\t\t\toutermostContext = contextBackup;\n\t\t\t}\n\n\t\t\treturn unmatched;\n\t\t};\n\n\treturn bySet ?\n\t\tmarkFunction( superMatcher ) :\n\t\tsuperMatcher;\n}\n\ncompile = Sizzle.compile = function( selector, group /* Internal Use Only */ ) {\n\tvar i,\n\t\tsetMatchers = [],\n\t\telementMatchers = [],\n\t\tcached = compilerCache[ selector + \" \" ];\n\n\tif ( !cached ) {\n\t\t// Generate a function of recursive functions that can be used to check each element\n\t\tif ( !group ) {\n\t\t\tgroup = tokenize( selector );\n\t\t}\n\t\ti = group.length;\n\t\twhile ( i-- ) {\n\t\t\tcached = matcherFromTokens( group[i] );\n\t\t\tif ( cached[ expando ] ) {\n\t\t\t\tsetMatchers.push( cached );\n\t\t\t} else {\n\t\t\t\telementMatchers.push( cached );\n\t\t\t}\n\t\t}\n\n\t\t// Cache the compiled function\n\t\tcached = compilerCache( selector, matcherFromGroupMatchers( elementMatchers, setMatchers ) );\n\t}\n\treturn cached;\n};\n\nfunction multipleContexts( selector, contexts, results ) {\n\tvar i = 0,\n\t\tlen = contexts.length;\n\tfor ( ; i < len; i++ ) {\n\t\tSizzle( selector, contexts[i], results );\n\t}\n\treturn results;\n}\n\nfunction select( selector, context, results, seed ) {\n\tvar i, tokens, token, type, find,\n\t\tmatch = tokenize( selector );\n\n\tif ( !seed ) {\n\t\t// Try to minimize operations if there is only one group\n\t\tif ( match.length === 1 ) {\n\n\t\t\t// Take a shortcut and set the context if the root selector is an ID\n\t\t\ttokens = match[0] = match[0].slice( 0 );\n\t\t\tif ( tokens.length > 2 && (token = tokens[0]).type === \"ID\" &&\n\t\t\t\t\tcontext.nodeType === 9 && documentIsHTML &&\n\t\t\t\t\tExpr.relative[ tokens[1].type ] ) {\n\n\t\t\t\tcontext = ( Expr.find[\"ID\"]( token.matches[0].replace(runescape, funescape), context ) || [] )[0];\n\t\t\t\tif ( !context ) {\n\t\t\t\t\treturn results;\n\t\t\t\t}\n\n\t\t\t\tselector = selector.slice( tokens.shift().value.length );\n\t\t\t}\n\n\t\t\t// Fetch a seed set for right-to-left matching\n\t\t\ti = matchExpr[\"needsContext\"].test( selector ) ? 0 : tokens.length;\n\t\t\twhile ( i-- ) {\n\t\t\t\ttoken = tokens[i];\n\n\t\t\t\t// Abort if we hit a combinator\n\t\t\t\tif ( Expr.relative[ (type = token.type) ] ) {\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t\tif ( (find = Expr.find[ type ]) ) {\n\t\t\t\t\t// Search, expanding context for leading sibling combinators\n\t\t\t\t\tif ( (seed = find(\n\t\t\t\t\t\ttoken.matches[0].replace( runescape, funescape ),\n\t\t\t\t\t\trsibling.test( tokens[0].type ) && context.parentNode || context\n\t\t\t\t\t)) ) {\n\n\t\t\t\t\t\t// If seed is empty or no tokens remain, we can return early\n\t\t\t\t\t\ttokens.splice( i, 1 );\n\t\t\t\t\t\tselector = seed.length && toSelector( tokens );\n\t\t\t\t\t\tif ( !selector ) {\n\t\t\t\t\t\t\tpush.apply( results, seed );\n\t\t\t\t\t\t\treturn results;\n\t\t\t\t\t\t}\n\n\t\t\t\t\t\tbreak;\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\t// Compile and execute a filtering function\n\t// Provide `match` to avoid retokenization if we modified the selector above\n\tcompile( selector, match )(\n\t\tseed,\n\t\tcontext,\n\t\t!documentIsHTML,\n\t\tresults,\n\t\trsibling.test( selector )\n\t);\n\treturn results;\n}\n\n// Deprecated\nExpr.pseudos[\"nth\"] = Expr.pseudos[\"eq\"];\n\n// Easy API for creating new setFilters\nfunction setFilters() {}\nsetFilters.prototype = Expr.filters = Expr.pseudos;\nExpr.setFilters = new setFilters();\n\n// One-time assignments\n\n// Sort stability\nsupport.sortStable = expando.split(\"\").sort( sortOrder ).join(\"\") === expando;\n\n// Initialize against the default document\nsetDocument();\n\n// Support: Chrome<<14\n// Always assume duplicates if they aren't passed to the comparison function\n[0, 0].sort( sortOrder );\nsupport.detectDuplicates = hasDuplicate;\n\n// Support: IE<8\n// Prevent attribute/property \"interpolation\"\nassert(function( div ) {\n\tdiv.innerHTML = \"<a href='#'></a>\";\n\tif ( div.firstChild.getAttribute(\"href\") !== \"#\" ) {\n\t\tvar attrs = \"type|href|height|width\".split(\"|\"),\n\t\t\ti = attrs.length;\n\t\twhile ( i-- ) {\n\t\t\tExpr.attrHandle[ attrs[i] ] = interpolationHandler;\n\t\t}\n\t}\n});\n\n// Support: IE<9\n// Use getAttributeNode to fetch booleans when getAttribute lies\nassert(function( div ) {\n\tif ( div.getAttribute(\"disabled\") != null ) {\n\t\tvar attrs = booleans.split(\"|\"),\n\t\t\ti = attrs.length;\n\t\twhile ( i-- ) {\n\t\t\tExpr.attrHandle[ attrs[i] ] = boolHandler;\n\t\t}\n\t}\n});\n\njQuery.find = Sizzle;\njQuery.expr = Sizzle.selectors;\njQuery.expr[\":\"] = jQuery.expr.pseudos;\njQuery.unique = Sizzle.uniqueSort;\njQuery.text = Sizzle.getText;\njQuery.isXMLDoc = Sizzle.isXML;\njQuery.contains = Sizzle.contains;\n\n\n})( window );\n// String to Object options format cache\nvar optionsCache = {};\n\n// Convert String-formatted options into Object-formatted ones and store in cache\nfunction createOptions( options ) {\n\tvar object = optionsCache[ options ] = {};\n\tjQuery.each( options.match( core_rnotwhite ) || [], function( _, flag ) {\n\t\tobject[ flag ] = true;\n\t});\n\treturn object;\n}\n\n/*\n * Create a callback list using the following parameters:\n *\n *\toptions: an optional list of space-separated options that will change how\n *\t\t\tthe callback list behaves or a more traditional option object\n *\n * By default a callback list will act like an event callback list and can be\n * \"fired\" multiple times.\n *\n * Possible options:\n *\n *\tonce:\t\t\twill ensure the callback list can only be fired once (like a Deferred)\n *\n *\tmemory:\t\t\twill keep track of previous values and will call any callback added\n *\t\t\t\t\tafter the list has been fired right away with the latest \"memorized\"\n *\t\t\t\t\tvalues (like a Deferred)\n *\n *\tunique:\t\t\twill ensure a callback can only be added once (no duplicate in the list)\n *\n *\tstopOnFalse:\tinterrupt callings when a callback returns false\n *\n */\njQuery.Callbacks = function( options ) {\n\n\t// Convert options from String-formatted to Object-formatted if needed\n\t// (we check in cache first)\n\toptions = typeof options === \"string\" ?\n\t\t( optionsCache[ options ] || createOptions( options ) ) :\n\t\tjQuery.extend( {}, options );\n\n\tvar // Last fire value (for non-forgettable lists)\n\t\tmemory,\n\t\t// Flag to know if list was already fired\n\t\tfired,\n\t\t// Flag to know if list is currently firing\n\t\tfiring,\n\t\t// First callback to fire (used internally by add and fireWith)\n\t\tfiringStart,\n\t\t// End of the loop when firing\n\t\tfiringLength,\n\t\t// Index of currently firing callback (modified by remove if needed)\n\t\tfiringIndex,\n\t\t// Actual callback list\n\t\tlist = [],\n\t\t// Stack of fire calls for repeatable lists\n\t\tstack = !options.once && [],\n\t\t// Fire callbacks\n\t\tfire = function( data ) {\n\t\t\tmemory = options.memory && data;\n\t\t\tfired = true;\n\t\t\tfiringIndex = firingStart || 0;\n\t\t\tfiringStart = 0;\n\t\t\tfiringLength = list.length;\n\t\t\tfiring = true;\n\t\t\tfor ( ; list && firingIndex < firingLength; firingIndex++ ) {\n\t\t\t\tif ( list[ firingIndex ].apply( data[ 0 ], data[ 1 ] ) === false && options.stopOnFalse ) {\n\t\t\t\t\tmemory = false; // To prevent further calls using add\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t}\n\t\t\tfiring = false;\n\t\t\tif ( list ) {\n\t\t\t\tif ( stack ) {\n\t\t\t\t\tif ( stack.length ) {\n\t\t\t\t\t\tfire( stack.shift() );\n\t\t\t\t\t}\n\t\t\t\t} else if ( memory ) {\n\t\t\t\t\tlist = [];\n\t\t\t\t} else {\n\t\t\t\t\tself.disable();\n\t\t\t\t}\n\t\t\t}\n\t\t},\n\t\t// Actual Callbacks object\n\t\tself = {\n\t\t\t// Add a callback or a collection of callbacks to the list\n\t\t\tadd: function() {\n\t\t\t\tif ( list ) {\n\t\t\t\t\t// First, we save the current length\n\t\t\t\t\tvar start = list.length;\n\t\t\t\t\t(function add( args ) {\n\t\t\t\t\t\tjQuery.each( args, function( _, arg ) {\n\t\t\t\t\t\t\tvar type = jQuery.type( arg );\n\t\t\t\t\t\t\tif ( type === \"function\" ) {\n\t\t\t\t\t\t\t\tif ( !options.unique || !self.has( arg ) ) {\n\t\t\t\t\t\t\t\t\tlist.push( arg );\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t} else if ( arg && arg.length && type !== \"string\" ) {\n\t\t\t\t\t\t\t\t// Inspect recursively\n\t\t\t\t\t\t\t\tadd( arg );\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t});\n\t\t\t\t\t})( arguments );\n\t\t\t\t\t// Do we need to add the callbacks to the\n\t\t\t\t\t// current firing batch?\n\t\t\t\t\tif ( firing ) {\n\t\t\t\t\t\tfiringLength = list.length;\n\t\t\t\t\t// With memory, if we're not firing then\n\t\t\t\t\t// we should call right away\n\t\t\t\t\t} else if ( memory ) {\n\t\t\t\t\t\tfiringStart = start;\n\t\t\t\t\t\tfire( memory );\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\treturn this;\n\t\t\t},\n\t\t\t// Remove a callback from the list\n\t\t\tremove: function() {\n\t\t\t\tif ( list ) {\n\t\t\t\t\tjQuery.each( arguments, function( _, arg ) {\n\t\t\t\t\t\tvar index;\n\t\t\t\t\t\twhile( ( index = jQuery.inArray( arg, list, index ) ) > -1 ) {\n\t\t\t\t\t\t\tlist.splice( index, 1 );\n\t\t\t\t\t\t\t// Handle firing indexes\n\t\t\t\t\t\t\tif ( firing ) {\n\t\t\t\t\t\t\t\tif ( index <= firingLength ) {\n\t\t\t\t\t\t\t\t\tfiringLength--;\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\tif ( index <= firingIndex ) {\n\t\t\t\t\t\t\t\t\tfiringIndex--;\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t});\n\t\t\t\t}\n\t\t\t\treturn this;\n\t\t\t},\n\t\t\t// Check if a given callback is in the list.\n\t\t\t// If no argument is given, return whether or not list has callbacks attached.\n\t\t\thas: function( fn ) {\n\t\t\t\treturn fn ? jQuery.inArray( fn, list ) > -1 : !!( list && list.length );\n\t\t\t},\n\t\t\t// Remove all callbacks from the list\n\t\t\tempty: function() {\n\t\t\t\tlist = [];\n\t\t\t\tfiringLength = 0;\n\t\t\t\treturn this;\n\t\t\t},\n\t\t\t// Have the list do nothing anymore\n\t\t\tdisable: function() {\n\t\t\t\tlist = stack = memory = undefined;\n\t\t\t\treturn this;\n\t\t\t},\n\t\t\t// Is it disabled?\n\t\t\tdisabled: function() {\n\t\t\t\treturn !list;\n\t\t\t},\n\t\t\t// Lock the list in its current state\n\t\t\tlock: function() {\n\t\t\t\tstack = undefined;\n\t\t\t\tif ( !memory ) {\n\t\t\t\t\tself.disable();\n\t\t\t\t}\n\t\t\t\treturn this;\n\t\t\t},\n\t\t\t// Is it locked?\n\t\t\tlocked: function() {\n\t\t\t\treturn !stack;\n\t\t\t},\n\t\t\t// Call all callbacks with the given context and arguments\n\t\t\tfireWith: function( context, args ) {\n\t\t\t\targs = args || [];\n\t\t\t\targs = [ context, args.slice ? args.slice() : args ];\n\t\t\t\tif ( list && ( !fired || stack ) ) {\n\t\t\t\t\tif ( firing ) {\n\t\t\t\t\t\tstack.push( args );\n\t\t\t\t\t} else {\n\t\t\t\t\t\tfire( args );\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\treturn this;\n\t\t\t},\n\t\t\t// Call all the callbacks with the given arguments\n\t\t\tfire: function() {\n\t\t\t\tself.fireWith( this, arguments );\n\t\t\t\treturn this;\n\t\t\t},\n\t\t\t// To know if the callbacks have already been called at least once\n\t\t\tfired: function() {\n\t\t\t\treturn !!fired;\n\t\t\t}\n\t\t};\n\n\treturn self;\n};\njQuery.extend({\n\n\tDeferred: function( func ) {\n\t\tvar tuples = [\n\t\t\t\t// action, add listener, listener list, final state\n\t\t\t\t[ \"resolve\", \"done\", jQuery.Callbacks(\"once memory\"), \"resolved\" ],\n\t\t\t\t[ \"reject\", \"fail\", jQuery.Callbacks(\"once memory\"), \"rejected\" ],\n\t\t\t\t[ \"notify\", \"progress\", jQuery.Callbacks(\"memory\") ]\n\t\t\t],\n\t\t\tstate = \"pending\",\n\t\t\tpromise = {\n\t\t\t\tstate: function() {\n\t\t\t\t\treturn state;\n\t\t\t\t},\n\t\t\t\talways: function() {\n\t\t\t\t\tdeferred.done( arguments ).fail( arguments );\n\t\t\t\t\treturn this;\n\t\t\t\t},\n\t\t\t\tthen: function( /* fnDone, fnFail, fnProgress */ ) {\n\t\t\t\t\tvar fns = arguments;\n\t\t\t\t\treturn jQuery.Deferred(function( newDefer ) {\n\t\t\t\t\t\tjQuery.each( tuples, function( i, tuple ) {\n\t\t\t\t\t\t\tvar action = tuple[ 0 ],\n\t\t\t\t\t\t\t\tfn = jQuery.isFunction( fns[ i ] ) && fns[ i ];\n\t\t\t\t\t\t\t// deferred[ done | fail | progress ] for forwarding actions to newDefer\n\t\t\t\t\t\t\tdeferred[ tuple[1] ](function() {\n\t\t\t\t\t\t\t\tvar returned = fn && fn.apply( this, arguments );\n\t\t\t\t\t\t\t\tif ( returned && jQuery.isFunction( returned.promise ) ) {\n\t\t\t\t\t\t\t\t\treturned.promise()\n\t\t\t\t\t\t\t\t\t\t.done( newDefer.resolve )\n\t\t\t\t\t\t\t\t\t\t.fail( newDefer.reject )\n\t\t\t\t\t\t\t\t\t\t.progress( newDefer.notify );\n\t\t\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\t\t\tnewDefer[ action + \"With\" ]( this === promise ? newDefer.promise() : this, fn ? [ returned ] : arguments );\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t});\n\t\t\t\t\t\t});\n\t\t\t\t\t\tfns = null;\n\t\t\t\t\t}).promise();\n\t\t\t\t},\n\t\t\t\t// Get a promise for this deferred\n\t\t\t\t// If obj is provided, the promise aspect is added to the object\n\t\t\t\tpromise: function( obj ) {\n\t\t\t\t\treturn obj != null ? jQuery.extend( obj, promise ) : promise;\n\t\t\t\t}\n\t\t\t},\n\t\t\tdeferred = {};\n\n\t\t// Keep pipe for back-compat\n\t\tpromise.pipe = promise.then;\n\n\t\t// Add list-specific methods\n\t\tjQuery.each( tuples, function( i, tuple ) {\n\t\t\tvar list = tuple[ 2 ],\n\t\t\t\tstateString = tuple[ 3 ];\n\n\t\t\t// promise[ done | fail | progress ] = list.add\n\t\t\tpromise[ tuple[1] ] = list.add;\n\n\t\t\t// Handle state\n\t\t\tif ( stateString ) {\n\t\t\t\tlist.add(function() {\n\t\t\t\t\t// state = [ resolved | rejected ]\n\t\t\t\t\tstate = stateString;\n\n\t\t\t\t// [ reject_list | resolve_list ].disable; progress_list.lock\n\t\t\t\t}, tuples[ i ^ 1 ][ 2 ].disable, tuples[ 2 ][ 2 ].lock );\n\t\t\t}\n\n\t\t\t// deferred[ resolve | reject | notify ]\n\t\t\tdeferred[ tuple[0] ] = function() {\n\t\t\t\tdeferred[ tuple[0] + \"With\" ]( this === deferred ? promise : this, arguments );\n\t\t\t\treturn this;\n\t\t\t};\n\t\t\tdeferred[ tuple[0] + \"With\" ] = list.fireWith;\n\t\t});\n\n\t\t// Make the deferred a promise\n\t\tpromise.promise( deferred );\n\n\t\t// Call given func if any\n\t\tif ( func ) {\n\t\t\tfunc.call( deferred, deferred );\n\t\t}\n\n\t\t// All done!\n\t\treturn deferred;\n\t},\n\n\t// Deferred helper\n\twhen: function( subordinate /* , ..., subordinateN */ ) {\n\t\tvar i = 0,\n\t\t\tresolveValues = core_slice.call( arguments ),\n\t\t\tlength = resolveValues.length,\n\n\t\t\t// the count of uncompleted subordinates\n\t\t\tremaining = length !== 1 || ( subordinate && jQuery.isFunction( subordinate.promise ) ) ? length : 0,\n\n\t\t\t// the master Deferred. If resolveValues consist of only a single Deferred, just use that.\n\t\t\tdeferred = remaining === 1 ? subordinate : jQuery.Deferred(),\n\n\t\t\t// Update function for both resolve and progress values\n\t\t\tupdateFunc = function( i, contexts, values ) {\n\t\t\t\treturn function( value ) {\n\t\t\t\t\tcontexts[ i ] = this;\n\t\t\t\t\tvalues[ i ] = arguments.length > 1 ? core_slice.call( arguments ) : value;\n\t\t\t\t\tif( values === progressValues ) {\n\t\t\t\t\t\tdeferred.notifyWith( contexts, values );\n\t\t\t\t\t} else if ( !( --remaining ) ) {\n\t\t\t\t\t\tdeferred.resolveWith( contexts, values );\n\t\t\t\t\t}\n\t\t\t\t};\n\t\t\t},\n\n\t\t\tprogressValues, progressContexts, resolveContexts;\n\n\t\t// add listeners to Deferred subordinates; treat others as resolved\n\t\tif ( length > 1 ) {\n\t\t\tprogressValues = new Array( length );\n\t\t\tprogressContexts = new Array( length );\n\t\t\tresolveContexts = new Array( length );\n\t\t\tfor ( ; i < length; i++ ) {\n\t\t\t\tif ( resolveValues[ i ] && jQuery.isFunction( resolveValues[ i ].promise ) ) {\n\t\t\t\t\tresolveValues[ i ].promise()\n\t\t\t\t\t\t.done( updateFunc( i, resolveContexts, resolveValues ) )\n\t\t\t\t\t\t.fail( deferred.reject )\n\t\t\t\t\t\t.progress( updateFunc( i, progressContexts, progressValues ) );\n\t\t\t\t} else {\n\t\t\t\t\t--remaining;\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\t// if we're not waiting on anything, resolve the master\n\t\tif ( !remaining ) {\n\t\t\tdeferred.resolveWith( resolveContexts, resolveValues );\n\t\t}\n\n\t\treturn deferred.promise();\n\t}\n});\njQuery.support = (function( support ) {\n\tvar input = document.createElement(\"input\"),\n\t\tfragment = document.createDocumentFragment(),\n\t\tdiv = document.createElement(\"div\"),\n\t\tselect = document.createElement(\"select\"),\n\t\topt = select.appendChild( document.createElement(\"option\") );\n\n\t// Finish early in limited environments\n\tif ( !input.type ) {\n\t\treturn support;\n\t}\n\n\tinput.type = \"checkbox\";\n\n\t// Support: Safari 5.1, iOS 5.1, Android 4.x, Android 2.3\n\t// Check the default checkbox/radio value (\"\" on old WebKit; \"on\" elsewhere)\n\tsupport.checkOn = input.value !== \"\";\n\n\t// Must access the parent to make an option select properly\n\t// Support: IE9, IE10\n\tsupport.optSelected = opt.selected;\n\n\t// Will be defined later\n\tsupport.reliableMarginRight = true;\n\tsupport.boxSizingReliable = true;\n\tsupport.pixelPosition = false;\n\n\t// Make sure checked status is properly cloned\n\t// Support: IE9, IE10\n\tinput.checked = true;\n\tsupport.noCloneChecked = input.cloneNode( true ).checked;\n\n\t// Make sure that the options inside disabled selects aren't marked as disabled\n\t// (WebKit marks them as disabled)\n\tselect.disabled = true;\n\tsupport.optDisabled = !opt.disabled;\n\n\t// Check if an input maintains its value after becoming a radio\n\t// Support: IE9, IE10\n\tinput = document.createElement(\"input\");\n\tinput.value = \"t\";\n\tinput.type = \"radio\";\n\tsupport.radioValue = input.value === \"t\";\n\n\t// #11217 - WebKit loses check when the name is after the checked attribute\n\tinput.setAttribute( \"checked\", \"t\" );\n\tinput.setAttribute( \"name\", \"t\" );\n\n\tfragment.appendChild( input );\n\n\t// Support: Safari 5.1, Android 4.x, Android 2.3\n\t// old WebKit doesn't clone checked state correctly in fragments\n\tsupport.checkClone = fragment.cloneNode( true ).cloneNode( true ).lastChild.checked;\n\n\t// Support: Firefox, Chrome, Safari\n\t// Beware of CSP restrictions (https://developer.mozilla.org/en/Security/CSP)\n\tsupport.focusinBubbles = \"onfocusin\" in window;\n\n\tdiv.style.backgroundClip = \"content-box\";\n\tdiv.cloneNode( true ).style.backgroundClip = \"\";\n\tsupport.clearCloneStyle = div.style.backgroundClip === \"content-box\";\n\n\t// Run tests that need a body at doc ready\n\tjQuery(function() {\n\t\tvar container, marginDiv,\n\t\t\t// Support: Firefox, Android 2.3 (Prefixed box-sizing versions).\n\t\t\tdivReset = \"padding:0;margin:0;border:0;display:block;-webkit-box-sizing:content-box;-moz-box-sizing:content-box;box-sizing:content-box\",\n\t\t\tbody = document.getElementsByTagName(\"body\")[ 0 ];\n\n\t\tif ( !body ) {\n\t\t\t// Return for frameset docs that don't have a body\n\t\t\treturn;\n\t\t}\n\n\t\tcontainer = document.createElement(\"div\");\n\t\tcontainer.style.cssText = \"border:0;width:0;height:0;position:absolute;top:0;left:-9999px;margin-top:1px\";\n\n\t\t// Check box-sizing and margin behavior.\n\t\tbody.appendChild( container ).appendChild( div );\n\t\tdiv.innerHTML = \"\";\n\t\t// Support: Firefox, Android 2.3 (Prefixed box-sizing versions).\n\t\tdiv.style.cssText = \"-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box;padding:1px;border:1px;display:block;width:4px;margin-top:1%;position:absolute;top:1%\";\n\n\t\t// Workaround failing boxSizing test due to offsetWidth returning wrong value\n\t\t// with some non-1 values of body zoom, ticket #13543\n\t\tjQuery.swap( body, body.style.zoom != null ? { zoom: 1 } : {}, function() {\n\t\t\tsupport.boxSizing = div.offsetWidth === 4;\n\t\t});\n\n\t\t// Use window.getComputedStyle because jsdom on node.js will break without it.\n\t\tif ( window.getComputedStyle ) {\n\t\t\tsupport.pixelPosition = ( window.getComputedStyle( div, null ) || {} ).top !== \"1%\";\n\t\t\tsupport.boxSizingReliable = ( window.getComputedStyle( div, null ) || { width: \"4px\" } ).width === \"4px\";\n\n\t\t\t// Support: Android 2.3\n\t\t\t// Check if div with explicit width and no margin-right incorrectly\n\t\t\t// gets computed margin-right based on width of container. (#3333)\n\t\t\t// WebKit Bug 13343 - getComputedStyle returns wrong value for margin-right\n\t\t\tmarginDiv = div.appendChild( document.createElement(\"div\") );\n\t\t\tmarginDiv.style.cssText = div.style.cssText = divReset;\n\t\t\tmarginDiv.style.marginRight = marginDiv.style.width = \"0\";\n\t\t\tdiv.style.width = \"1px\";\n\n\t\t\tsupport.reliableMarginRight =\n\t\t\t\t!parseFloat( ( window.getComputedStyle( marginDiv, null ) || {} ).marginRight );\n\t\t}\n\n\t\tbody.removeChild( container );\n\t});\n\n\treturn support;\n})( {} );\n\n/*\n\tImplementation Summary\n\n\t1. Enforce API surface and semantic compatibility with 1.9.x branch\n\t2. Improve the module's maintainability by reducing the storage\n\t\tpaths to a single mechanism.\n\t3. Use the same single mechanism to support \"private\" and \"user\" data.\n\t4. _Never_ expose \"private\" data to user code (TODO: Drop _data, _removeData)\n\t5. Avoid exposing implementation details on user objects (eg. expando properties)\n\t6. Provide a clear path for implementation upgrade to WeakMap in 2014\n*/\nvar data_user, data_priv,\n\trbrace = /(?:\\{[\\s\\S]*\\}|\\[[\\s\\S]*\\])$/,\n\trmultiDash = /([A-Z])/g;\n\nfunction Data() {\n\t// Support: Android < 4,\n\t// Old WebKit does not have Object.preventExtensions/freeze method,\n\t// return new empty object instead with no [[set]] accessor\n\tObject.defineProperty( this.cache = {}, 0, {\n\t\tget: function() {\n\t\t\treturn {};\n\t\t}\n\t});\n\n\tthis.expando = jQuery.expando + Math.random();\n}\n\nData.uid = 1;\n\nData.accepts = function( owner ) {\n\t// Accepts only:\n\t// - Node\n\t// - Node.ELEMENT_NODE\n\t// - Node.DOCUMENT_NODE\n\t// - Object\n\t// - Any\n\treturn owner.nodeType ?\n\t\towner.nodeType === 1 || owner.nodeType === 9 : true;\n};\n\nData.prototype = {\n\tkey: function( owner ) {\n\t\t// We can accept data for non-element nodes in modern browsers,\n\t\t// but we should not, see #8335.\n\t\t// Always return the key for a frozen object.\n\t\tif ( !Data.accepts( owner ) ) {\n\t\t\treturn 0;\n\t\t}\n\n\t\tvar descriptor = {},\n\t\t\t// Check if the owner object already has a cache key\n\t\t\tunlock = owner[ this.expando ];\n\n\t\t// If not, create one\n\t\tif ( !unlock ) {\n\t\t\tunlock = Data.uid++;\n\n\t\t\t// Secure it in a non-enumerable, non-writable property\n\t\t\ttry {\n\t\t\t\tdescriptor[ this.expando ] = { value: unlock };\n\t\t\t\tObject.defineProperties( owner, descriptor );\n\n\t\t\t// Support: Android < 4\n\t\t\t// Fallback to a less secure definition\n\t\t\t} catch ( e ) {\n\t\t\t\tdescriptor[ this.expando ] = unlock;\n\t\t\t\tjQuery.extend( owner, descriptor );\n\t\t\t}\n\t\t}\n\n\t\t// Ensure the cache object\n\t\tif ( !this.cache[ unlock ] ) {\n\t\t\tthis.cache[ unlock ] = {};\n\t\t}\n\n\t\treturn unlock;\n\t},\n\tset: function( owner, data, value ) {\n\t\tvar prop,\n\t\t\t// There may be an unlock assigned to this node,\n\t\t\t// if there is no entry for this \"owner\", create one inline\n\t\t\t// and set the unlock as though an owner entry had always existed\n\t\t\tunlock = this.key( owner ),\n\t\t\tcache = this.cache[ unlock ];\n\n\t\t// Handle: [ owner, key, value ] args\n\t\tif ( typeof data === \"string\" ) {\n\t\t\tcache[ data ] = value;\n\n\t\t// Handle: [ owner, { properties } ] args\n\t\t} else {\n\t\t\t// Support an expectation from the old data system where plain\n\t\t\t// objects used to initialize would be set to the cache by\n\t\t\t// reference, instead of having properties and values copied.\n\t\t\t// Note, this will kill the connection between\n\t\t\t// \"this.cache[ unlock ]\" and \"cache\"\n\t\t\tif ( jQuery.isEmptyObject( cache ) ) {\n\t\t\t\tthis.cache[ unlock ] = data;\n\t\t\t// Otherwise, copy the properties one-by-one to the cache object\n\t\t\t} else {\n\t\t\t\tfor ( prop in data ) {\n\t\t\t\t\tcache[ prop ] = data[ prop ];\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t},\n\tget: function( owner, key ) {\n\t\t// Either a valid cache is found, or will be created.\n\t\t// New caches will be created and the unlock returned,\n\t\t// allowing direct access to the newly created\n\t\t// empty data object. A valid owner object must be provided.\n\t\tvar cache = this.cache[ this.key( owner ) ];\n\n\t\treturn key === undefined ?\n\t\t\tcache : cache[ key ];\n\t},\n\taccess: function( owner, key, value ) {\n\t\t// In cases where either:\n\t\t//\n\t\t// 1. No key was specified\n\t\t// 2. A string key was specified, but no value provided\n\t\t//\n\t\t// Take the \"read\" path and allow the get method to determine\n\t\t// which value to return, respectively either:\n\t\t//\n\t\t// 1. The entire cache object\n\t\t// 2. The data stored at the key\n\t\t//\n\t\tif ( key === undefined ||\n\t\t\t\t((key && typeof key === \"string\") && value === undefined) ) {\n\t\t\treturn this.get( owner, key );\n\t\t}\n\n\t\t// [*]When the key is not a string, or both a key and value\n\t\t// are specified, set or extend (existing objects) with either:\n\t\t//\n\t\t// 1. An object of properties\n\t\t// 2. A key and value\n\t\t//\n\t\tthis.set( owner, key, value );\n\n\t\t// Since the \"set\" path can have two possible entry points\n\t\t// return the expected data based on which path was taken[*]\n\t\treturn value !== undefined ? value : key;\n\t},\n\tremove: function( owner, key ) {\n\t\tvar i, name,\n\t\t\tunlock = this.key( owner ),\n\t\t\tcache = this.cache[ unlock ];\n\n\t\tif ( key === undefined ) {\n\t\t\tthis.cache[ unlock ] = {};\n\n\t\t} else {\n\t\t\t// Support array or space separated string of keys\n\t\t\tif ( jQuery.isArray( key ) ) {\n\t\t\t\t// If \"name\" is an array of keys...\n\t\t\t\t// When data is initially created, via (\"key\", \"val\") signature,\n\t\t\t\t// keys will be converted to camelCase.\n\t\t\t\t// Since there is no way to tell _how_ a key was added, remove\n\t\t\t\t// both plain key and camelCase key. #12786\n\t\t\t\t// This will only penalize the array argument path.\n\t\t\t\tname = key.concat( key.map( jQuery.camelCase ) );\n\t\t\t} else {\n\t\t\t\t// Try the string as a key before any manipulation\n\t\t\t\tif ( key in cache ) {\n\t\t\t\t\tname = [ key ];\n\t\t\t\t} else {\n\t\t\t\t\t// If a key with the spaces exists, use it.\n\t\t\t\t\t// Otherwise, create an array by matching non-whitespace\n\t\t\t\t\tname = jQuery.camelCase( key );\n\t\t\t\t\tname = name in cache ?\n\t\t\t\t\t\t[ name ] : ( name.match( core_rnotwhite ) || [] );\n\t\t\t\t}\n\t\t\t}\n\n\t\t\ti = name.length;\n\t\t\twhile ( i-- ) {\n\t\t\t\tdelete cache[ name[ i ] ];\n\t\t\t}\n\t\t}\n\t},\n\thasData: function( owner ) {\n\t\treturn !jQuery.isEmptyObject(\n\t\t\tthis.cache[ owner[ this.expando ] ] || {}\n\t\t);\n\t},\n\tdiscard: function( owner ) {\n\t\tdelete this.cache[ this.key( owner ) ];\n\t}\n};\n\n// These may be used throughout the jQuery core codebase\ndata_user = new Data();\ndata_priv = new Data();\n\n\njQuery.extend({\n\tacceptData: Data.accepts,\n\n\thasData: function( elem ) {\n\t\treturn data_user.hasData( elem ) || data_priv.hasData( elem );\n\t},\n\n\tdata: function( elem, name, data ) {\n\t\treturn data_user.access( elem, name, data );\n\t},\n\n\tremoveData: function( elem, name ) {\n\t\tdata_user.remove( elem, name );\n\t},\n\n\t// TODO: Now that all calls to _data and _removeData have been replaced\n\t// with direct calls to data_priv methods, these can be deprecated.\n\t_data: function( elem, name, data ) {\n\t\treturn data_priv.access( elem, name, data );\n\t},\n\n\t_removeData: function( elem, name ) {\n\t\tdata_priv.remove( elem, name );\n\t}\n});\n\njQuery.fn.extend({\n\tdata: function( key, value ) {\n\t\tvar attrs, name,\n\t\t\telem = this[ 0 ],\n\t\t\ti = 0,\n\t\t\tdata = null;\n\n\t\t// Gets all values\n\t\tif ( key === undefined ) {\n\t\t\tif ( this.length ) {\n\t\t\t\tdata = data_user.get( elem );\n\n\t\t\t\tif ( elem.nodeType === 1 && !data_priv.get( elem, \"hasDataAttrs\" ) ) {\n\t\t\t\t\tattrs = elem.attributes;\n\t\t\t\t\tfor ( ; i < attrs.length; i++ ) {\n\t\t\t\t\t\tname = attrs[ i ].name;\n\n\t\t\t\t\t\tif ( name.indexOf( \"data-\" ) === 0 ) {\n\t\t\t\t\t\t\tname = jQuery.camelCase( name.substring(5) );\n\t\t\t\t\t\t\tdataAttr( elem, name, data[ name ] );\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tdata_priv.set( elem, \"hasDataAttrs\", true );\n\t\t\t\t}\n\t\t\t}\n\n\t\t\treturn data;\n\t\t}\n\n\t\t// Sets multiple values\n\t\tif ( typeof key === \"object\" ) {\n\t\t\treturn this.each(function() {\n\t\t\t\tdata_user.set( this, key );\n\t\t\t});\n\t\t}\n\n\t\treturn jQuery.access( this, function( value ) {\n\t\t\tvar data,\n\t\t\t\tcamelKey = jQuery.camelCase( key );\n\n\t\t\t// The calling jQuery object (element matches) is not empty\n\t\t\t// (and therefore has an element appears at this[ 0 ]) and the\n\t\t\t// `value` parameter was not undefined. An empty jQuery object\n\t\t\t// will result in `undefined` for elem = this[ 0 ] which will\n\t\t\t// throw an exception if an attempt to read a data cache is made.\n\t\t\tif ( elem && value === undefined ) {\n\t\t\t\t// Attempt to get data from the cache\n\t\t\t\t// with the key as-is\n\t\t\t\tdata = data_user.get( elem, key );\n\t\t\t\tif ( data !== undefined ) {\n\t\t\t\t\treturn data;\n\t\t\t\t}\n\n\t\t\t\t// Attempt to get data from the cache\n\t\t\t\t// with the key camelized\n\t\t\t\tdata = data_user.get( elem, camelKey );\n\t\t\t\tif ( data !== undefined ) {\n\t\t\t\t\treturn data;\n\t\t\t\t}\n\n\t\t\t\t// Attempt to \"discover\" the data in\n\t\t\t\t// HTML5 custom data-* attrs\n\t\t\t\tdata = dataAttr( elem, camelKey, undefined );\n\t\t\t\tif ( data !== undefined ) {\n\t\t\t\t\treturn data;\n\t\t\t\t}\n\n\t\t\t\t// We tried really hard, but the data doesn't exist.\n\t\t\t\treturn;\n\t\t\t}\n\n\t\t\t// Set the data...\n\t\t\tthis.each(function() {\n\t\t\t\t// First, attempt to store a copy or reference of any\n\t\t\t\t// data that might've been store with a camelCased key.\n\t\t\t\tvar data = data_user.get( this, camelKey );\n\n\t\t\t\t// For HTML5 data-* attribute interop, we have to\n\t\t\t\t// store property names with dashes in a camelCase form.\n\t\t\t\t// This might not apply to all properties...*\n\t\t\t\tdata_user.set( this, camelKey, value );\n\n\t\t\t\t// *... In the case of properties that might _actually_\n\t\t\t\t// have dashes, we need to also store a copy of that\n\t\t\t\t// unchanged property.\n\t\t\t\tif ( key.indexOf(\"-\") !== -1 && data !== undefined ) {\n\t\t\t\t\tdata_user.set( this, key, value );\n\t\t\t\t}\n\t\t\t});\n\t\t}, null, value, arguments.length > 1, null, true );\n\t},\n\n\tremoveData: function( key ) {\n\t\treturn this.each(function() {\n\t\t\tdata_user.remove( this, key );\n\t\t});\n\t}\n});\n\nfunction dataAttr( elem, key, data ) {\n\tvar name;\n\n\t// If nothing was found internally, try to fetch any\n\t// data from the HTML5 data-* attribute\n\tif ( data === undefined && elem.nodeType === 1 ) {\n\t\tname = \"data-\" + key.replace( rmultiDash, \"-$1\" ).toLowerCase();\n\t\tdata = elem.getAttribute( name );\n\n\t\tif ( typeof data === \"string\" ) {\n\t\t\ttry {\n\t\t\t\tdata = data === \"true\" ? true :\n\t\t\t\t\tdata === \"false\" ? false :\n\t\t\t\t\tdata === \"null\" ? null :\n\t\t\t\t\t// Only convert to a number if it doesn't change the string\n\t\t\t\t\t+data + \"\" === data ? +data :\n\t\t\t\t\trbrace.test( data ) ? JSON.parse( data ) :\n\t\t\t\t\tdata;\n\t\t\t} catch( e ) {}\n\n\t\t\t// Make sure we set the data so it isn't changed later\n\t\t\tdata_user.set( elem, key, data );\n\t\t} else {\n\t\t\tdata = undefined;\n\t\t}\n\t}\n\treturn data;\n}\njQuery.extend({\n\tqueue: function( elem, type, data ) {\n\t\tvar queue;\n\n\t\tif ( elem ) {\n\t\t\ttype = ( type || \"fx\" ) + \"queue\";\n\t\t\tqueue = data_priv.get( elem, type );\n\n\t\t\t// Speed up dequeue by getting out quickly if this is just a lookup\n\t\t\tif ( data ) {\n\t\t\t\tif ( !queue || jQuery.isArray( data ) ) {\n\t\t\t\t\tqueue = data_priv.access( elem, type, jQuery.makeArray(data) );\n\t\t\t\t} else {\n\t\t\t\t\tqueue.push( data );\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn queue || [];\n\t\t}\n\t},\n\n\tdequeue: function( elem, type ) {\n\t\ttype = type || \"fx\";\n\n\t\tvar queue = jQuery.queue( elem, type ),\n\t\t\tstartLength = queue.length,\n\t\t\tfn = queue.shift(),\n\t\t\thooks = jQuery._queueHooks( elem, type ),\n\t\t\tnext = function() {\n\t\t\t\tjQuery.dequeue( elem, type );\n\t\t\t};\n\n\t\t// If the fx queue is dequeued, always remove the progress sentinel\n\t\tif ( fn === \"inprogress\" ) {\n\t\t\tfn = queue.shift();\n\t\t\tstartLength--;\n\t\t}\n\n\t\thooks.cur = fn;\n\t\tif ( fn ) {\n\n\t\t\t// Add a progress sentinel to prevent the fx queue from being\n\t\t\t// automatically dequeued\n\t\t\tif ( type === \"fx\" ) {\n\t\t\t\tqueue.unshift( \"inprogress\" );\n\t\t\t}\n\n\t\t\t// clear up the last queue stop function\n\t\t\tdelete hooks.stop;\n\t\t\tfn.call( elem, next, hooks );\n\t\t}\n\n\t\tif ( !startLength && hooks ) {\n\t\t\thooks.empty.fire();\n\t\t}\n\t},\n\n\t// not intended for public consumption - generates a queueHooks object, or returns the current one\n\t_queueHooks: function( elem, type ) {\n\t\tvar key = type + \"queueHooks\";\n\t\treturn data_priv.get( elem, key ) || data_priv.access( elem, key, {\n\t\t\tempty: jQuery.Callbacks(\"once memory\").add(function() {\n\t\t\t\tdata_priv.remove( elem, [ type + \"queue\", key ] );\n\t\t\t})\n\t\t});\n\t}\n});\n\njQuery.fn.extend({\n\tqueue: function( type, data ) {\n\t\tvar setter = 2;\n\n\t\tif ( typeof type !== \"string\" ) {\n\t\t\tdata = type;\n\t\t\ttype = \"fx\";\n\t\t\tsetter--;\n\t\t}\n\n\t\tif ( arguments.length < setter ) {\n\t\t\treturn jQuery.queue( this[0], type );\n\t\t}\n\n\t\treturn data === undefined ?\n\t\t\tthis :\n\t\t\tthis.each(function() {\n\t\t\t\tvar queue = jQuery.queue( this, type, data );\n\n\t\t\t\t// ensure a hooks for this queue\n\t\t\t\tjQuery._queueHooks( this, type );\n\n\t\t\t\tif ( type === \"fx\" && queue[0] !== \"inprogress\" ) {\n\t\t\t\t\tjQuery.dequeue( this, type );\n\t\t\t\t}\n\t\t\t});\n\t},\n\tdequeue: function( type ) {\n\t\treturn this.each(function() {\n\t\t\tjQuery.dequeue( this, type );\n\t\t});\n\t},\n\t// Based off of the plugin by Clint Helfers, with permission.\n\t// http://blindsignals.com/index.php/2009/07/jquery-delay/\n\tdelay: function( time, type ) {\n\t\ttime = jQuery.fx ? jQuery.fx.speeds[ time ] || time : time;\n\t\ttype = type || \"fx\";\n\n\t\treturn this.queue( type, function( next, hooks ) {\n\t\t\tvar timeout = setTimeout( next, time );\n\t\t\thooks.stop = function() {\n\t\t\t\tclearTimeout( timeout );\n\t\t\t};\n\t\t});\n\t},\n\tclearQueue: function( type ) {\n\t\treturn this.queue( type || \"fx\", [] );\n\t},\n\t// Get a promise resolved when queues of a certain type\n\t// are emptied (fx is the type by default)\n\tpromise: function( type, obj ) {\n\t\tvar tmp,\n\t\t\tcount = 1,\n\t\t\tdefer = jQuery.Deferred(),\n\t\t\telements = this,\n\t\t\ti = this.length,\n\t\t\tresolve = function() {\n\t\t\t\tif ( !( --count ) ) {\n\t\t\t\t\tdefer.resolveWith( elements, [ elements ] );\n\t\t\t\t}\n\t\t\t};\n\n\t\tif ( typeof type !== \"string\" ) {\n\t\t\tobj = type;\n\t\t\ttype = undefined;\n\t\t}\n\t\ttype = type || \"fx\";\n\n\t\twhile( i-- ) {\n\t\t\ttmp = data_priv.get( elements[ i ], type + \"queueHooks\" );\n\t\t\tif ( tmp && tmp.empty ) {\n\t\t\t\tcount++;\n\t\t\t\ttmp.empty.add( resolve );\n\t\t\t}\n\t\t}\n\t\tresolve();\n\t\treturn defer.promise( obj );\n\t}\n});\nvar nodeHook, boolHook,\n\trclass = /[\\t\\r\\n]/g,\n\trreturn = /\\r/g,\n\trfocusable = /^(?:input|select|textarea|button)$/i;\n\njQuery.fn.extend({\n\tattr: function( name, value ) {\n\t\treturn jQuery.access( this, jQuery.attr, name, value, arguments.length > 1 );\n\t},\n\n\tremoveAttr: function( name ) {\n\t\treturn this.each(function() {\n\t\t\tjQuery.removeAttr( this, name );\n\t\t});\n\t},\n\n\tprop: function( name, value ) {\n\t\treturn jQuery.access( this, jQuery.prop, name, value, arguments.length > 1 );\n\t},\n\n\tremoveProp: function( name ) {\n\t\treturn this.each(function() {\n\t\t\tdelete this[ jQuery.propFix[ name ] || name ];\n\t\t});\n\t},\n\n\taddClass: function( value ) {\n\t\tvar classes, elem, cur, clazz, j,\n\t\t\ti = 0,\n\t\t\tlen = this.length,\n\t\t\tproceed = typeof value === \"string\" && value;\n\n\t\tif ( jQuery.isFunction( value ) ) {\n\t\t\treturn this.each(function( j ) {\n\t\t\t\tjQuery( this ).addClass( value.call( this, j, this.className ) );\n\t\t\t});\n\t\t}\n\n\t\tif ( proceed ) {\n\t\t\t// The disjunction here is for better compressibility (see removeClass)\n\t\t\tclasses = ( value || \"\" ).match( core_rnotwhite ) || [];\n\n\t\t\tfor ( ; i < len; i++ ) {\n\t\t\t\telem = this[ i ];\n\t\t\t\tcur = elem.nodeType === 1 && ( elem.className ?\n\t\t\t\t\t( \" \" + elem.className + \" \" ).replace( rclass, \" \" ) :\n\t\t\t\t\t\" \"\n\t\t\t\t);\n\n\t\t\t\tif ( cur ) {\n\t\t\t\t\tj = 0;\n\t\t\t\t\twhile ( (clazz = classes[j++]) ) {\n\t\t\t\t\t\tif ( cur.indexOf( \" \" + clazz + \" \" ) < 0 ) {\n\t\t\t\t\t\t\tcur += clazz + \" \";\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\telem.className = jQuery.trim( cur );\n\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\treturn this;\n\t},\n\n\tremoveClass: function( value ) {\n\t\tvar classes, elem, cur, clazz, j,\n\t\t\ti = 0,\n\t\t\tlen = this.length,\n\t\t\tproceed = arguments.length === 0 || typeof value === \"string\" && value;\n\n\t\tif ( jQuery.isFunction( value ) ) {\n\t\t\treturn this.each(function( j ) {\n\t\t\t\tjQuery( this ).removeClass( value.call( this, j, this.className ) );\n\t\t\t});\n\t\t}\n\t\tif ( proceed ) {\n\t\t\tclasses = ( value || \"\" ).match( core_rnotwhite ) || [];\n\n\t\t\tfor ( ; i < len; i++ ) {\n\t\t\t\telem = this[ i ];\n\t\t\t\t// This expression is here for better compressibility (see addClass)\n\t\t\t\tcur = elem.nodeType === 1 && ( elem.className ?\n\t\t\t\t\t( \" \" + elem.className + \" \" ).replace( rclass, \" \" ) :\n\t\t\t\t\t\"\"\n\t\t\t\t);\n\n\t\t\t\tif ( cur ) {\n\t\t\t\t\tj = 0;\n\t\t\t\t\twhile ( (clazz = classes[j++]) ) {\n\t\t\t\t\t\t// Remove *all* instances\n\t\t\t\t\t\twhile ( cur.indexOf( \" \" + clazz + \" \" ) >= 0 ) {\n\t\t\t\t\t\t\tcur = cur.replace( \" \" + clazz + \" \", \" \" );\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\telem.className = value ? jQuery.trim( cur ) : \"\";\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\treturn this;\n\t},\n\n\ttoggleClass: function( value, stateVal ) {\n\t\tvar type = typeof value,\n\t\t\tisBool = typeof stateVal === \"boolean\";\n\n\t\tif ( jQuery.isFunction( value ) ) {\n\t\t\treturn this.each(function( i ) {\n\t\t\t\tjQuery( this ).toggleClass( value.call(this, i, this.className, stateVal), stateVal );\n\t\t\t});\n\t\t}\n\n\t\treturn this.each(function() {\n\t\t\tif ( type === \"string\" ) {\n\t\t\t\t// toggle individual class names\n\t\t\t\tvar className,\n\t\t\t\t\ti = 0,\n\t\t\t\t\tself = jQuery( this ),\n\t\t\t\t\tstate = stateVal,\n\t\t\t\t\tclassNames = value.match( core_rnotwhite ) || [];\n\n\t\t\t\twhile ( (className = classNames[ i++ ]) ) {\n\t\t\t\t\t// check each className given, space separated list\n\t\t\t\t\tstate = isBool ? state : !self.hasClass( className );\n\t\t\t\t\tself[ state ? \"addClass\" : \"removeClass\" ]( className );\n\t\t\t\t}\n\n\t\t\t// Toggle whole class name\n\t\t\t} else if ( type === core_strundefined || type === \"boolean\" ) {\n\t\t\t\tif ( this.className ) {\n\t\t\t\t\t// store className if set\n\t\t\t\t\tdata_priv.set( this, \"__className__\", this.className );\n\t\t\t\t}\n\n\t\t\t\t// If the element has a class name or if we're passed \"false\",\n\t\t\t\t// then remove the whole classname (if there was one, the above saved it).\n\t\t\t\t// Otherwise bring back whatever was previously saved (if anything),\n\t\t\t\t// falling back to the empty string if nothing was stored.\n\t\t\t\tthis.className = this.className || value === false ? \"\" : data_priv.get( this, \"__className__\" ) || \"\";\n\t\t\t}\n\t\t});\n\t},\n\n\thasClass: function( selector ) {\n\t\tvar className = \" \" + selector + \" \",\n\t\t\ti = 0,\n\t\t\tl = this.length;\n\t\tfor ( ; i < l; i++ ) {\n\t\t\tif ( this[i].nodeType === 1 && (\" \" + this[i].className + \" \").replace(rclass, \" \").indexOf( className ) >= 0 ) {\n\t\t\t\treturn true;\n\t\t\t}\n\t\t}\n\n\t\treturn false;\n\t},\n\n\tval: function( value ) {\n\t\tvar hooks, ret, isFunction,\n\t\t\telem = this[0];\n\n\t\tif ( !arguments.length ) {\n\t\t\tif ( elem ) {\n\t\t\t\thooks = jQuery.valHooks[ elem.type ] || jQuery.valHooks[ elem.nodeName.toLowerCase() ];\n\n\t\t\t\tif ( hooks && \"get\" in hooks && (ret = hooks.get( elem, \"value\" )) !== undefined ) {\n\t\t\t\t\treturn ret;\n\t\t\t\t}\n\n\t\t\t\tret = elem.value;\n\n\t\t\t\treturn typeof ret === \"string\" ?\n\t\t\t\t\t// handle most common string cases\n\t\t\t\t\tret.replace(rreturn, \"\") :\n\t\t\t\t\t// handle cases where value is null/undef or number\n\t\t\t\t\tret == null ? \"\" : ret;\n\t\t\t}\n\n\t\t\treturn;\n\t\t}\n\n\t\tisFunction = jQuery.isFunction( value );\n\n\t\treturn this.each(function( i ) {\n\t\t\tvar val,\n\t\t\t\tself = jQuery(this);\n\n\t\t\tif ( this.nodeType !== 1 ) {\n\t\t\t\treturn;\n\t\t\t}\n\n\t\t\tif ( isFunction ) {\n\t\t\t\tval = value.call( this, i, self.val() );\n\t\t\t} else {\n\t\t\t\tval = value;\n\t\t\t}\n\n\t\t\t// Treat null/undefined as \"\"; convert numbers to string\n\t\t\tif ( val == null ) {\n\t\t\t\tval = \"\";\n\t\t\t} else if ( typeof val === \"number\" ) {\n\t\t\t\tval += \"\";\n\t\t\t} else if ( jQuery.isArray( val ) ) {\n\t\t\t\tval = jQuery.map(val, function ( value ) {\n\t\t\t\t\treturn value == null ? \"\" : value + \"\";\n\t\t\t\t});\n\t\t\t}\n\n\t\t\thooks = jQuery.valHooks[ this.type ] || jQuery.valHooks[ this.nodeName.toLowerCase() ];\n\n\t\t\t// If set returns undefined, fall back to normal setting\n\t\t\tif ( !hooks || !(\"set\" in hooks) || hooks.set( this, val, \"value\" ) === undefined ) {\n\t\t\t\tthis.value = val;\n\t\t\t}\n\t\t});\n\t}\n});\n\njQuery.extend({\n\tvalHooks: {\n\t\toption: {\n\t\t\tget: function( elem ) {\n\t\t\t\t// attributes.value is undefined in Blackberry 4.7 but\n\t\t\t\t// uses .value. See #6932\n\t\t\t\tvar val = elem.attributes.value;\n\t\t\t\treturn !val || val.specified ? elem.value : elem.text;\n\t\t\t}\n\t\t},\n\t\tselect: {\n\t\t\tget: function( elem ) {\n\t\t\t\tvar value, option,\n\t\t\t\t\toptions = elem.options,\n\t\t\t\t\tindex = elem.selectedIndex,\n\t\t\t\t\tone = elem.type === \"select-one\" || index < 0,\n\t\t\t\t\tvalues = one ? null : [],\n\t\t\t\t\tmax = one ? index + 1 : options.length,\n\t\t\t\t\ti = index < 0 ?\n\t\t\t\t\t\tmax :\n\t\t\t\t\t\tone ? index : 0;\n\n\t\t\t\t// Loop through all the selected options\n\t\t\t\tfor ( ; i < max; i++ ) {\n\t\t\t\t\toption = options[ i ];\n\n\t\t\t\t\t// IE6-9 doesn't update selected after form reset (#2551)\n\t\t\t\t\tif ( ( option.selected || i === index ) &&\n\t\t\t\t\t\t\t// Don't return options that are disabled or in a disabled optgroup\n\t\t\t\t\t\t\t( jQuery.support.optDisabled ? !option.disabled : option.getAttribute(\"disabled\") === null ) &&\n\t\t\t\t\t\t\t( !option.parentNode.disabled || !jQuery.nodeName( option.parentNode, \"optgroup\" ) ) ) {\n\n\t\t\t\t\t\t// Get the specific value for the option\n\t\t\t\t\t\tvalue = jQuery( option ).val();\n\n\t\t\t\t\t\t// We don't need an array for one selects\n\t\t\t\t\t\tif ( one ) {\n\t\t\t\t\t\t\treturn value;\n\t\t\t\t\t\t}\n\n\t\t\t\t\t\t// Multi-Selects return an array\n\t\t\t\t\t\tvalues.push( value );\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\treturn values;\n\t\t\t},\n\n\t\t\tset: function( elem, value ) {\n\t\t\t\tvar optionSet, option,\n\t\t\t\t\toptions = elem.options,\n\t\t\t\t\tvalues = jQuery.makeArray( value ),\n\t\t\t\t\ti = options.length;\n\n\t\t\t\twhile ( i-- ) {\n\t\t\t\t\toption = options[ i ];\n\t\t\t\t\tif ( (option.selected = jQuery.inArray( jQuery(option).val(), values ) >= 0) ) {\n\t\t\t\t\t\toptionSet = true;\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\t// force browsers to behave consistently when non-matching value is set\n\t\t\t\tif ( !optionSet ) {\n\t\t\t\t\telem.selectedIndex = -1;\n\t\t\t\t}\n\t\t\t\treturn values;\n\t\t\t}\n\t\t}\n\t},\n\n\tattr: function( elem, name, value ) {\n\t\tvar hooks, ret,\n\t\t\tnType = elem.nodeType;\n\n\t\t// don't get/set attributes on text, comment and attribute nodes\n\t\tif ( !elem || nType === 3 || nType === 8 || nType === 2 ) {\n\t\t\treturn;\n\t\t}\n\n\t\t// Fallback to prop when attributes are not supported\n\t\tif ( typeof elem.getAttribute === core_strundefined ) {\n\t\t\treturn jQuery.prop( elem, name, value );\n\t\t}\n\n\t\t// All attributes are lowercase\n\t\t// Grab necessary hook if one is defined\n\t\tif ( nType !== 1 || !jQuery.isXMLDoc( elem ) ) {\n\t\t\tname = name.toLowerCase();\n\t\t\thooks = jQuery.attrHooks[ name ] ||\n\t\t\t\t( jQuery.expr.match.boolean.test( name ) ? boolHook : nodeHook );\n\t\t}\n\n\t\tif ( value !== undefined ) {\n\n\t\t\tif ( value === null ) {\n\t\t\t\tjQuery.removeAttr( elem, name );\n\n\t\t\t} else if ( hooks && \"set\" in hooks && (ret = hooks.set( elem, value, name )) !== undefined ) {\n\t\t\t\treturn ret;\n\n\t\t\t} else {\n\t\t\t\telem.setAttribute( name, value + \"\" );\n\t\t\t\treturn value;\n\t\t\t}\n\n\t\t} else if ( hooks && \"get\" in hooks && (ret = hooks.get( elem, name )) !== null ) {\n\t\t\treturn ret;\n\n\t\t} else {\n\t\t\tret = jQuery.find.attr( elem, name );\n\n\t\t\t// Non-existent attributes return null, we normalize to undefined\n\t\t\treturn ret == null ?\n\t\t\t\tundefined :\n\t\t\t\tret;\n\t\t}\n\t},\n\n\tremoveAttr: function( elem, value ) {\n\t\tvar name, propName,\n\t\t\ti = 0,\n\t\t\tattrNames = value && value.match( core_rnotwhite );\n\n\t\tif ( attrNames && elem.nodeType === 1 ) {\n\t\t\twhile ( (name = attrNames[i++]) ) {\n\t\t\t\tpropName = jQuery.propFix[ name ] || name;\n\n\t\t\t\t// Boolean attributes get special treatment (#10870)\n\t\t\t\tif ( jQuery.expr.match.boolean.test( name ) ) {\n\t\t\t\t\t// Set corresponding property to false\n\t\t\t\t\telem[ propName ] = false;\n\t\t\t\t}\n\n\t\t\t\telem.removeAttribute( name );\n\t\t\t}\n\t\t}\n\t},\n\n\tattrHooks: {\n\t\ttype: {\n\t\t\tset: function( elem, value ) {\n\t\t\t\tif ( !jQuery.support.radioValue && value === \"radio\" && jQuery.nodeName(elem, \"input\") ) {\n\t\t\t\t\t// Setting the type on a radio button after the value resets the value in IE6-9\n\t\t\t\t\t// Reset value to default in case type is set after value during creation\n\t\t\t\t\tvar val = elem.value;\n\t\t\t\t\telem.setAttribute( \"type\", value );\n\t\t\t\t\tif ( val ) {\n\t\t\t\t\t\telem.value = val;\n\t\t\t\t\t}\n\t\t\t\t\treturn value;\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t},\n\n\tpropFix: {\n\t\t\"for\": \"htmlFor\",\n\t\t\"class\": \"className\"\n\t},\n\n\tprop: function( elem, name, value ) {\n\t\tvar ret, hooks, notxml,\n\t\t\tnType = elem.nodeType;\n\n\t\t// don't get/set properties on text, comment and attribute nodes\n\t\tif ( !elem || nType === 3 || nType === 8 || nType === 2 ) {\n\t\t\treturn;\n\t\t}\n\n\t\tnotxml = nType !== 1 || !jQuery.isXMLDoc( elem );\n\n\t\tif ( notxml ) {\n\t\t\t// Fix name and attach hooks\n\t\t\tname = jQuery.propFix[ name ] || name;\n\t\t\thooks = jQuery.propHooks[ name ];\n\t\t}\n\n\t\tif ( value !== undefined ) {\n\t\t\treturn hooks && \"set\" in hooks && (ret = hooks.set( elem, value, name )) !== undefined ?\n\t\t\t\tret :\n\t\t\t\t( elem[ name ] = value );\n\n\t\t} else {\n\t\t\treturn hooks && \"get\" in hooks && (ret = hooks.get( elem, name )) !== null ?\n\t\t\t\tret :\n\t\t\t\telem[ name ];\n\t\t}\n\t},\n\n\tpropHooks: {\n\t\ttabIndex: {\n\t\t\tget: function( elem ) {\n\t\t\t\treturn elem.hasAttribute( \"tabindex\" ) || rfocusable.test( elem.nodeName ) || elem.href ?\n\t\t\t\t\telem.tabIndex :\n\t\t\t\t\t-1;\n\t\t\t}\n\t\t}\n\t}\n});\n\n// Hooks for boolean attributes\nboolHook = {\n\tset: function( elem, value, name ) {\n\t\tif ( value === false ) {\n\t\t\t// Remove boolean attributes when set to false\n\t\t\tjQuery.removeAttr( elem, name );\n\t\t} else {\n\t\t\telem.setAttribute( name, name );\n\t\t}\n\t\treturn name;\n\t}\n};\njQuery.each( jQuery.expr.match.boolean.source.match( /\\w+/g ), function( i, name ) {\n\tvar getter = jQuery.expr.attrHandle[ name ] || jQuery.find.attr;\n\n\tjQuery.expr.attrHandle[ name ] = function( elem, name, isXML ) {\n\t\tvar fn = jQuery.expr.attrHandle[ name ],\n\t\t\tret = isXML ?\n\t\t\t\tundefined :\n\t\t\t\t/* jshint eqeqeq: false */\n\t\t\t\t// Temporarily disable this handler to check existence\n\t\t\t\t(jQuery.expr.attrHandle[ name ] = undefined) !=\n\t\t\t\t\tgetter( elem, name, isXML ) ?\n\n\t\t\t\t\tname.toLowerCase() :\n\t\t\t\t\tnull;\n\n\t\t// Restore handler\n\t\tjQuery.expr.attrHandle[ name ] = fn;\n\n\t\treturn ret;\n\t};\n});\n\n// Support: IE9+\n// Selectedness for an option in an optgroup can be inaccurate\nif ( !jQuery.support.optSelected ) {\n\tjQuery.propHooks.selected = {\n\t\tget: function( elem ) {\n\t\t\tvar parent = elem.parentNode;\n\t\t\tif ( parent && parent.parentNode ) {\n\t\t\t\tparent.parentNode.selectedIndex;\n\t\t\t}\n\t\t\treturn null;\n\t\t}\n\t};\n}\n\njQuery.each([\n\t\"tabIndex\",\n\t\"readOnly\",\n\t\"maxLength\",\n\t\"cellSpacing\",\n\t\"cellPadding\",\n\t\"rowSpan\",\n\t\"colSpan\",\n\t\"useMap\",\n\t\"frameBorder\",\n\t\"contentEditable\"\n], function() {\n\tjQuery.propFix[ this.toLowerCase() ] = this;\n});\n\n// Radios and checkboxes getter/setter\njQuery.each([ \"radio\", \"checkbox\" ], function() {\n\tjQuery.valHooks[ this ] = {\n\t\tset: function( elem, value ) {\n\t\t\tif ( jQuery.isArray( value ) ) {\n\t\t\t\treturn ( elem.checked = jQuery.inArray( jQuery(elem).val(), value ) >= 0 );\n\t\t\t}\n\t\t}\n\t};\n\tif ( !jQuery.support.checkOn ) {\n\t\tjQuery.valHooks[ this ].get = function( elem ) {\n\t\t\t// Support: Webkit\n\t\t\t// \"\" is returned instead of \"on\" if a value isn't specified\n\t\t\treturn elem.getAttribute(\"value\") === null ? \"on\" : elem.value;\n\t\t};\n\t}\n});\nvar rkeyEvent = /^key/,\n\trmouseEvent = /^(?:mouse|contextmenu)|click/,\n\trfocusMorph = /^(?:focusinfocus|focusoutblur)$/,\n\trtypenamespace = /^([^.]*)(?:\\.(.+)|)$/;\n\nfunction returnTrue() {\n\treturn true;\n}\n\nfunction returnFalse() {\n\treturn false;\n}\n\nfunction safeActiveElement() {\n\ttry {\n\t\treturn document.activeElement;\n\t} catch ( err ) { }\n}\n\n/*\n * Helper functions for managing events -- not part of the public interface.\n * Props to Dean Edwards' addEvent library for many of the ideas.\n */\njQuery.event = {\n\n\tglobal: {},\n\n\tadd: function( elem, types, handler, data, selector ) {\n\n\t\tvar handleObjIn, eventHandle, tmp,\n\t\t\tevents, t, handleObj,\n\t\t\tspecial, handlers, type, namespaces, origType,\n\t\t\telemData = data_priv.get( elem );\n\n\t\t// Don't attach events to noData or text/comment nodes (but allow plain objects)\n\t\tif ( !elemData ) {\n\t\t\treturn;\n\t\t}\n\n\t\t// Caller can pass in an object of custom data in lieu of the handler\n\t\tif ( handler.handler ) {\n\t\t\thandleObjIn = handler;\n\t\t\thandler = handleObjIn.handler;\n\t\t\tselector = handleObjIn.selector;\n\t\t}\n\n\t\t// Make sure that the handler has a unique ID, used to find/remove it later\n\t\tif ( !handler.guid ) {\n\t\t\thandler.guid = jQuery.guid++;\n\t\t}\n\n\t\t// Init the element's event structure and main handler, if this is the first\n\t\tif ( !(events = elemData.events) ) {\n\t\t\tevents = elemData.events = {};\n\t\t}\n\t\tif ( !(eventHandle = elemData.handle) ) {\n\t\t\teventHandle = elemData.handle = function( e ) {\n\t\t\t\t// Discard the second event of a jQuery.event.trigger() and\n\t\t\t\t// when an event is called after a page has unloaded\n\t\t\t\treturn typeof jQuery !== core_strundefined && (!e || jQuery.event.triggered !== e.type) ?\n\t\t\t\t\tjQuery.event.dispatch.apply( eventHandle.elem, arguments ) :\n\t\t\t\t\tundefined;\n\t\t\t};\n\t\t\t// Add elem as a property of the handle fn to prevent a memory leak with IE non-native events\n\t\t\teventHandle.elem = elem;\n\t\t}\n\n\t\t// Handle multiple events separated by a space\n\t\ttypes = ( types || \"\" ).match( core_rnotwhite ) || [\"\"];\n\t\tt = types.length;\n\t\twhile ( t-- ) {\n\t\t\ttmp = rtypenamespace.exec( types[t] ) || [];\n\t\t\ttype = origType = tmp[1];\n\t\t\tnamespaces = ( tmp[2] || \"\" ).split( \".\" ).sort();\n\n\t\t\t// There *must* be a type, no attaching namespace-only handlers\n\t\t\tif ( !type ) {\n\t\t\t\tcontinue;\n\t\t\t}\n\n\t\t\t// If event changes its type, use the special event handlers for the changed type\n\t\t\tspecial = jQuery.event.special[ type ] || {};\n\n\t\t\t// If selector defined, determine special event api type, otherwise given type\n\t\t\ttype = ( selector ? special.delegateType : special.bindType ) || type;\n\n\t\t\t// Update special based on newly reset type\n\t\t\tspecial = jQuery.event.special[ type ] || {};\n\n\t\t\t// handleObj is passed to all event handlers\n\t\t\thandleObj = jQuery.extend({\n\t\t\t\ttype: type,\n\t\t\t\torigType: origType,\n\t\t\t\tdata: data,\n\t\t\t\thandler: handler,\n\t\t\t\tguid: handler.guid,\n\t\t\t\tselector: selector,\n\t\t\t\tneedsContext: selector && jQuery.expr.match.needsContext.test( selector ),\n\t\t\t\tnamespace: namespaces.join(\".\")\n\t\t\t}, handleObjIn );\n\n\t\t\t// Init the event handler queue if we're the first\n\t\t\tif ( !(handlers = events[ type ]) ) {\n\t\t\t\thandlers = events[ type ] = [];\n\t\t\t\thandlers.delegateCount = 0;\n\n\t\t\t\t// Only use addEventListener if the special events handler returns false\n\t\t\t\tif ( !special.setup || special.setup.call( elem, data, namespaces, eventHandle ) === false ) {\n\t\t\t\t\tif ( elem.addEventListener ) {\n\t\t\t\t\t\telem.addEventListener( type, eventHandle, false );\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif ( special.add ) {\n\t\t\t\tspecial.add.call( elem, handleObj );\n\n\t\t\t\tif ( !handleObj.handler.guid ) {\n\t\t\t\t\thandleObj.handler.guid = handler.guid;\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Add to the element's handler list, delegates in front\n\t\t\tif ( selector ) {\n\t\t\t\thandlers.splice( handlers.delegateCount++, 0, handleObj );\n\t\t\t} else {\n\t\t\t\thandlers.push( handleObj );\n\t\t\t}\n\n\t\t\t// Keep track of which events have ever been used, for event optimization\n\t\t\tjQuery.event.global[ type ] = true;\n\t\t}\n\n\t\t// Nullify elem to prevent memory leaks in IE\n\t\telem = null;\n\t},\n\n\t// Detach an event or set of events from an element\n\tremove: function( elem, types, handler, selector, mappedTypes ) {\n\n\t\tvar j, origCount, tmp,\n\t\t\tevents, t, handleObj,\n\t\t\tspecial, handlers, type, namespaces, origType,\n\t\t\telemData = data_priv.hasData( elem ) && data_priv.get( elem );\n\n\t\tif ( !elemData || !(events = elemData.events) ) {\n\t\t\treturn;\n\t\t}\n\n\t\t// Once for each type.namespace in types; type may be omitted\n\t\ttypes = ( types || \"\" ).match( core_rnotwhite ) || [\"\"];\n\t\tt = types.length;\n\t\twhile ( t-- ) {\n\t\t\ttmp = rtypenamespace.exec( types[t] ) || [];\n\t\t\ttype = origType = tmp[1];\n\t\t\tnamespaces = ( tmp[2] || \"\" ).split( \".\" ).sort();\n\n\t\t\t// Unbind all events (on this namespace, if provided) for the element\n\t\t\tif ( !type ) {\n\t\t\t\tfor ( type in events ) {\n\t\t\t\t\tjQuery.event.remove( elem, type + types[ t ], handler, selector, true );\n\t\t\t\t}\n\t\t\t\tcontinue;\n\t\t\t}\n\n\t\t\tspecial = jQuery.event.special[ type ] || {};\n\t\t\ttype = ( selector ? special.delegateType : special.bindType ) || type;\n\t\t\thandlers = events[ type ] || [];\n\t\t\ttmp = tmp[2] && new RegExp( \"(^|\\\\.)\" + namespaces.join(\"\\\\.(?:.*\\\\.|)\") + \"(\\\\.|$)\" );\n\n\t\t\t// Remove matching events\n\t\t\torigCount = j = handlers.length;\n\t\t\twhile ( j-- ) {\n\t\t\t\thandleObj = handlers[ j ];\n\n\t\t\t\tif ( ( mappedTypes || origType === handleObj.origType ) &&\n\t\t\t\t\t( !handler || handler.guid === handleObj.guid ) &&\n\t\t\t\t\t( !tmp || tmp.test( handleObj.namespace ) ) &&\n\t\t\t\t\t( !selector || selector === handleObj.selector || selector === \"**\" && handleObj.selector ) ) {\n\t\t\t\t\thandlers.splice( j, 1 );\n\n\t\t\t\t\tif ( handleObj.selector ) {\n\t\t\t\t\t\thandlers.delegateCount--;\n\t\t\t\t\t}\n\t\t\t\t\tif ( special.remove ) {\n\t\t\t\t\t\tspecial.remove.call( elem, handleObj );\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Remove generic event handler if we removed something and no more handlers exist\n\t\t\t// (avoids potential for endless recursion during removal of special event handlers)\n\t\t\tif ( origCount && !handlers.length ) {\n\t\t\t\tif ( !special.teardown || special.teardown.call( elem, namespaces, elemData.handle ) === false ) {\n\t\t\t\t\tjQuery.removeEvent( elem, type, elemData.handle );\n\t\t\t\t}\n\n\t\t\t\tdelete events[ type ];\n\t\t\t}\n\t\t}\n\n\t\t// Remove the expando if it's no longer used\n\t\tif ( jQuery.isEmptyObject( events ) ) {\n\t\t\tdelete elemData.handle;\n\t\t\tdata_priv.remove( elem, \"events\" );\n\t\t}\n\t},\n\n\ttrigger: function( event, data, elem, onlyHandlers ) {\n\n\t\tvar i, cur, tmp, bubbleType, ontype, handle, special,\n\t\t\teventPath = [ elem || document ],\n\t\t\ttype = core_hasOwn.call( event, \"type\" ) ? event.type : event,\n\t\t\tnamespaces = core_hasOwn.call( event, \"namespace\" ) ? event.namespace.split(\".\") : [];\n\n\t\tcur = tmp = elem = elem || document;\n\n\t\t// Don't do events on text and comment nodes\n\t\tif ( elem.nodeType === 3 || elem.nodeType === 8 ) {\n\t\t\treturn;\n\t\t}\n\n\t\t// focus/blur morphs to focusin/out; ensure we're not firing them right now\n\t\tif ( rfocusMorph.test( type + jQuery.event.triggered ) ) {\n\t\t\treturn;\n\t\t}\n\n\t\tif ( type.indexOf(\".\") >= 0 ) {\n\t\t\t// Namespaced trigger; create a regexp to match event type in handle()\n\t\t\tnamespaces = type.split(\".\");\n\t\t\ttype = namespaces.shift();\n\t\t\tnamespaces.sort();\n\t\t}\n\t\tontype = type.indexOf(\":\") < 0 && \"on\" + type;\n\n\t\t// Caller can pass in a jQuery.Event object, Object, or just an event type string\n\t\tevent = event[ jQuery.expando ] ?\n\t\t\tevent :\n\t\t\tnew jQuery.Event( type, typeof event === \"object\" && event );\n\n\t\t// Trigger bitmask: & 1 for native handlers; & 2 for jQuery (always true)\n\t\tevent.isTrigger = onlyHandlers ? 2 : 3;\n\t\tevent.namespace = namespaces.join(\".\");\n\t\tevent.namespace_re = event.namespace ?\n\t\t\tnew RegExp( \"(^|\\\\.)\" + namespaces.join(\"\\\\.(?:.*\\\\.|)\") + \"(\\\\.|$)\" ) :\n\t\t\tnull;\n\n\t\t// Clean up the event in case it is being reused\n\t\tevent.result = undefined;\n\t\tif ( !event.target ) {\n\t\t\tevent.target = elem;\n\t\t}\n\n\t\t// Clone any incoming data and prepend the event, creating the handler arg list\n\t\tdata = data == null ?\n\t\t\t[ event ] :\n\t\t\tjQuery.makeArray( data, [ event ] );\n\n\t\t// Allow special events to draw outside the lines\n\t\tspecial = jQuery.event.special[ type ] || {};\n\t\tif ( !onlyHandlers && special.trigger && special.trigger.apply( elem, data ) === false ) {\n\t\t\treturn;\n\t\t}\n\n\t\t// Determine event propagation path in advance, per W3C events spec (#9951)\n\t\t// Bubble up to document, then to window; watch for a global ownerDocument var (#9724)\n\t\tif ( !onlyHandlers && !special.noBubble && !jQuery.isWindow( elem ) ) {\n\n\t\t\tbubbleType = special.delegateType || type;\n\t\t\tif ( !rfocusMorph.test( bubbleType + type ) ) {\n\t\t\t\tcur = cur.parentNode;\n\t\t\t}\n\t\t\tfor ( ; cur; cur = cur.parentNode ) {\n\t\t\t\teventPath.push( cur );\n\t\t\t\ttmp = cur;\n\t\t\t}\n\n\t\t\t// Only add window if we got to document (e.g., not plain obj or detached DOM)\n\t\t\tif ( tmp === (elem.ownerDocument || document) ) {\n\t\t\t\teventPath.push( tmp.defaultView || tmp.parentWindow || window );\n\t\t\t}\n\t\t}\n\n\t\t// Fire handlers on the event path\n\t\ti = 0;\n\t\twhile ( (cur = eventPath[i++]) && !event.isPropagationStopped() ) {\n\n\t\t\tevent.type = i > 1 ?\n\t\t\t\tbubbleType :\n\t\t\t\tspecial.bindType || type;\n\n\t\t\t// jQuery handler\n\t\t\thandle = ( data_priv.get( cur, \"events\" ) || {} )[ event.type ] && data_priv.get( cur, \"handle\" );\n\t\t\tif ( handle ) {\n\t\t\t\thandle.apply( cur, data );\n\t\t\t}\n\n\t\t\t// Native handler\n\t\t\thandle = ontype && cur[ ontype ];\n\t\t\tif ( handle && jQuery.acceptData( cur ) && handle.apply && handle.apply( cur, data ) === false ) {\n\t\t\t\tevent.preventDefault();\n\t\t\t}\n\t\t}\n\t\tevent.type = type;\n\n\t\t// If nobody prevented the default action, do it now\n\t\tif ( !onlyHandlers && !event.isDefaultPrevented() ) {\n\n\t\t\tif ( (!special._default || special._default.apply( eventPath.pop(), data ) === false) &&\n\t\t\t\tjQuery.acceptData( elem ) ) {\n\n\t\t\t\t// Call a native DOM method on the target with the same name name as the event.\n\t\t\t\t// Don't do default actions on window, that's where global variables be (#6170)\n\t\t\t\tif ( ontype && jQuery.isFunction( elem[ type ] ) && !jQuery.isWindow( elem ) ) {\n\n\t\t\t\t\t// Don't re-trigger an onFOO event when we call its FOO() method\n\t\t\t\t\ttmp = elem[ ontype ];\n\n\t\t\t\t\tif ( tmp ) {\n\t\t\t\t\t\telem[ ontype ] = null;\n\t\t\t\t\t}\n\n\t\t\t\t\t// Prevent re-triggering of the same event, since we already bubbled it above\n\t\t\t\t\tjQuery.event.triggered = type;\n\t\t\t\t\telem[ type ]();\n\t\t\t\t\tjQuery.event.triggered = undefined;\n\n\t\t\t\t\tif ( tmp ) {\n\t\t\t\t\t\telem[ ontype ] = tmp;\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\treturn event.result;\n\t},\n\n\tdispatch: function( event ) {\n\n\t\t// Make a writable jQuery.Event from the native event object\n\t\tevent = jQuery.event.fix( event );\n\n\t\tvar i, j, ret, matched, handleObj,\n\t\t\thandlerQueue = [],\n\t\t\targs = core_slice.call( arguments ),\n\t\t\thandlers = ( data_priv.get( this, \"events\" ) || {} )[ event.type ] || [],\n\t\t\tspecial = jQuery.event.special[ event.type ] || {};\n\n\t\t// Use the fix-ed jQuery.Event rather than the (read-only) native event\n\t\targs[0] = event;\n\t\tevent.delegateTarget = this;\n\n\t\t// Call the preDispatch hook for the mapped type, and let it bail if desired\n\t\tif ( special.preDispatch && special.preDispatch.call( this, event ) === false ) {\n\t\t\treturn;\n\t\t}\n\n\t\t// Determine handlers\n\t\thandlerQueue = jQuery.event.handlers.call( this, event, handlers );\n\n\t\t// Run delegates first; they may want to stop propagation beneath us\n\t\ti = 0;\n\t\twhile ( (matched = handlerQueue[ i++ ]) && !event.isPropagationStopped() ) {\n\t\t\tevent.currentTarget = matched.elem;\n\n\t\t\tj = 0;\n\t\t\twhile ( (handleObj = matched.handlers[ j++ ]) && !event.isImmediatePropagationStopped() ) {\n\n\t\t\t\t// Triggered event must either 1) have no namespace, or\n\t\t\t\t// 2) have namespace(s) a subset or equal to those in the bound event (both can have no namespace).\n\t\t\t\tif ( !event.namespace_re || event.namespace_re.test( handleObj.namespace ) ) {\n\n\t\t\t\t\tevent.handleObj = handleObj;\n\t\t\t\t\tevent.data = handleObj.data;\n\n\t\t\t\t\tret = ( (jQuery.event.special[ handleObj.origType ] || {}).handle || handleObj.handler )\n\t\t\t\t\t\t\t.apply( matched.elem, args );\n\n\t\t\t\t\tif ( ret !== undefined ) {\n\t\t\t\t\t\tif ( (event.result = ret) === false ) {\n\t\t\t\t\t\t\tevent.preventDefault();\n\t\t\t\t\t\t\tevent.stopPropagation();\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\t// Call the postDispatch hook for the mapped type\n\t\tif ( special.postDispatch ) {\n\t\t\tspecial.postDispatch.call( this, event );\n\t\t}\n\n\t\treturn event.result;\n\t},\n\n\thandlers: function( event, handlers ) {\n\t\tvar i, matches, sel, handleObj,\n\t\t\thandlerQueue = [],\n\t\t\tdelegateCount = handlers.delegateCount,\n\t\t\tcur = event.target;\n\n\t\t// Find delegate handlers\n\t\t// Black-hole SVG <use> instance trees (#13180)\n\t\t// Avoid non-left-click bubbling in Firefox (#3861)\n\t\tif ( delegateCount && cur.nodeType && (!event.button || event.type !== \"click\") ) {\n\n\t\t\tfor ( ; cur !== this; cur = cur.parentNode || this ) {\n\n\t\t\t\t// Don't process clicks on disabled elements (#6911, #8165, #11382, #11764)\n\t\t\t\tif ( cur.disabled !== true || event.type !== \"click\" ) {\n\t\t\t\t\tmatches = [];\n\t\t\t\t\tfor ( i = 0; i < delegateCount; i++ ) {\n\t\t\t\t\t\thandleObj = handlers[ i ];\n\n\t\t\t\t\t\t// Don't conflict with Object.prototype properties (#13203)\n\t\t\t\t\t\tsel = handleObj.selector + \" \";\n\n\t\t\t\t\t\tif ( matches[ sel ] === undefined ) {\n\t\t\t\t\t\t\tmatches[ sel ] = handleObj.needsContext ?\n\t\t\t\t\t\t\t\tjQuery( sel, this ).index( cur ) >= 0 :\n\t\t\t\t\t\t\t\tjQuery.find( sel, this, null, [ cur ] ).length;\n\t\t\t\t\t\t}\n\t\t\t\t\t\tif ( matches[ sel ] ) {\n\t\t\t\t\t\t\tmatches.push( handleObj );\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tif ( matches.length ) {\n\t\t\t\t\t\thandlerQueue.push({ elem: cur, handlers: matches });\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\t// Add the remaining (directly-bound) handlers\n\t\tif ( delegateCount < handlers.length ) {\n\t\t\thandlerQueue.push({ elem: this, handlers: handlers.slice( delegateCount ) });\n\t\t}\n\n\t\treturn handlerQueue;\n\t},\n\n\t// Includes some event props shared by KeyEvent and MouseEvent\n\tprops: \"altKey bubbles cancelable ctrlKey currentTarget eventPhase metaKey relatedTarget shiftKey target timeStamp view which\".split(\" \"),\n\n\tfixHooks: {},\n\n\tkeyHooks: {\n\t\tprops: \"char charCode key keyCode\".split(\" \"),\n\t\tfilter: function( event, original ) {\n\n\t\t\t// Add which for key events\n\t\t\tif ( event.which == null ) {\n\t\t\t\tevent.which = original.charCode != null ? original.charCode : original.keyCode;\n\t\t\t}\n\n\t\t\treturn event;\n\t\t}\n\t},\n\n\tmouseHooks: {\n\t\tprops: \"button buttons clientX clientY offsetX offsetY pageX pageY screenX screenY toElement\".split(\" \"),\n\t\tfilter: function( event, original ) {\n\t\t\tvar eventDoc, doc, body,\n\t\t\t\tbutton = original.button;\n\n\t\t\t// Calculate pageX/Y if missing and clientX/Y available\n\t\t\tif ( event.pageX == null && original.clientX != null ) {\n\t\t\t\teventDoc = event.target.ownerDocument || document;\n\t\t\t\tdoc = eventDoc.documentElement;\n\t\t\t\tbody = eventDoc.body;\n\n\t\t\t\tevent.pageX = original.clientX + ( doc && doc.scrollLeft || body && body.scrollLeft || 0 ) - ( doc && doc.clientLeft || body && body.clientLeft || 0 );\n\t\t\t\tevent.pageY = original.clientY + ( doc && doc.scrollTop || body && body.scrollTop || 0 ) - ( doc && doc.clientTop || body && body.clientTop || 0 );\n\t\t\t}\n\n\t\t\t// Add which for click: 1 === left; 2 === middle; 3 === right\n\t\t\t// Note: button is not normalized, so don't use it\n\t\t\tif ( !event.which && button !== undefined ) {\n\t\t\t\tevent.which = ( button & 1 ? 1 : ( button & 2 ? 3 : ( button & 4 ? 2 : 0 ) ) );\n\t\t\t}\n\n\t\t\treturn event;\n\t\t}\n\t},\n\n\tfix: function( event ) {\n\t\tif ( event[ jQuery.expando ] ) {\n\t\t\treturn event;\n\t\t}\n\n\t\t// Create a writable copy of the event object and normalize some properties\n\t\tvar i, prop, copy,\n\t\t\ttype = event.type,\n\t\t\toriginalEvent = event,\n\t\t\tfixHook = this.fixHooks[ type ];\n\n\t\tif ( !fixHook ) {\n\t\t\tthis.fixHooks[ type ] = fixHook =\n\t\t\t\trmouseEvent.test( type ) ? this.mouseHooks :\n\t\t\t\trkeyEvent.test( type ) ? this.keyHooks :\n\t\t\t\t{};\n\t\t}\n\t\tcopy = fixHook.props ? this.props.concat( fixHook.props ) : this.props;\n\n\t\tevent = new jQuery.Event( originalEvent );\n\n\t\ti = copy.length;\n\t\twhile ( i-- ) {\n\t\t\tprop = copy[ i ];\n\t\t\tevent[ prop ] = originalEvent[ prop ];\n\t\t}\n\n\t\t// Support: Safari 6.0+, Chrome < 28\n\t\t// Target should not be a text node (#504, #13143)\n\t\tif ( event.target.nodeType === 3 ) {\n\t\t\tevent.target = event.target.parentNode;\n\t\t}\n\n\t\treturn fixHook.filter? fixHook.filter( event, originalEvent ) : event;\n\t},\n\n\tspecial: {\n\t\tload: {\n\t\t\t// Prevent triggered image.load events from bubbling to window.load\n\t\t\tnoBubble: true\n\t\t},\n\t\tfocus: {\n\t\t\t// Fire native event if possible so blur/focus sequence is correct\n\t\t\ttrigger: function() {\n\t\t\t\tif ( this !== safeActiveElement() && this.focus ) {\n\t\t\t\t\tthis.focus();\n\t\t\t\t\treturn false;\n\t\t\t\t}\n\t\t\t},\n\t\t\tdelegateType: \"focusin\"\n\t\t},\n\t\tblur: {\n\t\t\ttrigger: function() {\n\t\t\t\tif ( this === safeActiveElement() && this.blur ) {\n\t\t\t\t\tthis.blur();\n\t\t\t\t\treturn false;\n\t\t\t\t}\n\t\t\t},\n\t\t\tdelegateType: \"focusout\"\n\t\t},\n\t\tclick: {\n\t\t\t// For checkbox, fire native event so checked state will be right\n\t\t\ttrigger: function() {\n\t\t\t\tif ( this.type === \"checkbox\" && this.click && jQuery.nodeName( this, \"input\" ) ) {\n\t\t\t\t\tthis.click();\n\t\t\t\t\treturn false;\n\t\t\t\t}\n\t\t\t},\n\n\t\t\t// For cross-browser consistency, don't fire native .click() on links\n\t\t\t_default: function( event ) {\n\t\t\t\treturn jQuery.nodeName( event.target, \"a\" );\n\t\t\t}\n\t\t},\n\n\t\tbeforeunload: {\n\t\t\tpostDispatch: function( event ) {\n\n\t\t\t\t// Support: Firefox 20+\n\t\t\t\t// Firefox doesn't alert if the returnValue field is not set.\n\t\t\t\tif ( event.result !== undefined ) {\n\t\t\t\t\tevent.originalEvent.returnValue = event.result;\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t},\n\n\tsimulate: function( type, elem, event, bubble ) {\n\t\t// Piggyback on a donor event to simulate a different one.\n\t\t// Fake originalEvent to avoid donor's stopPropagation, but if the\n\t\t// simulated event prevents default then we do the same on the donor.\n\t\tvar e = jQuery.extend(\n\t\t\tnew jQuery.Event(),\n\t\t\tevent,\n\t\t\t{\n\t\t\t\ttype: type,\n\t\t\t\tisSimulated: true,\n\t\t\t\toriginalEvent: {}\n\t\t\t}\n\t\t);\n\t\tif ( bubble ) {\n\t\t\tjQuery.event.trigger( e, null, elem );\n\t\t} else {\n\t\t\tjQuery.event.dispatch.call( elem, e );\n\t\t}\n\t\tif ( e.isDefaultPrevented() ) {\n\t\t\tevent.preventDefault();\n\t\t}\n\t}\n};\n\njQuery.removeEvent = function( elem, type, handle ) {\n\tif ( elem.removeEventListener ) {\n\t\telem.removeEventListener( type, handle, false );\n\t}\n};\n\njQuery.Event = function( src, props ) {\n\t// Allow instantiation without the 'new' keyword\n\tif ( !(this instanceof jQuery.Event) ) {\n\t\treturn new jQuery.Event( src, props );\n\t}\n\n\t// Event object\n\tif ( src && src.type ) {\n\t\tthis.originalEvent = src;\n\t\tthis.type = src.type;\n\n\t\t// Events bubbling up the document may have been marked as prevented\n\t\t// by a handler lower down the tree; reflect the correct value.\n\t\tthis.isDefaultPrevented = ( src.defaultPrevented ||\n\t\t\tsrc.getPreventDefault && src.getPreventDefault() ) ? returnTrue : returnFalse;\n\n\t// Event type\n\t} else {\n\t\tthis.type = src;\n\t}\n\n\t// Put explicitly provided properties onto the event object\n\tif ( props ) {\n\t\tjQuery.extend( this, props );\n\t}\n\n\t// Create a timestamp if incoming event doesn't have one\n\tthis.timeStamp = src && src.timeStamp || jQuery.now();\n\n\t// Mark it as fixed\n\tthis[ jQuery.expando ] = true;\n};\n\n// jQuery.Event is based on DOM3 Events as specified by the ECMAScript Language Binding\n// http://www.w3.org/TR/2003/WD-DOM-Level-3-Events-20030331/ecma-script-binding.html\njQuery.Event.prototype = {\n\tisDefaultPrevented: returnFalse,\n\tisPropagationStopped: returnFalse,\n\tisImmediatePropagationStopped: returnFalse,\n\n\tpreventDefault: function() {\n\t\tvar e = this.originalEvent;\n\n\t\tthis.isDefaultPrevented = returnTrue;\n\n\t\tif ( e && e.preventDefault ) {\n\t\t\te.preventDefault();\n\t\t}\n\t},\n\tstopPropagation: function() {\n\t\tvar e = this.originalEvent;\n\n\t\tthis.isPropagationStopped = returnTrue;\n\n\t\tif ( e && e.stopPropagation ) {\n\t\t\te.stopPropagation();\n\t\t}\n\t},\n\tstopImmediatePropagation: function() {\n\t\tthis.isImmediatePropagationStopped = returnTrue;\n\t\tthis.stopPropagation();\n\t}\n};\n\n// Create mouseenter/leave events using mouseover/out and event-time checks\n// Support: Chrome 15+\njQuery.each({\n\tmouseenter: \"mouseover\",\n\tmouseleave: \"mouseout\"\n}, function( orig, fix ) {\n\tjQuery.event.special[ orig ] = {\n\t\tdelegateType: fix,\n\t\tbindType: fix,\n\n\t\thandle: function( event ) {\n\t\t\tvar ret,\n\t\t\t\ttarget = this,\n\t\t\t\trelated = event.relatedTarget,\n\t\t\t\thandleObj = event.handleObj;\n\n\t\t\t// For mousenter/leave call the handler if related is outside the target.\n\t\t\t// NB: No relatedTarget if the mouse left/entered the browser window\n\t\t\tif ( !related || (related !== target && !jQuery.contains( target, related )) ) {\n\t\t\t\tevent.type = handleObj.origType;\n\t\t\t\tret = handleObj.handler.apply( this, arguments );\n\t\t\t\tevent.type = fix;\n\t\t\t}\n\t\t\treturn ret;\n\t\t}\n\t};\n});\n\n// Create \"bubbling\" focus and blur events\n// Support: Firefox, Chrome, Safari\nif ( !jQuery.support.focusinBubbles ) {\n\tjQuery.each({ focus: \"focusin\", blur: \"focusout\" }, function( orig, fix ) {\n\n\t\t// Attach a single capturing handler while someone wants focusin/focusout\n\t\tvar attaches = 0,\n\t\t\thandler = function( event ) {\n\t\t\t\tjQuery.event.simulate( fix, event.target, jQuery.event.fix( event ), true );\n\t\t\t};\n\n\t\tjQuery.event.special[ fix ] = {\n\t\t\tsetup: function() {\n\t\t\t\tif ( attaches++ === 0 ) {\n\t\t\t\t\tdocument.addEventListener( orig, handler, true );\n\t\t\t\t}\n\t\t\t},\n\t\t\tteardown: function() {\n\t\t\t\tif ( --attaches === 0 ) {\n\t\t\t\t\tdocument.removeEventListener( orig, handler, true );\n\t\t\t\t}\n\t\t\t}\n\t\t};\n\t});\n}\n\njQuery.fn.extend({\n\n\ton: function( types, selector, data, fn, /*INTERNAL*/ one ) {\n\t\tvar origFn, type;\n\n\t\t// Types can be a map of types/handlers\n\t\tif ( typeof types === \"object\" ) {\n\t\t\t// ( types-Object, selector, data )\n\t\t\tif ( typeof selector !== \"string\" ) {\n\t\t\t\t// ( types-Object, data )\n\t\t\t\tdata = data || selector;\n\t\t\t\tselector = undefined;\n\t\t\t}\n\t\t\tfor ( type in types ) {\n\t\t\t\tthis.on( type, selector, data, types[ type ], one );\n\t\t\t}\n\t\t\treturn this;\n\t\t}\n\n\t\tif ( data == null && fn == null ) {\n\t\t\t// ( types, fn )\n\t\t\tfn = selector;\n\t\t\tdata = selector = undefined;\n\t\t} else if ( fn == null ) {\n\t\t\tif ( typeof selector === \"string\" ) {\n\t\t\t\t// ( types, selector, fn )\n\t\t\t\tfn = data;\n\t\t\t\tdata = undefined;\n\t\t\t} else {\n\t\t\t\t// ( types, data, fn )\n\t\t\t\tfn = data;\n\t\t\t\tdata = selector;\n\t\t\t\tselector = undefined;\n\t\t\t}\n\t\t}\n\t\tif ( fn === false ) {\n\t\t\tfn = returnFalse;\n\t\t} else if ( !fn ) {\n\t\t\treturn this;\n\t\t}\n\n\t\tif ( one === 1 ) {\n\t\t\torigFn = fn;\n\t\t\tfn = function( event ) {\n\t\t\t\t// Can use an empty set, since event contains the info\n\t\t\t\tjQuery().off( event );\n\t\t\t\treturn origFn.apply( this, arguments );\n\t\t\t};\n\t\t\t// Use same guid so caller can remove using origFn\n\t\t\tfn.guid = origFn.guid || ( origFn.guid = jQuery.guid++ );\n\t\t}\n\t\treturn this.each( function() {\n\t\t\tjQuery.event.add( this, types, fn, data, selector );\n\t\t});\n\t},\n\tone: function( types, selector, data, fn ) {\n\t\treturn this.on( types, selector, data, fn, 1 );\n\t},\n\toff: function( types, selector, fn ) {\n\t\tvar handleObj, type;\n\t\tif ( types && types.preventDefault && types.handleObj ) {\n\t\t\t// ( event ) dispatched jQuery.Event\n\t\t\thandleObj = types.handleObj;\n\t\t\tjQuery( types.delegateTarget ).off(\n\t\t\t\thandleObj.namespace ? handleObj.origType + \".\" + handleObj.namespace : handleObj.origType,\n\t\t\t\thandleObj.selector,\n\t\t\t\thandleObj.handler\n\t\t\t);\n\t\t\treturn this;\n\t\t}\n\t\tif ( typeof types === \"object\" ) {\n\t\t\t// ( types-object [, selector] )\n\t\t\tfor ( type in types ) {\n\t\t\t\tthis.off( type, selector, types[ type ] );\n\t\t\t}\n\t\t\treturn this;\n\t\t}\n\t\tif ( selector === false || typeof selector === \"function\" ) {\n\t\t\t// ( types [, fn] )\n\t\t\tfn = selector;\n\t\t\tselector = undefined;\n\t\t}\n\t\tif ( fn === false ) {\n\t\t\tfn = returnFalse;\n\t\t}\n\t\treturn this.each(function() {\n\t\t\tjQuery.event.remove( this, types, fn, selector );\n\t\t});\n\t},\n\n\ttrigger: function( type, data ) {\n\t\treturn this.each(function() {\n\t\t\tjQuery.event.trigger( type, data, this );\n\t\t});\n\t},\n\ttriggerHandler: function( type, data ) {\n\t\tvar elem = this[0];\n\t\tif ( elem ) {\n\t\t\treturn jQuery.event.trigger( type, data, elem, true );\n\t\t}\n\t}\n});\nvar isSimple = /^.[^:#\\[\\.,]*$/,\n\trneedsContext = jQuery.expr.match.needsContext,\n\t// methods guaranteed to produce a unique set when starting from a unique set\n\tguaranteedUnique = {\n\t\tchildren: true,\n\t\tcontents: true,\n\t\tnext: true,\n\t\tprev: true\n\t};\n\njQuery.fn.extend({\n\tfind: function( selector ) {\n\t\tvar self, matched, i,\n\t\t\tl = this.length;\n\n\t\tif ( typeof selector !== \"string\" ) {\n\t\t\tself = this;\n\t\t\treturn this.pushStack( jQuery( selector ).filter(function() {\n\t\t\t\tfor ( i = 0; i < l; i++ ) {\n\t\t\t\t\tif ( jQuery.contains( self[ i ], this ) ) {\n\t\t\t\t\t\treturn true;\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}) );\n\t\t}\n\n\t\tmatched = [];\n\t\tfor ( i = 0; i < l; i++ ) {\n\t\t\tjQuery.find( selector, this[ i ], matched );\n\t\t}\n\n\t\t// Needed because $( selector, context ) becomes $( context ).find( selector )\n\t\tmatched = this.pushStack( l > 1 ? jQuery.unique( matched ) : matched );\n\t\tmatched.selector = ( this.selector ? this.selector + \" \" : \"\" ) + selector;\n\t\treturn matched;\n\t},\n\n\thas: function( target ) {\n\t\tvar targets = jQuery( target, this ),\n\t\t\tl = targets.length;\n\n\t\treturn this.filter(function() {\n\t\t\tvar i = 0;\n\t\t\tfor ( ; i < l; i++ ) {\n\t\t\t\tif ( jQuery.contains( this, targets[i] ) ) {\n\t\t\t\t\treturn true;\n\t\t\t\t}\n\t\t\t}\n\t\t});\n\t},\n\n\tnot: function( selector ) {\n\t\treturn this.pushStack( winnow(this, selector || [], true) );\n\t},\n\n\tfilter: function( selector ) {\n\t\treturn this.pushStack( winnow(this, selector || [], false) );\n\t},\n\n\tis: function( selector ) {\n\t\treturn !!selector && (\n\t\t\ttypeof selector === \"string\" ?\n\t\t\t\t// If this is a positional/relative selector, check membership in the returned set\n\t\t\t\t// so $(\"p:first\").is(\"p:last\") won't return true for a doc with two \"p\".\n\t\t\t\trneedsContext.test( selector ) ?\n\t\t\t\t\tjQuery( selector, this.context ).index( this[ 0 ] ) >= 0 :\n\t\t\t\t\tjQuery.filter( selector, this ).length > 0 :\n\t\t\t\tthis.filter( selector ).length > 0 );\n\t},\n\n\tclosest: function( selectors, context ) {\n\t\tvar cur,\n\t\t\ti = 0,\n\t\t\tl = this.length,\n\t\t\tmatched = [],\n\t\t\tpos = ( rneedsContext.test( selectors ) || typeof selectors !== \"string\" ) ?\n\t\t\t\tjQuery( selectors, context || this.context ) :\n\t\t\t\t0;\n\n\t\tfor ( ; i < l; i++ ) {\n\t\t\tfor ( cur = this[i]; cur && cur !== context; cur = cur.parentNode ) {\n\t\t\t\t// Always skip document fragments\n\t\t\t\tif ( cur.nodeType < 11 && (pos ?\n\t\t\t\t\tpos.index(cur) > -1 :\n\n\t\t\t\t\t// Don't pass non-elements to Sizzle\n\t\t\t\t\tcur.nodeType === 1 &&\n\t\t\t\t\t\tjQuery.find.matchesSelector(cur, selectors)) ) {\n\n\t\t\t\t\tcur = matched.push( cur );\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\treturn this.pushStack( matched.length > 1 ? jQuery.unique( matched ) : matched );\n\t},\n\n\t// Determine the position of an element within\n\t// the matched set of elements\n\tindex: function( elem ) {\n\n\t\t// No argument, return index in parent\n\t\tif ( !elem ) {\n\t\t\treturn ( this[ 0 ] && this[ 0 ].parentNode ) ? this.first().prevAll().length : -1;\n\t\t}\n\n\t\t// index in selector\n\t\tif ( typeof elem === \"string\" ) {\n\t\t\treturn core_indexOf.call( jQuery( elem ), this[ 0 ] );\n\t\t}\n\n\t\t// Locate the position of the desired element\n\t\treturn core_indexOf.call( this,\n\n\t\t\t// If it receives a jQuery object, the first element is used\n\t\t\telem.jquery ? elem[ 0 ] : elem\n\t\t);\n\t},\n\n\tadd: function( selector, context ) {\n\t\tvar set = typeof selector === \"string\" ?\n\t\t\t\tjQuery( selector, context ) :\n\t\t\t\tjQuery.makeArray( selector && selector.nodeType ? [ selector ] : selector ),\n\t\t\tall = jQuery.merge( this.get(), set );\n\n\t\treturn this.pushStack( jQuery.unique(all) );\n\t},\n\n\taddBack: function( selector ) {\n\t\treturn this.add( selector == null ?\n\t\t\tthis.prevObject : this.prevObject.filter(selector)\n\t\t);\n\t}\n});\n\nfunction sibling( cur, dir ) {\n\twhile ( (cur = cur[dir]) && cur.nodeType !== 1 ) {}\n\n\treturn cur;\n}\n\njQuery.each({\n\tparent: function( elem ) {\n\t\tvar parent = elem.parentNode;\n\t\treturn parent && parent.nodeType !== 11 ? parent : null;\n\t},\n\tparents: function( elem ) {\n\t\treturn jQuery.dir( elem, \"parentNode\" );\n\t},\n\tparentsUntil: function( elem, i, until ) {\n\t\treturn jQuery.dir( elem, \"parentNode\", until );\n\t},\n\tnext: function( elem ) {\n\t\treturn sibling( elem, \"nextSibling\" );\n\t},\n\tprev: function( elem ) {\n\t\treturn sibling( elem, \"previousSibling\" );\n\t},\n\tnextAll: function( elem ) {\n\t\treturn jQuery.dir( elem, \"nextSibling\" );\n\t},\n\tprevAll: function( elem ) {\n\t\treturn jQuery.dir( elem, \"previousSibling\" );\n\t},\n\tnextUntil: function( elem, i, until ) {\n\t\treturn jQuery.dir( elem, \"nextSibling\", until );\n\t},\n\tprevUntil: function( elem, i, until ) {\n\t\treturn jQuery.dir( elem, \"previousSibling\", until );\n\t},\n\tsiblings: function( elem ) {\n\t\treturn jQuery.sibling( ( elem.parentNode || {} ).firstChild, elem );\n\t},\n\tchildren: function( elem ) {\n\t\treturn jQuery.sibling( elem.firstChild );\n\t},\n\tcontents: function( elem ) {\n\t\treturn jQuery.nodeName( elem, \"iframe\" ) ?\n\t\t\telem.contentDocument || elem.contentWindow.document :\n\t\t\tjQuery.merge( [], elem.childNodes );\n\t}\n}, function( name, fn ) {\n\tjQuery.fn[ name ] = function( until, selector ) {\n\t\tvar matched = jQuery.map( this, fn, until );\n\n\t\tif ( name.slice( -5 ) !== \"Until\" ) {\n\t\t\tselector = until;\n\t\t}\n\n\t\tif ( selector && typeof selector === \"string\" ) {\n\t\t\tmatched = jQuery.filter( selector, matched );\n\t\t}\n\n\t\tif ( this.length > 1 ) {\n\t\t\t// Remove duplicates\n\t\t\tif ( !guaranteedUnique[ name ] ) {\n\t\t\t\tjQuery.unique( matched );\n\t\t\t}\n\n\t\t\t// Reverse order for parents* and prev*\n\t\t\tif ( name[ 0 ] === \"p\" ) {\n\t\t\t\tmatched.reverse();\n\t\t\t}\n\t\t}\n\n\t\treturn this.pushStack( matched );\n\t};\n});\n\njQuery.extend({\n\tfilter: function( expr, elems, not ) {\n\t\tvar elem = elems[ 0 ];\n\n\t\tif ( not ) {\n\t\t\texpr = \":not(\" + expr + \")\";\n\t\t}\n\n\t\treturn elems.length === 1 && elem.nodeType === 1 ?\n\t\t\tjQuery.find.matchesSelector( elem, expr ) ? [ elem ] : [] :\n\t\t\tjQuery.find.matches( expr, jQuery.grep( elems, function( elem ) {\n\t\t\t\treturn elem.nodeType === 1;\n\t\t\t}));\n\t},\n\n\tdir: function( elem, dir, until ) {\n\t\tvar matched = [],\n\t\t\ttruncate = until !== undefined;\n\n\t\twhile ( (elem = elem[ dir ]) && elem.nodeType !== 9 ) {\n\t\t\tif ( elem.nodeType === 1 ) {\n\t\t\t\tif ( truncate && jQuery( elem ).is( until ) ) {\n\t\t\t\t\tbreak;\n\t\t\t\t}\n\t\t\t\tmatched.push( elem );\n\t\t\t}\n\t\t}\n\t\treturn matched;\n\t},\n\n\tsibling: function( n, elem ) {\n\t\tvar matched = [];\n\n\t\tfor ( ; n; n = n.nextSibling ) {\n\t\t\tif ( n.nodeType === 1 && n !== elem ) {\n\t\t\t\tmatched.push( n );\n\t\t\t}\n\t\t}\n\n\t\treturn matched;\n\t}\n});\n\n// Implement the identical functionality for filter and not\nfunction winnow( elements, qualifier, not ) {\n\tif ( jQuery.isFunction( qualifier ) ) {\n\t\treturn jQuery.grep( elements, function( elem, i ) {\n\t\t\t/* jshint -W018 */\n\t\t\treturn !!qualifier.call( elem, i, elem ) !== not;\n\t\t});\n\n\t}\n\n\tif ( qualifier.nodeType ) {\n\t\treturn jQuery.grep( elements, function( elem ) {\n\t\t\treturn ( elem === qualifier ) !== not;\n\t\t});\n\n\t}\n\n\tif ( typeof qualifier === \"string\" ) {\n\t\tif ( isSimple.test( qualifier ) ) {\n\t\t\treturn jQuery.filter( qualifier, elements, not );\n\t\t}\n\n\t\tqualifier = jQuery.filter( qualifier, elements );\n\t}\n\n\treturn jQuery.grep( elements, function( elem ) {\n\t\treturn ( core_indexOf.call( qualifier, elem ) >= 0 ) !== not;\n\t});\n}\nvar rxhtmlTag = /<(?!area|br|col|embed|hr|img|input|link|meta|param)(([\\w:]+)[^>]*)\\/>/gi,\n\trtagName = /<([\\w:]+)/,\n\trhtml = /<|&#?\\w+;/,\n\trnoInnerhtml = /<(?:script|style|link)/i,\n\tmanipulation_rcheckableType = /^(?:checkbox|radio)$/i,\n\t// checked=\"checked\" or checked\n\trchecked = /checked\\s*(?:[^=]|=\\s*.checked.)/i,\n\trscriptType = /^$|\\/(?:java|ecma)script/i,\n\trscriptTypeMasked = /^true\\/(.*)/,\n\trcleanScript = /^\\s*<!(?:\\[CDATA\\[|--)|(?:\\]\\]|--)>\\s*$/g,\n\n\t// We have to close these tags to support XHTML (#13200)\n\twrapMap = {\n\n\t\t// Support: IE 9\n\t\toption: [ 1, \"<select multiple='multiple'>\", \"</select>\" ],\n\n\t\tthead: [ 1, \"<table>\", \"</table>\" ],\n\t\ttr: [ 2, \"<table><tbody>\", \"</tbody></table>\" ],\n\t\ttd: [ 3, \"<table><tbody><tr>\", \"</tr></tbody></table>\" ],\n\n\t\t_default: [ 0, \"\", \"\" ]\n\t};\n\n// Support: IE 9\nwrapMap.optgroup = wrapMap.option;\n\nwrapMap.tbody = wrapMap.tfoot = wrapMap.colgroup = wrapMap.caption = wrapMap.col = wrapMap.thead;\nwrapMap.th = wrapMap.td;\n\njQuery.fn.extend({\n\ttext: function( value ) {\n\t\treturn jQuery.access( this, function( value ) {\n\t\t\treturn value === undefined ?\n\t\t\t\tjQuery.text( this ) :\n\t\t\t\tthis.empty().append( ( this[ 0 ] && this[ 0 ].ownerDocument || document ).createTextNode( value ) );\n\t\t}, null, value, arguments.length );\n\t},\n\n\tappend: function() {\n\t\treturn this.domManip( arguments, function( elem ) {\n\t\t\tif ( this.nodeType === 1 || this.nodeType === 11 || this.nodeType === 9 ) {\n\t\t\t\tvar target = manipulationTarget( this, elem );\n\t\t\t\ttarget.appendChild( elem );\n\t\t\t}\n\t\t});\n\t},\n\n\tprepend: function() {\n\t\treturn this.domManip( arguments, function( elem ) {\n\t\t\tif ( this.nodeType === 1 || this.nodeType === 11 || this.nodeType === 9 ) {\n\t\t\t\tvar target = manipulationTarget( this, elem );\n\t\t\t\ttarget.insertBefore( elem, target.firstChild );\n\t\t\t}\n\t\t});\n\t},\n\n\tbefore: function() {\n\t\treturn this.domManip( arguments, function( elem ) {\n\t\t\tif ( this.parentNode ) {\n\t\t\t\tthis.parentNode.insertBefore( elem, this );\n\t\t\t}\n\t\t});\n\t},\n\n\tafter: function() {\n\t\treturn this.domManip( arguments, function( elem ) {\n\t\t\tif ( this.parentNode ) {\n\t\t\t\tthis.parentNode.insertBefore( elem, this.nextSibling );\n\t\t\t}\n\t\t});\n\t},\n\n\t// keepData is for internal use only--do not document\n\tremove: function( selector, keepData ) {\n\t\tvar elem,\n\t\t\telems = selector ? jQuery.filter( selector, this ) : this,\n\t\t\ti = 0;\n\n\t\tfor ( ; (elem = elems[i]) != null; i++ ) {\n\t\t\tif ( !keepData && elem.nodeType === 1 ) {\n\t\t\t\tjQuery.cleanData( getAll( elem ) );\n\t\t\t}\n\n\t\t\tif ( elem.parentNode ) {\n\t\t\t\tif ( keepData && jQuery.contains( elem.ownerDocument, elem ) ) {\n\t\t\t\t\tsetGlobalEval( getAll( elem, \"script\" ) );\n\t\t\t\t}\n\t\t\t\telem.parentNode.removeChild( elem );\n\t\t\t}\n\t\t}\n\n\t\treturn this;\n\t},\n\n\tempty: function() {\n\t\tvar elem,\n\t\t\ti = 0;\n\n\t\tfor ( ; (elem = this[i]) != null; i++ ) {\n\t\t\tif ( elem.nodeType === 1 ) {\n\n\t\t\t\t// Prevent memory leaks\n\t\t\t\tjQuery.cleanData( getAll( elem, false ) );\n\n\t\t\t\t// Remove any remaining nodes\n\t\t\t\telem.textContent = \"\";\n\t\t\t}\n\t\t}\n\n\t\treturn this;\n\t},\n\n\tclone: function( dataAndEvents, deepDataAndEvents ) {\n\t\tdataAndEvents = dataAndEvents == null ? false : dataAndEvents;\n\t\tdeepDataAndEvents = deepDataAndEvents == null ? dataAndEvents : deepDataAndEvents;\n\n\t\treturn this.map( function () {\n\t\t\treturn jQuery.clone( this, dataAndEvents, deepDataAndEvents );\n\t\t});\n\t},\n\n\thtml: function( value ) {\n\t\treturn jQuery.access( this, function( value ) {\n\t\t\tvar elem = this[ 0 ] || {},\n\t\t\t\ti = 0,\n\t\t\t\tl = this.length;\n\n\t\t\tif ( value === undefined && elem.nodeType === 1 ) {\n\t\t\t\treturn elem.innerHTML;\n\t\t\t}\n\n\t\t\t// See if we can take a shortcut and just use innerHTML\n\t\t\tif ( typeof value === \"string\" && !rnoInnerhtml.test( value ) &&\n\t\t\t\t!wrapMap[ ( rtagName.exec( value ) || [ \"\", \"\" ] )[ 1 ].toLowerCase() ] ) {\n\n\t\t\t\tvalue = value.replace( rxhtmlTag, \"<$1></$2>\" );\n\n\t\t\t\ttry {\n\t\t\t\t\tfor ( ; i < l; i++ ) {\n\t\t\t\t\t\telem = this[ i ] || {};\n\n\t\t\t\t\t\t// Remove element nodes and prevent memory leaks\n\t\t\t\t\t\tif ( elem.nodeType === 1 ) {\n\t\t\t\t\t\t\tjQuery.cleanData( getAll( elem, false ) );\n\t\t\t\t\t\t\telem.innerHTML = value;\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\n\t\t\t\t\telem = 0;\n\n\t\t\t\t// If using innerHTML throws an exception, use the fallback method\n\t\t\t\t} catch( e ) {}\n\t\t\t}\n\n\t\t\tif ( elem ) {\n\t\t\t\tthis.empty().append( value );\n\t\t\t}\n\t\t}, null, value, arguments.length );\n\t},\n\n\treplaceWith: function() {\n\t\tvar\n\t\t\t// Snapshot the DOM in case .domManip sweeps something relevant into its fragment\n\t\t\targs = jQuery.map( this, function( elem ) {\n\t\t\t\treturn [ elem.nextSibling, elem.parentNode ];\n\t\t\t}),\n\t\t\ti = 0;\n\n\t\t// Make the changes, replacing each context element with the new content\n\t\tthis.domManip( arguments, function( elem ) {\n\t\t\tvar next = args[ i++ ],\n\t\t\t\tparent = args[ i++ ];\n\n\t\t\tif ( parent ) {\n\t\t\t\tjQuery( this ).remove();\n\t\t\t\tparent.insertBefore( elem, next );\n\t\t\t}\n\t\t// Allow new content to include elements from the context set\n\t\t}, true );\n\n\t\t// Force removal if there was no new content (e.g., from empty arguments)\n\t\treturn i ? this : this.remove();\n\t},\n\n\tdetach: function( selector ) {\n\t\treturn this.remove( selector, true );\n\t},\n\n\tdomManip: function( args, callback, allowIntersection ) {\n\n\t\t// Flatten any nested arrays\n\t\targs = core_concat.apply( [], args );\n\n\t\tvar fragment, first, scripts, hasScripts, node, doc,\n\t\t\ti = 0,\n\t\t\tl = this.length,\n\t\t\tset = this,\n\t\t\tiNoClone = l - 1,\n\t\t\tvalue = args[ 0 ],\n\t\t\tisFunction = jQuery.isFunction( value );\n\n\t\t// We can't cloneNode fragments that contain checked, in WebKit\n\t\tif ( isFunction || !( l <= 1 || typeof value !== \"string\" || jQuery.support.checkClone || !rchecked.test( value ) ) ) {\n\t\t\treturn this.each(function( index ) {\n\t\t\t\tvar self = set.eq( index );\n\t\t\t\tif ( isFunction ) {\n\t\t\t\t\targs[ 0 ] = value.call( this, index, self.html() );\n\t\t\t\t}\n\t\t\t\tself.domManip( args, callback, allowIntersection );\n\t\t\t});\n\t\t}\n\n\t\tif ( l ) {\n\t\t\tfragment = jQuery.buildFragment( args, this[ 0 ].ownerDocument, false, !allowIntersection && this );\n\t\t\tfirst = fragment.firstChild;\n\n\t\t\tif ( fragment.childNodes.length === 1 ) {\n\t\t\t\tfragment = first;\n\t\t\t}\n\n\t\t\tif ( first ) {\n\t\t\t\tscripts = jQuery.map( getAll( fragment, \"script\" ), disableScript );\n\t\t\t\thasScripts = scripts.length;\n\n\t\t\t\t// Use the original fragment for the last item instead of the first because it can end up\n\t\t\t\t// being emptied incorrectly in certain situations (#8070).\n\t\t\t\tfor ( ; i < l; i++ ) {\n\t\t\t\t\tnode = fragment;\n\n\t\t\t\t\tif ( i !== iNoClone ) {\n\t\t\t\t\t\tnode = jQuery.clone( node, true, true );\n\n\t\t\t\t\t\t// Keep references to cloned scripts for later restoration\n\t\t\t\t\t\tif ( hasScripts ) {\n\t\t\t\t\t\t\t// Support: QtWebKit\n\t\t\t\t\t\t\t// jQuery.merge because core_push.apply(_, arraylike) throws\n\t\t\t\t\t\t\tjQuery.merge( scripts, getAll( node, \"script\" ) );\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\n\t\t\t\t\tcallback.call( this[ i ], node, i );\n\t\t\t\t}\n\n\t\t\t\tif ( hasScripts ) {\n\t\t\t\t\tdoc = scripts[ scripts.length - 1 ].ownerDocument;\n\n\t\t\t\t\t// Reenable scripts\n\t\t\t\t\tjQuery.map( scripts, restoreScript );\n\n\t\t\t\t\t// Evaluate executable scripts on first document insertion\n\t\t\t\t\tfor ( i = 0; i < hasScripts; i++ ) {\n\t\t\t\t\t\tnode = scripts[ i ];\n\t\t\t\t\t\tif ( rscriptType.test( node.type || \"\" ) &&\n\t\t\t\t\t\t\t!data_priv.access( node, \"globalEval\" ) && jQuery.contains( doc, node ) ) {\n\n\t\t\t\t\t\t\tif ( node.src ) {\n\t\t\t\t\t\t\t\t// Hope ajax is available...\n\t\t\t\t\t\t\t\tjQuery._evalUrl( node.src );\n\t\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\t\tjQuery.globalEval( node.textContent.replace( rcleanScript, \"\" ) );\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\treturn this;\n\t}\n});\n\njQuery.each({\n\tappendTo: \"append\",\n\tprependTo: \"prepend\",\n\tinsertBefore: \"before\",\n\tinsertAfter: \"after\",\n\treplaceAll: \"replaceWith\"\n}, function( name, original ) {\n\tjQuery.fn[ name ] = function( selector ) {\n\t\tvar elems,\n\t\t\tret = [],\n\t\t\tinsert = jQuery( selector ),\n\t\t\tlast = insert.length - 1,\n\t\t\ti = 0;\n\n\t\tfor ( ; i <= last; i++ ) {\n\t\t\telems = i === last ? this : this.clone( true );\n\t\t\tjQuery( insert[ i ] )[ original ]( elems );\n\n\t\t\t// Support: QtWebKit\n\t\t\t// .get() because core_push.apply(_, arraylike) throws\n\t\t\tcore_push.apply( ret, elems.get() );\n\t\t}\n\n\t\treturn this.pushStack( ret );\n\t};\n});\n\njQuery.extend({\n\tclone: function( elem, dataAndEvents, deepDataAndEvents ) {\n\t\tvar i, l, srcElements, destElements,\n\t\t\tclone = elem.cloneNode( true ),\n\t\t\tinPage = jQuery.contains( elem.ownerDocument, elem );\n\n\t\t// Support: IE >= 9\n\t\t// Fix Cloning issues\n\t\tif ( !jQuery.support.noCloneChecked && ( elem.nodeType === 1 || elem.nodeType === 11 ) && !jQuery.isXMLDoc( elem ) ) {\n\n\t\t\t// We eschew Sizzle here for performance reasons: http://jsperf.com/getall-vs-sizzle/2\n\t\t\tdestElements = getAll( clone );\n\t\t\tsrcElements = getAll( elem );\n\n\t\t\tfor ( i = 0, l = srcElements.length; i < l; i++ ) {\n\t\t\t\tfixInput( srcElements[ i ], destElements[ i ] );\n\t\t\t}\n\t\t}\n\n\t\t// Copy the events from the original to the clone\n\t\tif ( dataAndEvents ) {\n\t\t\tif ( deepDataAndEvents ) {\n\t\t\t\tsrcElements = srcElements || getAll( elem );\n\t\t\t\tdestElements = destElements || getAll( clone );\n\n\t\t\t\tfor ( i = 0, l = srcElements.length; i < l; i++ ) {\n\t\t\t\t\tcloneCopyEvent( srcElements[ i ], destElements[ i ] );\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tcloneCopyEvent( elem, clone );\n\t\t\t}\n\t\t}\n\n\t\t// Preserve script evaluation history\n\t\tdestElements = getAll( clone, \"script\" );\n\t\tif ( destElements.length > 0 ) {\n\t\t\tsetGlobalEval( destElements, !inPage && getAll( elem, \"script\" ) );\n\t\t}\n\n\t\t// Return the cloned set\n\t\treturn clone;\n\t},\n\n\tbuildFragment: function( elems, context, scripts, selection ) {\n\t\tvar elem, tmp, tag, wrap, contains, j,\n\t\t\ti = 0,\n\t\t\tl = elems.length,\n\t\t\tfragment = context.createDocumentFragment(),\n\t\t\tnodes = [];\n\n\t\tfor ( ; i < l; i++ ) {\n\t\t\telem = elems[ i ];\n\n\t\t\tif ( elem || elem === 0 ) {\n\n\t\t\t\t// Add nodes directly\n\t\t\t\tif ( jQuery.type( elem ) === \"object\" ) {\n\t\t\t\t\t// Support: QtWebKit\n\t\t\t\t\t// jQuery.merge because core_push.apply(_, arraylike) throws\n\t\t\t\t\tjQuery.merge( nodes, elem.nodeType ? [ elem ] : elem );\n\n\t\t\t\t// Convert non-html into a text node\n\t\t\t\t} else if ( !rhtml.test( elem ) ) {\n\t\t\t\t\tnodes.push( context.createTextNode( elem ) );\n\n\t\t\t\t// Convert html into DOM nodes\n\t\t\t\t} else {\n\t\t\t\t\ttmp = tmp || fragment.appendChild( context.createElement(\"div\") );\n\n\t\t\t\t\t// Deserialize a standard representation\n\t\t\t\t\ttag = ( rtagName.exec( elem ) || [\"\", \"\"] )[ 1 ].toLowerCase();\n\t\t\t\t\twrap = wrapMap[ tag ] || wrapMap._default;\n\t\t\t\t\ttmp.innerHTML = wrap[ 1 ] + elem.replace( rxhtmlTag, \"<$1></$2>\" ) + wrap[ 2 ];\n\n\t\t\t\t\t// Descend through wrappers to the right content\n\t\t\t\t\tj = wrap[ 0 ];\n\t\t\t\t\twhile ( j-- ) {\n\t\t\t\t\t\ttmp = tmp.firstChild;\n\t\t\t\t\t}\n\n\t\t\t\t\t// Support: QtWebKit\n\t\t\t\t\t// jQuery.merge because core_push.apply(_, arraylike) throws\n\t\t\t\t\tjQuery.merge( nodes, tmp.childNodes );\n\n\t\t\t\t\t// Remember the top-level container\n\t\t\t\t\ttmp = fragment.firstChild;\n\n\t\t\t\t\t// Fixes #12346\n\t\t\t\t\t// Support: Webkit, IE\n\t\t\t\t\ttmp.textContent = \"\";\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\t// Remove wrapper from fragment\n\t\tfragment.textContent = \"\";\n\n\t\ti = 0;\n\t\twhile ( (elem = nodes[ i++ ]) ) {\n\n\t\t\t// #4087 - If origin and destination elements are the same, and this is\n\t\t\t// that element, do not do anything\n\t\t\tif ( selection && jQuery.inArray( elem, selection ) !== -1 ) {\n\t\t\t\tcontinue;\n\t\t\t}\n\n\t\t\tcontains = jQuery.contains( elem.ownerDocument, elem );\n\n\t\t\t// Append to fragment\n\t\t\ttmp = getAll( fragment.appendChild( elem ), \"script\" );\n\n\t\t\t// Preserve script evaluation history\n\t\t\tif ( contains ) {\n\t\t\t\tsetGlobalEval( tmp );\n\t\t\t}\n\n\t\t\t// Capture executables\n\t\t\tif ( scripts ) {\n\t\t\t\tj = 0;\n\t\t\t\twhile ( (elem = tmp[ j++ ]) ) {\n\t\t\t\t\tif ( rscriptType.test( elem.type || \"\" ) ) {\n\t\t\t\t\t\tscripts.push( elem );\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\treturn fragment;\n\t},\n\n\tcleanData: function( elems ) {\n\t\tvar data, elem, type,\n\t\t\tl = elems.length,\n\t\t\ti = 0,\n\t\t\tspecial = jQuery.event.special;\n\n\t\tfor ( ; i < l; i++ ) {\n\t\t\telem = elems[ i ];\n\n\t\t\tif ( jQuery.acceptData( elem ) ) {\n\n\t\t\t\tdata = data_priv.access( elem );\n\n\t\t\t\tif ( data ) {\n\t\t\t\t\tfor ( type in data.events ) {\n\t\t\t\t\t\tif ( special[ type ] ) {\n\t\t\t\t\t\t\tjQuery.event.remove( elem, type );\n\n\t\t\t\t\t\t// This is a shortcut to avoid jQuery.event.remove's overhead\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\tjQuery.removeEvent( elem, type, data.handle );\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\t// Discard any remaining `private` and `user` data\n\t\t\t// One day we'll replace the dual arrays with a WeakMap and this won't be an issue.\n\t\t\t// (Splices the data objects out of the internal cache arrays)\n\t\t\tdata_user.discard( elem );\n\t\t\tdata_priv.discard( elem );\n\t\t}\n\t},\n\n\t_evalUrl: function( url ) {\n\t\treturn jQuery.ajax({\n\t\t\turl: url,\n\t\t\ttype: \"GET\",\n\t\t\tdataType: \"text\",\n\t\t\tasync: false,\n\t\t\tglobal: false,\n\t\t\tsuccess: jQuery.globalEval\n\t\t});\n\t}\n});\n\n// Support: 1.x compatibility\n// Manipulating tables requires a tbody\nfunction manipulationTarget( elem, content ) {\n\treturn jQuery.nodeName( elem, \"table\" ) &&\n\t\tjQuery.nodeName( content.nodeType === 1 ? content : content.firstChild, \"tr\" ) ?\n\n\t\telem.getElementsByTagName(\"tbody\")[0] ||\n\t\t\telem.appendChild( elem.ownerDocument.createElement(\"tbody\") ) :\n\t\telem;\n}\n\n// Replace/restore the type attribute of script elements for safe DOM manipulation\nfunction disableScript( elem ) {\n\telem.type = (elem.getAttribute(\"type\") !== null) + \"/\" + elem.type;\n\treturn elem;\n}\nfunction restoreScript( elem ) {\n\tvar match = rscriptTypeMasked.exec( elem.type );\n\n\tif ( match ) {\n\t\telem.type = match[ 1 ];\n\t} else {\n\t\telem.removeAttribute(\"type\");\n\t}\n\n\treturn elem;\n}\n\n// Mark scripts as having already been evaluated\nfunction setGlobalEval( elems, refElements ) {\n\tvar l = elems.length,\n\t\ti = 0;\n\n\tfor ( ; i < l; i++ ) {\n\t\tdata_priv.set(\n\t\t\telems[ i ], \"globalEval\", !refElements || data_priv.get( refElements[ i ], \"globalEval\" )\n\t\t);\n\t}\n}\n\nfunction cloneCopyEvent( src, dest ) {\n\tvar i, l, type, pdataOld, pdataCur, udataOld, udataCur, events;\n\n\tif ( dest.nodeType !== 1 ) {\n\t\treturn;\n\t}\n\n\t// 1. Copy private data: events, handlers, etc.\n\tif ( data_priv.hasData( src ) ) {\n\t\tpdataOld = data_priv.access( src );\n\t\tpdataCur = jQuery.extend( {}, pdataOld );\n\t\tevents = pdataOld.events;\n\n\t\tdata_priv.set( dest, pdataCur );\n\n\t\tif ( events ) {\n\t\t\tdelete pdataCur.handle;\n\t\t\tpdataCur.events = {};\n\n\t\t\tfor ( type in events ) {\n\t\t\t\tfor ( i = 0, l = events[ type ].length; i < l; i++ ) {\n\t\t\t\t\tjQuery.event.add( dest, type, events[ type ][ i ] );\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\t// 2. Copy user data\n\tif ( data_user.hasData( src ) ) {\n\t\tudataOld = data_user.access( src );\n\t\tudataCur = jQuery.extend( {}, udataOld );\n\n\t\tdata_user.set( dest, udataCur );\n\t}\n}\n\n\nfunction getAll( context, tag ) {\n\tvar ret = context.getElementsByTagName ? context.getElementsByTagName( tag || \"*\" ) :\n\t\t\tcontext.querySelectorAll ? context.querySelectorAll( tag || \"*\" ) :\n\t\t\t[];\n\n\treturn tag === undefined || tag && jQuery.nodeName( context, tag ) ?\n\t\tjQuery.merge( [ context ], ret ) :\n\t\tret;\n}\n\n// Support: IE >= 9\nfunction fixInput( src, dest ) {\n\tvar nodeName = dest.nodeName.toLowerCase();\n\n\t// Fails to persist the checked state of a cloned checkbox or radio button.\n\tif ( nodeName === \"input\" && manipulation_rcheckableType.test( src.type ) ) {\n\t\tdest.checked = src.checked;\n\n\t// Fails to return the selected option to the default selected state when cloning options\n\t} else if ( nodeName === \"input\" || nodeName === \"textarea\" ) {\n\t\tdest.defaultValue = src.defaultValue;\n\t}\n}\njQuery.fn.extend({\n\twrapAll: function( html ) {\n\t\tvar wrap;\n\n\t\tif ( jQuery.isFunction( html ) ) {\n\t\t\treturn this.each(function( i ) {\n\t\t\t\tjQuery( this ).wrapAll( html.call(this, i) );\n\t\t\t});\n\t\t}\n\n\t\tif ( this[ 0 ] ) {\n\n\t\t\t// The elements to wrap the target around\n\t\t\twrap = jQuery( html, this[ 0 ].ownerDocument ).eq( 0 ).clone( true );\n\n\t\t\tif ( this[ 0 ].parentNode ) {\n\t\t\t\twrap.insertBefore( this[ 0 ] );\n\t\t\t}\n\n\t\t\twrap.map(function() {\n\t\t\t\tvar elem = this;\n\n\t\t\t\twhile ( elem.firstElementChild ) {\n\t\t\t\t\telem = elem.firstElementChild;\n\t\t\t\t}\n\n\t\t\t\treturn elem;\n\t\t\t}).append( this );\n\t\t}\n\n\t\treturn this;\n\t},\n\n\twrapInner: function( html ) {\n\t\tif ( jQuery.isFunction( html ) ) {\n\t\t\treturn this.each(function( i ) {\n\t\t\t\tjQuery( this ).wrapInner( html.call(this, i) );\n\t\t\t});\n\t\t}\n\n\t\treturn this.each(function() {\n\t\t\tvar self = jQuery( this ),\n\t\t\t\tcontents = self.contents();\n\n\t\t\tif ( contents.length ) {\n\t\t\t\tcontents.wrapAll( html );\n\n\t\t\t} else {\n\t\t\t\tself.append( html );\n\t\t\t}\n\t\t});\n\t},\n\n\twrap: function( html ) {\n\t\tvar isFunction = jQuery.isFunction( html );\n\n\t\treturn this.each(function( i ) {\n\t\t\tjQuery( this ).wrapAll( isFunction ? html.call(this, i) : html );\n\t\t});\n\t},\n\n\tunwrap: function() {\n\t\treturn this.parent().each(function() {\n\t\t\tif ( !jQuery.nodeName( this, \"body\" ) ) {\n\t\t\t\tjQuery( this ).replaceWith( this.childNodes );\n\t\t\t}\n\t\t}).end();\n\t}\n});\nvar curCSS, iframe,\n\t// swappable if display is none or starts with table except \"table\", \"table-cell\", or \"table-caption\"\n\t// see here for display values: https://developer.mozilla.org/en-US/docs/CSS/display\n\trdisplayswap = /^(none|table(?!-c[ea]).+)/,\n\trmargin = /^margin/,\n\trnumsplit = new RegExp( \"^(\" + core_pnum + \")(.*)$\", \"i\" ),\n\trnumnonpx = new RegExp( \"^(\" + core_pnum + \")(?!px)[a-z%]+$\", \"i\" ),\n\trrelNum = new RegExp( \"^([+-])=(\" + core_pnum + \")\", \"i\" ),\n\telemdisplay = { BODY: \"block\" },\n\n\tcssShow = { position: \"absolute\", visibility: \"hidden\", display: \"block\" },\n\tcssNormalTransform = {\n\t\tletterSpacing: 0,\n\t\tfontWeight: 400\n\t},\n\n\tcssExpand = [ \"Top\", \"Right\", \"Bottom\", \"Left\" ],\n\tcssPrefixes = [ \"Webkit\", \"O\", \"Moz\", \"ms\" ];\n\n// return a css property mapped to a potentially vendor prefixed property\nfunction vendorPropName( style, name ) {\n\n\t// shortcut for names that are not vendor prefixed\n\tif ( name in style ) {\n\t\treturn name;\n\t}\n\n\t// check for vendor prefixed names\n\tvar capName = name.charAt(0).toUpperCase() + name.slice(1),\n\t\torigName = name,\n\t\ti = cssPrefixes.length;\n\n\twhile ( i-- ) {\n\t\tname = cssPrefixes[ i ] + capName;\n\t\tif ( name in style ) {\n\t\t\treturn name;\n\t\t}\n\t}\n\n\treturn origName;\n}\n\nfunction isHidden( elem, el ) {\n\t// isHidden might be called from jQuery#filter function;\n\t// in that case, element will be second argument\n\telem = el || elem;\n\treturn jQuery.css( elem, \"display\" ) === \"none\" || !jQuery.contains( elem.ownerDocument, elem );\n}\n\n// NOTE: we've included the \"window\" in window.getComputedStyle\n// because jsdom on node.js will break without it.\nfunction getStyles( elem ) {\n\treturn window.getComputedStyle( elem, null );\n}\n\nfunction showHide( elements, show ) {\n\tvar display, elem, hidden,\n\t\tvalues = [],\n\t\tindex = 0,\n\t\tlength = elements.length;\n\n\tfor ( ; index < length; index++ ) {\n\t\telem = elements[ index ];\n\t\tif ( !elem.style ) {\n\t\t\tcontinue;\n\t\t}\n\n\t\tvalues[ index ] = data_priv.get( elem, \"olddisplay\" );\n\t\tdisplay = elem.style.display;\n\t\tif ( show ) {\n\t\t\t// Reset the inline display of this element to learn if it is\n\t\t\t// being hidden by cascaded rules or not\n\t\t\tif ( !values[ index ] && display === \"none\" ) {\n\t\t\t\telem.style.display = \"\";\n\t\t\t}\n\n\t\t\t// Set elements which have been overridden with display: none\n\t\t\t// in a stylesheet to whatever the default browser style is\n\t\t\t// for such an element\n\t\t\tif ( elem.style.display === \"\" && isHidden( elem ) ) {\n\t\t\t\tvalues[ index ] = data_priv.access( elem, \"olddisplay\", css_defaultDisplay(elem.nodeName) );\n\t\t\t}\n\t\t} else {\n\n\t\t\tif ( !values[ index ] ) {\n\t\t\t\thidden = isHidden( elem );\n\n\t\t\t\tif ( display && display !== \"none\" || !hidden ) {\n\t\t\t\t\tdata_priv.set( elem, \"olddisplay\", hidden ? display : jQuery.css(elem, \"display\") );\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\t// Set the display of most of the elements in a second loop\n\t// to avoid the constant reflow\n\tfor ( index = 0; index < length; index++ ) {\n\t\telem = elements[ index ];\n\t\tif ( !elem.style ) {\n\t\t\tcontinue;\n\t\t}\n\t\tif ( !show || elem.style.display === \"none\" || elem.style.display === \"\" ) {\n\t\t\telem.style.display = show ? values[ index ] || \"\" : \"none\";\n\t\t}\n\t}\n\n\treturn elements;\n}\n\njQuery.fn.extend({\n\tcss: function( name, value ) {\n\t\treturn jQuery.access( this, function( elem, name, value ) {\n\t\t\tvar styles, len,\n\t\t\t\tmap = {},\n\t\t\t\ti = 0;\n\n\t\t\tif ( jQuery.isArray( name ) ) {\n\t\t\t\tstyles = getStyles( elem );\n\t\t\t\tlen = name.length;\n\n\t\t\t\tfor ( ; i < len; i++ ) {\n\t\t\t\t\tmap[ name[ i ] ] = jQuery.css( elem, name[ i ], false, styles );\n\t\t\t\t}\n\n\t\t\t\treturn map;\n\t\t\t}\n\n\t\t\treturn value !== undefined ?\n\t\t\t\tjQuery.style( elem, name, value ) :\n\t\t\t\tjQuery.css( elem, name );\n\t\t}, name, value, arguments.length > 1 );\n\t},\n\tshow: function() {\n\t\treturn showHide( this, true );\n\t},\n\thide: function() {\n\t\treturn showHide( this );\n\t},\n\ttoggle: function( state ) {\n\t\tvar bool = typeof state === \"boolean\";\n\n\t\treturn this.each(function() {\n\t\t\tif ( bool ? state : isHidden( this ) ) {\n\t\t\t\tjQuery( this ).show();\n\t\t\t} else {\n\t\t\t\tjQuery( this ).hide();\n\t\t\t}\n\t\t});\n\t}\n});\n\njQuery.extend({\n\t// Add in style property hooks for overriding the default\n\t// behavior of getting and setting a style property\n\tcssHooks: {\n\t\topacity: {\n\t\t\tget: function( elem, computed ) {\n\t\t\t\tif ( computed ) {\n\t\t\t\t\t// We should always get a number back from opacity\n\t\t\t\t\tvar ret = curCSS( elem, \"opacity\" );\n\t\t\t\t\treturn ret === \"\" ? \"1\" : ret;\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t},\n\n\t// Exclude the following css properties to add px\n\tcssNumber: {\n\t\t\"columnCount\": true,\n\t\t\"fillOpacity\": true,\n\t\t\"fontWeight\": true,\n\t\t\"lineHeight\": true,\n\t\t\"opacity\": true,\n\t\t\"orphans\": true,\n\t\t\"widows\": true,\n\t\t\"zIndex\": true,\n\t\t\"zoom\": true\n\t},\n\n\t// Add in properties whose names you wish to fix before\n\t// setting or getting the value\n\tcssProps: {\n\t\t// normalize float css property\n\t\t\"float\": \"cssFloat\"\n\t},\n\n\t// Get and set the style property on a DOM Node\n\tstyle: function( elem, name, value, extra ) {\n\t\t// Don't set styles on text and comment nodes\n\t\tif ( !elem || elem.nodeType === 3 || elem.nodeType === 8 || !elem.style ) {\n\t\t\treturn;\n\t\t}\n\n\t\t// Make sure that we're working with the right name\n\t\tvar ret, type, hooks,\n\t\t\torigName = jQuery.camelCase( name ),\n\t\t\tstyle = elem.style;\n\n\t\tname = jQuery.cssProps[ origName ] || ( jQuery.cssProps[ origName ] = vendorPropName( style, origName ) );\n\n\t\t// gets hook for the prefixed version\n\t\t// followed by the unprefixed version\n\t\thooks = jQuery.cssHooks[ name ] || jQuery.cssHooks[ origName ];\n\n\t\t// Check if we're setting a value\n\t\tif ( value !== undefined ) {\n\t\t\ttype = typeof value;\n\n\t\t\t// convert relative number strings (+= or -=) to relative numbers. #7345\n\t\t\tif ( type === \"string\" && (ret = rrelNum.exec( value )) ) {\n\t\t\t\tvalue = ( ret[1] + 1 ) * ret[2] + parseFloat( jQuery.css( elem, name ) );\n\t\t\t\t// Fixes bug #9237\n\t\t\t\ttype = \"number\";\n\t\t\t}\n\n\t\t\t// Make sure that NaN and null values aren't set. See: #7116\n\t\t\tif ( value == null || type === \"number\" && isNaN( value ) ) {\n\t\t\t\treturn;\n\t\t\t}\n\n\t\t\t// If a number was passed in, add 'px' to the (except for certain CSS properties)\n\t\t\tif ( type === \"number\" && !jQuery.cssNumber[ origName ] ) {\n\t\t\t\tvalue += \"px\";\n\t\t\t}\n\n\t\t\t// Fixes #8908, it can be done more correctly by specifying setters in cssHooks,\n\t\t\t// but it would mean to define eight (for every problematic property) identical functions\n\t\t\tif ( !jQuery.support.clearCloneStyle && value === \"\" && name.indexOf(\"background\") === 0 ) {\n\t\t\t\tstyle[ name ] = \"inherit\";\n\t\t\t}\n\n\t\t\t// If a hook was provided, use that value, otherwise just set the specified value\n\t\t\tif ( !hooks || !(\"set\" in hooks) || (value = hooks.set( elem, value, extra )) !== undefined ) {\n\t\t\t\tstyle[ name ] = value;\n\t\t\t}\n\n\t\t} else {\n\t\t\t// If a hook was provided get the non-computed value from there\n\t\t\tif ( hooks && \"get\" in hooks && (ret = hooks.get( elem, false, extra )) !== undefined ) {\n\t\t\t\treturn ret;\n\t\t\t}\n\n\t\t\t// Otherwise just get the value from the style object\n\t\t\treturn style[ name ];\n\t\t}\n\t},\n\n\tcss: function( elem, name, extra, styles ) {\n\t\tvar val, num, hooks,\n\t\t\torigName = jQuery.camelCase( name );\n\n\t\t// Make sure that we're working with the right name\n\t\tname = jQuery.cssProps[ origName ] || ( jQuery.cssProps[ origName ] = vendorPropName( elem.style, origName ) );\n\n\t\t// gets hook for the prefixed version\n\t\t// followed by the unprefixed version\n\t\thooks = jQuery.cssHooks[ name ] || jQuery.cssHooks[ origName ];\n\n\t\t// If a hook was provided get the computed value from there\n\t\tif ( hooks && \"get\" in hooks ) {\n\t\t\tval = hooks.get( elem, true, extra );\n\t\t}\n\n\t\t// Otherwise, if a way to get the computed value exists, use that\n\t\tif ( val === undefined ) {\n\t\t\tval = curCSS( elem, name, styles );\n\t\t}\n\n\t\t//convert \"normal\" to computed value\n\t\tif ( val === \"normal\" && name in cssNormalTransform ) {\n\t\t\tval = cssNormalTransform[ name ];\n\t\t}\n\n\t\t// Return, converting to number if forced or a qualifier was provided and val looks numeric\n\t\tif ( extra === \"\" || extra ) {\n\t\t\tnum = parseFloat( val );\n\t\t\treturn extra === true || jQuery.isNumeric( num ) ? num || 0 : val;\n\t\t}\n\t\treturn val;\n\t}\n});\n\ncurCSS = function( elem, name, _computed ) {\n\tvar width, minWidth, maxWidth,\n\t\tcomputed = _computed || getStyles( elem ),\n\n\t\t// Support: IE9\n\t\t// getPropertyValue is only needed for .css('filter') in IE9, see #12537\n\t\tret = computed ? computed.getPropertyValue( name ) || computed[ name ] : undefined,\n\t\tstyle = elem.style;\n\n\tif ( computed ) {\n\n\t\tif ( ret === \"\" && !jQuery.contains( elem.ownerDocument, elem ) ) {\n\t\t\tret = jQuery.style( elem, name );\n\t\t}\n\n\t\t// Support: Safari 5.1\n\t\t// A tribute to the \"awesome hack by Dean Edwards\"\n\t\t// Safari 5.1.7 (at least) returns percentage for a larger set of values, but width seems to be reliably pixels\n\t\t// this is against the CSSOM draft spec: http://dev.w3.org/csswg/cssom/#resolved-values\n\t\tif ( rnumnonpx.test( ret ) && rmargin.test( name ) ) {\n\n\t\t\t// Remember the original values\n\t\t\twidth = style.width;\n\t\t\tminWidth = style.minWidth;\n\t\t\tmaxWidth = style.maxWidth;\n\n\t\t\t// Put in the new values to get a computed value out\n\t\t\tstyle.minWidth = style.maxWidth = style.width = ret;\n\t\t\tret = computed.width;\n\n\t\t\t// Revert the changed values\n\t\t\tstyle.width = width;\n\t\t\tstyle.minWidth = minWidth;\n\t\t\tstyle.maxWidth = maxWidth;\n\t\t}\n\t}\n\n\treturn ret;\n};\n\n\nfunction setPositiveNumber( elem, value, subtract ) {\n\tvar matches = rnumsplit.exec( value );\n\treturn matches ?\n\t\t// Guard against undefined \"subtract\", e.g., when used as in cssHooks\n\t\tMath.max( 0, matches[ 1 ] - ( subtract || 0 ) ) + ( matches[ 2 ] || \"px\" ) :\n\t\tvalue;\n}\n\nfunction augmentWidthOrHeight( elem, name, extra, isBorderBox, styles ) {\n\tvar i = extra === ( isBorderBox ? \"border\" : \"content\" ) ?\n\t\t// If we already have the right measurement, avoid augmentation\n\t\t4 :\n\t\t// Otherwise initialize for horizontal or vertical properties\n\t\tname === \"width\" ? 1 : 0,\n\n\t\tval = 0;\n\n\tfor ( ; i < 4; i += 2 ) {\n\t\t// both box models exclude margin, so add it if we want it\n\t\tif ( extra === \"margin\" ) {\n\t\t\tval += jQuery.css( elem, extra + cssExpand[ i ], true, styles );\n\t\t}\n\n\t\tif ( isBorderBox ) {\n\t\t\t// border-box includes padding, so remove it if we want content\n\t\t\tif ( extra === \"content\" ) {\n\t\t\t\tval -= jQuery.css( elem, \"padding\" + cssExpand[ i ], true, styles );\n\t\t\t}\n\n\t\t\t// at this point, extra isn't border nor margin, so remove border\n\t\t\tif ( extra !== \"margin\" ) {\n\t\t\t\tval -= jQuery.css( elem, \"border\" + cssExpand[ i ] + \"Width\", true, styles );\n\t\t\t}\n\t\t} else {\n\t\t\t// at this point, extra isn't content, so add padding\n\t\t\tval += jQuery.css( elem, \"padding\" + cssExpand[ i ], true, styles );\n\n\t\t\t// at this point, extra isn't content nor padding, so add border\n\t\t\tif ( extra !== \"padding\" ) {\n\t\t\t\tval += jQuery.css( elem, \"border\" + cssExpand[ i ] + \"Width\", true, styles );\n\t\t\t}\n\t\t}\n\t}\n\n\treturn val;\n}\n\nfunction getWidthOrHeight( elem, name, extra ) {\n\n\t// Start with offset property, which is equivalent to the border-box value\n\tvar valueIsBorderBox = true,\n\t\tval = name === \"width\" ? elem.offsetWidth : elem.offsetHeight,\n\t\tstyles = getStyles( elem ),\n\t\tisBorderBox = jQuery.support.boxSizing && jQuery.css( elem, \"boxSizing\", false, styles ) === \"border-box\";\n\n\t// some non-html elements return undefined for offsetWidth, so check for null/undefined\n\t// svg - https://bugzilla.mozilla.org/show_bug.cgi?id=649285\n\t// MathML - https://bugzilla.mozilla.org/show_bug.cgi?id=491668\n\tif ( val <= 0 || val == null ) {\n\t\t// Fall back to computed then uncomputed css if necessary\n\t\tval = curCSS( elem, name, styles );\n\t\tif ( val < 0 || val == null ) {\n\t\t\tval = elem.style[ name ];\n\t\t}\n\n\t\t// Computed unit is not pixels. Stop here and return.\n\t\tif ( rnumnonpx.test(val) ) {\n\t\t\treturn val;\n\t\t}\n\n\t\t// we need the check for style in case a browser which returns unreliable values\n\t\t// for getComputedStyle silently falls back to the reliable elem.style\n\t\tvalueIsBorderBox = isBorderBox && ( jQuery.support.boxSizingReliable || val === elem.style[ name ] );\n\n\t\t// Normalize \"\", auto, and prepare for extra\n\t\tval = parseFloat( val ) || 0;\n\t}\n\n\t// use the active box-sizing model to add/subtract irrelevant styles\n\treturn ( val +\n\t\taugmentWidthOrHeight(\n\t\t\telem,\n\t\t\tname,\n\t\t\textra || ( isBorderBox ? \"border\" : \"content\" ),\n\t\t\tvalueIsBorderBox,\n\t\t\tstyles\n\t\t)\n\t) + \"px\";\n}\n\n// Try to determine the default display value of an element\nfunction css_defaultDisplay( nodeName ) {\n\tvar doc = document,\n\t\tdisplay = elemdisplay[ nodeName ];\n\n\tif ( !display ) {\n\t\tdisplay = actualDisplay( nodeName, doc );\n\n\t\t// If the simple way fails, read from inside an iframe\n\t\tif ( display === \"none\" || !display ) {\n\t\t\t// Use the already-created iframe if possible\n\t\t\tiframe = ( iframe ||\n\t\t\t\tjQuery(\"<iframe frameborder='0' width='0' height='0'/>\")\n\t\t\t\t.css( \"cssText\", \"display:block !important\" )\n\t\t\t).appendTo( doc.documentElement );\n\n\t\t\t// Always write a new HTML skeleton so Webkit and Firefox don't choke on reuse\n\t\t\tdoc = ( iframe[0].contentWindow || iframe[0].contentDocument ).document;\n\t\t\tdoc.write(\"<!doctype html><html><body>\");\n\t\t\tdoc.close();\n\n\t\t\tdisplay = actualDisplay( nodeName, doc );\n\t\t\tiframe.detach();\n\t\t}\n\n\t\t// Store the correct default display\n\t\telemdisplay[ nodeName ] = display;\n\t}\n\n\treturn display;\n}\n\n// Called ONLY from within css_defaultDisplay\nfunction actualDisplay( name, doc ) {\n\tvar elem = jQuery( doc.createElement( name ) ).appendTo( doc.body ),\n\t\tdisplay = jQuery.css( elem[0], \"display\" );\n\telem.remove();\n\treturn display;\n}\n\njQuery.each([ \"height\", \"width\" ], function( i, name ) {\n\tjQuery.cssHooks[ name ] = {\n\t\tget: function( elem, computed, extra ) {\n\t\t\tif ( computed ) {\n\t\t\t\t// certain elements can have dimension info if we invisibly show them\n\t\t\t\t// however, it must have a current display style that would benefit from this\n\t\t\t\treturn elem.offsetWidth === 0 && rdisplayswap.test( jQuery.css( elem, \"display\" ) ) ?\n\t\t\t\t\tjQuery.swap( elem, cssShow, function() {\n\t\t\t\t\t\treturn getWidthOrHeight( elem, name, extra );\n\t\t\t\t\t}) :\n\t\t\t\t\tgetWidthOrHeight( elem, name, extra );\n\t\t\t}\n\t\t},\n\n\t\tset: function( elem, value, extra ) {\n\t\t\tvar styles = extra && getStyles( elem );\n\t\t\treturn setPositiveNumber( elem, value, extra ?\n\t\t\t\taugmentWidthOrHeight(\n\t\t\t\t\telem,\n\t\t\t\t\tname,\n\t\t\t\t\textra,\n\t\t\t\t\tjQuery.support.boxSizing && jQuery.css( elem, \"boxSizing\", false, styles ) === \"border-box\",\n\t\t\t\t\tstyles\n\t\t\t\t) : 0\n\t\t\t);\n\t\t}\n\t};\n});\n\n// These hooks cannot be added until DOM ready because the support test\n// for it is not run until after DOM ready\njQuery(function() {\n\t// Support: Android 2.3\n\tif ( !jQuery.support.reliableMarginRight ) {\n\t\tjQuery.cssHooks.marginRight = {\n\t\t\tget: function( elem, computed ) {\n\t\t\t\tif ( computed ) {\n\t\t\t\t\t// Support: Android 2.3\n\t\t\t\t\t// WebKit Bug 13343 - getComputedStyle returns wrong value for margin-right\n\t\t\t\t\t// Work around by temporarily setting element display to inline-block\n\t\t\t\t\treturn jQuery.swap( elem, { \"display\": \"inline-block\" },\n\t\t\t\t\t\tcurCSS, [ elem, \"marginRight\" ] );\n\t\t\t\t}\n\t\t\t}\n\t\t};\n\t}\n\n\t// Webkit bug: https://bugs.webkit.org/show_bug.cgi?id=29084\n\t// getComputedStyle returns percent when specified for top/left/bottom/right\n\t// rather than make the css module depend on the offset module, we just check for it here\n\tif ( !jQuery.support.pixelPosition && jQuery.fn.position ) {\n\t\tjQuery.each( [ \"top\", \"left\" ], function( i, prop ) {\n\t\t\tjQuery.cssHooks[ prop ] = {\n\t\t\t\tget: function( elem, computed ) {\n\t\t\t\t\tif ( computed ) {\n\t\t\t\t\t\tcomputed = curCSS( elem, prop );\n\t\t\t\t\t\t// if curCSS returns percentage, fallback to offset\n\t\t\t\t\t\treturn rnumnonpx.test( computed ) ?\n\t\t\t\t\t\t\tjQuery( elem ).position()[ prop ] + \"px\" :\n\t\t\t\t\t\t\tcomputed;\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t};\n\t\t});\n\t}\n\n});\n\nif ( jQuery.expr && jQuery.expr.filters ) {\n\tjQuery.expr.filters.hidden = function( elem ) {\n\t\t// Support: Opera <= 12.12\n\t\t// Opera reports offsetWidths and offsetHeights less than zero on some elements\n\t\treturn elem.offsetWidth <= 0 && elem.offsetHeight <= 0;\n\t};\n\n\tjQuery.expr.filters.visible = function( elem ) {\n\t\treturn !jQuery.expr.filters.hidden( elem );\n\t};\n}\n\n// These hooks are used by animate to expand properties\njQuery.each({\n\tmargin: \"\",\n\tpadding: \"\",\n\tborder: \"Width\"\n}, function( prefix, suffix ) {\n\tjQuery.cssHooks[ prefix + suffix ] = {\n\t\texpand: function( value ) {\n\t\t\tvar i = 0,\n\t\t\t\texpanded = {},\n\n\t\t\t\t// assumes a single number if not a string\n\t\t\t\tparts = typeof value === \"string\" ? value.split(\" \") : [ value ];\n\n\t\t\tfor ( ; i < 4; i++ ) {\n\t\t\t\texpanded[ prefix + cssExpand[ i ] + suffix ] =\n\t\t\t\t\tparts[ i ] || parts[ i - 2 ] || parts[ 0 ];\n\t\t\t}\n\n\t\t\treturn expanded;\n\t\t}\n\t};\n\n\tif ( !rmargin.test( prefix ) ) {\n\t\tjQuery.cssHooks[ prefix + suffix ].set = setPositiveNumber;\n\t}\n});\nvar r20 = /%20/g,\n\trbracket = /\\[\\]$/,\n\trCRLF = /\\r?\\n/g,\n\trsubmitterTypes = /^(?:submit|button|image|reset|file)$/i,\n\trsubmittable = /^(?:input|select|textarea|keygen)/i;\n\njQuery.fn.extend({\n\tserialize: function() {\n\t\treturn jQuery.param( this.serializeArray() );\n\t},\n\tserializeArray: function() {\n\t\treturn this.map(function(){\n\t\t\t// Can add propHook for \"elements\" to filter or add form elements\n\t\t\tvar elements = jQuery.prop( this, \"elements\" );\n\t\t\treturn elements ? jQuery.makeArray( elements ) : this;\n\t\t})\n\t\t.filter(function(){\n\t\t\tvar type = this.type;\n\t\t\t// Use .is(\":disabled\") so that fieldset[disabled] works\n\t\t\treturn this.name && !jQuery( this ).is( \":disabled\" ) &&\n\t\t\t\trsubmittable.test( this.nodeName ) && !rsubmitterTypes.test( type ) &&\n\t\t\t\t( this.checked || !manipulation_rcheckableType.test( type ) );\n\t\t})\n\t\t.map(function( i, elem ){\n\t\t\tvar val = jQuery( this ).val();\n\n\t\t\treturn val == null ?\n\t\t\t\tnull :\n\t\t\t\tjQuery.isArray( val ) ?\n\t\t\t\t\tjQuery.map( val, function( val ){\n\t\t\t\t\t\treturn { name: elem.name, value: val.replace( rCRLF, \"\\r\\n\" ) };\n\t\t\t\t\t}) :\n\t\t\t\t\t{ name: elem.name, value: val.replace( rCRLF, \"\\r\\n\" ) };\n\t\t}).get();\n\t}\n});\n\n//Serialize an array of form elements or a set of\n//key/values into a query string\njQuery.param = function( a, traditional ) {\n\tvar prefix,\n\t\ts = [],\n\t\tadd = function( key, value ) {\n\t\t\t// If value is a function, invoke it and return its value\n\t\t\tvalue = jQuery.isFunction( value ) ? value() : ( value == null ? \"\" : value );\n\t\t\ts[ s.length ] = encodeURIComponent( key ) + \"=\" + encodeURIComponent( value );\n\t\t};\n\n\t// Set traditional to true for jQuery <= 1.3.2 behavior.\n\tif ( traditional === undefined ) {\n\t\ttraditional = jQuery.ajaxSettings && jQuery.ajaxSettings.traditional;\n\t}\n\n\t// If an array was passed in, assume that it is an array of form elements.\n\tif ( jQuery.isArray( a ) || ( a.jquery && !jQuery.isPlainObject( a ) ) ) {\n\t\t// Serialize the form elements\n\t\tjQuery.each( a, function() {\n\t\t\tadd( this.name, this.value );\n\t\t});\n\n\t} else {\n\t\t// If traditional, encode the \"old\" way (the way 1.3.2 or older\n\t\t// did it), otherwise encode params recursively.\n\t\tfor ( prefix in a ) {\n\t\t\tbuildParams( prefix, a[ prefix ], traditional, add );\n\t\t}\n\t}\n\n\t// Return the resulting serialization\n\treturn s.join( \"&\" ).replace( r20, \"+\" );\n};\n\nfunction buildParams( prefix, obj, traditional, add ) {\n\tvar name;\n\n\tif ( jQuery.isArray( obj ) ) {\n\t\t// Serialize array item.\n\t\tjQuery.each( obj, function( i, v ) {\n\t\t\tif ( traditional || rbracket.test( prefix ) ) {\n\t\t\t\t// Treat each array item as a scalar.\n\t\t\t\tadd( prefix, v );\n\n\t\t\t} else {\n\t\t\t\t// Item is non-scalar (array or object), encode its numeric index.\n\t\t\t\tbuildParams( prefix + \"[\" + ( typeof v === \"object\" ? i : \"\" ) + \"]\", v, traditional, add );\n\t\t\t}\n\t\t});\n\n\t} else if ( !traditional && jQuery.type( obj ) === \"object\" ) {\n\t\t// Serialize object item.\n\t\tfor ( name in obj ) {\n\t\t\tbuildParams( prefix + \"[\" + name + \"]\", obj[ name ], traditional, add );\n\t\t}\n\n\t} else {\n\t\t// Serialize scalar item.\n\t\tadd( prefix, obj );\n\t}\n}\njQuery.each( (\"blur focus focusin focusout load resize scroll unload click dblclick \" +\n\t\"mousedown mouseup mousemove mouseover mouseout mouseenter mouseleave \" +\n\t\"change select submit keydown keypress keyup error contextmenu\").split(\" \"), function( i, name ) {\n\n\t// Handle event binding\n\tjQuery.fn[ name ] = function( data, fn ) {\n\t\treturn arguments.length > 0 ?\n\t\t\tthis.on( name, null, data, fn ) :\n\t\t\tthis.trigger( name );\n\t};\n});\n\njQuery.fn.extend({\n\thover: function( fnOver, fnOut ) {\n\t\treturn this.mouseenter( fnOver ).mouseleave( fnOut || fnOver );\n\t},\n\n\tbind: function( types, data, fn ) {\n\t\treturn this.on( types, null, data, fn );\n\t},\n\tunbind: function( types, fn ) {\n\t\treturn this.off( types, null, fn );\n\t},\n\n\tdelegate: function( selector, types, data, fn ) {\n\t\treturn this.on( types, selector, data, fn );\n\t},\n\tundelegate: function( selector, types, fn ) {\n\t\t// ( namespace ) or ( selector, types [, fn] )\n\t\treturn arguments.length === 1 ? this.off( selector, \"**\" ) : this.off( types, selector || \"**\", fn );\n\t}\n});\nvar\n\t// Document location\n\tajaxLocParts,\n\tajaxLocation,\n\n\tajax_nonce = jQuery.now(),\n\n\tajax_rquery = /\\?/,\n\trhash = /#.*$/,\n\trts = /([?&])_=[^&]*/,\n\trheaders = /^(.*?):[ \\t]*([^\\r\\n]*)$/mg,\n\t// #7653, #8125, #8152: local protocol detection\n\trlocalProtocol = /^(?:about|app|app-storage|.+-extension|file|res|widget):$/,\n\trnoContent = /^(?:GET|HEAD)$/,\n\trprotocol = /^\\/\\//,\n\trurl = /^([\\w.+-]+:)(?:\\/\\/([^\\/?#:]*)(?::(\\d+)|)|)/,\n\n\t// Keep a copy of the old load method\n\t_load = jQuery.fn.load,\n\n\t/* Prefilters\n\t * 1) They are useful to introduce custom dataTypes (see ajax/jsonp.js for an example)\n\t * 2) These are called:\n\t * - BEFORE asking for a transport\n\t * - AFTER param serialization (s.data is a string if s.processData is true)\n\t * 3) key is the dataType\n\t * 4) the catchall symbol \"*\" can be used\n\t * 5) execution will start with transport dataType and THEN continue down to \"*\" if needed\n\t */\n\tprefilters = {},\n\n\t/* Transports bindings\n\t * 1) key is the dataType\n\t * 2) the catchall symbol \"*\" can be used\n\t * 3) selection will start with transport dataType and THEN go to \"*\" if needed\n\t */\n\ttransports = {},\n\n\t// Avoid comment-prolog char sequence (#10098); must appease lint and evade compression\n\tallTypes = \"*/\".concat(\"*\");\n\n// #8138, IE may throw an exception when accessing\n// a field from window.location if document.domain has been set\ntry {\n\tajaxLocation = location.href;\n} catch( e ) {\n\t// Use the href attribute of an A element\n\t// since IE will modify it given document.location\n\tajaxLocation = document.createElement( \"a\" );\n\tajaxLocation.href = \"\";\n\tajaxLocation = ajaxLocation.href;\n}\n\n// Segment location into parts\najaxLocParts = rurl.exec( ajaxLocation.toLowerCase() ) || [];\n\n// Base \"constructor\" for jQuery.ajaxPrefilter and jQuery.ajaxTransport\nfunction addToPrefiltersOrTransports( structure ) {\n\n\t// dataTypeExpression is optional and defaults to \"*\"\n\treturn function( dataTypeExpression, func ) {\n\n\t\tif ( typeof dataTypeExpression !== \"string\" ) {\n\t\t\tfunc = dataTypeExpression;\n\t\t\tdataTypeExpression = \"*\";\n\t\t}\n\n\t\tvar dataType,\n\t\t\ti = 0,\n\t\t\tdataTypes = dataTypeExpression.toLowerCase().match( core_rnotwhite ) || [];\n\n\t\tif ( jQuery.isFunction( func ) ) {\n\t\t\t// For each dataType in the dataTypeExpression\n\t\t\twhile ( (dataType = dataTypes[i++]) ) {\n\t\t\t\t// Prepend if requested\n\t\t\t\tif ( dataType[0] === \"+\" ) {\n\t\t\t\t\tdataType = dataType.slice( 1 ) || \"*\";\n\t\t\t\t\t(structure[ dataType ] = structure[ dataType ] || []).unshift( func );\n\n\t\t\t\t// Otherwise append\n\t\t\t\t} else {\n\t\t\t\t\t(structure[ dataType ] = structure[ dataType ] || []).push( func );\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t};\n}\n\n// Base inspection function for prefilters and transports\nfunction inspectPrefiltersOrTransports( structure, options, originalOptions, jqXHR ) {\n\n\tvar inspected = {},\n\t\tseekingTransport = ( structure === transports );\n\n\tfunction inspect( dataType ) {\n\t\tvar selected;\n\t\tinspected[ dataType ] = true;\n\t\tjQuery.each( structure[ dataType ] || [], function( _, prefilterOrFactory ) {\n\t\t\tvar dataTypeOrTransport = prefilterOrFactory( options, originalOptions, jqXHR );\n\t\t\tif( typeof dataTypeOrTransport === \"string\" && !seekingTransport && !inspected[ dataTypeOrTransport ] ) {\n\t\t\t\toptions.dataTypes.unshift( dataTypeOrTransport );\n\t\t\t\tinspect( dataTypeOrTransport );\n\t\t\t\treturn false;\n\t\t\t} else if ( seekingTransport ) {\n\t\t\t\treturn !( selected = dataTypeOrTransport );\n\t\t\t}\n\t\t});\n\t\treturn selected;\n\t}\n\n\treturn inspect( options.dataTypes[ 0 ] ) || !inspected[ \"*\" ] && inspect( \"*\" );\n}\n\n// A special extend for ajax options\n// that takes \"flat\" options (not to be deep extended)\n// Fixes #9887\nfunction ajaxExtend( target, src ) {\n\tvar key, deep,\n\t\tflatOptions = jQuery.ajaxSettings.flatOptions || {};\n\n\tfor ( key in src ) {\n\t\tif ( src[ key ] !== undefined ) {\n\t\t\t( flatOptions[ key ] ? target : ( deep || (deep = {}) ) )[ key ] = src[ key ];\n\t\t}\n\t}\n\tif ( deep ) {\n\t\tjQuery.extend( true, target, deep );\n\t}\n\n\treturn target;\n}\n\njQuery.fn.load = function( url, params, callback ) {\n\tif ( typeof url !== \"string\" && _load ) {\n\t\treturn _load.apply( this, arguments );\n\t}\n\n\tvar selector, type, response,\n\t\tself = this,\n\t\toff = url.indexOf(\" \");\n\n\tif ( off >= 0 ) {\n\t\tselector = url.slice( off );\n\t\turl = url.slice( 0, off );\n\t}\n\n\t// If it's a function\n\tif ( jQuery.isFunction( params ) ) {\n\n\t\t// We assume that it's the callback\n\t\tcallback = params;\n\t\tparams = undefined;\n\n\t// Otherwise, build a param string\n\t} else if ( params && typeof params === \"object\" ) {\n\t\ttype = \"POST\";\n\t}\n\n\t// If we have elements to modify, make the request\n\tif ( self.length > 0 ) {\n\t\tjQuery.ajax({\n\t\t\turl: url,\n\n\t\t\t// if \"type\" variable is undefined, then \"GET\" method will be used\n\t\t\ttype: type,\n\t\t\tdataType: \"html\",\n\t\t\tdata: params\n\t\t}).done(function( responseText ) {\n\n\t\t\t// Save response for use in complete callback\n\t\t\tresponse = arguments;\n\n\t\t\tself.html( selector ?\n\n\t\t\t\t// If a selector was specified, locate the right elements in a dummy div\n\t\t\t\t// Exclude scripts to avoid IE 'Permission Denied' errors\n\t\t\t\tjQuery(\"<div>\").append( jQuery.parseHTML( responseText ) ).find( selector ) :\n\n\t\t\t\t// Otherwise use the full result\n\t\t\t\tresponseText );\n\n\t\t}).complete( callback && function( jqXHR, status ) {\n\t\t\tself.each( callback, response || [ jqXHR.responseText, status, jqXHR ] );\n\t\t});\n\t}\n\n\treturn this;\n};\n\n// Attach a bunch of functions for handling common AJAX events\njQuery.each( [ \"ajaxStart\", \"ajaxStop\", \"ajaxComplete\", \"ajaxError\", \"ajaxSuccess\", \"ajaxSend\" ], function( i, type ){\n\tjQuery.fn[ type ] = function( fn ){\n\t\treturn this.on( type, fn );\n\t};\n});\n\njQuery.extend({\n\n\t// Counter for holding the number of active queries\n\tactive: 0,\n\n\t// Last-Modified header cache for next request\n\tlastModified: {},\n\tetag: {},\n\n\tajaxSettings: {\n\t\turl: ajaxLocation,\n\t\ttype: \"GET\",\n\t\tisLocal: rlocalProtocol.test( ajaxLocParts[ 1 ] ),\n\t\tglobal: true,\n\t\tprocessData: true,\n\t\tasync: true,\n\t\tcontentType: \"application/x-www-form-urlencoded; charset=UTF-8\",\n\t\t/*\n\t\ttimeout: 0,\n\t\tdata: null,\n\t\tdataType: null,\n\t\tusername: null,\n\t\tpassword: null,\n\t\tcache: null,\n\t\tthrows: false,\n\t\ttraditional: false,\n\t\theaders: {},\n\t\t*/\n\n\t\taccepts: {\n\t\t\t\"*\": allTypes,\n\t\t\ttext: \"text/plain\",\n\t\t\thtml: \"text/html\",\n\t\t\txml: \"application/xml, text/xml\",\n\t\t\tjson: \"application/json, text/javascript\"\n\t\t},\n\n\t\tcontents: {\n\t\t\txml: /xml/,\n\t\t\thtml: /html/,\n\t\t\tjson: /json/\n\t\t},\n\n\t\tresponseFields: {\n\t\t\txml: \"responseXML\",\n\t\t\ttext: \"responseText\",\n\t\t\tjson: \"responseJSON\"\n\t\t},\n\n\t\t// Data converters\n\t\t// Keys separate source (or catchall \"*\") and destination types with a single space\n\t\tconverters: {\n\n\t\t\t// Convert anything to text\n\t\t\t\"* text\": String,\n\n\t\t\t// Text to html (true = no transformation)\n\t\t\t\"text html\": true,\n\n\t\t\t// Evaluate text as a json expression\n\t\t\t\"text json\": jQuery.parseJSON,\n\n\t\t\t// Parse text as xml\n\t\t\t\"text xml\": jQuery.parseXML\n\t\t},\n\n\t\t// For options that shouldn't be deep extended:\n\t\t// you can add your own custom options here if\n\t\t// and when you create one that shouldn't be\n\t\t// deep extended (see ajaxExtend)\n\t\tflatOptions: {\n\t\t\turl: true,\n\t\t\tcontext: true\n\t\t}\n\t},\n\n\t// Creates a full fledged settings object into target\n\t// with both ajaxSettings and settings fields.\n\t// If target is omitted, writes into ajaxSettings.\n\tajaxSetup: function( target, settings ) {\n\t\treturn settings ?\n\n\t\t\t// Building a settings object\n\t\t\tajaxExtend( ajaxExtend( target, jQuery.ajaxSettings ), settings ) :\n\n\t\t\t// Extending ajaxSettings\n\t\t\tajaxExtend( jQuery.ajaxSettings, target );\n\t},\n\n\tajaxPrefilter: addToPrefiltersOrTransports( prefilters ),\n\tajaxTransport: addToPrefiltersOrTransports( transports ),\n\n\t// Main method\n\tajax: function( url, options ) {\n\n\t\t// If url is an object, simulate pre-1.5 signature\n\t\tif ( typeof url === \"object\" ) {\n\t\t\toptions = url;\n\t\t\turl = undefined;\n\t\t}\n\n\t\t// Force options to be an object\n\t\toptions = options || {};\n\n\t\tvar transport,\n\t\t\t// URL without anti-cache param\n\t\t\tcacheURL,\n\t\t\t// Response headers\n\t\t\tresponseHeadersString,\n\t\t\tresponseHeaders,\n\t\t\t// timeout handle\n\t\t\ttimeoutTimer,\n\t\t\t// Cross-domain detection vars\n\t\t\tparts,\n\t\t\t// To know if global events are to be dispatched\n\t\t\tfireGlobals,\n\t\t\t// Loop variable\n\t\t\ti,\n\t\t\t// Create the final options object\n\t\t\ts = jQuery.ajaxSetup( {}, options ),\n\t\t\t// Callbacks context\n\t\t\tcallbackContext = s.context || s,\n\t\t\t// Context for global events is callbackContext if it is a DOM node or jQuery collection\n\t\t\tglobalEventContext = s.context && ( callbackContext.nodeType || callbackContext.jquery ) ?\n\t\t\t\tjQuery( callbackContext ) :\n\t\t\t\tjQuery.event,\n\t\t\t// Deferreds\n\t\t\tdeferred = jQuery.Deferred(),\n\t\t\tcompleteDeferred = jQuery.Callbacks(\"once memory\"),\n\t\t\t// Status-dependent callbacks\n\t\t\tstatusCode = s.statusCode || {},\n\t\t\t// Headers (they are sent all at once)\n\t\t\trequestHeaders = {},\n\t\t\trequestHeadersNames = {},\n\t\t\t// The jqXHR state\n\t\t\tstate = 0,\n\t\t\t// Default abort message\n\t\t\tstrAbort = \"canceled\",\n\t\t\t// Fake xhr\n\t\t\tjqXHR = {\n\t\t\t\treadyState: 0,\n\n\t\t\t\t// Builds headers hashtable if needed\n\t\t\t\tgetResponseHeader: function( key ) {\n\t\t\t\t\tvar match;\n\t\t\t\t\tif ( state === 2 ) {\n\t\t\t\t\t\tif ( !responseHeaders ) {\n\t\t\t\t\t\t\tresponseHeaders = {};\n\t\t\t\t\t\t\twhile ( (match = rheaders.exec( responseHeadersString )) ) {\n\t\t\t\t\t\t\t\tresponseHeaders[ match[1].toLowerCase() ] = match[ 2 ];\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t\tmatch = responseHeaders[ key.toLowerCase() ];\n\t\t\t\t\t}\n\t\t\t\t\treturn match == null ? null : match;\n\t\t\t\t},\n\n\t\t\t\t// Raw string\n\t\t\t\tgetAllResponseHeaders: function() {\n\t\t\t\t\treturn state === 2 ? responseHeadersString : null;\n\t\t\t\t},\n\n\t\t\t\t// Caches the header\n\t\t\t\tsetRequestHeader: function( name, value ) {\n\t\t\t\t\tvar lname = name.toLowerCase();\n\t\t\t\t\tif ( !state ) {\n\t\t\t\t\t\tname = requestHeadersNames[ lname ] = requestHeadersNames[ lname ] || name;\n\t\t\t\t\t\trequestHeaders[ name ] = value;\n\t\t\t\t\t}\n\t\t\t\t\treturn this;\n\t\t\t\t},\n\n\t\t\t\t// Overrides response content-type header\n\t\t\t\toverrideMimeType: function( type ) {\n\t\t\t\t\tif ( !state ) {\n\t\t\t\t\t\ts.mimeType = type;\n\t\t\t\t\t}\n\t\t\t\t\treturn this;\n\t\t\t\t},\n\n\t\t\t\t// Status-dependent callbacks\n\t\t\t\tstatusCode: function( map ) {\n\t\t\t\t\tvar code;\n\t\t\t\t\tif ( map ) {\n\t\t\t\t\t\tif ( state < 2 ) {\n\t\t\t\t\t\t\tfor ( code in map ) {\n\t\t\t\t\t\t\t\t// Lazy-add the new callback in a way that preserves old ones\n\t\t\t\t\t\t\t\tstatusCode[ code ] = [ statusCode[ code ], map[ code ] ];\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\t// Execute the appropriate callbacks\n\t\t\t\t\t\t\tjqXHR.always( map[ jqXHR.status ] );\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\treturn this;\n\t\t\t\t},\n\n\t\t\t\t// Cancel the request\n\t\t\t\tabort: function( statusText ) {\n\t\t\t\t\tvar finalText = statusText || strAbort;\n\t\t\t\t\tif ( transport ) {\n\t\t\t\t\t\ttransport.abort( finalText );\n\t\t\t\t\t}\n\t\t\t\t\tdone( 0, finalText );\n\t\t\t\t\treturn this;\n\t\t\t\t}\n\t\t\t};\n\n\t\t// Attach deferreds\n\t\tdeferred.promise( jqXHR ).complete = completeDeferred.add;\n\t\tjqXHR.success = jqXHR.done;\n\t\tjqXHR.error = jqXHR.fail;\n\n\t\t// Remove hash character (#7531: and string promotion)\n\t\t// Add protocol if not provided (prefilters might expect it)\n\t\t// Handle falsy url in the settings object (#10093: consistency with old signature)\n\t\t// We also use the url parameter if available\n\t\ts.url = ( ( url || s.url || ajaxLocation ) + \"\" ).replace( rhash, \"\" )\n\t\t\t.replace( rprotocol, ajaxLocParts[ 1 ] + \"//\" );\n\n\t\t// Alias method option to type as per ticket #12004\n\t\ts.type = options.method || options.type || s.method || s.type;\n\n\t\t// Extract dataTypes list\n\t\ts.dataTypes = jQuery.trim( s.dataType || \"*\" ).toLowerCase().match( core_rnotwhite ) || [\"\"];\n\n\t\t// A cross-domain request is in order when we have a protocol:host:port mismatch\n\t\tif ( s.crossDomain == null ) {\n\t\t\tparts = rurl.exec( s.url.toLowerCase() );\n\t\t\ts.crossDomain = !!( parts &&\n\t\t\t\t( parts[ 1 ] !== ajaxLocParts[ 1 ] || parts[ 2 ] !== ajaxLocParts[ 2 ] ||\n\t\t\t\t\t( parts[ 3 ] || ( parts[ 1 ] === \"http:\" ? \"80\" : \"443\" ) ) !==\n\t\t\t\t\t\t( ajaxLocParts[ 3 ] || ( ajaxLocParts[ 1 ] === \"http:\" ? \"80\" : \"443\" ) ) )\n\t\t\t);\n\t\t}\n\n\t\t// Convert data if not already a string\n\t\tif ( s.data && s.processData && typeof s.data !== \"string\" ) {\n\t\t\ts.data = jQuery.param( s.data, s.traditional );\n\t\t}\n\n\t\t// Apply prefilters\n\t\tinspectPrefiltersOrTransports( prefilters, s, options, jqXHR );\n\n\t\t// If request was aborted inside a prefilter, stop there\n\t\tif ( state === 2 ) {\n\t\t\treturn jqXHR;\n\t\t}\n\n\t\t// We can fire global events as of now if asked to\n\t\tfireGlobals = s.global;\n\n\t\t// Watch for a new set of requests\n\t\tif ( fireGlobals && jQuery.active++ === 0 ) {\n\t\t\tjQuery.event.trigger(\"ajaxStart\");\n\t\t}\n\n\t\t// Uppercase the type\n\t\ts.type = s.type.toUpperCase();\n\n\t\t// Determine if request has content\n\t\ts.hasContent = !rnoContent.test( s.type );\n\n\t\t// Save the URL in case we're toying with the If-Modified-Since\n\t\t// and/or If-None-Match header later on\n\t\tcacheURL = s.url;\n\n\t\t// More options handling for requests with no content\n\t\tif ( !s.hasContent ) {\n\n\t\t\t// If data is available, append data to url\n\t\t\tif ( s.data ) {\n\t\t\t\tcacheURL = ( s.url += ( ajax_rquery.test( cacheURL ) ? \"&\" : \"?\" ) + s.data );\n\t\t\t\t// #9682: remove data so that it's not used in an eventual retry\n\t\t\t\tdelete s.data;\n\t\t\t}\n\n\t\t\t// Add anti-cache in url if needed\n\t\t\tif ( s.cache === false ) {\n\t\t\t\ts.url = rts.test( cacheURL ) ?\n\n\t\t\t\t\t// If there is already a '_' parameter, set its value\n\t\t\t\t\tcacheURL.replace( rts, \"$1_=\" + ajax_nonce++ ) :\n\n\t\t\t\t\t// Otherwise add one to the end\n\t\t\t\t\tcacheURL + ( ajax_rquery.test( cacheURL ) ? \"&\" : \"?\" ) + \"_=\" + ajax_nonce++;\n\t\t\t}\n\t\t}\n\n\t\t// Set the If-Modified-Since and/or If-None-Match header, if in ifModified mode.\n\t\tif ( s.ifModified ) {\n\t\t\tif ( jQuery.lastModified[ cacheURL ] ) {\n\t\t\t\tjqXHR.setRequestHeader( \"If-Modified-Since\", jQuery.lastModified[ cacheURL ] );\n\t\t\t}\n\t\t\tif ( jQuery.etag[ cacheURL ] ) {\n\t\t\t\tjqXHR.setRequestHeader( \"If-None-Match\", jQuery.etag[ cacheURL ] );\n\t\t\t}\n\t\t}\n\n\t\t// Set the correct header, if data is being sent\n\t\tif ( s.data && s.hasContent && s.contentType !== false || options.contentType ) {\n\t\t\tjqXHR.setRequestHeader( \"Content-Type\", s.contentType );\n\t\t}\n\n\t\t// Set the Accepts header for the server, depending on the dataType\n\t\tjqXHR.setRequestHeader(\n\t\t\t\"Accept\",\n\t\t\ts.dataTypes[ 0 ] && s.accepts[ s.dataTypes[0] ] ?\n\t\t\t\ts.accepts[ s.dataTypes[0] ] + ( s.dataTypes[ 0 ] !== \"*\" ? \", \" + allTypes + \"; q=0.01\" : \"\" ) :\n\t\t\t\ts.accepts[ \"*\" ]\n\t\t);\n\n\t\t// Check for headers option\n\t\tfor ( i in s.headers ) {\n\t\t\tjqXHR.setRequestHeader( i, s.headers[ i ] );\n\t\t}\n\n\t\t// Allow custom headers/mimetypes and early abort\n\t\tif ( s.beforeSend && ( s.beforeSend.call( callbackContext, jqXHR, s ) === false || state === 2 ) ) {\n\t\t\t// Abort if not done already and return\n\t\t\treturn jqXHR.abort();\n\t\t}\n\n\t\t// aborting is no longer a cancellation\n\t\tstrAbort = \"abort\";\n\n\t\t// Install callbacks on deferreds\n\t\tfor ( i in { success: 1, error: 1, complete: 1 } ) {\n\t\t\tjqXHR[ i ]( s[ i ] );\n\t\t}\n\n\t\t// Get transport\n\t\ttransport = inspectPrefiltersOrTransports( transports, s, options, jqXHR );\n\n\t\t// If no transport, we auto-abort\n\t\tif ( !transport ) {\n\t\t\tdone( -1, \"No Transport\" );\n\t\t} else {\n\t\t\tjqXHR.readyState = 1;\n\n\t\t\t// Send global event\n\t\t\tif ( fireGlobals ) {\n\t\t\t\tglobalEventContext.trigger( \"ajaxSend\", [ jqXHR, s ] );\n\t\t\t}\n\t\t\t// Timeout\n\t\t\tif ( s.async && s.timeout > 0 ) {\n\t\t\t\ttimeoutTimer = setTimeout(function() {\n\t\t\t\t\tjqXHR.abort(\"timeout\");\n\t\t\t\t}, s.timeout );\n\t\t\t}\n\n\t\t\ttry {\n\t\t\t\tstate = 1;\n\t\t\t\ttransport.send( requestHeaders, done );\n\t\t\t} catch ( e ) {\n\t\t\t\t// Propagate exception as error if not done\n\t\t\t\tif ( state < 2 ) {\n\t\t\t\t\tdone( -1, e );\n\t\t\t\t// Simply rethrow otherwise\n\t\t\t\t} else {\n\t\t\t\t\tthrow e;\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\t// Callback for when everything is done\n\t\tfunction done( status, nativeStatusText, responses, headers ) {\n\t\t\tvar isSuccess, success, error, response, modified,\n\t\t\t\tstatusText = nativeStatusText;\n\n\t\t\t// Called once\n\t\t\tif ( state === 2 ) {\n\t\t\t\treturn;\n\t\t\t}\n\n\t\t\t// State is \"done\" now\n\t\t\tstate = 2;\n\n\t\t\t// Clear timeout if it exists\n\t\t\tif ( timeoutTimer ) {\n\t\t\t\tclearTimeout( timeoutTimer );\n\t\t\t}\n\n\t\t\t// Dereference transport for early garbage collection\n\t\t\t// (no matter how long the jqXHR object will be used)\n\t\t\ttransport = undefined;\n\n\t\t\t// Cache response headers\n\t\t\tresponseHeadersString = headers || \"\";\n\n\t\t\t// Set readyState\n\t\t\tjqXHR.readyState = status > 0 ? 4 : 0;\n\n\t\t\t// Determine if successful\n\t\t\tisSuccess = status >= 200 && status < 300 || status === 304;\n\n\t\t\t// Get response data\n\t\t\tif ( responses ) {\n\t\t\t\tresponse = ajaxHandleResponses( s, jqXHR, responses );\n\t\t\t}\n\n\t\t\t// Convert no matter what (that way responseXXX fields are always set)\n\t\t\tresponse = ajaxConvert( s, response, jqXHR, isSuccess );\n\n\t\t\t// If successful, handle type chaining\n\t\t\tif ( isSuccess ) {\n\n\t\t\t\t// Set the If-Modified-Since and/or If-None-Match header, if in ifModified mode.\n\t\t\t\tif ( s.ifModified ) {\n\t\t\t\t\tmodified = jqXHR.getResponseHeader(\"Last-Modified\");\n\t\t\t\t\tif ( modified ) {\n\t\t\t\t\t\tjQuery.lastModified[ cacheURL ] = modified;\n\t\t\t\t\t}\n\t\t\t\t\tmodified = jqXHR.getResponseHeader(\"etag\");\n\t\t\t\t\tif ( modified ) {\n\t\t\t\t\t\tjQuery.etag[ cacheURL ] = modified;\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\t// if no content\n\t\t\t\tif ( status === 204 ) {\n\t\t\t\t\tstatusText = \"nocontent\";\n\n\t\t\t\t// if not modified\n\t\t\t\t} else if ( status === 304 ) {\n\t\t\t\t\tstatusText = \"notmodified\";\n\n\t\t\t\t// If we have data, let's convert it\n\t\t\t\t} else {\n\t\t\t\t\tstatusText = response.state;\n\t\t\t\t\tsuccess = response.data;\n\t\t\t\t\terror = response.error;\n\t\t\t\t\tisSuccess = !error;\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\t// We extract error from statusText\n\t\t\t\t// then normalize statusText and status for non-aborts\n\t\t\t\terror = statusText;\n\t\t\t\tif ( status || !statusText ) {\n\t\t\t\t\tstatusText = \"error\";\n\t\t\t\t\tif ( status < 0 ) {\n\t\t\t\t\t\tstatus = 0;\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Set data for the fake xhr object\n\t\t\tjqXHR.status = status;\n\t\t\tjqXHR.statusText = ( nativeStatusText || statusText ) + \"\";\n\n\t\t\t// Success/Error\n\t\t\tif ( isSuccess ) {\n\t\t\t\tdeferred.resolveWith( callbackContext, [ success, statusText, jqXHR ] );\n\t\t\t} else {\n\t\t\t\tdeferred.rejectWith( callbackContext, [ jqXHR, statusText, error ] );\n\t\t\t}\n\n\t\t\t// Status-dependent callbacks\n\t\t\tjqXHR.statusCode( statusCode );\n\t\t\tstatusCode = undefined;\n\n\t\t\tif ( fireGlobals ) {\n\t\t\t\tglobalEventContext.trigger( isSuccess ? \"ajaxSuccess\" : \"ajaxError\",\n\t\t\t\t\t[ jqXHR, s, isSuccess ? success : error ] );\n\t\t\t}\n\n\t\t\t// Complete\n\t\t\tcompleteDeferred.fireWith( callbackContext, [ jqXHR, statusText ] );\n\n\t\t\tif ( fireGlobals ) {\n\t\t\t\tglobalEventContext.trigger( \"ajaxComplete\", [ jqXHR, s ] );\n\t\t\t\t// Handle the global AJAX counter\n\t\t\t\tif ( !( --jQuery.active ) ) {\n\t\t\t\t\tjQuery.event.trigger(\"ajaxStop\");\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\treturn jqXHR;\n\t},\n\n\tgetJSON: function( url, data, callback ) {\n\t\treturn jQuery.get( url, data, callback, \"json\" );\n\t},\n\n\tgetScript: function( url, callback ) {\n\t\treturn jQuery.get( url, undefined, callback, \"script\" );\n\t}\n});\n\njQuery.each( [ \"get\", \"post\" ], function( i, method ) {\n\tjQuery[ method ] = function( url, data, callback, type ) {\n\t\t// shift arguments if data argument was omitted\n\t\tif ( jQuery.isFunction( data ) ) {\n\t\t\ttype = type || callback;\n\t\t\tcallback = data;\n\t\t\tdata = undefined;\n\t\t}\n\n\t\treturn jQuery.ajax({\n\t\t\turl: url,\n\t\t\ttype: method,\n\t\t\tdataType: type,\n\t\t\tdata: data,\n\t\t\tsuccess: callback\n\t\t});\n\t};\n});\n\n/* Handles responses to an ajax request:\n * - finds the right dataType (mediates between content-type and expected dataType)\n * - returns the corresponding response\n */\nfunction ajaxHandleResponses( s, jqXHR, responses ) {\n\n\tvar ct, type, finalDataType, firstDataType,\n\t\tcontents = s.contents,\n\t\tdataTypes = s.dataTypes;\n\n\t// Remove auto dataType and get content-type in the process\n\twhile( dataTypes[ 0 ] === \"*\" ) {\n\t\tdataTypes.shift();\n\t\tif ( ct === undefined ) {\n\t\t\tct = s.mimeType || jqXHR.getResponseHeader(\"Content-Type\");\n\t\t}\n\t}\n\n\t// Check if we're dealing with a known content-type\n\tif ( ct ) {\n\t\tfor ( type in contents ) {\n\t\t\tif ( contents[ type ] && contents[ type ].test( ct ) ) {\n\t\t\t\tdataTypes.unshift( type );\n\t\t\t\tbreak;\n\t\t\t}\n\t\t}\n\t}\n\n\t// Check to see if we have a response for the expected dataType\n\tif ( dataTypes[ 0 ] in responses ) {\n\t\tfinalDataType = dataTypes[ 0 ];\n\t} else {\n\t\t// Try convertible dataTypes\n\t\tfor ( type in responses ) {\n\t\t\tif ( !dataTypes[ 0 ] || s.converters[ type + \" \" + dataTypes[0] ] ) {\n\t\t\t\tfinalDataType = type;\n\t\t\t\tbreak;\n\t\t\t}\n\t\t\tif ( !firstDataType ) {\n\t\t\t\tfirstDataType = type;\n\t\t\t}\n\t\t}\n\t\t// Or just use first one\n\t\tfinalDataType = finalDataType || firstDataType;\n\t}\n\n\t// If we found a dataType\n\t// We add the dataType to the list if needed\n\t// and return the corresponding response\n\tif ( finalDataType ) {\n\t\tif ( finalDataType !== dataTypes[ 0 ] ) {\n\t\t\tdataTypes.unshift( finalDataType );\n\t\t}\n\t\treturn responses[ finalDataType ];\n\t}\n}\n\n/* Chain conversions given the request and the original response\n * Also sets the responseXXX fields on the jqXHR instance\n */\nfunction ajaxConvert( s, response, jqXHR, isSuccess ) {\n\tvar conv2, current, conv, tmp, prev,\n\t\tconverters = {},\n\t\t// Work with a copy of dataTypes in case we need to modify it for conversion\n\t\tdataTypes = s.dataTypes.slice();\n\n\t// Create converters map with lowercased keys\n\tif ( dataTypes[ 1 ] ) {\n\t\tfor ( conv in s.converters ) {\n\t\t\tconverters[ conv.toLowerCase() ] = s.converters[ conv ];\n\t\t}\n\t}\n\n\tcurrent = dataTypes.shift();\n\n\t// Convert to each sequential dataType\n\twhile ( current ) {\n\n\t\tif ( s.responseFields[ current ] ) {\n\t\t\tjqXHR[ s.responseFields[ current ] ] = response;\n\t\t}\n\n\t\t// Apply the dataFilter if provided\n\t\tif ( !prev && isSuccess && s.dataFilter ) {\n\t\t\tresponse = s.dataFilter( response, s.dataType );\n\t\t}\n\n\t\tprev = current;\n\t\tcurrent = dataTypes.shift();\n\n\t\tif ( current ) {\n\n\t\t// There's only work to do if current dataType is non-auto\n\t\t\tif ( current === \"*\" ) {\n\n\t\t\t\tcurrent = prev;\n\n\t\t\t// Convert response if prev dataType is non-auto and differs from current\n\t\t\t} else if ( prev !== \"*\" && prev !== current ) {\n\n\t\t\t\t// Seek a direct converter\n\t\t\t\tconv = converters[ prev + \" \" + current ] || converters[ \"* \" + current ];\n\n\t\t\t\t// If none found, seek a pair\n\t\t\t\tif ( !conv ) {\n\t\t\t\t\tfor ( conv2 in converters ) {\n\n\t\t\t\t\t\t// If conv2 outputs current\n\t\t\t\t\t\ttmp = conv2.split( \" \" );\n\t\t\t\t\t\tif ( tmp[ 1 ] === current ) {\n\n\t\t\t\t\t\t\t// If prev can be converted to accepted input\n\t\t\t\t\t\t\tconv = converters[ prev + \" \" + tmp[ 0 ] ] ||\n\t\t\t\t\t\t\t\tconverters[ \"* \" + tmp[ 0 ] ];\n\t\t\t\t\t\t\tif ( conv ) {\n\t\t\t\t\t\t\t\t// Condense equivalence converters\n\t\t\t\t\t\t\t\tif ( conv === true ) {\n\t\t\t\t\t\t\t\t\tconv = converters[ conv2 ];\n\n\t\t\t\t\t\t\t\t// Otherwise, insert the intermediate dataType\n\t\t\t\t\t\t\t\t} else if ( converters[ conv2 ] !== true ) {\n\t\t\t\t\t\t\t\t\tcurrent = tmp[ 0 ];\n\t\t\t\t\t\t\t\t\tdataTypes.unshift( tmp[ 1 ] );\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\tbreak;\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\t// Apply converter (if not an equivalence)\n\t\t\t\tif ( conv !== true ) {\n\n\t\t\t\t\t// Unless errors are allowed to bubble, catch and return them\n\t\t\t\t\tif ( conv && s[ \"throws\" ] ) {\n\t\t\t\t\t\tresponse = conv( response );\n\t\t\t\t\t} else {\n\t\t\t\t\t\ttry {\n\t\t\t\t\t\t\tresponse = conv( response );\n\t\t\t\t\t\t} catch ( e ) {\n\t\t\t\t\t\t\treturn { state: \"parsererror\", error: conv ? e : \"No conversion from \" + prev + \" to \" + current };\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\treturn { state: \"success\", data: response };\n}\n// Install script dataType\njQuery.ajaxSetup({\n\taccepts: {\n\t\tscript: \"text/javascript, application/javascript, application/ecmascript, application/x-ecmascript\"\n\t},\n\tcontents: {\n\t\tscript: /(?:java|ecma)script/\n\t},\n\tconverters: {\n\t\t\"text script\": function( text ) {\n\t\t\tjQuery.globalEval( text );\n\t\t\treturn text;\n\t\t}\n\t}\n});\n\n// Handle cache's special case and crossDomain\njQuery.ajaxPrefilter( \"script\", function( s ) {\n\tif ( s.cache === undefined ) {\n\t\ts.cache = false;\n\t}\n\tif ( s.crossDomain ) {\n\t\ts.type = \"GET\";\n\t}\n});\n\n// Bind script tag hack transport\njQuery.ajaxTransport( \"script\", function( s ) {\n\t// This transport only deals with cross domain requests\n\tif ( s.crossDomain ) {\n\t\tvar script, callback;\n\t\treturn {\n\t\t\tsend: function( _, complete ) {\n\t\t\t\tscript = jQuery(\"<script>\").prop({\n\t\t\t\t\tasync: true,\n\t\t\t\t\tcharset: s.scriptCharset,\n\t\t\t\t\tsrc: s.url\n\t\t\t\t}).on(\n\t\t\t\t\t\"load error\",\n\t\t\t\t\tcallback = function( evt ) {\n\t\t\t\t\t\tscript.remove();\n\t\t\t\t\t\tcallback = null;\n\t\t\t\t\t\tif ( evt ) {\n\t\t\t\t\t\t\tcomplete( evt.type === \"error\" ? 404 : 200, evt.type );\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t);\n\t\t\t\tdocument.head.appendChild( script[ 0 ] );\n\t\t\t},\n\t\t\tabort: function() {\n\t\t\t\tif ( callback ) {\n\t\t\t\t\tcallback();\n\t\t\t\t}\n\t\t\t}\n\t\t};\n\t}\n});\nvar oldCallbacks = [],\n\trjsonp = /(=)\\?(?=&|$)|\\?\\?/;\n\n// Default jsonp settings\njQuery.ajaxSetup({\n\tjsonp: \"callback\",\n\tjsonpCallback: function() {\n\t\tvar callback = oldCallbacks.pop() || ( jQuery.expando + \"_\" + ( ajax_nonce++ ) );\n\t\tthis[ callback ] = true;\n\t\treturn callback;\n\t}\n});\n\n// Detect, normalize options and install callbacks for jsonp requests\njQuery.ajaxPrefilter( \"json jsonp\", function( s, originalSettings, jqXHR ) {\n\n\tvar callbackName, overwritten, responseContainer,\n\t\tjsonProp = s.jsonp !== false && ( rjsonp.test( s.url ) ?\n\t\t\t\"url\" :\n\t\t\ttypeof s.data === \"string\" && !( s.contentType || \"\" ).indexOf(\"application/x-www-form-urlencoded\") && rjsonp.test( s.data ) && \"data\"\n\t\t);\n\n\t// Handle iff the expected data type is \"jsonp\" or we have a parameter to set\n\tif ( jsonProp || s.dataTypes[ 0 ] === \"jsonp\" ) {\n\n\t\t// Get callback name, remembering preexisting value associated with it\n\t\tcallbackName = s.jsonpCallback = jQuery.isFunction( s.jsonpCallback ) ?\n\t\t\ts.jsonpCallback() :\n\t\t\ts.jsonpCallback;\n\n\t\t// Insert callback into url or form data\n\t\tif ( jsonProp ) {\n\t\t\ts[ jsonProp ] = s[ jsonProp ].replace( rjsonp, \"$1\" + callbackName );\n\t\t} else if ( s.jsonp !== false ) {\n\t\t\ts.url += ( ajax_rquery.test( s.url ) ? \"&\" : \"?\" ) + s.jsonp + \"=\" + callbackName;\n\t\t}\n\n\t\t// Use data converter to retrieve json after script execution\n\t\ts.converters[\"script json\"] = function() {\n\t\t\tif ( !responseContainer ) {\n\t\t\t\tjQuery.error( callbackName + \" was not called\" );\n\t\t\t}\n\t\t\treturn responseContainer[ 0 ];\n\t\t};\n\n\t\t// force json dataType\n\t\ts.dataTypes[ 0 ] = \"json\";\n\n\t\t// Install callback\n\t\toverwritten = window[ callbackName ];\n\t\twindow[ callbackName ] = function() {\n\t\t\tresponseContainer = arguments;\n\t\t};\n\n\t\t// Clean-up function (fires after converters)\n\t\tjqXHR.always(function() {\n\t\t\t// Restore preexisting value\n\t\t\twindow[ callbackName ] = overwritten;\n\n\t\t\t// Save back as free\n\t\t\tif ( s[ callbackName ] ) {\n\t\t\t\t// make sure that re-using the options doesn't screw things around\n\t\t\t\ts.jsonpCallback = originalSettings.jsonpCallback;\n\n\t\t\t\t// save the callback name for future use\n\t\t\t\toldCallbacks.push( callbackName );\n\t\t\t}\n\n\t\t\t// Call if it was a function and we have a response\n\t\t\tif ( responseContainer && jQuery.isFunction( overwritten ) ) {\n\t\t\t\toverwritten( responseContainer[ 0 ] );\n\t\t\t}\n\n\t\t\tresponseContainer = overwritten = undefined;\n\t\t});\n\n\t\t// Delegate to script\n\t\treturn \"script\";\n\t}\n});\njQuery.ajaxSettings.xhr = function() {\n\ttry {\n\t\treturn new XMLHttpRequest();\n\t} catch( e ) {}\n};\n\nvar xhrSupported = jQuery.ajaxSettings.xhr(),\n\txhrSuccessStatus = {\n\t\t// file protocol always yields status code 0, assume 200\n\t\t0: 200,\n\t\t// Support: IE9\n\t\t// #1450: sometimes IE returns 1223 when it should be 204\n\t\t1223: 204\n\t},\n\t// Support: IE9\n\t// We need to keep track of outbound xhr and abort them manually\n\t// because IE is not smart enough to do it all by itself\n\txhrId = 0,\n\txhrCallbacks = {};\n\nif ( window.ActiveXObject ) {\n\tjQuery( window ).on( \"unload\", function() {\n\t\tfor( var key in xhrCallbacks ) {\n\t\t\txhrCallbacks[ key ]();\n\t\t}\n\t\txhrCallbacks = undefined;\n\t});\n}\n\njQuery.support.cors = !!xhrSupported && ( \"withCredentials\" in xhrSupported );\njQuery.support.ajax = xhrSupported = !!xhrSupported;\n\njQuery.ajaxTransport(function( options ) {\n\tvar callback;\n\t// Cross domain only allowed if supported through XMLHttpRequest\n\tif ( jQuery.support.cors || xhrSupported && !options.crossDomain ) {\n\t\treturn {\n\t\t\tsend: function( headers, complete ) {\n\t\t\t\tvar i, id,\n\t\t\t\t\txhr = options.xhr();\n\t\t\t\txhr.open( options.type, options.url, options.async, options.username, options.password );\n\t\t\t\t// Apply custom fields if provided\n\t\t\t\tif ( options.xhrFields ) {\n\t\t\t\t\tfor ( i in options.xhrFields ) {\n\t\t\t\t\t\txhr[ i ] = options.xhrFields[ i ];\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\t// Override mime type if needed\n\t\t\t\tif ( options.mimeType && xhr.overrideMimeType ) {\n\t\t\t\t\txhr.overrideMimeType( options.mimeType );\n\t\t\t\t}\n\t\t\t\t// X-Requested-With header\n\t\t\t\t// For cross-domain requests, seeing as conditions for a preflight are\n\t\t\t\t// akin to a jigsaw puzzle, we simply never set it to be sure.\n\t\t\t\t// (it can always be set on a per-request basis or even using ajaxSetup)\n\t\t\t\t// For same-domain requests, won't change header if already provided.\n\t\t\t\tif ( !options.crossDomain && !headers[\"X-Requested-With\"] ) {\n\t\t\t\t\theaders[\"X-Requested-With\"] = \"XMLHttpRequest\";\n\t\t\t\t}\n\t\t\t\t// Set headers\n\t\t\t\tfor ( i in headers ) {\n\t\t\t\t\txhr.setRequestHeader( i, headers[ i ] );\n\t\t\t\t}\n\t\t\t\t// Callback\n\t\t\t\tcallback = function( type ) {\n\t\t\t\t\treturn function() {\n\t\t\t\t\t\tif ( callback ) {\n\t\t\t\t\t\t\tdelete xhrCallbacks[ id ];\n\t\t\t\t\t\t\tcallback = xhr.onload = xhr.onerror = null;\n\t\t\t\t\t\t\tif ( type === \"abort\" ) {\n\t\t\t\t\t\t\t\txhr.abort();\n\t\t\t\t\t\t\t} else if ( type === \"error\" ) {\n\t\t\t\t\t\t\t\tcomplete(\n\t\t\t\t\t\t\t\t\t// file protocol always yields status 0, assume 404\n\t\t\t\t\t\t\t\t\txhr.status || 404,\n\t\t\t\t\t\t\t\t\txhr.statusText\n\t\t\t\t\t\t\t\t);\n\t\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\t\tcomplete(\n\t\t\t\t\t\t\t\t\txhrSuccessStatus[ xhr.status ] || xhr.status,\n\t\t\t\t\t\t\t\t\txhr.statusText,\n\t\t\t\t\t\t\t\t\t// Support: IE9\n\t\t\t\t\t\t\t\t\t// #11426: When requesting binary data, IE9 will throw an exception\n\t\t\t\t\t\t\t\t\t// on any attempt to access responseText\n\t\t\t\t\t\t\t\t\ttypeof xhr.responseText === \"string\" ? {\n\t\t\t\t\t\t\t\t\t\ttext: xhr.responseText\n\t\t\t\t\t\t\t\t\t} : undefined,\n\t\t\t\t\t\t\t\t\txhr.getAllResponseHeaders()\n\t\t\t\t\t\t\t\t);\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t};\n\t\t\t\t};\n\t\t\t\t// Listen to events\n\t\t\t\txhr.onload = callback();\n\t\t\t\txhr.onerror = callback(\"error\");\n\t\t\t\t// Create the abort callback\n\t\t\t\tcallback = xhrCallbacks[( id = xhrId++ )] = callback(\"abort\");\n\t\t\t\t// Do send the request\n\t\t\t\t// This may raise an exception which is actually\n\t\t\t\t// handled in jQuery.ajax (so no try/catch here)\n\t\t\t\txhr.send( options.hasContent && options.data || null );\n\t\t\t},\n\t\t\tabort: function() {\n\t\t\t\tif ( callback ) {\n\t\t\t\t\tcallback();\n\t\t\t\t}\n\t\t\t}\n\t\t};\n\t}\n});\nvar fxNow, timerId,\n\trfxtypes = /^(?:toggle|show|hide)$/,\n\trfxnum = new RegExp( \"^(?:([+-])=|)(\" + core_pnum + \")([a-z%]*)$\", \"i\" ),\n\trrun = /queueHooks$/,\n\tanimationPrefilters = [ defaultPrefilter ],\n\ttweeners = {\n\t\t\"*\": [function( prop, value ) {\n\t\t\tvar end, unit,\n\t\t\t\ttween = this.createTween( prop, value ),\n\t\t\t\tparts = rfxnum.exec( value ),\n\t\t\t\ttarget = tween.cur(),\n\t\t\t\tstart = +target || 0,\n\t\t\t\tscale = 1,\n\t\t\t\tmaxIterations = 20;\n\n\t\t\tif ( parts ) {\n\t\t\t\tend = +parts[2];\n\t\t\t\tunit = parts[3] || ( jQuery.cssNumber[ prop ] ? \"\" : \"px\" );\n\n\t\t\t\t// We need to compute starting value\n\t\t\t\tif ( unit !== \"px\" && start ) {\n\t\t\t\t\t// Iteratively approximate from a nonzero starting point\n\t\t\t\t\t// Prefer the current property, because this process will be trivial if it uses the same units\n\t\t\t\t\t// Fallback to end or a simple constant\n\t\t\t\t\tstart = jQuery.css( tween.elem, prop, true ) || end || 1;\n\n\t\t\t\t\tdo {\n\t\t\t\t\t\t// If previous iteration zeroed out, double until we get *something*\n\t\t\t\t\t\t// Use a string for doubling factor so we don't accidentally see scale as unchanged below\n\t\t\t\t\t\tscale = scale || \".5\";\n\n\t\t\t\t\t\t// Adjust and apply\n\t\t\t\t\t\tstart = start / scale;\n\t\t\t\t\t\tjQuery.style( tween.elem, prop, start + unit );\n\n\t\t\t\t\t// Update scale, tolerating zero or NaN from tween.cur()\n\t\t\t\t\t// And breaking the loop if scale is unchanged or perfect, or if we've just had enough\n\t\t\t\t\t} while ( scale !== (scale = tween.cur() / target) && scale !== 1 && --maxIterations );\n\t\t\t\t}\n\n\t\t\t\ttween.unit = unit;\n\t\t\t\ttween.start = start;\n\t\t\t\t// If a +=/-= token was provided, we're doing a relative animation\n\t\t\t\ttween.end = parts[1] ? start + ( parts[1] + 1 ) * end : end;\n\t\t\t}\n\t\t\treturn tween;\n\t\t}]\n\t};\n\n// Animations created synchronously will run synchronously\nfunction createFxNow() {\n\tsetTimeout(function() {\n\t\tfxNow = undefined;\n\t});\n\treturn ( fxNow = jQuery.now() );\n}\n\nfunction createTweens( animation, props ) {\n\tjQuery.each( props, function( prop, value ) {\n\t\tvar collection = ( tweeners[ prop ] || [] ).concat( tweeners[ \"*\" ] ),\n\t\t\tindex = 0,\n\t\t\tlength = collection.length;\n\t\tfor ( ; index < length; index++ ) {\n\t\t\tif ( collection[ index ].call( animation, prop, value ) ) {\n\n\t\t\t\t// we're done with this property\n\t\t\t\treturn;\n\t\t\t}\n\t\t}\n\t});\n}\n\nfunction Animation( elem, properties, options ) {\n\tvar result,\n\t\tstopped,\n\t\tindex = 0,\n\t\tlength = animationPrefilters.length,\n\t\tdeferred = jQuery.Deferred().always( function() {\n\t\t\t// don't match elem in the :animated selector\n\t\t\tdelete tick.elem;\n\t\t}),\n\t\ttick = function() {\n\t\t\tif ( stopped ) {\n\t\t\t\treturn false;\n\t\t\t}\n\t\t\tvar currentTime = fxNow || createFxNow(),\n\t\t\t\tremaining = Math.max( 0, animation.startTime + animation.duration - currentTime ),\n\t\t\t\t// archaic crash bug won't allow us to use 1 - ( 0.5 || 0 ) (#12497)\n\t\t\t\ttemp = remaining / animation.duration || 0,\n\t\t\t\tpercent = 1 - temp,\n\t\t\t\tindex = 0,\n\t\t\t\tlength = animation.tweens.length;\n\n\t\t\tfor ( ; index < length ; index++ ) {\n\t\t\t\tanimation.tweens[ index ].run( percent );\n\t\t\t}\n\n\t\t\tdeferred.notifyWith( elem, [ animation, percent, remaining ]);\n\n\t\t\tif ( percent < 1 && length ) {\n\t\t\t\treturn remaining;\n\t\t\t} else {\n\t\t\t\tdeferred.resolveWith( elem, [ animation ] );\n\t\t\t\treturn false;\n\t\t\t}\n\t\t},\n\t\tanimation = deferred.promise({\n\t\t\telem: elem,\n\t\t\tprops: jQuery.extend( {}, properties ),\n\t\t\topts: jQuery.extend( true, { specialEasing: {} }, options ),\n\t\t\toriginalProperties: properties,\n\t\t\toriginalOptions: options,\n\t\t\tstartTime: fxNow || createFxNow(),\n\t\t\tduration: options.duration,\n\t\t\ttweens: [],\n\t\t\tcreateTween: function( prop, end ) {\n\t\t\t\tvar tween = jQuery.Tween( elem, animation.opts, prop, end,\n\t\t\t\t\t\tanimation.opts.specialEasing[ prop ] || animation.opts.easing );\n\t\t\t\tanimation.tweens.push( tween );\n\t\t\t\treturn tween;\n\t\t\t},\n\t\t\tstop: function( gotoEnd ) {\n\t\t\t\tvar index = 0,\n\t\t\t\t\t// if we are going to the end, we want to run all the tweens\n\t\t\t\t\t// otherwise we skip this part\n\t\t\t\t\tlength = gotoEnd ? animation.tweens.length : 0;\n\t\t\t\tif ( stopped ) {\n\t\t\t\t\treturn this;\n\t\t\t\t}\n\t\t\t\tstopped = true;\n\t\t\t\tfor ( ; index < length ; index++ ) {\n\t\t\t\t\tanimation.tweens[ index ].run( 1 );\n\t\t\t\t}\n\n\t\t\t\t// resolve when we played the last frame\n\t\t\t\t// otherwise, reject\n\t\t\t\tif ( gotoEnd ) {\n\t\t\t\t\tdeferred.resolveWith( elem, [ animation, gotoEnd ] );\n\t\t\t\t} else {\n\t\t\t\t\tdeferred.rejectWith( elem, [ animation, gotoEnd ] );\n\t\t\t\t}\n\t\t\t\treturn this;\n\t\t\t}\n\t\t}),\n\t\tprops = animation.props;\n\n\tpropFilter( props, animation.opts.specialEasing );\n\n\tfor ( ; index < length ; index++ ) {\n\t\tresult = animationPrefilters[ index ].call( animation, elem, props, animation.opts );\n\t\tif ( result ) {\n\t\t\treturn result;\n\t\t}\n\t}\n\n\tcreateTweens( animation, props );\n\n\tif ( jQuery.isFunction( animation.opts.start ) ) {\n\t\tanimation.opts.start.call( elem, animation );\n\t}\n\n\tjQuery.fx.timer(\n\t\tjQuery.extend( tick, {\n\t\t\telem: elem,\n\t\t\tanim: animation,\n\t\t\tqueue: animation.opts.queue\n\t\t})\n\t);\n\n\t// attach callbacks from options\n\treturn animation.progress( animation.opts.progress )\n\t\t.done( animation.opts.done, animation.opts.complete )\n\t\t.fail( animation.opts.fail )\n\t\t.always( animation.opts.always );\n}\n\nfunction propFilter( props, specialEasing ) {\n\tvar index, name, easing, value, hooks;\n\n\t// camelCase, specialEasing and expand cssHook pass\n\tfor ( index in props ) {\n\t\tname = jQuery.camelCase( index );\n\t\teasing = specialEasing[ name ];\n\t\tvalue = props[ index ];\n\t\tif ( jQuery.isArray( value ) ) {\n\t\t\teasing = value[ 1 ];\n\t\t\tvalue = props[ index ] = value[ 0 ];\n\t\t}\n\n\t\tif ( index !== name ) {\n\t\t\tprops[ name ] = value;\n\t\t\tdelete props[ index ];\n\t\t}\n\n\t\thooks = jQuery.cssHooks[ name ];\n\t\tif ( hooks && \"expand\" in hooks ) {\n\t\t\tvalue = hooks.expand( value );\n\t\t\tdelete props[ name ];\n\n\t\t\t// not quite $.extend, this wont overwrite keys already present.\n\t\t\t// also - reusing 'index' from above because we have the correct \"name\"\n\t\t\tfor ( index in value ) {\n\t\t\t\tif ( !( index in props ) ) {\n\t\t\t\t\tprops[ index ] = value[ index ];\n\t\t\t\t\tspecialEasing[ index ] = easing;\n\t\t\t\t}\n\t\t\t}\n\t\t} else {\n\t\t\tspecialEasing[ name ] = easing;\n\t\t}\n\t}\n}\n\njQuery.Animation = jQuery.extend( Animation, {\n\n\ttweener: function( props, callback ) {\n\t\tif ( jQuery.isFunction( props ) ) {\n\t\t\tcallback = props;\n\t\t\tprops = [ \"*\" ];\n\t\t} else {\n\t\t\tprops = props.split(\" \");\n\t\t}\n\n\t\tvar prop,\n\t\t\tindex = 0,\n\t\t\tlength = props.length;\n\n\t\tfor ( ; index < length ; index++ ) {\n\t\t\tprop = props[ index ];\n\t\t\ttweeners[ prop ] = tweeners[ prop ] || [];\n\t\t\ttweeners[ prop ].unshift( callback );\n\t\t}\n\t},\n\n\tprefilter: function( callback, prepend ) {\n\t\tif ( prepend ) {\n\t\t\tanimationPrefilters.unshift( callback );\n\t\t} else {\n\t\t\tanimationPrefilters.push( callback );\n\t\t}\n\t}\n});\n\nfunction defaultPrefilter( elem, props, opts ) {\n\t/* jshint validthis: true */\n\tvar index, prop, value, length, dataShow, toggle, tween, hooks, oldfire,\n\t\tanim = this,\n\t\tstyle = elem.style,\n\t\torig = {},\n\t\thandled = [],\n\t\thidden = elem.nodeType && isHidden( elem );\n\n\t// handle queue: false promises\n\tif ( !opts.queue ) {\n\t\thooks = jQuery._queueHooks( elem, \"fx\" );\n\t\tif ( hooks.unqueued == null ) {\n\t\t\thooks.unqueued = 0;\n\t\t\toldfire = hooks.empty.fire;\n\t\t\thooks.empty.fire = function() {\n\t\t\t\tif ( !hooks.unqueued ) {\n\t\t\t\t\toldfire();\n\t\t\t\t}\n\t\t\t};\n\t\t}\n\t\thooks.unqueued++;\n\n\t\tanim.always(function() {\n\t\t\t// doing this makes sure that the complete handler will be called\n\t\t\t// before this completes\n\t\t\tanim.always(function() {\n\t\t\t\thooks.unqueued--;\n\t\t\t\tif ( !jQuery.queue( elem, \"fx\" ).length ) {\n\t\t\t\t\thooks.empty.fire();\n\t\t\t\t}\n\t\t\t});\n\t\t});\n\t}\n\n\t// height/width overflow pass\n\tif ( elem.nodeType === 1 && ( \"height\" in props || \"width\" in props ) ) {\n\t\t// Make sure that nothing sneaks out\n\t\t// Record all 3 overflow attributes because IE9-10 do not\n\t\t// change the overflow attribute when overflowX and\n\t\t// overflowY are set to the same value\n\t\topts.overflow = [ style.overflow, style.overflowX, style.overflowY ];\n\n\t\t// Set display property to inline-block for height/width\n\t\t// animations on inline elements that are having width/height animated\n\t\tif ( jQuery.css( elem, \"display\" ) === \"inline\" &&\n\t\t\t\tjQuery.css( elem, \"float\" ) === \"none\" ) {\n\n\t\t\tstyle.display = \"inline-block\";\n\t\t}\n\t}\n\n\tif ( opts.overflow ) {\n\t\tstyle.overflow = \"hidden\";\n\t\tanim.always(function() {\n\t\t\tstyle.overflow = opts.overflow[ 0 ];\n\t\t\tstyle.overflowX = opts.overflow[ 1 ];\n\t\t\tstyle.overflowY = opts.overflow[ 2 ];\n\t\t});\n\t}\n\n\n\t// show/hide pass\n\tdataShow = data_priv.get( elem, \"fxshow\" );\n\tfor ( index in props ) {\n\t\tvalue = props[ index ];\n\t\tif ( rfxtypes.exec( value ) ) {\n\t\t\tdelete props[ index ];\n\t\t\ttoggle = toggle || value === \"toggle\";\n\t\t\tif ( value === ( hidden ? \"hide\" : \"show\" ) ) {\n\n\t\t\t\t// If there is dataShow left over from a stopped hide or show and we are going to proceed with show, we should pretend to be hidden\n\t\t\t\tif( value === \"show\" && dataShow !== undefined && dataShow[ index ] !== undefined ) {\n\t\t\t\t\thidden = true;\n\t\t\t\t} else {\n\t\t\t\t\tcontinue;\n\t\t\t\t}\n\t\t\t}\n\t\t\thandled.push( index );\n\t\t}\n\t}\n\n\tlength = handled.length;\n\tif ( length ) {\n\t\tdataShow = data_priv.get( elem, \"fxshow\" ) || data_priv.access( elem, \"fxshow\", {} );\n\t\tif ( \"hidden\" in dataShow ) {\n\t\t\thidden = dataShow.hidden;\n\t\t}\n\n\t\t// store state if its toggle - enables .stop().toggle() to \"reverse\"\n\t\tif ( toggle ) {\n\t\t\tdataShow.hidden = !hidden;\n\t\t}\n\t\tif ( hidden ) {\n\t\t\tjQuery( elem ).show();\n\t\t} else {\n\t\t\tanim.done(function() {\n\t\t\t\tjQuery( elem ).hide();\n\t\t\t});\n\t\t}\n\t\tanim.done(function() {\n\t\t\tvar prop;\n\n\t\t\tdata_priv.remove( elem, \"fxshow\" );\n\t\t\tfor ( prop in orig ) {\n\t\t\t\tjQuery.style( elem, prop, orig[ prop ] );\n\t\t\t}\n\t\t});\n\t\tfor ( index = 0 ; index < length ; index++ ) {\n\t\t\tprop = handled[ index ];\n\t\t\ttween = anim.createTween( prop, hidden ? dataShow[ prop ] : 0 );\n\t\t\torig[ prop ] = dataShow[ prop ] || jQuery.style( elem, prop );\n\n\t\t\tif ( !( prop in dataShow ) ) {\n\t\t\t\tdataShow[ prop ] = tween.start;\n\t\t\t\tif ( hidden ) {\n\t\t\t\t\ttween.end = tween.start;\n\t\t\t\t\ttween.start = prop === \"width\" || prop === \"height\" ? 1 : 0;\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunction Tween( elem, options, prop, end, easing ) {\n\treturn new Tween.prototype.init( elem, options, prop, end, easing );\n}\njQuery.Tween = Tween;\n\nTween.prototype = {\n\tconstructor: Tween,\n\tinit: function( elem, options, prop, end, easing, unit ) {\n\t\tthis.elem = elem;\n\t\tthis.prop = prop;\n\t\tthis.easing = easing || \"swing\";\n\t\tthis.options = options;\n\t\tthis.start = this.now = this.cur();\n\t\tthis.end = end;\n\t\tthis.unit = unit || ( jQuery.cssNumber[ prop ] ? \"\" : \"px\" );\n\t},\n\tcur: function() {\n\t\tvar hooks = Tween.propHooks[ this.prop ];\n\n\t\treturn hooks && hooks.get ?\n\t\t\thooks.get( this ) :\n\t\t\tTween.propHooks._default.get( this );\n\t},\n\trun: function( percent ) {\n\t\tvar eased,\n\t\t\thooks = Tween.propHooks[ this.prop ];\n\n\t\tif ( this.options.duration ) {\n\t\t\tthis.pos = eased = jQuery.easing[ this.easing ](\n\t\t\t\tpercent, this.options.duration * percent, 0, 1, this.options.duration\n\t\t\t);\n\t\t} else {\n\t\t\tthis.pos = eased = percent;\n\t\t}\n\t\tthis.now = ( this.end - this.start ) * eased + this.start;\n\n\t\tif ( this.options.step ) {\n\t\t\tthis.options.step.call( this.elem, this.now, this );\n\t\t}\n\n\t\tif ( hooks && hooks.set ) {\n\t\t\thooks.set( this );\n\t\t} else {\n\t\t\tTween.propHooks._default.set( this );\n\t\t}\n\t\treturn this;\n\t}\n};\n\nTween.prototype.init.prototype = Tween.prototype;\n\nTween.propHooks = {\n\t_default: {\n\t\tget: function( tween ) {\n\t\t\tvar result;\n\n\t\t\tif ( tween.elem[ tween.prop ] != null &&\n\t\t\t\t(!tween.elem.style || tween.elem.style[ tween.prop ] == null) ) {\n\t\t\t\treturn tween.elem[ tween.prop ];\n\t\t\t}\n\n\t\t\t// passing an empty string as a 3rd parameter to .css will automatically\n\t\t\t// attempt a parseFloat and fallback to a string if the parse fails\n\t\t\t// so, simple values such as \"10px\" are parsed to Float.\n\t\t\t// complex values such as \"rotate(1rad)\" are returned as is.\n\t\t\tresult = jQuery.css( tween.elem, tween.prop, \"\" );\n\t\t\t// Empty strings, null, undefined and \"auto\" are converted to 0.\n\t\t\treturn !result || result === \"auto\" ? 0 : result;\n\t\t},\n\t\tset: function( tween ) {\n\t\t\t// use step hook for back compat - use cssHook if its there - use .style if its\n\t\t\t// available and use plain properties where available\n\t\t\tif ( jQuery.fx.step[ tween.prop ] ) {\n\t\t\t\tjQuery.fx.step[ tween.prop ]( tween );\n\t\t\t} else if ( tween.elem.style && ( tween.elem.style[ jQuery.cssProps[ tween.prop ] ] != null || jQuery.cssHooks[ tween.prop ] ) ) {\n\t\t\t\tjQuery.style( tween.elem, tween.prop, tween.now + tween.unit );\n\t\t\t} else {\n\t\t\t\ttween.elem[ tween.prop ] = tween.now;\n\t\t\t}\n\t\t}\n\t}\n};\n\n// Support: IE9\n// Panic based approach to setting things on disconnected nodes\n\nTween.propHooks.scrollTop = Tween.propHooks.scrollLeft = {\n\tset: function( tween ) {\n\t\tif ( tween.elem.nodeType && tween.elem.parentNode ) {\n\t\t\ttween.elem[ tween.prop ] = tween.now;\n\t\t}\n\t}\n};\n\njQuery.each([ \"toggle\", \"show\", \"hide\" ], function( i, name ) {\n\tvar cssFn = jQuery.fn[ name ];\n\tjQuery.fn[ name ] = function( speed, easing, callback ) {\n\t\treturn speed == null || typeof speed === \"boolean\" ?\n\t\t\tcssFn.apply( this, arguments ) :\n\t\t\tthis.animate( genFx( name, true ), speed, easing, callback );\n\t};\n});\n\njQuery.fn.extend({\n\tfadeTo: function( speed, to, easing, callback ) {\n\n\t\t// show any hidden elements after setting opacity to 0\n\t\treturn this.filter( isHidden ).css( \"opacity\", 0 ).show()\n\n\t\t\t// animate to the value specified\n\t\t\t.end().animate({ opacity: to }, speed, easing, callback );\n\t},\n\tanimate: function( prop, speed, easing, callback ) {\n\t\tvar empty = jQuery.isEmptyObject( prop ),\n\t\t\toptall = jQuery.speed( speed, easing, callback ),\n\t\t\tdoAnimation = function() {\n\t\t\t\t// Operate on a copy of prop so per-property easing won't be lost\n\t\t\t\tvar anim = Animation( this, jQuery.extend( {}, prop ), optall );\n\t\t\t\tdoAnimation.finish = function() {\n\t\t\t\t\tanim.stop( true );\n\t\t\t\t};\n\t\t\t\t// Empty animations, or finishing resolves immediately\n\t\t\t\tif ( empty || data_priv.get( this, \"finish\" ) ) {\n\t\t\t\t\tanim.stop( true );\n\t\t\t\t}\n\t\t\t};\n\t\t\tdoAnimation.finish = doAnimation;\n\n\t\treturn empty || optall.queue === false ?\n\t\t\tthis.each( doAnimation ) :\n\t\t\tthis.queue( optall.queue, doAnimation );\n\t},\n\tstop: function( type, clearQueue, gotoEnd ) {\n\t\tvar stopQueue = function( hooks ) {\n\t\t\tvar stop = hooks.stop;\n\t\t\tdelete hooks.stop;\n\t\t\tstop( gotoEnd );\n\t\t};\n\n\t\tif ( typeof type !== \"string\" ) {\n\t\t\tgotoEnd = clearQueue;\n\t\t\tclearQueue = type;\n\t\t\ttype = undefined;\n\t\t}\n\t\tif ( clearQueue && type !== false ) {\n\t\t\tthis.queue( type || \"fx\", [] );\n\t\t}\n\n\t\treturn this.each(function() {\n\t\t\tvar dequeue = true,\n\t\t\t\tindex = type != null && type + \"queueHooks\",\n\t\t\t\ttimers = jQuery.timers,\n\t\t\t\tdata = data_priv.get( this );\n\n\t\t\tif ( index ) {\n\t\t\t\tif ( data[ index ] && data[ index ].stop ) {\n\t\t\t\t\tstopQueue( data[ index ] );\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tfor ( index in data ) {\n\t\t\t\t\tif ( data[ index ] && data[ index ].stop && rrun.test( index ) ) {\n\t\t\t\t\t\tstopQueue( data[ index ] );\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tfor ( index = timers.length; index--; ) {\n\t\t\t\tif ( timers[ index ].elem === this && (type == null || timers[ index ].queue === type) ) {\n\t\t\t\t\ttimers[ index ].anim.stop( gotoEnd );\n\t\t\t\t\tdequeue = false;\n\t\t\t\t\ttimers.splice( index, 1 );\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// start the next in the queue if the last step wasn't forced\n\t\t\t// timers currently will call their complete callbacks, which will dequeue\n\t\t\t// but only if they were gotoEnd\n\t\t\tif ( dequeue || !gotoEnd ) {\n\t\t\t\tjQuery.dequeue( this, type );\n\t\t\t}\n\t\t});\n\t},\n\tfinish: function( type ) {\n\t\tif ( type !== false ) {\n\t\t\ttype = type || \"fx\";\n\t\t}\n\t\treturn this.each(function() {\n\t\t\tvar index,\n\t\t\t\tdata = data_priv.get( this ),\n\t\t\t\tqueue = data[ type + \"queue\" ],\n\t\t\t\thooks = data[ type + \"queueHooks\" ],\n\t\t\t\ttimers = jQuery.timers,\n\t\t\t\tlength = queue ? queue.length : 0;\n\n\t\t\t// enable finishing flag on private data\n\t\t\tdata.finish = true;\n\n\t\t\t// empty the queue first\n\t\t\tjQuery.queue( this, type, [] );\n\n\t\t\tif ( hooks && hooks.cur && hooks.cur.finish ) {\n\t\t\t\thooks.cur.finish.call( this );\n\t\t\t}\n\n\t\t\t// look for any active animations, and finish them\n\t\t\tfor ( index = timers.length; index--; ) {\n\t\t\t\tif ( timers[ index ].elem === this && timers[ index ].queue === type ) {\n\t\t\t\t\ttimers[ index ].anim.stop( true );\n\t\t\t\t\ttimers.splice( index, 1 );\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// look for any animations in the old queue and finish them\n\t\t\tfor ( index = 0; index < length; index++ ) {\n\t\t\t\tif ( queue[ index ] && queue[ index ].finish ) {\n\t\t\t\t\tqueue[ index ].finish.call( this );\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// turn off finishing flag\n\t\t\tdelete data.finish;\n\t\t});\n\t}\n});\n\n// Generate parameters to create a standard animation\nfunction genFx( type, includeWidth ) {\n\tvar which,\n\t\tattrs = { height: type },\n\t\ti = 0;\n\n\t// if we include width, step value is 1 to do all cssExpand values,\n\t// if we don't include width, step value is 2 to skip over Left and Right\n\tincludeWidth = includeWidth? 1 : 0;\n\tfor( ; i < 4 ; i += 2 - includeWidth ) {\n\t\twhich = cssExpand[ i ];\n\t\tattrs[ \"margin\" + which ] = attrs[ \"padding\" + which ] = type;\n\t}\n\n\tif ( includeWidth ) {\n\t\tattrs.opacity = attrs.width = type;\n\t}\n\n\treturn attrs;\n}\n\n// Generate shortcuts for custom animations\njQuery.each({\n\tslideDown: genFx(\"show\"),\n\tslideUp: genFx(\"hide\"),\n\tslideToggle: genFx(\"toggle\"),\n\tfadeIn: { opacity: \"show\" },\n\tfadeOut: { opacity: \"hide\" },\n\tfadeToggle: { opacity: \"toggle\" }\n}, function( name, props ) {\n\tjQuery.fn[ name ] = function( speed, easing, callback ) {\n\t\treturn this.animate( props, speed, easing, callback );\n\t};\n});\n\njQuery.speed = function( speed, easing, fn ) {\n\tvar opt = speed && typeof speed === \"object\" ? jQuery.extend( {}, speed ) : {\n\t\tcomplete: fn || !fn && easing ||\n\t\t\tjQuery.isFunction( speed ) && speed,\n\t\tduration: speed,\n\t\teasing: fn && easing || easing && !jQuery.isFunction( easing ) && easing\n\t};\n\n\topt.duration = jQuery.fx.off ? 0 : typeof opt.duration === \"number\" ? opt.duration :\n\t\topt.duration in jQuery.fx.speeds ? jQuery.fx.speeds[ opt.duration ] : jQuery.fx.speeds._default;\n\n\t// normalize opt.queue - true/undefined/null -> \"fx\"\n\tif ( opt.queue == null || opt.queue === true ) {\n\t\topt.queue = \"fx\";\n\t}\n\n\t// Queueing\n\topt.old = opt.complete;\n\n\topt.complete = function() {\n\t\tif ( jQuery.isFunction( opt.old ) ) {\n\t\t\topt.old.call( this );\n\t\t}\n\n\t\tif ( opt.queue ) {\n\t\t\tjQuery.dequeue( this, opt.queue );\n\t\t}\n\t};\n\n\treturn opt;\n};\n\njQuery.easing = {\n\tlinear: function( p ) {\n\t\treturn p;\n\t},\n\tswing: function( p ) {\n\t\treturn 0.5 - Math.cos( p*Math.PI ) / 2;\n\t}\n};\n\njQuery.timers = [];\njQuery.fx = Tween.prototype.init;\njQuery.fx.tick = function() {\n\tvar timer,\n\t\ttimers = jQuery.timers,\n\t\ti = 0;\n\n\tfxNow = jQuery.now();\n\n\tfor ( ; i < timers.length; i++ ) {\n\t\ttimer = timers[ i ];\n\t\t// Checks the timer has not already been removed\n\t\tif ( !timer() && timers[ i ] === timer ) {\n\t\t\ttimers.splice( i--, 1 );\n\t\t}\n\t}\n\n\tif ( !timers.length ) {\n\t\tjQuery.fx.stop();\n\t}\n\tfxNow = undefined;\n};\n\njQuery.fx.timer = function( timer ) {\n\tif ( timer() && jQuery.timers.push( timer ) ) {\n\t\tjQuery.fx.start();\n\t}\n};\n\njQuery.fx.interval = 13;\n\njQuery.fx.start = function() {\n\tif ( !timerId ) {\n\t\ttimerId = setInterval( jQuery.fx.tick, jQuery.fx.interval );\n\t}\n};\n\njQuery.fx.stop = function() {\n\tclearInterval( timerId );\n\ttimerId = null;\n};\n\njQuery.fx.speeds = {\n\tslow: 600,\n\tfast: 200,\n\t// Default speed\n\t_default: 400\n};\n\n// Back Compat <1.8 extension point\njQuery.fx.step = {};\n\nif ( jQuery.expr && jQuery.expr.filters ) {\n\tjQuery.expr.filters.animated = function( elem ) {\n\t\treturn jQuery.grep(jQuery.timers, function( fn ) {\n\t\t\treturn elem === fn.elem;\n\t\t}).length;\n\t};\n}\njQuery.fn.offset = function( options ) {\n\tif ( arguments.length ) {\n\t\treturn options === undefined ?\n\t\t\tthis :\n\t\t\tthis.each(function( i ) {\n\t\t\t\tjQuery.offset.setOffset( this, options, i );\n\t\t\t});\n\t}\n\n\tvar docElem, win,\n\t\telem = this[ 0 ],\n\t\tbox = { top: 0, left: 0 },\n\t\tdoc = elem && elem.ownerDocument;\n\n\tif ( !doc ) {\n\t\treturn;\n\t}\n\n\tdocElem = doc.documentElement;\n\n\t// Make sure it's not a disconnected DOM node\n\tif ( !jQuery.contains( docElem, elem ) ) {\n\t\treturn box;\n\t}\n\n\t// If we don't have gBCR, just use 0,0 rather than error\n\t// BlackBerry 5, iOS 3 (original iPhone)\n\tif ( typeof elem.getBoundingClientRect !== core_strundefined ) {\n\t\tbox = elem.getBoundingClientRect();\n\t}\n\twin = getWindow( doc );\n\treturn {\n\t\ttop: box.top + win.pageYOffset - docElem.clientTop,\n\t\tleft: box.left + win.pageXOffset - docElem.clientLeft\n\t};\n};\n\njQuery.offset = {\n\n\tsetOffset: function( elem, options, i ) {\n\t\tvar curPosition, curLeft, curCSSTop, curTop, curOffset, curCSSLeft, calculatePosition,\n\t\t\tposition = jQuery.css( elem, \"position\" ),\n\t\t\tcurElem = jQuery( elem ),\n\t\t\tprops = {};\n\n\t\t// Set position first, in-case top/left are set even on static elem\n\t\tif ( position === \"static\" ) {\n\t\t\telem.style.position = \"relative\";\n\t\t}\n\n\t\tcurOffset = curElem.offset();\n\t\tcurCSSTop = jQuery.css( elem, \"top\" );\n\t\tcurCSSLeft = jQuery.css( elem, \"left\" );\n\t\tcalculatePosition = ( position === \"absolute\" || position === \"fixed\" ) && ( curCSSTop + curCSSLeft ).indexOf(\"auto\") > -1;\n\n\t\t// Need to be able to calculate position if either top or left is auto and position is either absolute or fixed\n\t\tif ( calculatePosition ) {\n\t\t\tcurPosition = curElem.position();\n\t\t\tcurTop = curPosition.top;\n\t\t\tcurLeft = curPosition.left;\n\n\t\t} else {\n\t\t\tcurTop = parseFloat( curCSSTop ) || 0;\n\t\t\tcurLeft = parseFloat( curCSSLeft ) || 0;\n\t\t}\n\n\t\tif ( jQuery.isFunction( options ) ) {\n\t\t\toptions = options.call( elem, i, curOffset );\n\t\t}\n\n\t\tif ( options.top != null ) {\n\t\t\tprops.top = ( options.top - curOffset.top ) + curTop;\n\t\t}\n\t\tif ( options.left != null ) {\n\t\t\tprops.left = ( options.left - curOffset.left ) + curLeft;\n\t\t}\n\n\t\tif ( \"using\" in options ) {\n\t\t\toptions.using.call( elem, props );\n\n\t\t} else {\n\t\t\tcurElem.css( props );\n\t\t}\n\t}\n};\n\n\njQuery.fn.extend({\n\n\tposition: function() {\n\t\tif ( !this[ 0 ] ) {\n\t\t\treturn;\n\t\t}\n\n\t\tvar offsetParent, offset,\n\t\t\telem = this[ 0 ],\n\t\t\tparentOffset = { top: 0, left: 0 };\n\n\t\t// Fixed elements are offset from window (parentOffset = {top:0, left: 0}, because it is it's only offset parent\n\t\tif ( jQuery.css( elem, \"position\" ) === \"fixed\" ) {\n\t\t\t// We assume that getBoundingClientRect is available when computed position is fixed\n\t\t\toffset = elem.getBoundingClientRect();\n\n\t\t} else {\n\t\t\t// Get *real* offsetParent\n\t\t\toffsetParent = this.offsetParent();\n\n\t\t\t// Get correct offsets\n\t\t\toffset = this.offset();\n\t\t\tif ( !jQuery.nodeName( offsetParent[ 0 ], \"html\" ) ) {\n\t\t\t\tparentOffset = offsetParent.offset();\n\t\t\t}\n\n\t\t\t// Add offsetParent borders\n\t\t\tparentOffset.top += jQuery.css( offsetParent[ 0 ], \"borderTopWidth\", true );\n\t\t\tparentOffset.left += jQuery.css( offsetParent[ 0 ], \"borderLeftWidth\", true );\n\t\t}\n\n\t\t// Subtract parent offsets and element margins\n\t\treturn {\n\t\t\ttop: offset.top - parentOffset.top - jQuery.css( elem, \"marginTop\", true ),\n\t\t\tleft: offset.left - parentOffset.left - jQuery.css( elem, \"marginLeft\", true )\n\t\t};\n\t},\n\n\toffsetParent: function() {\n\t\treturn this.map(function() {\n\t\t\tvar offsetParent = this.offsetParent || docElem;\n\n\t\t\twhile ( offsetParent && ( !jQuery.nodeName( offsetParent, \"html\" ) && jQuery.css( offsetParent, \"position\") === \"static\" ) ) {\n\t\t\t\toffsetParent = offsetParent.offsetParent;\n\t\t\t}\n\n\t\t\treturn offsetParent || docElem;\n\t\t});\n\t}\n});\n\n\n// Create scrollLeft and scrollTop methods\njQuery.each( {scrollLeft: \"pageXOffset\", scrollTop: \"pageYOffset\"}, function( method, prop ) {\n\tvar top = \"pageYOffset\" === prop;\n\n\tjQuery.fn[ method ] = function( val ) {\n\t\treturn jQuery.access( this, function( elem, method, val ) {\n\t\t\tvar win = getWindow( elem );\n\n\t\t\tif ( val === undefined ) {\n\t\t\t\treturn win ? win[ prop ] : elem[ method ];\n\t\t\t}\n\n\t\t\tif ( win ) {\n\t\t\t\twin.scrollTo(\n\t\t\t\t\t!top ? val : window.pageXOffset,\n\t\t\t\t\ttop ? val : window.pageYOffset\n\t\t\t\t);\n\n\t\t\t} else {\n\t\t\t\telem[ method ] = val;\n\t\t\t}\n\t\t}, method, val, arguments.length, null );\n\t};\n});\n\nfunction getWindow( elem ) {\n\treturn jQuery.isWindow( elem ) ? elem : elem.nodeType === 9 && elem.defaultView;\n}\n// Create innerHeight, innerWidth, height, width, outerHeight and outerWidth methods\njQuery.each( { Height: \"height\", Width: \"width\" }, function( name, type ) {\n\tjQuery.each( { padding: \"inner\" + name, content: type, \"\": \"outer\" + name }, function( defaultExtra, funcName ) {\n\t\t// margin is only for outerHeight, outerWidth\n\t\tjQuery.fn[ funcName ] = function( margin, value ) {\n\t\t\tvar chainable = arguments.length && ( defaultExtra || typeof margin !== \"boolean\" ),\n\t\t\t\textra = defaultExtra || ( margin === true || value === true ? \"margin\" : \"border\" );\n\n\t\t\treturn jQuery.access( this, function( elem, type, value ) {\n\t\t\t\tvar doc;\n\n\t\t\t\tif ( jQuery.isWindow( elem ) ) {\n\t\t\t\t\t// As of 5/8/2012 this will yield incorrect results for Mobile Safari, but there\n\t\t\t\t\t// isn't a whole lot we can do. See pull request at this URL for discussion:\n\t\t\t\t\t// https://github.com/jquery/jquery/pull/764\n\t\t\t\t\treturn elem.document.documentElement[ \"client\" + name ];\n\t\t\t\t}\n\n\t\t\t\t// Get document width or height\n\t\t\t\tif ( elem.nodeType === 9 ) {\n\t\t\t\t\tdoc = elem.documentElement;\n\n\t\t\t\t\t// Either scroll[Width/Height] or offset[Width/Height] or client[Width/Height],\n\t\t\t\t\t// whichever is greatest\n\t\t\t\t\treturn Math.max(\n\t\t\t\t\t\telem.body[ \"scroll\" + name ], doc[ \"scroll\" + name ],\n\t\t\t\t\t\telem.body[ \"offset\" + name ], doc[ \"offset\" + name ],\n\t\t\t\t\t\tdoc[ \"client\" + name ]\n\t\t\t\t\t);\n\t\t\t\t}\n\n\t\t\t\treturn value === undefined ?\n\t\t\t\t\t// Get width or height on the element, requesting but not forcing parseFloat\n\t\t\t\t\tjQuery.css( elem, type, extra ) :\n\n\t\t\t\t\t// Set width or height on the element\n\t\t\t\t\tjQuery.style( elem, type, value, extra );\n\t\t\t}, type, chainable ? margin : undefined, chainable, null );\n\t\t};\n\t});\n});\n// Limit scope pollution from any deprecated API\n// (function() {\n\n// The number of elements contained in the matched element set\njQuery.fn.size = function() {\n\treturn this.length;\n};\n\njQuery.fn.andSelf = jQuery.fn.addBack;\n\n// })();\nif ( typeof module === \"object\" && typeof module.exports === \"object\" ) {\n\t// Expose jQuery as module.exports in loaders that implement the Node\n\t// module pattern (including browserify). Do not create the global, since\n\t// the user will be storing it themselves locally, and globals are frowned\n\t// upon in the Node module world.\n\tmodule.exports = jQuery;\n} else {\n\t// Register as a named AMD module, since jQuery can be concatenated with other\n\t// files that may use define, but not via a proper concatenation script that\n\t// understands anonymous AMD modules. A named AMD is safest and most robust\n\t// way to register. Lowercase jquery is used because AMD module names are\n\t// derived from file names, and jQuery is normally delivered in a lowercase\n\t// file name. Do this after creating the global so that if an AMD module wants\n\t// to call noConflict to hide this version of jQuery, it will work.\n\tif ( typeof define === \"function\" && define.amd ) {\n\t\tdefine( \"jquery\", [], function () { return jQuery; } );\n\t}\n}\n\n// If there is a window object, that at least has a document property,\n// define jQuery and $ identifiers\nif ( typeof window === \"object\" && typeof window.document === \"object\" ) {\n\twindow.jQuery = window.$ = jQuery;\n}\n\n})( window );\n" }, { "path": "extensions/chrome/js/jquery/package.json", "content": "{\n \"name\": \"components-jquery\",\n \"version\": \"2.0.0\",\n \"description\": \"jQuery component\",\n \"keywords\": [\"jquery\"],\n \"main\": \"./jquery.js\"\n}\n" }, { "path": "extensions/chrome/manifest.json", "content": "{\n \"name\": \"DocsGPT - Documentation AI butler\",\n \"version\": \"0.0.1\",\n \"manifest_version\": 3,\n \"description\": \"AI assistant for developers, that helps you answer your questions about the documentation you are reading.\",\n \"icons\": {\n \"16\": \"icons/icon16.png\",\n \"48\": \"icons/icon48.png\",\n \"128\": \"icons/icon128.png\"\n },\n \"default_locale\": \"en\",\n \"background\": {\n \"service_worker\": \"src/bg/service-worker.js\"\n },\n \"action\": {\n \"default_title\": \"DocsGPT - Documentation AI butler\",\n \"default_popup\": \"popup.html\"\n },\n \"permissions\": [\"activeTab\", \"storage\"],\n \"host_permissions\": [\n \"*://*/*\"\n ],\n \"content_scripts\": [{\n \"js\": [\"popup.js\"],\n \"matches\": [\"https://github.com/*\"]\n }]\n\n}\n" }, { "path": "extensions/chrome/package.json", "content": "{\n \"name\": \"docsgpt-chrome-extension\",\n \"version\": \"0.0.1\", \n \"description\": \"DocsGPT - Documentation AI butler\",\n \"main\": \"popup.js\",\n \"author\": \"\",\n \"license\": \"MIT\",\n \"scripts\": {\n \"dev\": \"npx tailwindcss -i ./styles.css -o ./dist/output.css --watch\"\n },\n \"keywords\": [\n \"DocsGPT\",\n \"Documentation\",\n \"Chrome\",\n \"extension\"\n ],\n \"devDependencies\": {\n \"tailwindcss\": \"^3.2.4\"\n }\n}\n" }, { "path": "extensions/chrome/popup.html", "content": "<!DOCTYPE html>\n<html>\n <head>\n <title>Chat Extension\n \n \n\n\n\n \n \n
\n

DocsGPT

\n About\n
\n
\n
\n \n
\n
\n

Hello, ask me anything about this library. Im here to help

\n
\n
\n

How to create API key for Api gateway?

\n
\n
\n

Import the boto3 library and create a client for the API Gateway service:

\n \n\n
\n
\n client = boto3.client('apigateway') \n \n\n
\n
\n

Create an API key:

\n \n
\n
\n response = client.create_api_key(
name='API_KEY_NAME',
description='API key description',
enabled=True)
api_key = response['value']\n\n
\n
\n
\n
\n
\n \n \n
\n
\n\n
\n \n \n\n" }, { "path": "extensions/chrome/popup.js", "content": "document.getElementById(\"message-form\").addEventListener(\"submit\", function(event) {\n event.preventDefault();\n var message = document.getElementById(\"message-input\").value;\n chrome.runtime.sendMessage({msg: \"sendMessage\", message: message}, function(response) {\n console.log(response.response);\n msg_html = '

'\n msg_html += message\n msg_html += '

'\n document.getElementById(\"messages\").innerHTML += msg_html;\n let chatWindow = document.getElementById(\"chat-container\");\n chatWindow.scrollTop = chatWindow.scrollHeight;\n\n });\n\n document.getElementById(\"message-input\").value = \"\";\n var conversation_state = localStorage.getItem(\"conversation_state\");\n // check if conversation state is null\n if (conversation_state == null) {\n conversation_state = 0;\n localStorage.setItem(\"conversation_state\", conversation_state);\n }\n\n // send post request to server http://127.0.0.1:5000/ with message in json body\n fetch('http://127.0.0.1:7091/api/answer', {\n method: 'POST',\n headers: {\n 'Content-Type': 'application/json',\n },\n body: JSON.stringify({question: message, history: null}),\n })\n .then(response => response.json())\n .then(data => {\n console.log('Success:', data);\n\n \n msg_html = '
'\n msg_html += data.answer\n msg_html += '
'\n document.getElementById(\"messages\").innerHTML += msg_html;\n let chatWindow = document.getElementById(\"chat-container\");\n chatWindow.scrollTop = chatWindow.scrollHeight;\n })\n\n\n });\n " }, { "path": "extensions/chrome/src/bg/service-worker.js", "content": "// This is the service worker script, which executes in its own context\n// when the extension is installed or refreshed (or when you access its console).\n// It would correspond to the background script in chrome extensions v2.\n\nconsole.log(\"This prints to the console of the service worker (background script)\");\nchrome.runtime.onMessage.addListener(\n function(request, sender, sendResponse) {\n if (request.msg === \"sendMessage\") {\n sendResponse({response: \"Message received\"});\n }\n }\n);" }, { "path": "extensions/chrome/styles.css", "content": "@tailwind base;\n@tailwind components;\n@tailwind utilities;\n\n\n#chat-container {\n width: 500px;\n height: 450px;\n background-color: white;\n padding: 10px;\n overflow: auto;\n}\n\n\n\n.bg-gray-200 {\n background-color: #edf2f7;\n }\n \n .bg-gray-900 {\n background-color: #1a202c;\n }\n \n .rounded-lg {\n border-radius: 0.5rem;\n }\n \n .shadow {\n box-shadow: 0 1px 3px rgba(0, 0, 0, 0.12), 0 1px 2px rgba(0, 0, 0, 0.24);\n }\n \n .text-gray-700 {\n color: #4a5568;\n }\n \n .text-sm {\n font-size: 0.875rem;\n }\n \n .p-4 {\n padding: 1.5rem;\n }\n \n\n\n" }, { "path": "extensions/chrome/tailwind.config.js", "content": "module.exports = {\n content: [\"./src/**/*.{html,js}\", \"./*.{html,js,css}\"],\n theme: {\n extend: {},\n },\n plugins: [],\n }" }, { "path": "extensions/discord/__init__.py", "content": "" }, { "path": "extensions/discord/bot.py", "content": "import os\nimport re\nimport logging\nimport aiohttp\nimport discord\nfrom discord.ext import commands\nimport dotenv\n\ndotenv.load_dotenv()\n\n# Enable logging\nlogging.basicConfig(level=logging.INFO)\nlogger = logging.getLogger(__name__)\n\n# Bot configuration\nTOKEN = os.getenv(\"DISCORD_TOKEN\")\nPREFIX = '!' # Command prefix\nBASE_API_URL = os.getenv(\"API_BASE\", \"https://gptcloud.arc53.com\")\nAPI_URL = BASE_API_URL + \"/api/answer\"\nAPI_KEY = os.getenv(\"API_KEY\")\n\nintents = discord.Intents.default()\nintents.message_content = True\n\nbot = commands.Bot(command_prefix=PREFIX, intents=intents)\n\n# Store conversation history per user\nconversation_histories = {}\n\ndef chunk_string(text, max_length=2000):\n \"\"\"Splits a string into chunks of a specified maximum length.\"\"\"\n # Create list to store the split strings\n chunks = []\n # Loop through the text, create substrings with max_length\n while len(text) > max_length:\n # Find last space within the limit\n idx = text.rfind(' ', 0, max_length)\n # Ensure we don't have an empty part\n if idx == -1:\n # If no spaces, just take chunk\n chunks.append(text[:max_length])\n text = text[max_length:]\n else:\n # Push whatever we've got up to the last space\n chunks.append(text[:idx])\n text = text[idx+1:]\n # Catches the remaining part\n chunks.append(text)\n return chunks\n\ndef escape_markdown(text):\n \"\"\"Escapes Discord markdown characters.\"\"\"\n escape_chars = r'\\*_$$$$()~>#+-=|{}.!'\n return re.sub(f'([{re.escape(escape_chars)}])', r'\\\\\\1', text)\n\ndef split_string(input_str):\n \"\"\"Splits the input string to detect bot mentions.\"\"\"\n pattern = r'^<@!?{0}>\\s*'.format(bot.user.id)\n match = re.match(pattern, input_str)\n if match:\n content = input_str[match.end():].strip()\n return str(bot.user.id), content\n return None, input_str\n\n@bot.event\nasync def on_ready():\n print(f'{bot.user.name} has connected to Discord!')\n\nasync def generate_answer(question, messages, conversation_id):\n \"\"\"Generates an answer using the external API.\"\"\"\n payload = {\n \"question\": question,\n \"api_key\": API_KEY,\n \"history\": messages,\n \"conversation_id\": conversation_id\n }\n headers = {\n \"Content-Type\": \"application/json; charset=utf-8\"\n }\n timeout = aiohttp.ClientTimeout(total=60)\n async with aiohttp.ClientSession(timeout=timeout) as session:\n async with session.post(API_URL, json=payload, headers=headers) as resp:\n if resp.status == 200:\n data = await resp.json()\n conversation_id = data.get(\"conversation_id\")\n answer = data.get(\"answer\", \"Sorry, I couldn't find an answer.\")\n return {\"answer\": answer, \"conversation_id\": conversation_id}\n else:\n return {\"answer\": \"Sorry, I couldn't find an answer.\", \"conversation_id\": None}\n\n@bot.command(name=\"start\")\nasync def start(ctx):\n \"\"\"Handles the /start command.\"\"\"\n await ctx.send(f\"Hi {ctx.author.mention}! How can I assist you today?\")\n\n@bot.command(name=\"custom_help\")\nasync def custom_help_command(ctx):\n \"\"\"Handles the /custom_help command.\"\"\"\n help_text = (\n \"Here are the available commands:\\n\"\n \"`!start` - Begin a new conversation with the bot\\n\"\n \"`!help` - Display this help message\\n\\n\"\n \"You can also mention me or send a direct message to ask a question!\"\n )\n await ctx.send(help_text)\n\n@bot.event\nasync def on_message(message):\n if message.author == bot.user:\n return\n\n # Process commands first\n await bot.process_commands(message)\n\n # Check if the message is in a DM channel\n if isinstance(message.channel, discord.DMChannel):\n content = message.content.strip()\n else:\n # In guild channels, check if the message mentions the bot at the start\n content = message.content.strip()\n prefix, content = split_string(content)\n if prefix is None:\n return\n part_prefix = str(bot.user.id)\n if part_prefix != prefix:\n return # Bot not mentioned at the start, so do not process\n\n # Now process the message\n user_id = message.author.id\n if user_id not in conversation_histories:\n conversation_histories[user_id] = {\n \"history\": [],\n \"conversation_id\": None\n }\n\n conversation = conversation_histories[user_id]\n conversation[\"history\"].append({\"prompt\": content})\n\n # Generate the answer\n response_doc = await generate_answer(\n content,\n conversation[\"history\"],\n conversation[\"conversation_id\"]\n )\n answer = response_doc[\"answer\"]\n conversation_id = response_doc[\"conversation_id\"]\n\n answer_chunks = chunk_string(answer)\n for chunk in answer_chunks:\n await message.channel.send(chunk)\n\n conversation[\"history\"][-1][\"response\"] = answer\n conversation[\"conversation_id\"] = conversation_id\n\n # Keep conversation history to last 10 exchanges\n conversation[\"history\"] = conversation[\"history\"][-10:]\n\nbot.run(TOKEN)" }, { "path": "extensions/react-widget/.gitignore", "content": "node_modules\ndist\n.parcel-cache" }, { "path": "extensions/react-widget/.parcelrc", "content": "{\n \"extends\": \"@parcel/config-default\",\n \"resolvers\": [\"@parcel/resolver-glob\",\"...\"],\n \"transformers\": {\n \"*.svg\": [\"...\", \"@parcel/transformer-svg-react\", \"@parcel/transformer-typescript-tsc\"]\n},\n \"validators\": {\n \"*.{ts,tsx}\": [\"@parcel/validator-typescript\"]\n }\n}" }, { "path": "extensions/react-widget/README.md", "content": "# DocsGPT react widget\n\nThis widget will allow you to embed a DocsGPT assistant in your React app.\n\n## Installation\n\n```bash\nnpm install docsgpt\n```\n\n## Usage\n\n### React\n\n```javascript\n import { DocsGPTWidget } from \"docsgpt-react\";\n\n const App = () => {\n return ;\n };\n```\n\nTo link the widget to your api and your documents you can pass parameters to the component.\n\n```javascript\n import { DocsGPTWidget } from \"docsgpt-react\";\n\n const App = () => {\n return ;\n };\n```\n\n### Html\n\n```html\n \n \n \n \n \n \n DocsGPT Widget\n \n \n
\n \n \n \n \n \n```\n\nTo link the widget to your api and your documents you can pass parameters to the **renderDocsGPTWidget('div id', { parameters })**.\n\n```html\n \n \n \n \n \n \n DocsGPT Widget\n \n \n
\n \n \n \n \n \n```\n\n# SearchBar\n\nThe `SearchBar` component is an interactive search bar designed to provide search results based on **vector similarity search**. It also includes the capability to open the AI Chatbot, enabling users to query.\n\n---\n\n### Importing the Component\n```tsx\nimport { SearchBar } from \"docsgpt-react\";\n```\n\n---\n\n### Usage Example\n```tsx\n\n```\n\n---\n\n## HTML embedding for Search bar\n\n```html\n\n\n\n \n \n SearchBar Embedding\n \n\n\n \n
\n\n \n\n\n\n```\n\n### Props\n\n| **Prop** | **Type** | **Default Value** | **Description** |\n|-----------------|-----------|-------------------------------------|--------------------------------------------------------------------------------------------------|\n| **`apiKey`** | `string` | `\"74039c6d-bff7-44ce-ae55-2973cbf13837\"` | Your API key generated from the app. Used for authenticating requests. |\n| **`apiHost`** | `string` | `\"https://gptcloud.arc53.com\"` | The base URL of the server hosting the vector similarity search and chatbot services. |\n| **`theme`** | `\"dark\" \\| \"light\"` | `\"dark\"` | The theme of the search bar. Accepts `\"dark\"` or `\"light\"`. |\n| **`placeholder`** | `string` | `\"Search or Ask AI...\"` | Placeholder text displayed in the search input field. |\n| **`width`** | `string` | `\"256px\"` | Width of the search bar. Accepts any valid CSS width value (e.g., `\"300px\"`, `\"100%\"`, `\"20rem\"`). |\n\n\nFeel free to reach out if you need help customizing or extending the `SearchBar`!\n\n## Our github\n\n[DocsGPT](https://github.com/arc53/DocsGPT)\n\nYou can find the source code in the extensions/react-widget folder.\n" }, { "path": "extensions/react-widget/custom.d.ts", "content": "declare module \"*.svg\" {\n import * as React from \"react\";\n \n const ReactComponent: React.FunctionComponent<\n React.SVGProps & { title?: string }\n >;\n \n export default ReactComponent;\n }" }, { "path": "extensions/react-widget/package.json", "content": "{\n \"name\": \"docsgpt\",\n \"version\": \"0.5.1\",\n \"private\": false,\n \"description\": \"DocsGPT 🦖 is an innovative open-source tool designed to simplify the retrieval of information from project documentation using advanced GPT models 🤖.\",\n \"source\": \"./src/index.html\",\n \"main\": \"dist/main.js\",\n \"module\": \"dist/module.js\",\n \"types\": \"dist/types.d.ts\",\n \"files\": [\n \"dist\",\n \"package.json\"\n ],\n \"targets\": {\n \"modern\": {\n \"engines\": {\n \"browsers\": \"Chrome 80\"\n }\n },\n \"legacy\": {\n \"engines\": {\n \"browsers\": \"> 0.5%, last 2 versions, not dead\"\n }\n }\n },\n \"@parcel/resolver-default\": {\n \"packageExports\": true\n },\n \"resolution\": {\n \"styled-components\": \"^5\"\n },\n \"scripts\": {\n \"build\": \"parcel build src/browser.tsx --public-url ./\",\n \"build:react\": \"parcel build src/index.ts\",\n \"serve\": \"parcel serve -p 3000\",\n \"dev\": \"parcel -p 3000\",\n \"test\": \"jest\",\n \"lint\": \"eslint\",\n \"check\": \"tsc --noEmit\",\n \"ci\": \"yarn build && yarn test && yarn lint && yarn check\"\n },\n \"dependencies\": {\n \"@babel/plugin-transform-flow-strip-types\": \"^7.23.3\",\n \"@bpmn-io/snarkdown\": \"^2.2.0\",\n \"@parcel/resolver-glob\": \"^2.16.4\",\n \"@parcel/transformer-svg-react\": \"^2.16.4\",\n \"@parcel/transformer-typescript-tsc\": \"^2.16.4\",\n \"@parcel/validator-typescript\": \"^2.16.4\",\n \"@radix-ui/react-icons\": \"^1.3.0\",\n \"class-variance-authority\": \"^0.7.0\",\n \"clsx\": \"^2.1.0\",\n \"dompurify\": \"^3.1.5\",\n \"flow-bin\": \"^0.305.0\",\n \"markdown-it\": \"^14.1.0\",\n \"react\": \"^18.2.0\",\n \"react-dom\": \"^18.2.0\",\n \"styled-components\": \"^6.1.8\"\n },\n \"devDependencies\": {\n \"@babel/core\": \"^7.24.0\",\n \"@babel/preset-env\": \"^7.24.0\",\n \"@babel/preset-react\": \"^7.23.3\",\n \"@parcel/packager-ts\": \"^2.16.4\",\n \"@parcel/transformer-typescript-types\": \"^2.16.4\",\n \"@types/dompurify\": \"^3.0.5\",\n \"@types/markdown-it\": \"^14.1.2\",\n \"@types/react\": \"^18.3.3\",\n \"@types/react-dom\": \"^18.3.0\",\n \"babel-loader\": \"^10.1.1\",\n \"parcel\": \"^2.16.4\",\n \"process\": \"^0.11.10\",\n \"svgo\": \"^3.3.3\",\n \"typescript\": \"^5.3.3\"\n },\n \"publishConfig\": {\n \"access\": \"public\"\n },\n \"repository\": {\n \"type\": \"git\",\n \"url\": \"git+https://github.com/arc53/DocsGPT.git\"\n },\n \"keywords\": [\n \"docsgpt\",\n \"chatbot\",\n \"assistant\",\n \"ai\",\n \"chatdocs\",\n \"widget\"\n ],\n \"author\": \"Arc53\",\n \"license\": \"Apache-2.0\",\n \"bugs\": {\n \"url\": \"https://github.com/arc53/DocsGPT/issues\"\n },\n \"homepage\": \"https://github.com/arc53/DocsGPT#readme\"\n}\n" }, { "path": "extensions/react-widget/publish.sh", "content": "#!/bin/bash\nset -e\n\n# Create backup of original files\ncp package.json package_original.json\ncp package-lock.json package-lock_original.json\n\n# Store the latest version after publishing\nLATEST_VERSION=\"\"\n\n# Check if a specific version was provided\nif [ \"$1\" ]; then\n VERSION_UPDATE_TYPE=\"$1\"\n echo \"Using custom version update: $VERSION_UPDATE_TYPE\"\nelse\n VERSION_UPDATE_TYPE=\"patch\"\n echo \"No version specified, defaulting to patch update\"\nfi\n\npublish_package() {\n PACKAGE_NAME=$1\n BUILD_COMMAND=$2\n IS_REACT=$3\n\n echo \"Preparing to publish ${PACKAGE_NAME}...\"\n \n # Restore original package.json state before each publish\n cp package_original.json package.json\n cp package-lock_original.json package-lock.json\n\n # Update package name in package.json\n jq --arg name \"$PACKAGE_NAME\" '.name=$name' package.json > temp.json && mv temp.json package.json\n\n # Handle targets based on package type\n if [ \"$IS_REACT\" = \"true\" ]; then\n echo \"Removing targets for React library build...\"\n jq 'del(.targets)' package.json > temp.json && mv temp.json package.json\n fi\n\n # Clean dist directory\n if [ -d \"dist\" ]; then\n echo \"Cleaning dist directory...\"\n rm -rf dist\n fi\n\n # Update version based on input parameter or default to patch\n if [[ \"$VERSION_UPDATE_TYPE\" =~ ^[0-9]+\\.[0-9]+\\.[0-9]+$ ]]; then\n # If full version number is provided (e.g., 0.5.0)\n LATEST_VERSION=$(npm version \"$VERSION_UPDATE_TYPE\" --no-git-tag-version)\n else\n # If update type is provided (patch, minor, major)\n LATEST_VERSION=$(npm version \"$VERSION_UPDATE_TYPE\" --no-git-tag-version)\n fi\n \n echo \"New version: ${LATEST_VERSION}\"\n\n # Build package\n npm run \"$BUILD_COMMAND\"\n\n # Publish package\n npm publish\n\n echo \"Successfully published ${PACKAGE_NAME} version ${LATEST_VERSION}\"\n}\n\n# First publish docsgpt (HTML bundle)\npublish_package \"docsgpt\" \"build\" \"false\"\n\n# Then publish docsgpt-react (React library)\npublish_package \"docsgpt-react\" \"build:react\" \"true\"\n\n# Restore original state but keep the updated version\ncp package_original.json package.json\ncp package-lock_original.json package-lock.json\n\n# Update the version in the final package.json\njq --arg version \"${LATEST_VERSION#v}\" '.version=$version' package.json > temp.json && mv temp.json package.json\n\n# Run npm install to update package-lock-only\nnpm install --package-lock-only\n\n# Cleanup backup files\nrm -f package_original.json\nrm -f package-lock_original.json\nrm -f temp.json\n\necho \"---Process completed---\"\necho \"Final version in package.json: $(jq -r '.version' package.json)\"\necho \"Final version in package-lock.json: $(jq -r '.version' package-lock.json)\"\n\n" }, { "path": "extensions/react-widget/src/App.tsx", "content": "import React from \"react\"\nimport {DocsGPTWidget} from \"./components/DocsGPTWidget\"\nimport {SearchBar} from \"./components/SearchBar\"\nexport const App = () => {\n return (\n
\n \n \n
\n )\n}" }, { "path": "extensions/react-widget/src/browser.tsx", "content": "//exports browser ready methods\n\nimport { createRoot } from \"react-dom/client\";\n\nimport { DocsGPTWidget } from './components/DocsGPTWidget';\nimport { SearchBar } from './components/SearchBar';\nimport React from \"react\";\nif (typeof window !== 'undefined') {\n const renderWidget = (elementId: string, props = {}) => {\n const root = createRoot(document.getElementById(elementId) as HTMLElement);\n root.render();\n };\n const renderSearchBar = (elementId: string, props = {}) => {\n const root = createRoot(document.getElementById(elementId) as HTMLElement);\n root.render();\n };\n (window as any).renderDocsGPTWidget = renderWidget;\n\n (window as any).renderSearchBar = renderSearchBar;\n}\n\nexport { DocsGPTWidget, SearchBar };\n" }, { "path": "extensions/react-widget/src/components/DocsGPTWidget.tsx", "content": "\"use client\";\nimport React, { useRef, useState, useEffect } from 'react'\nimport DOMPurify from 'dompurify';\nimport styled, { keyframes, css } from 'styled-components';\nimport { PaperPlaneIcon, RocketIcon, ExclamationTriangleIcon, Cross2Icon } from '@radix-ui/react-icons';\nimport { FEEDBACK, MESSAGE_TYPE, Query, Status, WidgetCoreProps, WidgetProps } from '../types/index';\nimport { fetchAnswerStreaming, sendFeedback } from '../requests/streamingApi';\nimport { ThemeProvider } from 'styled-components';\nimport Like from '../assets/like.svg';\nimport Dislike from '../assets/dislike.svg';\nimport MarkdownIt from 'markdown-it';\n\nconst themes = {\n dark: {\n bg: '#222327',\n text: '#fff',\n primary: {\n text: \"#FAFAFA\",\n bg: '#222327'\n },\n secondary: {\n text: \"#A1A1AA\",\n bg: \"#38383b\"\n }\n },\n light: {\n bg: '#fff',\n text: '#000',\n primary: {\n text: \"#222327\",\n bg: \"#fff\"\n },\n secondary: {\n text: \"#A1A1AA\",\n bg: \"#F6F6F6\"\n }\n }\n};\n\nconst sizesConfig = {\n small: { size: 'small', width: '320px', height: '400px' },\n medium: { size: 'medium', width: '400px', height: '80vh' },\n large: { size: 'large', width: '666px', height: '75vh' },\n getCustom: (custom: { width: string; height: string; maxWidth?: string; maxHeight?: string }) => ({\n size: 'custom',\n width: custom.width,\n height: custom.height,\n maxWidth: custom.maxWidth || '968px',\n maxHeight: custom.maxHeight || '70vh',\n }),\n};\nconst createBox = keyframes`\n 0% {\n transform: scale(0.6);\n }\n 90% {\n transform: scale(1.02);\n }\n 100% {\n transform: scale(1);\n }\n`\nconst closeBox = keyframes`\n 0% {\n transform: scale(1); \n }\n 10% {\n transform: scale(1.02); \n }\n 100% {\n transform: scale(0.6);\n }\n`\n\nconst openContainer = keyframes`\n 0% {\n width: 200px;\n height: 100px;\n }\n 100% {\n width: ${(props) => props.theme.dimensions.width};\n height: ${(props) => props.theme.dimensions.height};\n border-radius: 12px;\n }`\nconst closeContainer = keyframes`\n 0% {\n width: ${(props) => props.theme.dimensions.width};\n height: ${(props) => props.theme.dimensions.height};\n border-radius: 12px;\n }\n 100% {\n width: 200px;\n height: 100px;\n }\n`\nconst fadeIn = keyframes`\n from {\n opacity: 0;\n width: ${(props) => props.theme.dimensions.width};\n height: ${(props) => props.theme.dimensions.height};\n transform: scale(0.9);\n }\n to {\n opacity: 1;\n transform: scale(1);\n width: ${(props) => props.theme.dimensions.width};\n height: ${(props) => props.theme.dimensions.height};\n }\n`\n\nconst fadeOut = keyframes`\n from {\n opacity: 1;\n width: ${(props) => props.theme.dimensions.width};\n height: ${(props) => props.theme.dimensions.height};\n }\n to {\n opacity: 0;\n transform: scale(0.9);\n width: ${(props) => props.theme.dimensions.width};\n height: ${(props) => props.theme.dimensions.height};\n }\n`\nconst scaleAnimation = keyframes`\n from {\n transform: scale(1.2);\n }\n to {\n transform: scale(1);\n }\n`\nconst Overlay = styled.div`\n position: fixed;\n top: 0;\n left: 0;\n width: 100%;\n height: 100%;\n background-color: rgba(0, 0, 0, 0.5);\n z-index: 999;\n transition: opacity 0.5s;\n`\n\n\nconst WidgetContainer = styled.div<{ $modal?: boolean }>`\n all: initial;\n position: fixed;\n right: ${props => props.$modal ? '50%' : '10px'};\n bottom: ${props => props.$modal ? '50%' : '10px'};\n z-index: 1001;\n transform-origin:100% 100%;\n display: block;\n &.modal{\n transform : translate(50%,50%);\n }\n &.open {\n animation: css ${createBox} 250ms cubic-bezier(0.25, 0.1, 0.25, 1) forwards;\n }\n &.close {\n animation: css ${closeBox} 250ms cubic-bezier(0.25, 0.1, 0.25, 1) forwards;\n }\n align-items: center;\n text-align: left;\n`;\n\nconst StyledContainer = styled.div<{ $isOpen: boolean }>`\n all: initial;\n max-height: ${(props) => props.theme.dimensions.maxHeight};\n max-width: ${(props) => props.theme.dimensions.maxWidth};\n width: ${(props) => props.theme.dimensions.width};\n height: ${(props) => props.theme.dimensions.height} ;\n position: relative;\n flex-direction: column;\n justify-content: space-between;\n bottom: 0;\n left: 0;\n background-color: ${(props) => props.theme.primary.bg};\n font-family: sans-serif;\n display: flex;\n border-radius: 12px;\n box-shadow: 0 1px 2px rgba(0, 0, 0, 0.05), 0 2px 4px rgba(0, 0, 0, 0.1);\n padding: 26px 26px 0px 26px;\n animation: ${({ $isOpen, theme }) =>\n theme.dimensions.size === 'large'\n ? $isOpen\n ? css`${fadeIn} 150ms ease-in forwards`\n : css` ${fadeOut} 150ms ease-in forwards`\n : $isOpen\n ? css`${openContainer} 150ms ease-in forwards`\n : css`${closeContainer} 250ms ease-in forwards`};\n @media only screen and (max-width: 768px) {\n max-height: 100vh;\n max-width: 80vw;\n overflow: auto;\n }\n`;\n\nconst FloatingButton = styled.div<{ $bgcolor: string, $hidden: boolean, $isAnimatingButton: boolean }>`\n position: fixed;\n display: ${props => props.$hidden ? \"none\" : \"flex\"};\n z-index: 500;\n justify-content: center;\n gap: 8px;\n padding: 14px;\n align-items: center;\n bottom: 16px;\n color: white;\n font-family: sans-serif;\n right: 16px;\n font-weight: 500;\n border-radius: 9999px;\n background: ${props => props.$bgcolor};\n box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1);\n cursor: pointer;\n animation: ${props => props.$isAnimatingButton ? css`${scaleAnimation} 200ms forwards` : 'none'};\n &:hover {\n transform: scale(1.1);\n transition: transform 0.2s ease-in-out;\n }\n &:not(:hover) {\n transition: transform 0.2s ease-in-out;\n }\n`;\nconst CancelButton = styled.button`\n cursor: pointer;\n position: absolute;\n top: 0;\n right: 0;\n margin: 8px;\n width: 30px;\n padding: 0;\n background-color: transparent;\n border: none;\n outline: none;\n color: inherit;\n transition: opacity 0.3s ease;\n opacity: 0.6;\n &:hover {\n opacity: 1;\n }\n .white-filter {\n filter: invert(100%);\n }\n`;\n\nconst Header = styled.div`\n display: flex;\n align-items: flex-start;\n`;\n\nconst ContentWrapper = styled.div`\n display: flex;\n flex-direction: column;\n gap:2px; \n margin-left: 8px;\n`;\n\nconst Title = styled.h3`\n font-size: 14px;\n font-weight: normal;\n color: ${props => props.theme.primary.text};\n margin: 0;\n`;\n\nconst Description = styled.p`\n font-size: 13.75px;\n color: ${props => props.theme.secondary.text};\n margin: 0 ;\n padding: 0 ;\n`;\n\nconst Conversation = styled.div`\n height: 70%;\n border-radius: 6px;\n text-align: left;\n overflow-y: auto;\n scrollbar-width: thin;\n scrollbar-color: ${props => props.theme.secondary.bg} transparent; /* thumb color track color */\n`;\nconst Feedback = styled.div`\n background-color: transparent;\n font-weight: normal;\n gap: 12px;\n display: flex;\n padding: 6px;\n clear: both;\n`;\nconst MessageBubble = styled.div<{ $type: MESSAGE_TYPE }>`\n display: block;\n font-size: 16px;\n position: relative;\n width: 100%;;\n float: right;\n margin: 0px;\n &:hover ${Feedback} * {\n visibility: visible ;\n \n \n }\n`;\nconst Message = styled.div<{ $type: MESSAGE_TYPE }>`\n background: ${props => props.$type === 'QUESTION' ?\n 'linear-gradient(to bottom right, #8860DB, #6D42C5)' :\n props.theme.secondary.bg};\n color: ${props => props.$type === 'ANSWER' ? props.theme.primary.text : '#fff'};\n border: none;\n float: ${props => props.$type === 'QUESTION' ? 'right' : 'left'};\n max-width: ${props => props.$type === 'ANSWER' ? '90%' : '80%'};\n overflow: auto;\n margin: 4px;\n display: block;\n line-height: 1.5;\n padding: 12px;\n border-radius: 6px;\n overflow-wrap: break-word; \n`;\nconst Markdown = styled.div`\n pre {\n padding: 8px;\n width: 90%;\n font-size: 12px;\n border-radius: 6px;\n overflow-x: auto;\n background-color: #1B1C1F;\n color: #fff ;\n }\n\n h1 {\n font-size: clamp(14px,40vw,16px);\n }\n\n h2 {\n font-size: 14px;\n }\n\n h3 {\n font-size: 14px;\n }\n\n p {\n \n margin: 0px;\n }\n\n code:not(pre code) {\n border-radius: 6px;\n padding: 1px 3px;\n font-size: 12px;\n display: inline-block;\n background-color: #646464;\n color: #fff ;\n }\n\n code {\n white-space: pre-wrap ;\n overflow-wrap: break-word;\n word-break: break-all;\n }\n\n ul{\n padding:0px;\n margin: 1rem 0; \n list-style-position: outside; \n list-style-type: disc; \n padding-left: 1rem; \n white-space: normal;\n }\n \n ol{\n padding:0px;\n margin: 1rem 0; \n list-style-position: outside; \n list-style-type: decimal; \n padding-left: 1rem; \n white-space: normal;\n }\n \n li{\n line-height: 1.625;\n }\n .dgpt-table-container { \n margin: 20px 0;\n width:100%;\n overflow-x: scroll !important; \n border: 1px solid #a2a2ab; \n border-radius: 6px; \n -webkit-overflow-scrolling: touch;\n -ms-overflow-style: scrollbar;\n scrollbar-width: thin; \n scrollbar-color: #a2a2ab #38383b;\n }\n \n\n table, .dgpt-table { \n width: 100%; \n border-collapse: collapse; \n text-align: left; \n min-width:600px;\n \n }\n thead, .dgpt-thead { \n font-size: 12px; \n text-transform: uppercase; \n \n }\n \n \n th, .dgpt-th, td, .dgpt-td { \n padding: 10px;\n border-bottom: 1px solid #a2a2ab; \n font-size:14px;\n \n }\n th{\n font-weight: normal !important;\n }\n td{\n font-weight: bold;\n }\n \n \n \n`\nconst ErrorAlert = styled.div`\n color: #b91c1c;\n border:0.1px solid #b91c1c;\n display: flex;\n padding:4px;\n margin:11.2px;\n opacity: 90%;\n max-width: 70%;\n font-weight: 400;\n border-radius: 6px;\n justify-content: space-evenly;\n`\n//dot loading animation\nconst dotBounce = keyframes`\n 0%, 80%, 100% {\n transform: translateY(0);\n }\n 40% {\n transform: translateY(-5px);\n }\n`;\n\nconst DotAnimation = styled.div`\n display: inline-block;\n animation: ${dotBounce} 1s infinite ease-in-out;\n`;\n// delay classes as styled components\nconst Delay = styled(DotAnimation) <{ $delay: number }>`\n animation-delay: ${props => props.$delay + 'ms'};\n`;\nconst PromptContainer = styled.form`\n background-color: transparent;\n min-height: ${props => props.theme.dimensions.size == 'large' ? '40px' : '23px'};\n max-height:150px;\n display: flex;\n align-items: end;\n justify-content: space-evenly;\n`;\nconst StyledTextarea = styled.textarea`\n box-sizing: border-box;\n width: 100%;\n border: 1px solid #686877;\n padding: ${props => props.theme.dimensions.size === 'large' ? '18px 12px 14px 12px' : '8px 12px 4px 12px'};\n background-color: transparent;\n font-size: 16px;\n font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;\n border-radius: 6px;\n color: ${props => props.theme.text};\n outline: none;\n resize: none;\n transition: height 0.1s ease;\n overflow-wrap: break-word;\n white-space: pre-wrap;\n line-height: 1.4;\n text-align: left;\n min-height: ${props => props.theme.dimensions.size === 'large' ? '60px' : '40px'};\n max-height: 140px;\n overflow-y: auto;\n scrollbar-width: thin;\n scrollbar-color: #38383b transparent;\n &::-webkit-scrollbar {\n width: 6px;\n height: 6px;\n }\n &::-webkit-scrollbar-thumb {\n background-color: #38383b;\n border-radius: 6px;\n }\n &::-webkit-scrollbar-track {\n background: transparent;\n }\n &::placeholder {\n text-align: left;\n\n }\n`;\nconst StyledButton = styled.button`\n display: flex;\n justify-content: center;\n align-items: center;\n background-image: linear-gradient(to bottom right, #5AF0EC, #E80D9D);\n background-color: rgba(0, 0, 0, 0.3);\n border-radius: 6px;\n min-width: ${props => props.theme.dimensions.size === 'large' ? '60px' : '40px'};\n height: ${props => props.theme.dimensions.size === 'large' ? '60px' : '40px'};\n margin-left:8px;\n padding: 0px;\n \n border: none;\n cursor: pointer;\n outline: none;\n &:hover{\n opacity: 90%;\n }\n &:disabled {\n background-image: linear-gradient(to bottom right, #2d938f, #b31877);\n }`;\nconst HeroContainer = styled.div`\n position: relative;\n width: 90%;\n max-width: 500px;\n background-image: linear-gradient(to bottom right, #5AF0EC, #ff1bf4);\n border-radius: 10px;\n margin: 16px auto;\n padding: 2px;\n`;\nconst HeroWrapper = styled.div`\n display: flex;\n flex-direction: column;\n justify-content: flex-start;\n gap: 8px;\n align-items: middle;\n background-color: ${props => props.theme.primary.bg};\n border-radius: 10px; \n font-weight: normal;\n padding: 12px;\n`\nconst HeroTitle = styled.h3`\n color: ${props => props.theme.text};\n font-size: 16px;\n margin:0px ;\n padding: 0px;\n`;\nconst HeroDescription = styled.p`\n color: ${props => props.theme.text};\n font-size: 12px;\n line-height: 1.5;\n margin: 0px;\n padding: 0px;\n`;\nconst Hyperlink = styled.a`\n color: #9971EC;\n text-decoration: none;\n`;\nconst Tagline = styled.div`\n text-align: center;\n display: block;\n color: ${props => props.theme.secondary.text};\n padding: 12px ;\n font-size: 12px;\n`;\n\n\nconst SourcesList = styled.div`\n display: flex;\n margin:12px 0px;\n flex-wrap: wrap;\n gap: 8px;\n`;\n\nconst SourceLink = styled.a`\n color: ${props => props.theme.primary.text};\n text-decoration: none;\n background: ${props => props.theme.secondary.bg};\n padding: 4px 12px;\n border-radius: 85px;\n font-size: 14px;\n transition: opacity 0.2s ease;\n display: inline-block;\n text-align: center;\n max-width: 25%;\n white-space: nowrap;\n overflow: hidden;\n text-overflow: ellipsis;\n line-height: 1.5;\n \n &:hover {\n opacity: 0.8;\n }\n`;\n\nconst ExtraButton = styled.button`\n color: #9971EC;\n background: transparent;\n border-radius: 85px;\n padding: 4px 12px;\n font-size: 14px;\n border: none;\n cursor: pointer;\n transition: opacity 0.2s ease;\n text-align: center;\n height:auto;\n &:hover {\n opacity: 0.8;\n }\n`;\nconst SourcesComponent = ({ sources }: { sources: Array<{ source: string; title: string }> }) => {\n const [showAll, setShowAll] = React.useState(false);\n const visibleSources = showAll ? sources : sources.slice(0, 3);\n const extraCount = sources.length - 3;\n \n return (\n \n {visibleSources.map((source, idx) => (\n \n {source.title}\n \n ))}\n {sources.length > 3 && (\n setShowAll(!showAll)}>\n {showAll ? \"Show less\" : `+ ${extraCount} more`}\n \n )}\n \n );\n};\n\nconst Hero = ({ title, description, theme }: { title: string, description: string, theme: string }) => {\n return (\n \n \n \n {title}\n {description}\n \n \n );\n};\nexport const DocsGPTWidget = (props: WidgetProps) => {\n\n const {\n buttonIcon = 'https://d3dg1063dc54p9.cloudfront.net/widget/chat.svg',\n buttonText = 'Ask a question',\n buttonBg = 'linear-gradient(to bottom right, #5AF0EC, #E80D9D)',\n defaultOpen = false,\n ...coreProps\n } = props\n\n const [open, setOpen] = React.useState(defaultOpen);\n const [isAnimatingButton, setIsAnimatingButton] = React.useState(false);\n const [isFloatingButtonVisible, setIsFloatingButtonVisible] = React.useState(true);\n\n React.useEffect(() => {\n if (isFloatingButtonVisible)\n setTimeout(() => setIsAnimatingButton(true), 250);\n return () => {\n setIsAnimatingButton(false)\n }\n }, [isFloatingButtonVisible])\n\n const handleClose = () => {\n setIsFloatingButtonVisible(true);\n setOpen(false);\n };\n const handleOpen = () => {\n setOpen(true);\n setIsFloatingButtonVisible(false);\n }\n return (\n <>\n \n \n {buttonText}\n \n \n \n )\n}\n\nexport const WidgetCore = ({\n apiHost = 'https://gptcloud.arc53.com',\n apiKey = \"527686a3-e867-4b4d-9fec-f5f45fdb613a\",\n avatar = 'https://d3dg1063dc54p9.cloudfront.net/cute-docsgpt.png',\n title = 'Get AI assistance',\n description = 'DocsGPT\\'s AI Chatbot is here to help',\n heroTitle = 'Welcome to DocsGPT !',\n heroDescription = 'This chatbot is built with DocsGPT and utilises GenAI, please review important information using sources.',\n size = 'small',\n theme = 'dark',\n collectFeedback = true,\n isOpen = false,\n showSources = true,\n handleClose,\n prefilledQuery = ''\n}: WidgetCoreProps) => {\n const [prompt, setPrompt] = React.useState(\"\");\n const [mounted, setMounted] = React.useState(false);\n const [status, setStatus] = React.useState('idle');\n const [queries, setQueries] = React.useState([]);\n const [conversationId, setConversationId] = React.useState(null);\n const [eventInterrupt, setEventInterrupt] = React.useState(false); //click or scroll by user while autoScrolling\n const [hasScrolledToLast, setHasScrolledToLast] = useState(true);\n\n const isBubbleHovered = useRef(false);\n const conversationRef = useRef(null);\n const endMessageRef = React.useRef(null);\n const promptRef = React.useRef(null);\n const md = new MarkdownIt();\n //Custom markdown for the table\n md.renderer.rules.table_open = () => '
';\n md.renderer.rules.table_close = () => '
';\n md.renderer.rules.thead_open = () => '';\n md.renderer.rules.tr_open = () => '';\n md.renderer.rules.td_open = () => '';\n md.renderer.rules.th_open = () => '';\n\n\n\n React.useEffect(() => {\n if (isOpen) {\n setMounted(true); // Mount the component\n appendQuery(prefilledQuery)\n } else {\n // Wait for animations before unmounting\n const timeout = setTimeout(() => {\n setMounted(false)\n }, 250);\n return () => clearTimeout(timeout);\n }\n }, [isOpen]);\n\n const handleUserInterrupt = () => {\n if (!eventInterrupt && status === 'loading') setEventInterrupt(true);\n }\n\n const scrollIntoView = () => {\n if (!conversationRef?.current || eventInterrupt) return;\n\n if (status === 'idle' || !queries.length || !queries[queries.length - 1].response) {\n conversationRef.current.scrollTo({\n behavior: 'smooth',\n top: conversationRef.current.scrollHeight,\n });\n } else {\n conversationRef.current.scrollTop = conversationRef.current.scrollHeight;\n }\n setHasScrolledToLast(true);\n };\n\n const checkScroll = () => {\n const el = conversationRef.current;\n if (!el) return;\n const isBottom = el.scrollHeight - el.scrollTop - el.clientHeight < 10;\n setHasScrolledToLast(isBottom);\n };\n\n React.useEffect(() => {\n !eventInterrupt && scrollIntoView();\n \n conversationRef.current?.addEventListener('scroll', checkScroll);\n return () => {\n conversationRef.current?.removeEventListener('scroll', checkScroll);\n };\n }, [queries.length, queries[queries.length - 1]?.response]);\n\n async function handleFeedback(feedback: FEEDBACK, index: number) {\n let query = queries[index];\n if (!query.response || !conversationId) {\n console.log(\"Cannot submit feedback: missing response or conversation ID\");\n return;\n }\n\n // If clicking the same feedback button that's already active, remove the feedback by sending null\n if (query.feedback === feedback) {\n try {\n const response = await sendFeedback({\n question: query.prompt,\n answer: query.response,\n feedback: null,\n apikey: apiKey,\n conversation_id: conversationId,\n question_index: index,\n }, apiHost);\n\n if (response.status === 200) {\n const updatedQuery = { ...query };\n delete updatedQuery.feedback;\n setQueries((prev: Query[]) =>\n prev.map((q, i) => (i === index ? updatedQuery : q))\n );\n }\n } catch (err) {\n console.error(\"Failed to submit feedback:\", err);\n }\n return;\n }\n\n try {\n const response = await sendFeedback({\n question: query.prompt,\n answer: query.response,\n feedback: feedback,\n apikey: apiKey,\n conversation_id: conversationId,\n question_index: index,\n }, apiHost);\n\n if (response.status === 200) {\n setQueries((prev: Query[]) => {\n return prev.map((q, i) => {\n if (i === index) {\n return { ...q, feedback: feedback };\n }\n return q;\n });\n });\n }\n } catch (err) {\n console.error(\"Failed to submit feedback:\", err);\n }\n }\n\n async function stream(question: string) {\n setStatus('loading')\n try {\n await fetchAnswerStreaming(\n {\n question: question,\n apiKey: apiKey,\n apiHost: apiHost,\n history: queries,\n conversationId: conversationId,\n onEvent: (event: MessageEvent) => {\n const data = JSON.parse(event.data);\n // check if the 'end' event has been received\n if (data.type === 'end') {\n setStatus('idle');\n }\n else if (data.type === 'id') {\n setConversationId(data.id)\n }\n else if (data.type === 'error') {\n const updatedQueries = [...queries];\n updatedQueries[updatedQueries.length - 1].error = data.error;\n setQueries(updatedQueries);\n setStatus('idle')\n }\n else if (data.type === 'source' && showSources) {\n const updatedQueries = [...queries];\n updatedQueries[updatedQueries.length - 1].sources = data.source;\n setQueries(updatedQueries);\n }\n else {\n const result = data.answer ? data.answer : ''; //Fallback to an empty string if data.answer is undefined\n const streamingResponse = queries[queries.length - 1].response ? queries[queries.length - 1].response : '';\n const updatedQueries = [...queries];\n updatedQueries[updatedQueries.length - 1].response = streamingResponse + result;\n setQueries(updatedQueries);\n }\n }\n }\n );\n } catch (error) {\n const updatedQueries = [...queries];\n updatedQueries[updatedQueries.length - 1].error = 'Something went wrong !'\n setQueries(updatedQueries);\n setStatus('idle')\n //setEventInterrupt(false)\n }\n\n }\n\n\n const appendQuery = async (userQuery: string) => {\n if (!userQuery)\n return;\n\n setEventInterrupt(false);\n queries.push({ prompt: userQuery });\n setPrompt('');\n await stream(userQuery);\n }\n // submit handler\n const handleSubmit = async (e: React.FormEvent) => {\n e.preventDefault();\n if (!prompt.trim()) return;\n if (promptRef.current) {\n promptRef.current.style.height = \"auto\";\n }\n await appendQuery(prompt);\n }\n const handlePromptKeyDown = async (e: React.KeyboardEvent) => {\n if (e.key === 'Enter' && !e.shiftKey) {\n \n e.preventDefault();\n // Prevent sending empty messages\n if (promptRef.current && promptRef.current.value.trim() === \"\") return;\n //Rest the input to it's original size after submitting\n if(promptRef.current){\n promptRef.current.value = \"\";\n promptRef.current.style.height = \"auto\"; \n }\n await appendQuery(prompt);\n }\n}\n // Auto-resize the input textarea while typing, clamping to base or max height\n const handleUserInput = (e: React.KeyboardEvent) =>{\n const el = promptRef.current;\n if (!el) return;\n const baseHeight = size === 'large' ? 60 : 40;\n const maxHeight = 140;\n el.style.height = 'auto';\n const next = Math.min(el.scrollHeight, maxHeight);\n el.style.height = Math.max(baseHeight, next) + 'px';\n\n }\n \n // Update prompt state, auto resize textarea to content, and maintain scroll on new lines\n const handlePromptChange = (event: React.ChangeEvent) => {\n const value = event.target.value;\n setPrompt(value);\n const el = event.currentTarget;\n const baseHeight = size === 'large' ? 60 : 40;\n const maxHeight = 140;\n el.style.height = 'auto';\n const next = Math.min(el.scrollHeight, maxHeight);\n el.style.height = Math.max(baseHeight, next) + 'px';\n if(value.includes(\"\\n\")){\n el.scrollTop = el.scrollHeight;\n\n }\n \n \n }\n const handleImageError = (event: React.SyntheticEvent) => {\n event.currentTarget.src = \"https://d3dg1063dc54p9.cloudfront.net/cute-docsgpt.png\";\n };\n\n const dimensions =\n typeof size === 'object' && 'custom' in size\n ? sizesConfig.getCustom(size.custom)\n : sizesConfig[size];\n if (!mounted) return null;\n\n\n \n return (\n \n {isOpen && size === 'large' &&\n \n }\n {(\n \n \n
\n \n \n \n
\n docs-gpt\n \n {title}\n {description}\n \n
\n
\n \n {\n queries.length > 0 ? queries?.map((query, index) => {\n return (\n \n {\n query.prompt &&\n \n \n {query.prompt}\n \n \n }\n {\n query.response ? { isBubbleHovered.current = true }} $type='ANSWER'>\n {showSources && query.sources && query.sources.length > 0 && query.sources.some(source => source.source !== 'local') && (\n source.source !== 'local')} />\n )}\n \n \n \n\n {collectFeedback &&\n \n {\n e.stopPropagation()\n handleFeedback(\"LIKE\", index)}\n }>\n \n \n {\n e.stopPropagation()\n handleFeedback(\"DISLIKE\", index)}\n }>\n \n \n }\n \n :
\n {\n query.error ? \n\n \n
\n
Network Error
\n {query.error}\n
\n \n : \n \n .\n .\n .\n \n \n }\n
\n }\n )\n })\n : \n }\n \n
\n \n \n \n \n \n \n \n Powered by \n DocsGPT\n \n
\n \n \n )\n }\n \n )\n}" }, { "path": "extensions/react-widget/src/components/SearchBar.tsx", "content": "import React from 'react';\nimport styled, { ThemeProvider, createGlobalStyle } from 'styled-components';\nimport { WidgetCore } from './DocsGPTWidget';\nimport { SearchBarProps } from '@/types';\nimport { getSearchResults } from '../requests/searchAPI';\nimport { Result } from '@/types';\nimport { getOS, processMarkdownString } from '../utils/helper';\nimport DOMPurify from 'dompurify';\nimport {\n CodeIcon,\n TextAlignLeftIcon,\n HeadingIcon,\n ReaderIcon,\n ListBulletIcon,\n QuoteIcon\n} from '@radix-ui/react-icons';\nconst themes = {\n dark: {\n name: 'dark',\n bg: '#202124',\n text: '#EDEDED',\n primary: {\n text: \"#FAFAFA\",\n bg: '#111111'\n },\n secondary: {\n text: \"#A1A1AA\",\n bg: \"#38383b\"\n }\n },\n light: {\n name: 'light',\n bg: '#EAEAEA',\n text: '#171717',\n primary: {\n text: \"#222327\",\n bg: \"#fff\"\n },\n secondary: {\n text: \"#A1A1AA\",\n bg: \"#F6F6F6\"\n }\n }\n}\n\nconst GlobalStyle = createGlobalStyle`\n .highlight {\n color: ${props => props.theme.name === 'dark' ? '#4B9EFF' : '#0066CC'};\n font-weight: 500;\n }\n`;\n\nconst loadGeistFont = () => {\n const link = document.createElement('link');\n link.href = 'https://fonts.googleapis.com/css2?family=Geist:wght@100..900&display=swap';\n link.rel = 'stylesheet';\n document.head.appendChild(link);\n};\n\nconst Main = styled.div`\n all: initial;\n font-family: 'Geist', sans-serif;\n`\nconst SearchButton = styled.button<{ $inputWidth: string }>`\n padding: 6px 6px;\n font-family: inherit;\n width: ${({ $inputWidth }) => $inputWidth};\n border-radius: 8px;\n display: inline;\n color: ${props => props.theme.secondary.text};\n outline: none;\n border: none;\n background-color: ${props => props.theme.secondary.bg};\n -webkit-appearance: none;\n -moz-appearance: none;\n appearance: none;\n transition: background-color 128ms linear;\n text-align: left;\n cursor: pointer;\n`\n\nconst Container = styled.div`\n position: relative;\n display: inline-block;\n`\nconst SearchOverlay = styled.div`\n position: fixed;\n top: 0;\n left: 0;\n width: 100%;\n height: 100%;\n background-color: #0000001A;\n backdrop-filter: blur(8px);\n -webkit-backdrop-filter: blur(8px);\n z-index: 99;\n`;\n\n\nconst SearchResults = styled.div`\n position: fixed;\n display: flex;\n flex-direction: column;\n background-color: ${props => props.theme.name === 'dark' ? \n 'rgba(0, 0, 0, 0.15)' : \n 'rgba(255, 255, 255, 0.4)'};\n border: 1px solid rgba(255, 255, 255, 0.18);\n border-radius: 15px;\n padding: 8px 0px 8px 0px;\n width: 792px;\n max-width: 90vw;\n height: 396px;\n z-index: 100;\n left: 50%;\n top: 50%;\n transform: translate(-50%, -50%);\n color: ${props => props.theme.primary.text};\n \n box-shadow: 0 8px 32px 0 rgba(31, 38, 135, 0.37);\n backdrop-filter: blur(82px);\n -webkit-backdrop-filter: blur(82px);\n border-radius: 10px;\n \n box-sizing: border-box;\n\n @media only screen and (max-width: 768px) {\n height: 80vh;\n width: 90vw;\n }\n`;\n\nconst SearchResultsScroll = styled.div`\n flex: 1;\n overflow-y: auto;\n overflow-x: hidden;\n scrollbar-gutter: stable;\n scrollbar-width: thin;\n scrollbar-color: #383838 transparent;\n padding: 0 16px;\n`;\n\nconst IconTitleWrapper = styled.div`\n display: flex;\n align-items: center;\n gap: 8px;\n\n .element-icon{\n margin: 4px;\n }\n`;\n\nconst Title = styled.h3`\n font-size: 15px;\n font-weight: 400;\n color: ${props => props.theme.primary.text};\n margin: 0;\n overflow-wrap: break-word;\n white-space: normal;\n overflow: hidden;\n text-overflow: ellipsis;\n`;\nconst ContentWrapper = styled.div`\n display: flex;\n flex-direction: column;\n gap: 12px; \n`;\n\n\n\nconst ResultWrapper = styled.div`\n display: flex;\n align-items: flex-start;\n width: 100%;\n box-sizing: border-box;\n padding: 8px 16px;\n cursor: pointer;\n background-color: transparent;\n font-family: 'Geist', sans-serif;\n border-radius: 8px;\n\n word-wrap: break-word;\n overflow-wrap: break-word;\n word-break: break-word;\n white-space: normal;\n overflow: hidden;\n text-overflow: ellipsis;\n\n &:hover {\n backdrop-filter: blur(8px);\n -webkit-backdrop-filter: blur(8px);\n }\n`;\n\nconst Content = styled.div`\n display: flex;\n margin-left: 8px;\n flex-direction: column;\n gap: 8px;\n padding: 4px 0px 0px 12px;\n font-size: 15px;\n color: ${props => props.theme.primary.text};\n line-height: 1.6;\n border-left: 2px solid ${props => props.theme.primary.text}CC;\n overflow: hidden;\n \n`;\nconst ContentSegment = styled.div`\n display: flex;\n align-items: flex-start;\n gap: 8px;\n padding-right: 16px;\n overflow-wrap: break-word;\n white-space: normal;\n overflow: hidden; \n text-overflow: ellipsis;\n`\n\nconst Toolkit = styled.kbd`\n position: absolute;\n right: 4px;\n top: 50%;\n transform: translateY(-50%);\n background-color: ${(props) => props.theme.primary.bg};\n color: ${(props) => props.theme.secondary.text};\n font-weight: 600;\n font-size: 10px;\n padding: 3px 6px;\n border: 1px solid ${(props) => props.theme.secondary.text};\n border-radius: 4px;\n display: flex;\n align-items: center;\n justify-content: center;\n z-index: 1;\n pointer-events: none;\n`\nconst Loader = styled.div`\n margin: 2rem auto;\n border: 4px solid ${props => props.theme.name === 'dark' ? 'rgba(255, 255, 255, 0.2)' : 'rgba(0, 0, 0, 0.1)'};\n border-top: 4px solid ${props => props.theme.name === 'dark' ? '#FFFFFF' : props.theme.primary.bg};\n border-radius: 50%;\n width: 12px;\n height: 12px;\n animation: spin 1s linear infinite;\n\n @keyframes spin {\n 0% {\n transform: rotate(0deg);\n }\n 100% {\n transform: rotate(360deg);\n }\n }\n`;\n\nconst NoResults = styled.div`\n margin-top: 2rem;\n text-align: center;\n font-size: 14px;\n color: ${props => props.theme.name === 'dark' ? '#E0E0E0' : '#505050'};\n font-weight: 500;\n`;\nconst AskAIButton = styled.button`\n display: flex;\n align-items: center;\n justify-content: flex-start;\n gap: 12px;\n width: calc(100% - 32px);\n margin: 0 16px 16px 16px;\n box-sizing: border-box;\n height: 50px;\n padding: 8px 24px;\n border: none;\n border-radius: 8px;\n color: ${props => props.theme.text}; \n cursor: pointer;\n font-size: 16px;\n backdrop-filter: blur(16px);\n -webkit-backdrop-filter: blur(16px);\n background-color: ${props => props.theme.name === 'dark' ? \n 'rgba(255, 255, 255, 0.05)' : \n 'rgba(0, 0, 0, 0.03)'};\n\n &:hover {\n backdrop-filter: blur(20px);\n -webkit-backdrop-filter: blur(20px);\n background-color: ${props => props.theme.name === 'dark' ? \n 'rgba(255, 255, 255, 0.1)' : \n 'rgba(0, 0, 0, 0.06)'}; \n }\n`;\n\nconst SearchHeader = styled.div`\n display: flex;\n align-items: center;\n gap: 8px;\n margin-bottom: 12px;\n padding-bottom: 12px;\n border-bottom: 1px solid ${props => props.theme.name === 'dark' ? '#FFFFFF24' : 'rgba(0, 0, 0, 0.14)'};\n`;\n\n\n\nconst TextField = styled.input`\n width: calc(100% - 32px);\n margin: 0 16px;\n padding: 12px 16px;\n border: none;\n background-color: transparent;\n color: ${props => props.theme.text}; \n font-size: 20px;\n font-weight: 400; \n outline: none;\n \n &:focus {\n border-color: none;\n }\n\n &::placeholder {\n color: ${props => props.theme.name === 'dark' ? 'rgba(255, 255, 255, 0.6)' : 'rgba(0, 0, 0, 0.5)'} !important;\n opacity: 100%; /* Force opacity to ensure placeholder is visible */\n font-weight: 500;\n }\n`\n\n\n\nconst EscapeInstruction = styled.kbd`\n display: flex;\n align-items: center;\n justify-content: center;\n margin: 12px 16px 0;\n padding: 4px 8px;\n border-radius: 4px;\n background-color: transparent;\n border: 1px solid ${props => props.theme.name === 'dark' ? \n 'rgba(237, 237, 237, 0.6)' : \n 'rgba(23, 23, 23, 0.6)'};\n color: ${props => props.theme.name === 'dark' ? '#EDEDED' : '#171717'};\n font-size: 12px;\n font-family: 'Geist', sans-serif;\n white-space: nowrap;\n cursor: pointer;\n width: fit-content;\n -webkit-appearance: none;\n -moz-appearance: none;\n appearance: none;\n`;\n\n\nexport const SearchBar = ({\n apiKey = \"74039c6d-bff7-44ce-ae55-2973cbf13837\",\n apiHost = \"https://gptcloud.arc53.com\",\n theme = \"dark\",\n placeholder = \"Search or Ask AI...\",\n width = \"256px\",\n buttonText = \"Search here\"\n}: SearchBarProps) => {\n const [input, setInput] = React.useState(\"\");\n const [loading, setLoading] = React.useState(false);\n const [isWidgetOpen, setIsWidgetOpen] = React.useState(false);\n const inputRef = React.useRef(null);\n const containerRef = React.useRef(null);\n const [isResultVisible, setIsResultVisible] = React.useState(false);\n const [results, setResults] = React.useState([]);\n const debounceTimeout = React.useRef | null>(null);\n const abortControllerRef = React.useRef(null);\n const browserOS = getOS();\n const isTouch = 'ontouchstart' in window;\n\n const getKeyboardInstruction = () => {\n if (isResultVisible) return \"Enter\";\n return browserOS === 'mac' ? '⌘ + K' : 'Ctrl + K';\n };\n\n React.useEffect(() => {\n loadGeistFont()\n const handleClickOutside = (event: MouseEvent) => {\n if (containerRef.current && !containerRef.current.contains(event.target as Node)) {\n setIsResultVisible(false);\n }\n };\n\n const handleKeyDown = (event: KeyboardEvent) => {\n if (\n ((browserOS === 'win' || browserOS === 'linux') && event.ctrlKey && event.key === 'k') ||\n (browserOS === 'mac' && event.metaKey && event.key === 'k')\n ) {\n event.preventDefault();\n inputRef.current?.focus();\n setIsResultVisible(true);\n } else if (event.key === 'Escape') {\n setIsResultVisible(false);\n }\n };\n\n\n document.addEventListener('mousedown', handleClickOutside);\n document.addEventListener('keydown', handleKeyDown);\n return () => {\n document.removeEventListener('mousedown', handleClickOutside);\n document.removeEventListener('keydown', handleKeyDown);\n };\n }, []);\n\n React.useEffect(() => {\n if (!input) {\n setResults([]);\n setLoading(false);\n return;\n }\n setLoading(true);\n if (debounceTimeout.current) {\n clearTimeout(debounceTimeout.current);\n }\n\n if (abortControllerRef.current) {\n abortControllerRef.current.abort();\n }\n const abortController = new AbortController();\n abortControllerRef.current = abortController;\n\n debounceTimeout.current = setTimeout(() => {\n getSearchResults(input, apiKey, apiHost, abortController.signal)\n .then((data) => setResults(data))\n .catch((err) => !abortController.signal.aborted && console.log(err))\n .finally(() => setLoading(false));\n }, 500);\n\n return () => {\n abortController.abort();\n clearTimeout(debounceTimeout.current ?? undefined);\n };\n}, [input])\n\n const handleKeyDown = (event: React.KeyboardEvent) => {\n if (event.key === 'Enter') {\n event.preventDefault();\n openWidget();\n }\n };\n\n const openWidget = () => {\n setIsWidgetOpen(true);\n setIsResultVisible(false);\n };\n\n const handleClose = () => {\n setIsWidgetOpen(false);\n setIsResultVisible(true);\n };\n\n return (\n \n
\n \n \n setIsResultVisible(true)}\n $inputWidth={width}\n >\n {buttonText}\n \n {\n isResultVisible && (\n <>\n setIsResultVisible(false)} />\n \n \n setInput(e.target.value)}\n onKeyDown={(e) => handleKeyDown(e)}\n placeholder={placeholder}\n autoFocus\n />\n setIsResultVisible(false)}>\n Esc\n \n \n \n \n Ask the AI\n \n \n {!loading ? (\n results.length > 0 ? (\n results.map((res, key) => {\n const containsSource = res.source !== 'local';\n const processedResults = processMarkdownString(res.text, input);\n if (processedResults)\n return (\n {\n if (!containsSource) return;\n window.open(res.source, '_blank', 'noopener, noreferrer');\n }}\n >\n
\n \n \n \n {res.title}\n \n \n {processedResults.map((element, index) => (\n \n \n {element.tag === 'code' && }\n {(element.tag === 'bulletList' || element.tag === 'numberedList') && }\n {element.tag === 'text' && }\n {element.tag === 'heading' && }\n {element.tag === 'blockquote' && }\n \n \n \n ))}\n \n \n
\n \n );\n return null;\n })\n ) : (\n No results found\n )\n ) : (\n \n )}\n \n \n \n )\n }\n {\n isTouch ?\n\n {\n setIsWidgetOpen(true)\n }}\n title={\"Tap to Ask the AI\"}>\n Tap\n \n :\n \n {getKeyboardInstruction()}\n \n }\n \n \n
\n \n )\n}\n" }, { "path": "extensions/react-widget/src/index.html", "content": "\n\n \n \n \n \n DocsGPT Widget\n \n \n
\n \n \n \n\n" }, { "path": "extensions/react-widget/src/index.ts", "content": "//exports methods for React\nexport {SearchBar} from \"./components/SearchBar\"\nexport { DocsGPTWidget } from \"./components/DocsGPTWidget\";\n" }, { "path": "extensions/react-widget/src/main.tsx", "content": "\n//development\nimport { createRoot } from \"react-dom/client\";\nimport { App } from \"./App\";\nimport React from \"react\";\nconst container = document.getElementById(\"app\") as HTMLElement;\nconst root = createRoot(container)\nroot.render();\n" }, { "path": "extensions/react-widget/src/requests/searchAPI.ts", "content": "import { Result } from \"@/types\";\n\nasync function getSearchResults(question: string, apiKey: string, apiHost: string, signal: AbortSignal): Promise {\n\n const payload = {\n question,\n api_key: apiKey\n };\n\n try {\n const response = await fetch(`${apiHost}/api/search`, {\n method: \"POST\",\n headers: {\n \"Content-Type\": \"application/json\",\n },\n body: JSON.stringify(payload),\n signal: signal\n });\n\n if (!response.ok) {\n throw new Error(`Error: ${response.status}`);\n }\n\n const data: Result[] = await response.json();\n return data;\n\n } catch (error) {\n if (!(error instanceof DOMException && error.name == \"AbortError\")) {\n console.error(\"Failed to fetch documents:\", error);\n }\n throw error;\n }\n}\n\nexport {\n getSearchResults\n}" }, { "path": "extensions/react-widget/src/requests/streamingApi.ts", "content": "import { FEEDBACK } from \"@/types\";\n\ninterface HistoryItem {\n prompt: string;\n response?: string;\n}\n\ninterface FetchAnswerStreamingProps {\n question?: string;\n apiKey?: string;\n selectedDocs?: string;\n history?: HistoryItem[];\n conversationId?: string | null;\n apiHost?: string;\n onEvent?: (event: MessageEvent) => void;\n}\n\nexport interface FeedbackPayload {\n question?: string;\n answer?: string;\n feedback: string | null;\n apikey?: string;\n conversation_id: string;\n question_index: number;\n}\n\nexport function fetchAnswerStreaming({\n question = '',\n apiKey = '',\n history = [],\n conversationId = null,\n apiHost = '',\n onEvent = () => { console.log(\"Event triggered, but no handler provided.\"); }\n}: FetchAnswerStreamingProps): Promise {\n return new Promise((resolve, reject) => {\n const body = {\n question: question,\n history: JSON.stringify(history),\n conversation_id: conversationId,\n model: 'default',\n api_key: apiKey\n };\n fetch(apiHost + '/stream', {\n method: 'POST',\n headers: {\n 'Content-Type': 'application/json',\n },\n body: JSON.stringify(body),\n })\n .then((response) => {\n if (!response.body) throw Error('No response body');\n\n const reader = response.body.getReader();\n const decoder = new TextDecoder('utf-8');\n let counter = 0;\n const processStream = ({\n done,\n value,\n }: ReadableStreamReadResult) => {\n if (done) {\n resolve();\n return;\n }\n\n counter += 1;\n\n const chunk = decoder.decode(value);\n\n const lines = chunk.split('\\n');\n\n for (let line of lines) {\n if (line.trim() == '') {\n continue;\n }\n if (line.startsWith('data:')) {\n line = line.substring(5);\n }\n\n const messageEvent = new MessageEvent('message', {\n data: line,\n });\n\n onEvent(messageEvent); // handle each message\n }\n\n reader.read().then(processStream).catch(reject);\n };\n\n reader.read().then(processStream).catch(reject);\n })\n .catch((error) => {\n console.error('Connection failed:', error);\n reject(error);\n });\n });\n}\n\n\nexport const sendFeedback = (payload: FeedbackPayload, apiHost: string): Promise => {\n return fetch(`${apiHost}/api/feedback`, {\n method: 'POST',\n headers: {\n 'Content-Type': 'application/json'\n },\n body: JSON.stringify({\n question: payload.question,\n answer: payload.answer,\n feedback: payload.feedback,\n api_key: payload.apikey,\n conversation_id: payload.conversation_id,\n question_index: payload.question_index\n }),\n });\n};\n" }, { "path": "extensions/react-widget/src/types/index.ts", "content": "export type MESSAGE_TYPE = 'QUESTION' | 'ANSWER' | 'ERROR';\n\nexport type Status = 'idle' | 'loading' | 'failed';\n\nexport type FEEDBACK = 'LIKE' | 'DISLIKE';\n\nexport type THEME = 'light' | 'dark';\n\nexport interface Query {\n prompt: string;\n response?: string;\n feedback?: FEEDBACK;\n error?: string;\n sources?: { title: string; text: string, source:string }[];\n conversationId?: string | null;\n title?: string | null;\n}\n\nexport interface WidgetProps {\n apiHost?: string;\n apiKey?: string;\n avatar?: string;\n title?: string;\n description?: string;\n heroTitle?: string;\n heroDescription?: string;\n size?: 'small' | 'medium' | 'large' | {\n custom: {\n width: string;\n height: string;\n maxWidth?: string;\n maxHeight?: string;\n };\n };\n theme?:THEME,\n buttonIcon?:string;\n buttonText?:string;\n buttonBg?:string;\n collectFeedback?:boolean;\n showSources?: boolean;\n defaultOpen?: boolean;\n}\nexport interface WidgetCoreProps extends WidgetProps { \n widgetRef?:React.RefObject | null;\n handleClose?:React.MouseEventHandler | undefined;\n isOpen:boolean;\n prefilledQuery?: string;\n}\n\nexport interface SearchBarProps { \n apiHost?: string;\n apiKey?: string;\n theme?: THEME;\n placeholder?: string;\n width?: string;\n buttonText?: string;\n}\n\nexport interface Result {\n text:string;\n title:string;\n source:string;\n}\n" }, { "path": "extensions/react-widget/src/utils/helper.ts", "content": "export const getOS = () => {\n const platform = window.navigator.platform;\n const userAgent = window.navigator.userAgent || window.navigator.vendor;\n\n if (/Mac/i.test(platform)) {\n return 'mac';\n }\n\n if (/Win/i.test(platform)) {\n return 'win';\n }\n\n if (/Linux/i.test(platform) && !/Android/i.test(userAgent)) {\n return 'linux';\n }\n\n if (/Android/i.test(userAgent)) {\n return 'android';\n }\n\n if (/iPhone|iPad|iPod/i.test(userAgent)) {\n return 'ios';\n }\n\n return 'other';\n};\n\ninterface ParsedElement {\n content: string;\n tag: string;\n}\n\nexport const processMarkdownString = (markdown: string, keyword?: string): ParsedElement[] => {\n const lines = markdown.trim().split('\\n');\n const keywordLower = keyword?.toLowerCase();\n\n const escapeRegExp = (str: string) => str.replace(/[-\\/\\\\^$*+?.()|[\\]{}]/g, '\\\\$&');\n const escapedKeyword = keyword ? escapeRegExp(keyword) : '';\n const keywordRegex = keyword ? new RegExp(`(${escapedKeyword})`, 'gi') : null;\n\n let isInCodeBlock = false;\n let codeBlockContent: string[] = [];\n let matchingLines: ParsedElement[] = [];\n let firstLine: ParsedElement | null = null;\n\n for (let i = 0; i < lines.length; i++) {\n const trimmedLine = lines[i].trim();\n if (!trimmedLine) continue;\n\n if (trimmedLine.startsWith('```')) {\n if (!isInCodeBlock) {\n isInCodeBlock = true;\n codeBlockContent = [];\n } else {\n isInCodeBlock = false;\n const codeContent = codeBlockContent.join('\\n');\n const parsedElement: ParsedElement = {\n content: codeContent,\n tag: 'code'\n };\n\n if (!firstLine) {\n firstLine = parsedElement;\n }\n\n if (keywordLower && codeContent.toLowerCase().includes(keywordLower)) {\n parsedElement.content = parsedElement.content.replace(keywordRegex!, '$1');\n matchingLines.push(parsedElement);\n }\n }\n continue;\n }\n\n if (isInCodeBlock) {\n codeBlockContent.push(trimmedLine);\n continue;\n }\n\n let parsedElement: ParsedElement | null = null;\n\n const headingMatch = trimmedLine.match(/^(#{1,6})\\s+(.+)$/);\n const bulletMatch = trimmedLine.match(/^[-*]\\s+(.+)$/);\n const numberedMatch = trimmedLine.match(/^\\d+\\.\\s+(.+)$/);\n const blockquoteMatch = trimmedLine.match(/^>+\\s*(.+)$/);\n\n let content = trimmedLine;\n\n if (headingMatch) {\n content = headingMatch[2];\n parsedElement = {\n content: content,\n tag: 'heading'\n };\n } else if (bulletMatch) {\n content = bulletMatch[1];\n parsedElement = {\n content: content,\n tag: 'bulletList'\n };\n } else if (numberedMatch) {\n content = numberedMatch[1];\n parsedElement = {\n content: content,\n tag: 'numberedList'\n };\n } else if (blockquoteMatch) {\n content = blockquoteMatch[1];\n parsedElement = {\n content: content,\n tag: 'blockquote'\n };\n } else {\n parsedElement = {\n content: content,\n tag: 'text'\n };\n }\n\n if (!firstLine) {\n firstLine = parsedElement;\n }\n\n if (keywordLower && parsedElement.content.toLowerCase().includes(keywordLower)) {\n parsedElement.content = parsedElement.content.replace(keywordRegex!, '$1');\n matchingLines.push(parsedElement);\n }\n }\n\n if (isInCodeBlock && codeBlockContent.length > 0) {\n const codeContent = codeBlockContent.join('\\n');\n const parsedElement: ParsedElement = {\n content: codeContent,\n tag: 'code'\n };\n\n if (!firstLine) {\n firstLine = parsedElement;\n }\n\n if (keywordLower && codeContent.toLowerCase().includes(keywordLower)) {\n parsedElement.content = parsedElement.content.replace(keywordRegex!, '$1');\n matchingLines.push(parsedElement);\n }\n }\n\n if (keywordLower && matchingLines.length > 0) {\n return matchingLines;\n }\n\n return firstLine ? [firstLine] : [];\n};\n" }, { "path": "extensions/react-widget/tsconfig.json", "content": "{\n \"compilerOptions\": {\n \"baseUrl\": \".\",\n \"paths\": {\n \"@/*\": [\"src/*\", \"@/*\"]\n },\n \"target\": \"ES2020\",\n \"useDefineForClassFields\": true,\n \"lib\": [\"ES2020\", \"DOM\", \"DOM.Iterable\"],\n \"module\": \"ESNext\",\n \"skipLibCheck\": true,\n \n /* Bundler mode */\n \"moduleResolution\": \"bundler\",\n \"allowImportingTsExtensions\": true,\n \"resolveJsonModule\": true,\n \"isolatedModules\": true,\n \"noEmit\": true,\n \"jsx\": \"react-jsx\",\n \n /* Linting */\n \"strict\": true,\n \"noUnusedLocals\": false,\n \"noUnusedParameters\": false,\n \"noFallthroughCasesInSwitch\": true,\n /* The \"typeRoots\" configuration specifies the locations where \n TypeScript looks for type definitions (.d.ts files) to \n include in the compilation process.*/\n \"typeRoots\": [\"./dist/index.d.ts\", \"node_modules/@types\"]\n },\n /* include /index.ts*/\n \"include\": [\"src/index.ts\",\"custom.d.ts\"],\n \"exclude\": [\"node_modules\"],\n }" }, { "path": "extensions/slack-bot/.gitignore", "content": ".env\n.venv/\nget-pip.py" }, { "path": "extensions/slack-bot/Readme.md", "content": "\n# Slack Bot Configuration Guide\n\n> **Note:** The following guidelines must be followed on the [Slack API website](https://api.slack.com/) for setting up your Slack app and generating the necessary tokens.\n\n## Step-by-Step Instructions\n\n### 1. Navigate to Your Apps\n- Go to the Slack API page for apps and select **Create an App** from the “From Scratch” option.\n\n### 2. App Creation\n- Name your app and choose the workspace where you wish to add the assistant. \n\n### 3. Enabling Socket Mode\n- Navigate to **Settings > Socket Mode** and enable **Socket Mode**. \n- This action will generate an App-level token. Select the `connections:write` scope and copy the App-level token for future use.\n\n### 4. Socket Naming\n- Assign a name to your socket as per your preference.\n\n### 5. Basic Information Setup\n- Go to **Basic Information** (under **Settings**) and configure the following:\n - Assistant name\n - App icon\n - Background color \n\n### 6. Bot Token and Permissions\n- In the **OAuth & Permissions** option found under the **Features** section, retrieve the Bot Token. Save it for future usage.\n- You will also need to add specific bot token scopes:\n - `app_mentions:read`\n - `assistant:write`\n - `chat:write`\n - `chat:write.public`\n - `im:history`\n\n### 7. Enable Events\n- From **Event Subscriptions**, enable events and add the following Bot User events:\n - `app_mention`\n - `assistant_thread_context_changed`\n - `assistant_thread_started`\n - `message.im`\n\n### 8. Agent/Assistant Toggle\n- In the **Features > Agent & Assistants** section, toggle on the Agent or Assistant option. \n- In the **Suggested Prompts** setting, leave it as `dynamic` (this is the default setting).\n\n---\n\n## Code-Side Configuration Guide\n\nThis section focuses on generating and setting up the necessary tokens in the `.env` file, using the `.env-example` as a template.\n\n### Step 1: Generating Required Keys\n\n1. **SLACK_APP_TOKEN**\n - Navigate to **Settings > Socket Mode** in the Slack API and enable **Socket Mode**.\n - Copy the App-level token generated (usually starts with `xapp-`).\n\n2. **SLACK_BOT_TOKEN**\n - Go to **OAuth & Permissions** (under the **Features** section in Slack API).\n - Retrieve the **Bot Token** (starts with `xoxb-`).\n\n3. **DOCSGPT_API_KEY**\n - Go to the **DocsGPT website**.\n - Navigate to **Settings > Chatbots > Create New** to generate a DocsGPT API Key.\n - Copy the generated key for use.\n\n### Step 2: Creating and Updating the `.env` File\n\n1. Create a new `.env` file in the root of your project (if it doesn’t already exist).\n2. Use the `.env-example` as a reference and update the file with the following keys and values:\n\n```bash\n# .env file\nSLACK_APP_TOKEN=xapp-your-generated-app-token\nSLACK_BOT_TOKEN=xoxb-your-generated-bot-token\nDOCSGPT_API_KEY=your-docsgpt-generated-api-key\n```\n\nReplace the placeholder values with the actual tokens generated from the Slack API and DocsGPT as per the steps outlined above.\n\n---\n\nThis concludes the guide for both setting up the Slack API and configuring the `.env` file on the code side.\n" }, { "path": "extensions/slack-bot/app.py", "content": "import os\nimport hashlib\nimport httpx\nimport re\nfrom slack_bolt.async_app import AsyncApp\nfrom slack_bolt.adapter.socket_mode.async_handler import AsyncSocketModeHandler\nfrom dotenv import load_dotenv\n\nload_dotenv()\nAPI_BASE = os.getenv(\"API_BASE\", \"https://gptcloud.arc53.com\")\nAPI_URL = API_BASE + \"/api/answer\"\n\n# Slack bot token and signing secret\nSLACK_BOT_TOKEN = os.getenv(\"SLACK_BOT_TOKEN\")\nSLACK_APP_TOKEN = os.getenv(\"SLACK_APP_TOKEN\")\n\n# OpenAI API key for DocsGPT (replace this with your actual API key)\nDOCSGPT_API_KEY = os.getenv(\"DOCSGPT_API_KEY\")\n\n# Initialize Slack app\napp = AsyncApp(token=SLACK_BOT_TOKEN)\n\ndef encode_conversation_id(conversation_id: str) -> str:\n \"\"\"\n Encodes 11 length Slack conversation_id to 12 length string\n Args:\n conversation_id (str): The 11 digit slack conversation_id.\n Returns:\n str: Hashed id.\n \"\"\" \n # Create a SHA-256 hash of the string\n hashed_id = hashlib.sha256(conversation_id.encode()).hexdigest()\n\n # Take the first 24 characters of the hash\n hashed_24_char_id = hashed_id[:24]\n return hashed_24_char_id\n\nasync def generate_answer(question: str, messages: list, conversation_id: str | None) -> dict:\n \"\"\"Generates an answer using the external API.\"\"\"\n payload = {\n \"question\": question,\n \"api_key\": DOCSGPT_API_KEY,\n \"history\": messages,\n \"conversation_id\": conversation_id,\n }\n headers = {\n \"Content-Type\": \"application/json; charset=utf-8\"\n }\n timeout = 60.0\n async with httpx.AsyncClient() as client:\n response = await client.post(API_URL, json=payload, headers=headers, timeout=timeout)\n\n if response.status_code == 200:\n data = response.json()\n conversation_id = data.get(\"conversation_id\")\n answer = data.get(\"answer\", \"Sorry, I couldn't find an answer.\")\n return {\"answer\": answer, \"conversation_id\": conversation_id}\n else:\n print(response.json())\n return {\"answer\": \"Sorry, I couldn't find an answer.\", \"conversation_id\": None}\n\n@app.message(\".*\")\nasync def message_docs(message, say):\n client = app.client\n channel = message['channel']\n thread_ts = message['thread_ts']\n user_query = message['text'] \n await client.assistant_threads_setStatus(\n channel_id = channel,\n thread_ts = thread_ts,\n status = \"is generating your answer...\",\n )\n\n docs_gpt_channel_id = encode_conversation_id(thread_ts)\n \n # Get response from DocsGPT\n response = await generate_answer(user_query,[], docs_gpt_channel_id)\n answer = convert_to_slack_markdown(response['answer'])\n\n # Respond in Slack\n await client.chat_postMessage(text = answer, mrkdwn= True, channel= message['channel'],\n thread_ts = message['thread_ts'],)\n\ndef convert_to_slack_markdown(markdown_text: str):\n # Convert bold **text** to *text* for Slack\n slack_text = re.sub(r'\\*\\*(.*?)\\*\\*', r'*\\1*', markdown_text) # **text** to *text*\n\n # Convert italics _text_ to _text_ for Slack\n slack_text = re.sub(r'_(.*?)_', r'_\\1_', slack_text) # _text_ to _text_\n\n # Convert inline code `code` to `code` (Slack supports backticks for inline code)\n slack_text = re.sub(r'`(.*?)`', r'`\\1`', slack_text)\n\n # Convert bullet points with single or no spaces to filled bullets (•)\n slack_text = re.sub(r'^\\s{0,1}[-*]\\s+', ' • ', slack_text, flags=re.MULTILINE)\n\n # Convert bullet points with multiple spaces to hollow bullets (◦)\n slack_text = re.sub(r'^\\s{2,}[-*]\\s+', '\\t◦ ', slack_text, flags=re.MULTILINE)\n\n # Convert headers (##) to bold in Slack\n slack_text = re.sub(r'^\\s*#{1,6}\\s*(.*?)$', r'*\\1*', slack_text, flags=re.MULTILINE)\n\n return slack_text\n\nasync def main():\n handler = AsyncSocketModeHandler(app, os.environ[\"SLACK_APP_TOKEN\"])\n await handler.start_async()\n\n# Start the app\nif __name__ == \"__main__\":\n import asyncio\n asyncio.run(main())" }, { "path": "extensions/slack-bot/requirements.txt", "content": "aiohttp>=3,<4\ncertifi==2024.7.4\nh11==0.14.0\nhttpcore==1.0.5\nhttpx==0.27.0\nidna==3.7\npython-dotenv==1.0.1\nsniffio==1.3.1\nslack-bolt==1.21.0\nbson==0.5.10\n" }, { "path": "extensions/web-widget/README.md", "content": "# Chat Widget\n\nA simple chat widget that can be easily integrated into any website.\n\n## Installation\n\n1. Host the `widget.html`, `styles.css`, and `script.js` files from the `src` folder on your own server or a Content Delivery Network (CDN). Make sure to note the URLs for these files.\n\n2. Update the URLs in the `dist/chat-widget.js` file to match the locations of your hosted files:\n\n ```javascript\n fetch(\"https://your-server-or-cdn.com/path/to/widget.html\"),\n fetch(\"https://your-server-or-cdn.com/path/to/styles.css\"),\n fetch(\"https://your-server-or-cdn.com/path/to/script.js\"),\n ```\n \n3. Host the `dist/chat-widget.js` file on your own server or a Content Delivery Network (CDN). Make sure to note the URL for this file.\n\n\n##Integration\n\nTo integrate the chat widget into a website, add the following script tag to the HTML file, replacing URL_TO_CHAT_WIDGET_JS with the actual URL of your hosted chat-widget.js file:\n```javascript\n\n```" }, { "path": "extensions/web-widget/dist/chat-widget.js", "content": "(async function () {\n // Fetch the HTML, CSS, and JavaScript from your server or CDN\n const [htmlRes, jsRes] = await Promise.all([\n fetch(\"https://s3-eu-west-2.amazonaws.com/arc53data/widget.html\"),\n // fetch(\"https://s3-eu-west-2.amazonaws.com/arc53data/tailwind.css\"),\n fetch(\"https://s3-eu-west-2.amazonaws.com/arc53data/script.js\"),\n ]);\n\n const html = await htmlRes.text();\n //const css = await cssRes.text();\n const js = await jsRes.text();\n\n // create a new link element\n const link = document.createElement(\"link\");\n\n //set the rel, href, type, and integrity attributes\n link.rel = \"stylesheet\";\n link.href = \"https://cdn.tailwindcss.com/\";\n link.type = \"text/css\";\n link.integrity = \"sha384-PDOmVviaTm8N1W35y1NSmo80w6GPaGhbDuOBAF/5hRffaeGc6yOwIo1qAt4gqLGA%\";\n\n // get the document head and append the link element to it\n // document.head.appendChild(link);\n\n\n\n // Create a style element for the CSS\n // const style = document.createElement(\"style\");\n // style.innerHTML = css;\n // document.head.appendChild(style);\n\n // Create a container for the chat widget and inject the HTML\n const chatWidgetContainer = document.createElement(\"div\");\n chatWidgetContainer.innerHTML = html;\n document.body.appendChild(chatWidgetContainer);\n\n // Execute the JavaScript code\n const script = document.createElement(\"script\");\n script.innerHTML = js;\n document.body.appendChild(script);\n})();\n" }, { "path": "extensions/web-widget/dist/output.css", "content": "/*\n! tailwindcss v3.3.1 | MIT License | https://tailwindcss.com\n*/\n\n/*\n1. Prevent padding and border from affecting element width. (https://github.com/mozdevs/cssremedy/issues/4)\n2. Allow adding a border to an element by just adding a border-width. (https://github.com/tailwindcss/tailwindcss/pull/116)\n*/\n\n*,\n::before,\n::after {\n box-sizing: border-box;\n /* 1 */\n border-width: 0;\n /* 2 */\n border-style: solid;\n /* 2 */\n border-color: #e5e7eb;\n /* 2 */\n}\n\n::before,\n::after {\n --tw-content: '';\n}\n\n/*\n1. Use a consistent sensible line-height in all browsers.\n2. Prevent adjustments of font size after orientation changes in iOS.\n3. Use a more readable tab size.\n4. Use the user's configured `sans` font-family by default.\n5. Use the user's configured `sans` font-feature-settings by default.\n6. Use the user's configured `sans` font-variation-settings by default.\n*/\n\nhtml {\n line-height: 1.5;\n /* 1 */\n -webkit-text-size-adjust: 100%;\n /* 2 */\n -moz-tab-size: 4;\n /* 3 */\n -o-tab-size: 4;\n tab-size: 4;\n /* 3 */\n font-family: ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, \"Segoe UI\", Roboto, \"Helvetica Neue\", Arial, \"Noto Sans\", sans-serif, \"Apple Color Emoji\", \"Segoe UI Emoji\", \"Segoe UI Symbol\", \"Noto Color Emoji\";\n /* 4 */\n font-feature-settings: normal;\n /* 5 */\n font-variation-settings: normal;\n /* 6 */\n}\n\n/*\n1. Remove the margin in all browsers.\n2. Inherit line-height from `html` so users can set them as a class directly on the `html` element.\n*/\n\nbody {\n margin: 0;\n /* 1 */\n line-height: inherit;\n /* 2 */\n}\n\n/*\n1. Add the correct height in Firefox.\n2. Correct the inheritance of border color in Firefox. (https://bugzilla.mozilla.org/show_bug.cgi?id=190655)\n3. Ensure horizontal rules are visible by default.\n*/\n\nhr {\n height: 0;\n /* 1 */\n color: inherit;\n /* 2 */\n border-top-width: 1px;\n /* 3 */\n}\n\n/*\nAdd the correct text decoration in Chrome, Edge, and Safari.\n*/\n\nabbr:where([title]) {\n -webkit-text-decoration: underline dotted;\n text-decoration: underline dotted;\n}\n\n/*\nRemove the default font size and weight for headings.\n*/\n\nh1,\nh2,\nh3,\nh4,\nh5,\nh6 {\n font-size: inherit;\n font-weight: inherit;\n}\n\n/*\nReset links to optimize for opt-in styling instead of opt-out.\n*/\n\na {\n color: inherit;\n text-decoration: inherit;\n}\n\n/*\nAdd the correct font weight in Edge and Safari.\n*/\n\nb,\nstrong {\n font-weight: bolder;\n}\n\n/*\n1. Use the user's configured `mono` font family by default.\n2. Correct the odd `em` font sizing in all browsers.\n*/\n\ncode,\nkbd,\nsamp,\npre {\n font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, \"Liberation Mono\", \"Courier New\", monospace;\n /* 1 */\n font-size: 1em;\n /* 2 */\n}\n\n/*\nAdd the correct font size in all browsers.\n*/\n\nsmall {\n font-size: 80%;\n}\n\n/*\nPrevent `sub` and `sup` elements from affecting the line height in all browsers.\n*/\n\nsub,\nsup {\n font-size: 75%;\n line-height: 0;\n position: relative;\n vertical-align: baseline;\n}\n\nsub {\n bottom: -0.25em;\n}\n\nsup {\n top: -0.5em;\n}\n\n/*\n1. Remove text indentation from table contents in Chrome and Safari. (https://bugs.chromium.org/p/chromium/issues/detail?id=999088, https://bugs.webkit.org/show_bug.cgi?id=201297)\n2. Correct table border color inheritance in all Chrome and Safari. (https://bugs.chromium.org/p/chromium/issues/detail?id=935729, https://bugs.webkit.org/show_bug.cgi?id=195016)\n3. Remove gaps between table borders by default.\n*/\n\ntable {\n text-indent: 0;\n /* 1 */\n border-color: inherit;\n /* 2 */\n border-collapse: collapse;\n /* 3 */\n}\n\n/*\n1. Change the font styles in all browsers.\n2. Remove the margin in Firefox and Safari.\n3. Remove default padding in all browsers.\n*/\n\nbutton,\ninput,\noptgroup,\nselect,\ntextarea {\n font-family: inherit;\n /* 1 */\n font-size: 100%;\n /* 1 */\n font-weight: inherit;\n /* 1 */\n line-height: inherit;\n /* 1 */\n color: inherit;\n /* 1 */\n margin: 0;\n /* 2 */\n padding: 0;\n /* 3 */\n}\n\n/*\nRemove the inheritance of text transform in Edge and Firefox.\n*/\n\nbutton,\nselect {\n text-transform: none;\n}\n\n/*\n1. Correct the inability to style clickable types in iOS and Safari.\n2. Remove default button styles.\n*/\n\nbutton,\n[type='button'],\n[type='reset'],\n[type='submit'] {\n -webkit-appearance: button;\n /* 1 */\n background-color: transparent;\n /* 2 */\n background-image: none;\n /* 2 */\n}\n\n/*\nUse the modern Firefox focus style for all focusable elements.\n*/\n\n:-moz-focusring {\n outline: auto;\n}\n\n/*\nRemove the additional `:invalid` styles in Firefox. (https://github.com/mozilla/gecko-dev/blob/2f9eacd9d3d995c937b4251a5557d95d494c9be1/layout/style/res/forms.css#L728-L737)\n*/\n\n:-moz-ui-invalid {\n box-shadow: none;\n}\n\n/*\nAdd the correct vertical alignment in Chrome and Firefox.\n*/\n\nprogress {\n vertical-align: baseline;\n}\n\n/*\nCorrect the cursor style of increment and decrement buttons in Safari.\n*/\n\n::-webkit-inner-spin-button,\n::-webkit-outer-spin-button {\n height: auto;\n}\n\n/*\n1. Correct the odd appearance in Chrome and Safari.\n2. Correct the outline style in Safari.\n*/\n\n[type='search'] {\n -webkit-appearance: textfield;\n /* 1 */\n outline-offset: -2px;\n /* 2 */\n}\n\n/*\nRemove the inner padding in Chrome and Safari on macOS.\n*/\n\n::-webkit-search-decoration {\n -webkit-appearance: none;\n}\n\n/*\n1. Correct the inability to style clickable types in iOS and Safari.\n2. Change font properties to `inherit` in Safari.\n*/\n\n::-webkit-file-upload-button {\n -webkit-appearance: button;\n /* 1 */\n font: inherit;\n /* 2 */\n}\n\n/*\nAdd the correct display in Chrome and Safari.\n*/\n\nsummary {\n display: list-item;\n}\n\n/*\nRemoves the default spacing and border for appropriate elements.\n*/\n\nblockquote,\ndl,\ndd,\nh1,\nh2,\nh3,\nh4,\nh5,\nh6,\nhr,\nfigure,\np,\npre {\n margin: 0;\n}\n\nfieldset {\n margin: 0;\n padding: 0;\n}\n\nlegend {\n padding: 0;\n}\n\nol,\nul,\nmenu {\n list-style: none;\n margin: 0;\n padding: 0;\n}\n\n/*\nPrevent resizing textareas horizontally by default.\n*/\n\ntextarea {\n resize: vertical;\n}\n\n/*\n1. Reset the default placeholder opacity in Firefox. (https://github.com/tailwindlabs/tailwindcss/issues/3300)\n2. Set the default placeholder color to the user's configured gray 400 color.\n*/\n\ninput::-moz-placeholder, textarea::-moz-placeholder {\n opacity: 1;\n /* 1 */\n color: #9ca3af;\n /* 2 */\n}\n\ninput::placeholder,\ntextarea::placeholder {\n opacity: 1;\n /* 1 */\n color: #9ca3af;\n /* 2 */\n}\n\n/*\nSet the default cursor for buttons.\n*/\n\nbutton,\n[role=\"button\"] {\n cursor: pointer;\n}\n\n/*\nMake sure disabled buttons don't get the pointer cursor.\n*/\n\n:disabled {\n cursor: default;\n}\n\n/*\n1. Make replaced elements `display: block` by default. (https://github.com/mozdevs/cssremedy/issues/14)\n2. Add `vertical-align: middle` to align replaced elements more sensibly by default. (https://github.com/jensimmons/cssremedy/issues/14#issuecomment-634934210)\n This can trigger a poorly considered lint error in some tools but is included by design.\n*/\n\nimg,\nsvg,\nvideo,\ncanvas,\naudio,\niframe,\nembed,\nobject {\n display: block;\n /* 1 */\n vertical-align: middle;\n /* 2 */\n}\n\n/*\nConstrain images and videos to the parent width and preserve their intrinsic aspect ratio. (https://github.com/mozdevs/cssremedy/issues/14)\n*/\n\nimg,\nvideo {\n max-width: 100%;\n height: auto;\n}\n\n/* Make elements with the HTML hidden attribute stay hidden by default */\n\n[hidden] {\n display: none;\n}\n\n*, ::before, ::after {\n --tw-border-spacing-x: 0;\n --tw-border-spacing-y: 0;\n --tw-translate-x: 0;\n --tw-translate-y: 0;\n --tw-rotate: 0;\n --tw-skew-x: 0;\n --tw-skew-y: 0;\n --tw-scale-x: 1;\n --tw-scale-y: 1;\n --tw-pan-x: ;\n --tw-pan-y: ;\n --tw-pinch-zoom: ;\n --tw-scroll-snap-strictness: proximity;\n --tw-ordinal: ;\n --tw-slashed-zero: ;\n --tw-numeric-figure: ;\n --tw-numeric-spacing: ;\n --tw-numeric-fraction: ;\n --tw-ring-inset: ;\n --tw-ring-offset-width: 0px;\n --tw-ring-offset-color: #fff;\n --tw-ring-color: rgb(59 130 246 / 0.5);\n --tw-ring-offset-shadow: 0 0 #0000;\n --tw-ring-shadow: 0 0 #0000;\n --tw-shadow: 0 0 #0000;\n --tw-shadow-colored: 0 0 #0000;\n --tw-blur: ;\n --tw-brightness: ;\n --tw-contrast: ;\n --tw-grayscale: ;\n --tw-hue-rotate: ;\n --tw-invert: ;\n --tw-saturate: ;\n --tw-sepia: ;\n --tw-drop-shadow: ;\n --tw-backdrop-blur: ;\n --tw-backdrop-brightness: ;\n --tw-backdrop-contrast: ;\n --tw-backdrop-grayscale: ;\n --tw-backdrop-hue-rotate: ;\n --tw-backdrop-invert: ;\n --tw-backdrop-opacity: ;\n --tw-backdrop-saturate: ;\n --tw-backdrop-sepia: ;\n}\n\n::backdrop {\n --tw-border-spacing-x: 0;\n --tw-border-spacing-y: 0;\n --tw-translate-x: 0;\n --tw-translate-y: 0;\n --tw-rotate: 0;\n --tw-skew-x: 0;\n --tw-skew-y: 0;\n --tw-scale-x: 1;\n --tw-scale-y: 1;\n --tw-pan-x: ;\n --tw-pan-y: ;\n --tw-pinch-zoom: ;\n --tw-scroll-snap-strictness: proximity;\n --tw-ordinal: ;\n --tw-slashed-zero: ;\n --tw-numeric-figure: ;\n --tw-numeric-spacing: ;\n --tw-numeric-fraction: ;\n --tw-ring-inset: ;\n --tw-ring-offset-width: 0px;\n --tw-ring-offset-color: #fff;\n --tw-ring-color: rgb(59 130 246 / 0.5);\n --tw-ring-offset-shadow: 0 0 #0000;\n --tw-ring-shadow: 0 0 #0000;\n --tw-shadow: 0 0 #0000;\n --tw-shadow-colored: 0 0 #0000;\n --tw-blur: ;\n --tw-brightness: ;\n --tw-contrast: ;\n --tw-grayscale: ;\n --tw-hue-rotate: ;\n --tw-invert: ;\n --tw-saturate: ;\n --tw-sepia: ;\n --tw-drop-shadow: ;\n --tw-backdrop-blur: ;\n --tw-backdrop-brightness: ;\n --tw-backdrop-contrast: ;\n --tw-backdrop-grayscale: ;\n --tw-backdrop-hue-rotate: ;\n --tw-backdrop-invert: ;\n --tw-backdrop-opacity: ;\n --tw-backdrop-saturate: ;\n --tw-backdrop-sepia: ;\n}\n\n.fixed {\n position: fixed;\n}\n\n.absolute {\n position: absolute;\n}\n\n.relative {\n position: relative;\n}\n\n.inset-y-0 {\n top: 0px;\n bottom: 0px;\n}\n\n.bottom-5 {\n bottom: 1.25rem;\n}\n\n.left-5 {\n left: 1.25rem;\n}\n\n.right-2 {\n right: 0.5rem;\n}\n\n.z-50 {\n z-index: 50;\n}\n\n.m-0 {\n margin: 0px;\n}\n\n.-mx-2 {\n margin-left: -0.5rem;\n margin-right: -0.5rem;\n}\n\n.mt-1 {\n margin-top: 0.25rem;\n}\n\n.flex {\n display: flex;\n}\n\n.hidden {\n display: none;\n}\n\n.w-full {\n width: 100%;\n}\n\n.flex-1 {\n flex: 1 1 0%;\n}\n\n.transform {\n transform: translate(var(--tw-translate-x), var(--tw-translate-y)) rotate(var(--tw-rotate)) skewX(var(--tw-skew-x)) skewY(var(--tw-skew-y)) scaleX(var(--tw-scale-x)) scaleY(var(--tw-scale-y));\n}\n\n.items-center {\n align-items: center;\n}\n\n.justify-center {\n justify-content: center;\n}\n\n.gap-2 {\n gap: 0.5rem;\n}\n\n.divide-y > :not([hidden]) ~ :not([hidden]) {\n --tw-divide-y-reverse: 0;\n border-top-width: calc(1px * calc(1 - var(--tw-divide-y-reverse)));\n border-bottom-width: calc(1px * var(--tw-divide-y-reverse));\n}\n\n.rounded-md {\n border-radius: 0.375rem;\n}\n\n.rounded-b {\n border-bottom-right-radius: 0.25rem;\n border-bottom-left-radius: 0.25rem;\n}\n\n.border {\n border-width: 1px;\n}\n\n.bg-transparent {\n background-color: transparent;\n}\n\n.bg-gradient-to-br {\n background-image: linear-gradient(to bottom right, var(--tw-gradient-stops));\n}\n\n.from-gray-100\\/80 {\n --tw-gradient-from: rgb(243 244 246 / 0.8) var(--tw-gradient-from-position);\n --tw-gradient-from-position: ;\n --tw-gradient-to: rgb(243 244 246 / 0) var(--tw-gradient-from-position);\n --tw-gradient-to-position: ;\n --tw-gradient-stops: var(--tw-gradient-from), var(--tw-gradient-to);\n}\n\n.via-white {\n --tw-gradient-via-position: ;\n --tw-gradient-to: rgb(255 255 255 / 0) var(--tw-gradient-to-position);\n --tw-gradient-to-position: ;\n --tw-gradient-stops: var(--tw-gradient-from), #fff var(--tw-gradient-via-position), var(--tw-gradient-to);\n}\n\n.to-white {\n --tw-gradient-to: #fff var(--tw-gradient-to-position);\n --tw-gradient-to-position: ;\n}\n\n.p-3 {\n padding: 0.75rem;\n}\n\n.px-2 {\n padding-left: 0.5rem;\n padding-right: 0.5rem;\n}\n\n.px-5 {\n padding-left: 1.25rem;\n padding-right: 1.25rem;\n}\n\n.py-3 {\n padding-top: 0.75rem;\n padding-bottom: 0.75rem;\n}\n\n.pl-5 {\n padding-left: 1.25rem;\n}\n\n.pr-8 {\n padding-right: 2rem;\n}\n\n.font-sans {\n font-family: ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, \"Segoe UI\", Roboto, \"Helvetica Neue\", Arial, \"Noto Sans\", sans-serif, \"Apple Color Emoji\", \"Segoe UI Emoji\", \"Segoe UI Symbol\", \"Noto Color Emoji\";\n}\n\n.text-sm {\n font-size: 0.875rem;\n line-height: 1.25rem;\n}\n\n.text-xs {\n font-size: 0.75rem;\n line-height: 1rem;\n}\n\n.font-bold {\n font-weight: 700;\n}\n\n.text-gray-400 {\n --tw-text-opacity: 1;\n color: rgb(156 163 175 / var(--tw-text-opacity));\n}\n\n.text-gray-600 {\n --tw-text-opacity: 1;\n color: rgb(75 85 99 / var(--tw-text-opacity));\n}\n\n.text-gray-700 {\n --tw-text-opacity: 1;\n color: rgb(55 65 81 / var(--tw-text-opacity));\n}\n\n.text-gray-800 {\n --tw-text-opacity: 1;\n color: rgb(31 41 55 / var(--tw-text-opacity));\n}\n\n.shadow {\n --tw-shadow: 0 1px 3px 0 rgb(0 0 0 / 0.1), 0 1px 2px -1px rgb(0 0 0 / 0.1);\n --tw-shadow-colored: 0 1px 3px 0 var(--tw-shadow-color), 0 1px 2px -1px var(--tw-shadow-color);\n box-shadow: var(--tw-ring-offset-shadow, 0 0 #0000), var(--tw-ring-shadow, 0 0 #0000), var(--tw-shadow);\n}\n\n.backdrop-blur-sm {\n --tw-backdrop-blur: blur(4px);\n -webkit-backdrop-filter: var(--tw-backdrop-blur) var(--tw-backdrop-brightness) var(--tw-backdrop-contrast) var(--tw-backdrop-grayscale) var(--tw-backdrop-hue-rotate) var(--tw-backdrop-invert) var(--tw-backdrop-opacity) var(--tw-backdrop-saturate) var(--tw-backdrop-sepia);\n backdrop-filter: var(--tw-backdrop-blur) var(--tw-backdrop-brightness) var(--tw-backdrop-contrast) var(--tw-backdrop-grayscale) var(--tw-backdrop-hue-rotate) var(--tw-backdrop-invert) var(--tw-backdrop-opacity) var(--tw-backdrop-saturate) var(--tw-backdrop-sepia);\n}\n\n.transition {\n transition-property: color, background-color, border-color, text-decoration-color, fill, stroke, opacity, box-shadow, transform, filter, -webkit-backdrop-filter;\n transition-property: color, background-color, border-color, text-decoration-color, fill, stroke, opacity, box-shadow, transform, filter, backdrop-filter;\n transition-property: color, background-color, border-color, text-decoration-color, fill, stroke, opacity, box-shadow, transform, filter, backdrop-filter, -webkit-backdrop-filter;\n transition-timing-function: cubic-bezier(0.4, 0, 0.2, 1);\n transition-duration: 150ms;\n}\n\n.delay-200 {\n transition-delay: 200ms;\n}\n\n.duration-300 {\n transition-duration: 300ms;\n}\n\n.hover\\:bg-gray-100:hover {\n --tw-bg-opacity: 1;\n background-color: rgb(243 244 246 / var(--tw-bg-opacity));\n}\n\n.focus\\:outline-none:focus {\n outline: 2px solid transparent;\n outline-offset: 2px;\n}\n\n@media (prefers-color-scheme: dark) {\n .dark\\:divide-gray-700 > :not([hidden]) ~ :not([hidden]) {\n --tw-divide-opacity: 1;\n border-color: rgb(55 65 81 / var(--tw-divide-opacity));\n }\n\n .dark\\:border-gray-700 {\n --tw-border-opacity: 1;\n border-color: rgb(55 65 81 / var(--tw-border-opacity));\n }\n\n .dark\\:from-gray-900\\/80 {\n --tw-gradient-from: rgb(17 24 39 / 0.8) var(--tw-gradient-from-position);\n --tw-gradient-from-position: ;\n --tw-gradient-to: rgb(17 24 39 / 0) var(--tw-gradient-from-position);\n --tw-gradient-to-position: ;\n --tw-gradient-stops: var(--tw-gradient-from), var(--tw-gradient-to);\n }\n\n .dark\\:via-gray-900 {\n --tw-gradient-via-position: ;\n --tw-gradient-to: rgb(17 24 39 / 0) var(--tw-gradient-to-position);\n --tw-gradient-to-position: ;\n --tw-gradient-stops: var(--tw-gradient-from), #111827 var(--tw-gradient-via-position), var(--tw-gradient-to);\n }\n\n .dark\\:to-gray-900 {\n --tw-gradient-to: #111827 var(--tw-gradient-to-position);\n --tw-gradient-to-position: ;\n }\n\n .dark\\:text-gray-200 {\n --tw-text-opacity: 1;\n color: rgb(229 231 235 / var(--tw-text-opacity));\n }\n\n .dark\\:text-gray-300 {\n --tw-text-opacity: 1;\n color: rgb(209 213 219 / var(--tw-text-opacity));\n }\n\n .dark\\:text-gray-500 {\n --tw-text-opacity: 1;\n color: rgb(107 114 128 / var(--tw-text-opacity));\n }\n\n .dark\\:text-white {\n --tw-text-opacity: 1;\n color: rgb(255 255 255 / var(--tw-text-opacity));\n }\n\n .dark\\:hover\\:bg-gray-800\\/70:hover {\n background-color: rgb(31 41 55 / 0.7);\n }\n}\n\n@media (min-width: 768px) {\n .md\\:pl-0 {\n padding-left: 0px;\n }\n}" }, { "path": "extensions/web-widget/index.html", "content": "\n\n\n \n \n Chat Widget Test\n \n\n\n \n\n\n" }, { "path": "extensions/web-widget/package.json", "content": "{\n \"name\": \"web-widget\",\n \"version\": \"1.0.0\",\n \"description\": \"\",\n \"main\": \"index.js\",\n \"scripts\": {\n \"test\": \"echo \\\"Error: no test specified\\\" && exit 1\"\n },\n \"keywords\": [],\n \"author\": \"\",\n \"license\": \"ISC\",\n \"devDependencies\": {\n \"tailwindcss\": \"^3.3.1\"\n }\n}\n" }, { "path": "extensions/web-widget/src/html/widget.html", "content": "
\n\n\n\n
\n
\n
\n

Looking for help with documentation?

\n

DocsGPT AI assistant will help you with docs

\n
\n
\n

Come cool answer

\n
\n\n
\n
\n
\n \n\n
\n \n \n
\n

\n Processing...\n

\n\n\n\n
\n
\n
" }, { "path": "extensions/web-widget/src/input.css", "content": "@tailwind base;\n@tailwind components;\n@tailwind utilities;" }, { "path": "extensions/web-widget/src/js/script.js", "content": "const API_ENDPOINT = \"http://localhost:7091/api/answer\"; // Replace with your API endpoint\n\nconst widgetInitMessage = document.getElementById(\"docsgpt-init-message\");\nconst widgetAnswerMessage = document.getElementById(\"docsgpt-answer\");\nconst widgetAnswerMessageP = widgetAnswerMessage.querySelector(\"p\");\nconst askDocsGPTButton = document.getElementById(\"ask-docsgpt\");\nconst chatInput = document.getElementById(\"docsgpt-chat-input\");\nconst chatForm = document.getElementById(\"docsgpt-chat-form\");\nconst chatProcessing = document.getElementById(\"docsgpt-chat-processing\");\n\nasync function sendMessage(message) {\n const requestData = {\n \"question\": message,\n \"active_docs\": \"default\",\n \"api_key\": \"token\",\n \"embeddings_key\": \"token\",\n \"model\": \"default\",\n \"history\": null,\n }\n const response = await fetch(API_ENDPOINT, {\n method: \"POST\",\n headers: { \"Content-Type\": \"application/json\" },\n body: JSON.stringify(requestData),\n });\n const data = await response.json();\n return data.answer;\n}\n\naskDocsGPTButton.addEventListener(\"click\", () => {\n askDocsGPTButton.classList.add(\"hidden\");\n chatForm.classList.remove(\"hidden\");\n chatForm.focus();\n widgetInitMessage.classList.remove(\"hidden\");\n widgetAnswerMessage.classList.add(\"hidden\");\n\n\n});\n\nchatForm.addEventListener(\"submit\", async (e) => {\n e.preventDefault();\n const message = chatInput.value.trim();\n if (!message) return;\n\n chatInput.value = \"\";\n chatForm.classList.add(\"hidden\");\n chatProcessing.classList.remove(\"hidden\");\n\nconst reply = await sendMessage(message);\nchatProcessing.classList.add(\"hidden\");\n\n// inside

tag\nwidgetAnswerMessageP.innerHTML = reply;\nwidgetAnswerMessage.classList.remove(\"hidden\");\nwidgetInitMessage.classList.add(\"hidden\");\naskDocsGPTButton.classList.remove(\"hidden\");\n});" }, { "path": "extensions/web-widget/tailwind.config.js", "content": "/** @type {import('tailwindcss').Config} */\nmodule.exports = {\n content: [\"./src/**/*.{html,js}\"],\n theme: {\n extend: {},\n },\n plugins: [],\n}\n\n\n" }, { "path": "frontend/.husky/pre-commit", "content": "#!/usr/bin/env sh\n. \"$(dirname -- \"$0\")/_/husky.sh\"\n\n# npm test\ncd frontend\nnpx lint-staged" }, { "path": "frontend/.prettierignore", "content": "node_modules/\ndist/\nprettier.config.cjs\n.eslintrc.cjs\nenv.d.ts\npublic/\nassets/\nvite-env.d.ts\n.prettierignore\npackage-lock.json\npackage.json\npostcss.config.cjs\nprettier.config.cjs\ntailwind.config.cjs\ntsconfig.json\ntsconfig.node.json\nvite.config.ts" }, { "path": "frontend/Dockerfile", "content": "FROM node:22-bullseye-slim\n\n\nWORKDIR /app\nCOPY package*.json ./\nRUN npm install\nCOPY . .\n\nEXPOSE 5173\n\nCMD [ \"npm\", \"run\", \"dev\", \"--\" , \"--host\"]\n" }, { "path": "frontend/components.json", "content": "{\n \"$schema\": \"https://ui.shadcn.com/schema.json\",\n \"style\": \"new-york\",\n \"rsc\": false,\n \"tsx\": true,\n \"tailwind\": {\n \"config\": \"\",\n \"css\": \"src/index.css\",\n \"baseColor\": \"neutral\",\n \"cssVariables\": true,\n \"prefix\": \"\"\n },\n \"iconLibrary\": \"lucide\",\n \"aliases\": {\n \"components\": \"@/components\",\n \"utils\": \"@/lib/utils\",\n \"ui\": \"@/components/ui\",\n \"lib\": \"@/lib\",\n \"hooks\": \"@/hooks\"\n },\n \"registries\": {}\n}\n" }, { "path": "frontend/eslint.config.js", "content": "import js from '@eslint/js'\nimport tsParser from '@typescript-eslint/parser'\nimport tsPlugin from '@typescript-eslint/eslint-plugin'\nimport react from 'eslint-plugin-react'\nimport unusedImports from 'eslint-plugin-unused-imports'\nimport prettier from 'eslint-plugin-prettier'\nimport globals from 'globals'\n\nexport default [\n {\n ignores: [\n 'node_modules/',\n 'dist/',\n 'prettier.config.cjs',\n '.eslintrc.cjs',\n 'env.d.ts',\n 'public/',\n 'assets/',\n 'vite-env.d.ts',\n '.prettierignore',\n 'package-lock.json',\n 'package.json',\n 'postcss.config.cjs',\n 'tailwind.config.cjs',\n 'tsconfig.json',\n 'tsconfig.node.json',\n 'vite.config.ts',\n ],\n },\n {\n files: ['**/*.{js,jsx,ts,tsx}'],\n languageOptions: {\n ecmaVersion: 'latest',\n sourceType: 'module',\n parser: tsParser,\n parserOptions: {\n ecmaFeatures: {\n jsx: true,\n },\n },\n globals: {\n ...globals.browser,\n ...globals.es2021,\n ...globals.node,\n },\n },\n plugins: {\n '@typescript-eslint': tsPlugin,\n react,\n 'unused-imports': unusedImports,\n prettier,\n },\n rules: {\n ...js.configs.recommended.rules,\n ...tsPlugin.configs.recommended.rules,\n ...react.configs.recommended.rules,\n ...prettier.configs.recommended.rules,\n 'react/prop-types': 'off',\n 'unused-imports/no-unused-imports': 'error',\n 'react/react-in-jsx-scope': 'off',\n 'no-undef': 'off',\n '@typescript-eslint/no-explicit-any': 'warn',\n '@typescript-eslint/no-unused-vars': 'warn',\n '@typescript-eslint/no-unused-expressions': 'warn',\n 'prettier/prettier': [\n 'error',\n {\n endOfLine: 'auto',\n },\n ],\n },\n settings: {\n react: {\n version: 'detect',\n },\n },\n },\n]\n" }, { "path": "frontend/index.html", "content": "\n\n\n\n \n \n \n \n \n DocsGPT\n \n\n\n\n

\n \n\n\n" }, { "path": "frontend/package.json", "content": "{\n \"name\": \"frontend\",\n \"private\": true,\n \"version\": \"0.0.0\",\n \"type\": \"module\",\n \"scripts\": {\n \"dev\": \"vite\",\n \"build\": \"tsc && vite build\",\n \"preview\": \"vite preview\",\n \"lint\": \"eslint ./src --ext .jsx,.js,.ts,.tsx\",\n \"lint-fix\": \"eslint ./src --ext .jsx,.js,.ts,.tsx --fix\",\n \"format\": \"prettier ./src --write\",\n \"prepare\": \"cd .. && husky install frontend/.husky\"\n },\n \"lint-staged\": {\n \"**/*.{js,jsx,ts,tsx}\": [\n \"npm run lint-fix\",\n \"npm run format\"\n ]\n },\n \"dependencies\": {\n \"@radix-ui/react-dialog\": \"^1.1.15\",\n \"@radix-ui/react-popover\": \"^1.1.15\",\n \"@radix-ui/react-select\": \"^2.2.6\",\n \"@radix-ui/react-slot\": \"^1.2.4\",\n \"@reduxjs/toolkit\": \"^2.10.1\",\n \"chart.js\": \"^4.4.4\",\n \"class-variance-authority\": \"^0.7.1\",\n \"clsx\": \"^2.1.1\",\n \"cmdk\": \"^1.1.1\",\n \"copy-to-clipboard\": \"^3.3.3\",\n \"i18next\": \"^25.8.18\",\n \"i18next-browser-languagedetector\": \"^8.2.0\",\n \"lodash\": \"^4.17.21\",\n \"lucide-react\": \"^0.562.0\",\n \"mermaid\": \"^11.12.1\",\n \"prop-types\": \"^15.8.1\",\n \"radix-ui\": \"^1.4.3\",\n \"react\": \"^19.1.0\",\n \"react-chartjs-2\": \"^5.3.0\",\n \"react-dom\": \"^19.1.1\",\n \"react-dropzone\": \"^14.3.8\",\n \"react-google-drive-picker\": \"^1.2.2\",\n \"react-i18next\": \"^16.2.4\",\n \"react-markdown\": \"^9.0.1\",\n \"react-redux\": \"^9.2.0\",\n \"react-router-dom\": \"^7.6.1\",\n \"react-syntax-highlighter\": \"^16.1.1\",\n \"reactflow\": \"^11.11.4\",\n \"rehype-katex\": \"^7.0.1\",\n \"remark-gfm\": \"^4.0.0\",\n \"remark-math\": \"^6.0.0\",\n \"tailwind-merge\": \"^3.4.0\"\n },\n \"devDependencies\": {\n \"@tailwindcss/postcss\": \"^4.1.10\",\n \"@types/lodash\": \"^4.17.20\",\n \"@types/react\": \"^19.1.8\",\n \"@types/react-dom\": \"^19.1.7\",\n \"@types/react-syntax-highlighter\": \"^15.5.13\",\n \"@typescript-eslint/eslint-plugin\": \"^8.46.3\",\n \"@typescript-eslint/parser\": \"^8.46.3\",\n \"@vitejs/plugin-react\": \"^6.0.1\",\n \"eslint\": \"^9.39.1\",\n \"eslint-config-prettier\": \"^10.1.5\",\n \"eslint-plugin-import\": \"^2.31.0\",\n \"eslint-plugin-n\": \"^17.23.1\",\n \"eslint-plugin-prettier\": \"^5.5.4\",\n \"eslint-plugin-promise\": \"^6.6.0\",\n \"eslint-plugin-react\": \"^7.37.5\",\n \"eslint-plugin-unused-imports\": \"^4.1.4\",\n \"husky\": \"^9.1.7\",\n \"lint-staged\": \"^16.4.0\",\n \"postcss\": \"^8.4.49\",\n \"prettier\": \"^3.5.3\",\n \"prettier-plugin-tailwindcss\": \"^0.7.1\",\n \"tailwindcss\": \"^4.2.1\",\n \"tw-animate-css\": \"^1.4.0\",\n \"typescript\": \"^5.8.3\",\n \"vite\": \"^8.0.0\",\n \"vite-plugin-svgr\": \"^4.3.0\"\n }\n}\n" }, { "path": "frontend/postcss.config.cjs", "content": "module.exports = {\n plugins: {\n '@tailwindcss/postcss': {},\n },\n}\n" }, { "path": "frontend/prettier.config.cjs", "content": "module.exports = {\n trailingComma: 'all',\n tabWidth: 2,\n semi: true,\n singleQuote: true,\n printWidth: 80,\n plugins: ['prettier-plugin-tailwindcss'],\n};\n" }, { "path": "frontend/src/App.tsx", "content": "import './locale/i18n';\n\nimport { useState } from 'react';\nimport { Outlet, Route, Routes } from 'react-router-dom';\n\nimport Agents from './agents';\nimport SharedAgentGate from './agents/SharedAgentGate';\nimport ActionButtons from './components/ActionButtons';\nimport Spinner from './components/Spinner';\nimport UploadToast from './components/UploadToast';\nimport Conversation from './conversation/Conversation';\nimport { SharedConversation } from './conversation/SharedConversation';\nimport { useDarkTheme, useMediaQuery } from './hooks';\nimport useDataInitializer from './hooks/useDataInitializer';\nimport useTokenAuth from './hooks/useTokenAuth';\nimport Navigation from './Navigation';\nimport PageNotFound from './PageNotFound';\nimport Setting from './settings';\nimport Notification from './components/Notification';\n\nfunction AuthWrapper({ children }: { children: React.ReactNode }) {\n const { isAuthLoading } = useTokenAuth();\n useDataInitializer(isAuthLoading);\n\n if (isAuthLoading) {\n return (\n
\n \n
\n );\n }\n return <>{children};\n}\n\nfunction MainLayout() {\n const { isMobile, isTablet } = useMediaQuery();\n const [navOpen, setNavOpen] = useState(!(isMobile || isTablet));\n\n return (\n
\n \n \n \n \n
\n \n
\n );\n}\nexport default function App() {\n const [, , componentMounted] = useDarkTheme();\n const [showNotification, setShowNotification] = useState(() => {\n const saved = localStorage.getItem('showNotification');\n return saved ? JSON.parse(saved) : true;\n });\n const notificationText = import.meta.env.VITE_NOTIFICATION_TEXT;\n const notificationLink = import.meta.env.VITE_NOTIFICATION_LINK;\n if (!componentMounted) {\n return
;\n }\n return (\n
\n {notificationLink && notificationText && showNotification && (\n {\n setShowNotification(false);\n localStorage.setItem('showNotification', 'false');\n }}\n />\n )}\n \n \n \n \n }\n >\n } />\n } />\n } />\n \n } />\n } />\n } />\n \n
\n );\n}\n" }, { "path": "frontend/src/Hero.tsx", "content": "import { useTranslation } from 'react-i18next';\n\nimport DocsGPT3 from './assets/cute_docsgpt3.svg';\nimport DropdownModel from './components/DropdownModel';\n\nexport default function Hero({\n handleQuestion,\n}: {\n handleQuestion: ({\n question,\n isRetry,\n }: {\n question: string;\n isRetry?: boolean;\n }) => void;\n}) {\n const { t } = useTranslation();\n const demos = t('demo', { returnObjects: true }) as Array<{\n header: string;\n query: string;\n }>;\n\n return (\n
\n {/* Header Section */}\n
\n
\n DocsGPT\n \"docsgpt\"\n
\n {/* Model Selector */}\n
\n \n
\n
\n\n {/* Demo Buttons Section */}\n
\n
\n {demos?.map(\n (demo: { header: string; query: string }, key: number) =>\n demo.header &&\n demo.query && (\n handleQuestion({ question: demo.query })}\n className={`border-dark-gray text-just-black hover:bg-cultured dark:border-dim-gray dark:text-chinese-white dark:hover:bg-charleston-green w-full rounded-[66px] border bg-transparent px-6 py-[14px] text-left transition-colors ${key >= 2 ? 'hidden md:block' : ''}`}\n >\n

\n {demo.header}\n

\n \n {demo.query}\n \n \n ),\n )}\n
\n
\n
\n );\n}\n" }, { "path": "frontend/src/Navigation.tsx", "content": "import { useEffect, useRef, useState } from 'react';\nimport { useTranslation } from 'react-i18next';\nimport { useDispatch, useSelector } from 'react-redux';\nimport { NavLink, useNavigate } from 'react-router-dom';\n\nimport { Agent } from './agents/types';\nimport conversationService from './api/services/conversationService';\nimport userService from './api/services/userService';\nimport Add from './assets/add.svg';\nimport DocsGPT3 from './assets/cute_docsgpt3.svg';\nimport Discord from './assets/discord.svg';\nimport PanelLeftClose from './assets/panel-left-close.svg';\nimport PanelLeftOpen from './assets/panel-left-open.svg';\nimport Github from './assets/git_nav.svg';\nimport Hamburger from './assets/hamburger.svg';\nimport openNewChat from './assets/openNewChat.svg';\nimport Pin from './assets/pin.svg';\nimport AgentImage from './components/AgentImage';\nimport SettingGear from './assets/settingGear.svg';\nimport Spark from './assets/spark.svg';\nimport SpinnerDark from './assets/spinner-dark.svg';\nimport Spinner from './assets/spinner.svg';\nimport Twitter from './assets/TwitterX.svg';\nimport UnPin from './assets/unpin.svg';\nimport Help from './components/Help';\nimport {\n handleAbort,\n selectQueries,\n setConversation,\n updateConversationId,\n} from './conversation/conversationSlice';\nimport ConversationTile from './conversation/ConversationTile';\nimport { useDarkTheme, useMediaQuery } from './hooks';\nimport useTokenAuth from './hooks/useTokenAuth';\nimport DeleteConvModal from './modals/DeleteConvModal';\nimport JWTModal from './modals/JWTModal';\nimport { ActiveState } from './models/misc';\nimport { getConversations } from './preferences/preferenceApi';\nimport {\n selectAgents,\n selectConversationId,\n selectConversations,\n selectModalStateDeleteConv,\n selectSelectedAgent,\n selectSharedAgents,\n selectToken,\n setAgents,\n setConversations,\n setModalStateDeleteConv,\n setSelectedAgent,\n setSharedAgents,\n} from './preferences/preferenceSlice';\nimport Upload from './upload/Upload';\n\ninterface NavigationProps {\n navOpen: boolean;\n setNavOpen: React.Dispatch>;\n}\n\nexport default function Navigation({ navOpen, setNavOpen }: NavigationProps) {\n const dispatch = useDispatch();\n const navigate = useNavigate();\n\n const { t } = useTranslation();\n\n const token = useSelector(selectToken);\n const queries = useSelector(selectQueries);\n const conversations = useSelector(selectConversations);\n const conversationId = useSelector(selectConversationId);\n const modalStateDeleteConv = useSelector(selectModalStateDeleteConv);\n const agents = useSelector(selectAgents);\n const sharedAgents = useSelector(selectSharedAgents);\n const selectedAgent = useSelector(selectSelectedAgent);\n\n const { isMobile, isTablet } = useMediaQuery();\n const [isDarkTheme] = useDarkTheme();\n const { showTokenModal, handleTokenSubmit } = useTokenAuth();\n\n const [isDeletingConversation, setIsDeletingConversation] = useState(false);\n const [uploadModalState, setUploadModalState] =\n useState('INACTIVE');\n const [recentAgents, setRecentAgents] = useState([]);\n\n const navRef = useRef(null);\n useEffect(() => {\n function handleClickOutside(event: MouseEvent) {\n if (\n navRef.current &&\n !navRef.current.contains(event.target as Node) &&\n (isMobile || isTablet) &&\n navOpen\n ) {\n setNavOpen(false);\n }\n }\n\n //event listener only for mobile/tablet when nav is open\n if ((isMobile || isTablet) && navOpen) {\n document.addEventListener('mousedown', handleClickOutside);\n return () => {\n document.removeEventListener('mousedown', handleClickOutside);\n };\n }\n }, [navOpen, isMobile, isTablet, setNavOpen]);\n async function fetchRecentAgents() {\n try {\n const response = await userService.getPinnedAgents(token);\n if (!response.ok) throw new Error('Failed to fetch pinned agents');\n const pinnedAgents: Agent[] = await response.json();\n if (pinnedAgents.length >= 3) {\n setRecentAgents(pinnedAgents);\n return;\n }\n let tempAgents: Agent[] = [];\n if (!agents) {\n const response = await userService.getAgents(token);\n if (!response.ok) throw new Error('Failed to fetch agents');\n const data: Agent[] = await response.json();\n dispatch(setAgents(data));\n tempAgents = data;\n } else tempAgents = agents;\n const additionalAgents = tempAgents\n .filter(\n (agent: Agent) =>\n agent.status === 'published' &&\n !pinnedAgents.some((pinned) => pinned.id === agent.id),\n )\n .sort(\n (a: Agent, b: Agent) =>\n new Date(b.last_used_at ?? 0).getTime() -\n new Date(a.last_used_at ?? 0).getTime(),\n )\n .slice(0, 3 - pinnedAgents.length);\n setRecentAgents([...pinnedAgents, ...additionalAgents]);\n } catch (error) {\n console.error('Failed to fetch recent agents: ', error);\n }\n }\n\n async function fetchConversations() {\n dispatch(setConversations({ ...conversations, loading: true }));\n return await getConversations(token)\n .then((fetchedConversations) => {\n dispatch(setConversations(fetchedConversations));\n })\n .catch((error) => {\n console.error('Failed to fetch conversations: ', error);\n dispatch(setConversations({ data: null, loading: false }));\n });\n }\n\n useEffect(() => {\n fetchRecentAgents();\n }, [agents, sharedAgents, token, dispatch]);\n\n useEffect(() => {\n if (queries.length === 0) resetConversation();\n }, [conversations?.data, dispatch]);\n\n const handleDeleteAllConversations = () => {\n setIsDeletingConversation(true);\n conversationService\n .deleteAll(token)\n .then(() => {\n fetchConversations();\n })\n .catch((error) => console.error(error));\n };\n\n const handleDeleteConversation = (id: string) => {\n setIsDeletingConversation(true);\n conversationService\n .delete(id, {}, token)\n .then(() => {\n fetchConversations();\n resetConversation();\n })\n .catch((error) => console.error(error));\n };\n\n const handleAgentClick = (agent: Agent) => {\n resetConversation();\n dispatch(setSelectedAgent(agent));\n if (isMobile || isTablet) setNavOpen(!navOpen);\n navigate('/');\n };\n\n const handleTogglePin = (agent: Agent) => {\n userService.togglePinAgent(agent.id ?? '', token).then((response) => {\n if (response.ok) {\n const updatePinnedStatus = (a: Agent) =>\n a.id === agent.id ? { ...a, pinned: !a.pinned } : a;\n dispatch(setAgents(agents?.map(updatePinnedStatus)));\n dispatch(setSharedAgents(sharedAgents?.map(updatePinnedStatus)));\n }\n });\n };\n\n const handleConversationClick = async (index: string) => {\n try {\n dispatch(setSelectedAgent(null));\n\n const response = await conversationService.getConversation(index, token);\n if (!response.ok) {\n navigate('/');\n return;\n }\n\n const data = await response.json();\n if (!data) return;\n\n dispatch(setConversation(data.queries));\n dispatch(updateConversationId({ query: { conversationId: index } }));\n\n if (!data.agent_id) {\n navigate('/');\n return;\n }\n\n let agent: Agent;\n if (data.is_shared_usage) {\n const sharedResponse = await userService.getSharedAgent(\n data.shared_token,\n token,\n );\n if (!sharedResponse.ok) {\n navigate('/');\n return;\n }\n agent = await sharedResponse.json();\n navigate(`/agents/shared/${agent.shared_token}`);\n } else {\n const agentResponse = await userService.getAgent(data.agent_id, token);\n if (!agentResponse.ok) {\n navigate('/');\n return;\n }\n agent = await agentResponse.json();\n if (agent.shared_token) {\n navigate(`/agents/shared/${agent.shared_token}`);\n } else {\n await Promise.resolve(dispatch(setSelectedAgent(agent)));\n navigate('/');\n }\n }\n } catch (error) {\n console.error('Error handling conversation click:', error);\n navigate('/');\n }\n };\n\n const resetConversation = () => {\n handleAbort();\n dispatch(setConversation([]));\n dispatch(\n updateConversationId({\n query: { conversationId: null },\n }),\n );\n dispatch(setSelectedAgent(null));\n };\n\n const newChat = () => {\n if (queries && queries?.length > 0) {\n resetConversation();\n }\n };\n\n async function updateConversationName(updatedConversation: {\n name: string;\n id: string;\n }) {\n await conversationService\n .update(updatedConversation, token)\n .then((response) => response.json())\n .then((data) => {\n if (data) {\n navigate('/');\n fetchConversations();\n }\n })\n .catch((err) => {\n console.error(err);\n });\n }\n\n useEffect(() => {\n setNavOpen(!(isMobile || isTablet));\n }, [isMobile, isTablet]);\n\n return (\n <>\n {(isMobile || isTablet) && navOpen && (\n setNavOpen(false)}\n />\n )}\n\n {\n \n }\n \n \n {\n if (isMobile) {\n setNavOpen(!navOpen);\n }\n }}\n >\n \n \"DocsGPT\n

DocsGPT

\n \n
\n {\n setNavOpen(!navOpen);\n }}\n >\n \n \n
\n {\n if (isMobile || isTablet) {\n setNavOpen(!navOpen);\n }\n resetConversation();\n }}\n className={({ isActive }) =>\n `${\n isActive ? 'bg-transparent' : ''\n } group border-silver hover:border-rainy-gray dark:border-purple-taupe sticky mx-4 mt-4 flex cursor-pointer gap-2.5 rounded-3xl border p-3 hover:bg-transparent dark:text-white`\n }\n >\n \n

\n {t('newChat')}\n

\n \n \n {conversations?.loading && !isDeletingConversation && (\n
\n \n
\n )}\n {recentAgents?.length > 0 ? (\n
\n
\n

\n {t('navigation.agents')}\n

\n
\n
\n
\n {recentAgents.map((agent, idx) => (\n handleAgentClick(agent)}\n >\n
\n
\n \n
\n

\n {agent.name}\n

\n
\n \n {\n e.stopPropagation();\n handleTogglePin(agent);\n }}\n >\n \n \n
\n
\n ))}\n
\n {\n dispatch(setSelectedAgent(null));\n if (isMobile || isTablet) {\n setNavOpen(false);\n }\n navigate('/agents');\n }}\n >\n
\n \n
\n

\n {t('manageAgents')}\n

\n \n \n \n ) : (\n {\n if (isMobile || isTablet) {\n setNavOpen(false);\n }\n dispatch(setSelectedAgent(null));\n navigate('/agents');\n }}\n >\n
\n \n
\n

\n {t('manageAgents')}\n

\n \n )}\n {conversations?.data && conversations.data.length > 0 ? (\n
\n
\n

{t('chats')}

\n
\n
\n {conversations.data?.map((conversation) => (\n handleConversationClick(id)}\n onConversationClick={() => {\n if (isMobile) {\n setNavOpen(false);\n }\n }}\n onDeleteConversation={(id) => handleDeleteConversation(id)}\n onSave={(conversation) =>\n updateConversationName(conversation)\n }\n />\n ))}\n
\n
\n ) : (\n <>\n )}\n \n
\n
\n {\n if (isMobile || isTablet) {\n setNavOpen(false);\n }\n resetConversation();\n }}\n to=\"/settings\"\n className={({ isActive }) =>\n `mx-4 my-auto flex h-9 cursor-pointer items-center gap-4 rounded-3xl hover:bg-gray-100 dark:hover:bg-[#28292E] ${\n isActive ? 'bg-gray-3000 dark:bg-transparent' : ''\n }`\n }\n >\n \n

\n {t('settings.label')}\n

\n \n
\n
\n
\n \n\n
\n \n \n \n \n \n \n \n \n \n
\n
\n
\n
\n \n
\n
\n setNavOpen(true)}\n >\n \n \n
DocsGPT
\n
\n
\n \n {uploadModalState === 'ACTIVE' && (\n setUploadModalState('INACTIVE')}\n >\n )}\n \n \n );\n}\n" }, { "path": "frontend/src/PageNotFound.tsx", "content": "import { Link } from 'react-router-dom';\nimport { useTranslation } from 'react-i18next';\n\nexport default function PageNotFound() {\n const { t } = useTranslation();\n\n return (\n
\n

\n

{t('pageNotFound.title')}

\n

{t('pageNotFound.message')}

\n \n

\n
\n );\n}\n" }, { "path": "frontend/src/agents/AgentCard.tsx", "content": "import { SyntheticEvent, useRef, useState } from 'react';\nimport { useDispatch, useSelector } from 'react-redux';\nimport { useNavigate } from 'react-router-dom';\nimport { useTranslation } from 'react-i18next';\n\nimport userService from '../api/services/userService';\nimport Duplicate from '../assets/duplicate.svg';\nimport Edit from '../assets/edit.svg';\nimport FolderIcon from '../assets/folder.svg';\nimport Link from '../assets/link-gray.svg';\nimport Monitoring from '../assets/monitoring.svg';\nimport Pin from '../assets/pin.svg';\nimport Trash from '../assets/red-trash.svg';\nimport ThreeDots from '../assets/three-dots.svg';\nimport UnPin from '../assets/unpin.svg';\nimport AgentImage from '../components/AgentImage';\nimport ContextMenu, { MenuOption } from '../components/ContextMenu';\nimport ConfirmationModal from '../modals/ConfirmationModal';\nimport MoveToFolderModal from '../modals/MoveToFolderModal';\nimport { ActiveState } from '../models/misc';\nimport {\n selectAgents,\n selectToken,\n setAgents,\n setSelectedAgent,\n} from '../preferences/preferenceSlice';\nimport { Agent } from './types';\n\ntype AgentCardProps = {\n agent: Agent;\n agents: Agent[];\n updateAgents?: (agents: Agent[]) => void;\n section: string;\n};\n\nexport default function AgentCard({\n agent,\n agents,\n updateAgents,\n section,\n}: AgentCardProps) {\n const { t } = useTranslation();\n const navigate = useNavigate();\n const dispatch = useDispatch();\n const token = useSelector(selectToken);\n const userAgents = useSelector(selectAgents);\n\n const [isMenuOpen, setIsMenuOpen] = useState(false);\n const [deleteConfirmation, setDeleteConfirmation] =\n useState('INACTIVE');\n const [moveModalState, setMoveModalState] = useState('INACTIVE');\n\n const menuRef = useRef(null);\n\n const menuOptionsConfig: Record = {\n template: [\n {\n icon: Duplicate,\n label: 'Duplicate',\n onClick: (e: SyntheticEvent) => {\n e.stopPropagation();\n handleDuplicate();\n },\n variant: 'primary',\n iconWidth: 18,\n iconHeight: 18,\n },\n ],\n user: [\n {\n icon: Monitoring,\n label: 'Logs',\n onClick: (e: SyntheticEvent) => {\n e.stopPropagation();\n navigate(`/agents/logs/${agent.id}`);\n },\n variant: 'primary',\n iconWidth: 14,\n iconHeight: 14,\n },\n {\n icon: Edit,\n label: 'Edit',\n onClick: (e: SyntheticEvent) => {\n e.stopPropagation();\n if (agent.agent_type === 'workflow') {\n navigate(`/agents/workflow/edit/${agent.id}`);\n } else {\n navigate(`/agents/edit/${agent.id}`);\n }\n },\n variant: 'primary',\n iconWidth: 14,\n iconHeight: 14,\n },\n ...(agent.status === 'published'\n ? [\n {\n icon: agent.pinned ? UnPin : Pin,\n label: agent.pinned ? 'Unpin' : 'Pin agent',\n onClick: (e: SyntheticEvent) => {\n e.stopPropagation();\n togglePin();\n },\n variant: 'primary' as const,\n iconWidth: 18,\n iconHeight: 18,\n },\n ]\n : []),\n {\n icon: FolderIcon,\n label: t('agents.folders.moveToFolder'),\n onClick: (e: SyntheticEvent) => {\n e.stopPropagation();\n setMoveModalState('ACTIVE');\n setIsMenuOpen(false);\n },\n variant: 'primary',\n iconWidth: 16,\n iconHeight: 15,\n },\n {\n icon: Trash,\n label: 'Delete',\n onClick: (e: SyntheticEvent) => {\n e.stopPropagation();\n setDeleteConfirmation('ACTIVE');\n },\n variant: 'danger',\n iconWidth: 13,\n iconHeight: 13,\n },\n ],\n shared: [\n {\n icon: Link,\n label: 'Open',\n onClick: (e: SyntheticEvent) => {\n e.stopPropagation();\n navigate(`/agents/shared/${agent.shared_token}`);\n },\n variant: 'primary',\n iconWidth: 12,\n iconHeight: 12,\n },\n {\n icon: agent.pinned ? UnPin : Pin,\n label: agent.pinned ? 'Unpin' : 'Pin agent',\n onClick: (e: SyntheticEvent) => {\n e.stopPropagation();\n togglePin();\n },\n variant: 'primary',\n iconWidth: 18,\n iconHeight: 18,\n },\n {\n icon: Trash,\n label: 'Remove',\n onClick: (e: SyntheticEvent) => {\n e.stopPropagation();\n handleHideSharedAgent();\n },\n variant: 'danger',\n iconWidth: 13,\n iconHeight: 13,\n },\n ],\n };\n const menuOptions = menuOptionsConfig[section] || [];\n\n const handleClick = () => {\n if (section === 'user') {\n if (agent.status === 'published') {\n dispatch(setSelectedAgent(agent));\n navigate(`/`);\n }\n }\n if (section === 'shared') {\n navigate(`/agents/shared/${agent.shared_token}`);\n }\n };\n\n const togglePin = async () => {\n try {\n const response = await userService.togglePinAgent(agent.id ?? '', token);\n if (!response.ok) throw new Error('Failed to pin agent');\n const updatedAgents = agents.map((prevAgent) => {\n if (prevAgent.id === agent.id)\n return { ...prevAgent, pinned: !prevAgent.pinned };\n return prevAgent;\n });\n updateAgents?.(updatedAgents);\n } catch (error) {\n console.error('Error:', error);\n }\n };\n\n const handleHideSharedAgent = async () => {\n try {\n const response = await userService.removeSharedAgent(\n agent.id ?? '',\n token,\n );\n if (!response.ok) throw new Error('Failed to hide shared agent');\n const updatedAgents = agents.filter(\n (prevAgent) => prevAgent.id !== agent.id,\n );\n updateAgents?.(updatedAgents);\n } catch (error) {\n console.error('Error:', error);\n }\n };\n\n const handleDelete = async () => {\n try {\n const response = await userService.deleteAgent(agent.id ?? '', token);\n if (!response.ok) throw new Error('Failed to delete agent');\n const updatedAgents = agents.filter(\n (prevAgent) => prevAgent.id !== agent.id,\n );\n updateAgents?.(updatedAgents);\n } catch (error) {\n console.error('Error:', error);\n }\n };\n\n const handleDuplicate = async () => {\n try {\n const response = await userService.adoptAgent(agent.id ?? '', token);\n if (!response.ok) throw new Error('Failed to duplicate agent');\n const data = await response.json();\n if (userAgents) {\n const updatedAgents = [...userAgents, data.agent];\n dispatch(setAgents(updatedAgents));\n } else dispatch(setAgents([data.agent]));\n } catch (error) {\n console.error('Error:', error);\n }\n };\n\n const handleMoveSuccess = (folderId: string | null) => {\n const updatedAgents = agents.map((prevAgent) => {\n if (prevAgent.id === agent.id) {\n return { ...prevAgent, folder_id: folderId ?? undefined };\n }\n return prevAgent;\n });\n updateAgents?.(updatedAgents);\n };\n return (\n {\n e.stopPropagation();\n handleClick();\n }}\n >\n {\n e.stopPropagation();\n setIsMenuOpen(true);\n }}\n className=\"absolute top-4 right-4 z-10 cursor-pointer\"\n >\n {'use-agent'}\n \n \n
\n
\n \n {agent.status === 'draft' && (\n

{`(Draft)`}

\n )}\n
\n
\n \n {agent.name}\n

\n

\n {agent.description}\n

\n
\n
\n {\n handleDelete();\n setDeleteConfirmation('INACTIVE');\n }}\n cancelLabel=\"Cancel\"\n variant=\"danger\"\n />\n \n \n );\n}" }, { "path": "frontend/src/agents/AgentLogs.tsx", "content": "import { useEffect, useState } from 'react';\nimport { useTranslation } from 'react-i18next';\nimport { useSelector } from 'react-redux';\nimport { useNavigate, useParams } from 'react-router-dom';\n\nimport userService from '../api/services/userService';\nimport ArrowLeft from '../assets/arrow-left.svg';\nimport Spinner from '../components/Spinner';\nimport { selectToken } from '../preferences/preferenceSlice';\nimport Analytics from '../settings/Analytics';\nimport Logs from '../settings/Logs';\nimport { Agent } from './types';\n\nexport default function AgentLogs() {\n const { t } = useTranslation();\n const navigate = useNavigate();\n const { agentId } = useParams();\n const token = useSelector(selectToken);\n\n const [agent, setAgent] = useState();\n const [loadingAgent, setLoadingAgent] = useState(true);\n\n const fetchAgent = async (agentId: string) => {\n setLoadingAgent(true);\n try {\n const response = await userService.getAgent(agentId ?? '', token);\n if (!response.ok) throw new Error('Failed to fetch Chatbots');\n const agent = await response.json();\n setAgent(agent);\n } catch (error) {\n console.error(error);\n } finally {\n setLoadingAgent(false);\n }\n };\n\n useEffect(() => {\n if (agentId) fetchAgent(agentId);\n }, [agentId, token]);\n return (\n
\n
\n navigate('/agents')}\n >\n \"left-arrow\"\n \n

\n {t('agents.backToAll')}\n

\n
\n
\n

\n {t('agents.logs.title')}\n

\n
\n
\n {agent && (\n
\n

{agent.name}

\n

\n {agent.last_used_at\n ? t('agents.logs.lastUsedAt') +\n ' ' +\n new Date(agent.last_used_at).toLocaleString()\n : t('agents.logs.noUsageHistory')}\n

\n
\n )}\n
\n {loadingAgent ? (\n
\n \n
\n ) : (\n agent && \n )}\n {loadingAgent ? (\n
\n {' '}\n \n
\n ) : (\n agent && (\n \n )\n )}\n
\n );\n}\n" }, { "path": "frontend/src/agents/AgentPreview.tsx", "content": "import { useCallback, useEffect, useRef, useState } from 'react';\nimport { useTranslation } from 'react-i18next';\nimport { useDispatch, useSelector } from 'react-redux';\n\nimport MessageInput from '../components/MessageInput';\nimport ConversationMessages from '../conversation/ConversationMessages';\nimport { Query } from '../conversation/conversationModels';\nimport { selectSelectedAgent } from '../preferences/preferenceSlice';\nimport { AppDispatch } from '../store';\nimport {\n addQuery,\n fetchPreviewAnswer,\n handlePreviewAbort,\n resendQuery,\n resetPreview,\n selectPreviewQueries,\n selectPreviewStatus,\n} from './agentPreviewSlice';\n\nexport default function AgentPreview() {\n const { t } = useTranslation();\n const dispatch = useDispatch();\n\n const queries = useSelector(selectPreviewQueries);\n const status = useSelector(selectPreviewStatus);\n const selectedAgent = useSelector(selectSelectedAgent);\n\n const [lastQueryReturnedErr, setLastQueryReturnedErr] = useState(false);\n\n const fetchStream = useRef(null);\n\n const handleFetchAnswer = useCallback(\n ({ question, index }: { question: string; index?: number }) => {\n fetchStream.current = dispatch(\n fetchPreviewAnswer({ question, indx: index }),\n );\n },\n [dispatch],\n );\n\n const handleQuestion = useCallback(\n ({\n question,\n isRetry = false,\n index = undefined,\n }: {\n question: string;\n isRetry?: boolean;\n index?: number;\n }) => {\n const trimmedQuestion = question.trim();\n if (trimmedQuestion === '') return;\n\n if (index !== undefined) {\n if (!isRetry) dispatch(resendQuery({ index, prompt: trimmedQuestion }));\n handleFetchAnswer({ question: trimmedQuestion, index });\n } else {\n if (!isRetry) {\n const newQuery: Query = { prompt: trimmedQuestion };\n dispatch(addQuery(newQuery));\n }\n handleFetchAnswer({ question: trimmedQuestion, index: undefined });\n }\n },\n [dispatch, handleFetchAnswer],\n );\n\n const handleQuestionSubmission = (\n question?: string,\n updated?: boolean,\n indx?: number,\n ) => {\n if (updated === true && question !== undefined && indx !== undefined) {\n handleQuestion({\n question,\n index: indx,\n isRetry: false,\n });\n } else if (question && status !== 'loading') {\n const currentInput = question.trim();\n if (lastQueryReturnedErr && queries.length > 0) {\n const lastQueryIndex = queries.length - 1;\n handleQuestion({\n question: currentInput,\n isRetry: true,\n index: lastQueryIndex,\n });\n } else {\n handleQuestion({\n question: currentInput,\n isRetry: false,\n index: undefined,\n });\n }\n }\n };\n\n useEffect(() => {\n dispatch(resetPreview());\n return () => {\n if (fetchStream.current) fetchStream.current.abort();\n handlePreviewAbort();\n dispatch(resetPreview());\n };\n }, [dispatch]);\n\n useEffect(() => {\n if (queries.length > 0) {\n const lastQuery = queries[queries.length - 1];\n setLastQueryReturnedErr(!!lastQuery.error);\n } else setLastQueryReturnedErr(false);\n }, [queries]);\n return (\n
\n
div>div]:w-full! [&>div>div]:max-w-none!\">\n \n
\n
\n
\n handleQuestionSubmission(text)}\n loading={status === 'loading'}\n showSourceButton={selectedAgent ? false : true}\n showToolButton={selectedAgent ? false : true}\n autoFocus={false}\n />\n
\n

\n {t('agents.preview.testMessage')}\n

\n
\n
\n );\n}\n" }, { "path": "frontend/src/agents/AgentsList.tsx", "content": "import { useCallback, useEffect, useMemo, useRef, useState } from 'react';\nimport { useTranslation } from 'react-i18next';\nimport { useDispatch, useSelector } from 'react-redux';\nimport { useNavigate, useSearchParams } from 'react-router-dom';\n\nimport userService from '../api/services/userService';\nimport Search from '../assets/search.svg';\nimport Spinner from '../components/Spinner';\nimport {\n setConversation,\n updateConversationId,\n} from '../conversation/conversationSlice';\nimport {\n selectAgentFolders,\n selectSelectedAgent,\n selectToken,\n setAgentFolders,\n setSelectedAgent,\n} from '../preferences/preferenceSlice';\nimport AgentCard from './AgentCard';\nimport { AgentSectionId, agentSectionsConfig } from './agents.config';\nimport AgentTypeModal from './components/AgentTypeModal';\nimport FolderCard from './FolderCard';\nimport { AgentFilterTab, useAgentSearch } from './hooks/useAgentSearch';\nimport { useAgentsFetch } from './hooks/useAgentsFetch';\nimport { Agent, AgentFolder } from './types';\n\nconst FILTER_TABS: { id: AgentFilterTab; labelKey: string }[] = [\n { id: 'all', labelKey: 'agents.filters.all' },\n { id: 'template', labelKey: 'agents.filters.byDocsGPT' },\n { id: 'user', labelKey: 'agents.filters.byMe' },\n { id: 'shared', labelKey: 'agents.filters.shared' },\n];\n\nexport default function AgentsList() {\n const { t } = useTranslation();\n const dispatch = useDispatch();\n const navigate = useNavigate();\n const [searchParams] = useSearchParams();\n const token = useSelector(selectToken);\n const selectedAgent = useSelector(selectSelectedAgent);\n const folders = useSelector(selectAgentFolders);\n const [folderPath, setFolderPath] = useState(() => {\n const folderIdFromUrl = searchParams.get('folder');\n return folderIdFromUrl ? [folderIdFromUrl] : [];\n });\n const [showAgentTypeModal, setShowAgentTypeModal] = useState(false);\n const [modalFolderId, setModalFolderId] = useState(null);\n\n // Sync folder path with URL\n useEffect(() => {\n const currentFolderInUrl = searchParams.get('folder');\n const currentFolderId =\n folderPath.length > 0 ? folderPath[folderPath.length - 1] : null;\n\n if (currentFolderId !== currentFolderInUrl) {\n const newUrl = currentFolderId\n ? `/agents?folder=${currentFolderId}`\n : '/agents';\n navigate(newUrl, { replace: true });\n }\n }, [folderPath, searchParams, navigate]);\n\n const { isLoading, refetchFolders, refetchUserAgents } = useAgentsFetch();\n\n const {\n searchQuery,\n setSearchQuery,\n activeFilter,\n setActiveFilter,\n filteredAgentsBySection,\n totalAgentsBySection,\n hasAnyAgents,\n hasFilteredResults,\n isDataLoaded,\n } = useAgentSearch();\n\n useEffect(() => {\n dispatch(setConversation([]));\n dispatch(\n updateConversationId({\n query: { conversationId: null },\n }),\n );\n if (selectedAgent) dispatch(setSelectedAgent(null));\n }, []);\n\n const handleCreateFolder = useCallback(\n async (name: string, parentId?: string) => {\n const response = await userService.createAgentFolder(\n { name, parent_id: parentId },\n token,\n );\n if (response.ok) {\n await refetchFolders();\n return true;\n }\n return false;\n },\n [token, refetchFolders],\n );\n\n const handleDeleteFolder = useCallback(\n async (folderId: string) => {\n const response = await userService.deleteAgentFolder(folderId, token);\n if (response.ok) {\n await Promise.all([refetchFolders(), refetchUserAgents()]);\n return true;\n }\n return false;\n },\n [token, refetchFolders, refetchUserAgents],\n );\n\n const handleRenameFolder = useCallback(\n async (folderId: string, newName: string) => {\n const response = await userService.updateAgentFolder(\n folderId,\n { name: newName },\n token,\n );\n if (response.ok) {\n dispatch(\n setAgentFolders(\n (folders || []).map((f) =>\n f.id === folderId ? { ...f, name: newName } : f,\n ),\n ),\n );\n }\n },\n [token, folders, dispatch],\n );\n\n const handleSubmitNewFolder = async (name: string, parentId?: string) => {\n await handleCreateFolder(name, parentId);\n };\n\n const visibleSections = agentSectionsConfig.filter((config) => {\n if (activeFilter !== 'all') {\n return config.id === activeFilter;\n }\n const sectionId = config.id as AgentSectionId;\n const hasAgentsInSection = totalAgentsBySection[sectionId] > 0;\n const hasFilteredAgents = filteredAgentsBySection[sectionId].length > 0;\n const sectionDataLoaded = isDataLoaded[sectionId];\n\n if (!sectionDataLoaded) return true;\n if (searchQuery) return hasFilteredAgents;\n if (config.id === 'user') return true;\n return hasAgentsInSection;\n });\n\n const showSearchEmptyState =\n searchQuery &&\n hasAnyAgents &&\n !hasFilteredResults &&\n activeFilter === 'all';\n\n return (\n
\n

\n {t('agents.title')}\n

\n

\n {t('agents.description')}\n

\n\n
\n
\n \n setSearchQuery(e.target.value)}\n placeholder={t('agents.searchPlaceholder')}\n className=\"h-11 w-full rounded-full border border-[#E5E5E5] bg-white py-2 pr-5 pl-11 text-sm shadow-[0_1px_4px_rgba(0,0,0,0.06)] transition-shadow outline-none placeholder:text-[#9CA3AF] focus:shadow-[0_2px_8px_rgba(0,0,0,0.1)] dark:border-[#3A3A3A] dark:bg-[#2C2C2C] dark:text-white dark:shadow-none dark:placeholder:text-[#6B7280]\"\n />\n
\n\n
\n {FILTER_TABS.map((tab) => (\n setActiveFilter(tab.id)}\n className={`rounded-full px-4 py-2 text-sm transition-colors ${\n activeFilter === tab.id\n ? 'bg-[#E0E0E0] text-[#18181B] dark:bg-[#4A4A4A] dark:text-white'\n : 'dark:text-gray bg-transparent text-[#71717A] hover:bg-[#F5F5F5] dark:hover:bg-[#383838]/50'\n }`}\n >\n {t(tab.labelKey)}\n \n ))}\n
\n
\n\n {visibleSections.map((sectionConfig) => (\n \n ))}\n\n {showSearchEmptyState && (\n
\n

{t('agents.noSearchResults')}

\n

{t('agents.tryDifferentSearch')}

\n
\n )}\n\n setShowAgentTypeModal(false)}\n folderId={modalFolderId}\n />\n
\n );\n}\n\ninterface AgentSectionProps {\n config: (typeof agentSectionsConfig)[number];\n filteredAgents: Agent[];\n totalAgents: number;\n searchQuery: string;\n isFilteredView: boolean;\n isLoading: boolean;\n folders: AgentFolder[] | null;\n folderPath: string[];\n onFolderPathChange?: (path: string[]) => void;\n onCreateFolder: (name: string, parentId?: string) => void;\n onDeleteFolder: (id: string) => Promise;\n onRenameFolder: (id: string, name: string) => void;\n setModalFolderId: (folderId: string | null) => void;\n setShowAgentTypeModal: (show: boolean) => void;\n}\n\nfunction AgentSection({\n config,\n filteredAgents,\n totalAgents,\n searchQuery,\n isFilteredView,\n isLoading,\n folders,\n folderPath,\n onFolderPathChange,\n onCreateFolder,\n onDeleteFolder,\n onRenameFolder,\n setModalFolderId,\n setShowAgentTypeModal,\n}: AgentSectionProps) {\n const { t } = useTranslation();\n const navigate = useNavigate();\n const dispatch = useDispatch();\n const allAgents = useSelector(config.selectData);\n const [isCreatingFolder, setIsCreatingFolder] = useState(false);\n const [newFolderName, setNewFolderName] = useState('');\n const newFolderInputRef = useRef(null);\n\n const currentFolderId =\n folderPath.length > 0 ? folderPath[folderPath.length - 1] : null;\n\n const setFolderPath = useCallback(\n (updater: string[] | ((prev: string[]) => string[])) => {\n if (!onFolderPathChange) return;\n if (typeof updater === 'function') {\n onFolderPathChange(updater(folderPath));\n } else {\n onFolderPathChange(updater);\n }\n },\n [onFolderPathChange, folderPath],\n );\n\n const updateAgents = (updatedAgents: Agent[]) => {\n dispatch(config.updateAction(updatedAgents));\n };\n\n const currentFolderDescendantIds = useMemo(() => {\n if (config.id !== 'user' || !folders || currentFolderId === null)\n return null;\n\n const getDescendants = (folderId: string): string[] => {\n const children = folders.filter((f) => f.parent_id === folderId);\n return children.flatMap((child) => [\n child.id,\n ...getDescendants(child.id),\n ]);\n };\n\n return new Set([currentFolderId, ...getDescendants(currentFolderId)]);\n }, [folders, currentFolderId, config.id]);\n\n const folderHasMatchingAgents = useCallback(\n (folderId: string): boolean => {\n const directMatches = filteredAgents.some(\n (a) => a.folder_id === folderId,\n );\n if (directMatches) return true;\n const childFolders = (folders || []).filter(\n (f) => f.parent_id === folderId,\n );\n return childFolders.some((f) => folderHasMatchingAgents(f.id));\n },\n [filteredAgents, folders],\n );\n\n // Get folders at the current level (root or inside current folder)\n const currentLevelFolders = useMemo(() => {\n if (config.id !== 'user' || !folders) return [];\n const foldersAtLevel = folders.filter(\n (f) => (f.parent_id || null) === currentFolderId,\n );\n if (searchQuery) {\n return foldersAtLevel.filter((f) => folderHasMatchingAgents(f.id));\n }\n return foldersAtLevel;\n }, [\n folders,\n currentFolderId,\n config.id,\n searchQuery,\n folderHasMatchingAgents,\n ]);\n\n const unfolderedAgents = useMemo(() => {\n if (config.id !== 'user' || !folders) return filteredAgents;\n\n if (searchQuery) {\n // When searching at root: return ALL filtered agents\n if (currentFolderId === null) {\n return filteredAgents;\n }\n // When searching inside a folder: return agents in current folder OR any descendant\n return filteredAgents.filter(\n (a) => currentFolderDescendantIds?.has(a.folder_id ?? '') ?? false,\n );\n }\n\n // No search: show agents that belong to the current folder level only\n return filteredAgents.filter(\n (a) => (a.folder_id || null) === currentFolderId,\n );\n }, [\n filteredAgents,\n folders,\n config.id,\n currentFolderId,\n searchQuery,\n currentFolderDescendantIds,\n ]);\n\n const getAgentsForFolder = (folderId: string) => {\n return filteredAgents.filter((a) => a.folder_id === folderId);\n };\n\n const handleNavigateIntoFolder = (folderId: string) => {\n setFolderPath((prev) => [...prev, folderId]);\n };\n\n const handleNavigateToPath = (index: number) => {\n if (index < 0) {\n setFolderPath([]);\n } else {\n setFolderPath((prev) => prev.slice(0, index + 1));\n }\n };\n\n const handleSubmitNewFolder = (name: string) => {\n onCreateFolder(name, currentFolderId || undefined);\n };\n\n const hasNoAgentsAtAll = !isLoading && totalAgents === 0;\n const isSearchingWithNoResults =\n !isLoading && searchQuery && filteredAgents.length === 0 && totalAgents > 0;\n\n if (isFilteredView && isSearchingWithNoResults) {\n return (\n
\n

{t('agents.noSearchResults')}

\n

{t('agents.tryDifferentSearch')}

\n
\n );\n }\n\n if (isFilteredView && hasNoAgentsAtAll) {\n return (\n
\n

{t(`agents.sections.${config.id}.emptyState`)}

\n {config.showNewAgentButton && (\n {\n setModalFolderId(null);\n setShowAgentTypeModal(true);\n }}\n >\n {t('agents.newAgent')}\n \n )}\n
\n );\n }\n\n // Build breadcrumb items from folder path\n const breadcrumbItems = useMemo(() => {\n if (!folders || folderPath.length === 0) return [];\n return folderPath.map((folderId) => {\n const folder = folders.find((f) => f.id === folderId);\n return { id: folderId, name: folder?.name || '' };\n });\n }, [folders, folderPath]);\n\n const ChevronIcon = () => (\n \n \n \n );\n\n return (\n
\n
\n
\n

\n {config.id === 'user' && folderPath.length > 0 ? (\n <>\n handleNavigateToPath(-1)}\n className=\"text-[#71717A] hover:text-[#18181B] dark:hover:text-white\"\n >\n {t(`agents.sections.${config.id}.title`)}\n \n {breadcrumbItems.map((item, index) => (\n \n \n {index === breadcrumbItems.length - 1 ? (\n {item.name}\n ) : (\n handleNavigateToPath(index)}\n className=\"text-[#71717A] hover:text-[#18181B] dark:hover:text-white\"\n >\n {item.name}\n \n )}\n \n ))}\n \n ) : (\n t(`agents.sections.${config.id}.title`)\n )}\n

\n

\n {t(`agents.sections.${config.id}.description`)}\n

\n
\n
\n {config.id === 'user' &&\n (isCreatingFolder ? (\n setNewFolderName(e.target.value)}\n onKeyDown={(e) => {\n if (e.key === 'Enter' && newFolderName.trim()) {\n handleSubmitNewFolder(newFolderName.trim());\n setNewFolderName('');\n setIsCreatingFolder(false);\n } else if (e.key === 'Escape') {\n setNewFolderName('');\n setIsCreatingFolder(false);\n }\n }}\n onBlur={() => {\n if (!newFolderName.trim()) {\n setIsCreatingFolder(false);\n }\n }}\n placeholder={t('agents.folders.newFolder')}\n className=\"w-28 rounded-full border border-[#E5E5E5] bg-white px-4 py-2 text-sm text-[#18181B] outline-none placeholder:text-[#9CA3AF] sm:w-auto dark:border-[#3A3A3A] dark:bg-[#2C2C2C] dark:text-white dark:placeholder:text-[#6B7280]\"\n autoFocus\n />\n ) : (\n {\n setIsCreatingFolder(true);\n setTimeout(() => newFolderInputRef.current?.focus(), 0);\n }}\n >\n {t('agents.folders.newFolder')}\n \n ))}\n {config.showNewAgentButton && (\n {\n setModalFolderId(currentFolderId);\n setShowAgentTypeModal(true);\n }}\n >\n {t('agents.newAgent')}\n \n )}\n
\n
\n\n
\n {isLoading ? (\n
\n \n
\n ) : (\n <>\n {/* Show subfolders at current level */}\n {config.id === 'user' && currentLevelFolders.length > 0 && (\n
\n {currentLevelFolders.map((folder) => (\n \n ))}\n
\n )}\n\n {/* Show agents at current level */}\n {unfolderedAgents.length > 0 ? (\n
\n {unfolderedAgents.map((agent) => (\n \n ))}\n
\n ) : hasNoAgentsAtAll && currentLevelFolders.length === 0 ? (\n
\n

\n {currentFolderId\n ? t('agents.folders.empty')\n : t(`agents.sections.${config.id}.emptyState`)}\n

\n {config.showNewAgentButton && !currentFolderId && (\n {\n setModalFolderId(currentFolderId);\n setShowAgentTypeModal(true);\n }}\n >\n {t('agents.newAgent')}\n \n )}\n
\n ) : null}\n \n )}\n
\n
\n );\n}" }, { "path": "frontend/src/agents/FolderCard.tsx", "content": "import { SyntheticEvent, useRef, useState } from 'react';\nimport { useTranslation } from 'react-i18next';\n\nimport Edit from '../assets/edit.svg';\nimport Trash from '../assets/red-trash.svg';\nimport ThreeDots from '../assets/three-dots.svg';\nimport ContextMenu, { MenuOption } from '../components/ContextMenu';\nimport ConfirmationModal from '../modals/ConfirmationModal';\nimport FolderNameModal from '../modals/FolderManagementModal';\nimport { ActiveState } from '../models/misc';\nimport { AgentFolder } from './types';\n\ntype FolderCardProps = {\n folder: AgentFolder;\n agentCount: number;\n onDelete: (folderId: string) => Promise;\n onRename: (folderId: string, newName: string) => void;\n isExpanded: boolean;\n onToggleExpand: (folderId: string) => void;\n};\n\nexport default function FolderCard({\n folder,\n agentCount,\n onDelete,\n onRename,\n isExpanded,\n onToggleExpand,\n}: FolderCardProps) {\n const { t } = useTranslation();\n const [isMenuOpen, setIsMenuOpen] = useState(false);\n const [deleteConfirmation, setDeleteConfirmation] =\n useState('INACTIVE');\n const [renameModalState, setRenameModalState] =\n useState('INACTIVE');\n const menuRef = useRef(null);\n\n const menuOptions: MenuOption[] = [\n {\n icon: Edit,\n label: t('agents.folders.rename'),\n onClick: (e: SyntheticEvent) => {\n e.stopPropagation();\n setRenameModalState('ACTIVE');\n setIsMenuOpen(false);\n },\n variant: 'primary',\n iconWidth: 14,\n iconHeight: 14,\n },\n {\n icon: Trash,\n label: t('agents.folders.delete'),\n onClick: (e: SyntheticEvent) => {\n e.stopPropagation();\n setDeleteConfirmation('ACTIVE');\n setIsMenuOpen(false);\n },\n variant: 'danger',\n iconWidth: 13,\n iconHeight: 13,\n },\n ];\n\n const handleRename = (newName: string) => {\n onRename(folder.id, newName);\n };\n\n return (\n <>\n onToggleExpand(folder.id)}\n >\n
\n \n {folder.name}\n \n \n ({agentCount})\n \n
\n {\n e.stopPropagation();\n setIsMenuOpen(true);\n }}\n className=\"ml-2 shrink-0 cursor-pointer\"\n >\n \"menu\"\n \n \n \n {\n onDelete(folder.id);\n setDeleteConfirmation('INACTIVE');\n }}\n cancelLabel={t('cancel')}\n variant=\"danger\"\n />\n \n \n );\n}\n" }, { "path": "frontend/src/agents/NewAgent.tsx", "content": "import isEqual from 'lodash/isEqual';\nimport React, { useCallback, useEffect, useRef, useState } from 'react';\nimport { useTranslation } from 'react-i18next';\nimport { useDispatch, useSelector } from 'react-redux';\nimport { useNavigate, useParams, useSearchParams } from 'react-router-dom';\n\nimport modelService from '../api/services/modelService';\nimport userService from '../api/services/userService';\nimport ArrowLeft from '../assets/arrow-left.svg';\nimport SourceIcon from '../assets/source.svg';\nimport Dropdown from '../components/Dropdown';\nimport { FileUpload } from '../components/FileUpload';\nimport MultiSelectPopup, { OptionType } from '../components/MultiSelectPopup';\nimport Spinner from '../components/Spinner';\nimport AgentDetailsModal from '../modals/AgentDetailsModal';\nimport ConfirmationModal from '../modals/ConfirmationModal';\nimport { ActiveState, Doc, Prompt } from '../models/misc';\nimport {\n selectAgentFolders,\n selectSelectedAgent,\n selectSourceDocs,\n selectToken,\n selectPrompts,\n setAgentFolders,\n setSelectedAgent,\n setPrompts,\n} from '../preferences/preferenceSlice';\nimport PromptsModal from '../preferences/PromptsModal';\nimport Prompts from '../settings/Prompts';\nimport { UserToolType } from '../settings/types';\nimport AgentPreview from './AgentPreview';\nimport { Agent, ToolSummary } from './types';\nimport WorkflowBuilder from './workflow/WorkflowBuilder';\n\nimport type { Model } from '../models/types';\n\nexport default function NewAgent({ mode }: { mode: 'new' | 'edit' | 'draft' }) {\n const { t } = useTranslation();\n const navigate = useNavigate();\n const dispatch = useDispatch();\n const { agentId } = useParams();\n\n const [searchParams] = useSearchParams();\n const folderIdFromUrl = searchParams.get('folder_id');\n\n const token = useSelector(selectToken);\n const sourceDocs = useSelector(selectSourceDocs);\n const selectedAgent = useSelector(selectSelectedAgent);\n const prompts = useSelector(selectPrompts);\n const agentFolders = useSelector(selectAgentFolders);\n\n const [validatedFolderId, setValidatedFolderId] = useState(\n null,\n );\n\n const [effectiveMode, setEffectiveMode] = useState(mode);\n const [agent, setAgent] = useState({\n id: agentId || '',\n name: '',\n description: '',\n image: '',\n source: '',\n sources: [],\n chunks: '2',\n retriever: 'classic',\n prompt_id: 'default',\n tools: [],\n agent_type: 'classic',\n status: '',\n json_schema: undefined,\n limited_token_mode: false,\n token_limit: undefined,\n limited_request_mode: false,\n request_limit: undefined,\n models: [],\n default_model_id: '',\n });\n const [imageFile, setImageFile] = useState(null);\n const [userTools, setUserTools] = useState([]);\n const [availableModels, setAvailableModels] = useState([]);\n const [isSourcePopupOpen, setIsSourcePopupOpen] = useState(false);\n const [isToolsPopupOpen, setIsToolsPopupOpen] = useState(false);\n const [isModelsPopupOpen, setIsModelsPopupOpen] = useState(false);\n const [selectedSourceIds, setSelectedSourceIds] = useState<\n Set\n >(new Set());\n const [selectedTools, setSelectedTools] = useState([]);\n const [selectedModelIds, setSelectedModelIds] = useState>(\n new Set(),\n );\n const [deleteConfirmation, setDeleteConfirmation] =\n useState('INACTIVE');\n const [agentDetails, setAgentDetails] = useState('INACTIVE');\n const [addPromptModal, setAddPromptModal] = useState('INACTIVE');\n const [hasChanges, setHasChanges] = useState(false);\n const [draftLoading, setDraftLoading] = useState(false);\n const [publishLoading, setPublishLoading] = useState(false);\n const [jsonSchemaText, setJsonSchemaText] = useState('');\n const [jsonSchemaValid, setJsonSchemaValid] = useState(true);\n const [isAdvancedSectionExpanded, setIsAdvancedSectionExpanded] =\n useState(false);\n\n const initialAgentRef = useRef(null);\n const sourceAnchorButtonRef = useRef(null);\n const toolAnchorButtonRef = useRef(null);\n const modelAnchorButtonRef = useRef(null);\n\n const modeConfig = {\n new: {\n heading: t('agents.form.headings.new'),\n buttonText: t('agents.form.buttons.publish'),\n showDelete: false,\n showSaveDraft: true,\n showLogs: false,\n showAccessDetails: false,\n trackChanges: false,\n },\n edit: {\n heading: t('agents.form.headings.edit'),\n buttonText: t('agents.form.buttons.save'),\n showDelete: true,\n showSaveDraft: false,\n showLogs: true,\n showAccessDetails: true,\n trackChanges: true,\n },\n draft: {\n heading: t('agents.form.headings.draft'),\n buttonText: t('agents.form.buttons.publish'),\n showDelete: true,\n showSaveDraft: true,\n showLogs: false,\n showAccessDetails: false,\n trackChanges: false,\n },\n };\n const chunks = ['0', '2', '4', '6', '8', '10'];\n const agentTypes = [\n { label: t('agents.form.agentTypes.classic'), value: 'classic' },\n { label: t('agents.form.agentTypes.react'), value: 'react' },\n ];\n\n const isPublishable = () => {\n const hasRequiredFields =\n agent.name && agent.description && agent.prompt_id && agent.agent_type;\n const isJsonSchemaValidOrEmpty =\n jsonSchemaText.trim() === '' || jsonSchemaValid;\n const hasSource = selectedSourceIds.size > 0;\n return hasRequiredFields && isJsonSchemaValidOrEmpty && hasSource;\n };\n\n const isJsonSchemaInvalid = () => {\n return jsonSchemaText.trim() !== '' && !jsonSchemaValid;\n };\n\n const handleUpload = useCallback((files: File[]) => {\n if (files && files.length > 0) {\n const file = files[0];\n setImageFile(file);\n }\n }, []);\n\n const navigateBackToAgents = useCallback(() => {\n const targetPath = validatedFolderId\n ? `/agents?folder=${validatedFolderId}`\n : '/agents';\n navigate(targetPath);\n }, [navigate, validatedFolderId]);\n\n const handleCancel = () => {\n if (selectedAgent) dispatch(setSelectedAgent(null));\n navigateBackToAgents();\n };\n\n const handleDelete = async (agentId: string) => {\n const response = await userService.deleteAgent(agentId, token);\n if (!response.ok) throw new Error('Failed to delete agent');\n navigateBackToAgents();\n };\n\n const handleSaveDraft = async () => {\n const formData = new FormData();\n formData.append('name', agent.name);\n formData.append('description', agent.description);\n\n if (selectedSourceIds.size > 1) {\n const sourcesArray = Array.from(selectedSourceIds)\n .map((id) => {\n const sourceDoc = sourceDocs?.find(\n (source) =>\n source.id === id || source.retriever === id || source.name === id,\n );\n if (sourceDoc?.name === 'Default' && !sourceDoc?.id) {\n return 'default';\n }\n return sourceDoc?.id || id;\n })\n .filter(Boolean);\n formData.append('sources', JSON.stringify(sourcesArray));\n formData.append('source', '');\n } else if (selectedSourceIds.size === 1) {\n const singleSourceId = Array.from(selectedSourceIds)[0];\n const sourceDoc = sourceDocs?.find(\n (source) =>\n source.id === singleSourceId ||\n source.retriever === singleSourceId ||\n source.name === singleSourceId,\n );\n let finalSourceId;\n if (sourceDoc?.name === 'Default' && !sourceDoc?.id)\n finalSourceId = 'default';\n else finalSourceId = sourceDoc?.id || singleSourceId;\n formData.append('source', String(finalSourceId));\n formData.append('sources', JSON.stringify([]));\n } else {\n formData.append('source', '');\n formData.append('sources', JSON.stringify([]));\n }\n\n formData.append('chunks', agent.chunks);\n formData.append('retriever', agent.retriever);\n formData.append('prompt_id', agent.prompt_id);\n formData.append('agent_type', agent.agent_type);\n formData.append('status', 'draft');\n\n if (agent.limited_token_mode && agent.token_limit) {\n formData.append('limited_token_mode', 'True');\n formData.append('token_limit', agent.token_limit.toString());\n } else {\n formData.append('limited_token_mode', 'False');\n formData.append('token_limit', '0');\n }\n\n if (agent.limited_request_mode && agent.request_limit) {\n formData.append('limited_request_mode', 'True');\n formData.append('request_limit', agent.request_limit.toString());\n } else {\n formData.append('limited_request_mode', 'False');\n formData.append('request_limit', '0');\n }\n\n if (imageFile) formData.append('image', imageFile);\n\n if (agent.tools && agent.tools.length > 0)\n formData.append('tools', JSON.stringify(agent.tools));\n else formData.append('tools', '[]');\n\n if (agent.json_schema) {\n formData.append('json_schema', JSON.stringify(agent.json_schema));\n }\n\n if (agent.models && agent.models.length > 0) {\n formData.append('models', JSON.stringify(agent.models));\n }\n if (agent.default_model_id) {\n formData.append('default_model_id', agent.default_model_id);\n }\n if (agent.agent_type === 'workflow' && agent.workflow) {\n formData.append('workflow', JSON.stringify(agent.workflow));\n }\n\n if (effectiveMode === 'new' && validatedFolderId) {\n formData.append('folder_id', validatedFolderId);\n }\n\n try {\n setDraftLoading(true);\n const response =\n effectiveMode === 'new'\n ? await userService.createAgent(formData, token)\n : await userService.updateAgent(agent.id || '', formData, token);\n if (!response.ok) throw new Error('Failed to create agent draft');\n const data = await response.json();\n\n const updatedAgent = {\n ...agent,\n id: data.id || agent.id,\n image: data.image || agent.image,\n };\n setAgent(updatedAgent);\n\n if (effectiveMode === 'new') setEffectiveMode('draft');\n } catch (error) {\n console.error('Error saving draft:', error);\n throw new Error('Failed to save draft');\n } finally {\n setDraftLoading(false);\n }\n };\n\n const handlePublish = async () => {\n const formData = new FormData();\n formData.append('name', agent.name);\n formData.append('description', agent.description);\n\n if (selectedSourceIds.size > 1) {\n const sourcesArray = Array.from(selectedSourceIds)\n .map((id) => {\n const sourceDoc = sourceDocs?.find(\n (source) =>\n source.id === id || source.retriever === id || source.name === id,\n );\n if (sourceDoc?.name === 'Default' && !sourceDoc?.id) {\n return 'default';\n }\n return sourceDoc?.id || id;\n })\n .filter(Boolean);\n formData.append('sources', JSON.stringify(sourcesArray));\n formData.append('source', '');\n } else if (selectedSourceIds.size === 1) {\n const singleSourceId = Array.from(selectedSourceIds)[0];\n const sourceDoc = sourceDocs?.find(\n (source) =>\n source.id === singleSourceId ||\n source.retriever === singleSourceId ||\n source.name === singleSourceId,\n );\n let finalSourceId;\n if (sourceDoc?.name === 'Default' && !sourceDoc?.id)\n finalSourceId = 'default';\n else finalSourceId = sourceDoc?.id || singleSourceId;\n formData.append('source', String(finalSourceId));\n formData.append('sources', JSON.stringify([]));\n } else {\n formData.append('source', '');\n formData.append('sources', JSON.stringify([]));\n }\n\n formData.append('chunks', agent.chunks);\n formData.append('retriever', agent.retriever);\n formData.append('prompt_id', agent.prompt_id);\n formData.append('agent_type', agent.agent_type);\n formData.append('status', 'published');\n\n if (imageFile) formData.append('image', imageFile);\n if (agent.tools && agent.tools.length > 0)\n formData.append('tools', JSON.stringify(agent.tools));\n else formData.append('tools', '[]');\n\n if (agent.json_schema) {\n formData.append('json_schema', JSON.stringify(agent.json_schema));\n }\n\n // Always send the limited mode fields\n if (agent.limited_token_mode && agent.token_limit) {\n formData.append('limited_token_mode', 'True');\n formData.append('token_limit', agent.token_limit.toString());\n } else {\n formData.append('limited_token_mode', 'False');\n formData.append('token_limit', '0');\n }\n\n if (agent.limited_request_mode && agent.request_limit) {\n formData.append('limited_request_mode', 'True');\n formData.append('request_limit', agent.request_limit.toString());\n } else {\n formData.append('limited_request_mode', 'False');\n formData.append('request_limit', '0');\n }\n\n if (agent.models && agent.models.length > 0) {\n formData.append('models', JSON.stringify(agent.models));\n }\n if (agent.default_model_id) {\n formData.append('default_model_id', agent.default_model_id);\n }\n if (agent.agent_type === 'workflow' && agent.workflow) {\n formData.append('workflow', JSON.stringify(agent.workflow));\n }\n\n if (effectiveMode === 'new' && validatedFolderId) {\n formData.append('folder_id', validatedFolderId);\n }\n\n try {\n setPublishLoading(true);\n const response =\n effectiveMode === 'new'\n ? await userService.createAgent(formData, token)\n : await userService.updateAgent(agent.id || '', formData, token);\n if (!response.ok) throw new Error('Failed to publish agent');\n const data = await response.json();\n\n const updatedAgent = {\n ...agent,\n id: data.id || agent.id,\n key: data.key || agent.key,\n status: 'published',\n image: data.image || agent.image,\n };\n setAgent(updatedAgent);\n initialAgentRef.current = updatedAgent;\n\n if (effectiveMode === 'new' || effectiveMode === 'draft') {\n setEffectiveMode('edit');\n setAgentDetails('ACTIVE');\n }\n setImageFile(null);\n } catch (error) {\n console.error('Error publishing agent:', error);\n throw new Error('Failed to publish agent');\n } finally {\n setPublishLoading(false);\n }\n };\n\n const validateAndSetJsonSchema = (text: string) => {\n setJsonSchemaText(text);\n if (text.trim() === '') {\n setAgent({ ...agent, json_schema: undefined });\n setJsonSchemaValid(true);\n return;\n }\n try {\n const parsed = JSON.parse(text);\n setAgent({ ...agent, json_schema: parsed });\n setJsonSchemaValid(true);\n } catch (error) {\n setJsonSchemaValid(false);\n }\n };\n\n useEffect(() => {\n const getTools = async () => {\n const response = await userService.getUserTools(token);\n if (!response.ok) throw new Error('Failed to fetch tools');\n const data = await response.json();\n const tools: OptionType[] = data.tools.map((tool: UserToolType) => ({\n id: tool.id,\n label: tool.customName ? tool.customName : tool.displayName,\n icon: `/toolIcons/tool_${tool.name}.svg`,\n }));\n setUserTools(tools);\n };\n const getModels = async () => {\n const response = await modelService.getModels(null);\n if (!response.ok) throw new Error('Failed to fetch models');\n const data = await response.json();\n const transformed = modelService.transformModels(data.models || []);\n setAvailableModels(transformed);\n\n if (mode === 'new' && transformed.length > 0) {\n const preferredDefaultModelId =\n transformed.find((model) => model.id === data.default_model_id)?.id ||\n transformed[0].id;\n\n if (preferredDefaultModelId) {\n setSelectedModelIds((prevSelectedModelIds) =>\n prevSelectedModelIds.size > 0\n ? prevSelectedModelIds\n : new Set([preferredDefaultModelId]),\n );\n }\n }\n };\n getTools();\n getModels();\n }, [token, mode]);\n\n // Validate folder_id from URL against user's folders\n useEffect(() => {\n const validateAndSetFolder = async () => {\n if (!folderIdFromUrl) {\n setValidatedFolderId(null);\n return;\n }\n\n let folders = agentFolders;\n if (!folders) {\n try {\n const response = await userService.getAgentFolders(token);\n if (response.ok) {\n const data = await response.json();\n folders = data.folders || [];\n dispatch(setAgentFolders(folders));\n }\n } catch {\n setValidatedFolderId(null);\n return;\n }\n }\n\n const folderExists = folders?.some((f) => f.id === folderIdFromUrl);\n setValidatedFolderId(folderExists ? folderIdFromUrl : null);\n };\n\n validateAndSetFolder();\n }, [folderIdFromUrl, agentFolders, token, dispatch]);\n\n // Auto-select default source if none selected\n useEffect(() => {\n if (sourceDocs && sourceDocs.length > 0 && selectedSourceIds.size === 0) {\n const defaultSource = sourceDocs.find((s) => s.name === 'Default');\n if (defaultSource) {\n setSelectedSourceIds(\n new Set([\n defaultSource.id || defaultSource.retriever || defaultSource.name,\n ]),\n );\n } else {\n setSelectedSourceIds(\n new Set([\n sourceDocs[0].id || sourceDocs[0].retriever || sourceDocs[0].name,\n ]),\n );\n }\n }\n }, [sourceDocs, selectedSourceIds.size]);\n\n useEffect(() => {\n if ((mode === 'edit' || mode === 'draft') && agentId) {\n const getAgent = async () => {\n const response = await userService.getAgent(agentId, token);\n if (!response.ok) {\n navigate('/agents');\n throw new Error('Failed to fetch agent');\n }\n const data = await response.json();\n\n if (data.sources && data.sources.length > 0) {\n const mappedSources = data.sources.map((sourceId: string) => {\n if (sourceId === 'default') {\n const defaultSource = sourceDocs?.find(\n (source) => source.name === 'Default',\n );\n return defaultSource?.retriever || 'classic';\n }\n return sourceId;\n });\n setSelectedSourceIds(new Set(mappedSources));\n } else if (data.source) {\n if (data.source === 'default') {\n const defaultSource = sourceDocs?.find(\n (source) => source.name === 'Default',\n );\n setSelectedSourceIds(\n new Set([defaultSource?.retriever || 'classic']),\n );\n } else {\n setSelectedSourceIds(new Set([data.source]));\n }\n } else if (data.retriever) {\n setSelectedSourceIds(new Set([data.retriever]));\n }\n\n if (data.tool_details) setSelectedTools(data.tool_details);\n if (data.status === 'draft') setEffectiveMode('draft');\n if (data.json_schema) {\n const jsonText = JSON.stringify(data.json_schema, null, 2);\n setJsonSchemaText(jsonText);\n setJsonSchemaValid(true);\n }\n setAgent(data);\n initialAgentRef.current = data;\n };\n getAgent();\n }\n }, [agentId, mode, token]);\n\n useEffect(() => {\n if (agent.models && agent.models.length > 0 && availableModels.length > 0) {\n const agentModelIds = new Set(agent.models);\n if (agentModelIds.size > 0 && selectedModelIds.size === 0) {\n setSelectedModelIds(agentModelIds);\n }\n }\n }, [agent.models, availableModels.length]);\n\n useEffect(() => {\n const modelsArray = Array.from(selectedModelIds);\n if (modelsArray.length > 0) {\n setAgent((prev) => ({\n ...prev,\n models: modelsArray,\n default_model_id: modelsArray.includes(prev.default_model_id || '')\n ? prev.default_model_id\n : modelsArray[0],\n }));\n } else {\n setAgent((prev) => ({\n ...prev,\n models: [],\n default_model_id: '',\n }));\n }\n }, [selectedModelIds]);\n\n useEffect(() => {\n const selectedSources = Array.from(selectedSourceIds)\n .map((id) =>\n sourceDocs?.find(\n (source) =>\n source.id === id || source.retriever === id || source.name === id,\n ),\n )\n .filter(Boolean);\n\n if (selectedSources.length > 0) {\n // Handle multiple sources\n if (selectedSources.length > 1) {\n // Multiple sources selected - store in sources array\n const sourceIds = selectedSources\n .map((source) => source?.id)\n .filter((id): id is string => Boolean(id));\n setAgent((prev) => ({\n ...prev,\n sources: sourceIds,\n source: '', // Clear single source for multiple sources\n retriever: '',\n }));\n } else {\n // Single source selected - maintain backward compatibility\n const selectedSource = selectedSources[0];\n if (selectedSource && 'id' in selectedSource) {\n setAgent((prev) => ({\n ...prev,\n source: selectedSource?.id || 'default',\n sources: [], // Clear sources array for single source\n retriever: '',\n }));\n } else {\n setAgent((prev) => ({\n ...prev,\n source: '',\n sources: [], // Clear sources array\n retriever: selectedSource?.retriever || 'classic',\n }));\n }\n }\n } else {\n // No sources selected\n setAgent((prev) => ({\n ...prev,\n source: '',\n sources: [],\n retriever: '',\n }));\n }\n }, [selectedSourceIds]);\n\n useEffect(() => {\n setAgent((prev) => ({\n ...prev,\n tools: Array.from(selectedTools)\n .map((tool) => tool?.id)\n .filter((id): id is string => typeof id === 'string'),\n }));\n }, [selectedTools]);\n\n useEffect(() => {\n if (isPublishable()) dispatch(setSelectedAgent(agent));\n\n if (!modeConfig[effectiveMode].trackChanges) {\n setHasChanges(true);\n return;\n }\n if (!initialAgentRef.current) {\n setHasChanges(false);\n return;\n }\n\n const initialJsonSchemaText = initialAgentRef.current.json_schema\n ? JSON.stringify(initialAgentRef.current.json_schema, null, 2)\n : '';\n\n const isChanged =\n !isEqual(agent, initialAgentRef.current) ||\n imageFile !== null ||\n jsonSchemaText !== initialJsonSchemaText;\n setHasChanges(isChanged);\n }, [agent, dispatch, effectiveMode, imageFile, jsonSchemaText]);\n return (\n
\n
\n \n \"left-arrow\"\n \n

\n {t('agents.backToAll')}\n

\n
\n
\n

\n {modeConfig[effectiveMode].heading}\n

\n {agent.agent_type === 'workflow' && (\n
\n \n
\n )}\n
\n \n {t('agents.form.buttons.cancel')}\n \n {modeConfig[effectiveMode].showDelete && agent.id && (\n setDeleteConfirmation('ACTIVE')}\n >\n \n {t('agents.form.buttons.delete')}\n \n )}\n {modeConfig[effectiveMode].showSaveDraft && (\n \n \n {draftLoading ? (\n \n ) : (\n t('agents.form.buttons.saveDraft')\n )}\n \n \n )}\n {modeConfig[effectiveMode].showAccessDetails && (\n navigate(`/agents/logs/${agent.id}`)}\n >\n \n {t('agents.form.buttons.logs')}\n \n )}\n {modeConfig[effectiveMode].showAccessDetails && (\n olets-are-blue border-violets-are-blue text-violets-are-blue hover:bg-violets-are-blue rounded-3xl border border-solid px-5 py-2 text-sm font-medium transition-colors hover:text-white\"\n onClick={() => setAgentDetails('ACTIVE')}\n >\n {t('agents.form.buttons.accessDetails')}\n \n )}\n \n \n {publishLoading ? (\n \n ) : (\n modeConfig[effectiveMode].buttonText\n )}\n \n \n
\n
\n
\n
\n
\n

\n {t('agents.form.sections.meta')}\n

\n setAgent({ ...agent, name: e.target.value })}\n />\n \n setAgent({ ...agent, description: e.target.value })\n }\n />\n
\n setImageFile(null)}\n uploadText={[\n {\n text: t('agents.form.upload.clickToUpload'),\n colorClass: 'text-[#7D54D1]',\n },\n {\n text: t('agents.form.upload.dragAndDrop'),\n colorClass: 'text-[#525252]',\n },\n ]}\n />\n
\n
\n
\n

\n {t('agents.form.sections.source')}\n

\n
\n
\n setIsSourcePopupOpen(!isSourcePopupOpen)}\n className={`border-silver dark:bg-raisin-black w-full truncate rounded-3xl border bg-white px-5 py-3 text-left text-sm dark:border-[#7E7E7E] ${\n selectedSourceIds.size > 0\n ? 'text-jet dark:text-bright-gray'\n : 'dark:text-silver text-gray-400'\n }`}\n >\n {selectedSourceIds.size > 0\n ? Array.from(selectedSourceIds)\n .map((id) => {\n const matchedDoc = sourceDocs?.find(\n (source) =>\n source.id === id ||\n source.name === id ||\n source.retriever === id,\n );\n return (\n matchedDoc?.name || t('agents.form.externalKb')\n );\n })\n .filter(Boolean)\n .join(', ')\n : t('agents.form.placeholders.selectSources')}\n \n setIsSourcePopupOpen(false)}\n anchorRef={sourceAnchorButtonRef}\n options={\n sourceDocs?.map((doc: Doc) => ({\n id: doc.id || doc.retriever || doc.name,\n label: doc.name,\n icon: \"\",\n })) || []\n }\n selectedIds={selectedSourceIds}\n onSelectionChange={(newSelectedIds: Set) => {\n if (\n newSelectedIds.size === 0 &&\n sourceDocs &&\n sourceDocs.length > 0\n ) {\n const defaultSource = sourceDocs.find(\n (s) => s.name === 'Default',\n );\n if (defaultSource) {\n setSelectedSourceIds(\n new Set([\n defaultSource.id ||\n defaultSource.retriever ||\n defaultSource.name,\n ]),\n );\n } else {\n setSelectedSourceIds(\n new Set([\n sourceDocs[0].id ||\n sourceDocs[0].retriever ||\n sourceDocs[0].name,\n ]),\n );\n }\n } else {\n setSelectedSourceIds(newSelectedIds);\n }\n }}\n title={t('agents.form.sourcePopup.title')}\n searchPlaceholder={t(\n 'agents.form.sourcePopup.searchPlaceholder',\n )}\n noOptionsMessage={t(\n 'agents.form.sourcePopup.noOptionsMessage',\n )}\n />\n
\n
\n \n setAgent({ ...agent, chunks: value })\n }\n size=\"w-full\"\n rounded=\"3xl\"\n border=\"border\"\n buttonClassName=\"bg-white dark:bg-[#222327] border-silver dark:border-[#7E7E7E]\"\n optionsClassName=\"bg-white dark:bg-[#383838] border-silver dark:border-[#7E7E7E]\"\n placeholder={t('agents.form.placeholders.chunksPerQuery')}\n placeholderClassName=\"text-gray-400 dark:text-silver\"\n contentSize=\"text-sm\"\n />\n
\n
\n
\n
\n
\n
\n prompt.id === agent.prompt_id) ||\n prompts[0] || {\n name: 'default',\n id: 'default',\n type: 'public',\n }\n }\n onSelectPrompt={(name, id, type) =>\n setAgent({ ...agent, prompt_id: id })\n }\n setPrompts={(newPrompts) => dispatch(setPrompts(newPrompts))}\n title={t('agents.form.sections.prompt')}\n titleClassName=\"text-lg font-semibold dark:text-[#E0E0E0]\"\n showAddButton={false}\n dropdownProps={{\n size: 'w-full',\n rounded: '3xl',\n border: 'border',\n buttonClassName:\n 'bg-white dark:bg-[#222327] border-silver dark:border-[#7E7E7E]',\n optionsClassName:\n 'bg-white dark:bg-[#383838] border-silver dark:border-[#7E7E7E]',\n placeholderClassName: 'text-gray-400 dark:text-silver',\n contentSize: 'text-sm',\n }}\n />\n
\n setAddPromptModal('ACTIVE')}\n >\n {t('agents.form.buttons.add')}\n \n
\n
\n
\n

\n {t('agents.form.sections.tools')}\n

\n
\n setIsToolsPopupOpen(!isToolsPopupOpen)}\n className={`border-silver dark:bg-raisin-black w-full truncate rounded-3xl border bg-white px-5 py-3 text-left text-sm dark:border-[#7E7E7E] ${\n selectedTools.length > 0\n ? 'text-jet dark:text-bright-gray'\n : 'dark:text-silver text-gray-400'\n }`}\n >\n {selectedTools.length > 0\n ? selectedTools\n .map((tool) => tool.display_name || tool.name)\n .filter(Boolean)\n .join(', ')\n : t('agents.form.placeholders.selectTools')}\n \n setIsToolsPopupOpen(false)}\n anchorRef={toolAnchorButtonRef}\n options={userTools}\n selectedIds={new Set(selectedTools.map((tool) => tool.id))}\n onSelectionChange={(newSelectedIds: Set) =>\n setSelectedTools(\n userTools\n .filter((tool) => newSelectedIds.has(tool.id))\n .map((tool) => ({\n id: String(tool.id),\n name: tool.label,\n display_name: tool.label,\n })),\n )\n }\n title={t('agents.form.toolsPopup.title')}\n searchPlaceholder={t(\n 'agents.form.toolsPopup.searchPlaceholder',\n )}\n noOptionsMessage={t('agents.form.toolsPopup.noOptionsMessage')}\n />\n
\n
\n
\n

\n {t('agents.form.sections.agentType')}\n

\n
\n type.value === agent.agent_type)\n ?.label || null\n : null\n }\n onSelect={(option: { label: string; value: string }) =>\n setAgent({ ...agent, agent_type: option.value })\n }\n size=\"w-full\"\n rounded=\"3xl\"\n border=\"border\"\n buttonClassName=\"bg-white dark:bg-[#222327] border-silver dark:border-[#7E7E7E]\"\n optionsClassName=\"bg-white dark:bg-[#383838] border-silver dark:border-[#7E7E7E]\"\n placeholder={t('agents.form.placeholders.selectType')}\n placeholderClassName=\"text-gray-400 dark:text-silver\"\n contentSize=\"text-sm\"\n />\n
\n
\n
\n

\n {t('agents.form.sections.models')}\n

\n
\n setIsModelsPopupOpen(!isModelsPopupOpen)}\n className={`border-silver dark:bg-raisin-black w-full truncate rounded-3xl border bg-white px-5 py-3 text-left text-sm dark:border-[#7E7E7E] ${\n selectedModelIds.size > 0\n ? 'text-jet dark:text-bright-gray'\n : 'dark:text-silver text-gray-400'\n }`}\n >\n {selectedModelIds.size > 0\n ? availableModels\n .filter((m) => selectedModelIds.has(m.id))\n .map((m) => m.display_name)\n .join(', ')\n : t('agents.form.placeholders.selectModels')}\n \n setIsModelsPopupOpen(false)}\n anchorRef={modelAnchorButtonRef}\n options={availableModels.map((model) => ({\n id: model.id,\n label: model.display_name,\n }))}\n selectedIds={selectedModelIds}\n onSelectionChange={(newSelectedIds: Set) =>\n setSelectedModelIds(\n new Set(Array.from(newSelectedIds).map(String)),\n )\n }\n title={t('agents.form.modelsPopup.title')}\n searchPlaceholder={t(\n 'agents.form.modelsPopup.searchPlaceholder',\n )}\n noOptionsMessage={t('agents.form.modelsPopup.noOptionsMessage')}\n />\n {selectedModelIds.size > 0 && (\n
\n \n selectedModelIds.has(m.id))\n .map((m) => ({\n label: m.display_name,\n value: m.id,\n }))}\n selectedValue={\n availableModels.find(\n (m) => m.id === agent.default_model_id,\n )?.display_name || null\n }\n onSelect={(option: { label: string; value: string }) =>\n setAgent({ ...agent, default_model_id: option.value })\n }\n size=\"w-full\"\n rounded=\"3xl\"\n border=\"border\"\n buttonClassName=\"bg-white dark:bg-[#222327] border-silver dark:border-[#7E7E7E]\"\n optionsClassName=\"bg-white dark:bg-[#383838] border-silver dark:border-[#7E7E7E]\"\n placeholder={t(\n 'agents.form.placeholders.selectDefaultModel',\n )}\n placeholderClassName=\"text-gray-400 dark:text-silver\"\n contentSize=\"text-sm\"\n />\n
\n )}\n
\n
\n
\n \n setIsAdvancedSectionExpanded(!isAdvancedSectionExpanded)\n }\n className=\"flex w-full items-center justify-between text-left focus:outline-none\"\n >\n
\n

\n {t('agents.form.sections.advanced')}\n

\n
\n
\n \n \n \n
\n \n {isAdvancedSectionExpanded && (\n
\n
\n

\n {t('agents.form.advanced.jsonSchema')}\n

\n

\n {t('agents.form.advanced.jsonSchemaDescription')}\n

\n
\n validateAndSetJsonSchema(e.target.value)}\n placeholder={`{\n \"type\": \"object\",\n \"properties\": {\n \"name\": {\"type\": \"string\"},\n \"email\": {\"type\": \"string\"}\n },\n \"required\": [\"name\", \"email\"],\n \"additionalProperties\": false\n}`}\n rows={9}\n className={`border-silver text-jet dark:bg-raisin-black dark:text-bright-gray mt-2 w-full rounded-2xl border bg-white px-4 py-3 font-mono text-sm outline-hidden dark:border-[#7E7E7E]`}\n />\n {jsonSchemaText.trim() !== '' && (\n \n \n {jsonSchemaValid\n ? t('agents.form.advanced.validJson')\n : t('agents.form.advanced.invalidJson')}\n
\n )}\n\n
\n
\n
\n

\n {t('agents.form.advanced.tokenLimiting')}\n

\n

\n {t('agents.form.advanced.tokenLimitingDescription')}\n

\n
\n {\n const newTokenMode = !agent.limited_token_mode;\n setAgent({\n ...agent,\n limited_token_mode: newTokenMode,\n limited_request_mode: newTokenMode\n ? false\n : agent.limited_request_mode,\n });\n }}\n className={`relative h-6 w-11 rounded-full transition-colors ${\n agent.limited_token_mode\n ? 'bg-purple-30'\n : 'bg-gray-300 dark:bg-gray-600'\n }`}\n >\n \n \n
\n \n setAgent({\n ...agent,\n token_limit: e.target.value\n ? parseInt(e.target.value)\n : undefined,\n })\n }\n disabled={!agent.limited_token_mode}\n placeholder={t('agents.form.placeholders.enterTokenLimit')}\n className={`border-silver text-jet dark:bg-raisin-black dark:text-bright-gray dark:placeholder:text-silver mt-2 w-full rounded-3xl border bg-white px-5 py-3 text-sm outline-hidden placeholder:text-gray-400 dark:border-[#7E7E7E] ${\n !agent.limited_token_mode\n ? 'cursor-not-allowed opacity-50'\n : ''\n }`}\n />\n
\n\n
\n
\n
\n

\n {t('agents.form.advanced.requestLimiting')}\n

\n

\n {t('agents.form.advanced.requestLimitingDescription')}\n

\n
\n {\n const newRequestMode = !agent.limited_request_mode;\n setAgent({\n ...agent,\n limited_request_mode: newRequestMode,\n limited_token_mode: newRequestMode\n ? false\n : agent.limited_token_mode,\n });\n }}\n className={`relative h-6 w-11 rounded-full transition-colors ${\n agent.limited_request_mode\n ? 'bg-purple-30'\n : 'bg-gray-300 dark:bg-gray-600'\n }`}\n >\n \n \n
\n \n setAgent({\n ...agent,\n request_limit: e.target.value\n ? parseInt(e.target.value)\n : undefined,\n })\n }\n disabled={!agent.limited_request_mode}\n placeholder={t(\n 'agents.form.placeholders.enterRequestLimit',\n )}\n className={`border-silver text-jet dark:bg-raisin-black dark:text-bright-gray dark:placeholder:text-silver mt-2 w-full rounded-3xl border bg-white px-5 py-3 text-sm outline-hidden placeholder:text-gray-400 dark:border-[#7E7E7E] ${\n !agent.limited_request_mode\n ? 'cursor-not-allowed opacity-50'\n : ''\n }`}\n />\n
\n
\n )}\n
\n
\n
\n

\n {t('agents.form.sections.preview')}\n

\n
\n \n
\n
\n
\n {\n handleDelete(agent.id || '');\n setDeleteConfirmation('INACTIVE');\n }}\n cancelLabel={t('agents.form.buttons.cancel')}\n variant=\"danger\"\n />\n \n setAddPromptModal('INACTIVE')}\n onSelect={(name: string, id: string, type: string) => {\n setAgent({ ...agent, prompt_id: id });\n }}\n />\n \n );\n}\n\nfunction AgentPreviewArea() {\n const { t } = useTranslation();\n const selectedAgent = useSelector(selectSelectedAgent);\n return (\n
\n {selectedAgent?.status === 'published' ? (\n
\n \n
\n ) : (\n
\n {' '}\n

\n {t('agents.form.preview.publishedPreview')}\n

\n
\n )}\n
\n );\n}\n\nfunction AddPromptModal({\n prompts,\n isOpen,\n onClose,\n onSelect,\n}: {\n prompts: Prompt[];\n isOpen: ActiveState;\n onClose: () => void;\n onSelect?: (name: string, id: string, type: string) => void;\n}) {\n const dispatch = useDispatch();\n const token = useSelector(selectToken);\n\n const [newPromptName, setNewPromptName] = useState('');\n const [newPromptContent, setNewPromptContent] = useState('');\n\n const handleAddPrompt = async () => {\n try {\n const response = await userService.createPrompt(\n {\n name: newPromptName,\n content: newPromptContent,\n },\n token,\n );\n if (!response.ok) {\n throw new Error('Failed to add prompt');\n }\n const newPrompt = await response.json();\n // Update Redux store with new prompt\n dispatch(\n setPrompts([\n ...prompts,\n { name: newPromptName, id: newPrompt.id, type: 'private' },\n ]),\n );\n onClose();\n setNewPromptName('');\n setNewPromptContent('');\n onSelect?.(newPromptName, newPrompt.id, newPromptContent);\n } catch (error) {\n console.error('Error adding prompt:', error);\n }\n };\n return (\n undefined}\n editPromptContent={''}\n setEditPromptContent={() => undefined}\n currentPromptEdit={{ id: '', name: '', type: '' }}\n handleAddPrompt={handleAddPrompt}\n />\n );\n}\n" }, { "path": "frontend/src/agents/SharedAgent.tsx", "content": "import { useCallback, useEffect, useRef, useState } from 'react';\nimport { useTranslation } from 'react-i18next';\nimport { useDispatch, useSelector } from 'react-redux';\nimport { useParams } from 'react-router-dom';\n\nimport userService from '../api/services/userService';\nimport NoFilesDarkIcon from '../assets/no-files-dark.svg';\nimport NoFilesIcon from '../assets/no-files.svg';\nimport AgentImage from '../components/AgentImage';\nimport MessageInput from '../components/MessageInput';\nimport Spinner from '../components/Spinner';\nimport ConversationMessages from '../conversation/ConversationMessages';\nimport { Query } from '../conversation/conversationModels';\nimport {\n addQuery,\n fetchAnswer,\n resendQuery,\n selectQueries,\n selectStatus,\n} from '../conversation/conversationSlice';\nimport { useDarkTheme } from '../hooks';\nimport { selectToken, setSelectedAgent } from '../preferences/preferenceSlice';\nimport { AppDispatch } from '../store';\nimport SharedAgentCard from './SharedAgentCard';\nimport { Agent } from './types';\n\nexport default function SharedAgent() {\n const { t } = useTranslation();\n const { agentId } = useParams();\n const dispatch = useDispatch();\n const [isDarkTheme] = useDarkTheme();\n\n const token = useSelector(selectToken);\n const queries = useSelector(selectQueries);\n const status = useSelector(selectStatus);\n\n const [sharedAgent, setSharedAgent] = useState();\n const [isLoading, setIsLoading] = useState(true);\n const [input, setInput] = useState('');\n const [lastQueryReturnedErr, setLastQueryReturnedErr] = useState(false);\n\n const fetchStream = useRef(null);\n\n const getSharedAgent = async () => {\n try {\n setIsLoading(true);\n const response = await userService.getSharedAgent(agentId ?? '', token);\n if (!response.ok) throw new Error('Failed to fetch Shared Agent');\n const agent: Agent = await response.json();\n setSharedAgent(agent);\n } catch (error) {\n console.error('Error: ', error);\n } finally {\n setIsLoading(false);\n }\n };\n\n const handleFetchAnswer = useCallback(\n ({ question, index }: { question: string; index?: number }) => {\n fetchStream.current = dispatch(fetchAnswer({ question, indx: index }));\n },\n [dispatch],\n );\n\n const handleQuestion = useCallback(\n ({\n question,\n isRetry = false,\n index = undefined,\n }: {\n question: string;\n isRetry?: boolean;\n index?: number;\n }) => {\n const trimmedQuestion = question.trim();\n if (trimmedQuestion === '') return;\n\n if (index !== undefined) {\n if (!isRetry) dispatch(resendQuery({ index, prompt: trimmedQuestion }));\n handleFetchAnswer({ question: trimmedQuestion, index });\n } else {\n if (!isRetry) {\n const newQuery: Query = { prompt: trimmedQuestion };\n dispatch(addQuery(newQuery));\n }\n handleFetchAnswer({ question: trimmedQuestion, index: undefined });\n }\n },\n [dispatch, handleFetchAnswer],\n );\n\n const handleQuestionSubmission = (\n question?: string,\n updated?: boolean,\n indx?: number,\n ) => {\n if (updated === true && question !== undefined && indx !== undefined) {\n handleQuestion({\n question,\n index: indx,\n isRetry: false,\n });\n } else if (question && status !== 'loading') {\n const currentInput = question.trim();\n if (lastQueryReturnedErr && queries.length > 0) {\n const lastQueryIndex = queries.length - 1;\n handleQuestion({\n question: currentInput,\n isRetry: true,\n index: lastQueryIndex,\n });\n } else {\n handleQuestion({\n question: currentInput,\n isRetry: false,\n index: undefined,\n });\n }\n setInput('');\n }\n };\n\n useEffect(() => {\n if (agentId) getSharedAgent();\n }, [agentId, token]);\n\n useEffect(() => {\n if (sharedAgent) dispatch(setSelectedAgent(sharedAgent));\n }, [sharedAgent, dispatch]);\n\n if (isLoading)\n return (\n
\n \n
\n );\n if (!sharedAgent)\n return (\n
\n
\n \n

\n {t('agents.shared.notFound')}\n

\n
\n
\n );\n return (\n
\n \n
\n
\n \n \n
\n }\n />\n
\n
\n
\n handleQuestionSubmission(text)}\n loading={status === 'loading'}\n showSourceButton={sharedAgent ? false : true}\n showToolButton={sharedAgent ? false : true}\n autoFocus={false}\n />\n
\n \n
\n
\n \n );\n}\n" }, { "path": "frontend/src/agents/SharedAgentCard.tsx", "content": "import AgentImage from '../components/AgentImage';\nimport { Agent } from './types';\n\nexport default function SharedAgentCard({ agent }: { agent: Agent }) {\n // Check if shared metadata exists and has properties (type is 'any' so we validate it's a non-empty object)\n const hasSharedMetadata =\n agent.shared_metadata &&\n typeof agent.shared_metadata === 'object' &&\n agent.shared_metadata !== null &&\n Object.keys(agent.shared_metadata).length > 0;\n return (\n
\n
\n
\n \n
\n
\n

\n {agent.name}\n

\n

\n {agent.description}\n

\n
\n
\n {hasSharedMetadata && (\n
\n {agent.shared_metadata?.shared_by && (\n

\n by {agent.shared_metadata.shared_by}\n

\n )}\n {agent.shared_metadata?.shared_at && (\n

\n Shared on{' '}\n {new Date(agent.shared_metadata.shared_at).toLocaleString(\n 'en-US',\n {\n month: 'long',\n day: 'numeric',\n year: 'numeric',\n hour: '2-digit',\n minute: '2-digit',\n hour12: true,\n },\n )}\n

\n )}\n
\n )}\n {agent.tool_details && agent.tool_details.length > 0 && (\n
\n

\n Connected Tools\n

\n
\n {agent.tool_details.map((tool, index) => (\n \n {' '}\n {tool.name}\n \n ))}\n
\n
\n )}\n
\n );\n}\n" }, { "path": "frontend/src/agents/SharedAgentGate.tsx", "content": "import { Navigate, useParams } from 'react-router-dom';\n\nexport default function SharedAgentGate() {\n const { agentId } = useParams();\n\n return ;\n}\n" }, { "path": "frontend/src/agents/WorkflowBuilder.tsx", "content": "import 'reactflow/dist/style.css';\n\nimport {\n AlertCircle,\n Bot,\n Database,\n Flag,\n Play,\n Settings,\n StickyNote,\n Trash2,\n X,\n} from 'lucide-react';\nimport { useCallback, useEffect, useMemo, useRef, useState } from 'react';\nimport { useNavigate, useParams, useSearchParams } from 'react-router-dom';\nimport ReactFlow, {\n addEdge,\n applyEdgeChanges,\n applyNodeChanges,\n Background,\n Connection,\n Controls,\n Edge,\n EdgeChange,\n Node,\n NodeChange,\n NodeTypes,\n ReactFlowProvider,\n useReactFlow,\n} from 'reactflow';\n\nimport { Alert, AlertDescription, AlertTitle } from '@/components/ui/alert';\nimport { MultiSelect } from '@/components/ui/multi-select';\nimport {\n Select,\n SelectContent,\n SelectItem,\n SelectTrigger,\n SelectValue,\n} from '@/components/ui/select';\nimport { Sheet, SheetContent } from '@/components/ui/sheet';\n\nimport modelService from '../api/services/modelService';\nimport userService from '../api/services/userService';\nimport ArrowLeft from '../assets/arrow-left.svg';\nimport { WorkflowNode } from './types/workflow';\nimport {\n AgentNode,\n EndNode,\n NoteNode,\n SetStateNode,\n StartNode,\n} from './workflow/nodes';\nimport WorkflowPreview from './workflow/WorkflowPreview';\n\nimport type { Model } from '../models/types';\ninterface AgentNodeConfig {\n agent_type: 'classic' | 'react';\n llm_name?: string;\n model_id?: string;\n system_prompt: string;\n prompt_template: string;\n output_variable?: string;\n stream_to_user: boolean;\n sources: string[];\n tools: string[];\n chunks?: string;\n retriever?: string;\n json_schema?: Record;\n}\n\ninterface UserTool {\n id: string;\n name: string;\n displayName: string;\n}\n\nfunction WorkflowBuilderInner() {\n const navigate = useNavigate();\n const { agentId } = useParams<{ agentId?: string }>();\n const [searchParams] = useSearchParams();\n const folderId = searchParams.get('folder_id');\n const [workflowId, setWorkflowId] = useState(\n searchParams.get('workflow_id'),\n );\n const reactFlowInstance = useReactFlow();\n const [currentAgentId, setCurrentAgentId] = useState(\n agentId || null,\n );\n\n const reactFlowWrapper = useRef(null);\n\n const [selectedNode, setSelectedNode] = useState(null);\n const [workflowName, setWorkflowName] = useState('New Workflow');\n const [workflowDescription, setWorkflowDescription] = useState('');\n const [showWorkflowSettings, setShowWorkflowSettings] = useState(false);\n const [isPublishing, setIsPublishing] = useState(false);\n const [publishErrors, setPublishErrors] = useState([]);\n const [errorContext, setErrorContext] = useState<'preview' | 'publish'>(\n 'publish',\n );\n const [showNodeConfig, setShowNodeConfig] = useState(false);\n const [showPreview, setShowPreview] = useState(false);\n const configPanelRef = useRef(null);\n const workflowSettingsRef = useRef(null);\n const [availableModels, setAvailableModels] = useState([]);\n const [availableTools, setAvailableTools] = useState([]);\n\n const nodeTypes = useMemo(\n () => ({\n start: StartNode,\n agent: AgentNode,\n end: EndNode,\n note: NoteNode,\n state: SetStateNode,\n }),\n [],\n );\n\n const initialNodes: Node[] = useMemo(\n () => [\n {\n id: 'start',\n type: 'start',\n data: { label: 'Start' },\n position: { x: 250, y: 50 },\n },\n ],\n [],\n );\n\n const [nodes, setNodes] = useState(initialNodes);\n const [edges, setEdges] = useState([]);\n\n const onNodesChange = useCallback(\n (changes: NodeChange[]) =>\n setNodes((nds) => applyNodeChanges(changes, nds)),\n [],\n );\n\n const onEdgesChange = useCallback(\n (changes: EdgeChange[]) =>\n setEdges((eds) => applyEdgeChanges(changes, eds)),\n [],\n );\n\n const onConnect = useCallback(\n (params: Connection) => setEdges((eds) => addEdge(params, eds)),\n [],\n );\n\n const onDragOver = useCallback((event: React.DragEvent) => {\n event.preventDefault();\n event.dataTransfer.dropEffect = 'move';\n }, []);\n\n const onDrop = useCallback(\n (event: React.DragEvent) => {\n event.preventDefault();\n\n const type = event.dataTransfer.getData('application/reactflow');\n if (!type) return;\n\n // Use screenToFlowPosition to correctly convert screen coordinates to flow coordinates\n // This accounts for viewport pan and zoom\n const position = reactFlowInstance.screenToFlowPosition({\n x: event.clientX,\n y: event.clientY,\n });\n\n const baseNode: Node = {\n id: `${type}_${Date.now()}`,\n type,\n position,\n data: {\n title: `${type} node`,\n label: `${type} node`,\n },\n };\n\n if (type === 'agent') {\n baseNode.data.config = {\n agent_type: 'classic',\n system_prompt: 'You are a helpful assistant.',\n prompt_template: '',\n stream_to_user: true,\n sources: [],\n tools: [],\n } as AgentNodeConfig;\n } else if (type === 'state') {\n baseNode.data.title = 'Set State';\n baseNode.data.variable = '';\n baseNode.data.value = '';\n } else if (type === 'note') {\n baseNode.data.title = 'Note';\n baseNode.data.label = 'Note';\n }\n\n setNodes((nds) => nds.concat(baseNode));\n },\n [reactFlowInstance],\n );\n\n const handleNodeClick = useCallback(\n (_event: React.MouseEvent, node: Node) => {\n setSelectedNode(node);\n setShowNodeConfig(true);\n },\n [],\n );\n\n const handleDeleteNode = useCallback(() => {\n if (!selectedNode || selectedNode.type === 'start') return;\n setNodes((nds) => nds.filter((n) => n.id !== selectedNode.id));\n setEdges((eds) =>\n eds.filter(\n (e) => e.source !== selectedNode.id && e.target !== selectedNode.id,\n ),\n );\n setSelectedNode(null);\n setShowNodeConfig(false);\n }, [selectedNode]);\n\n const handleUpdateNodeData = useCallback(\n (data: Record) => {\n if (!selectedNode) return;\n setNodes((nds) =>\n nds.map((n) =>\n n.id === selectedNode.id ? { ...n, data: { ...n.data, ...data } } : n,\n ),\n );\n setSelectedNode((prev) =>\n prev ? { ...prev, data: { ...prev.data, ...data } } : null,\n );\n },\n [selectedNode],\n );\n\n useEffect(() => {\n if (publishErrors.length > 0) {\n const timer = setTimeout(() => {\n setPublishErrors([]);\n }, 6000);\n return () => clearTimeout(timer);\n }\n }, [publishErrors.length]);\n\n useEffect(() => {\n const handleKeyDown = (e: KeyboardEvent) => {\n if (e.key === 'Delete' && selectedNode) {\n handleDeleteNode();\n }\n if (e.key === 'Escape') {\n setShowNodeConfig(false);\n setSelectedNode(null);\n }\n };\n window.addEventListener('keydown', handleKeyDown);\n return () => window.removeEventListener('keydown', handleKeyDown);\n }, [selectedNode, handleDeleteNode]);\n\n useEffect(() => {\n const handleClickOutside = (e: MouseEvent) => {\n const target = e.target as HTMLElement;\n const isInsidePanel = configPanelRef.current?.contains(target) ?? false;\n const isInsideRadixPortal =\n target.closest('[data-radix-popper-content-wrapper]') !== null ||\n target.closest('[data-radix-select-content]') !== null ||\n target.closest('[role=\"listbox\"]') !== null ||\n target.closest('[cmdk-root]') !== null;\n if (!isInsidePanel && !isInsideRadixPortal) {\n setShowNodeConfig(false);\n }\n };\n if (showNodeConfig) {\n document.addEventListener('mousedown', handleClickOutside);\n return () =>\n document.removeEventListener('mousedown', handleClickOutside);\n }\n }, [showNodeConfig]);\n\n useEffect(() => {\n const handleClickOutside = (e: MouseEvent) => {\n if (\n workflowSettingsRef.current &&\n !workflowSettingsRef.current.contains(e.target as HTMLElement)\n ) {\n setShowWorkflowSettings(false);\n }\n };\n if (showWorkflowSettings) {\n document.addEventListener('mousedown', handleClickOutside);\n return () =>\n document.removeEventListener('mousedown', handleClickOutside);\n }\n }, [showWorkflowSettings]);\n\n useEffect(() => {\n const loadModelsAndTools = async () => {\n try {\n const modelsResponse = await modelService.getModels(null);\n if (modelsResponse.ok) {\n const modelsData = await modelsResponse.json();\n setAvailableModels(modelService.transformModels(modelsData.models));\n }\n\n const toolsResponse = await userService.getUserTools(null);\n if (toolsResponse.ok) {\n const toolsData = await toolsResponse.json();\n setAvailableTools(toolsData.tools);\n }\n } catch (error) {\n console.error('Failed to load models or tools:', error);\n }\n };\n loadModelsAndTools();\n }, []);\n\n useEffect(() => {\n const loadAgentDetails = async () => {\n if (!agentId) return;\n try {\n const response = await userService.getAgent(agentId, null);\n if (!response.ok) throw new Error('Failed to fetch agent');\n const agent = await response.json();\n if (agent.agent_type === 'workflow' && agent.workflow) {\n setWorkflowId(agent.workflow);\n setCurrentAgentId(agent.id);\n setWorkflowName(agent.name);\n setWorkflowDescription(agent.description || '');\n }\n } catch (error) {\n console.error('Failed to load agent:', error);\n }\n };\n loadAgentDetails();\n }, [agentId]);\n\n useEffect(() => {\n const loadWorkflow = async () => {\n if (!workflowId) return;\n try {\n const response = await userService.getWorkflow(workflowId, null);\n if (!response.ok) throw new Error('Failed to fetch workflow');\n const responseData = await response.json();\n const { workflow, nodes: apiNodes, edges: apiEdges } = responseData;\n setWorkflowName(workflow.name);\n setWorkflowDescription(workflow.description || '');\n setNodes(\n apiNodes.map((n: WorkflowNode) => {\n const nodeData: Record = {\n title: n.title,\n label: n.title,\n };\n if (n.type === 'agent' && n.data) {\n nodeData.config = n.data;\n } else if (n.data) {\n Object.assign(nodeData, n.data);\n }\n return {\n id: n.id,\n type: n.type,\n position: n.position,\n data: nodeData,\n };\n }),\n );\n setEdges(\n apiEdges.map(\n (e: {\n id: string;\n source: string;\n target: string;\n sourceHandle?: string;\n targetHandle?: string;\n }) => ({\n id: e.id,\n source: e.source,\n target: e.target,\n sourceHandle: e.sourceHandle,\n targetHandle: e.targetHandle,\n }),\n ),\n );\n // Fit view after loading with slight delay to ensure nodes are rendered\n setTimeout(() => {\n reactFlowInstance.fitView({\n padding: 0.2,\n maxZoom: 0.8,\n duration: 300,\n });\n }, 100);\n } catch (error) {\n console.error('Failed to load workflow:', error);\n }\n };\n loadWorkflow();\n }, [workflowId, reactFlowInstance]);\n\n const validateWorkflow = useCallback((): string[] => {\n const errors: string[] = [];\n\n if (!workflowName.trim()) {\n errors.push('Workflow name is required');\n }\n\n const startNodes = nodes.filter((n) => n.type === 'start');\n if (startNodes.length !== 1) {\n errors.push('Workflow must have exactly one start node');\n }\n\n const endNodes = nodes.filter((n) => n.type === 'end');\n if (endNodes.length === 0) {\n errors.push('Workflow must have at least one end node');\n }\n\n const agentNodes = nodes.filter((n) => n.type === 'agent');\n if (agentNodes.length === 0) {\n errors.push('Workflow must have at least one AI agent node');\n }\n\n agentNodes.forEach((node) => {\n const config = node.data?.config;\n if (!config?.llm_name && !config?.model_id) {\n errors.push(\n `Agent \"${node.data?.title || node.id}\" must have a model selected`,\n );\n }\n });\n\n if (startNodes.length === 1) {\n const startId = startNodes[0].id;\n const hasOutgoing = edges.some((e) => e.source === startId);\n if (!hasOutgoing) {\n errors.push('Start node must be connected to another node');\n }\n }\n\n endNodes.forEach((endNode) => {\n const hasIncoming = edges.some((e) => e.target === endNode.id);\n if (!hasIncoming) {\n errors.push(\n `End node \"${endNode.id}\" must have an incoming connection`,\n );\n }\n });\n\n const nodeIds = new Set(nodes.map((n) => n.id));\n edges.forEach((edge) => {\n if (!nodeIds.has(edge.source)) {\n errors.push(`Edge references non-existent source node`);\n }\n if (!nodeIds.has(edge.target)) {\n errors.push(`Edge references non-existent target node`);\n }\n });\n\n return errors;\n }, [workflowName, nodes, edges]);\n\n const handlePublish = useCallback(async () => {\n setPublishErrors([]);\n setErrorContext('publish');\n\n const validationErrors = validateWorkflow();\n if (validationErrors.length > 0) {\n setPublishErrors(validationErrors);\n return;\n }\n\n setIsPublishing(true);\n try {\n const workflowPayload = {\n name: workflowName,\n description: workflowDescription,\n nodes: nodes.map((n) => ({\n id: n.id,\n type: n.type as 'start' | 'end' | 'agent' | 'note' | 'state',\n title: n.data.title || n.data.label || n.type,\n position: n.position,\n data: n.type === 'agent' ? n.data.config : n.data,\n })),\n edges: edges.map((e) => ({\n id: e.id,\n source: e.source,\n target: e.target,\n sourceHandle: e.sourceHandle || undefined,\n targetHandle: e.targetHandle || undefined,\n })),\n };\n\n let savedWorkflowId = workflowId;\n if (workflowId) {\n const updateResponse = await userService.updateWorkflow(\n workflowId,\n workflowPayload,\n null,\n );\n if (!updateResponse.ok) {\n const errorData = await updateResponse.json().catch(() => ({}));\n throw new Error(errorData.message || 'Failed to update workflow');\n }\n\n if (currentAgentId) {\n const agentFormData = new FormData();\n agentFormData.append('name', workflowName);\n agentFormData.append(\n 'description',\n workflowDescription || `Workflow agent: ${workflowName}`,\n );\n agentFormData.append('status', 'published');\n const agentUpdateResponse = await userService.updateAgent(\n currentAgentId,\n agentFormData,\n null,\n );\n if (!agentUpdateResponse.ok) {\n throw new Error('Failed to update agent');\n }\n }\n } else {\n const createResponse = await userService.createWorkflow(\n workflowPayload,\n null,\n );\n if (!createResponse.ok) {\n const errorData = await createResponse.json().catch(() => ({}));\n const backendErrors = errorData.errors || [];\n if (backendErrors.length > 0) {\n setPublishErrors(backendErrors);\n return;\n }\n throw new Error(errorData.message || 'Failed to create workflow');\n }\n const responseData = await createResponse.json();\n savedWorkflowId = responseData.id;\n\n const agentFormData = new FormData();\n agentFormData.append('name', workflowName);\n agentFormData.append(\n 'description',\n workflowDescription || `Workflow agent: ${workflowName}`,\n );\n agentFormData.append('agent_type', 'workflow');\n agentFormData.append('status', 'published');\n agentFormData.append('workflow', savedWorkflowId || '');\n if (folderId) agentFormData.append('folder_id', folderId);\n\n const agentResponse = await userService.createAgent(\n agentFormData,\n null,\n );\n if (!agentResponse.ok) throw new Error('Failed to create agent');\n }\n\n navigate(folderId ? `/agents?folder=${folderId}` : '/agents');\n } catch (error) {\n console.error('Failed to publish workflow:', error);\n setPublishErrors([\n error instanceof Error ? error.message : 'Failed to publish workflow',\n ]);\n } finally {\n setIsPublishing(false);\n }\n }, [\n workflowName,\n workflowDescription,\n nodes,\n edges,\n navigate,\n folderId,\n workflowId,\n currentAgentId,\n validateWorkflow,\n ]);\n\n return (\n
\n
\n
\n navigate('/agents')}\n className=\"rounded-full border p-3 text-sm text-gray-400 dark:border-0 dark:bg-[#28292D] dark:text-gray-500 dark:hover:bg-[#2E2F34]\"\n >\n \"left-arrow\"\n \n
\n setShowWorkflowSettings(!showWorkflowSettings)}\n className=\"flex items-center gap-2 text-left\"\n >\n
\n
\n {workflowName || 'New Workflow'}\n
\n {workflowDescription && (\n
\n {workflowDescription.length > 50\n ? `${workflowDescription.slice(0, 50)}...`\n : workflowDescription}\n
\n )}\n
\n \n \n {showWorkflowSettings && (\n \n
\n \n setWorkflowName(e.target.value)}\n className=\"focus:ring-purple-30 w-full rounded-lg border border-[#E5E5E5] bg-white px-3 py-2 text-sm outline-none focus:ring-2 dark:border-[#3A3A3A] dark:bg-[#2C2C2C] dark:text-white\"\n placeholder=\"Enter workflow name\"\n />\n
\n
\n \n setWorkflowDescription(e.target.value)}\n className=\"focus:ring-purple-30 w-full rounded-lg border border-[#E5E5E5] bg-white px-3 py-2 text-sm outline-none focus:ring-2 dark:border-[#3A3A3A] dark:bg-[#2C2C2C] dark:text-white\"\n rows={3}\n placeholder=\"Describe what this workflow does\"\n />\n
\n setShowWorkflowSettings(false)}\n className=\"bg-violets-are-blue hover:bg-purple-30 w-full rounded-lg px-3 py-2 text-sm font-medium text-white\"\n >\n Done\n \n
\n )}\n
\n
\n
\n {\n const validationErrors = validateWorkflow();\n if (validationErrors.length > 0) {\n setErrorContext('preview');\n setPublishErrors(validationErrors);\n return;\n }\n setShowPreview(true);\n }}\n className=\"flex items-center gap-2 rounded-full border border-gray-200 bg-white px-4 py-2 text-sm font-medium text-gray-700 transition-colors hover:bg-gray-50 dark:border-[#3A3A3A] dark:bg-[#2C2C2C] dark:text-gray-200 dark:hover:bg-[#383838]\"\n >\n \n Preview\n \n \n {isPublishing ? 'Publishing...' : 'Publish'}\n \n
\n
\n\n {publishErrors.length > 0 && (\n
\n \n \n \n {errorContext === 'preview'\n ? 'Unable to preview workflow'\n : 'Unable to publish workflow'}\n \n \n
    \n {publishErrors.map((error, index) => (\n
  • {error}
    • \n ))}\n
\n \n setPublishErrors([])}\n className=\"absolute top-4 right-4 text-red-700 hover:text-red-900 dark:text-red-300 dark:hover:text-red-100\"\n >\n \n \n \n
\n )}\n\n
\n
\n
\n

\n Core Nodes\n

\n
\n \n e.dataTransfer.setData('application/reactflow', 'agent')\n }\n >\n
\n \n
\n \n AI Agent\n \n
\n \n e.dataTransfer.setData('application/reactflow', 'end')\n }\n >\n
\n \n
\n \n End\n \n
\n \n e.dataTransfer.setData('application/reactflow', 'note')\n }\n >\n
\n \n
\n \n Note\n \n
\n
\n \n\n
\n

\n Logic & Data\n

\n
\n \n e.dataTransfer.setData('application/reactflow', 'state')\n }\n >\n
\n \n
\n
\n \n Set State\n \n \n Modify workflow variables\n \n
\n
\n
\n \n \n\n \n \n \n \n \n\n {showNodeConfig && selectedNode && (\n \n
\n

\n {selectedNode.type === 'start' && 'Start Node'}\n {selectedNode.type === 'end' && 'End Node'}\n {selectedNode.type === 'agent' && 'AI Agent'}\n {selectedNode.type === 'note' && 'Note'}\n {selectedNode.type === 'state' && 'Set State'}\n

\n setShowNodeConfig(false)}\n className=\"text-gray-400 transition-colors hover:text-gray-600 dark:hover:text-gray-200\"\n >\n \n \n
\n\n
\n
\n
\n
\n Node ID\n
\n
\n {selectedNode.id}\n
\n
\n\n {selectedNode.type !== 'start' &&\n selectedNode.type !== 'end' && (\n <>\n
\n \n \n handleUpdateNodeData({\n title: e.target.value,\n label: e.target.value,\n })\n }\n className=\"border-light-silver focus:ring-purple-30 w-full rounded-xl border bg-white px-3 py-2 text-sm transition-all outline-none focus:ring-2 dark:border-[#3A3A3A] dark:bg-[#2C2C2C] dark:text-white\"\n placeholder=\"Enter node title\"\n />\n
\n\n {selectedNode.type === 'agent' && (\n <>\n
\n \n \n handleUpdateNodeData({\n config: {\n ...(selectedNode.data.config || {}),\n agent_type: value,\n },\n })\n }\n >\n \n \n \n \n \n Classic\n \n ReAct\n \n \n
\n
\n \n {\n const selectedModel = availableModels.find(\n (m) => m.id === value,\n );\n handleUpdateNodeData({\n config: {\n ...(selectedNode.data.config || {}),\n model_id: value,\n llm_name: selectedModel?.provider || '',\n },\n });\n }}\n >\n \n \n \n \n {availableModels.map((model) => (\n \n {model.display_name} · {model.provider}\n \n ))}\n \n \n
\n
\n \n \n handleUpdateNodeData({\n config: {\n ...(selectedNode.data.config || {}),\n system_prompt: e.target.value,\n },\n })\n }\n className=\"border-light-silver focus:ring-purple-30 w-full rounded-xl border bg-white px-3 py-2 text-sm transition-all outline-none focus:ring-2 dark:border-[#3A3A3A] dark:bg-[#2C2C2C] dark:text-white\"\n rows={3}\n placeholder=\"System prompt for the agent\"\n />\n
\n
\n \n \n handleUpdateNodeData({\n config: {\n ...(selectedNode.data.config || {}),\n prompt_template: e.target.value,\n },\n })\n }\n className=\"border-light-silver focus:ring-purple-30 w-full rounded-xl border bg-white px-3 py-2 text-sm transition-all outline-none focus:ring-2 dark:border-[#3A3A3A] dark:bg-[#2C2C2C] dark:text-white\"\n rows={4}\n placeholder=\"Use {{variable}} for dynamic content\"\n />\n
\n
\n \n \n handleUpdateNodeData({\n config: {\n ...(selectedNode.data.config || {}),\n output_variable: e.target.value,\n },\n })\n }\n className=\"border-light-silver focus:ring-purple-30 w-full rounded-xl border bg-white px-3 py-2 text-sm transition-all outline-none focus:ring-2 dark:border-[#3A3A3A] dark:bg-[#2C2C2C] dark:text-white\"\n placeholder=\"Variable name for output\"\n />\n
\n
\n \n handleUpdateNodeData({\n config: {\n ...(selectedNode.data.config || {}),\n stream_to_user: e.target.checked,\n },\n })\n }\n className=\"h-4 w-4\"\n />\n \n Stream output to user\n \n
{' '}\n
\n \n ({\n value: tool.id,\n label: tool.displayName,\n }))}\n selected={selectedNode.data.config?.tools || []}\n onChange={(newTools) =>\n handleUpdateNodeData({\n config: {\n ...(selectedNode.data.config || {}),\n tools: newTools,\n },\n })\n }\n placeholder=\"Select tools...\"\n searchPlaceholder=\"Search tools...\"\n emptyText=\"No tools available\"\n />\n
\n \n )}\n\n {selectedNode.type === 'note' && (\n
\n \n \n handleUpdateNodeData({\n content: e.target.value,\n })\n }\n className=\"border-light-silver focus:ring-purple-30 w-full rounded-xl border bg-white px-3 py-2 text-sm transition-all outline-none focus:ring-2 dark:border-[#3A3A3A] dark:bg-[#2C2C2C] dark:text-white\"\n rows={4}\n placeholder=\"Enter note content\"\n />\n
\n )}\n\n {selectedNode.type === 'state' && (\n <>\n
\n \n \n handleUpdateNodeData({\n variable: e.target.value,\n })\n }\n className=\"border-light-silver focus:ring-purple-30 w-full rounded-xl border bg-white px-3 py-2 text-sm transition-all outline-none focus:ring-2 dark:border-[#3A3A3A] dark:bg-[#2C2C2C] dark:text-white\"\n placeholder=\"e.g. analysis_type\"\n />\n
\n
\n \n \n handleUpdateNodeData({\n value: e.target.value,\n })\n }\n className=\"border-light-silver focus:ring-purple-30 w-full rounded-xl border bg-white px-3 py-2 text-sm transition-all outline-none focus:ring-2 dark:border-[#3A3A3A] dark:bg-[#2C2C2C] dark:text-white\"\n placeholder=\"e.g. price_check\"\n />\n
\n \n )}\n \n )}\n
\n\n \n \n {selectedNode?.type === 'start'\n ? 'Cannot Delete Start Node'\n : 'Delete Node'}\n \n
\n \n )}\n \n \n\n {/* Preview Panel */}\n \n \n ({\n id: n.id,\n type: n.type as 'start' | 'end' | 'agent' | 'note' | 'state',\n title: n.data.title || n.data.label || n.type,\n position: n.position,\n data: n.type === 'agent' ? n.data.config : n.data,\n })),\n edges: edges.map((e) => ({\n id: e.id,\n source: e.source,\n target: e.target,\n sourceHandle: e.sourceHandle || undefined,\n targetHandle: e.targetHandle || undefined,\n })),\n }}\n />\n \n \n \n );\n}\n\nexport default function WorkflowBuilder() {\n return (\n \n \n \n );\n}\n" }, { "path": "frontend/src/agents/agentPreviewSlice.ts", "content": "import { createAsyncThunk, createSlice, PayloadAction } from '@reduxjs/toolkit';\n\nimport {\n handleFetchAnswer,\n handleFetchAnswerSteaming,\n} from '../conversation/conversationHandlers';\nimport {\n Answer,\n ConversationState,\n Query,\n Status,\n} from '../conversation/conversationModels';\nimport store from '../store';\nimport {\n clearAttachments,\n selectCompletedAttachments,\n} from '../upload/uploadSlice';\n\nconst initialState: ConversationState = {\n queries: [],\n status: 'idle',\n conversationId: null,\n};\n\nconst API_STREAMING = import.meta.env.VITE_API_STREAMING === 'true';\n\nlet abortController: AbortController | null = null;\nexport function handlePreviewAbort() {\n if (abortController) {\n abortController.abort();\n abortController = null;\n }\n}\n\nexport const fetchPreviewAnswer = createAsyncThunk<\n Answer,\n { question: string; indx?: number }\n>(\n 'agentPreview/fetchAnswer',\n async ({ question, indx }, { dispatch, getState }) => {\n if (abortController) abortController.abort();\n abortController = new AbortController();\n const { signal } = abortController;\n\n const state = getState() as RootState;\n const attachmentIds = selectCompletedAttachments(state)\n .filter((a) => a.id)\n .map((a) => a.id) as string[];\n\n if (attachmentIds.length > 0) {\n dispatch(clearAttachments());\n }\n\n if (state.preference) {\n const modelId =\n state.preference.selectedAgent?.default_model_id ||\n state.preference.selectedModel?.id;\n\n if (API_STREAMING) {\n await handleFetchAnswerSteaming(\n question,\n signal,\n state.preference.token,\n state.preference.selectedDocs,\n null, // No conversation ID for previews\n state.preference.prompt.id,\n state.preference.chunks,\n (event: MessageEvent) => {\n const data = JSON.parse(event.data);\n const targetIndex = indx ?? state.agentPreview.queries.length - 1;\n\n if (data.type === 'end') {\n dispatch(agentPreviewSlice.actions.setStatus('idle'));\n } else if (data.type === 'thought') {\n dispatch(\n updateThought({\n index: targetIndex,\n query: { thought: data.thought },\n }),\n );\n } else if (data.type === 'source') {\n dispatch(\n updateStreamingSource({\n index: targetIndex,\n query: { sources: data.source ?? [] },\n }),\n );\n } else if (data.type === 'tool_call') {\n dispatch(\n updateToolCall({\n index: targetIndex,\n tool_call: data.data,\n }),\n );\n } else if (data.type === 'error') {\n dispatch(agentPreviewSlice.actions.setStatus('failed'));\n dispatch(\n agentPreviewSlice.actions.raiseError({\n index: targetIndex,\n message: data.error,\n }),\n );\n } else if (data.type === 'structured_answer') {\n dispatch(\n updateStreamingQuery({\n index: targetIndex,\n query: {\n response: data.answer,\n structured: data.structured,\n schema: data.schema,\n },\n }),\n );\n } else {\n dispatch(\n updateStreamingQuery({\n index: targetIndex,\n query: { response: data.answer },\n }),\n );\n }\n },\n indx,\n state.preference.selectedAgent?.id,\n attachmentIds,\n false,\n modelId,\n );\n } else {\n const answer = await handleFetchAnswer(\n question,\n signal,\n state.preference.token,\n state.preference.selectedDocs,\n null,\n state.preference.prompt.id,\n state.preference.chunks,\n state.preference.selectedAgent?.id,\n attachmentIds,\n false,\n modelId,\n );\n\n if (answer) {\n const sourcesPrepped = answer.sources.map(\n (source: { title: string }) => {\n if (source && source.title) {\n const titleParts = source.title.split('/');\n return {\n ...source,\n title: titleParts[titleParts.length - 1],\n };\n }\n return source;\n },\n );\n\n const targetIndex = indx ?? state.agentPreview.queries.length - 1;\n\n dispatch(\n updateQuery({\n index: targetIndex,\n query: {\n response: answer.answer,\n thought: answer.thought,\n sources: sourcesPrepped,\n tool_calls: answer.toolCalls,\n },\n }),\n );\n dispatch(agentPreviewSlice.actions.setStatus('idle'));\n }\n }\n }\n\n return {\n conversationId: null,\n title: null,\n answer: '',\n query: question,\n result: '',\n thought: '',\n sources: [],\n tool_calls: [],\n };\n },\n);\n\nexport const agentPreviewSlice = createSlice({\n name: 'agentPreview',\n initialState,\n reducers: {\n addQuery(state, action: PayloadAction) {\n state.queries.push(action.payload);\n },\n resendQuery(\n state,\n action: PayloadAction<{ index: number; prompt: string }>,\n ) {\n const { index, prompt } = action.payload;\n if (index < 0 || index >= state.queries.length) return;\n\n state.queries.splice(index + 1);\n state.queries[index].prompt = prompt;\n delete state.queries[index].response;\n delete state.queries[index].thought;\n delete state.queries[index].sources;\n delete state.queries[index].tool_calls;\n delete state.queries[index].error;\n delete state.queries[index].structured;\n delete state.queries[index].schema;\n delete state.queries[index].feedback;\n },\n updateStreamingQuery(\n state,\n action: PayloadAction<{\n index: number;\n query: Partial;\n }>,\n ) {\n const { index, query } = action.payload;\n if (state.status === 'idle') return;\n\n if (query.response != undefined) {\n state.queries[index].response =\n (state.queries[index].response || '') + query.response;\n }\n\n if (query.structured !== undefined) {\n state.queries[index].structured = query.structured;\n }\n\n if (query.schema !== undefined) {\n state.queries[index].schema = query.schema;\n }\n },\n updateThought(\n state,\n action: PayloadAction<{\n index: number;\n query: Partial;\n }>,\n ) {\n const { index, query } = action.payload;\n if (query.thought != undefined) {\n state.queries[index].thought =\n (state.queries[index].thought || '') + query.thought;\n }\n },\n updateStreamingSource(\n state,\n action: PayloadAction<{\n index: number;\n query: Partial;\n }>,\n ) {\n const { index, query } = action.payload;\n if (!state.queries[index].sources) {\n state.queries[index].sources = query?.sources;\n } else if (query.sources) {\n state.queries[index].sources!.push(...query.sources);\n }\n },\n updateToolCall(state, action) {\n const { index, tool_call } = action.payload;\n\n if (!state.queries[index].tool_calls) {\n state.queries[index].tool_calls = [];\n }\n\n const existingIndex = state.queries[index].tool_calls.findIndex(\n (call) => call.call_id === tool_call.call_id,\n );\n\n if (existingIndex !== -1) {\n const existingCall = state.queries[index].tool_calls[existingIndex];\n state.queries[index].tool_calls[existingIndex] = {\n ...existingCall,\n ...tool_call,\n };\n } else state.queries[index].tool_calls.push(tool_call);\n },\n updateQuery(\n state,\n action: PayloadAction<{ index: number; query: Partial }>,\n ) {\n const { index, query } = action.payload;\n state.queries[index] = {\n ...state.queries[index],\n ...query,\n };\n },\n setStatus(state, action: PayloadAction) {\n state.status = action.payload;\n },\n raiseError(\n state,\n action: PayloadAction<{\n index: number;\n message: string;\n }>,\n ) {\n const { index, message } = action.payload;\n state.queries[index].error = message;\n },\n resetPreview: (state) => {\n state.queries = initialState.queries;\n state.status = initialState.status;\n state.conversationId = initialState.conversationId;\n handlePreviewAbort();\n },\n },\n extraReducers(builder) {\n builder\n .addCase(fetchPreviewAnswer.pending, (state) => {\n state.status = 'loading';\n })\n .addCase(fetchPreviewAnswer.rejected, (state, action) => {\n if (action.meta.aborted) {\n state.status = 'idle';\n return;\n }\n state.status = 'failed';\n if (state.queries.length > 0) {\n state.queries[state.queries.length - 1].error =\n 'Something went wrong';\n }\n });\n },\n});\n\ntype RootState = ReturnType;\n\nexport const selectPreviewQueries = (state: RootState) =>\n state.agentPreview.queries;\nexport const selectPreviewStatus = (state: RootState) =>\n state.agentPreview.status;\n\nexport const {\n addQuery,\n updateQuery,\n resendQuery,\n updateStreamingQuery,\n updateThought,\n updateStreamingSource,\n updateToolCall,\n setStatus,\n raiseError,\n resetPreview,\n} = agentPreviewSlice.actions;\n\nexport default agentPreviewSlice.reducer;\n" }, { "path": "frontend/src/agents/agents.config.ts", "content": "import userService from '../api/services/userService';\nimport {\n selectAgents,\n selectSharedAgents,\n selectTemplateAgents,\n setAgents,\n setSharedAgents,\n setTemplateAgents,\n} from '../preferences/preferenceSlice';\n\nexport type AgentSectionId = 'template' | 'user' | 'shared';\n\nexport const agentSectionsConfig = [\n {\n id: 'template' as const,\n title: 'By DocsGPT',\n description: 'Agents provided by DocsGPT',\n showNewAgentButton: false,\n emptyStateDescription: 'No template agents found.',\n fetchAgents: (token: string | null) => userService.getTemplateAgents(token),\n selectData: selectTemplateAgents,\n updateAction: setTemplateAgents,\n },\n {\n id: 'user' as const,\n title: 'By me',\n description: 'Agents created or published by you',\n showNewAgentButton: true,\n emptyStateDescription: 'You don’t have any created agents yet.',\n fetchAgents: (token: string | null) => userService.getAgents(token),\n selectData: selectAgents,\n updateAction: setAgents,\n },\n {\n id: 'shared' as const,\n title: 'Shared with me',\n description: 'Agents imported by using a public link',\n showNewAgentButton: false,\n emptyStateDescription: 'No shared agents found.',\n fetchAgents: (token: string | null) => userService.getSharedAgents(token),\n selectData: selectSharedAgents,\n updateAction: setSharedAgents,\n },\n];" }, { "path": "frontend/src/agents/components/AgentTypeModal.tsx", "content": "import { Bot, Workflow, X } from 'lucide-react';\nimport { useNavigate } from 'react-router-dom';\n\ninterface AgentTypeModalProps {\n isOpen: boolean;\n onClose: () => void;\n folderId?: string | null;\n}\n\nexport default function AgentTypeModal({\n isOpen,\n onClose,\n folderId,\n}: AgentTypeModalProps) {\n const navigate = useNavigate();\n\n if (!isOpen) return null;\n\n const handleSelect = (type: 'normal' | 'workflow') => {\n if (type === 'workflow') {\n navigate(\n `/agents/workflow/new${folderId ? `?folder_id=${folderId}` : ''}`,\n );\n } else {\n navigate(`/agents/new${folderId ? `?folder_id=${folderId}` : ''}`);\n }\n onClose();\n };\n\n return (\n \n e.stopPropagation()}\n >\n \n \n \n\n

\n Create New Agent\n

\n

\n Choose the type of agent you want to create\n

\n\n
\n handleSelect('normal')}\n className=\"hover:border-purple-30 hover:bg-purple-30/5 dark:hover:border-purple-30 dark:hover:bg-purple-30/10 group flex items-start gap-5 rounded-xl border-2 border-gray-200 p-5 text-left transition-all dark:border-[#2E2F34]\"\n >\n
\n \n
\n
\n

\n Classic Agent\n

\n

\n Create a standard AI agent with a single model, tools, and\n knowledge sources\n

\n
\n \n\n handleSelect('workflow')}\n className=\"hover:border-violets-are-blue hover:bg-violets-are-blue/5 dark:hover:border-violets-are-blue dark:hover:bg-violets-are-blue/10 group flex items-start gap-5 rounded-xl border-2 border-gray-200 p-5 text-left transition-all dark:border-[#2E2F34]\"\n >\n
\n \n
\n
\n

\n Workflow Agent\n

\n

\n Design complex multi-step workflows with different models,\n conditional logic, and state management\n

\n
\n \n
\n \n \n );\n}\n" }, { "path": "frontend/src/agents/hooks/useAgentSearch.ts", "content": "import { useCallback, useMemo, useState } from 'react';\nimport { useSelector } from 'react-redux';\n\nimport {\n selectAgents,\n selectSharedAgents,\n selectTemplateAgents,\n} from '../../preferences/preferenceSlice';\nimport { AgentSectionId } from '../agents.config';\nimport { Agent } from '../types';\n\nexport type AgentFilterTab = 'all' | AgentSectionId;\n\nexport type AgentsBySection = Record;\n\ninterface UseAgentSearchResult {\n searchQuery: string;\n setSearchQuery: (query: string) => void;\n activeFilter: AgentFilterTab;\n setActiveFilter: (filter: AgentFilterTab) => void;\n filteredAgentsBySection: AgentsBySection;\n totalAgentsBySection: Record;\n hasAnyAgents: boolean;\n hasFilteredResults: boolean;\n isDataLoaded: Record;\n}\n\nconst filterAgentsByQuery = (\n agents: Agent[] | null,\n query: string,\n): Agent[] => {\n if (!agents) return [];\n if (!query.trim()) return agents;\n\n const normalizedQuery = query.toLowerCase().trim();\n return agents.filter(\n (agent) =>\n agent.name.toLowerCase().includes(normalizedQuery) ||\n agent.description?.toLowerCase().includes(normalizedQuery),\n );\n};\n\nexport function useAgentSearch(): UseAgentSearchResult {\n const [searchQuery, setSearchQuery] = useState('');\n const [activeFilter, setActiveFilter] = useState('all');\n\n const templateAgents = useSelector(selectTemplateAgents);\n const userAgents = useSelector(selectAgents);\n const sharedAgents = useSelector(selectSharedAgents);\n\n const handleSearchChange = useCallback((query: string) => {\n setSearchQuery(query);\n }, []);\n\n const handleFilterChange = useCallback((filter: AgentFilterTab) => {\n setActiveFilter(filter);\n }, []);\n\n const isDataLoaded = useMemo(\n (): Record => ({\n template: templateAgents !== null,\n user: userAgents !== null,\n shared: sharedAgents !== null,\n }),\n [templateAgents, userAgents, sharedAgents],\n );\n\n const totalAgentsBySection = useMemo(\n (): Record => ({\n template: templateAgents?.length ?? 0,\n user: userAgents?.length ?? 0,\n shared: sharedAgents?.length ?? 0,\n }),\n [templateAgents, userAgents, sharedAgents],\n );\n\n const filteredAgentsBySection = useMemo((): AgentsBySection => {\n const filtered = {\n template: filterAgentsByQuery(templateAgents, searchQuery),\n user: filterAgentsByQuery(userAgents, searchQuery),\n shared: filterAgentsByQuery(sharedAgents, searchQuery),\n };\n\n if (activeFilter === 'all') {\n return filtered;\n }\n\n return {\n template: activeFilter === 'template' ? filtered.template : [],\n user: activeFilter === 'user' ? filtered.user : [],\n shared: activeFilter === 'shared' ? filtered.shared : [],\n };\n }, [templateAgents, userAgents, sharedAgents, searchQuery, activeFilter]);\n\n const hasAnyAgents = useMemo(() => {\n return (\n totalAgentsBySection.template > 0 ||\n totalAgentsBySection.user > 0 ||\n totalAgentsBySection.shared > 0\n );\n }, [totalAgentsBySection]);\n\n const hasFilteredResults = useMemo(() => {\n return (\n filteredAgentsBySection.template.length > 0 ||\n filteredAgentsBySection.user.length > 0 ||\n filteredAgentsBySection.shared.length > 0\n );\n }, [filteredAgentsBySection]);\n\n return {\n searchQuery,\n setSearchQuery: handleSearchChange,\n activeFilter,\n setActiveFilter: handleFilterChange,\n filteredAgentsBySection,\n totalAgentsBySection,\n hasAnyAgents,\n hasFilteredResults,\n isDataLoaded,\n };\n}\n" }, { "path": "frontend/src/agents/hooks/useAgentsFetch.ts", "content": "import { useCallback, useEffect, useState } from 'react';\nimport { useDispatch, useSelector } from 'react-redux';\n\nimport userService from '../../api/services/userService';\nimport {\n selectToken,\n setAgentFolders,\n setAgents,\n setSharedAgents,\n setTemplateAgents,\n} from '../../preferences/preferenceSlice';\nimport { AgentSectionId } from '../agents.config';\n\ninterface UseAgentsFetchResult {\n isLoading: Record;\n isAllLoaded: boolean;\n refetchFolders: () => Promise;\n refetchUserAgents: () => Promise;\n}\n\nexport function useAgentsFetch(): UseAgentsFetchResult {\n const dispatch = useDispatch();\n const token = useSelector(selectToken);\n\n const [isLoading, setIsLoading] = useState>({\n template: true,\n user: true,\n shared: true,\n });\n\n const fetchTemplateAgents = useCallback(async () => {\n try {\n const response = await userService.getTemplateAgents(token);\n if (!response.ok) throw new Error('Failed to fetch template agents');\n const data = await response.json();\n dispatch(setTemplateAgents(data));\n } catch (error) {\n dispatch(setTemplateAgents([]));\n } finally {\n setIsLoading((prev) => ({ ...prev, template: false }));\n }\n }, [token, dispatch]);\n\n const fetchUserAgents = useCallback(async () => {\n try {\n const response = await userService.getAgents(token);\n if (!response.ok) throw new Error('Failed to fetch user agents');\n const data = await response.json();\n dispatch(setAgents(data));\n } catch (error) {\n dispatch(setAgents([]));\n } finally {\n setIsLoading((prev) => ({ ...prev, user: false }));\n }\n }, [token, dispatch]);\n\n const fetchSharedAgents = useCallback(async () => {\n try {\n const response = await userService.getSharedAgents(token);\n if (!response.ok) throw new Error('Failed to fetch shared agents');\n const data = await response.json();\n dispatch(setSharedAgents(data));\n } catch (error) {\n dispatch(setSharedAgents([]));\n } finally {\n setIsLoading((prev) => ({ ...prev, shared: false }));\n }\n }, [token, dispatch]);\n\n const fetchFolders = useCallback(async () => {\n try {\n const response = await userService.getAgentFolders(token);\n if (!response.ok) throw new Error('Failed to fetch folders');\n const data = await response.json();\n dispatch(setAgentFolders(data.folders || []));\n } catch (error) {\n dispatch(setAgentFolders([]));\n }\n }, [token, dispatch]);\n\n useEffect(() => {\n setIsLoading({ template: true, user: true, shared: true });\n Promise.all([\n fetchTemplateAgents(),\n fetchUserAgents(),\n fetchSharedAgents(),\n fetchFolders(),\n ]);\n }, [fetchTemplateAgents, fetchUserAgents, fetchSharedAgents, fetchFolders]);\n\n const isAllLoaded =\n !isLoading.template && !isLoading.user && !isLoading.shared;\n\n return {\n isLoading,\n isAllLoaded,\n refetchFolders: fetchFolders,\n refetchUserAgents: fetchUserAgents,\n };\n}\n" }, { "path": "frontend/src/agents/index.tsx", "content": "import { Route, Routes } from 'react-router-dom';\n\nimport AgentLogs from './AgentLogs';\nimport AgentsList from './AgentsList';\nimport NewAgent from './NewAgent';\nimport SharedAgent from './SharedAgent';\nimport WorkflowBuilder from './workflow/WorkflowBuilder';\n\nexport default function Agents() {\n return (\n \n } />\n } />\n } />\n } />\n } />\n } />\n } />\n \n );\n}" }, { "path": "frontend/src/agents/types/index.ts", "content": "export type ToolSummary = {\n id: string;\n name: string;\n display_name: string;\n};\n\nexport type Agent = {\n id?: string;\n name: string;\n description: string;\n image: string;\n source: string;\n sources?: string[];\n chunks: string;\n retriever: string;\n prompt_id: string;\n tools: string[];\n tool_details?: ToolSummary[];\n agent_type: string;\n status: string;\n key?: string;\n incoming_webhook_token?: string;\n pinned?: boolean;\n shared?: boolean;\n shared_token?: string;\n shared_metadata?: any;\n created_at?: string;\n updated_at?: string;\n last_used_at?: string;\n json_schema?: object;\n limited_token_mode?: boolean;\n token_limit?: number;\n limited_request_mode?: boolean;\n request_limit?: number;\n models?: string[];\n default_model_id?: string;\n folder_id?: string;\n workflow?: string;\n};\n\nexport type AgentFolder = {\n id: string;\n name: string;\n parent_id?: string | null;\n created_at?: string;\n updated_at?: string;\n};\n\nexport * from './workflow';\n" }, { "path": "frontend/src/agents/types/workflow.ts", "content": "export type NodeType = 'start' | 'end' | 'agent' | 'note' | 'state' | 'condition';\n\nexport interface ConditionCase {\n name?: string;\n expression: string;\n sourceHandle: string;\n}\n\nexport interface ConditionNodeConfig {\n mode: 'simple' | 'advanced';\n cases: ConditionCase[];\n}\n\nexport interface StateOperationConfig {\n expression: string;\n target_variable: string;\n}\n\nexport interface WorkflowEdge {\n id: string;\n source: string;\n target: string;\n sourceHandle?: string;\n targetHandle?: string;\n}\n\nexport interface WorkflowNode {\n id: string;\n type: NodeType;\n title: string;\n description?: string;\n position: { x: number; y: number };\n data: Record;\n}\n\nexport interface WorkflowDefinition {\n id?: string;\n name: string;\n nodes: WorkflowNode[];\n edges: WorkflowEdge[];\n created_at?: string;\n updated_at?: string;\n}\n\nexport type ExecutionStatus = 'pending' | 'running' | 'completed' | 'failed';\n\nexport interface NodeExecutionLog {\n node_id: string;\n status: ExecutionStatus;\n input_state: Record;\n output_state: Record;\n error?: string;\n started_at: number;\n completed_at?: number;\n logs: string[];\n}\n\nexport interface WorkflowRun {\n workflow_id: string;\n status: ExecutionStatus;\n state: Record;\n node_logs: NodeExecutionLog[];\n created_at: number;\n completed_at?: number;\n}\n" }, { "path": "frontend/src/agents/workflow/WorkflowBuilder.tsx", "content": "import 'reactflow/dist/style.css';\n\nimport {\n AlertCircle,\n ChartColumn,\n Bot,\n Database,\n Flag,\n GitBranch,\n Loader2,\n Link,\n Pencil,\n Play,\n Plus,\n Settings2,\n StickyNote,\n Trash2,\n X,\n} from 'lucide-react';\nimport { useCallback, useEffect, useMemo, useRef, useState } from 'react';\nimport { useSelector } from 'react-redux';\nimport { useNavigate, useParams, useSearchParams } from 'react-router-dom';\nimport ReactFlow, {\n addEdge,\n applyEdgeChanges,\n applyNodeChanges,\n Background,\n Connection,\n Controls,\n Edge,\n EdgeChange,\n Node,\n NodeChange,\n NodeTypes,\n ReactFlowProvider,\n useReactFlow,\n} from 'reactflow';\n\nimport { Alert, AlertDescription, AlertTitle } from '@/components/ui/alert';\nimport { MultiSelect } from '@/components/ui/multi-select';\nimport {\n Select,\n SelectContent,\n SelectItem,\n SelectTrigger,\n SelectValue,\n} from '@/components/ui/select';\nimport { Sheet, SheetContent } from '@/components/ui/sheet';\n\nimport modelService from '../../api/services/modelService';\nimport userService from '../../api/services/userService';\nimport ArrowLeft from '../../assets/arrow-left.svg';\nimport { FileUpload } from '../../components/FileUpload';\nimport AgentDetailsModal from '../../modals/AgentDetailsModal';\nimport ConfirmationModal from '../../modals/ConfirmationModal';\nimport { ActiveState } from '../../models/misc';\nimport { selectToken } from '../../preferences/preferenceSlice';\nimport { Agent } from '../types';\nimport { ConditionCase, WorkflowNode } from '../types/workflow';\nimport MobileBlocker from './components/MobileBlocker';\nimport PromptTextArea from './components/PromptTextArea';\nimport {\n AgentNode,\n ConditionNode,\n EndNode,\n NoteNode,\n SetStateNode,\n StartNode,\n} from './nodes';\nimport WorkflowPreview from './WorkflowPreview';\n\nimport type { Model } from '../../models/types';\n\nconst PRIMARY_ACTION_SPINNER_DELAY_MS = 180;\n\ninterface AgentNodeConfig {\n agent_type: 'classic' | 'react';\n llm_name?: string;\n model_id?: string;\n system_prompt: string;\n prompt_template: string;\n output_variable?: string;\n stream_to_user: boolean;\n sources: string[];\n tools: string[];\n chunks?: string;\n retriever?: string;\n json_schema?: Record;\n}\n\ninterface UserTool {\n id: string;\n name: string;\n displayName: string;\n}\n\nfunction validateJsonSchemaConfig(schema: unknown): string | null {\n if (schema === undefined || schema === null) return null;\n if (typeof schema !== 'object' || Array.isArray(schema)) {\n return 'must be a valid JSON object';\n }\n\n const schemaObject = schema as Record;\n if (!('schema' in schemaObject) && !('type' in schemaObject)) {\n return 'must include either a \"type\" or \"schema\" field';\n }\n\n return null;\n}\n\nfunction createEmptyWorkflowAgent(): Agent {\n return {\n id: '',\n name: '',\n description: '',\n image: '',\n source: '',\n chunks: '2',\n retriever: '',\n prompt_id: '',\n tools: [],\n agent_type: 'workflow',\n status: 'published',\n };\n}\n\nfunction canReachEnd(\n nodeId: string,\n edges: Edge[],\n nodeIds: Set,\n endIds: Set,\n visited: Set = new Set(),\n): boolean {\n if (endIds.has(nodeId)) return true;\n if (visited.has(nodeId) || !nodeIds.has(nodeId)) return false;\n visited.add(nodeId);\n return edges\n .filter((e) => e.source === nodeId)\n .some((e) => canReachEnd(e.target, edges, nodeIds, endIds, visited));\n}\n\nfunction parseSimpleCel(expression: string): {\n variable: string;\n operator: string;\n value: string;\n} {\n const trimmedExpression = expression.trim();\n\n let match = trimmedExpression.match(\n /^(\\w+)\\.(contains|startsWith)\\([\"'](.*)[\"']\\)$/,\n );\n if (match) return { variable: match[1], operator: match[2], value: match[3] };\n\n match = trimmedExpression.match(/^(\\w+)\\.(contains|startsWith)\\((.*)\\)$/);\n if (match) {\n const rawValue = match[3].trim();\n const unquotedValue = rawValue.replace(/^[\"'](.*)[\"']$/, '$1');\n return {\n variable: match[1],\n operator: match[2],\n value: unquotedValue,\n };\n }\n\n match = trimmedExpression.match(/^(contains|startsWith)\\([\"'](.*)[\"']\\)$/);\n if (match) return { variable: '', operator: match[1], value: match[2] };\n\n match = trimmedExpression.match(/^(contains|startsWith)\\((.*)\\)$/);\n if (match) {\n const rawValue = match[2].trim();\n const unquotedValue = rawValue.replace(/^[\"'](.*)[\"']$/, '$1');\n return { variable: '', operator: match[1], value: unquotedValue };\n }\n\n match = trimmedExpression.match(/^(\\w+)\\s*(==|!=|>=|<=|>|<)\\s*[\"'](.*)[\"']$/);\n if (match) return { variable: match[1], operator: match[2], value: match[3] };\n\n match = trimmedExpression.match(/^(==|!=|>=|<=|>|<)\\s*[\"'](.*)[\"']$/);\n if (match) return { variable: '', operator: match[1], value: match[2] };\n\n match = trimmedExpression.match(/^(\\w+)\\s*(==|!=|>=|<=|>|<)\\s*(.*)$/);\n if (match) return { variable: match[1], operator: match[2], value: match[3] };\n\n match = trimmedExpression.match(/^(==|!=|>=|<=|>|<)\\s*(.*)$/);\n if (match) return { variable: '', operator: match[1], value: match[2] };\n\n return { variable: '', operator: '==', value: '' };\n}\n\nfunction buildSimpleCel(\n variable: string,\n operator: string,\n value: string,\n): string {\n const trimmedValue = value.trim();\n const isNumeric = trimmedValue !== '' && !isNaN(Number(trimmedValue));\n const isBool = trimmedValue === 'true' || trimmedValue === 'false';\n const literalValue =\n isNumeric || isBool ? trimmedValue : JSON.stringify(value);\n const stringValue = JSON.stringify(value);\n if (operator === 'contains') {\n return variable\n ? `${variable}.contains(${stringValue})`\n : `contains(${stringValue})`;\n }\n if (operator === 'startsWith') {\n return variable\n ? `${variable}.startsWith(${stringValue})`\n : `startsWith(${stringValue})`;\n }\n if (!variable) return `${operator} ${literalValue}`;\n return `${variable} ${operator} ${literalValue}`;\n}\n\nfunction normalizeConditionCases(cases: ConditionCase[]): ConditionCase[] {\n const usedHandles = new Set();\n let nextIndex = 0;\n\n return cases.map((conditionCase) => {\n const candidate = (conditionCase.sourceHandle || '').trim();\n if (candidate && !usedHandles.has(candidate)) {\n usedHandles.add(candidate);\n const match = candidate.match(/^case_(\\d+)$/);\n if (match) {\n nextIndex = Math.max(nextIndex, Number(match[1]) + 1);\n }\n return conditionCase;\n }\n\n while (usedHandles.has(`case_${nextIndex}`)) {\n nextIndex += 1;\n }\n const generatedHandle = `case_${nextIndex}`;\n usedHandles.add(generatedHandle);\n nextIndex += 1;\n\n return {\n ...conditionCase,\n sourceHandle: generatedHandle,\n };\n });\n}\n\nfunction getNextConditionHandle(cases: ConditionCase[]): string {\n const usedHandles = new Set(\n cases.map((conditionCase) => conditionCase.sourceHandle).filter(Boolean),\n );\n const usedIndices = Array.from(usedHandles)\n .map((handle) => handle.match(/^case_(\\d+)$/))\n .filter((match): match is RegExpMatchArray => Boolean(match))\n .map((match) => Number(match[1]));\n\n let nextIndex = usedIndices.length > 0 ? Math.max(...usedIndices) + 1 : 0;\n while (usedHandles.has(`case_${nextIndex}`)) {\n nextIndex += 1;\n }\n\n return `case_${nextIndex}`;\n}\n\nfunction createWorkflowPayload(\n name: string,\n description: string,\n workflowNodes: Node[],\n workflowEdges: Edge[],\n) {\n return {\n name,\n description,\n nodes: workflowNodes.map((node) => ({\n id: node.id,\n type: node.type as\n | 'start'\n | 'end'\n | 'agent'\n | 'note'\n | 'state'\n | 'condition',\n title: node.data.title || node.data.label || node.type,\n position: node.position,\n data:\n node.type === 'agent' ||\n node.type === 'condition' ||\n node.type === 'state'\n ? node.data.config\n : node.data,\n })),\n edges: workflowEdges.map((edge) => ({\n id: edge.id,\n source: edge.source,\n target: edge.target,\n sourceHandle: edge.sourceHandle || undefined,\n targetHandle: edge.targetHandle || undefined,\n })),\n };\n}\n\nfunction WorkflowBuilderInner() {\n const navigate = useNavigate();\n const token = useSelector(selectToken);\n const { agentId } = useParams<{ agentId?: string }>();\n const [searchParams] = useSearchParams();\n const folderId = searchParams.get('folder_id');\n const [workflowId, setWorkflowId] = useState(\n searchParams.get('workflow_id'),\n );\n const reactFlowInstance = useReactFlow();\n const [currentAgentId, setCurrentAgentId] = useState(\n agentId || null,\n );\n\n const reactFlowWrapper = useRef(null);\n\n const [selectedNode, setSelectedNode] = useState(null);\n const [workflowName, setWorkflowName] = useState('New Workflow');\n const [workflowDescription, setWorkflowDescription] = useState('');\n const [showWorkflowSettings, setShowWorkflowSettings] = useState(false);\n const [isPublishing, setIsPublishing] = useState(false);\n const [showPrimaryActionSpinner, setShowPrimaryActionSpinner] =\n useState(false);\n const [publishErrors, setPublishErrors] = useState([]);\n const [errorContext, setErrorContext] = useState<'preview' | 'publish'>(\n 'publish',\n );\n const [showNodeConfig, setShowNodeConfig] = useState(false);\n const [showPreview, setShowPreview] = useState(false);\n const [deleteConfirmation, setDeleteConfirmation] =\n useState('INACTIVE');\n const [agentDetails, setAgentDetails] = useState('INACTIVE');\n const [isDeletingAgent, setIsDeletingAgent] = useState(false);\n const [currentAgent, setCurrentAgent] = useState(\n createEmptyWorkflowAgent(),\n );\n const [imageFile, setImageFile] = useState(null);\n const [savedWorkflowSignature, setSavedWorkflowSignature] = useState<\n string | null\n >(null);\n const workflowSettingsRef = useRef(null);\n const [availableModels, setAvailableModels] = useState([]);\n const [defaultAgentModelId, setDefaultAgentModelId] = useState('');\n const [availableTools, setAvailableTools] = useState([]);\n const [agentJsonSchemaDrafts, setAgentJsonSchemaDrafts] = useState<\n Record\n >({});\n const [agentJsonSchemaErrors, setAgentJsonSchemaErrors] = useState<\n Record\n >({});\n\n const nodeTypes = useMemo(\n () => ({\n start: StartNode,\n agent: AgentNode,\n end: EndNode,\n note: NoteNode,\n state: SetStateNode,\n condition: ConditionNode,\n }),\n [],\n );\n\n const initialNodes: Node[] = useMemo(\n () => [\n {\n id: 'start',\n type: 'start',\n data: { label: 'Start' },\n position: { x: 250, y: 50 },\n },\n ],\n [],\n );\n\n const [nodes, setNodes] = useState(initialNodes);\n const [edges, setEdges] = useState([]);\n\n const onNodesChange = useCallback(\n (changes: NodeChange[]) =>\n setNodes((nds) => applyNodeChanges(changes, nds)),\n [],\n );\n\n const onEdgesChange = useCallback(\n (changes: EdgeChange[]) =>\n setEdges((eds) => applyEdgeChanges(changes, eds)),\n [],\n );\n\n const onConnect = useCallback((params: Connection) => {\n setEdges((eds) => {\n const exists = eds.some(\n (e) =>\n e.source === params.source &&\n e.sourceHandle === params.sourceHandle &&\n e.target === params.target &&\n e.targetHandle === params.targetHandle,\n );\n if (exists) return eds;\n\n const filtered = eds.filter(\n (e) =>\n !(\n e.source === params.source &&\n e.sourceHandle === (params.sourceHandle ?? null)\n ) &&\n !(\n e.target === params.target &&\n e.targetHandle === (params.targetHandle ?? null)\n ),\n );\n return addEdge(params, filtered);\n });\n }, []);\n\n const onEdgeClick = useCallback((_event: React.MouseEvent, edge: Edge) => {\n setEdges((eds) => eds.filter((e) => e.id !== edge.id));\n }, []);\n\n const handleNodeDragStart = useCallback(\n (e: React.DragEvent, nodeType: string) => {\n e.dataTransfer.setData('application/reactflow', nodeType);\n e.dataTransfer.effectAllowed = 'move';\n const el = e.currentTarget as HTMLElement;\n const clone = el.cloneNode(true) as HTMLElement;\n clone.style.position = 'absolute';\n clone.style.top = '-9999px';\n clone.style.width = `${el.offsetWidth}px`;\n clone.style.borderRadius = '9999px';\n clone.style.overflow = 'hidden';\n document.body.appendChild(clone);\n e.dataTransfer.setDragImage(\n clone,\n clone.offsetWidth / 2,\n clone.offsetHeight / 2,\n );\n requestAnimationFrame(() => document.body.removeChild(clone));\n },\n [],\n );\n\n const onDragOver = useCallback((event: React.DragEvent) => {\n event.preventDefault();\n event.dataTransfer.dropEffect = 'move';\n }, []);\n\n const onDrop = useCallback(\n (event: React.DragEvent) => {\n event.preventDefault();\n\n const type = event.dataTransfer.getData('application/reactflow');\n if (!type) return;\n\n const position = reactFlowInstance.screenToFlowPosition({\n x: event.clientX,\n y: event.clientY,\n });\n\n const baseNode: Node = {\n id: `${type}_${Date.now()}`,\n type,\n position,\n data: {\n title: `${type} node`,\n label: `${type} node`,\n },\n };\n\n if (type === 'agent') {\n const defaultModelId = defaultAgentModelId || availableModels[0]?.id;\n const defaultModelProvider = availableModels.find(\n (model) => model.id === defaultModelId,\n )?.provider;\n baseNode.data.config = {\n agent_type: 'classic',\n model_id: defaultModelId,\n llm_name: defaultModelProvider || '',\n system_prompt: 'You are a helpful assistant.',\n prompt_template: '',\n stream_to_user: true,\n sources: [],\n tools: [],\n } as AgentNodeConfig;\n } else if (type === 'state') {\n baseNode.data.title = 'Set State';\n baseNode.data.config = {\n operations: [{ expression: '', target_variable: '' }],\n };\n } else if (type === 'condition') {\n baseNode.data.title = 'If / Else';\n baseNode.data.config = {\n mode: 'simple',\n cases: [{ name: '', expression: '', sourceHandle: 'case_0' }],\n };\n } else if (type === 'note') {\n baseNode.data.title = 'Note';\n baseNode.data.label = 'Note';\n }\n\n setNodes((nds) => nds.concat(baseNode));\n },\n [reactFlowInstance, availableModels, defaultAgentModelId],\n );\n\n const handleNodeClick = useCallback(\n (_event: React.MouseEvent, node: Node) => {\n setSelectedNode(node);\n setShowNodeConfig(true);\n },\n [],\n );\n\n const handleDeleteNode = useCallback(() => {\n if (!selectedNode || selectedNode.type === 'start') return;\n setNodes((nds) => nds.filter((n) => n.id !== selectedNode.id));\n setEdges((eds) =>\n eds.filter(\n (e) => e.source !== selectedNode.id && e.target !== selectedNode.id,\n ),\n );\n setAgentJsonSchemaDrafts((prev) => {\n if (!(selectedNode.id in prev)) return prev;\n const next = { ...prev };\n delete next[selectedNode.id];\n return next;\n });\n setAgentJsonSchemaErrors((prev) => {\n if (!(selectedNode.id in prev)) return prev;\n const next = { ...prev };\n delete next[selectedNode.id];\n return next;\n });\n setSelectedNode(null);\n setShowNodeConfig(false);\n }, [selectedNode]);\n\n const handleUpdateNodeData = useCallback(\n (data: Record) => {\n if (!selectedNode) return;\n setNodes((nds) =>\n nds.map((n) =>\n n.id === selectedNode.id ? { ...n, data: { ...n.data, ...data } } : n,\n ),\n );\n setSelectedNode((prev) =>\n prev ? { ...prev, data: { ...prev.data, ...data } } : null,\n );\n },\n [selectedNode],\n );\n\n const handleAgentJsonSchemaChange = useCallback(\n (text: string) => {\n if (!selectedNode || selectedNode.type !== 'agent') return;\n\n const nodeId = selectedNode.id;\n setAgentJsonSchemaDrafts((prev) => ({ ...prev, [nodeId]: text }));\n\n if (text.trim() === '') {\n setAgentJsonSchemaErrors((prev) => ({ ...prev, [nodeId]: null }));\n handleUpdateNodeData({\n config: {\n ...(selectedNode.data.config || {}),\n json_schema: undefined,\n },\n });\n return;\n }\n\n try {\n const parsed = JSON.parse(text);\n const validationError = validateJsonSchemaConfig(parsed);\n setAgentJsonSchemaErrors((prev) => ({\n ...prev,\n [nodeId]: validationError,\n }));\n if (!validationError) {\n handleUpdateNodeData({\n config: {\n ...(selectedNode.data.config || {}),\n json_schema: parsed,\n },\n });\n }\n } catch {\n setAgentJsonSchemaErrors((prev) => ({\n ...prev,\n [nodeId]: 'must be valid JSON',\n }));\n }\n },\n [handleUpdateNodeData, selectedNode],\n );\n\n const handleUpload = useCallback((files: File[]) => {\n if (files && files.length > 0) {\n setImageFile(files[0]);\n }\n }, []);\n\n const navigateBackToAgents = useCallback(() => {\n navigate(folderId ? `/agents?folder=${folderId}` : '/agents');\n }, [navigate, folderId]);\n\n const handleDeleteAgent = useCallback(async () => {\n const agentToDelete = currentAgentId || currentAgent.id;\n if (!agentToDelete) return;\n setIsDeletingAgent(true);\n try {\n const response = await userService.deleteAgent(agentToDelete, token);\n if (!response.ok) {\n const errorData = await response.json().catch(() => ({}));\n throw new Error(errorData.message || 'Failed to delete workflow agent');\n }\n navigateBackToAgents();\n } catch (error) {\n setPublishErrors([\n error instanceof Error\n ? error.message\n : 'Failed to delete workflow agent',\n ]);\n setErrorContext('publish');\n } finally {\n setIsDeletingAgent(false);\n }\n }, [currentAgentId, currentAgent.id, token, navigateBackToAgents]);\n\n useEffect(() => {\n if (publishErrors.length > 0) {\n const timer = setTimeout(() => {\n setPublishErrors([]);\n }, 6000);\n return () => clearTimeout(timer);\n }\n }, [publishErrors.length]);\n\n useEffect(() => {\n if (!isPublishing) {\n setShowPrimaryActionSpinner(false);\n return;\n }\n\n const spinnerTimer = window.setTimeout(() => {\n setShowPrimaryActionSpinner(true);\n }, PRIMARY_ACTION_SPINNER_DELAY_MS);\n\n return () => window.clearTimeout(spinnerTimer);\n }, [isPublishing]);\n\n useEffect(() => {\n const handleKeyDown = (e: KeyboardEvent) => {\n if (e.key === 'Delete' && selectedNode) {\n handleDeleteNode();\n }\n if (e.key === 'Escape') {\n setShowNodeConfig(false);\n setSelectedNode(null);\n }\n };\n window.addEventListener('keydown', handleKeyDown);\n return () => window.removeEventListener('keydown', handleKeyDown);\n }, [selectedNode, handleDeleteNode]);\n\n const handlePanelBackdropClick = useCallback(() => {\n setShowNodeConfig(false);\n setSelectedNode(null);\n }, []);\n\n useEffect(() => {\n const handleClickOutside = (e: MouseEvent) => {\n if (\n workflowSettingsRef.current &&\n !workflowSettingsRef.current.contains(e.target as HTMLElement)\n ) {\n setShowWorkflowSettings(false);\n }\n };\n if (showWorkflowSettings) {\n document.addEventListener('mousedown', handleClickOutside);\n return () =>\n document.removeEventListener('mousedown', handleClickOutside);\n }\n }, [showWorkflowSettings]);\n\n useEffect(() => {\n const loadModelsAndTools = async () => {\n try {\n const modelsResponse = await modelService.getModels(null);\n if (modelsResponse.ok) {\n const modelsData = await modelsResponse.json();\n const transformedModels = modelService.transformModels(\n modelsData.models || [],\n );\n setAvailableModels(transformedModels);\n const preferredDefaultModel =\n transformedModels.find(\n (model) => model.id === modelsData.default_model_id,\n )?.id ||\n transformedModels[0]?.id ||\n '';\n setDefaultAgentModelId(preferredDefaultModel);\n }\n\n const toolsResponse = await userService.getUserTools(null);\n if (toolsResponse.ok) {\n const toolsData = await toolsResponse.json();\n setAvailableTools(toolsData.tools);\n }\n } catch (error) {\n console.error('Failed to load models or tools:', error);\n }\n };\n loadModelsAndTools();\n }, []);\n\n useEffect(() => {\n if (!selectedNode || selectedNode.type !== 'agent') return;\n if (!defaultAgentModelId) return;\n if (selectedNode.data.config?.model_id) return;\n\n handleUpdateNodeData({\n config: {\n ...(selectedNode.data.config || {}),\n model_id: defaultAgentModelId,\n llm_name:\n availableModels.find((model) => model.id === defaultAgentModelId)\n ?.provider || '',\n },\n });\n }, [\n selectedNode,\n defaultAgentModelId,\n availableModels,\n handleUpdateNodeData,\n ]);\n\n useEffect(() => {\n if (!selectedNode || selectedNode.type !== 'agent') return;\n const nodeId = selectedNode.id;\n const rawSchema = selectedNode.data.config?.json_schema;\n\n setAgentJsonSchemaDrafts((prev) => {\n if (prev[nodeId] !== undefined) return prev;\n if (rawSchema === undefined || rawSchema === null) {\n return { ...prev, [nodeId]: '' };\n }\n\n try {\n return { ...prev, [nodeId]: JSON.stringify(rawSchema, null, 2) };\n } catch {\n return { ...prev, [nodeId]: String(rawSchema) };\n }\n });\n\n setAgentJsonSchemaErrors((prev) => {\n if (prev[nodeId] !== undefined) return prev;\n return { ...prev, [nodeId]: validateJsonSchemaConfig(rawSchema) };\n });\n }, [selectedNode]);\n\n useEffect(() => {\n const loadAgentDetails = async () => {\n if (!agentId) return;\n try {\n const response = await userService.getAgent(agentId, token);\n if (!response.ok) throw new Error('Failed to fetch agent');\n const agent = await response.json();\n setCurrentAgent({\n ...createEmptyWorkflowAgent(),\n ...agent,\n agent_type: 'workflow',\n });\n if (agent.agent_type === 'workflow' && agent.workflow) {\n setWorkflowId(agent.workflow);\n setCurrentAgentId(agent.id);\n setWorkflowName(agent.name);\n setWorkflowDescription(agent.description || '');\n }\n } catch (error) {\n console.error('Failed to load agent:', error);\n }\n };\n loadAgentDetails();\n }, [agentId, token]);\n\n useEffect(() => {\n const loadWorkflow = async () => {\n if (!workflowId) return;\n try {\n const response = await userService.getWorkflow(workflowId, token);\n if (!response.ok) throw new Error('Failed to fetch workflow');\n const responseData = await response.json();\n const { workflow, nodes: apiNodes, edges: apiEdges } = responseData;\n const nextWorkflowName = workflow.name;\n const nextWorkflowDescription = workflow.description || '';\n const mappedNodes = apiNodes.map((n: WorkflowNode) => {\n const nodeData: Record = {\n title: n.title,\n label: n.title,\n };\n if (n.type === 'agent' && n.data) {\n nodeData.config = n.data;\n } else if (n.type === 'condition' && n.data) {\n nodeData.config = {\n ...n.data,\n cases: normalizeConditionCases(n.data.cases || []),\n };\n } else if (n.type === 'state' && n.data) {\n nodeData.config = n.data;\n } else if (n.data) {\n Object.assign(nodeData, n.data);\n }\n return {\n id: n.id,\n type: n.type,\n position: n.position,\n data: nodeData,\n };\n });\n const mappedEdges = apiEdges.map(\n (e: {\n id: string;\n source: string;\n target: string;\n sourceHandle?: string;\n targetHandle?: string;\n }) => ({\n id: e.id,\n source: e.source,\n target: e.target,\n sourceHandle: e.sourceHandle,\n targetHandle: e.targetHandle,\n }),\n );\n setWorkflowName(nextWorkflowName);\n setWorkflowDescription(nextWorkflowDescription);\n setAgentJsonSchemaDrafts({});\n setAgentJsonSchemaErrors({});\n setNodes(mappedNodes);\n setEdges(mappedEdges);\n setSavedWorkflowSignature(\n JSON.stringify(\n createWorkflowPayload(\n nextWorkflowName,\n nextWorkflowDescription,\n mappedNodes,\n mappedEdges,\n ),\n ),\n );\n setTimeout(() => {\n reactFlowInstance.fitView({\n padding: 0.2,\n maxZoom: 0.8,\n duration: 300,\n });\n }, 100);\n } catch (error) {\n console.error('Failed to load workflow:', error);\n }\n };\n loadWorkflow();\n }, [workflowId, reactFlowInstance, token]);\n\n const validateWorkflow = useCallback((): string[] => {\n const errors: string[] = [];\n\n if (!workflowName.trim()) {\n errors.push('Workflow name is required');\n }\n\n const startNodes = nodes.filter((n) => n.type === 'start');\n if (startNodes.length !== 1) {\n errors.push('Workflow must have exactly one start node');\n }\n\n const endNodes = nodes.filter((n) => n.type === 'end');\n const endNodeIds = new Set(endNodes.map((n) => n.id));\n if (endNodes.length === 0) {\n errors.push('Workflow must have at least one end node');\n }\n\n const agentNodes = nodes.filter((n) => n.type === 'agent');\n if (agentNodes.length === 0) {\n errors.push('Workflow must have at least one AI agent node');\n }\n\n agentNodes.forEach((node) => {\n const config = node.data?.config;\n if (!config?.llm_name && !config?.model_id) {\n errors.push(\n `Agent \"${node.data?.title || node.id}\" must have a model selected`,\n );\n }\n\n const hasSchema =\n config?.json_schema !== undefined && config?.json_schema !== null;\n if (hasSchema && config?.model_id) {\n const selectedModel = availableModels.find(\n (model) => model.id === config.model_id,\n );\n if (selectedModel && !selectedModel.supports_structured_output) {\n errors.push(\n `Agent \"${node.data?.title || node.id}\" selected model does not support structured output`,\n );\n }\n }\n\n const schemaValidationError = validateJsonSchemaConfig(\n config?.json_schema,\n );\n const draftSchemaError = agentJsonSchemaErrors[node.id];\n const effectiveSchemaError =\n draftSchemaError !== undefined\n ? draftSchemaError\n : schemaValidationError;\n if (effectiveSchemaError) {\n errors.push(\n `Agent \"${node.data?.title || node.id}\" JSON schema ${effectiveSchemaError}`,\n );\n }\n });\n\n if (startNodes.length === 1) {\n const startId = startNodes[0].id;\n const hasOutgoing = edges.some((e) => e.source === startId);\n if (!hasOutgoing) {\n errors.push('Start node must be connected to another node');\n }\n }\n\n endNodes.forEach((endNode) => {\n const hasIncoming = edges.some((e) => e.target === endNode.id);\n if (!hasIncoming) {\n errors.push(\n `End node \"${endNode.id}\" must have an incoming connection`,\n );\n }\n });\n\n const nodeIds = new Set(nodes.map((n) => n.id));\n edges.forEach((edge) => {\n if (!nodeIds.has(edge.source)) {\n errors.push(`Edge references non-existent source node`);\n }\n if (!nodeIds.has(edge.target)) {\n errors.push(`Edge references non-existent target node`);\n }\n });\n\n const conditionNodes = nodes.filter((n) => n.type === 'condition');\n conditionNodes.forEach((node) => {\n const conditionTitle = node.data?.title || node.id;\n const conditionMode = node.data?.config?.mode || 'simple';\n const cases = (node.data?.config?.cases || []) as ConditionCase[];\n if (\n !cases.length ||\n !cases.some((c: ConditionCase) => Boolean((c.expression || '').trim()))\n ) {\n errors.push(\n `Condition \"${conditionTitle}\" must have at least one case with an expression`,\n );\n }\n\n const caseHandles = new Set();\n const duplicateCaseHandles = new Set();\n cases.forEach((conditionCase: ConditionCase) => {\n const handle = (conditionCase.sourceHandle || '').trim();\n if (!handle) {\n errors.push(\n `Condition \"${conditionTitle}\" has a case without a branch handle`,\n );\n return;\n }\n if (caseHandles.has(handle)) {\n duplicateCaseHandles.add(handle);\n }\n caseHandles.add(handle);\n });\n duplicateCaseHandles.forEach((handle) => {\n errors.push(\n `Condition \"${conditionTitle}\" has duplicate case handle \"${handle}\"`,\n );\n });\n\n const outgoing = edges.filter((e) => e.source === node.id);\n if (outgoing.length < 2) {\n errors.push(\n `Condition \"${conditionTitle}\" must have at least 2 outgoing connections`,\n );\n }\n\n const outgoingByHandle = new Map();\n outgoing.forEach((edge) => {\n const handle = (edge.sourceHandle || '').trim();\n const handleEdges = outgoingByHandle.get(handle);\n if (handleEdges) {\n handleEdges.push(edge);\n return;\n }\n outgoingByHandle.set(handle, [edge]);\n });\n\n for (const [handle, handleEdges] of outgoingByHandle.entries()) {\n if (!handle) {\n errors.push(\n `Condition \"${conditionTitle}\" has a connection without a branch handle`,\n );\n continue;\n }\n if (handle !== 'else' && !caseHandles.has(handle)) {\n errors.push(\n `Condition \"${conditionTitle}\" has a connection from unknown branch \"${handle}\"`,\n );\n }\n if (handleEdges.length > 1) {\n errors.push(\n `Condition \"${conditionTitle}\" has multiple connections from branch \"${handle}\"`,\n );\n }\n }\n\n if (!outgoingByHandle.has('else')) {\n errors.push(`Condition \"${conditionTitle}\" must have an Else branch`);\n }\n\n cases.forEach((conditionCase: ConditionCase) => {\n const handle = (conditionCase.sourceHandle || '').trim();\n if (!handle) return;\n\n const hasExpression = Boolean((conditionCase.expression || '').trim());\n const hasOutgoing = Boolean(outgoingByHandle.get(handle)?.length);\n if (hasExpression && !hasOutgoing) {\n errors.push(\n `Condition \"${conditionTitle}\" case \"${handle}\" has an expression but no branch connection`,\n );\n }\n if (!hasExpression && hasOutgoing) {\n errors.push(\n `Condition \"${conditionTitle}\" case \"${handle}\" has a branch connection but no expression`,\n );\n }\n if (conditionMode === 'simple' && hasExpression) {\n const parsedCondition = parseSimpleCel(\n conditionCase.expression || '',\n );\n if (!parsedCondition.variable.trim()) {\n errors.push(\n `Condition \"${conditionTitle}\" case \"${handle}\" must specify a variable in Simple mode`,\n );\n }\n }\n });\n\n outgoing.forEach((edge) => {\n if (!canReachEnd(edge.target, edges, nodeIds, endNodeIds)) {\n const handle = edge.sourceHandle || 'branch';\n errors.push(\n `Branch \"${handle}\" of condition \"${conditionTitle}\" must eventually reach an end node`,\n );\n }\n });\n });\n\n return errors;\n }, [workflowName, nodes, edges, agentJsonSchemaErrors, availableModels]);\n\n const canManageAgent = Boolean(currentAgentId || currentAgent.id);\n const effectiveAgentId = currentAgentId || currentAgent.id || '';\n const currentAgentImage = currentAgent.image || '';\n\n const buildWorkflowPayload = useCallback(\n () =>\n createWorkflowPayload(workflowName, workflowDescription, nodes, edges),\n [workflowName, workflowDescription, nodes, edges],\n );\n\n const workflowPayloadSignature = useMemo(\n () => JSON.stringify(buildWorkflowPayload()),\n [buildWorkflowPayload],\n );\n\n const hasSavableChanges =\n canManageAgent && savedWorkflowSignature !== null\n ? workflowPayloadSignature !== savedWorkflowSignature ||\n imageFile !== null\n : false;\n\n const persistWorkflow = useCallback(\n async (navigateAfterSuccess: boolean): Promise => {\n setPublishErrors([]);\n setErrorContext('publish');\n\n const validationErrors = validateWorkflow();\n if (validationErrors.length > 0) {\n setPublishErrors(validationErrors);\n return false;\n }\n\n setIsPublishing(true);\n let createdWorkflowId: string | null = null;\n try {\n const workflowPayload = buildWorkflowPayload();\n\n let savedWorkflowId = workflowId;\n if (workflowId) {\n const updateResponse = await userService.updateWorkflow(\n workflowId,\n workflowPayload,\n token,\n );\n if (!updateResponse.ok) {\n const errorData = await updateResponse.json().catch(() => ({}));\n throw new Error(errorData.message || 'Failed to update workflow');\n }\n\n if (effectiveAgentId) {\n const agentFormData = new FormData();\n agentFormData.append('name', workflowName);\n agentFormData.append(\n 'description',\n workflowDescription || `Workflow agent: ${workflowName}`,\n );\n agentFormData.append('status', 'published');\n if (imageFile) {\n agentFormData.append('image', imageFile);\n }\n const agentUpdateResponse = await userService.updateAgent(\n effectiveAgentId,\n agentFormData,\n token,\n );\n if (!agentUpdateResponse.ok) {\n throw new Error('Failed to update agent');\n }\n const updatedAgent = await agentUpdateResponse\n .json()\n .catch(() => null);\n setCurrentAgent((prev) => ({\n ...prev,\n ...(updatedAgent || {}),\n id: effectiveAgentId,\n name: workflowName,\n description:\n workflowDescription || `Workflow agent: ${workflowName}`,\n image: updatedAgent?.image || prev.image || '',\n }));\n }\n setImageFile(null);\n setSavedWorkflowSignature(JSON.stringify(workflowPayload));\n if (navigateAfterSuccess) {\n navigateBackToAgents();\n }\n return true;\n }\n\n const createResponse = await userService.createWorkflow(\n workflowPayload,\n token,\n );\n if (!createResponse.ok) {\n const errorData = await createResponse.json().catch(() => ({}));\n const backendErrors = errorData.errors || [];\n if (backendErrors.length > 0) {\n setPublishErrors(backendErrors);\n return false;\n }\n throw new Error(errorData.message || 'Failed to create workflow');\n }\n const responseData = await createResponse.json();\n savedWorkflowId = responseData.id;\n createdWorkflowId = savedWorkflowId || null;\n if (savedWorkflowId) {\n setWorkflowId(savedWorkflowId);\n }\n\n const agentFormData = new FormData();\n agentFormData.append('name', workflowName);\n agentFormData.append(\n 'description',\n workflowDescription || `Workflow agent: ${workflowName}`,\n );\n agentFormData.append('agent_type', 'workflow');\n agentFormData.append('status', 'published');\n agentFormData.append('workflow', savedWorkflowId || '');\n if (imageFile) {\n agentFormData.append('image', imageFile);\n }\n if (folderId) agentFormData.append('folder_id', folderId);\n\n const agentResponse = await userService.createAgent(\n agentFormData,\n token,\n );\n if (!agentResponse.ok) {\n const errorData = await agentResponse.json().catch(() => ({}));\n throw new Error(errorData.message || 'Failed to create agent');\n }\n const agentData = await agentResponse.json().catch(() => ({}));\n if (agentData?.id) {\n setCurrentAgentId(agentData.id);\n setCurrentAgent({\n ...createEmptyWorkflowAgent(),\n ...agentData,\n id: agentData.id,\n name: workflowName,\n description:\n workflowDescription || `Workflow agent: ${workflowName}`,\n image: agentData.image || '',\n workflow: savedWorkflowId || undefined,\n agent_type: 'workflow',\n status: 'published',\n });\n }\n\n setImageFile(null);\n setSavedWorkflowSignature(JSON.stringify(workflowPayload));\n if (navigateAfterSuccess) {\n navigateBackToAgents();\n }\n return true;\n } catch (error) {\n if (createdWorkflowId) {\n try {\n const cleanupResponse = await userService.deleteWorkflow(\n createdWorkflowId,\n token,\n );\n if (cleanupResponse.ok) {\n setWorkflowId(null);\n }\n } catch (cleanupError) {\n console.error(\n 'Failed to clean up workflow after publish error:',\n cleanupError,\n );\n }\n }\n console.error('Failed to save workflow:', error);\n setPublishErrors([\n error instanceof Error ? error.message : 'Failed to save workflow',\n ]);\n return false;\n } finally {\n setIsPublishing(false);\n }\n },\n [\n validateWorkflow,\n buildWorkflowPayload,\n workflowId,\n token,\n effectiveAgentId,\n workflowName,\n workflowDescription,\n imageFile,\n folderId,\n navigateBackToAgents,\n ],\n );\n\n const handleWorkflowSettingsDone = useCallback(() => {\n setShowWorkflowSettings(false);\n if (!canManageAgent || !hasSavableChanges || isPublishing) return;\n void persistWorkflow(false);\n }, [canManageAgent, hasSavableChanges, isPublishing, persistWorkflow]);\n\n const isPrimaryActionDisabled =\n isPublishing || (canManageAgent && !hasSavableChanges);\n const primaryActionLabel = canManageAgent ? 'Save' : 'Publish';\n\n const handlePrimaryAction = useCallback(() => {\n if (isPrimaryActionDisabled) return;\n void persistWorkflow(!canManageAgent);\n }, [isPrimaryActionDisabled, persistWorkflow, canManageAgent]);\n\n const agentForDetails = useMemo(\n () => ({\n ...createEmptyWorkflowAgent(),\n ...currentAgent,\n id: effectiveAgentId,\n name: workflowName,\n description: workflowDescription || `Workflow agent: ${workflowName}`,\n image: currentAgentImage,\n agent_type: 'workflow',\n status: currentAgent.status || 'published',\n workflow: workflowId || currentAgent.workflow,\n }),\n [\n currentAgent,\n effectiveAgentId,\n workflowName,\n workflowDescription,\n currentAgentImage,\n workflowId,\n ],\n );\n\n const selectedAgentJsonSchemaText = useMemo(() => {\n if (!selectedNode || selectedNode.type !== 'agent') return '';\n\n const draft = agentJsonSchemaDrafts[selectedNode.id];\n if (draft !== undefined) return draft;\n\n const schema = selectedNode.data.config?.json_schema;\n if (schema === undefined || schema === null) return '';\n\n try {\n return JSON.stringify(schema, null, 2);\n } catch {\n return String(schema);\n }\n }, [selectedNode, agentJsonSchemaDrafts]);\n\n const selectedAgentJsonSchemaError = useMemo(() => {\n if (!selectedNode || selectedNode.type !== 'agent') return null;\n\n const cachedError = agentJsonSchemaErrors[selectedNode.id];\n if (cachedError !== undefined) return cachedError;\n\n return validateJsonSchemaConfig(selectedNode.data.config?.json_schema);\n }, [selectedNode, agentJsonSchemaErrors]);\n\n const selectedAgentModelSupportsStructuredOutput = useMemo(() => {\n if (!selectedNode || selectedNode.type !== 'agent') return true;\n const modelId = selectedNode.data.config?.model_id;\n if (!modelId) return true;\n\n const selectedModel = availableModels.find((model) => model.id === modelId);\n if (!selectedModel) return true;\n\n return selectedModel.supports_structured_output;\n }, [selectedNode, availableModels]);\n\n return (\n <>\n \n \n \n
\n setShowWorkflowSettings((prev) => !prev)}\n className=\"flex items-center gap-2 rounded-full border border-gray-200 bg-white px-4 py-2 text-sm font-medium text-gray-700 transition-colors hover:bg-gray-50 dark:border-[#3A3A3A] dark:bg-[#2C2C2C] dark:text-gray-200 dark:hover:bg-[#383838]\"\n >\n \n Details\n \n {canManageAgent && (\n navigate(`/agents/logs/${effectiveAgentId}`)}\n className=\"flex items-center gap-2 rounded-full border border-gray-200 bg-white px-4 py-2 text-sm font-medium text-gray-700 transition-colors hover:bg-gray-50 dark:border-[#3A3A3A] dark:bg-[#2C2C2C] dark:text-gray-200 dark:hover:bg-[#383838]\"\n >\n \n Logs\n \n )}\n {canManageAgent && (\n setAgentDetails('ACTIVE')}\n className=\"flex items-center gap-2 rounded-full border border-gray-200 bg-white px-4 py-2 text-sm font-medium text-gray-700 transition-colors hover:bg-gray-50 dark:border-[#3A3A3A] dark:bg-[#2C2C2C] dark:text-gray-200 dark:hover:bg-[#383838]\"\n >\n \n Access Details\n \n )}\n {canManageAgent && (\n setDeleteConfirmation('ACTIVE')}\n disabled={isDeletingAgent}\n className=\"flex items-center gap-2 rounded-full border border-red-200 bg-white px-4 py-2 text-sm font-medium text-red-600 transition-colors hover:bg-red-50 disabled:cursor-not-allowed disabled:opacity-50 dark:border-red-900/30 dark:bg-[#2C2C2C] dark:text-red-400 dark:hover:bg-red-900/10\"\n >\n \n {isDeletingAgent ? 'Deleting...' : 'Delete'}\n \n )}\n {\n const validationErrors = validateWorkflow();\n if (validationErrors.length > 0) {\n setErrorContext('preview');\n setPublishErrors(validationErrors);\n return;\n }\n setShowPreview(true);\n }}\n className=\"flex items-center gap-2 rounded-full border border-gray-200 bg-white px-4 py-2 text-sm font-medium text-gray-700 transition-colors hover:bg-gray-50 dark:border-[#3A3A3A] dark:bg-[#2C2C2C] dark:text-gray-200 dark:hover:bg-[#383838]\"\n >\n \n Preview\n \n \n \n {primaryActionLabel}\n \n {showPrimaryActionSpinner ? (\n \n ) : null}\n \n
\n \n\n {publishErrors.length > 0 && (\n
\n \n \n \n {errorContext === 'preview'\n ? 'Unable to preview workflow'\n : canManageAgent\n ? 'Unable to save workflow'\n : 'Unable to publish workflow'}\n \n \n
    \n {publishErrors.map((error, index) => (\n
  • {error}
    • \n ))}\n
\n \n setPublishErrors([])}\n className=\"absolute top-4 right-4 text-red-700 hover:text-red-900 dark:text-red-300 dark:hover:text-red-100\"\n >\n \n \n \n
\n )}\n\n
\n
\n
\n

\n Core Nodes\n

\n
\n handleNodeDragStart(e, 'agent')}\n >\n
\n \n
\n \n AI Agent\n \n
\n handleNodeDragStart(e, 'end')}\n >\n
\n \n
\n \n End\n \n
\n handleNodeDragStart(e, 'note')}\n >\n
\n \n
\n \n Note\n \n
\n
\n \n\n
\n

\n Logic & Data\n

\n
\n handleNodeDragStart(e, 'state')}\n >\n
\n \n
\n
\n \n Set State\n \n \n Modify workflow variables\n \n
\n
\n handleNodeDragStart(e, 'condition')}\n >\n
\n \n
\n
\n \n If / Else\n \n \n Conditional branching\n \n
\n
\n \n \n \n\n \n \n \n \n \n\n {showNodeConfig && selectedNode && (\n <>\n \n
\n
\n

\n {selectedNode.type === 'start' && 'Start Node'}\n {selectedNode.type === 'end' && 'End Node'}\n {selectedNode.type === 'agent' && 'AI Agent'}\n {selectedNode.type === 'note' && 'Note'}\n {selectedNode.type === 'state' && 'Set global variables'}\n {selectedNode.type === 'condition' && 'If / Else'}\n

\n setShowNodeConfig(false)}\n className=\"text-gray-400 transition-colors hover:text-gray-600 dark:hover:text-gray-200\"\n >\n \n \n
\n\n
\n
\n
\n
\n Node ID\n
\n
\n {selectedNode.id}\n
\n
\n\n {selectedNode.type !== 'start' &&\n selectedNode.type !== 'end' && (\n <>\n
\n \n \n handleUpdateNodeData({\n title: e.target.value,\n label: e.target.value,\n })\n }\n className=\"border-light-silver focus:ring-purple-30 w-full rounded-xl border bg-white px-3 py-2 text-sm transition-all outline-none focus:ring-2 dark:border-[#3A3A3A] dark:bg-[#2C2C2C] dark:text-white\"\n placeholder=\"Enter node title\"\n />\n
\n\n {selectedNode.type === 'agent' && (\n <>\n
\n \n \n handleUpdateNodeData({\n config: {\n ...(selectedNode.data.config || {}),\n agent_type: value,\n },\n })\n }\n >\n \n \n \n \n \n Classic\n \n \n ReAct\n \n \n \n
\n
\n \n {\n const selectedModel =\n availableModels.find(\n (m) => m.id === value,\n );\n handleUpdateNodeData({\n config: {\n ...(selectedNode.data.config || {}),\n model_id: value,\n llm_name:\n selectedModel?.provider || '',\n },\n });\n }}\n >\n \n \n \n \n {availableModels.map((model) => (\n \n {model.display_name} ·{' '}\n {model.provider}\n \n ))}\n \n \n
\n
\n \n \n handleUpdateNodeData({\n config: {\n ...(selectedNode.data.config || {}),\n system_prompt: e.target.value,\n },\n })\n }\n className=\"border-light-silver focus:ring-purple-30 w-full rounded-xl border bg-white px-3 py-2 text-sm transition-all outline-none focus:ring-2 dark:border-[#3A3A3A] dark:bg-[#2C2C2C] dark:text-white\"\n rows={3}\n placeholder=\"System prompt for the agent\"\n />\n
\n \n handleUpdateNodeData({\n config: {\n ...(selectedNode.data.config || {}),\n prompt_template: val,\n },\n })\n }\n nodes={nodes}\n edges={edges}\n selectedNodeId={selectedNode.id}\n placeholder=\"Use {{ agent.variable }} for dynamic content\"\n />\n
\n \n {\n const nextOutputVariable = e.target.value;\n handleUpdateNodeData({\n config: {\n ...(selectedNode.data.config || {}),\n output_variable: nextOutputVariable,\n },\n });\n }}\n className=\"border-light-silver focus:ring-purple-30 w-full rounded-xl border bg-white px-3 py-2 text-sm transition-all outline-none focus:ring-2 dark:border-[#3A3A3A] dark:bg-[#2C2C2C] dark:text-white\"\n placeholder=\"Variable name for output\"\n />\n
\n
\n \n handleUpdateNodeData({\n config: {\n ...(selectedNode.data.config || {}),\n stream_to_user: e.target.checked,\n },\n })\n }\n className=\"h-4 w-4\"\n />\n \n Stream output to user\n \n
{' '}\n
\n \n ({\n value: tool.id,\n label: tool.displayName,\n }))}\n selected={\n selectedNode.data.config?.tools || []\n }\n onChange={(newTools) =>\n handleUpdateNodeData({\n config: {\n ...(selectedNode.data.config || {}),\n tools: newTools,\n },\n })\n }\n placeholder=\"Select tools...\"\n searchPlaceholder=\"Search tools...\"\n emptyText=\"No tools available\"\n />\n
\n
\n \n {!selectedAgentModelSupportsStructuredOutput && (\n

\n Selected model does not support structured\n output.\n

\n )}\n \n handleAgentJsonSchemaChange(\n e.target.value,\n )\n }\n className=\"border-light-silver focus:ring-purple-30 w-full rounded-xl border bg-white px-3 py-2 font-mono text-xs transition-all outline-none focus:ring-2 dark:border-[#3A3A3A] dark:bg-[#2C2C2C] dark:text-white\"\n rows={8}\n placeholder={`{\n \"type\": \"object\",\n \"properties\": {\n \"summary\": { \"type\": \"string\" }\n },\n \"required\": [\"summary\"]\n}`}\n />\n {selectedAgentJsonSchemaText.trim() !==\n '' && (\n \n {selectedAgentJsonSchemaError\n ? `Invalid JSON schema: ${selectedAgentJsonSchemaError}`\n : 'Valid JSON schema'}\n

\n )}\n
\n \n )}\n\n {selectedNode.type === 'note' && (\n
\n \n \n handleUpdateNodeData({\n content: e.target.value,\n })\n }\n className=\"border-light-silver focus:ring-purple-30 w-full rounded-xl border bg-white px-3 py-2 text-sm transition-all outline-none focus:ring-2 dark:border-[#3A3A3A] dark:bg-[#2C2C2C] dark:text-white\"\n rows={4}\n placeholder=\"Enter note content\"\n />\n
\n )}\n\n {selectedNode.type === 'state' && (\n <>\n

\n Assign values to workflow's state\n variables\n

\n {(\n selectedNode.data.config?.operations || []\n ).map(\n (\n op: {\n expression: string;\n target_variable: string;\n },\n idx: number,\n ) => (\n \n
\n \n Assign value\n \n {(\n selectedNode.data.config\n ?.operations || []\n ).length > 1 && (\n {\n const ops = [\n ...(selectedNode.data.config\n ?.operations || []),\n ];\n ops.splice(idx, 1);\n handleUpdateNodeData({\n config: {\n ...(selectedNode.data\n .config || {}),\n operations: ops,\n },\n });\n }}\n className=\"text-gray-400 transition-colors hover:text-red-500\"\n >\n \n \n )}\n
\n {\n const ops = [\n ...(selectedNode.data.config\n ?.operations || []),\n ];\n ops[idx] = {\n ...ops[idx],\n expression: e.target.value,\n };\n handleUpdateNodeData({\n config: {\n ...(selectedNode.data.config ||\n {}),\n operations: ops,\n },\n });\n }}\n className=\"border-light-silver focus:ring-purple-30 mb-1 w-full rounded-xl border bg-white px-3 py-2 text-sm transition-all outline-none focus:ring-2 dark:border-[#3A3A3A] dark:bg-[#383838] dark:text-white\"\n rows={2}\n placeholder=\"input.foo + 1\"\n />\n

\n Use Common Expression Language to create\n a custom expression.{' '}\n \n Learn more\n \n

\n
\n \n To variable\n \n {\n const ops = [\n ...(selectedNode.data.config\n ?.operations || []),\n ];\n ops[idx] = {\n ...ops[idx],\n target_variable: e.target.value,\n };\n handleUpdateNodeData({\n config: {\n ...(selectedNode.data.config ||\n {}),\n operations: ops,\n },\n });\n }}\n className=\"border-light-silver focus:ring-purple-30 w-full rounded-xl border bg-white px-3 py-2 text-sm transition-all outline-none focus:ring-2 dark:border-[#3A3A3A] dark:bg-[#383838] dark:text-white\"\n placeholder=\"variable_name\"\n />\n
\n
\n ),\n )}\n {\n const ops = [\n ...(selectedNode.data.config\n ?.operations || []),\n { expression: '', target_variable: '' },\n ];\n handleUpdateNodeData({\n config: {\n ...(selectedNode.data.config || {}),\n operations: ops,\n },\n });\n }}\n className=\"flex items-center gap-1 rounded-lg px-3 py-1.5 text-sm font-medium text-gray-600 transition-colors hover:bg-gray-100 dark:text-gray-400 dark:hover:bg-[#383838]\"\n >\n \n Add\n \n \n )}\n\n {selectedNode.type === 'condition' && (\n <>\n

\n Create conditions to branch your workflow\n

\n
\n \n handleUpdateNodeData({\n config: {\n ...(selectedNode.data.config || {}),\n mode: 'simple',\n },\n })\n }\n className={`flex-1 px-3 py-1.5 text-xs font-medium transition-colors ${\n (selectedNode.data.config?.mode ||\n 'simple') === 'simple'\n ? 'bg-violets-are-blue text-white'\n : 'text-gray-600 hover:bg-gray-50 dark:text-gray-400 dark:hover:bg-[#383838]'\n }`}\n >\n Simple\n \n \n handleUpdateNodeData({\n config: {\n ...(selectedNode.data.config || {}),\n mode: 'advanced',\n },\n })\n }\n className={`flex-1 px-3 py-1.5 text-xs font-medium transition-colors ${\n selectedNode.data.config?.mode ===\n 'advanced'\n ? 'bg-violets-are-blue text-white'\n : 'text-gray-600 hover:bg-gray-50 dark:text-gray-400 dark:hover:bg-[#383838]'\n }`}\n >\n Advanced\n \n
\n\n {(selectedNode.data.config?.cases || []).map(\n (c: ConditionCase, idx: number) => (\n \n
\n \n {idx === 0 ? 'If' : 'Else if'}\n \n {(selectedNode.data.config?.cases || [])\n .length > 1 && (\n {\n const cases =\n normalizeConditionCases([\n ...(selectedNode.data.config\n ?.cases || []),\n ]);\n const removedHandle =\n cases[idx]?.sourceHandle;\n cases.splice(idx, 1);\n handleUpdateNodeData({\n config: {\n ...(selectedNode.data\n .config || {}),\n cases,\n },\n });\n if (removedHandle) {\n setEdges((eds) =>\n eds.filter(\n (edge) =>\n !(\n edge.source ===\n selectedNode.id &&\n edge.sourceHandle ===\n removedHandle\n ),\n ),\n );\n }\n }}\n className=\"text-gray-400 transition-colors hover:text-red-500\"\n >\n \n \n )}\n
\n {\n const cases = [\n ...(selectedNode.data.config\n ?.cases || []),\n ];\n cases[idx] = {\n ...cases[idx],\n name: e.target.value,\n };\n handleUpdateNodeData({\n config: {\n ...(selectedNode.data.config ||\n {}),\n cases,\n },\n });\n }}\n className=\"border-light-silver focus:ring-purple-30 mb-2 w-full rounded-xl border bg-white px-3 py-2 text-sm transition-all outline-none focus:ring-2 dark:border-[#3A3A3A] dark:bg-[#383838] dark:text-white\"\n placeholder=\"Case name (optional)\"\n />\n {(selectedNode.data.config?.mode ||\n 'simple') === 'simple' ? (\n
\n {\n const parsed = parseSimpleCel(\n c.expression,\n );\n const cases = [\n ...(selectedNode.data.config\n ?.cases || []),\n ];\n cases[idx] = {\n ...cases[idx],\n expression: buildSimpleCel(\n e.target.value,\n parsed.operator,\n parsed.value,\n ),\n };\n handleUpdateNodeData({\n config: {\n ...(selectedNode.data\n .config || {}),\n cases,\n },\n });\n }}\n className=\"border-light-silver focus:ring-purple-30 w-full rounded-xl border bg-white px-3 py-2 text-sm outline-none focus:ring-2 dark:border-[#3A3A3A] dark:bg-[#383838] dark:text-white\"\n placeholder=\"Variable\"\n />\n {\n const parsed = parseSimpleCel(\n c.expression,\n );\n const cases = [\n ...(selectedNode.data.config\n ?.cases || []),\n ];\n cases[idx] = {\n ...cases[idx],\n expression: buildSimpleCel(\n parsed.variable,\n op,\n parsed.value,\n ),\n };\n handleUpdateNodeData({\n config: {\n ...(selectedNode.data\n .config || {}),\n cases,\n },\n });\n }}\n >\n \n \n \n \n \n =\n \n \n !=\n \n \">\n >\n \n \n <\n \n =\">\n >=\n \n \n <=\n \n \n contains\n \n \n starts\n \n \n \n {\n const parsed = parseSimpleCel(\n c.expression,\n );\n const cases = [\n ...(selectedNode.data.config\n ?.cases || []),\n ];\n cases[idx] = {\n ...cases[idx],\n expression: buildSimpleCel(\n parsed.variable,\n parsed.operator,\n e.target.value,\n ),\n };\n handleUpdateNodeData({\n config: {\n ...(selectedNode.data\n .config || {}),\n cases,\n },\n });\n }}\n className=\"border-light-silver focus:ring-purple-30 w-full rounded-xl border bg-white px-3 py-2 text-sm outline-none focus:ring-2 dark:border-[#3A3A3A] dark:bg-[#383838] dark:text-white\"\n placeholder=\"Value\"\n />\n
\n ) : (\n <>\n {\n const cases = [\n ...(selectedNode.data.config\n ?.cases || []),\n ];\n cases[idx] = {\n ...cases[idx],\n expression: e.target.value,\n };\n handleUpdateNodeData({\n config: {\n ...(selectedNode.data\n .config || {}),\n cases,\n },\n });\n }}\n className=\"border-light-silver focus:ring-purple-30 w-full rounded-xl border bg-white px-3 py-2 text-sm transition-all outline-none focus:ring-2 dark:border-[#3A3A3A] dark:bg-[#383838] dark:text-white\"\n rows={2}\n placeholder=\"Enter condition, e.g. input == 5\"\n />\n

\n Use Common Expression Language to\n create a custom expression.{' '}\n \n Learn more\n \n

\n \n )}\n
\n ),\n )}\n\n {\n const cases = normalizeConditionCases([\n ...(selectedNode.data.config?.cases ||\n []),\n ]);\n const nextHandle =\n getNextConditionHandle(cases);\n cases.push({\n name: '',\n expression: '',\n sourceHandle: nextHandle,\n });\n handleUpdateNodeData({\n config: {\n ...(selectedNode.data.config || {}),\n cases,\n },\n });\n }}\n className=\"flex items-center gap-1 rounded-lg px-3 py-1.5 text-sm font-medium text-gray-600 transition-colors hover:bg-gray-100 dark:text-gray-400 dark:hover:bg-[#383838]\"\n >\n \n Add\n \n \n )}\n \n )}\n
\n\n \n \n {selectedNode?.type === 'start'\n ? 'Cannot Delete Start Node'\n : 'Delete Node'}\n \n \n \n \n )}\n \n \n\n \n \n n.type !== 'note')\n .map((n) => ({\n id: n.id,\n type: n.type as 'start' | 'end' | 'agent' | 'state',\n title: n.data.title || n.data.label || n.type,\n position: n.position,\n data: n.type === 'agent' ? n.data.config : n.data,\n })),\n edges: edges.map((e) => ({\n id: e.id,\n source: e.source,\n target: e.target,\n sourceHandle: e.sourceHandle || undefined,\n targetHandle: e.targetHandle || undefined,\n })),\n }}\n />\n \n \n \n {canManageAgent && (\n \n )}\n \n \n );\n}\n\nexport default function WorkflowBuilder() {\n return (\n \n \n \n );\n}\n" }, { "path": "frontend/src/agents/workflow/WorkflowPreview.tsx", "content": "import {\n Bot,\n CheckCircle2,\n Circle,\n Database,\n Flag,\n GitBranch,\n Loader2,\n MessageSquare,\n Play,\n StickyNote,\n Workflow,\n XCircle,\n} from 'lucide-react';\nimport { useCallback, useEffect, useRef, useState } from 'react';\nimport { useDispatch, useSelector } from 'react-redux';\n\nimport { cn } from '@/lib/utils';\n\nimport ChevronDownIcon from '../../assets/chevron-down.svg';\nimport MessageInput from '../../components/MessageInput';\nimport ConversationBubble from '../../conversation/ConversationBubble';\nimport { Query } from '../../conversation/conversationModels';\nimport { AppDispatch } from '../../store';\nimport { WorkflowEdge, WorkflowNode } from '../types/workflow';\nimport {\n addQuery,\n fetchWorkflowPreviewAnswer,\n handleWorkflowPreviewAbort,\n resendQuery,\n resetWorkflowPreview,\n selectActiveNodeId,\n selectWorkflowExecutionSteps,\n selectWorkflowPreviewQueries,\n selectWorkflowPreviewStatus,\n WorkflowExecutionStep,\n WorkflowQuery,\n} from './workflowPreviewSlice';\n\ninterface WorkflowData {\n name: string;\n description?: string;\n nodes: WorkflowNode[];\n edges: WorkflowEdge[];\n}\n\ninterface WorkflowPreviewProps {\n workflowData: WorkflowData;\n}\n\nconst NODE_ICONS: Record = {\n start: ,\n agent: ,\n end: ,\n note: ,\n state: ,\n condition: ,\n};\n\nconst NODE_COLORS: Record = {\n start: 'text-green-600 dark:text-green-400',\n agent: 'text-purple-600 dark:text-purple-400',\n end: 'text-gray-600 dark:text-gray-400',\n note: 'text-yellow-600 dark:text-yellow-400',\n state: 'text-blue-600 dark:text-blue-400',\n condition: 'text-orange-600 dark:text-orange-400',\n};\n\nfunction ExecutionDetails({\n steps,\n nodes,\n isOpen,\n onToggle,\n stepRefs,\n}: {\n steps: WorkflowExecutionStep[];\n nodes: WorkflowNode[];\n isOpen: boolean;\n onToggle: () => void;\n stepRefs?: React.RefObject>;\n}) {\n const completedSteps = steps.filter(\n (s) => s.status === 'completed' || s.status === 'failed',\n );\n\n if (completedSteps.length === 0) return null;\n\n const formatValue = (value: unknown): string => {\n if (typeof value === 'string') return value;\n if (value === undefined) return '';\n const formatted = JSON.stringify(value, null, 2);\n return formatted ?? String(value);\n };\n\n return (\n
\n
\n
\n \n
\n \n
\n \n
\n
\n {completedSteps.map((step, stepIndex) => {\n const node = nodes.find((n) => n.id === step.nodeId);\n const displayName =\n node?.title || node?.data?.title || step.nodeTitle;\n const stateVars = step.stateSnapshot\n ? Object.entries(step.stateSnapshot).filter(\n ([key]) => !['query', 'chat_history'].includes(key),\n )\n : [];\n\n const truncateText = (text: string, maxLength: number) => {\n if (text.length <= maxLength) return text;\n return text.slice(0, maxLength) + '...';\n };\n const hasOutput =\n step.output !== undefined &&\n step.output !== null &&\n formatValue(step.output) !== '';\n const formattedOutput = hasOutput ? formatValue(step.output) : '';\n\n return (\n {\n if (el && stepRefs) stepRefs.current.set(step.nodeId, el);\n }}\n className=\"rounded-xl bg-[#F5F5F5] p-3 dark:bg-[#383838]\"\n >\n
\n \n {stepIndex + 1}.\n \n \n {NODE_ICONS[step.nodeType] || (\n \n )}\n
\n \n {displayName}\n \n
\n {step.status === 'completed' && (\n \n )}\n {step.status === 'failed' && (\n \n )}\n
\n
\n {(hasOutput || step.error || stateVars.length > 0) && (\n
\n {hasOutput && (\n
\n \n Output:{' '}\n \n \n {truncateText(formattedOutput, 300)}\n \n
\n )}\n {step.error && (\n
\n \n Error:{' '}\n \n \n {step.error}\n \n
\n )}\n {stateVars.length > 0 && (\n
\n {stateVars.map(([key, value]) => (\n \n \n {key}:\n \n \n {truncateText(formatValue(value), 50)}\n \n \n ))}\n
\n )}\n
\n )}\n
\n );\n })}\n
\n \n \n \n );\n}\n\nfunction WorkflowMiniMap({\n nodes,\n activeNodeId,\n executionSteps,\n onNodeClick,\n}: {\n nodes: WorkflowNode[];\n activeNodeId: string | null;\n executionSteps: WorkflowExecutionStep[];\n onNodeClick?: (nodeId: string) => void;\n}) {\n const getNodeDisplayName = (node: WorkflowNode) => {\n if (node.type === 'start') return 'Start';\n if (node.type === 'end') return 'End';\n return node.title || node.data?.title || node.type;\n };\n\n const getNodeSubtitle = (node: WorkflowNode) => {\n if (node.type === 'agent' && node.data?.model_id) {\n return node.data.model_id;\n }\n return null;\n };\n\n const getNodeStatus = (nodeId: string) => {\n const step = executionSteps.find((s) => s.nodeId === nodeId);\n return step?.status || 'pending';\n };\n\n const getStatusColor = (nodeId: string) => {\n const status = getNodeStatus(nodeId);\n const isActive = nodeId === activeNodeId;\n\n if (isActive) {\n return 'ring-2 ring-purple-500 bg-purple-100 dark:bg-purple-900/50';\n }\n\n switch (status) {\n case 'completed':\n return 'bg-green-100 dark:bg-green-900/30 border-green-300 dark:border-green-700';\n case 'running':\n return 'bg-purple-100 dark:bg-purple-900/30 border-purple-300 dark:border-purple-700 animate-pulse';\n case 'failed':\n return 'bg-red-100 dark:bg-red-900/30 border-red-300 dark:border-red-700';\n default:\n return 'bg-gray-50 dark:bg-gray-800 border-gray-200 dark:border-gray-700';\n }\n };\n\n const executedOrder = new Map(executionSteps.map((s, i) => [s.nodeId, i]));\n const startNode = nodes.find((node) => node.type === 'start');\n const visibleNodeIds = new Set(executionSteps.map((step) => step.nodeId));\n if (activeNodeId) {\n visibleNodeIds.add(activeNodeId);\n }\n if (startNode) {\n visibleNodeIds.add(startNode.id);\n }\n\n const sortedNodes = nodes\n .filter((node) => visibleNodeIds.has(node.id))\n .sort((a, b) => {\n if (a.type === 'start') return -1;\n if (b.type === 'start') return 1;\n\n const aIdx = executedOrder.get(a.id);\n const bIdx = executedOrder.get(b.id);\n if (aIdx !== undefined && bIdx !== undefined) return aIdx - bIdx;\n if (aIdx !== undefined) return -1;\n if (bIdx !== undefined) return 1;\n return (a.position?.y || 0) - (b.position?.y || 0);\n });\n\n const hasStepData = (nodeId: string) => {\n const step = executionSteps.find((s) => s.nodeId === nodeId);\n return step && (step.status === 'completed' || step.status === 'failed');\n };\n\n return (\n
\n {sortedNodes.map((node, index) => (\n
\n {index < sortedNodes.length - 1 && (\n
\n )}\n\n hasStepData(node.id) && onNodeClick?.(node.id)}\n disabled={!hasStepData(node.id)}\n className={cn(\n 'flex h-12 w-full items-center gap-2 rounded-lg border px-3 text-xs transition-all',\n getStatusColor(node.id),\n hasStepData(node.id) && 'cursor-pointer hover:opacity-80',\n )}\n >\n \n {NODE_ICONS[node.type] || }\n
\n
\n
\n {getNodeDisplayName(node)}\n
\n {getNodeSubtitle(node) && (\n
\n {getNodeSubtitle(node)}\n
\n )}\n
\n
\n {getNodeStatus(node.id) === 'running' && (\n \n )}\n {getNodeStatus(node.id) === 'completed' && (\n \n )}\n {getNodeStatus(node.id) === 'failed' && (\n \n )}\n
\n \n
\n ))}\n
\n );\n}\n\nexport default function WorkflowPreview({\n workflowData,\n}: WorkflowPreviewProps) {\n const dispatch = useDispatch();\n\n const queries = useSelector(selectWorkflowPreviewQueries) as WorkflowQuery[];\n const status = useSelector(selectWorkflowPreviewStatus);\n const executionSteps = useSelector(selectWorkflowExecutionSteps);\n const activeNodeId = useSelector(selectActiveNodeId);\n\n const [lastQueryReturnedErr, setLastQueryReturnedErr] = useState(false);\n const [openDetailsIndex, setOpenDetailsIndex] = useState(null);\n\n const fetchStream = useRef<{ abort: () => void } | null>(null);\n const stepRefs = useRef>(new Map());\n const chatContainerRef = useRef(null);\n\n const scrollToStep = useCallback(\n (nodeId: string) => {\n const lastQueryIndex = queries.length - 1;\n if (lastQueryIndex >= 0) {\n setOpenDetailsIndex(lastQueryIndex);\n setTimeout(() => {\n const stepEl = stepRefs.current.get(nodeId);\n if (stepEl) {\n stepEl.scrollIntoView({ behavior: 'smooth', block: 'center' });\n }\n }, 100);\n }\n },\n [queries.length],\n );\n\n const handleFetchAnswer = useCallback(\n ({ question, index }: { question: string; index?: number }) => {\n const promise = dispatch(\n fetchWorkflowPreviewAnswer({\n question,\n workflowData,\n indx: index,\n }),\n );\n fetchStream.current = promise;\n },\n [dispatch, workflowData],\n );\n\n const handleQuestion = useCallback(\n ({\n question,\n isRetry = false,\n index = undefined,\n }: {\n question: string;\n isRetry?: boolean;\n index?: number;\n }) => {\n const trimmedQuestion = question.trim();\n if (trimmedQuestion === '') return;\n\n if (index !== undefined) {\n if (!isRetry) dispatch(resendQuery({ index, prompt: trimmedQuestion }));\n handleFetchAnswer({ question: trimmedQuestion, index });\n } else {\n if (!isRetry) {\n const newQuery: Query = { prompt: trimmedQuestion };\n dispatch(addQuery(newQuery));\n }\n handleFetchAnswer({ question: trimmedQuestion, index: undefined });\n }\n },\n [dispatch, handleFetchAnswer],\n );\n\n const handleQuestionSubmission = (\n question?: string,\n updated?: boolean,\n indx?: number,\n ) => {\n if (updated === true && question !== undefined && indx !== undefined) {\n handleQuestion({\n question,\n index: indx,\n isRetry: false,\n });\n } else if (question && status !== 'loading') {\n const currentInput = question.trim();\n if (lastQueryReturnedErr && queries.length > 0) {\n const lastQueryIndex = queries.length - 1;\n handleQuestion({\n question: currentInput,\n isRetry: true,\n index: lastQueryIndex,\n });\n } else {\n handleQuestion({\n question: currentInput,\n isRetry: false,\n index: undefined,\n });\n }\n }\n };\n\n useEffect(() => {\n dispatch(resetWorkflowPreview());\n return () => {\n if (fetchStream.current) fetchStream.current.abort();\n handleWorkflowPreviewAbort();\n dispatch(resetWorkflowPreview());\n };\n }, [dispatch]);\n\n useEffect(() => {\n if (queries.length > 0) {\n const lastQuery = queries[queries.length - 1];\n setLastQueryReturnedErr(!!lastQuery.error);\n } else setLastQueryReturnedErr(false);\n }, [queries]);\n\n const lastQuerySteps =\n queries.length > 0 ? queries[queries.length - 1].executionSteps || [] : [];\n\n return (\n
\n
\n
\n
\n \n
\n
\n

\n Preview\n

\n

\n {workflowData.name}\n {workflowData.description && ` - ${workflowData.description}`}\n

\n
\n
\n {status === 'loading' && (\n \n \n Running\n \n )}\n
\n\n
\n
\n
\n

\n Workflow\n

\n
\n
\n 0 ? lastQuerySteps : executionSteps\n }\n onNodeClick={scrollToStep}\n />\n
\n
\n\n
\n \n {queries.length === 0 ? (\n
\n
\n \n
\n

\n Test the workflow\n

\n
\n ) : (\n
\n {queries.map((query, index) => {\n const querySteps = query.executionSteps || [];\n const hasResponse = !!(query.response || query.error);\n const isLastQuery = index === queries.length - 1;\n const isStreamingLastQuery =\n status === 'loading' && isLastQuery;\n const shouldShowThought =\n !isStreamingLastQuery && Boolean(query.thought);\n const isOpen =\n openDetailsIndex === index ||\n (!hasResponse && isLastQuery && querySteps.length > 0);\n\n return (\n
\n {/* Query bubble */}\n \n\n {/* Execution Details */}\n {querySteps.length > 0 && (\n \n setOpenDetailsIndex(\n openDetailsIndex === index ? null : index,\n )\n }\n stepRefs={isLastQuery ? stepRefs : undefined}\n />\n )}\n\n {/* Response bubble */}\n {(query.response ||\n shouldShowThought ||\n query.tool_calls) && (\n \n )}\n\n {/* Error bubble */}\n {query.error && (\n \n )}\n
\n );\n })}\n
\n )}\n
\n
\n handleQuestionSubmission(text)}\n loading={status === 'loading'}\n showSourceButton={false}\n showToolButton={false}\n autoFocus={true}\n />\n
\n
\n
\n \n );\n}\n" }, { "path": "frontend/src/agents/workflow/components/MobileBlocker.tsx", "content": "import { Monitor } from 'lucide-react';\n\nexport default function MobileBlocker() {\n return (\n
\n
\n \n
\n

\n Desktop Required\n

\n

\n The Workflow Builder requires a larger screen for the best experience.\n Please open this page on a desktop or laptop computer.\n

\n
\n );\n}\n" }, { "path": "frontend/src/agents/workflow/components/PromptTextArea.tsx", "content": "import { Braces, Plus, Search } from 'lucide-react';\nimport { useCallback, useEffect, useMemo, useRef, useState } from 'react';\nimport { Edge, Node } from 'reactflow';\n\nimport {\n Popover,\n PopoverContent,\n PopoverTrigger,\n} from '@/components/ui/popover';\n\ninterface WorkflowVariable {\n label: string;\n templatePath: string;\n section: string;\n}\n\nconst GLOBAL_CONTEXT_VARIABLES: WorkflowVariable[] = [\n {\n label: 'source.content',\n templatePath: 'source.content',\n section: 'Global context',\n },\n {\n label: 'source.summaries',\n templatePath: 'source.summaries',\n section: 'Global context',\n },\n {\n label: 'source.documents',\n templatePath: 'source.documents',\n section: 'Global context',\n },\n {\n label: 'source.count',\n templatePath: 'source.count',\n section: 'Global context',\n },\n {\n label: 'system.date',\n templatePath: 'system.date',\n section: 'Global context',\n },\n {\n label: 'system.time',\n templatePath: 'system.time',\n section: 'Global context',\n },\n {\n label: 'system.timestamp',\n templatePath: 'system.timestamp',\n section: 'Global context',\n },\n {\n label: 'system.request_id',\n templatePath: 'system.request_id',\n section: 'Global context',\n },\n {\n label: 'system.user_id',\n templatePath: 'system.user_id',\n section: 'Global context',\n },\n];\n\nfunction toAgentTemplatePath(variableName: string): string {\n const trimmed = variableName.trim();\n if (!trimmed) return 'agent';\n\n if (/^[A-Za-z_][A-Za-z0-9_]*$/.test(trimmed)) {\n return `agent.${trimmed}`;\n }\n\n const escaped = trimmed.replace(/\\\\/g, '\\\\\\\\').replace(/'/g, \"\\\\'\");\n return `agent['${escaped}']`;\n}\n\nfunction getUpstreamNodeIds(nodeId: string, edges: Edge[]): Set {\n const upstream = new Set();\n const queue = [nodeId];\n\n while (queue.length > 0) {\n const current = queue.shift()!;\n for (const edge of edges) {\n if (edge.target === current && !upstream.has(edge.source)) {\n upstream.add(edge.source);\n queue.push(edge.source);\n }\n }\n }\n\n return upstream;\n}\n\nfunction extractUpstreamVariables(\n nodes: Node[],\n edges: Edge[],\n selectedNodeId: string,\n): WorkflowVariable[] {\n const variables: WorkflowVariable[] = [\n {\n label: 'agent.query',\n templatePath: 'agent.query',\n section: 'Workflow input',\n },\n {\n label: 'agent.chat_history',\n templatePath: 'agent.chat_history',\n section: 'Workflow input',\n },\n ...GLOBAL_CONTEXT_VARIABLES,\n ];\n const seen = new Set(variables.map((variable) => variable.templatePath));\n const upstreamIds = getUpstreamNodeIds(selectedNodeId, edges);\n\n for (const node of nodes) {\n if (!upstreamIds.has(node.id)) continue;\n\n if (node.type === 'agent') {\n const defaultOutputTemplatePath = toAgentTemplatePath(\n `node_${node.id}_output`,\n );\n if (!seen.has(defaultOutputTemplatePath)) {\n seen.add(defaultOutputTemplatePath);\n variables.push({\n label: defaultOutputTemplatePath,\n templatePath: defaultOutputTemplatePath,\n section: node.data.title || node.data.label || 'Agent',\n });\n }\n\n const outputVariable = String(\n node.data?.config?.output_variable || '',\n ).trim();\n if (outputVariable) {\n const templatePath = toAgentTemplatePath(outputVariable);\n if (!seen.has(templatePath)) {\n seen.add(templatePath);\n variables.push({\n label: templatePath,\n templatePath,\n section: node.data.title || node.data.label || 'Agent',\n });\n }\n }\n }\n\n if (node.type === 'state') {\n const operations = node.data?.config?.operations;\n if (!Array.isArray(operations)) continue;\n\n for (const operation of operations) {\n const targetVariable = String(operation?.target_variable || '').trim();\n if (!targetVariable) continue;\n\n const templatePath = toAgentTemplatePath(targetVariable);\n if (seen.has(templatePath)) continue;\n\n seen.add(templatePath);\n variables.push({\n label: templatePath,\n templatePath,\n section: node.data.title || node.data.label || 'Set State',\n });\n }\n }\n }\n\n return variables;\n}\n\nfunction groupBySection(\n vars: WorkflowVariable[],\n): Map {\n const groups = new Map();\n for (const v of vars) {\n const list = groups.get(v.section) ?? [];\n list.push(v);\n groups.set(v.section, list);\n }\n return groups;\n}\n\nfunction HighlightedOverlay({ text }: { text: string }) {\n const parts = text.split(/(\\{\\{[^}]*\\}\\})/g);\n return (\n <>\n {parts.map((part, i) =>\n /^\\{\\{[^}]*\\}\\}$/.test(part) ? (\n \n {part}\n \n ) : (\n \n {part}\n \n ),\n )}\n \n );\n}\n\nfunction VariableListWithSearch({\n variables,\n onSelect,\n}: {\n variables: WorkflowVariable[];\n onSelect: (templatePath: string) => void;\n}) {\n const [search, setSearch] = useState('');\n\n const filtered = useMemo(\n () =>\n variables.filter((v) =>\n `${v.label} ${v.templatePath}`\n .toLowerCase()\n .includes(search.toLowerCase()),\n ),\n [variables, search],\n );\n\n const grouped = useMemo(() => groupBySection(filtered), [filtered]);\n\n return (\n
\n
\n \n setSearch(e.target.value)}\n placeholder=\"Search variables...\"\n className=\"placeholder:text-muted-foreground w-full bg-transparent text-sm text-gray-800 outline-none dark:text-gray-200\"\n />\n
\n\n
\n {filtered.length === 0 ? (\n
\n No variables found\n
\n ) : (\n Array.from(grouped.entries()).map(([section, vars]) => (\n
\n
\n {section}\n
\n {vars.map((v) => (\n {\n e.preventDefault();\n e.stopPropagation();\n onSelect(v.templatePath);\n }}\n className=\"flex w-full cursor-pointer items-center gap-2 px-3 py-1.5 text-left text-sm transition-colors hover:bg-gray-50 dark:hover:bg-[#383838]\"\n >\n \n \n {v.label}\n \n \n ))}\n
\n ))\n )}\n
\n
\n );\n}\n\ninterface PromptTextAreaProps {\n value: string;\n onChange: (value: string) => void;\n nodes: Node[];\n edges: Edge[];\n selectedNodeId: string;\n placeholder?: string;\n rows?: number;\n label?: string;\n}\n\nexport default function PromptTextArea({\n value,\n onChange,\n nodes,\n edges,\n selectedNodeId,\n placeholder,\n rows = 4,\n label,\n}: PromptTextAreaProps) {\n const textareaRef = useRef(null);\n const overlayRef = useRef(null);\n const dropdownRef = useRef(null);\n const wrapperRef = useRef(null);\n const [showDropdown, setShowDropdown] = useState(false);\n const [dropdownPos, setDropdownPos] = useState({ top: 0, left: 0 });\n const [filterText, setFilterText] = useState('');\n const [cursorInsertPos, setCursorInsertPos] = useState(null);\n const [contextOpen, setContextOpen] = useState(false);\n\n const variables = useMemo(\n () => extractUpstreamVariables(nodes, edges, selectedNodeId),\n [nodes, edges, selectedNodeId],\n );\n const filtered = useMemo(\n () =>\n variables.filter((v) =>\n `${v.label} ${v.templatePath}`\n .toLowerCase()\n .includes(filterText.toLowerCase()),\n ),\n [variables, filterText],\n );\n\n const checkForTrigger = useCallback(() => {\n const textarea = textareaRef.current;\n if (!textarea) return;\n\n const cursorPos = textarea.selectionStart;\n const textBeforeCursor = value.slice(0, cursorPos);\n const triggerMatch = textBeforeCursor.match(\n /\\{\\{\\s*([A-Za-z0-9_.[\\]'\"]*)$/,\n );\n\n if (triggerMatch) {\n setFilterText(triggerMatch[1].trim());\n setCursorInsertPos(cursorPos);\n\n const wrapper = wrapperRef.current;\n if (!wrapper) return;\n\n setDropdownPos({\n top: wrapper.offsetHeight + 4,\n left: 0,\n });\n setShowDropdown(true);\n } else {\n setShowDropdown(false);\n }\n }, [value]);\n\n const insertVariable = useCallback(\n (templatePath: string) => {\n if (cursorInsertPos === null) return;\n\n const textBeforeCursor = value.slice(0, cursorInsertPos);\n const triggerMatch = textBeforeCursor.match(\n /\\{\\{\\s*([A-Za-z0-9_.[\\]'\"]*)$/,\n );\n if (!triggerMatch) return;\n\n const startPos = cursorInsertPos - triggerMatch[0].length;\n const insertion = `{{ ${templatePath} }}`;\n const newValue =\n value.slice(0, startPos) + insertion + value.slice(cursorInsertPos);\n\n onChange(newValue);\n setShowDropdown(false);\n\n requestAnimationFrame(() => {\n const newCursorPos = startPos + insertion.length;\n textareaRef.current?.setSelectionRange(newCursorPos, newCursorPos);\n textareaRef.current?.focus();\n });\n },\n [value, cursorInsertPos, onChange],\n );\n\n const insertVariableFromButton = useCallback(\n (templatePath: string) => {\n const textarea = textareaRef.current;\n const cursorPos = textarea?.selectionStart ?? value.length;\n const insertion = `{{ ${templatePath} }}`;\n const newValue =\n value.slice(0, cursorPos) + insertion + value.slice(cursorPos);\n\n onChange(newValue);\n setContextOpen(false);\n\n requestAnimationFrame(() => {\n const newCursorPos = cursorPos + insertion.length;\n textareaRef.current?.setSelectionRange(newCursorPos, newCursorPos);\n textareaRef.current?.focus();\n });\n },\n [value, onChange],\n );\n\n useEffect(() => {\n const handleClickOutside = (e: MouseEvent) => {\n if (\n dropdownRef.current &&\n !dropdownRef.current.contains(e.target as HTMLElement)\n ) {\n setShowDropdown(false);\n }\n };\n if (showDropdown) {\n document.addEventListener('mousedown', handleClickOutside);\n return () =>\n document.removeEventListener('mousedown', handleClickOutside);\n }\n }, [showDropdown]);\n\n return (\n
\n {label && (\n \n )}\n \n \n {value ? (\n \n ) : (\n \n {placeholder}\n \n )}\n
\n\n {\n onChange(e.target.value);\n setTimeout(checkForTrigger, 0);\n }}\n onKeyUp={checkForTrigger}\n onKeyDown={(e) => {\n if (showDropdown && e.key === 'Escape') {\n e.preventDefault();\n e.stopPropagation();\n setShowDropdown(false);\n }\n }}\n onScroll={() => {\n if (overlayRef.current && textareaRef.current) {\n overlayRef.current.scrollTop = textareaRef.current.scrollTop;\n }\n }}\n className=\"relative w-full rounded-xl bg-transparent px-3 pt-2 pb-8 text-sm caret-black outline-none dark:caret-white\"\n style={{\n color: 'transparent',\n WebkitTextFillColor: 'transparent',\n }}\n rows={rows}\n placeholder={placeholder}\n spellCheck={false}\n />\n\n
\n \n \n \n \n Add context\n \n \n e.preventDefault()}\n >\n \n \n \n
\n\n {showDropdown && filtered.length > 0 && (\n \n \n \n )}\n \n \n );\n}\n" }, { "path": "frontend/src/agents/workflow/nodes/BaseNode.tsx", "content": "import React, { ReactNode } from 'react';\nimport { Handle, Position } from 'reactflow';\n\ninterface BaseNodeProps {\n title: string;\n children?: ReactNode;\n selected?: boolean;\n type?: 'start' | 'end' | 'default' | 'state' | 'agent' | 'condition';\n icon?: ReactNode;\n handles?: {\n source?: boolean;\n target?: boolean;\n };\n}\n\nexport const BaseNode: React.FC = ({\n title,\n children,\n selected,\n type = 'default',\n icon,\n handles = { source: true, target: true },\n}) => {\n let bgColor = 'bg-white dark:bg-[#2C2C2C]';\n let borderColor = 'border-gray-200 dark:border-[#3A3A3A]';\n let iconBg = 'bg-gray-100 dark:bg-gray-800';\n let iconColor = 'text-gray-600 dark:text-gray-400';\n\n if (selected) {\n borderColor =\n 'border-violets-are-blue ring-2 ring-purple-300 dark:ring-violets-are-blue';\n }\n\n if (type === 'start') {\n iconBg = 'bg-green-100 dark:bg-green-900/30';\n iconColor = 'text-green-600 dark:text-green-400';\n } else if (type === 'end') {\n iconBg = 'bg-red-100 dark:bg-red-900/30';\n iconColor = 'text-red-600 dark:text-red-400';\n } else if (type === 'state') {\n iconBg = 'bg-gray-100 dark:bg-gray-800';\n iconColor = 'text-gray-600 dark:text-gray-400';\n } else if (type === 'condition') {\n iconBg = 'bg-orange-100 dark:bg-orange-900/30';\n iconColor = 'text-orange-600 dark:text-orange-400';\n }\n\n return (\n \n {handles.target && (\n \n )}\n\n
\n \n {icon}\n
\n
\n \n {title}\n
\n {children && (\n
\n {children}\n
\n )}\n \n \n\n {handles.source && (\n \n )}\n \n );\n};\n" }, { "path": "frontend/src/agents/workflow/nodes/ConditionNode.tsx", "content": "import { GitBranch } from 'lucide-react';\nimport { memo } from 'react';\nimport { Handle, NodeProps, Position } from 'reactflow';\n\nimport { ConditionCase } from '../../types/workflow';\n\ntype ConditionNodeData = {\n label?: string;\n title?: string;\n config?: {\n mode?: 'simple' | 'advanced';\n cases?: ConditionCase[];\n };\n};\n\nconst ROW_HEIGHT = 18;\nconst HEADER_HEIGHT = 52;\nconst PADDING_BOTTOM = 8;\n\nfunction getNodeHeight(caseCount: number): number {\n return (\n HEADER_HEIGHT + Math.max(caseCount + 1, 2) * ROW_HEIGHT + PADDING_BOTTOM\n );\n}\n\nfunction getHandleTop(index: number, total: number): string {\n const offset = HEADER_HEIGHT;\n return `${offset + ROW_HEIGHT * index + ROW_HEIGHT / 2}px`;\n}\n\nconst ConditionNode = ({ data, selected }: NodeProps) => {\n const title = data.title || data.label || 'If / Else';\n const cases = data.config?.cases || [];\n const totalOutputs = cases.length + 1;\n const height = getNodeHeight(cases.length);\n\n return (\n \n \n\n
\n
\n \n
\n
\n \n {title}\n
\n
\n {data.config?.mode || 'simple'}\n
\n
\n \n\n
\n {cases.map((c, i) => (\n \n \n {i === 0 ? 'If' : 'Else if'}\n \n {c.name && (\n \n {c.name}\n \n )}\n
\n ))}\n
\n Else\n
\n \n\n {cases.map((c, i) => (\n \n ))}\n \n \n );\n};\n\nexport default memo(ConditionNode);\n" }, { "path": "frontend/src/agents/workflow/nodes/SetStateNode.tsx", "content": "import { Database } from 'lucide-react';\nimport { memo } from 'react';\nimport { NodeProps } from 'reactflow';\n\nimport { StateOperationConfig } from '../../types/workflow';\nimport { BaseNode } from './BaseNode';\n\ntype SetStateNodeData = {\n label?: string;\n title?: string;\n variable?: string;\n value?: string;\n config?: {\n operations?: StateOperationConfig[];\n };\n};\n\nconst SetStateNode = ({ data, selected }: NodeProps) => {\n const title = data.title || data.label || 'Set State';\n const operations = data.config?.operations || [];\n const hasLegacy = !operations.length && data.variable;\n\n return (\n }\n handles={{ source: true, target: true }}\n >\n
\n {operations.length > 0 ? (\n \n {operations.length} variable{operations.length !== 1 ? 's' : ''}\n
\n ) : hasLegacy ? (\n <>\n \n {data.variable}\n \n {data.value && (\n \n {data.value}\n \n )}\n \n ) : null}\n \n \n );\n};\n\nexport default memo(SetStateNode);\n" }, { "path": "frontend/src/agents/workflow/nodes/index.tsx", "content": "import { Bot, Flag, Play, StickyNote } from 'lucide-react';\nimport { memo } from 'react';\n\nimport { BaseNode } from './BaseNode';\nimport ConditionNode from './ConditionNode';\nimport SetStateNode from './SetStateNode';\n\nexport const StartNode = memo(function StartNode({\n selected,\n}: {\n selected: boolean;\n}) {\n return (\n }\n >\n
Entry point of the workflow
\n \n );\n});\n\nexport const EndNode = memo(function EndNode({\n selected,\n}: {\n selected: boolean;\n}) {\n return (\n }\n >\n
Workflow completion
\n \n );\n});\n\nexport const AgentNode = memo(function AgentNode({\n data,\n selected,\n}: {\n data: {\n title?: string;\n label?: string;\n config?: {\n agent_type?: string;\n model_id?: string;\n prompt_template?: string;\n output_variable?: string;\n };\n };\n selected: boolean;\n}) {\n const title = data.title || data.label || 'Agent';\n const config = data.config || {};\n return (\n }\n >\n
\n {config.agent_type && (\n \n {config.agent_type}\n
\n )}\n {config.model_id && (\n \n {config.model_id}\n \n )}\n {config.output_variable && (\n \n Output: {config.output_variable}\n \n )}\n \n \n );\n});\n\nexport const NoteNode = memo(function NoteNode({\n data,\n selected,\n}: {\n data: { title?: string; label?: string; content?: string };\n selected: boolean;\n}) {\n const title = data.title || data.label || 'Note';\n const maxContentLength = 120;\n const displayContent =\n data.content && data.content.length > maxContentLength\n ? `${data.content.substring(0, maxContentLength)}...`\n : data.content;\n\n return (\n \n
\n
\n \n
\n
\n \n {title}\n
\n {displayContent && (\n \n {displayContent}\n
\n )}\n \n \n \n );\n});\n\nexport { SetStateNode };\nexport { ConditionNode };\n" }, { "path": "frontend/src/agents/workflow/workflowPreviewSlice.ts", "content": "import { createAsyncThunk, createSlice, PayloadAction } from '@reduxjs/toolkit';\n\nimport conversationService from '../../api/services/conversationService';\nimport { Query, Status } from '../../conversation/conversationModels';\nimport { WorkflowEdge, WorkflowNode } from '../types/workflow';\n\nexport interface WorkflowExecutionStep {\n nodeId: string;\n nodeType: string;\n nodeTitle: string;\n status: 'pending' | 'running' | 'completed' | 'failed';\n reasoning?: string;\n startedAt?: number;\n completedAt?: number;\n stateSnapshot?: Record;\n output?: unknown;\n error?: string;\n}\n\ninterface WorkflowData {\n name: string;\n description?: string;\n nodes: WorkflowNode[];\n edges: WorkflowEdge[];\n}\n\nexport interface WorkflowQuery extends Query {\n executionSteps?: WorkflowExecutionStep[];\n}\n\nexport interface WorkflowPreviewState {\n queries: WorkflowQuery[];\n status: Status;\n executionSteps: WorkflowExecutionStep[];\n activeNodeId: string | null;\n}\n\nconst initialState: WorkflowPreviewState = {\n queries: [],\n status: 'idle',\n executionSteps: [],\n activeNodeId: null,\n};\n\nlet abortController: AbortController | null = null;\n\nexport function handleWorkflowPreviewAbort() {\n if (abortController) {\n abortController.abort();\n abortController = null;\n }\n}\n\ninterface ThunkState {\n preference: {\n token: string | null;\n };\n workflowPreview: WorkflowPreviewState;\n}\n\nexport const fetchWorkflowPreviewAnswer = createAsyncThunk<\n void,\n {\n question: string;\n workflowData: WorkflowData;\n indx?: number;\n },\n { state: ThunkState }\n>(\n 'workflowPreview/fetchAnswer',\n async ({ question, workflowData, indx }, { dispatch, getState }) => {\n if (abortController) abortController.abort();\n abortController = new AbortController();\n const { signal } = abortController;\n\n const state = getState();\n\n if (state.preference) {\n const payload = {\n question,\n workflow: workflowData,\n save_conversation: false,\n };\n\n await new Promise((resolve, reject) => {\n conversationService\n .answerStream(payload, state.preference.token, signal)\n .then((response) => {\n if (!response.body) throw Error('No response body');\n\n let buffer = '';\n const reader = response.body.getReader();\n const decoder = new TextDecoder('utf-8');\n\n const processStream = ({\n done,\n value,\n }: ReadableStreamReadResult): Promise | void => {\n if (done) {\n resolve();\n return;\n }\n\n buffer += decoder.decode(value, { stream: true });\n const lines = buffer.split('\\n');\n buffer = lines.pop() || '';\n\n const currentState = getState();\n\n for (const line of lines) {\n if (line.startsWith('data:')) {\n try {\n const data = JSON.parse(line.slice(5));\n const targetIndex =\n indx ?? currentState.workflowPreview.queries.length - 1;\n\n if (data.type === 'end') {\n dispatch(workflowPreviewSlice.actions.setStatus('idle'));\n } else if (data.type === 'thought') {\n dispatch(\n updateThought({\n index: targetIndex,\n query: { thought: data.thought },\n }),\n );\n } else if (data.type === 'workflow_step') {\n dispatch(\n updateExecutionStep({\n index: targetIndex,\n step: {\n nodeId: data.node_id,\n nodeType: data.node_type,\n nodeTitle: data.node_title,\n status: data.status,\n reasoning: data.reasoning,\n stateSnapshot: data.state_snapshot,\n output: data.output,\n error: data.error,\n },\n }),\n );\n if (data.status === 'running') {\n dispatch(setActiveNodeId(data.node_id));\n }\n } else if (data.type === 'source') {\n dispatch(\n updateStreamingSource({\n index: targetIndex,\n query: { sources: data.source ?? [] },\n }),\n );\n } else if (data.type === 'tool_call') {\n dispatch(\n updateToolCall({\n index: targetIndex,\n tool_call: data.data,\n }),\n );\n } else if (data.type === 'error') {\n dispatch(\n workflowPreviewSlice.actions.setStatus('failed'),\n );\n dispatch(\n workflowPreviewSlice.actions.raiseError({\n index: targetIndex,\n message: data.error,\n }),\n );\n } else if (data.type === 'structured_answer') {\n dispatch(\n updateStreamingQuery({\n index: targetIndex,\n query: {\n response: data.answer,\n structured: data.structured,\n schema: data.schema,\n },\n }),\n );\n } else if (data.answer !== undefined) {\n dispatch(\n updateStreamingQuery({\n index: targetIndex,\n query: { response: data.answer },\n }),\n );\n }\n } catch {\n /* empty */\n }\n }\n }\n\n return reader.read().then(processStream);\n };\n\n reader.read().then(processStream).catch(reject);\n })\n .catch(reject);\n });\n }\n },\n);\n\nexport const workflowPreviewSlice = createSlice({\n name: 'workflowPreview',\n initialState,\n reducers: {\n addQuery(state, action: PayloadAction) {\n state.queries.push(action.payload);\n },\n resendQuery(\n state,\n action: PayloadAction<{ index: number; prompt: string; query?: Query }>,\n ) {\n state.queries = [\n ...state.queries.slice(0, action.payload.index),\n { prompt: action.payload.prompt },\n ];\n state.executionSteps = [];\n state.activeNodeId = null;\n },\n updateStreamingQuery(\n state,\n action: PayloadAction<{\n index: number;\n query: Partial;\n }>,\n ) {\n const { index, query } = action.payload;\n if (state.status === 'idle') return;\n\n if (query.response !== undefined) {\n state.queries[index].response =\n (state.queries[index].response || '') + query.response;\n }\n\n if (query.structured !== undefined) {\n state.queries[index].structured = query.structured;\n }\n\n if (query.schema !== undefined) {\n state.queries[index].schema = query.schema;\n }\n },\n updateThought(\n state,\n action: PayloadAction<{\n index: number;\n query: Partial;\n }>,\n ) {\n const { index, query } = action.payload;\n if (query.thought !== undefined) {\n state.queries[index].thought =\n (state.queries[index].thought || '') + query.thought;\n }\n },\n updateStreamingSource(\n state,\n action: PayloadAction<{\n index: number;\n query: Partial;\n }>,\n ) {\n const { index, query } = action.payload;\n if (!state.queries[index].sources) {\n state.queries[index].sources = query?.sources;\n } else if (query.sources) {\n state.queries[index].sources!.push(...query.sources);\n }\n },\n updateToolCall(state, action) {\n const { index, tool_call } = action.payload;\n\n if (!state.queries[index].tool_calls) {\n state.queries[index].tool_calls = [];\n }\n\n const existingIndex = state.queries[index].tool_calls.findIndex(\n (call: { call_id: string }) => call.call_id === tool_call.call_id,\n );\n\n if (existingIndex !== -1) {\n const existingCall = state.queries[index].tool_calls[existingIndex];\n state.queries[index].tool_calls[existingIndex] = {\n ...existingCall,\n ...tool_call,\n };\n } else {\n state.queries[index].tool_calls.push(tool_call);\n }\n },\n updateQuery(\n state,\n action: PayloadAction<{ index: number; query: Partial }>,\n ) {\n const { index, query } = action.payload;\n state.queries[index] = {\n ...state.queries[index],\n ...query,\n };\n },\n updateExecutionStep(\n state,\n action: PayloadAction<{\n index: number;\n step: Partial & {\n nodeId: string;\n nodeType: string;\n nodeTitle: string;\n status: WorkflowExecutionStep['status'];\n };\n }>,\n ) {\n const { index, step } = action.payload;\n\n if (!state.queries[index]) return;\n if (!state.queries[index].executionSteps) {\n state.queries[index].executionSteps = [];\n }\n\n const querySteps = state.queries[index].executionSteps!;\n const existingIndex = querySteps.findIndex(\n (s) => s.nodeId === step.nodeId,\n );\n\n const updatedStep: WorkflowExecutionStep = {\n nodeId: step.nodeId,\n nodeType: step.nodeType,\n nodeTitle: step.nodeTitle,\n status: step.status,\n reasoning: step.reasoning,\n stateSnapshot: step.stateSnapshot,\n output: step.output,\n error: step.error,\n startedAt:\n existingIndex !== -1\n ? querySteps[existingIndex].startedAt\n : Date.now(),\n completedAt:\n step.status === 'completed' || step.status === 'failed'\n ? Date.now()\n : existingIndex !== -1\n ? querySteps[existingIndex].completedAt\n : undefined,\n };\n\n if (existingIndex !== -1) {\n updatedStep.stateSnapshot =\n step.stateSnapshot ?? querySteps[existingIndex].stateSnapshot;\n updatedStep.output = step.output ?? querySteps[existingIndex].output;\n updatedStep.error = step.error ?? querySteps[existingIndex].error;\n querySteps[existingIndex] = updatedStep;\n } else {\n querySteps.push(updatedStep);\n }\n\n const globalIndex = state.executionSteps.findIndex(\n (s) => s.nodeId === step.nodeId,\n );\n if (globalIndex !== -1) {\n state.executionSteps[globalIndex] = updatedStep;\n } else {\n state.executionSteps.push(updatedStep);\n }\n },\n setActiveNodeId(state, action: PayloadAction) {\n state.activeNodeId = action.payload;\n },\n setStatus(state, action: PayloadAction) {\n state.status = action.payload;\n },\n raiseError(\n state,\n action: PayloadAction<{\n index: number;\n message: string;\n }>,\n ) {\n const { index, message } = action.payload;\n state.queries[index].error = message;\n },\n resetWorkflowPreview: (state) => {\n state.queries = initialState.queries;\n state.status = initialState.status;\n state.executionSteps = initialState.executionSteps;\n state.activeNodeId = initialState.activeNodeId;\n handleWorkflowPreviewAbort();\n },\n clearExecutionSteps: (state) => {\n state.executionSteps = [];\n state.activeNodeId = null;\n },\n },\n extraReducers(builder) {\n builder\n .addCase(fetchWorkflowPreviewAnswer.pending, (state) => {\n state.status = 'loading';\n state.executionSteps = [];\n state.activeNodeId = null;\n })\n .addCase(fetchWorkflowPreviewAnswer.rejected, (state, action) => {\n if (action.meta.aborted) {\n state.status = 'idle';\n return;\n }\n state.status = 'failed';\n if (state.queries.length > 0) {\n state.queries[state.queries.length - 1].error =\n 'Something went wrong';\n }\n });\n },\n});\n\ninterface RootStateWithWorkflowPreview {\n workflowPreview: WorkflowPreviewState;\n}\n\nexport const selectWorkflowPreviewQueries = (\n state: RootStateWithWorkflowPreview,\n) => state.workflowPreview.queries;\nexport const selectWorkflowPreviewStatus = (\n state: RootStateWithWorkflowPreview,\n) => state.workflowPreview.status;\nexport const selectWorkflowExecutionSteps = (\n state: RootStateWithWorkflowPreview,\n) => state.workflowPreview.executionSteps;\nexport const selectActiveNodeId = (state: RootStateWithWorkflowPreview) =>\n state.workflowPreview.activeNodeId;\n\nexport const {\n addQuery,\n updateQuery,\n resendQuery,\n updateStreamingQuery,\n updateThought,\n updateStreamingSource,\n updateToolCall,\n updateExecutionStep,\n setActiveNodeId,\n setStatus,\n raiseError,\n resetWorkflowPreview,\n clearExecutionSteps,\n} = workflowPreviewSlice.actions;\n\nexport default workflowPreviewSlice.reducer;\n" }, { "path": "frontend/src/api/client.ts", "content": "export const baseURL =\n import.meta.env.VITE_API_HOST || 'https://docsapi.arc53.com';\n\nconst getHeaders = (\n token: string | null,\n customHeaders = {},\n isFormData = false,\n): HeadersInit => {\n const headers: HeadersInit = {\n ...(token ? { Authorization: `Bearer ${token}` } : {}),\n ...customHeaders,\n };\n\n if (!isFormData) {\n headers['Content-Type'] = 'application/json';\n }\n\n return headers;\n};\n\nconst apiClient = {\n get: (\n url: string,\n token: string | null,\n headers = {},\n signal?: AbortSignal,\n ): Promise =>\n fetch(`${baseURL}${url}`, {\n method: 'GET',\n headers: getHeaders(token, headers),\n signal,\n }).then((response) => {\n return response;\n }),\n\n post: (\n url: string,\n data: any,\n token: string | null,\n headers = {},\n signal?: AbortSignal,\n ): Promise =>\n fetch(`${baseURL}${url}`, {\n method: 'POST',\n headers: getHeaders(token, headers),\n body: JSON.stringify(data),\n signal,\n }).then((response) => {\n return response;\n }),\n\n postFormData: (\n url: string,\n formData: FormData,\n token: string | null,\n headers = {},\n signal?: AbortSignal,\n ): Promise => {\n return fetch(`${baseURL}${url}`, {\n method: 'POST',\n headers: getHeaders(token, headers, true),\n body: formData,\n signal,\n });\n },\n\n put: (\n url: string,\n data: any,\n token: string | null,\n headers = {},\n signal?: AbortSignal,\n ): Promise =>\n fetch(`${baseURL}${url}`, {\n method: 'PUT',\n headers: getHeaders(token, headers),\n body: JSON.stringify(data),\n signal,\n }).then((response) => {\n return response;\n }),\n\n putFormData: (\n url: string,\n formData: FormData,\n token: string | null,\n headers = {},\n signal?: AbortSignal,\n ): Promise => {\n return fetch(`${baseURL}${url}`, {\n method: 'PUT',\n headers: getHeaders(token, headers, true),\n body: formData,\n signal,\n });\n },\n\n delete: (\n url: string,\n token: string | null,\n headers = {},\n signal?: AbortSignal,\n ): Promise =>\n fetch(`${baseURL}${url}`, {\n method: 'DELETE',\n headers: getHeaders(token, headers),\n signal,\n }).then((response) => {\n return response;\n }),\n};\n\nexport default apiClient;\n" }, { "path": "frontend/src/api/endpoints.ts", "content": "const endpoints = {\n USER: {\n CONFIG: '/api/config',\n NEW_TOKEN: '/api/generate_token',\n MODELS: '/api/models',\n DOCS: '/api/sources',\n DOCS_PAGINATED: '/api/sources/paginated',\n API_KEYS: '/api/get_api_keys',\n CREATE_API_KEY: '/api/create_api_key',\n DELETE_API_KEY: '/api/delete_api_key',\n AGENT: (id: string) => `/api/get_agent?id=${id}`,\n AGENTS: '/api/get_agents',\n CREATE_AGENT: '/api/create_agent',\n UPDATE_AGENT: (agent_id: string) => `/api/update_agent/${agent_id}`,\n DELETE_AGENT: (id: string) => `/api/delete_agent?id=${id}`,\n PINNED_AGENTS: '/api/pinned_agents',\n TOGGLE_PIN_AGENT: (id: string) => `/api/pin_agent?id=${id}`,\n SHARED_AGENT: (id: string) => `/api/shared_agent?token=${id}`,\n SHARED_AGENTS: '/api/shared_agents',\n SHARE_AGENT: `/api/share_agent`,\n REMOVE_SHARED_AGENT: (id: string) => `/api/remove_shared_agent?id=${id}`,\n TEMPLATE_AGENTS: '/api/template_agents',\n ADOPT_AGENT: (id: string) => `/api/adopt_agent?id=${id}`,\n AGENT_WEBHOOK: (id: string) => `/api/agent_webhook?id=${id}`,\n PROMPTS: '/api/get_prompts',\n CREATE_PROMPT: '/api/create_prompt',\n DELETE_PROMPT: '/api/delete_prompt',\n UPDATE_PROMPT: '/api/update_prompt',\n SINGLE_PROMPT: (id: string) => `/api/get_single_prompt?id=${id}`,\n DELETE_PATH: (docPath: string) => `/api/delete_old?source_id=${docPath}`,\n TASK_STATUS: (task_id: string) => `/api/task_status?task_id=${task_id}`,\n MESSAGE_ANALYTICS: '/api/get_message_analytics',\n TOKEN_ANALYTICS: '/api/get_token_analytics',\n FEEDBACK_ANALYTICS: '/api/get_feedback_analytics',\n LOGS: `/api/get_user_logs`,\n MANAGE_SYNC: '/api/manage_sync',\n SYNC_SOURCE: '/api/sync_source',\n GET_AVAILABLE_TOOLS: '/api/available_tools',\n GET_USER_TOOLS: '/api/get_tools',\n CREATE_TOOL: '/api/create_tool',\n UPDATE_TOOL_STATUS: '/api/update_tool_status',\n UPDATE_TOOL: '/api/update_tool',\n DELETE_TOOL: '/api/delete_tool',\n PARSE_SPEC: '/api/parse_spec',\n SYNC_CONNECTOR: '/api/connectors/sync',\n GET_CHUNKS: (\n docId: string,\n page: number,\n per_page: number,\n path?: string,\n search?: string,\n ) =>\n `/api/get_chunks?id=${docId}&page=${page}&per_page=${per_page}${\n path ? `&path=${encodeURIComponent(path)}` : ''\n }${search ? `&search=${encodeURIComponent(search)}` : ''}`,\n ADD_CHUNK: '/api/add_chunk',\n DELETE_CHUNK: (docId: string, chunkId: string) =>\n `/api/delete_chunk?id=${docId}&chunk_id=${chunkId}`,\n UPDATE_CHUNK: '/api/update_chunk',\n STORE_ATTACHMENT: '/api/store_attachment',\n STT: '/api/stt',\n LIVE_STT_START: '/api/stt/live/start',\n LIVE_STT_CHUNK: '/api/stt/live/chunk',\n LIVE_STT_FINISH: '/api/stt/live/finish',\n DIRECTORY_STRUCTURE: (docId: string) =>\n `/api/directory_structure?id=${docId}`,\n MANAGE_SOURCE_FILES: '/api/manage_source_files',\n MCP_TEST_CONNECTION: '/api/mcp_server/test',\n MCP_SAVE_SERVER: '/api/mcp_server/save',\n MCP_OAUTH_STATUS: (task_id: string) =>\n `/api/mcp_server/oauth_status/${task_id}`,\n MCP_AUTH_STATUS: '/api/mcp_server/auth_status',\n AGENT_FOLDERS: '/api/agents/folders/',\n AGENT_FOLDER: (id: string) => `/api/agents/folders/${id}`,\n MOVE_AGENT_TO_FOLDER: '/api/agents/folders/move_agent',\n GET_ARTIFACT: (artifactId: string) => `/api/artifact/${artifactId}`,\n WORKFLOWS: '/api/workflows',\n WORKFLOW: (id: string) => `/api/workflows/${id}`,\n },\n CONVERSATION: {\n ANSWER: '/api/answer',\n ANSWER_STREAMING: '/stream',\n SEARCH: '/api/search',\n FEEDBACK: '/api/feedback',\n CONVERSATION: (id: string) => `/api/get_single_conversation?id=${id}`,\n CONVERSATIONS: '/api/get_conversations',\n SHARE_CONVERSATION: (isPromptable: boolean) =>\n `/api/share?isPromptable=${isPromptable}`,\n SHARED_CONVERSATION: (identifier: string) =>\n `/api/shared_conversation/${identifier}`,\n DELETE: (id: string) => `/api/delete_conversation?id=${id}`,\n DELETE_ALL: '/api/delete_all_conversations',\n UPDATE: '/api/update_conversation_name',\n },\n};\n\nexport default endpoints;\n" }, { "path": "frontend/src/api/services/conversationService.ts", "content": "import apiClient from '../client';\nimport endpoints from '../endpoints';\n\nconst conversationService = {\n answer: (\n data: any,\n token: string | null,\n signal: AbortSignal,\n ): Promise =>\n apiClient.post(endpoints.CONVERSATION.ANSWER, data, token, {}, signal),\n answerStream: (\n data: any,\n token: string | null,\n signal: AbortSignal,\n ): Promise =>\n apiClient.post(\n endpoints.CONVERSATION.ANSWER_STREAMING,\n data,\n token,\n {},\n signal,\n ),\n search: (data: any, token: string | null): Promise =>\n apiClient.post(endpoints.CONVERSATION.SEARCH, data, token, {}),\n feedback: (data: any, token: string | null): Promise =>\n apiClient.post(endpoints.CONVERSATION.FEEDBACK, data, token, {}),\n getConversation: (id: string, token: string | null): Promise =>\n apiClient.get(endpoints.CONVERSATION.CONVERSATION(id), token, {}),\n getConversations: (token: string | null): Promise =>\n apiClient.get(endpoints.CONVERSATION.CONVERSATIONS, token, {}),\n shareConversation: (\n isPromptable: boolean,\n data: any,\n token: string | null,\n ): Promise =>\n apiClient.post(\n endpoints.CONVERSATION.SHARE_CONVERSATION(isPromptable),\n data,\n token,\n {},\n ),\n getSharedConversation: (\n identifier: string,\n token: string | null,\n ): Promise =>\n apiClient.get(\n endpoints.CONVERSATION.SHARED_CONVERSATION(identifier),\n token,\n {},\n ),\n delete: (id: string, data: any, token: string | null): Promise =>\n apiClient.post(endpoints.CONVERSATION.DELETE(id), data, token, {}),\n deleteAll: (token: string | null): Promise =>\n apiClient.get(endpoints.CONVERSATION.DELETE_ALL, token, {}),\n update: (data: any, token: string | null): Promise =>\n apiClient.post(endpoints.CONVERSATION.UPDATE, data, token, {}),\n};\n\nexport default conversationService;\n" }, { "path": "frontend/src/api/services/modelService.ts", "content": "import apiClient from '../client';\nimport endpoints from '../endpoints';\n\nimport type { AvailableModel, Model } from '../../models/types';\n\nconst modelService = {\n getModels: (token: string | null): Promise =>\n apiClient.get(endpoints.USER.MODELS, token, {}),\n\n transformModels: (models: AvailableModel[]): Model[] =>\n models.map((model) => ({\n id: model.id,\n value: model.id,\n provider: model.provider,\n display_name: model.display_name,\n description: model.description,\n context_window: model.context_window,\n supported_attachment_types: model.supported_attachment_types,\n supports_tools: model.supports_tools,\n supports_structured_output: model.supports_structured_output,\n supports_streaming: model.supports_streaming,\n })),\n};\n\nexport default modelService;\n" }, { "path": "frontend/src/api/services/userService.ts", "content": "import { getSessionToken } from '../../utils/providerUtils';\nimport apiClient from '../client';\nimport endpoints from '../endpoints';\n\nconst userService = {\n getConfig: (): Promise => apiClient.get(endpoints.USER.CONFIG, null),\n getNewToken: (): Promise =>\n apiClient.get(endpoints.USER.NEW_TOKEN, null),\n getDocs: (token: string | null): Promise =>\n apiClient.get(`${endpoints.USER.DOCS}`, token),\n getDocsWithPagination: (query: string, token: string | null): Promise =>\n apiClient.get(`${endpoints.USER.DOCS_PAGINATED}?${query}`, token),\n getAPIKeys: (token: string | null): Promise =>\n apiClient.get(endpoints.USER.API_KEYS, token),\n createAPIKey: (data: any, token: string | null): Promise =>\n apiClient.post(endpoints.USER.CREATE_API_KEY, data, token),\n deleteAPIKey: (data: any, token: string | null): Promise =>\n apiClient.post(endpoints.USER.DELETE_API_KEY, data, token),\n getAgent: (id: string, token: string | null): Promise =>\n apiClient.get(endpoints.USER.AGENT(id), token),\n getAgents: (token: string | null): Promise =>\n apiClient.get(endpoints.USER.AGENTS, token),\n createAgent: (data: any, token: string | null): Promise =>\n apiClient.postFormData(endpoints.USER.CREATE_AGENT, data, token),\n updateAgent: (\n agent_id: string,\n data: any,\n token: string | null,\n ): Promise =>\n apiClient.putFormData(endpoints.USER.UPDATE_AGENT(agent_id), data, token),\n deleteAgent: (id: string, token: string | null): Promise =>\n apiClient.delete(endpoints.USER.DELETE_AGENT(id), token),\n getPinnedAgents: (token: string | null): Promise =>\n apiClient.get(endpoints.USER.PINNED_AGENTS, token),\n togglePinAgent: (id: string, token: string | null): Promise =>\n apiClient.post(endpoints.USER.TOGGLE_PIN_AGENT(id), {}, token),\n getSharedAgent: (id: string, token: string | null): Promise =>\n apiClient.get(endpoints.USER.SHARED_AGENT(id), token),\n getSharedAgents: (token: string | null): Promise =>\n apiClient.get(endpoints.USER.SHARED_AGENTS, token),\n shareAgent: (data: any, token: string | null): Promise =>\n apiClient.put(endpoints.USER.SHARE_AGENT, data, token),\n removeSharedAgent: (id: string, token: string | null): Promise =>\n apiClient.delete(endpoints.USER.REMOVE_SHARED_AGENT(id), token),\n getTemplateAgents: (token: string | null): Promise =>\n apiClient.get(endpoints.USER.TEMPLATE_AGENTS, token),\n adoptAgent: (id: string, token: string | null): Promise =>\n apiClient.post(endpoints.USER.ADOPT_AGENT(id), {}, token),\n getAgentWebhook: (id: string, token: string | null): Promise =>\n apiClient.get(endpoints.USER.AGENT_WEBHOOK(id), token),\n getPrompts: (token: string | null): Promise =>\n apiClient.get(endpoints.USER.PROMPTS, token),\n createPrompt: (data: any, token: string | null): Promise =>\n apiClient.post(endpoints.USER.CREATE_PROMPT, data, token),\n deletePrompt: (data: any, token: string | null): Promise =>\n apiClient.post(endpoints.USER.DELETE_PROMPT, data, token),\n updatePrompt: (data: any, token: string | null): Promise =>\n apiClient.post(endpoints.USER.UPDATE_PROMPT, data, token),\n getSinglePrompt: (id: string, token: string | null): Promise =>\n apiClient.get(endpoints.USER.SINGLE_PROMPT(id), token),\n deletePath: (docPath: string, token: string | null): Promise =>\n apiClient.get(endpoints.USER.DELETE_PATH(docPath), token),\n getTaskStatus: (task_id: string, token: string | null): Promise =>\n apiClient.get(endpoints.USER.TASK_STATUS(task_id), token),\n getMessageAnalytics: (data: any, token: string | null): Promise =>\n apiClient.post(endpoints.USER.MESSAGE_ANALYTICS, data, token),\n getTokenAnalytics: (data: any, token: string | null): Promise =>\n apiClient.post(endpoints.USER.TOKEN_ANALYTICS, data, token),\n getFeedbackAnalytics: (data: any, token: string | null): Promise =>\n apiClient.post(endpoints.USER.FEEDBACK_ANALYTICS, data, token),\n getLogs: (data: any, token: string | null): Promise =>\n apiClient.post(endpoints.USER.LOGS, data, token),\n manageSync: (data: any, token: string | null): Promise =>\n apiClient.post(endpoints.USER.MANAGE_SYNC, data, token),\n syncSource: (data: any, token: string | null): Promise =>\n apiClient.post(endpoints.USER.SYNC_SOURCE, data, token),\n getAvailableTools: (token: string | null): Promise =>\n apiClient.get(endpoints.USER.GET_AVAILABLE_TOOLS, token),\n getUserTools: (token: string | null): Promise =>\n apiClient.get(endpoints.USER.GET_USER_TOOLS, token),\n createTool: (data: any, token: string | null): Promise =>\n apiClient.post(endpoints.USER.CREATE_TOOL, data, token),\n updateToolStatus: (data: any, token: string | null): Promise =>\n apiClient.post(endpoints.USER.UPDATE_TOOL_STATUS, data, token),\n updateTool: (data: any, token: string | null): Promise =>\n apiClient.post(endpoints.USER.UPDATE_TOOL, data, token),\n deleteTool: (data: any, token: string | null): Promise =>\n apiClient.post(endpoints.USER.DELETE_TOOL, data, token),\n parseSpec: (file: File, token: string | null): Promise => {\n const formData = new FormData();\n formData.append('file', file);\n return apiClient.postFormData(endpoints.USER.PARSE_SPEC, formData, token);\n },\n transcribeAudio: (\n file: File,\n token: string | null,\n language?: string,\n ): Promise => {\n const formData = new FormData();\n formData.append('file', file);\n if (language) {\n formData.append('language', language);\n }\n return apiClient.postFormData(endpoints.USER.STT, formData, token);\n },\n startLiveTranscription: (\n token: string | null,\n language?: string,\n ): Promise =>\n apiClient.post(\n endpoints.USER.LIVE_STT_START,\n language ? { language } : {},\n token,\n ),\n transcribeLiveAudioChunk: (\n sessionId: string,\n chunkIndex: number,\n file: File,\n token: string | null,\n isSilence?: boolean,\n ): Promise => {\n const formData = new FormData();\n formData.append('session_id', sessionId);\n formData.append('chunk_index', String(chunkIndex));\n if (typeof isSilence === 'boolean') {\n formData.append('is_silence', String(isSilence));\n }\n formData.append('file', file);\n return apiClient.postFormData(\n endpoints.USER.LIVE_STT_CHUNK,\n formData,\n token,\n );\n },\n finishLiveTranscription: (\n sessionId: string,\n token: string | null,\n ): Promise =>\n apiClient.post(\n endpoints.USER.LIVE_STT_FINISH,\n { session_id: sessionId },\n token,\n ),\n getDocumentChunks: (\n docId: string,\n page: number,\n perPage: number,\n token: string | null,\n path?: string,\n search?: string,\n ): Promise =>\n apiClient.get(\n endpoints.USER.GET_CHUNKS(docId, page, perPage, path, search),\n token,\n ),\n addChunk: (data: any, token: string | null): Promise =>\n apiClient.post(endpoints.USER.ADD_CHUNK, data, token),\n deleteChunk: (\n docId: string,\n chunkId: string,\n token: string | null,\n ): Promise =>\n apiClient.delete(endpoints.USER.DELETE_CHUNK(docId, chunkId), token),\n updateChunk: (data: any, token: string | null): Promise =>\n apiClient.put(endpoints.USER.UPDATE_CHUNK, data, token),\n getDirectoryStructure: (docId: string, token: string | null): Promise =>\n apiClient.get(endpoints.USER.DIRECTORY_STRUCTURE(docId), token),\n manageSourceFiles: (data: FormData, token: string | null): Promise =>\n apiClient.postFormData(endpoints.USER.MANAGE_SOURCE_FILES, data, token),\n testMCPConnection: (data: any, token: string | null): Promise =>\n apiClient.post(endpoints.USER.MCP_TEST_CONNECTION, data, token),\n saveMCPServer: (data: any, token: string | null): Promise =>\n apiClient.post(endpoints.USER.MCP_SAVE_SERVER, data, token),\n getMCPOAuthStatus: (task_id: string, token: string | null): Promise =>\n apiClient.get(endpoints.USER.MCP_OAUTH_STATUS(task_id), token),\n getMCPAuthStatus: (token: string | null): Promise =>\n apiClient.get(endpoints.USER.MCP_AUTH_STATUS, token),\n syncConnector: (\n docId: string,\n provider: string,\n token: string | null,\n ): Promise => {\n const sessionToken = getSessionToken(provider);\n return apiClient.post(\n endpoints.USER.SYNC_CONNECTOR,\n {\n source_id: docId,\n session_token: sessionToken,\n provider: provider,\n },\n token,\n );\n },\n getAgentFolders: (token: string | null): Promise =>\n apiClient.get(endpoints.USER.AGENT_FOLDERS, token),\n createAgentFolder: (\n data: { name: string; parent_id?: string },\n token: string | null,\n ): Promise => apiClient.post(endpoints.USER.AGENT_FOLDERS, data, token),\n getAgentFolder: (id: string, token: string | null): Promise =>\n apiClient.get(endpoints.USER.AGENT_FOLDER(id), token),\n updateAgentFolder: (\n id: string,\n data: { name?: string; parent_id?: string },\n token: string | null,\n ): Promise =>\n apiClient.put(endpoints.USER.AGENT_FOLDER(id), data, token),\n deleteAgentFolder: (id: string, token: string | null): Promise =>\n apiClient.delete(endpoints.USER.AGENT_FOLDER(id), token),\n moveAgentToFolder: (\n data: { agent_id: string; folder_id?: string | null },\n token: string | null,\n ): Promise =>\n apiClient.post(endpoints.USER.MOVE_AGENT_TO_FOLDER, data, token),\n getArtifact: (artifactId: string, token: string | null): Promise =>\n apiClient.get(endpoints.USER.GET_ARTIFACT(artifactId), token),\n getWorkflow: (id: string, token: string | null): Promise =>\n apiClient.get(endpoints.USER.WORKFLOW(id), token),\n createWorkflow: (data: any, token: string | null): Promise =>\n apiClient.post(endpoints.USER.WORKFLOWS, data, token),\n updateWorkflow: (id: string, data: any, token: string | null): Promise =>\n apiClient.put(endpoints.USER.WORKFLOW(id), data, token),\n deleteWorkflow: (id: string, token: string | null): Promise =>\n apiClient.delete(endpoints.USER.WORKFLOW(id), token),\n};\n\nexport default userService;\n" }, { "path": "frontend/src/components/Accordion.tsx", "content": "import React, { useRef, useState } from 'react';\n\nimport ChevronDown from '../assets/chevron-down.svg';\n\ntype AccordionProps = {\n title: string;\n children: React.ReactNode;\n className?: string;\n titleClassName?: string;\n contentClassName?: string;\n open?: boolean;\n};\n\nexport default function Accordion({\n title,\n children,\n className = '',\n titleClassName = '',\n contentClassName = '',\n open: initialOpen = false,\n}: AccordionProps) {\n const contentRef = useRef(null);\n const [isOpen, setIsOpen] = useState(initialOpen);\n\n const accordionContentStyle = {\n height: isOpen ? 'auto' : '0px',\n transition: 'height 0.3s ease-in-out, opacity 0.3s ease-in-out',\n overflow: 'hidden',\n } as React.CSSProperties;\n\n const toggleAccordion = () => {\n setIsOpen(!isOpen);\n };\n return (\n
\n \n

{title}

\n \n \n\n \n {children}\n
\n \n );\n}\n" }, { "path": "frontend/src/components/ActionButtons.tsx", "content": "import { useTranslation } from 'react-i18next';\nimport { useSelector } from 'react-redux';\nimport newChatIcon from '../assets/openNewChat.svg';\nimport ShareIcon from '../assets/share.svg';\nimport { ShareConversationModal } from '../modals/ShareConversationModal';\nimport { useState } from 'react';\nimport { selectConversationId } from '../preferences/preferenceSlice';\nimport { useDispatch } from 'react-redux';\nimport { AppDispatch } from '../store';\nimport {\n setConversation,\n updateConversationId,\n} from '../conversation/conversationSlice';\n\ninterface ActionButtonsProps {\n className?: string;\n showNewChat?: boolean;\n showShare?: boolean;\n isArtifactOpen?: boolean;\n}\n\nimport { useNavigate } from 'react-router-dom';\n\nexport default function ActionButtons({\n className = '',\n showNewChat = true,\n showShare = true,\n isArtifactOpen = false,\n}: ActionButtonsProps) {\n const { t } = useTranslation();\n const dispatch = useDispatch();\n const conversationId = useSelector(selectConversationId);\n const [isShareModalOpen, setShareModalState] = useState(false);\n const navigate = useNavigate();\n\n const newChat = () => {\n dispatch(setConversation([]));\n dispatch(\n updateConversationId({\n query: { conversationId: null },\n }),\n );\n navigate('/');\n };\n return (\n \n
\n {showNewChat && (\n \n \n \n )}\n\n {showShare && conversationId && (\n <>\n setShareModalState(true)}\n className=\"hover:bg-bright-gray rounded-full p-2 dark:hover:bg-[#28292E]\"\n >\n \n \n {isShareModalOpen && (\n setShareModalState(false)}\n conversationId={conversationId}\n />\n )}\n \n )}\n
{/* */}
\n
\n \n );\n}\n" }, { "path": "frontend/src/components/AgentImage.tsx", "content": "import { useState, useEffect } from 'react';\nimport Robot from '../assets/robot.svg';\n\ntype AgentImageProps = {\n src?: string | null;\n alt?: string;\n className?: string;\n fallbackSrc?: string;\n};\n\nexport default function AgentImage({\n src,\n alt = 'agent',\n className = '',\n fallbackSrc = Robot,\n}: AgentImageProps) {\n const [currentSrc, setCurrentSrc] = useState(\n src && src.trim() !== '' ? src : fallbackSrc,\n );\n\n useEffect(() => {\n const newSrc = src && src.trim() !== '' ? src : fallbackSrc;\n if (newSrc !== currentSrc) {\n setCurrentSrc(newSrc);\n }\n }, [src, fallbackSrc]);\n\n return (\n {\n if (currentSrc !== fallbackSrc) setCurrentSrc(fallbackSrc);\n }}\n />\n );\n}\n" }, { "path": "frontend/src/components/ArtifactSidebar.tsx", "content": "import React, { useEffect, useState } from 'react';\nimport { useSelector } from 'react-redux';\nimport ReactMarkdown from 'react-markdown';\nimport remarkGfm from 'remark-gfm';\nimport { Prism as SyntaxHighlighter } from 'react-syntax-highlighter';\nimport {\n oneLight,\n vscDarkPlus,\n} from 'react-syntax-highlighter/dist/cjs/styles/prism';\n\nimport Exit from '../assets/exit.svg';\nimport { selectToken } from '../preferences/preferenceSlice';\nimport userService from '../api/services/userService';\nimport Spinner from './Spinner';\nimport CopyButton from './CopyButton';\nimport { useDarkTheme } from '../hooks';\n\ntype TodoItem = {\n todo_id: number;\n title: string;\n status: 'open' | 'completed';\n created_at: string | null;\n updated_at: string | null;\n};\n\ntype TodoArtifactData = {\n items: TodoItem[];\n total_count: number;\n open_count: number;\n completed_count: number;\n};\n\ntype NoteArtifactData = {\n content: string;\n line_count: number;\n updated_at: string | null;\n};\n\ntype ArtifactData =\n | { artifact_type: 'todo_list'; data: TodoArtifactData }\n | { artifact_type: 'note'; data: NoteArtifactData }\n | { artifact_type: 'memory'; data: Record };\n\ntype ArtifactSidebarProps = {\n isOpen: boolean;\n onClose: () => void;\n artifactId: string | null;\n toolName?: string;\n conversationId: string | null;\n /**\n * overlay: current fixed slide-in sidebar\n * split: renders as a normal panel (to be placed in a split layout)\n */\n variant?: 'overlay' | 'split';\n};\n\nconst ARTIFACT_TITLE_BY_TYPE: Record = {\n todo_list: 'Todo List',\n note: 'Note',\n memory: 'Memory',\n};\n\nfunction getArtifactTitle(artifact: ArtifactData | null, toolName?: string) {\n if (artifact) return ARTIFACT_TITLE_BY_TYPE[artifact.artifact_type] ?? 'Artifact';\n\n const formattedToolName = (toolName ?? '')\n .replace(/_/g, ' ')\n .replace(/\\s+/g, ' ')\n .trim()\n .replace(/\\b\\w/g, (c) => c.toUpperCase());\n\n return formattedToolName || 'Artifact';\n}\n\nfunction TodoListView({ data }: { data: TodoArtifactData }) {\n return (\n
\n
\n
\n \n {data.completed_count} done\n \n \n {data.open_count} open\n \n
\n
\n
\n {data.items.length === 0 ? (\n

\n No todos yet\n

\n ) : (\n
    \n {data.items.map((item, index) => (\n \n \n {item.status === 'completed' && (\n \n \n \n )}\n \n
    \n \n {item.title}\n

    \n

    #{item.todo_id}

    \n
    \n \n ))}\n
\n )}\n
\n
\n );\n}\n\nfunction NoteView({ data }: { data: NoteArtifactData }) {\n const [isDarkTheme] = useDarkTheme();\n\n return (\n
\n
\n
\n \n {data.line_count} lines\n \n \n
\n
\n
\n {data.content ? (\n \n
\n \n {language}\n \n \n
\n \n {String(children).replace(/\\n$/, '')}\n \n
\n ) : (\n \n {children}\n \n );\n },\n ul({ children }) {\n return (\n
    \n {children}\n
\n );\n },\n ol({ children }) {\n return (\n
    \n {children}\n
\n );\n },\n a({ children, href }) {\n return (\n \n {children}\n \n );\n },\n p({ children }) {\n return

{children}

;\n },\n h1({ children }) {\n return

{children}

;\n },\n h2({ children }) {\n return

{children}

;\n },\n h3({ children }) {\n return

{children}

;\n },\n blockquote({ children }) {\n return (\n
\n {children}\n
\n );\n },\n }}\n >\n {data.content}\n \n ) : (\n

Empty note

\n )}\n
\n \n );\n}\n\nexport default function ArtifactSidebar({\n isOpen,\n onClose,\n artifactId,\n toolName,\n conversationId,\n variant = 'overlay',\n}: ArtifactSidebarProps) {\n const sidebarRef = React.useRef(null);\n const lastSuccessfulTodoArtifactIdRef = React.useRef(null);\n const currentFetchIdRef = React.useRef(null);\n const token = useSelector(selectToken);\n const [artifact, setArtifact] = useState(null);\n const [loading, setLoading] = useState(false);\n const [error, setError] = useState(null);\n const [effectiveArtifactId, setEffectiveArtifactId] = useState(\n artifactId,\n );\n\n const title = getArtifactTitle(artifact, toolName);\n\n // Reset last successful todo artifact ID when conversation changes\n useEffect(() => {\n lastSuccessfulTodoArtifactIdRef.current = null;\n }, [conversationId]);\n\n // Reset effectiveArtifactId when artifactId changes\n useEffect(() => {\n if (!isOpen) {\n setEffectiveArtifactId(null);\n return;\n }\n setEffectiveArtifactId(artifactId);\n }, [isOpen, artifactId]);\n\n // Fetch artifact when effectiveArtifactId changes\n useEffect(() => {\n if (!isOpen || !effectiveArtifactId) {\n setArtifact(null);\n setError(null);\n setLoading(false);\n currentFetchIdRef.current = null;\n return;\n }\n\n // Generate a unique ID for this fetch\n const fetchId = `${effectiveArtifactId}-${Date.now()}`;\n currentFetchIdRef.current = fetchId;\n \n setLoading(true);\n setError(null);\n \n // Note: For todo artifacts, the endpoint always returns all todos for the tool; will be coversation scoped later\n userService\n .getArtifact(effectiveArtifactId, token)\n .then(async (res: any) => {\n // Ignore if this is not the current fetch\n if (currentFetchIdRef.current !== fetchId) return;\n \n const isResponseLike = res && typeof res.json === 'function';\n const status = isResponseLike ? res.status : undefined;\n const ok = isResponseLike ? Boolean(res.ok) : true;\n\n let data: any = res;\n if (isResponseLike) {\n try {\n data = await res.json();\n } catch {\n data = null;\n }\n }\n\n // Check again after async operation\n if (currentFetchIdRef.current !== fetchId) return;\n\n if (ok && data?.success && data?.artifact) {\n setArtifact(data.artifact);\n // Remember the last successful todo artifact id so we can fallback if a newer id 404s.\n if (data.artifact?.artifact_type === 'todo_list') {\n lastSuccessfulTodoArtifactIdRef.current = effectiveArtifactId;\n }\n setLoading(false);\n return;\n }\n\n const isTodoTool = (toolName ?? '').toLowerCase().includes('todo');\n\n // If the latest todo artifact id is missing (404), fall back to the last known good one\n // so the backend can still resolve `tool_id` for the todo list.\n if (\n status === 404 &&\n isTodoTool &&\n lastSuccessfulTodoArtifactIdRef.current &&\n lastSuccessfulTodoArtifactIdRef.current !== effectiveArtifactId\n ) {\n // Update effectiveArtifactId to trigger a new fetch with the fallback id\n setEffectiveArtifactId(lastSuccessfulTodoArtifactIdRef.current);\n setLoading(false);\n return;\n }\n\n // Ensure we show a visible error state instead of rendering nothing.\n const message =\n data?.message ||\n (status === 404 ? 'Artifact not found' : null) ||\n 'Failed to load artifact';\n setError(message);\n setLoading(false);\n })\n .catch((err) => {\n // Ignore if this is not the current fetch\n if (currentFetchIdRef.current !== fetchId) return;\n setError('Failed to fetch artifact');\n setLoading(false);\n });\n }, [isOpen, effectiveArtifactId, token, toolName, conversationId]);\n\n const handleClickOutside = (event: MouseEvent) => {\n if (\n sidebarRef.current &&\n !sidebarRef.current.contains(event.target as Node)\n ) {\n onClose();\n }\n };\n\n useEffect(() => {\n if (variant === 'overlay' && isOpen) {\n document.addEventListener('mousedown', handleClickOutside);\n }\n return () => document.removeEventListener('mousedown', handleClickOutside);\n }, [isOpen, variant]);\n\n const renderContent = () => {\n if (loading) {\n return (\n
\n \n
\n );\n }\n if (error) {\n return (\n
\n

{error}

\n
\n );\n }\n // Avoid rendering an empty panel if the artifact couldn't be loaded for any reason.\n if (!artifact) {\n return (\n
\n

\n Artifact not found\n

\n
\n );\n }\n switch (artifact.artifact_type) {\n case 'todo_list':\n return ;\n case 'note':\n return ;\n default:\n return (\n 
\n            {JSON.stringify(artifact, null, 2)}\n
\n );\n }\n };\n\n if (variant === 'split') {\n if (!isOpen) return null;\n\n return (\n
\n {/* Space for top bar / actions */}\n
\n {/* Artifact panel */}\n
\n
\n \n {title}\n \n \n \n \n
\n
{renderContent()}
\n
\n
\n );\n }\n\n return (\n
\n \n
\n \n {title}\n \n \n \n \n
\n
{renderContent()}
\n
\n
\n );\n}\n" }, { "path": "frontend/src/components/Avatar.tsx", "content": "import { ReactNode } from 'react';\n\nexport default function Avatar({\n avatar,\n size,\n className,\n}: {\n avatar: ReactNode;\n size?: 'SMALL' | 'MEDIUM' | 'LARGE';\n className: string;\n}) {\n return
{avatar}
;\n}\n" }, { "path": "frontend/src/components/Chunks.tsx", "content": "import React, { useState, useEffect, useRef } from 'react';\nimport { useSelector } from 'react-redux';\nimport { useTranslation } from 'react-i18next';\nimport { selectToken } from '../preferences/preferenceSlice';\nimport {\n useDarkTheme,\n useLoaderState,\n useMediaQuery,\n useOutsideAlerter,\n} from '../hooks';\nimport userService from '../api/services/userService';\nimport ArrowLeft from '../assets/arrow-left.svg';\nimport NoFilesIcon from '../assets/no-files.svg';\nimport NoFilesDarkIcon from '../assets/no-files-dark.svg';\nimport SkeletonLoader from './SkeletonLoader';\nimport ConfirmationModal from '../modals/ConfirmationModal';\nimport { ActiveState } from '../models/misc';\nimport { ChunkType } from '../settings/types';\nimport Pagination from './DocumentPagination';\nimport FileIcon from '../assets/file.svg';\nimport FolderIcon from '../assets/folder.svg';\nimport SearchIcon from '../assets/search.svg';\ninterface LineNumberedTextareaProps {\n value: string;\n onChange: (value: string) => void;\n placeholder?: string;\n ariaLabel?: string;\n className?: string;\n editable?: boolean;\n onDoubleClick?: () => void;\n}\n\nconst LineNumberedTextarea: React.FC = ({\n value,\n onChange,\n placeholder,\n ariaLabel,\n className = '',\n editable = true,\n onDoubleClick,\n}) => {\n const { isMobile } = useMediaQuery();\n\n const handleChange = (e: React.ChangeEvent) => {\n onChange(e.target.value);\n };\n\n const lineHeight = 19.93;\n const contentLines = value.split('\\n').length;\n\n const heightOffset = isMobile ? 200 : 300;\n const minLinesForDisplay = Math.ceil(\n (typeof window !== 'undefined' ? window.innerHeight - heightOffset : 600) /\n lineHeight,\n );\n const totalLines = Math.max(contentLines, minLinesForDisplay);\n\n return (\n
\n \n {Array.from({ length: totalLines }, (_, i) => (\n \n {i + 1}\n
\n ))}\n \n \n \n );\n};\n\ninterface SearchResult {\n path: string;\n isFile: boolean;\n name?: string;\n}\n\ninterface ChunksProps {\n documentId: string;\n documentName?: string;\n handleGoBack: () => void;\n path?: string;\n displayPath?: string;\n onFileSearch?: (query: string) => SearchResult[];\n onFileSelect?: (path: string) => void;\n}\n\nconst Chunks: React.FC = ({\n documentId,\n documentName,\n handleGoBack,\n path,\n displayPath,\n onFileSearch,\n onFileSelect,\n}) => {\n const [fileSearchQuery, setFileSearchQuery] = useState('');\n const [fileSearchResults, setFileSearchResults] = useState(\n [],\n );\n const searchDropdownRef = useRef(null);\n const { t } = useTranslation();\n const token = useSelector(selectToken);\n const [isDarkTheme] = useDarkTheme();\n const [paginatedChunks, setPaginatedChunks] = useState([]);\n const [page, setPage] = useState(1);\n const [perPage, setPerPage] = useState(5);\n const [totalChunks, setTotalChunks] = useState(0);\n const [loading, setLoading] = useLoaderState(true);\n const [searchTerm, setSearchTerm] = useState('');\n const [editingChunk, setEditingChunk] = useState(null);\n const [editingTitle, setEditingTitle] = useState('');\n const [editingText, setEditingText] = useState('');\n const [isAddingChunk, setIsAddingChunk] = useState(false);\n const [deleteModalState, setDeleteModalState] =\n useState('INACTIVE');\n const [chunkToDelete, setChunkToDelete] = useState(null);\n const [isEditing, setIsEditing] = useState(false);\n\n const displayPathValue = displayPath ?? path ?? '';\n const pathParts = displayPathValue ? displayPathValue.split('/') : [];\n\n const fetchChunks = async () => {\n setLoading(true);\n try {\n const response = await userService.getDocumentChunks(\n documentId,\n page,\n perPage,\n token,\n path,\n searchTerm,\n );\n\n if (!response.ok) {\n throw new Error('Failed to fetch chunks data');\n }\n\n const data = await response.json();\n\n setPage(data.page);\n setPerPage(data.per_page);\n setTotalChunks(data.total);\n setPaginatedChunks(data.chunks);\n } catch (error) {\n setPaginatedChunks([]);\n console.error(error);\n } finally {\n // ✅ always runs, success or failure\n setLoading(false);\n }\n };\n\n const handleAddChunk = (title: string, text: string) => {\n if (!text.trim()) {\n return;\n }\n\n try {\n const metadata = {\n source: path || documentName,\n source_id: documentId,\n title: title,\n };\n\n userService\n .addChunk(\n {\n id: documentId,\n text: text,\n metadata: metadata,\n },\n token,\n )\n .then((response) => {\n if (!response.ok) {\n throw new Error('Failed to add chunk');\n }\n fetchChunks();\n });\n } catch (e) {\n console.log(e);\n }\n };\n\n const handleUpdateChunk = (title: string, text: string, chunk: ChunkType) => {\n if (!text.trim()) {\n return;\n }\n\n const originalTitle = chunk.metadata?.title || '';\n const originalText = chunk.text || '';\n\n if (title === originalTitle && text === originalText) {\n return;\n }\n\n try {\n userService\n .updateChunk(\n {\n id: documentId,\n chunk_id: chunk.doc_id,\n text: text,\n metadata: {\n title: title,\n },\n },\n token,\n )\n .then((response) => {\n if (!response.ok) {\n throw new Error('Failed to update chunk');\n }\n fetchChunks();\n });\n } catch (e) {\n console.log(e);\n }\n };\n\n const handleDeleteChunk = (chunk: ChunkType) => {\n try {\n userService\n .deleteChunk(documentId, chunk.doc_id, token)\n .then((response) => {\n if (!response.ok) {\n throw new Error('Failed to delete chunk');\n }\n setEditingChunk(null);\n fetchChunks();\n });\n } catch (e) {\n console.log(e);\n }\n };\n\n const confirmDeleteChunk = (chunk: ChunkType) => {\n setChunkToDelete(chunk);\n setDeleteModalState('ACTIVE');\n };\n\n const handleConfirmedDelete = () => {\n if (chunkToDelete) {\n handleDeleteChunk(chunkToDelete);\n setDeleteModalState('INACTIVE');\n setChunkToDelete(null);\n }\n };\n\n const handleCancelDelete = () => {\n setDeleteModalState('INACTIVE');\n setChunkToDelete(null);\n };\n\n useEffect(() => {\n const delayDebounceFn = setTimeout(() => {\n if (page !== 1) {\n setPage(1);\n } else {\n fetchChunks();\n }\n }, 300);\n\n return () => clearTimeout(delayDebounceFn);\n }, [searchTerm]);\n\n useEffect(() => {\n !loading && fetchChunks();\n }, [page, perPage, path]);\n\n useEffect(() => {\n setSearchTerm('');\n setPage(1);\n }, [path]);\n\n const filteredChunks = paginatedChunks;\n\n const renderPathNavigation = () => {\n return (\n
\n
\n setEditingChunk(null)\n : isAddingChunk\n ? () => setIsAddingChunk(false)\n : handleGoBack\n }\n >\n \"left-arrow\"\n \n\n
\n {/* Removed the directory icon */}\n \n {documentName}\n \n\n {pathParts.length > 0 && (\n <>\n /\n {pathParts.map((part, index) => (\n \n \n {part}\n \n {index < pathParts.length - 1 && (\n \n /\n \n )}\n \n ))}\n \n )}\n
\n
\n\n
\n {editingChunk ? (\n !isEditing ? (\n <>\n setIsEditing(true)}\n >\n {t('modals.chunk.edit')}\n \n {\n confirmDeleteChunk(editingChunk);\n }}\n >\n {t('modals.chunk.delete')}\n \n \n ) : (\n <>\n {\n setIsEditing(false);\n }}\n className=\"dark:text-light-gray flex h-[38px] min-w-[108px] cursor-pointer items-center justify-center rounded-full px-4 py-1 text-sm font-medium text-nowrap hover:bg-gray-100 dark:bg-transparent dark:hover:bg-[#767183]/50\"\n >\n {t('modals.chunk.cancel')}\n \n {\n if (editingText.trim()) {\n const hasChanges =\n editingTitle !==\n (editingChunk?.metadata?.title || '') ||\n editingText !== (editingChunk?.text || '');\n\n if (hasChanges) {\n handleUpdateChunk(\n editingTitle,\n editingText,\n editingChunk,\n );\n }\n setIsEditing(false);\n setEditingChunk(null);\n }\n }}\n disabled={\n !editingText.trim() ||\n (editingTitle === (editingChunk?.metadata?.title || '') &&\n editingText === (editingChunk?.text || ''))\n }\n className={`flex h-[38px] min-w-[108px] items-center justify-center rounded-full px-4 py-1 text-[14px] font-medium text-nowrap text-white transition-all ${\n editingText.trim() &&\n (editingTitle !== (editingChunk?.metadata?.title || '') ||\n editingText !== (editingChunk?.text || ''))\n ? 'bg-purple-30 hover:bg-violets-are-blue cursor-pointer'\n : 'cursor-not-allowed bg-gray-400'\n }`}\n >\n {t('modals.chunk.save')}\n \n \n )\n ) : isAddingChunk ? (\n <>\n setIsAddingChunk(false)}\n className=\"dark:text-light-gray flex h-[38px] min-w-[108px] cursor-pointer items-center justify-center rounded-full px-4 py-1 text-sm font-medium text-nowrap hover:bg-gray-100 dark:bg-transparent dark:hover:bg-[#767183]/50\"\n >\n {t('modals.chunk.cancel')}\n \n {\n if (editingText.trim()) {\n handleAddChunk(editingTitle, editingText);\n setIsAddingChunk(false);\n }\n }}\n disabled={!editingText.trim()}\n className={`flex h-[38px] min-w-[108px] items-center justify-center rounded-full px-4 py-1 text-[14px] font-medium text-nowrap text-white transition-all ${\n editingText.trim()\n ? 'bg-purple-30 hover:bg-violets-are-blue cursor-pointer'\n : 'cursor-not-allowed bg-gray-400'\n }`}\n >\n {t('modals.chunk.add')}\n \n \n ) : null}\n
\n
\n );\n };\n\n // File search handling\n const handleFileSearchChange = (query: string) => {\n setFileSearchQuery(query);\n if (query.trim() && onFileSearch) {\n const results = onFileSearch(query);\n setFileSearchResults(results);\n } else {\n setFileSearchResults([]);\n }\n };\n\n const handleSearchResultClick = (result: SearchResult) => {\n if (!onFileSelect) return;\n\n if (result.isFile) {\n onFileSelect(result.path);\n } else {\n // For directories, navigate to the directory and return to file tree\n onFileSelect(result.path);\n handleGoBack();\n }\n setFileSearchQuery('');\n setFileSearchResults([]);\n };\n\n useOutsideAlerter(\n searchDropdownRef,\n () => {\n setFileSearchQuery('');\n setFileSearchResults([]);\n },\n [], // No additional dependencies\n false, // Don't handle escape key\n );\n\n const renderFileSearch = () => {\n return (\n
\n
\n
\n \"Search\"\n
\n handleFileSearchChange(e.target.value)}\n placeholder={t('settings.sources.searchFiles')}\n className={`h-[38px] w-full border border-[#D1D9E0] py-2 pr-4 pl-10 dark:border-[#6A6A6A] ${\n fileSearchQuery ? 'rounded-t-[6px]' : 'rounded-[6px]'\n } bg-transparent transition-all duration-200 focus:outline-none dark:text-[#E0E0E0]`}\n />\n
\n\n {fileSearchQuery && (\n
\n
\n {fileSearchResults.length === 0 ? (\n
\n {t('settings.sources.noResults')}\n
\n ) : (\n fileSearchResults.map((result, index) => (\n handleSearchResultClick(result)}\n className={`flex cursor-pointer items-center px-3 py-2 hover:bg-[#ECEEEF] dark:hover:bg-[#27282D] ${\n index !== fileSearchResults.length - 1\n ? 'border-b border-[#D1D9E0] dark:border-[#6A6A6A]'\n : ''\n }`}\n >\n \n \n {result.name ||\n result.path.split('/').pop() ||\n result.path}\n \n
\n ))\n )}\n
\n
\n )}\n \n );\n };\n\n return (\n
\n
{renderPathNavigation()}
\n
\n {onFileSearch && onFileSelect && (\n
{renderFileSearch()}
\n )}\n\n {/* Right side: Chunks content */}\n
\n {!editingChunk && !isAddingChunk ? (\n <>\n
\n
\n
\n {totalChunks > 999999\n ? `${(totalChunks / 1000000).toFixed(2)}M`\n : totalChunks > 999\n ? `${(totalChunks / 1000).toFixed(2)}K`\n : totalChunks}{' '}\n {t('settings.sources.chunks')}\n
\n
\n
\n setSearchTerm(e.target.value)}\n className=\"h-full w-full border-none bg-transparent px-3 py-2 text-[13.56px] leading-[100%] font-normal outline-none dark:text-[#E0E0E0]\"\n />\n
\n
\n {\n setIsAddingChunk(true);\n setEditingTitle('');\n setEditingText('');\n }}\n >\n {t('settings.sources.addChunk')}\n \n
\n {loading ? (\n
\n \n
\n ) : (\n
\n {filteredChunks.length === 0 ? (\n
\n \n {t('settings.sources.noChunks')}\n
\n ) : (\n filteredChunks.map((chunk, index) => (\n {\n setEditingChunk(chunk);\n setEditingTitle(chunk.metadata?.title || '');\n setEditingText(chunk.text || '');\n }}\n >\n
\n
\n
\n {chunk.metadata.token_count\n ? chunk.metadata.token_count.toLocaleString()\n : '-'}{' '}\n {t('settings.sources.tokensUnit')}\n
\n
\n
\n

\n {chunk.text}\n

\n
\n
\n
\n ))\n )}\n
\n )}\n \n ) : isAddingChunk ? (\n
\n
\n \n
\n
\n ) : (\n editingChunk && (\n
\n
\n
\n
\n {editingChunk.metadata.token_count\n ? editingChunk.metadata.token_count.toLocaleString()\n : '-'}{' '}\n {t('settings.sources.tokensUnit')}\n
\n
\n
\n {\n if (!isEditing) {\n setIsEditing(true);\n setEditingTitle(editingChunk.metadata.title || '');\n setEditingText(editingChunk.text);\n }\n }}\n />\n
\n
\n
\n )\n )}\n\n {!loading &&\n totalChunks > perPage &&\n !editingChunk &&\n !isAddingChunk && (\n {\n setPerPage(rows);\n setPage(1);\n }}\n />\n )}\n
\n
\n\n {/* Delete Confirmation Modal */}\n \n \n );\n};\n\nexport default Chunks;\n" }, { "path": "frontend/src/components/ConfigFields.tsx", "content": "import { useMemo } from 'react';\n\nimport { cn } from '@/lib/utils';\n\nimport { ConfigRequirements } from '../modals/types';\nimport { Input } from './ui/input';\nimport { Label } from './ui/label';\nimport {\n Select,\n SelectContent,\n SelectItem,\n SelectTrigger,\n SelectValue,\n} from './ui/select';\n\ntype ConfigValues = { [key: string]: any };\n\ninterface ConfigFieldsProps {\n configRequirements: ConfigRequirements;\n values: ConfigValues;\n onChange: (key: string, value: any) => void;\n errors?: { [key: string]: string };\n isEditing?: boolean;\n hasEncryptedCredentials?: boolean;\n}\n\nfunction shouldShowField(\n spec: ConfigRequirements[string],\n values: ConfigValues,\n): boolean {\n if (!spec.depends_on) return true;\n return Object.entries(spec.depends_on).every(\n ([depKey, depValue]) => values[depKey] === depValue,\n );\n}\n\nexport default function ConfigFields({\n configRequirements,\n values,\n onChange,\n errors = {},\n isEditing = false,\n hasEncryptedCredentials = false,\n}: ConfigFieldsProps) {\n const sortedFields = useMemo(\n () =>\n Object.entries(configRequirements).sort(\n ([, a], [, b]) => (a.order ?? 99) - (b.order ?? 99),\n ),\n [configRequirements],\n );\n\n if (sortedFields.length === 0) return null;\n\n return (\n
\n {sortedFields.map(([key, spec]) => {\n if (!shouldShowField(spec, values)) return null;\n\n const value = values[key] ?? spec.default ?? '';\n const hasEncrypted =\n isEditing && spec.secret && hasEncryptedCredentials;\n const placeholder = hasEncrypted ? '••••••••' : '';\n const hasError = !!errors[key];\n\n if (spec.enum) {\n return (\n
\n \n onChange(key, v)}\n >\n \n \n \n \n {spec.enum.map((v) => (\n \n {v.charAt(0).toUpperCase() + v.slice(1).replace(/_/g, ' ')}\n \n ))}\n \n \n {hasError && (\n

{errors[key]}

\n )}\n
\n );\n }\n\n return (\n
\n \n {\n const v = e.target.value;\n if (spec.type === 'number') {\n if (v === '') onChange(key, '');\n else {\n const num = parseInt(v, 10);\n if (!isNaN(num)) onChange(key, num);\n }\n } else {\n onChange(key, v);\n }\n }}\n placeholder={placeholder || spec.description || ''}\n min={spec.type === 'number' ? 1 : undefined}\n max={spec.type === 'number' && key === 'timeout' ? 300 : undefined}\n aria-invalid={hasError || undefined}\n className={cn('rounded-xl', hasError && 'border-destructive')}\n />\n {hasError && (\n

{errors[key]}

\n )}\n
\n );\n })}\n
\n );\n}\n" }, { "path": "frontend/src/components/ConnectedStateSkeleton.tsx", "content": "const ConnectedStateSkeleton = () => (\n
\n
\n
\n
\n
\n
\n
\n
\n
\n);\n\nexport default ConnectedStateSkeleton;\n" }, { "path": "frontend/src/components/ConnectorAuth.tsx", "content": "import React, { useRef } from 'react';\nimport { useSelector } from 'react-redux';\nimport { useTranslation } from 'react-i18next';\nimport { useDarkTheme } from '../hooks';\nimport { selectToken } from '../preferences/preferenceSlice';\n\ninterface ConnectorAuthProps {\n provider: string;\n onSuccess: (data: { session_token: string; user_email: string }) => void;\n onError: (error: string) => void;\n label?: string;\n isConnected?: boolean;\n userEmail?: string;\n onDisconnect?: () => void;\n errorMessage?: string;\n}\n\nconst ConnectorAuth: React.FC = ({\n provider,\n onSuccess,\n onError,\n label,\n isConnected = false,\n userEmail = '',\n onDisconnect,\n errorMessage,\n}) => {\n const { t } = useTranslation();\n const token = useSelector(selectToken);\n const [isDarkTheme] = useDarkTheme();\n const completedRef = useRef(false);\n const intervalRef = useRef(null);\n\n const cleanup = () => {\n if (intervalRef.current) {\n clearInterval(intervalRef.current);\n intervalRef.current = null;\n }\n window.removeEventListener('message', handleAuthMessage as any);\n };\n\n const handleAuthMessage = (event: MessageEvent) => {\n const successGeneric = event.data?.type === 'connector_auth_success';\n const successProvider = event.data?.type === `${provider}_auth_success`;\n const errorProvider = event.data?.type === `${provider}_auth_error`;\n\n if (successGeneric || successProvider) {\n completedRef.current = true;\n cleanup();\n onSuccess({\n session_token: event.data.session_token,\n user_email:\n event.data.user_email ||\n t('modals.uploadDoc.connectors.auth.connectedUser'),\n });\n } else if (errorProvider) {\n completedRef.current = true;\n cleanup();\n onError(\n event.data.error || t('modals.uploadDoc.connectors.auth.authFailed'),\n );\n }\n };\n\n const handleAuth = async () => {\n try {\n completedRef.current = false;\n cleanup();\n\n const apiHost = import.meta.env.VITE_API_HOST;\n const authResponse = await fetch(\n `${apiHost}/api/connectors/auth?provider=${provider}`,\n {\n headers: { Authorization: `Bearer ${token}` },\n },\n );\n\n if (!authResponse.ok) {\n throw new Error(\n `${t('modals.uploadDoc.connectors.auth.authUrlFailed')}: ${authResponse.status}`,\n );\n }\n\n const authData = await authResponse.json();\n if (!authData.success || !authData.authorization_url) {\n throw new Error(\n authData.error || t('modals.uploadDoc.connectors.auth.authUrlFailed'),\n );\n }\n\n const authWindow = window.open(\n authData.authorization_url,\n `${provider}-auth`,\n 'width=500,height=600,scrollbars=yes,resizable=yes',\n );\n if (!authWindow) {\n throw new Error(t('modals.uploadDoc.connectors.auth.popupBlocked'));\n }\n\n window.addEventListener('message', handleAuthMessage as any);\n\n const checkClosed = window.setInterval(() => {\n if (authWindow.closed) {\n clearInterval(checkClosed);\n window.removeEventListener('message', handleAuthMessage as any);\n if (!completedRef.current) {\n onError(t('modals.uploadDoc.connectors.auth.authCancelled'));\n }\n }\n }, 1000);\n intervalRef.current = checkClosed;\n } catch (error) {\n onError(\n error instanceof Error\n ? error.message\n : t('modals.uploadDoc.connectors.auth.authFailed'),\n );\n }\n };\n\n return (\n <>\n {errorMessage && (\n
\n \n \n \n\n \n {errorMessage}\n \n
\n )}\n\n {isConnected ? (\n
\n
\n
\n \n \n \n \n {t('modals.uploadDoc.connectors.auth.connectedAs', {\n email: userEmail,\n })}\n \n
\n {onDisconnect && (\n \n {t('modals.uploadDoc.connectors.auth.disconnect')}\n \n )}\n
\n
\n ) : (\n \n \n \n \n {label}\n \n )}\n \n );\n};\n\nexport default ConnectorAuth;\n" }, { "path": "frontend/src/components/ConnectorTree.tsx", "content": "import React, { useState, useRef, useEffect } from 'react';\nimport { useTranslation } from 'react-i18next';\nimport { useSelector } from 'react-redux';\nimport { formatBytes } from '../utils/stringUtils';\nimport { selectToken } from '../preferences/preferenceSlice';\nimport { ActiveState } from '../models/misc';\nimport Chunks from './Chunks';\nimport ContextMenu, { MenuOption } from './ContextMenu';\nimport SkeletonLoader from './SkeletonLoader';\nimport ConfirmationModal from '../modals/ConfirmationModal';\nimport userService from '../api/services/userService';\nimport FileIcon from '../assets/file.svg';\nimport FolderIcon from '../assets/folder.svg';\nimport ArrowLeft from '../assets/arrow-left.svg';\nimport ThreeDots from '../assets/three-dots.svg';\nimport EyeView from '../assets/eye-view.svg';\nimport SyncIcon from '../assets/sync.svg';\nimport CheckmarkIcon from '../assets/checkMark2.svg';\nimport { useOutsideAlerter, useLoaderState } from '../hooks';\nimport {\n Table,\n TableContainer,\n TableHead,\n TableBody,\n TableRow,\n TableHeader,\n TableCell,\n} from './Table';\n\ninterface FileNode {\n type?: string;\n token_count?: number;\n size_bytes?: number;\n display_name?: string;\n [key: string]: any;\n}\n\ninterface DirectoryStructure {\n [key: string]: FileNode;\n}\n\ninterface ConnectorTreeProps {\n docId: string;\n sourceName: string;\n onBackToDocuments: () => void;\n}\n\ninterface SearchResult {\n name: string;\n path: string;\n isFile: boolean;\n}\n\nconst ConnectorTree: React.FC = ({\n docId,\n sourceName,\n onBackToDocuments,\n}) => {\n const { t } = useTranslation();\n const [loading, setLoading] = useLoaderState(true, 500);\n const [error, setError] = useState(null);\n const [directoryStructure, setDirectoryStructure] =\n useState(null);\n const [currentPath, setCurrentPath] = useState([]);\n const token = useSelector(selectToken);\n const [activeMenuId, setActiveMenuId] = useState(null);\n const menuRefs = useRef<{\n [key: string]: React.RefObject;\n }>({});\n const [selectedFile, setSelectedFile] = useState<{\n id: string;\n name: string;\n } | null>(null);\n const [searchQuery, setSearchQuery] = useState('');\n const [searchResults, setSearchResults] = useState([]);\n const searchDropdownRef = useRef(null);\n const [isSyncing, setIsSyncing] = useState(false);\n const [syncProgress, setSyncProgress] = useState(0);\n const [sourceProvider, setSourceProvider] = useState('');\n const [syncDone, setSyncDone] = useState(false);\n const [syncConfirmationModal, setSyncConfirmationModal] =\n useState('INACTIVE');\n\n useOutsideAlerter(\n searchDropdownRef,\n () => {\n setSearchQuery('');\n setSearchResults([]);\n },\n [],\n false,\n );\n\n const handleFileClick = (fileName: string, displayName?: string) => {\n const fullPath = [...currentPath, fileName].join('/');\n setSelectedFile({\n id: fullPath,\n name: displayName ?? fileName,\n });\n };\n\n const handleSync = async () => {\n if (isSyncing) return;\n\n const provider = sourceProvider;\n\n setIsSyncing(true);\n setSyncProgress(0);\n\n try {\n const response = await userService.syncConnector(docId, provider, token);\n const data = await response.json();\n\n if (data.success) {\n console.log('Sync started successfully:', data.task_id);\n setSyncProgress(10);\n\n // Poll task status using userService\n const maxAttempts = 30;\n const pollInterval = 2000;\n\n for (let attempt = 0; attempt < maxAttempts; attempt++) {\n try {\n const statusResponse = await userService.getTaskStatus(\n data.task_id,\n token,\n );\n const statusData = await statusResponse.json();\n\n console.log(\n `Task status (attempt ${attempt + 1}):`,\n statusData.status,\n );\n\n if (statusData.status === 'SUCCESS') {\n setSyncProgress(100);\n console.log('Sync completed successfully');\n\n // Refresh directory structure\n try {\n const refreshResponse = await userService.getDirectoryStructure(\n docId,\n token,\n );\n const refreshData = await refreshResponse.json();\n if (refreshData && refreshData.directory_structure) {\n setDirectoryStructure(refreshData.directory_structure);\n setCurrentPath([]);\n }\n if (refreshData && refreshData.provider) {\n setSourceProvider(refreshData.provider);\n }\n\n setSyncDone(true);\n setTimeout(() => setSyncDone(false), 5000);\n } catch (err) {\n console.error('Error refreshing directory structure:', err);\n }\n break;\n } else if (statusData.status === 'FAILURE') {\n console.error('Sync task failed:', statusData.result);\n break;\n } else if (statusData.status === 'PROGRESS') {\n const progress = Number(\n statusData.result && statusData.result.current != null\n ? statusData.result.current\n : statusData.meta && statusData.meta.current != null\n ? statusData.meta.current\n : 0,\n );\n setSyncProgress(Math.max(10, progress));\n }\n\n await new Promise((resolve) => setTimeout(resolve, pollInterval));\n } catch (error) {\n console.error('Error polling task status:', error);\n break;\n }\n }\n } else {\n console.error('Sync failed:', data.error);\n }\n } catch (err) {\n console.error('Error syncing connector:', err);\n } finally {\n setIsSyncing(false);\n setSyncProgress(0);\n }\n };\n\n useEffect(() => {\n const fetchDirectoryStructure = async () => {\n try {\n setLoading(true);\n\n const directoryResponse = await userService.getDirectoryStructure(\n docId,\n token,\n );\n const directoryData = await directoryResponse.json();\n\n if (directoryData && directoryData.directory_structure) {\n setDirectoryStructure(directoryData.directory_structure);\n } else {\n setError('Invalid response format');\n }\n\n if (directoryData && directoryData.provider) {\n setSourceProvider(directoryData.provider);\n }\n } catch (err) {\n setError('Failed to load source information');\n console.error(err);\n } finally {\n setLoading(false);\n }\n };\n\n if (docId) {\n fetchDirectoryStructure();\n }\n }, [docId, token]);\n\n const navigateToDirectory = (dirName: string) => {\n setCurrentPath([...currentPath, dirName]);\n };\n\n const navigateUp = () => {\n setCurrentPath(currentPath.slice(0, -1));\n };\n\n const getCurrentDirectory = (): DirectoryStructure => {\n if (!directoryStructure) return {};\n\n let current = directoryStructure;\n for (const dir of currentPath) {\n if (current[dir] && !current[dir].type) {\n current = current[dir] as DirectoryStructure;\n } else {\n return {};\n }\n }\n return current;\n };\n\n const getMenuRef = (id: string) => {\n if (!menuRefs.current[id]) {\n menuRefs.current[id] = React.createRef();\n }\n return menuRefs.current[id];\n };\n\n const handleMenuClick = (\n e: React.MouseEvent,\n id: string,\n ) => {\n e.stopPropagation();\n setActiveMenuId(activeMenuId === id ? null : id);\n };\n\n const getActionOptions = (\n name: string,\n isFile: boolean,\n _itemId: string,\n displayName?: string,\n ): MenuOption[] => {\n const options: MenuOption[] = [];\n\n options.push({\n icon: EyeView,\n label: t('settings.sources.view'),\n onClick: (event: React.SyntheticEvent) => {\n event.stopPropagation();\n if (isFile) {\n handleFileClick(name, displayName);\n } else {\n navigateToDirectory(name);\n }\n },\n iconWidth: 18,\n iconHeight: 18,\n variant: 'primary',\n });\n\n return options;\n };\n\n const calculateDirectoryStats = (\n structure: DirectoryStructure,\n ): { totalSize: number; totalTokens: number } => {\n let totalSize = 0;\n let totalTokens = 0;\n\n Object.entries(structure).forEach(([_, node]) => {\n if (node.type) {\n // It's a file\n totalSize += node.size_bytes || 0;\n totalTokens += node.token_count || 0;\n } else {\n // It's a directory, recurse\n const stats = calculateDirectoryStats(node);\n totalSize += stats.totalSize;\n totalTokens += stats.totalTokens;\n }\n });\n\n return { totalSize, totalTokens };\n };\n\n const handleBackNavigation = () => {\n if (selectedFile) {\n setSelectedFile(null);\n } else if (currentPath.length === 0) {\n if (onBackToDocuments) {\n onBackToDocuments();\n }\n } else {\n navigateUp();\n }\n };\n\n const renderPathNavigation = () => {\n return (\n
\n {/* Left side with path navigation */}\n
\n \n \"left-arrow\"\n \n\n
\n \n {sourceName}\n \n {currentPath.length > 0 && (\n <>\n /\n {currentPath.map((dir, index) => (\n \n \n {dir}\n \n {index < currentPath.length - 1 && (\n \n /\n \n )}\n \n ))}\n \n )}\n
\n
\n\n
\n {renderFileSearch()}\n\n {/* Sync button */}\n setSyncConfirmationModal('ACTIVE')}\n disabled={isSyncing}\n className={`flex h-[38px] min-w-[108px] items-center justify-center rounded-full px-4 text-[14px] font-medium whitespace-nowrap transition-colors ${\n isSyncing\n ? 'cursor-not-allowed bg-gray-300 text-gray-600 dark:bg-gray-600 dark:text-gray-400'\n : 'bg-purple-30 hover:bg-violets-are-blue text-white'\n }`}\n title={\n isSyncing\n ? `${t('settings.sources.syncing')} ${syncProgress}%`\n : syncDone\n ? 'Done'\n : t('settings.sources.sync')\n }\n >\n \n {isSyncing\n ? `${syncProgress}%`\n : syncDone\n ? 'Done'\n : t('settings.sources.sync')}\n \n
\n
\n );\n };\n\n const renderFileTree = (directory: DirectoryStructure): React.ReactNode[] => {\n // Create parent directory row\n const parentRow =\n currentPath.length > 0\n ? [\n \n \n
\n \n ..\n
\n \n \n -\n \n \n -\n \n \n ,\n ]\n : [];\n\n // Sort entries: directories first, then files, both alphabetically\n const sortedEntries = Object.entries(directory).sort(\n ([nameA, nodeA], [nameB, nodeB]) => {\n const isFileA = !!nodeA.type;\n const isFileB = !!nodeB.type;\n\n if (isFileA !== isFileB) {\n return isFileA ? 1 : -1; // Directories first\n }\n\n return nameA.localeCompare(nameB); // Alphabetical within each group\n },\n );\n\n // Process directories\n const directoryRows = sortedEntries\n .filter(([_, node]) => !node.type)\n .map(([name, node]) => {\n const itemId = `dir-${name}`;\n const menuRef = getMenuRef(itemId);\n\n // Calculate directory stats\n const dirStats = calculateDirectoryStats(node as DirectoryStructure);\n\n return (\n navigateToDirectory(name)}>\n \n
\n \n {name}\n
\n \n \n {dirStats.totalTokens > 0\n ? dirStats.totalTokens.toLocaleString()\n : '-'}\n \n \n {dirStats.totalSize > 0 ? formatBytes(dirStats.totalSize) : '-'}\n \n \n
\n handleMenuClick(e, itemId)}\n className=\"inline-flex h-[35px] w-[24px] shrink-0 items-center justify-center rounded-md font-medium transition-colors hover:bg-[#EBEBEB] dark:hover:bg-[#26272E]\"\n aria-label={t('settings.sources.menuAlt')}\n >\n \n \n \n setActiveMenuId(isOpen ? itemId : null)\n }\n options={getActionOptions(name, false, itemId)}\n anchorRef={menuRef}\n position=\"bottom-left\"\n offset={{ x: -4, y: 4 }}\n />\n
\n \n \n );\n });\n\n // Process files\n const fileRows = sortedEntries\n .filter(([_, node]) => !!node.type)\n .map(([name, node]) => {\n const itemId = `file-${name}`;\n const menuRef = getMenuRef(itemId);\n const displayName =\n typeof node.display_name === 'string' && node.display_name.trim()\n ? node.display_name\n : name;\n\n return (\n handleFileClick(name, displayName)}\n >\n \n
\n \n {displayName}\n
\n \n \n {node.token_count?.toLocaleString() || '-'}\n \n \n {node.size_bytes ? formatBytes(node.size_bytes) : '-'}\n \n \n
\n handleMenuClick(e, itemId)}\n className=\"inline-flex h-[35px] w-[24px] shrink-0 items-center justify-center rounded-md font-medium transition-colors hover:bg-[#EBEBEB] dark:hover:bg-[#26272E]\"\n aria-label={t('settings.sources.menuAlt')}\n >\n \n \n \n setActiveMenuId(isOpen ? itemId : null)\n }\n options={getActionOptions(name, true, itemId, displayName)}\n anchorRef={menuRef}\n position=\"bottom-left\"\n offset={{ x: -4, y: 4 }}\n />\n
\n \n \n );\n });\n\n return [...parentRow, ...directoryRows, ...fileRows];\n };\n\n const searchFiles = (\n query: string,\n structure: DirectoryStructure,\n currentPath: string[] = [],\n ): SearchResult[] => {\n let results: SearchResult[] = [];\n\n Object.entries(structure).forEach(([name, node]) => {\n const fullPath = [...currentPath, name].join('/');\n const displayName =\n typeof node.display_name === 'string' && node.display_name.trim()\n ? node.display_name\n : '';\n const queryLower = query.toLowerCase();\n const matchTarget = displayName ? `${name} ${displayName}` : name;\n\n if (matchTarget.toLowerCase().includes(queryLower)) {\n results.push({\n name: displayName || name,\n path: fullPath,\n isFile: !!node.type,\n });\n }\n\n if (!node.type) {\n // If it's a directory, search recursively\n results = [\n ...results,\n ...searchFiles(query, node as DirectoryStructure, [\n ...currentPath,\n name,\n ]),\n ];\n }\n });\n\n return results;\n };\n\n const handleSearchSelect = (result: SearchResult) => {\n if (result.isFile) {\n const pathParts = result.path.split('/');\n const fileName = pathParts.pop() || '';\n setCurrentPath(pathParts);\n\n setSelectedFile({\n id: result.path,\n name: result.name || fileName,\n });\n } else {\n setCurrentPath(result.path.split('/'));\n setSelectedFile(null);\n }\n setSearchQuery('');\n setSearchResults([]);\n };\n\n const renderFileSearch = () => {\n return (\n
\n {\n setSearchQuery(e.target.value);\n if (directoryStructure) {\n setSearchResults(searchFiles(e.target.value, directoryStructure));\n }\n }}\n placeholder={t('settings.sources.searchFiles')}\n className={`h-[38px] w-full border border-[#D1D9E0] px-4 py-2 dark:border-[#6A6A6A] ${searchQuery ? 'rounded-t-[24px]' : 'rounded-[24px]'} bg-transparent focus:outline-none dark:text-[#E0E0E0]`}\n />\n\n {searchQuery && (\n
\n
\n {searchResults.length === 0 ? (\n
\n {t('settings.sources.noResults')}\n
\n ) : (\n searchResults.map((result, index) => (\n handleSearchSelect(result)}\n title={result.path}\n className={`flex min-w-0 cursor-pointer items-center px-3 py-2 hover:bg-[#ECEEEF] dark:hover:bg-[#27282D] ${\n index !== searchResults.length - 1\n ? 'border-b border-[#D1D9E0] dark:border-[#6A6A6A]'\n : ''\n }`}\n >\n \n \n {result.name}\n \n
\n ))\n )}\n
\n
\n )}\n \n );\n };\n\n const handleFileSearch = (searchQuery: string) => {\n if (directoryStructure) {\n return searchFiles(searchQuery, directoryStructure);\n }\n return [];\n };\n\n const getDisplayNameForPath = (path: string) => {\n if (!directoryStructure) {\n return path.split('/').pop() || path;\n }\n let structure: any = directoryStructure;\n if (typeof structure === 'string') {\n try {\n structure = JSON.parse(structure);\n } catch (e) {\n return path.split('/').pop() || path;\n }\n }\n if (typeof structure !== 'object' || structure === null) {\n return path.split('/').pop() || path;\n }\n const parts = path.split('/').filter(Boolean);\n let current: any = structure;\n for (const part of parts) {\n if (!current || typeof current !== 'object') {\n return parts[parts.length - 1] || path;\n }\n current = current[part];\n }\n if (\n current &&\n typeof current === 'object' &&\n typeof current.display_name === 'string' &&\n current.display_name.trim()\n ) {\n return current.display_name;\n }\n return parts[parts.length - 1] || path;\n };\n\n const handleFileSelect = (path: string) => {\n const pathParts = path.split('/');\n const fileName = pathParts.pop() || '';\n setCurrentPath(pathParts);\n setSelectedFile({\n id: path,\n name: getDisplayNameForPath(path) || fileName,\n });\n };\n\n const currentDirectory = getCurrentDirectory();\n\n const navigateToPath = (index: number) => {\n setCurrentPath(currentPath.slice(0, index + 1));\n };\n\n return (\n
\n {selectedFile ? (\n
\n
\n setSelectedFile(null)}\n path={selectedFile.id}\n displayPath={[...currentPath, selectedFile.name].join('/')}\n onFileSearch={handleFileSearch}\n onFileSelect={handleFileSelect}\n />\n
\n
\n ) : (\n
\n
{renderPathNavigation()}
\n\n
\n \n \n \n \n \n {t('settings.sources.fileName')}\n \n \n {t('settings.sources.tokens')}\n \n \n {t('settings.sources.size')}\n \n \n \n {t('settings.sources.actions')}\n \n \n \n \n \n {loading ? (\n \n ) : (\n renderFileTree(getCurrentDirectory())\n )}\n \n
\n \n
\n
\n )}\n\n \n
\n );\n};\n\nexport default ConnectorTree;\n" }, { "path": "frontend/src/components/ContextMenu.tsx", "content": "import { CSSProperties, SyntheticEvent, useEffect, useRef } from 'react';\n\nimport type { LucideIcon } from 'lucide-react';\n\nexport interface MenuOption {\n icon?: string | LucideIcon;\n label: string;\n onClick: (event: SyntheticEvent) => void;\n variant?: 'primary' | 'danger';\n iconClassName?: string;\n iconWidth?: number;\n iconHeight?: number;\n}\n\ninterface ContextMenuProps {\n isOpen: boolean;\n setIsOpen: (isOpen: boolean) => void;\n options: MenuOption[];\n anchorRef: React.RefObject;\n position?: 'bottom-left' | 'bottom-right' | 'top-left' | 'top-right';\n offset?: { x: number; y: number };\n className?: string;\n}\n\nexport default function ContextMenu({\n isOpen,\n setIsOpen,\n options,\n anchorRef,\n className = '',\n position = 'bottom-right',\n offset = { x: 0, y: 8 },\n}: ContextMenuProps) {\n const menuRef = useRef(null);\n useEffect(() => {\n if (isOpen && menuRef.current) {\n const positionStyle = getMenuPosition();\n if (menuRef.current) {\n Object.assign(menuRef.current.style, {\n top: positionStyle.top,\n left: positionStyle.left,\n });\n }\n }\n }, [isOpen]);\n useEffect(() => {\n const handleClickOutside = (event: MouseEvent) => {\n if (\n menuRef.current &&\n !menuRef.current.contains(event.target as Node) &&\n !anchorRef.current?.contains(event.target as Node)\n ) {\n setIsOpen(false);\n }\n };\n\n if (isOpen) {\n document.addEventListener('mousedown', handleClickOutside);\n return () =>\n document.removeEventListener('mousedown', handleClickOutside);\n }\n }, [isOpen, setIsOpen]);\n\n if (!isOpen) return null;\n\n const getMenuPosition = (): CSSProperties => {\n if (!anchorRef.current) return {};\n\n const rect = anchorRef.current.getBoundingClientRect();\n const scrollY = window.scrollY || document.documentElement.scrollTop;\n const scrollX = window.scrollX || document.documentElement.scrollLeft;\n\n let top = rect.bottom + scrollY + offset.y;\n let left = rect.right + scrollX + offset.x;\n\n // Get menu dimensions (need ref to be available)\n const menuWidth = menuRef.current?.offsetWidth || 144; // Default min-width\n const menuHeight = menuRef.current?.offsetHeight || 0;\n\n // Get viewport dimensions\n const viewportWidth = window.innerWidth;\n const viewportHeight = window.innerHeight;\n\n // Adjust position based on specified position\n switch (position) {\n case 'bottom-left':\n left = rect.right + scrollX - menuWidth + offset.x;\n break;\n case 'top-right':\n top = rect.top + scrollY - offset.y - menuHeight;\n break;\n case 'top-left':\n top = rect.top + scrollY - offset.y - menuHeight;\n left = rect.right + scrollX - menuWidth + offset.x;\n break;\n // bottom-right is default\n }\n\n if (left + menuWidth > viewportWidth) {\n left = Math.max(5, viewportWidth - menuWidth - 5);\n }\n\n if (left < 5) {\n left = 5;\n }\n\n if (top + menuHeight > viewportHeight + scrollY) {\n top = rect.top + scrollY - menuHeight - offset.y;\n }\n\n if (top < scrollY + 5) {\n top = rect.bottom + scrollY + offset.y;\n }\n\n return {\n position: 'fixed',\n top: `${top}px`,\n left: `${left}px`,\n };\n };\n\n return (\n e.stopPropagation()}\n >\n \n {options.map((option, index) => (\n {\n event.preventDefault();\n event.stopPropagation();\n option.onClick(event);\n setIsOpen(false);\n }}\n className={`flex items-center justify-start gap-4 p-3 transition-colors duration-200 ease-in-out ${index === 0 ? 'rounded-t-xl' : ''} ${index === options.length - 1 ? 'rounded-b-xl' : ''} ${\n option.variant === 'danger'\n ? 'text-rosso-corsa hover:bg-bright-gray dark:text-red-2000 dark:hover:bg-charcoal-grey/20'\n : 'text-eerie-black hover:bg-bright-gray dark:text-bright-gray dark:hover:bg-charcoal-grey/20'\n } `}\n >\n {option.icon && (\n
\n {typeof option.icon === 'string' ? (\n \n ) : (\n \n )}\n
\n )}\n {option.label}\n \n ))}\n \n \n );\n}\n" }, { "path": "frontend/src/components/CopyButton.tsx", "content": "import clsx from 'clsx';\nimport copy from 'copy-to-clipboard';\nimport { useCallback, useEffect, useRef, useState } from 'react';\nimport { useTranslation } from 'react-i18next';\n\nimport CheckMark from '../assets/checkmark.svg?react';\nimport CopyIcon from '../assets/copy.svg?react';\n\ntype CopyButtonProps = {\n textToCopy: string;\n iconSize?: string;\n padding?: string;\n showText?: boolean;\n copiedDuration?: number;\n className?: string;\n iconWrapperClassName?: string;\n textClassName?: string;\n};\n\nconst DEFAULT_ICON_SIZE = 'w-4 h-4';\nconst DEFAULT_PADDING = 'p-2';\nconst DEFAULT_COPIED_DURATION = 2000;\n\nexport default function CopyButton({\n textToCopy,\n iconSize = DEFAULT_ICON_SIZE,\n padding = DEFAULT_PADDING,\n showText = false,\n copiedDuration = DEFAULT_COPIED_DURATION,\n className,\n iconWrapperClassName,\n textClassName,\n}: CopyButtonProps) {\n const { t } = useTranslation();\n const [isCopied, setIsCopied] = useState(false);\n const timeoutIdRef = useRef(null);\n\n const iconWrapperClasses = clsx(\n 'flex items-center justify-center rounded-full transition-colors duration-150 ease-in-out',\n padding,\n {\n [`bg-[#FFFFFF}] dark:bg-transparent hover:bg-[#EEEEEE] dark:hover:bg-purple-taupe`]:\n !isCopied,\n 'bg-green-100 dark:bg-green-900 hover:bg-green-100 dark:hover:bg-green-900':\n isCopied,\n },\n iconWrapperClassName,\n );\n\n const rootButtonClasses = clsx(\n 'flex items-center gap-2 group',\n 'focus:outline-hidden focus-visible:ring-2 focus-visible:ring-offset-2 focus-visible:ring-blue-500 rounded-full',\n className,\n );\n\n const textSpanClasses = clsx(\n 'text-xs text-gray-600 dark:text-gray-400 transition-opacity duration-150 ease-in-out',\n { 'opacity-75': isCopied },\n textClassName,\n );\n\n const IconComponent = isCopied ? CheckMark : CopyIcon;\n const iconClasses = clsx(iconSize, {\n 'stroke-green-600 dark:stroke-green-400': isCopied,\n 'fill-none text-gray-700 dark:text-gray-300': !isCopied,\n });\n\n const buttonTitle = isCopied\n ? t('conversation.copied')\n : t('conversation.copy');\n const displayedText = isCopied\n ? t('conversation.copied')\n : t('conversation.copy');\n\n const handleCopy = useCallback(() => {\n if (isCopied) return;\n\n try {\n const success = copy(textToCopy);\n if (success) {\n setIsCopied(true);\n\n if (timeoutIdRef.current) {\n clearTimeout(timeoutIdRef.current);\n }\n\n timeoutIdRef.current = setTimeout(() => {\n setIsCopied(false);\n timeoutIdRef.current = null;\n }, copiedDuration);\n } else {\n console.warn('Copy command failed.');\n }\n } catch (error) {\n console.error('Failed to copy text:', error);\n }\n }, [textToCopy, copiedDuration, isCopied]);\n\n useEffect(() => {\n return () => {\n if (timeoutIdRef.current) {\n clearTimeout(timeoutIdRef.current);\n }\n };\n }, []);\n return (\n \n
\n \n
\n {showText && {displayedText}}\n \n {isCopied ? t('conversation.copied', 'Copied to clipboard') : ''}\n \n \n );\n}\n" }, { "path": "frontend/src/components/DocumentPagination.tsx", "content": "import React, { useState } from 'react';\nimport { useTranslation } from 'react-i18next';\nimport SingleArrowLeft from '../assets/single-left-arrow.svg';\nimport SingleArrowRight from '../assets/single-right-arrow.svg';\nimport DoubleArrowLeft from '../assets/double-arrow-left.svg';\nimport DoubleArrowRight from '../assets/double-arrow-right.svg';\n\ninterface PaginationProps {\n currentPage: number;\n totalPages: number;\n rowsPerPage: number;\n onPageChange: (page: number) => void;\n onRowsPerPageChange: (rows: number) => void;\n}\n\nconst Pagination: React.FC = ({\n currentPage,\n totalPages,\n rowsPerPage,\n onPageChange,\n onRowsPerPageChange,\n}) => {\n const { t } = useTranslation();\n const [isDropdownOpen, setIsDropdownOpen] = useState(false);\n const rowsPerPageOptions = [5, 10, 20, 50];\n\n const toggleDropdown = () => setIsDropdownOpen((prev) => !prev);\n\n const handlePreviousPage = () => {\n if (currentPage > 1) {\n onPageChange(currentPage - 1);\n }\n };\n\n const handleNextPage = () => {\n if (currentPage < totalPages) {\n onPageChange(currentPage + 1);\n }\n };\n\n const handleFirstPage = () => {\n onPageChange(1);\n };\n\n const handleLastPage = () => {\n onPageChange(totalPages);\n };\n\n const handleSelectRowsPerPage = (rows: number) => {\n setIsDropdownOpen(false);\n onRowsPerPageChange(rows);\n };\n\n return (\n
\n {/* Rows per page dropdown */}\n
\n \n {t('pagination.rowsPerPage')}:\n \n
\n \n {rowsPerPage}\n \n \n {rowsPerPageOptions.map((option) => (\n handleSelectRowsPerPage(option)}\n className={`cursor-pointer px-4 py-2 text-xs hover:bg-gray-100 dark:hover:bg-neutral-700 ${\n rowsPerPage === option\n ? 'dark:text-light-gray bg-gray-100 dark:bg-neutral-700'\n : 'dark:bg-dark-charcoal dark:text-light-gray bg-white'\n }`}\n >\n {option}\n
\n ))}\n
\n
\n \n\n {/* Pagination controls */}\n
\n {t('pagination.pageOf', { currentPage, totalPages })}\n
\n
\n \n \n \n \n \n \n \n \n \n \n \n \n
\n \n );\n};\n\nexport default Pagination;\n" }, { "path": "frontend/src/components/Dropdown.tsx", "content": "import React from 'react';\n\nimport Arrow2 from '../assets/dropdown-arrow.svg';\nimport Edit from '../assets/edit.svg';\nimport Trash from '../assets/trash.svg';\nimport { DropdownOption, DropdownProps } from './types/Dropdown.types';\n\nfunction Dropdown({\n options,\n selectedValue,\n onSelect,\n size = 'w-32',\n rounded = 'xl',\n buttonClassName = 'border-silver bg-white dark:bg-transparent dark:border-dim-gray',\n optionsClassName = 'border-silver bg-white dark:border-dim-gray dark:bg-dark-charcoal',\n border = 'border-2',\n showEdit,\n onEdit,\n showDelete,\n onDelete,\n placeholder,\n placeholderClassName = 'text-gray-500 dark:text-gray-400',\n contentSize = 'text-base',\n}: DropdownProps) {\n const dropdownRef = React.useRef(null);\n const [isOpen, setIsOpen] = React.useState(false);\n const borderRadius = rounded === 'xl' ? 'rounded-xl' : 'rounded-3xl';\n const borderTopRadius = rounded === 'xl' ? 'rounded-t-xl' : 'rounded-t-3xl';\n\n const handleClickOutside = (event: MouseEvent) => {\n if (\n dropdownRef.current &&\n !dropdownRef.current.contains(event.target as Node)\n ) {\n setIsOpen(false);\n }\n };\n\n React.useEffect(() => {\n document.addEventListener('mousedown', handleClickOutside);\n return () => {\n document.removeEventListener('mousedown', handleClickOutside);\n };\n }, []);\n\n return (\n \n setIsOpen(!isOpen)}\n className={`flex w-full cursor-pointer items-center justify-between ${border} ${buttonClassName} px-5 py-3 ${\n isOpen ? `${borderTopRadius}` : `${borderRadius}`\n }`}\n >\n {typeof selectedValue === 'string' ? (\n \n {selectedValue}\n \n ) : (\n \n {selectedValue && 'label' in selectedValue\n ? selectedValue.label\n : selectedValue && 'description' in selectedValue\n ? `${\n selectedValue.value < 1e9\n ? selectedValue.value + ` (${selectedValue.description})`\n : selectedValue.description\n }`\n : placeholder\n ? placeholder\n : 'From URL'}\n \n )}\n \n \n {isOpen && (\n \n {options.map((option: any, index) => (\n \n {\n onSelect(option);\n setIsOpen(false);\n }}\n className={`dark:text-light-gray ml-5 flex-1 overflow-hidden py-3 text-ellipsis whitespace-nowrap ${contentSize}`}\n >\n {typeof option === 'string'\n ? option\n : option.name\n ? option.name\n : option.label\n ? option.label\n : `${\n option.value < 1e9\n ? option.value + ` (${option.description})`\n : option.description\n }`}\n \n {showEdit && onEdit && option.type !== 'public' && (\n {\n onEdit({\n id: option.id,\n name: option.name,\n type: option.type,\n });\n setIsOpen(false);\n }}\n />\n )}\n {showDelete && onDelete && (\n {\n e.stopPropagation();\n onDelete?.(typeof option === 'string' ? option : option.id);\n }}\n className={`${\n typeof showDelete === 'function' && !showDelete(option)\n ? 'hidden'\n : ''\n } mr-2 h-4 w-4 cursor-pointer hover:opacity-50`}\n >\n \n \n )}\n \n ))}\n \n )}\n \n );\n}\n\nexport default Dropdown;\n" }, { "path": "frontend/src/components/DropdownMenu.tsx", "content": "import React from 'react';\nimport ReactDOM from 'react-dom';\n\ntype DropdownMenuProps = {\n name: string;\n options: { label: string; value: string }[];\n onSelect: (value: string) => void;\n defaultValue?: string;\n icon?: string;\n isOpen: boolean;\n onOpenChange: (isOpen: boolean) => void;\n anchorRef: React.RefObject;\n position?: 'bottom-left' | 'bottom-right' | 'top-left' | 'top-right';\n offset?: { x: number; y: number };\n className?: string;\n};\n\nexport default function DropdownMenu({\n name,\n options,\n onSelect,\n defaultValue = 'none',\n icon,\n isOpen: controlledIsOpen,\n onOpenChange,\n anchorRef,\n className = '',\n position = 'bottom-right',\n offset = { x: 0, y: 8 },\n}: DropdownMenuProps) {\n const dropdownRef = React.useRef(null);\n const [internalIsOpen, setInternalIsOpen] = React.useState(false);\n const [selectedOption, setSelectedOption] = React.useState(\n options.find((option) => option.value === defaultValue) || options[0],\n );\n\n const isOpen =\n controlledIsOpen !== undefined ? controlledIsOpen : internalIsOpen;\n const setIsOpen = onOpenChange || setInternalIsOpen;\n\n const handleClickOutside = (event: MouseEvent) => {\n if (\n dropdownRef.current &&\n !dropdownRef.current.contains(event.target as Node) &&\n !anchorRef?.current?.contains(event.target as Node)\n ) {\n setIsOpen(false);\n }\n };\n\n const handleClickOption = (optionId: number) => {\n setIsOpen(false);\n setSelectedOption(options[optionId]);\n onSelect(options[optionId].value);\n };\n\n React.useEffect(() => {\n if (isOpen) {\n document.addEventListener('mousedown', handleClickOutside);\n return () =>\n document.removeEventListener('mousedown', handleClickOutside);\n }\n }, [isOpen]);\n\n if (!isOpen) return null;\n\n const getMenuPosition = (): React.CSSProperties => {\n if (!anchorRef?.current) return {};\n\n const rect = anchorRef.current.getBoundingClientRect();\n\n const top = rect.bottom + offset.y;\n const left = rect.right + offset.x;\n\n return {\n position: 'fixed',\n top: `${top}px`,\n left: `${left}px`,\n zIndex: 9999,\n };\n };\n\n // Use a portal to render the dropdown outside the table flow\n return ReactDOM.createPortal(\n e.stopPropagation()}\n >\n \n \n {options.map((option, idx) => (\n handleClickOption(idx)}\n >\n {option.label}\n \n ))}\n \n \n ,\n document.body,\n );\n}\n" }, { "path": "frontend/src/components/DropdownModel.tsx", "content": "import React, { useEffect } from 'react';\nimport { useDispatch, useSelector } from 'react-redux';\n\nimport modelService from '../api/services/modelService';\nimport Arrow2 from '../assets/dropdown-arrow.svg';\nimport RoundedTick from '../assets/rounded-tick.svg';\nimport {\n selectAvailableModels,\n selectSelectedModel,\n setAvailableModels,\n setModelsLoading,\n setSelectedModel,\n} from '../preferences/preferenceSlice';\n\nimport type { Model } from '../models/types';\n\nexport default function DropdownModel() {\n const dispatch = useDispatch();\n const selectedModel = useSelector(selectSelectedModel);\n const availableModels = useSelector(selectAvailableModels);\n const dropdownRef = React.useRef(null);\n const [isOpen, setIsOpen] = React.useState(false);\n\n useEffect(() => {\n const loadModels = async () => {\n if ((availableModels?.length ?? 0) > 0) {\n return;\n }\n dispatch(setModelsLoading(true));\n try {\n const response = await modelService.getModels(null);\n if (!response.ok) {\n throw new Error(`API error: ${response.status}`);\n }\n const data = await response.json();\n const models = data.models || [];\n const transformed = modelService.transformModels(models);\n\n dispatch(setAvailableModels(transformed));\n if (!selectedModel && transformed.length > 0) {\n const defaultModel =\n transformed.find((m) => m.id === data.default_model_id) ||\n transformed[0];\n dispatch(setSelectedModel(defaultModel));\n } else if (selectedModel && transformed.length > 0) {\n const isValid = transformed.find((m) => m.id === selectedModel.id);\n if (!isValid) {\n const defaultModel =\n transformed.find((m) => m.id === data.default_model_id) ||\n transformed[0];\n dispatch(setSelectedModel(defaultModel));\n }\n }\n } catch (error) {\n console.error('Failed to load models:', error);\n } finally {\n dispatch(setModelsLoading(false));\n }\n };\n\n loadModels();\n }, [availableModels?.length, dispatch, selectedModel]);\n\n const handleClickOutside = (event: MouseEvent) => {\n if (\n dropdownRef.current &&\n !dropdownRef.current.contains(event.target as Node)\n ) {\n setIsOpen(false);\n }\n };\n\n useEffect(() => {\n document.addEventListener('mousedown', handleClickOutside);\n return () => {\n document.removeEventListener('mousedown', handleClickOutside);\n };\n }, []);\n\n return (\n
\n setIsOpen(!isOpen)}\n >\n {selectedModel?.display_name ? (\n

\n {selectedModel.display_name}\n

\n ) : (\n

\n Select Model\n

\n )}\n \n
\n {isOpen && (\n
\n {availableModels && (availableModels?.length ?? 0) > 0 ? (\n availableModels.map((model: Model) => (\n {\n dispatch(setSelectedModel(model));\n setIsOpen(false);\n }}\n className={`border-gray-3000/75 dark:border-purple-taupe/50 hover:bg-gray-3000/75 dark:hover:bg-purple-taupe flex h-10 w-full cursor-pointer items-center justify-between border-t`}\n >\n
\n

\n {model.display_name}\n

\n {model.id === selectedModel?.id ? (\n \n ) : null}\n
\n
\n ))\n ) : (\n
\n

No models available

\n
\n )}\n \n )}\n \n );\n}\n" }, { "path": "frontend/src/components/FilePicker.tsx", "content": "import React, { useState, useEffect, useCallback, useRef } from 'react';\nimport { useTranslation } from 'react-i18next';\nimport { formatBytes } from '../utils/stringUtils';\nimport { formatDate } from '../utils/dateTimeUtils';\nimport {\n getSessionToken,\n setSessionToken,\n removeSessionToken,\n} from '../utils/providerUtils';\nimport ConnectorAuth from '../components/ConnectorAuth';\nimport FileIcon from '../assets/file.svg';\nimport FolderIcon from '../assets/folder.svg';\nimport CheckIcon from '../assets/checkmark.svg';\nimport SearchIcon from '../assets/search.svg';\nimport Input from './Input';\nimport {\n Table,\n TableContainer,\n TableHead,\n TableBody,\n TableRow,\n TableHeader,\n TableCell,\n} from './Table';\n\ninterface CloudFile {\n id: string;\n name: string;\n type: string;\n size?: number;\n modifiedTime: string;\n isFolder?: boolean;\n}\n\ninterface CloudFilePickerProps {\n onSelectionChange: (\n selectedFileIds: string[],\n selectedFolderIds?: string[],\n ) => void;\n onDisconnect?: () => void;\n provider: string;\n token: string | null;\n initialSelectedFiles?: string[];\n initialSelectedFolders?: string[];\n}\n\nexport const FilePicker: React.FC = ({\n onSelectionChange,\n onDisconnect,\n provider,\n token,\n initialSelectedFiles = [],\n}) => {\n const PROVIDER_CONFIG = {\n google_drive: {\n displayName: 'Drive',\n rootName: 'My Drive',\n },\n share_point: {\n displayName: 'SharePoint',\n rootName: 'My Files',\n },\n } as const;\n\n const getProviderConfig = (provider: string) => {\n return (\n PROVIDER_CONFIG[provider as keyof typeof PROVIDER_CONFIG] || {\n displayName: provider,\n rootName: 'Root',\n }\n );\n };\n\n const { t } = useTranslation();\n const [files, setFiles] = useState([]);\n const [selectedFiles, setSelectedFiles] =\n useState(initialSelectedFiles);\n const [selectedFolders, setSelectedFolders] = useState([]);\n const [isLoading, setIsLoading] = useState(false);\n const [hasMoreFiles, setHasMoreFiles] = useState(false);\n const [nextPageToken, setNextPageToken] = useState(null);\n const [currentFolderId, setCurrentFolderId] = useState(null);\n const [folderPath, setFolderPath] = useState<\n Array<{ id: string | null; name: string }>\n >([\n {\n id: null,\n name: getProviderConfig(provider).rootName,\n },\n ]);\n const [searchQuery, setSearchQuery] = useState('');\n const [authError, setAuthError] = useState('');\n const [isConnected, setIsConnected] = useState(false);\n const [userEmail, setUserEmail] = useState('');\n const [allowsSharedContent, setAllowsSharedContent] = useState(false);\n const [activeTab, setActiveTab] = useState<'my_files' | 'shared'>(\n 'my_files',\n );\n\n const scrollContainerRef = useRef(null);\n const searchTimeoutRef = useRef | null>(null);\n const abortControllerRef = useRef(null);\n\n const isFolder = (file: CloudFile) => {\n return (\n file.isFolder ||\n file.type === 'application/vnd.google-apps.folder' ||\n file.type === 'folder'\n );\n };\n\n const loadCloudFiles = useCallback(\n async (\n sessionToken: string,\n folderId: string | null,\n pageToken?: string,\n searchQuery = '',\n shared = false,\n ) => {\n // Cancel any in-flight request so stale responses never overwrite new state\n abortControllerRef.current?.abort();\n const controller = new AbortController();\n abortControllerRef.current = controller;\n\n setIsLoading(true);\n\n const apiHost = import.meta.env.VITE_API_HOST;\n if (!pageToken) {\n setFiles([]);\n }\n\n try {\n const body: Record = {\n provider: provider,\n session_token: sessionToken,\n folder_id: folderId,\n limit: 10,\n page_token: pageToken,\n search_query: searchQuery,\n shared: shared,\n };\n const response = await fetch(`${apiHost}/api/connectors/files`, {\n method: 'POST',\n headers: {\n 'Content-Type': 'application/json',\n Authorization: `Bearer ${token}`,\n },\n body: JSON.stringify(body),\n signal: controller.signal,\n });\n\n const data = await response.json();\n if (data.success) {\n setFiles((prev) =>\n pageToken ? [...prev, ...data.files] : data.files,\n );\n setNextPageToken(data.next_page_token);\n setHasMoreFiles(!!data.next_page_token);\n } else {\n console.error('Error loading files:', data.error);\n if (!pageToken) {\n setFiles([]);\n }\n }\n } catch (err) {\n if ((err as Error).name === 'AbortError') return;\n console.error('Error loading files:', err);\n if (!pageToken) {\n setFiles([]);\n }\n } finally {\n if (!controller.signal.aborted) {\n setIsLoading(false);\n }\n }\n },\n [token, provider],\n );\n\n const validateAndLoadFiles = useCallback(async () => {\n const sessionToken = getSessionToken(provider);\n if (!sessionToken) {\n setIsConnected(false);\n return;\n }\n\n try {\n const apiHost = import.meta.env.VITE_API_HOST;\n const validateResponse = await fetch(\n `${apiHost}/api/connectors/validate-session`,\n {\n method: 'POST',\n headers: {\n 'Content-Type': 'application/json',\n Authorization: `Bearer ${token}`,\n },\n body: JSON.stringify({\n provider: provider,\n session_token: sessionToken,\n }),\n },\n );\n\n if (!validateResponse.ok) {\n removeSessionToken(provider);\n setIsConnected(false);\n setAuthError('Session expired. Please reconnect to Google Drive.');\n return;\n }\n\n const validateData = await validateResponse.json();\n if (validateData.success) {\n setUserEmail(validateData.user_email || 'Connected User');\n setIsConnected(true);\n setAuthError('');\n if (provider === 'share_point') {\n setAllowsSharedContent(\n validateData.allows_shared_content ?? false,\n );\n }\n\n setFiles([]);\n setNextPageToken(null);\n setHasMoreFiles(false);\n setCurrentFolderId(null);\n setActiveTab('my_files');\n setFolderPath([\n {\n id: null,\n name: getProviderConfig(provider).rootName,\n },\n ]);\n loadCloudFiles(sessionToken, null, undefined, '');\n } else {\n removeSessionToken(provider);\n setIsConnected(false);\n setAuthError(\n validateData.error ||\n 'Session expired. Please reconnect your account.',\n );\n }\n } catch (error) {\n console.error('Error validating session:', error);\n setAuthError('Failed to validate session. Please reconnect.');\n setIsConnected(false);\n }\n }, [provider, token, loadCloudFiles]);\n\n useEffect(() => {\n validateAndLoadFiles();\n }, [validateAndLoadFiles]);\n\n const handleScroll = useCallback(() => {\n const scrollContainer = scrollContainerRef.current;\n if (!scrollContainer) return;\n\n const { scrollTop, scrollHeight, clientHeight } = scrollContainer;\n const isNearBottom = scrollHeight - scrollTop - clientHeight < 50;\n\n if (isNearBottom && hasMoreFiles && !isLoading && nextPageToken) {\n const sessionToken = getSessionToken(provider);\n if (sessionToken) {\n loadCloudFiles(\n sessionToken,\n currentFolderId,\n nextPageToken,\n searchQuery,\n activeTab === 'shared' && !currentFolderId,\n );\n }\n }\n }, [\n hasMoreFiles,\n isLoading,\n nextPageToken,\n currentFolderId,\n searchQuery,\n provider,\n loadCloudFiles,\n activeTab,\n ]);\n\n useEffect(() => {\n const scrollContainer = scrollContainerRef.current;\n if (scrollContainer) {\n scrollContainer.addEventListener('scroll', handleScroll);\n return () => scrollContainer.removeEventListener('scroll', handleScroll);\n }\n }, [handleScroll]);\n\n useEffect(() => {\n return () => {\n if (searchTimeoutRef.current) {\n clearTimeout(searchTimeoutRef.current);\n }\n abortControllerRef.current?.abort();\n };\n }, []);\n\n const handleSearchChange = (query: string) => {\n setSearchQuery(query);\n\n if (searchTimeoutRef.current) {\n clearTimeout(searchTimeoutRef.current);\n }\n\n searchTimeoutRef.current = setTimeout(() => {\n const sessionToken = getSessionToken(provider);\n if (sessionToken) {\n loadCloudFiles(\n sessionToken,\n currentFolderId,\n undefined,\n query,\n activeTab === 'shared' && !currentFolderId,\n );\n }\n }, 300);\n };\n\n const handleFolderClick = (folderId: string, folderName: string) => {\n if (folderId === currentFolderId) {\n return;\n }\n\n setIsLoading(true);\n\n setCurrentFolderId(folderId);\n setFolderPath((prev) => [...prev, { id: folderId, name: folderName }]);\n setSearchQuery('');\n\n const sessionToken = getSessionToken(provider);\n if (sessionToken) {\n loadCloudFiles(sessionToken, folderId, undefined, '', false);\n }\n };\n\n const navigateBack = (index: number) => {\n if (index >= folderPath.length - 1) return;\n\n const newFolderPath = folderPath.slice(0, index + 1);\n const newFolderId = newFolderPath[newFolderPath.length - 1].id;\n\n setFolderPath(newFolderPath);\n setCurrentFolderId(newFolderId);\n setSearchQuery('');\n\n const sessionToken = getSessionToken(provider);\n if (sessionToken) {\n loadCloudFiles(\n sessionToken,\n newFolderId,\n undefined,\n '',\n activeTab === 'shared' && !newFolderId,\n );\n }\n };\n\n const handleTabChange = (tab: 'my_files' | 'shared') => {\n if (tab === activeTab) return;\n setActiveTab(tab);\n setFiles([]);\n setNextPageToken(null);\n setHasMoreFiles(false);\n setCurrentFolderId(null);\n setSearchQuery('');\n setFolderPath([\n {\n id: null,\n name:\n tab === 'shared'\n ? 'Shared'\n : getProviderConfig(provider).rootName,\n },\n ]);\n const sessionToken = getSessionToken(provider);\n if (sessionToken) {\n loadCloudFiles(sessionToken, null, undefined, '', tab === 'shared');\n }\n };\n\n\n\n const handleFileSelect = (fileId: string, isFolder: boolean) => {\n if (isFolder) {\n const newSelectedFolders = selectedFolders.includes(fileId)\n ? selectedFolders.filter((id) => id !== fileId)\n : [...selectedFolders, fileId];\n setSelectedFolders(newSelectedFolders);\n onSelectionChange(selectedFiles, newSelectedFolders);\n } else {\n const newSelectedFiles = selectedFiles.includes(fileId)\n ? selectedFiles.filter((id) => id !== fileId)\n : [...selectedFiles, fileId];\n setSelectedFiles(newSelectedFiles);\n onSelectionChange(newSelectedFiles, selectedFolders);\n }\n };\n\n return (\n
\n {authError && (\n
{authError}
\n )}\n\n {\n setUserEmail(data.user_email || 'Connected User');\n setIsConnected(true);\n setAuthError('');\n\n if (data.session_token) {\n setSessionToken(provider, data.session_token);\n validateAndLoadFiles();\n }\n }}\n onError={(error) => {\n setAuthError(error);\n setIsConnected(false);\n }}\n isConnected={isConnected}\n userEmail={userEmail}\n onDisconnect={() => {\n const sessionToken = getSessionToken(provider);\n if (sessionToken) {\n const apiHost = import.meta.env.VITE_API_HOST;\n fetch(`${apiHost}/api/connectors/disconnect`, {\n method: 'POST',\n headers: {\n 'Content-Type': 'application/json',\n Authorization: `Bearer ${token}`,\n },\n body: JSON.stringify({\n provider: provider,\n session_token: sessionToken,\n }),\n }).catch((err) =>\n console.error(\n `Error disconnecting from ${getProviderConfig(provider).displayName}:`,\n err,\n ),\n );\n }\n\n removeSessionToken(provider);\n setIsConnected(false);\n setAllowsSharedContent(false);\n setActiveTab('my_files');\n setFiles([]);\n setSelectedFiles([]);\n onSelectionChange([]);\n\n if (onDisconnect) {\n onDisconnect();\n }\n }}\n />\n\n {isConnected && (\n
\n
\n {provider === 'share_point' && allowsSharedContent && (\n
\n handleTabChange('my_files')}\n className={`px-4 py-2 text-sm font-medium ${\n activeTab === 'my_files'\n ? 'border-b-2 border-[#A076F6] text-[#A076F6]'\n : 'text-gray-600 hover:text-gray-800 dark:text-gray-400 dark:hover:text-gray-200'\n }`}\n >\n {t('filePicker.myFiles')}\n \n handleTabChange('shared')}\n className={`px-4 py-2 text-sm font-medium ${\n activeTab === 'shared'\n ? 'border-b-2 border-[#A076F6] text-[#A076F6]'\n : 'text-gray-600 hover:text-gray-800 dark:text-gray-400 dark:hover:text-gray-200'\n }`}\n >\n {t('filePicker.sharedWithMe')}\n \n
\n )}\n
\n
\n {folderPath.map((path, index) => (\n \n {index > 0 && /}\n navigateBack(index)}\n className=\"text-sm text-[#A076F6] hover:text-[#8A5FD4] hover:underline\"\n disabled={index === folderPath.length - 1}\n >\n {path.name}\n \n
\n ))}\n
\n\n
\n Select Files from {getProviderConfig(provider).displayName}\n
\n\n
\n handleSearchChange(e.target.value)}\n colorVariant=\"silver\"\n borderVariant=\"thin\"\n labelBgClassName=\"bg-[#EEE6FF78] dark:bg-[#2A262E]\"\n leftIcon={\n \"Search\"\n }\n />\n
\n\n {/* Selected Files Message */}\n
\n {t('filePicker.itemsSelected', {\n count: selectedFiles.length + selectedFolders.length,\n })}\n
\n
\n\n
\n \n {\n <>\n \n \n \n \n \n {t('filePicker.name')}\n \n \n {t('filePicker.lastModified')}\n \n \n {t('filePicker.size')}\n \n \n \n \n {isLoading && files.length === 0\n ? Array.from({ length: 5 }).map((_, i) => (\n \n \n
\n \n \n
\n \n \n
\n \n \n
\n \n \n ))\n : files.map((file, index) => (\n {\n if (isFolder(file)) {\n handleFolderClick(file.id, file.name);\n } else {\n handleFileSelect(file.id, false);\n }\n }}\n >\n \n {\n e.stopPropagation();\n handleFileSelect(file.id, isFolder(file));\n }}\n >\n {(isFolder(file)\n ? selectedFolders\n : selectedFiles\n ).includes(file.id) && (\n \n )}\n
\n \n \n
\n
\n \n
\n {file.name}\n
\n \n \n {formatDate(file.modifiedTime)}\n \n \n {file.size ? formatBytes(file.size) : '-'}\n \n \n ))}\n {isLoading && files.length > 0 &&\n Array.from({ length: 3 }).map((_, i) => (\n \n \n
\n \n \n
\n \n \n
\n \n \n
\n \n \n ))}\n \n
\n \n }\n \n
\n
\n
\n )}\n \n );\n};\n" }, { "path": "frontend/src/components/FileSelectionSkeleton.tsx", "content": "const FilesSectionSkeleton = () => (\n
\n
\n
\n
\n
\n
\n
\n
\n
\n);\n\nexport default FilesSectionSkeleton;\n" }, { "path": "frontend/src/components/FileTree.tsx", "content": "import React, { useState, useRef, useEffect } from 'react';\nimport { useTranslation } from 'react-i18next';\nimport { useSelector } from 'react-redux';\nimport { selectToken } from '../preferences/preferenceSlice';\nimport { formatBytes } from '../utils/stringUtils';\nimport Chunks from './Chunks';\nimport ContextMenu, { MenuOption } from './ContextMenu';\nimport SkeletonLoader from './SkeletonLoader';\nimport userService from '../api/services/userService';\nimport FileIcon from '../assets/file.svg';\nimport FolderIcon from '../assets/folder.svg';\nimport ArrowLeft from '../assets/arrow-left.svg';\nimport ThreeDots from '../assets/three-dots.svg';\nimport EyeView from '../assets/eye-view.svg';\nimport Trash from '../assets/red-trash.svg';\nimport { SOURCE_FILE_TREE_ACCEPT_ATTR } from '../constants/fileUpload';\nimport { useOutsideAlerter, useLoaderState } from '../hooks';\nimport ConfirmationModal from '../modals/ConfirmationModal';\nimport {\n Table,\n TableContainer,\n TableHead,\n TableBody,\n TableRow,\n TableHeader,\n TableCell,\n} from './Table';\n\ninterface FileNode {\n type?: string;\n token_count?: number;\n size_bytes?: number;\n display_name?: string;\n [key: string]: any;\n}\n\ninterface DirectoryStructure {\n [key: string]: FileNode;\n}\n\ninterface FileTreeProps {\n docId: string;\n sourceName: string;\n onBackToDocuments: () => void;\n}\n\ninterface SearchResult {\n name: string;\n path: string;\n isFile: boolean;\n}\n\nconst FileTree: React.FC = ({\n docId,\n sourceName,\n onBackToDocuments,\n}) => {\n const { t } = useTranslation();\n const [loading, setLoading] = useLoaderState(true, 500);\n const [error, setError] = useState(null);\n const [directoryStructure, setDirectoryStructure] =\n useState(null);\n const [currentPath, setCurrentPath] = useState([]);\n const token = useSelector(selectToken);\n const [activeMenuId, setActiveMenuId] = useState(null);\n const menuRefs = useRef<{\n [key: string]: React.RefObject;\n }>({});\n const [selectedFile, setSelectedFile] = useState<{\n id: string;\n name: string;\n } | null>(null);\n const [searchQuery, setSearchQuery] = useState('');\n const [searchResults, setSearchResults] = useState([]);\n const searchDropdownRef = useRef(null);\n const currentOpRef = useRef(\n null,\n );\n\n const [deleteModalState, setDeleteModalState] = useState<\n 'ACTIVE' | 'INACTIVE'\n >('INACTIVE');\n const [itemToDelete, setItemToDelete] = useState<{\n name: string;\n isFile: boolean;\n } | null>(null);\n\n type QueuedOperation = {\n operation: 'add' | 'remove' | 'remove_directory';\n files?: File[];\n filePath?: string;\n directoryPath?: string;\n parentDirPath?: string;\n };\n const opQueueRef = useRef([]);\n const processingRef = useRef(false);\n const [queueLength, setQueueLength] = useState(0);\n\n useOutsideAlerter(\n searchDropdownRef,\n () => {\n setSearchQuery('');\n setSearchResults([]);\n },\n [],\n false,\n );\n\n const handleFileClick = (fileName: string, displayName?: string) => {\n const fullPath = [...currentPath, fileName].join('/');\n setSelectedFile({\n id: fullPath,\n name: displayName ?? fileName,\n });\n };\n\n useEffect(() => {\n const fetchDirectoryStructure = async () => {\n try {\n setLoading(true);\n const response = await userService.getDirectoryStructure(docId, token);\n const data = await response.json();\n\n if (data && data.directory_structure) {\n setDirectoryStructure(data.directory_structure);\n } else {\n setError('Invalid response format');\n }\n } catch (err) {\n setError('Failed to load directory structure');\n console.error(err);\n } finally {\n setLoading(false);\n }\n };\n\n if (docId) {\n fetchDirectoryStructure();\n }\n }, [docId, token]);\n\n const navigateToDirectory = (dirName: string) => {\n setCurrentPath((prev) => [...prev, dirName]);\n };\n\n const navigateUp = () => {\n setCurrentPath((prev) => prev.slice(0, -1));\n };\n\n const getCurrentDirectory = (): DirectoryStructure => {\n if (!directoryStructure) return {};\n\n let structure = directoryStructure;\n if (typeof structure === 'string') {\n try {\n structure = JSON.parse(structure);\n } catch (e) {\n console.error(\n 'Error parsing directory structure in getCurrentDirectory:',\n e,\n );\n return {};\n }\n }\n\n if (typeof structure !== 'object' || structure === null) {\n return {};\n }\n\n let current: any = structure;\n for (const dir of currentPath) {\n if (\n current[dir] &&\n typeof current[dir] === 'object' &&\n !current[dir].type\n ) {\n current = current[dir];\n } else {\n return {};\n }\n }\n return current;\n };\n\n const handleBackNavigation = () => {\n if (selectedFile) {\n setSelectedFile(null);\n } else if (currentPath.length === 0) {\n if (onBackToDocuments) {\n onBackToDocuments();\n }\n } else {\n navigateUp();\n }\n };\n\n const getMenuRef = (itemId: string) => {\n if (!menuRefs.current[itemId]) {\n menuRefs.current[itemId] = React.createRef();\n }\n return menuRefs.current[itemId];\n };\n\n const handleMenuClick = (e: React.MouseEvent, itemId: string) => {\n e.preventDefault();\n e.stopPropagation();\n\n if (activeMenuId === itemId) {\n setActiveMenuId(null);\n return;\n }\n setActiveMenuId(itemId);\n };\n\n const getActionOptions = (\n name: string,\n isFile: boolean,\n _itemId: string,\n displayName?: string,\n ): MenuOption[] => {\n const options: MenuOption[] = [];\n\n options.push({\n icon: EyeView,\n label: t('settings.sources.view'),\n onClick: (event: React.SyntheticEvent) => {\n event.stopPropagation();\n if (isFile) {\n handleFileClick(name, displayName);\n } else {\n navigateToDirectory(name);\n }\n },\n iconWidth: 18,\n iconHeight: 18,\n variant: 'primary',\n });\n\n options.push({\n icon: Trash,\n label: t('convTile.delete'),\n onClick: (event: React.SyntheticEvent) => {\n event.stopPropagation();\n confirmDeleteItem(name, isFile);\n },\n iconWidth: 18,\n iconHeight: 18,\n variant: 'danger',\n });\n\n return options;\n };\n\n const confirmDeleteItem = (name: string, isFile: boolean) => {\n setItemToDelete({ name, isFile });\n setDeleteModalState('ACTIVE');\n setActiveMenuId(null);\n };\n\n const handleConfirmedDelete = async () => {\n if (itemToDelete) {\n await handleDeleteFile(itemToDelete.name, itemToDelete.isFile);\n setDeleteModalState('INACTIVE');\n setItemToDelete(null);\n }\n };\n\n const handleCancelDelete = () => {\n setDeleteModalState('INACTIVE');\n setItemToDelete(null);\n };\n\n const manageSource = async (\n operation: 'add' | 'remove' | 'remove_directory',\n files?: File[] | null,\n filePath?: string,\n directoryPath?: string,\n parentDirPath?: string,\n ) => {\n currentOpRef.current = operation;\n\n try {\n const formData = new FormData();\n formData.append('source_id', docId);\n formData.append('operation', operation);\n\n if (operation === 'add' && files && files.length) {\n formData.append('parent_dir', parentDirPath ?? currentPath.join('/'));\n\n for (let i = 0; i < files.length; i++) {\n formData.append('file', files[i]);\n }\n } else if (operation === 'remove' && filePath) {\n const filePaths = JSON.stringify([filePath]);\n formData.append('file_paths', filePaths);\n } else if (operation === 'remove_directory' && directoryPath) {\n formData.append('directory_path', directoryPath);\n }\n\n const response = await userService.manageSourceFiles(formData, token);\n const result = await response.json();\n\n if (result.success && result.reingest_task_id) {\n if (operation === 'add') {\n console.log('Files uploaded successfully:', result.added_files);\n } else if (operation === 'remove') {\n console.log('Files deleted successfully:', result.removed_files);\n } else if (operation === 'remove_directory') {\n console.log(\n 'Directory deleted successfully:',\n result.removed_directory,\n );\n }\n console.log('Reingest task started:', result.reingest_task_id);\n\n const maxAttempts = 30;\n const pollInterval = 2000;\n\n for (let attempt = 0; attempt < maxAttempts; attempt++) {\n try {\n const statusResponse = await userService.getTaskStatus(\n result.reingest_task_id,\n token,\n );\n const statusData = await statusResponse.json();\n\n console.log(\n `Task status (attempt ${attempt + 1}):`,\n statusData.status,\n );\n\n if (statusData.status === 'SUCCESS') {\n console.log('Task completed successfully');\n\n const structureResponse = await userService.getDirectoryStructure(\n docId,\n token,\n );\n const structureData = await structureResponse.json();\n\n if (structureData && structureData.directory_structure) {\n setDirectoryStructure(structureData.directory_structure);\n currentOpRef.current = null;\n return true;\n }\n break;\n } else if (statusData.status === 'FAILURE') {\n console.error('Task failed');\n break;\n }\n\n await new Promise((resolve) => setTimeout(resolve, pollInterval));\n } catch (error) {\n console.error('Error polling task status:', error);\n break;\n }\n }\n } else {\n throw new Error(\n `Failed to ${operation} ${operation === 'remove_directory' ? 'directory' : 'file(s)'}`,\n );\n }\n } catch (error) {\n const actionText =\n operation === 'add'\n ? 'uploading'\n : operation === 'remove_directory'\n ? 'deleting directory'\n : 'deleting file(s)';\n const errorText =\n operation === 'add'\n ? 'upload'\n : operation === 'remove_directory'\n ? 'delete directory'\n : 'delete file(s)';\n console.error(`Error ${actionText}:`, error);\n setError(`Failed to ${errorText}`);\n } finally {\n currentOpRef.current = null;\n }\n\n return false;\n };\n\n const processQueue = async () => {\n if (processingRef.current) return;\n processingRef.current = true;\n try {\n while (opQueueRef.current.length > 0) {\n const nextOp = opQueueRef.current.shift()!;\n setQueueLength(opQueueRef.current.length);\n await manageSource(\n nextOp.operation,\n nextOp.files,\n nextOp.filePath,\n nextOp.directoryPath,\n nextOp.parentDirPath,\n );\n }\n } finally {\n processingRef.current = false;\n }\n };\n\n const enqueueOperation = (op: QueuedOperation) => {\n opQueueRef.current.push(op);\n setQueueLength(opQueueRef.current.length);\n if (!processingRef.current) {\n void processQueue();\n }\n };\n\n const handleAddFile = () => {\n const fileInput = document.createElement('input');\n fileInput.type = 'file';\n fileInput.multiple = true;\n fileInput.accept = SOURCE_FILE_TREE_ACCEPT_ATTR;\n\n fileInput.onchange = async (event) => {\n const fileList = (event.target as HTMLInputElement).files;\n if (!fileList || fileList.length === 0) return;\n const files = Array.from(fileList);\n enqueueOperation({\n operation: 'add',\n files,\n parentDirPath: currentPath.join('/'),\n });\n };\n\n fileInput.click();\n };\n\n const handleDeleteFile = async (name: string, isFile: boolean) => {\n // Construct the full path to the file or directory\n const itemPath = [...currentPath, name].join('/');\n\n if (isFile) {\n enqueueOperation({ operation: 'remove', filePath: itemPath });\n } else {\n enqueueOperation({\n operation: 'remove_directory',\n directoryPath: itemPath,\n });\n }\n };\n\n const renderPathNavigation = () => {\n return (\n
\n {/* Left side with path navigation */}\n
\n \n \"left-arrow\"\n \n\n
\n \n {sourceName}\n \n {currentPath.length > 0 && (\n <>\n /\n {currentPath.map((dir, index) => (\n \n \n {dir}\n \n {index < currentPath.length - 1 && (\n \n /\n \n )}\n \n ))}\n \n )}\n {selectedFile && (\n <>\n /\n \n {selectedFile.name}\n \n \n )}\n
\n
\n\n
\n {processingRef.current && (\n
\n {currentOpRef.current === 'add'\n ? t('settings.sources.uploadingFilesTitle')\n : t('settings.sources.deletingTitle')}\n
\n )}\n\n {renderFileSearch()}\n\n {/* Add file button */}\n {!processingRef.current && (\n \n {t('settings.sources.addFile')}\n \n )}\n
\n
\n );\n };\n const calculateDirectoryStats = (\n structure: DirectoryStructure,\n ): { totalSize: number; totalTokens: number } => {\n let totalSize = 0;\n let totalTokens = 0;\n\n Object.entries(structure).forEach(([_, node]) => {\n if (node.type) {\n // It's a file\n totalSize += node.size_bytes || 0;\n totalTokens += node.token_count || 0;\n } else {\n // It's a directory, recurse\n const stats = calculateDirectoryStats(node);\n totalSize += stats.totalSize;\n totalTokens += stats.totalTokens;\n }\n });\n\n return { totalSize, totalTokens };\n };\n\n const renderFileTree = (structure: DirectoryStructure): React.ReactNode[] => {\n // Separate directories and files\n const entries = Object.entries(structure);\n const directories = entries.filter(([_, node]) => !node.type);\n const files = entries.filter(([_, node]) => node.type);\n\n // Create parent directory row\n const parentRow =\n currentPath.length > 0\n ? [\n \n \n
\n \n ..\n
\n \n \n -\n \n \n -\n \n \n ,\n ]\n : [];\n\n // Render directories first, then files\n return [\n ...parentRow,\n ...directories.map(([name, node]) => {\n const itemId = `dir-${name}`;\n const menuRef = getMenuRef(itemId);\n const dirStats = calculateDirectoryStats(node as DirectoryStructure);\n\n return (\n navigateToDirectory(name)}>\n \n
\n \n {name}\n
\n \n \n {dirStats.totalSize > 0 ? formatBytes(dirStats.totalSize) : '-'}\n \n \n {dirStats.totalTokens > 0\n ? dirStats.totalTokens.toLocaleString()\n : '-'}\n \n \n
\n handleMenuClick(e, itemId)}\n className=\"inline-flex h-[35px] w-[24px] shrink-0 items-center justify-center rounded-md font-medium transition-colors hover:bg-[#EBEBEB] dark:hover:bg-[#26272E]\"\n aria-label={t('settings.sources.menuAlt')}\n >\n \n \n \n setActiveMenuId(isOpen ? itemId : null)\n }\n options={getActionOptions(name, false, itemId)}\n anchorRef={menuRef}\n position=\"bottom-left\"\n offset={{ x: -4, y: 4 }}\n />\n
\n \n \n );\n }),\n ...files.map(([name, node]) => {\n const itemId = `file-${name}`;\n const menuRef = getMenuRef(itemId);\n const displayName =\n typeof node.display_name === 'string' && node.display_name.trim()\n ? node.display_name\n : name;\n\n return (\n handleFileClick(name, displayName)}\n >\n \n
\n \n {displayName}\n
\n \n \n {node.size_bytes ? formatBytes(node.size_bytes) : '-'}\n \n \n {node.token_count?.toLocaleString() || '-'}\n \n \n
\n handleMenuClick(e, itemId)}\n className=\"inline-flex h-[35px] w-[24px] shrink-0 items-center justify-center rounded-md font-medium transition-colors hover:bg-[#EBEBEB] dark:hover:bg-[#26272E]\"\n aria-label={t('settings.sources.menuAlt')}\n >\n \n \n \n setActiveMenuId(isOpen ? itemId : null)\n }\n options={getActionOptions(name, true, itemId, displayName)}\n anchorRef={menuRef}\n position=\"bottom-left\"\n offset={{ x: -4, y: 4 }}\n />\n
\n \n \n );\n }),\n ];\n };\n const currentDirectory = getCurrentDirectory();\n\n const searchFiles = (\n query: string,\n structure: DirectoryStructure,\n currentPath: string[] = [],\n ): SearchResult[] => {\n let results: SearchResult[] = [];\n\n Object.entries(structure).forEach(([name, node]) => {\n const fullPath = [...currentPath, name].join('/');\n const displayName =\n typeof node.display_name === 'string' && node.display_name.trim()\n ? node.display_name\n : '';\n const queryLower = query.toLowerCase();\n const matchTarget = displayName ? `${name} ${displayName}` : name;\n\n if (matchTarget.toLowerCase().includes(queryLower)) {\n results.push({\n name: displayName || name,\n path: fullPath,\n isFile: !!node.type,\n });\n }\n\n if (!node.type) {\n // If it's a directory, search recursively\n results = [\n ...results,\n ...searchFiles(query, node as DirectoryStructure, [\n ...currentPath,\n name,\n ]),\n ];\n }\n });\n\n return results;\n };\n\n const handleSearchSelect = (result: SearchResult) => {\n if (result.isFile) {\n const pathParts = result.path.split('/');\n const fileName = pathParts.pop() || '';\n setCurrentPath(pathParts);\n\n setSelectedFile({\n id: result.path,\n name: result.name || fileName,\n });\n } else {\n setCurrentPath(result.path.split('/'));\n setSelectedFile(null);\n }\n setSearchQuery('');\n setSearchResults([]);\n };\n\n const renderFileSearch = () => {\n return (\n
\n {\n setSearchQuery(e.target.value);\n if (directoryStructure) {\n setSearchResults(searchFiles(e.target.value, directoryStructure));\n }\n }}\n placeholder={t('settings.sources.searchFiles')}\n className={`h-[38px] w-full border border-[#D1D9E0] px-4 py-2 dark:border-[#6A6A6A] ${searchQuery ? 'rounded-t-[24px]' : 'rounded-[24px]'} bg-transparent focus:outline-none dark:text-[#E0E0E0]`}\n />\n\n {searchQuery && (\n
\n
\n {searchResults.length === 0 ? (\n
\n {t('settings.sources.noResults')}\n
\n ) : (\n searchResults.map((result, index) => (\n handleSearchSelect(result)}\n title={result.path}\n className={`flex min-w-0 cursor-pointer items-center px-3 py-2 hover:bg-[#ECEEEF] dark:hover:bg-[#27282D] ${\n index !== searchResults.length - 1\n ? 'border-b border-[#D1D9E0] dark:border-[#6A6A6A]'\n : ''\n }`}\n >\n \n \n {result.name}\n \n
\n ))\n )}\n
\n
\n )}\n \n );\n };\n\n const handleFileSearch = (searchQuery: string) => {\n if (directoryStructure) {\n return searchFiles(searchQuery, directoryStructure);\n }\n return [];\n };\n\n const getDisplayNameForPath = (path: string) => {\n if (!directoryStructure) {\n return path.split('/').pop() || path;\n }\n let structure: any = directoryStructure;\n if (typeof structure === 'string') {\n try {\n structure = JSON.parse(structure);\n } catch (e) {\n return path.split('/').pop() || path;\n }\n }\n if (typeof structure !== 'object' || structure === null) {\n return path.split('/').pop() || path;\n }\n const parts = path.split('/').filter(Boolean);\n let current: any = structure;\n for (const part of parts) {\n if (!current || typeof current !== 'object') {\n return parts[parts.length - 1] || path;\n }\n current = current[part];\n }\n if (\n current &&\n typeof current === 'object' &&\n typeof current.display_name === 'string' &&\n current.display_name.trim()\n ) {\n return current.display_name;\n }\n return parts[parts.length - 1] || path;\n };\n\n const handleFileSelect = (path: string) => {\n const pathParts = path.split('/');\n const fileName = pathParts.pop() || '';\n setCurrentPath(pathParts);\n setSelectedFile({\n id: path,\n name: getDisplayNameForPath(path) || fileName,\n });\n };\n\n return (\n
\n {selectedFile ? (\n
\n
\n setSelectedFile(null)}\n path={selectedFile.id}\n displayPath={[...currentPath, selectedFile.name].join('/')}\n onFileSearch={handleFileSearch}\n onFileSelect={handleFileSelect}\n />\n
\n
\n ) : (\n
\n
{renderPathNavigation()}
\n\n
\n \n \n \n \n \n {t('settings.sources.fileName')}\n \n \n {t('settings.sources.size')}\n \n \n {t('settings.sources.tokens')}\n \n \n \n {t('settings.sources.actions')}\n \n \n \n \n \n {loading ? (\n \n ) : (\n renderFileTree(currentDirectory)\n )}\n \n
\n \n
\n
\n )}\n \n
\n );\n};\n\nexport default FileTree;\n" }, { "path": "frontend/src/components/FileUpload.tsx", "content": "import React, { useCallback, useState } from 'react';\nimport { useTranslation } from 'react-i18next';\nimport { useDropzone } from 'react-dropzone';\nimport { twMerge } from 'tailwind-merge';\n\nimport Cross from '../assets/cross.svg';\nimport ImagesIcon from '../assets/images.svg';\n\ninterface FileUploadProps {\n onUpload: (files: File[]) => void;\n onRemove?: (file: File) => void;\n multiple?: boolean;\n maxFiles?: number;\n maxSize?: number; // in bytes\n accept?: Record; // e.g. { 'image/*': ['.png', '.jpg'] }\n showPreview?: boolean;\n previewSize?: number;\n\n children?: React.ReactNode;\n className?: string;\n activeClassName?: string;\n acceptClassName?: string;\n rejectClassName?: string;\n\n uploadText?: string | { text: string; colorClass?: string }[];\n dragActiveText?: string;\n fileTypeText?: string;\n sizeLimitText?: string;\n\n disabled?: boolean;\n validator?: (file: File) => { isValid: boolean; error?: string };\n}\n\nexport const FileUpload = ({\n onUpload,\n onRemove,\n multiple = false,\n maxFiles = 1,\n maxSize = 5 * 1024 * 1024,\n accept = { 'image/*': ['.jpeg', '.png', '.jpg'] },\n showPreview = false,\n previewSize = 80,\n children,\n className = 'border-2 border-dashed rounded-3xl p-6 text-center cursor-pointer transition-colors border-silver dark:border-[#7E7E7E]',\n activeClassName = 'border-blue-500 bg-blue-50',\n acceptClassName = 'border-green-500 dark:border-green-500 bg-green-50 dark:bg-green-50/10',\n rejectClassName = 'border-red-500 bg-red-50 dark:bg-red-500/10 dark:border-red-500',\n uploadText,\n dragActiveText,\n fileTypeText,\n sizeLimitText,\n disabled = false,\n validator,\n}: FileUploadProps) => {\n const { t } = useTranslation();\n const [errors, setErrors] = useState([]);\n const [preview, setPreview] = useState(null);\n const [currentFile, setCurrentFile] = useState(null);\n\n const validateFile = (file: File) => {\n const defaultValidation = {\n isValid: true,\n error: '',\n };\n\n if (validator) {\n const customValidation = validator(file);\n if (!customValidation.isValid) {\n return customValidation;\n }\n }\n\n if (file.size > maxSize) {\n return {\n isValid: false,\n error: t('components.fileUpload.fileSizeError', {\n size: maxSize / 1024 / 1024,\n }),\n };\n }\n\n return defaultValidation;\n };\n\n const onDrop = useCallback(\n (acceptedFiles: File[], fileRejections: any[]) => {\n setErrors([]);\n\n if (fileRejections.length > 0) {\n const newErrors = fileRejections\n .map(({ errors }) => errors.map((e: any) => e.message))\n .flat();\n setErrors(newErrors);\n return;\n }\n\n const validationResults = acceptedFiles.map(validateFile);\n const invalidFiles = validationResults.filter((r) => !r.isValid);\n\n if (invalidFiles.length > 0) {\n setErrors(invalidFiles.map((f) => f.error!));\n return;\n }\n\n const filesToUpload = multiple ? acceptedFiles : [acceptedFiles[0]];\n onUpload(filesToUpload);\n\n const file = acceptedFiles[0];\n setCurrentFile(file);\n\n if (showPreview && file.type.startsWith('image/')) {\n const reader = new FileReader();\n reader.onload = () => setPreview(reader.result as string);\n reader.readAsDataURL(file);\n }\n },\n [onUpload, multiple, maxSize, validator],\n );\n\n const {\n getRootProps,\n getInputProps,\n isDragActive,\n isDragAccept,\n isDragReject,\n } = useDropzone({\n onDrop,\n multiple,\n maxFiles,\n maxSize,\n accept,\n disabled,\n });\n\n const currentClassName = twMerge(\n 'border-2 border-dashed rounded-3xl p-8 text-center cursor-pointer transition-colors border-silver dark:border-[#7E7E7E]',\n className,\n isDragActive && activeClassName,\n isDragAccept && acceptClassName,\n isDragReject && rejectClassName,\n disabled && 'opacity-50 cursor-not-allowed',\n );\n\n const handleRemove = () => {\n setPreview(null);\n setCurrentFile(null);\n if (onRemove && currentFile) onRemove(currentFile);\n };\n\n const renderPreview = () => (\n \n \n {\n e.stopPropagation();\n handleRemove();\n }}\n className=\"absolute -top-2 -right-2 rounded-full bg-[#7D54D1] p-1 transition-colors hover:bg-[#714cbc]\"\n >\n \"remove\"\n \n \n );\n\n const renderUploadText = () => {\n if (Array.isArray(uploadText)) {\n return (\n

\n {uploadText.map((segment, i) => (\n \n {segment.text}\n \n ))}\n

\n );\n }\n return (\n

\n {uploadText || t('components.fileUpload.clickToUpload')}\n

\n );\n };\n\n const defaultContent = (\n
\n {showPreview && preview ? (\n renderPreview()\n ) : (\n \n \n
\n )}\n
\n
\n {isDragActive ? (\n

\n {dragActiveText || t('components.fileUpload.dropFiles')}\n

\n ) : (\n renderUploadText()\n )}\n
\n

\n {fileTypeText || t('components.fileUpload.fileTypes')}{' '}\n {maxSize / 1024 / 1024}\n {sizeLimitText || t('components.fileUpload.sizeLimitUnit')}\n

\n
\n \n );\n\n return (\n
\n
\n \n {children || defaultContent}\n {errors.length > 0 && (\n
\n {errors.map((error, i) => (\n

\n {error}\n

\n ))}\n
\n )}\n
\n
\n );\n};\n" }, { "path": "frontend/src/components/GoogleDrivePicker.tsx", "content": "import React, { useState, useEffect } from 'react';\nimport { useTranslation } from 'react-i18next';\nimport useDrivePicker from 'react-google-drive-picker';\n\nimport ConnectorAuth from './ConnectorAuth';\nimport {\n getSessionToken,\n setSessionToken,\n removeSessionToken,\n validateProviderSession,\n} from '../utils/providerUtils';\nimport ConnectedStateSkeleton from './ConnectedStateSkeleton';\nimport FilesSectionSkeleton from './FileSelectionSkeleton';\n\ninterface PickerFile {\n id: string;\n name: string;\n mimeType: string;\n iconUrl: string;\n description?: string;\n sizeBytes?: string;\n}\n\ninterface GoogleDrivePickerProps {\n token: string | null;\n onSelectionChange: (fileIds: string[], folderIds?: string[]) => void;\n}\n\nconst GoogleDrivePicker: React.FC = ({\n token,\n onSelectionChange,\n}) => {\n const { t } = useTranslation();\n const [selectedFiles, setSelectedFiles] = useState([]);\n const [selectedFolders, setSelectedFolders] = useState([]);\n const [isLoading, setIsLoading] = useState(false);\n const [userEmail, setUserEmail] = useState('');\n const [isConnected, setIsConnected] = useState(false);\n const [authError, setAuthError] = useState('');\n const [accessToken, setAccessToken] = useState(null);\n const [isValidating, setIsValidating] = useState(false);\n\n const [openPicker] = useDrivePicker();\n\n useEffect(() => {\n const sessionToken = getSessionToken('google_drive');\n if (sessionToken) {\n setIsValidating(true);\n setIsConnected(true); // Optimistically set as connected for skeleton\n validateSession(sessionToken);\n }\n }, [token]);\n\n const validateSession = async (sessionToken: string) => {\n try {\n const validateResponse = await validateProviderSession(\n token,\n 'google_drive',\n );\n\n if (!validateResponse.ok) {\n setIsConnected(false);\n setAuthError(\n t('modals.uploadDoc.connectors.googleDrive.sessionExpired'),\n );\n setIsValidating(false);\n return false;\n }\n\n const validateData = await validateResponse.json();\n if (validateData.success) {\n setUserEmail(\n validateData.user_email ||\n t('modals.uploadDoc.connectors.auth.connectedUser'),\n );\n setIsConnected(true);\n setAuthError('');\n setAccessToken(validateData.access_token || null);\n setIsValidating(false);\n return true;\n } else {\n setIsConnected(false);\n setAuthError(\n validateData.error ||\n t('modals.uploadDoc.connectors.googleDrive.sessionExpiredGeneric'),\n );\n setIsValidating(false);\n return false;\n }\n } catch (error) {\n console.error('Error validating session:', error);\n setAuthError(t('modals.uploadDoc.connectors.googleDrive.validateFailed'));\n setIsConnected(false);\n setIsValidating(false);\n return false;\n }\n };\n\n const handleOpenPicker = async () => {\n setIsLoading(true);\n\n const sessionToken = getSessionToken('google_drive');\n\n if (!sessionToken) {\n setAuthError(t('modals.uploadDoc.connectors.googleDrive.noSession'));\n setIsLoading(false);\n return;\n }\n\n if (!accessToken) {\n setAuthError(t('modals.uploadDoc.connectors.googleDrive.noAccessToken'));\n setIsLoading(false);\n return;\n }\n\n try {\n const clientId: string = import.meta.env.VITE_GOOGLE_CLIENT_ID;\n\n // Derive appId from clientId (extract numeric part before first dash)\n const appId = clientId ? clientId.split('-')[0] : null;\n\n if (!clientId || !appId) {\n console.error('Missing Google Drive configuration');\n\n setIsLoading(false);\n return;\n }\n\n openPicker({\n clientId: clientId,\n developerKey: '',\n appId: appId,\n setSelectFolderEnabled: false,\n viewId: 'DOCS',\n showUploadView: false,\n showUploadFolders: false,\n supportDrives: false,\n multiselect: true,\n token: accessToken,\n viewMimeTypes:\n 'application/vnd.google-apps.document,application/vnd.google-apps.presentation,application/vnd.google-apps.spreadsheet,application/pdf,application/vnd.openxmlformats-officedocument.wordprocessingml.document,application/vnd.openxmlformats-officedocument.presentationml.presentation,application/vnd.openxmlformats-officedocument.spreadsheetml.sheet,application/msword,application/vnd.ms-powerpoint,application/vnd.ms-excel,text/plain,text/csv,text/html,text/markdown,text/x-rst,application/json,application/epub+zip,application/rtf,image/jpeg,image/jpg,image/png',\n callbackFunction: (data: any) => {\n setIsLoading(false);\n if (data.action === 'picked') {\n const docs = data.docs;\n\n const newFiles: PickerFile[] = [];\n const newFolders: PickerFile[] = [];\n\n docs.forEach((doc: any) => {\n const item = {\n id: doc.id,\n name: doc.name,\n mimeType: doc.mimeType,\n iconUrl: doc.iconUrl || '',\n description: doc.description,\n sizeBytes: doc.sizeBytes,\n };\n\n if (doc.mimeType === 'application/vnd.google-apps.folder') {\n newFolders.push(item);\n } else {\n newFiles.push(item);\n }\n });\n\n setSelectedFiles((prevFiles) => {\n const existingFileIds = new Set(prevFiles.map((file) => file.id));\n const uniqueNewFiles = newFiles.filter(\n (file) => !existingFileIds.has(file.id),\n );\n return [...prevFiles, ...uniqueNewFiles];\n });\n\n setSelectedFolders((prevFolders) => {\n const existingFolderIds = new Set(\n prevFolders.map((folder) => folder.id),\n );\n const uniqueNewFolders = newFolders.filter(\n (folder) => !existingFolderIds.has(folder.id),\n );\n return [...prevFolders, ...uniqueNewFolders];\n });\n onSelectionChange(\n [...selectedFiles, ...newFiles].map((file) => file.id),\n [...selectedFolders, ...newFolders].map((folder) => folder.id),\n );\n }\n },\n });\n } catch (error) {\n console.error('Error opening picker:', error);\n setAuthError(t('modals.uploadDoc.connectors.googleDrive.pickerFailed'));\n setIsLoading(false);\n }\n };\n\n const handleDisconnect = async () => {\n const sessionToken = getSessionToken('google_drive');\n if (sessionToken) {\n try {\n const apiHost = import.meta.env.VITE_API_HOST;\n await fetch(`${apiHost}/api/connectors/disconnect`, {\n method: 'POST',\n headers: {\n 'Content-Type': 'application/json',\n Authorization: `Bearer ${token}`,\n },\n body: JSON.stringify({\n provider: 'google_drive',\n session_token: sessionToken,\n }),\n });\n } catch (err) {\n console.error('Error disconnecting from Google Drive:', err);\n }\n }\n\n removeSessionToken('google_drive');\n setIsConnected(false);\n setSelectedFiles([]);\n setSelectedFolders([]);\n setAccessToken(null);\n setUserEmail('');\n setAuthError('');\n onSelectionChange([], []);\n };\n\n return (\n
\n {isValidating ? (\n <>\n \n \n \n ) : (\n <>\n {\n setUserEmail(\n data.user_email ||\n t('modals.uploadDoc.connectors.auth.connectedUser'),\n );\n setIsConnected(true);\n setAuthError('');\n\n if (data.session_token) {\n setSessionToken('google_drive', data.session_token);\n validateSession(data.session_token);\n }\n }}\n onError={(error) => {\n setAuthError(error);\n setIsConnected(false);\n }}\n isConnected={isConnected}\n userEmail={userEmail}\n onDisconnect={handleDisconnect}\n errorMessage={authError}\n />\n\n {isConnected && (\n
\n
\n
\n

\n {t('modals.uploadDoc.connectors.googleDrive.selectedFiles')}\n

\n handleOpenPicker()}\n className=\"rounded-md bg-[#A076F6] px-3 py-1 text-sm text-white hover:bg-[#8A5FD4]\"\n disabled={isLoading}\n >\n {isLoading\n ? t('modals.uploadDoc.connectors.googleDrive.loading')\n : t(\n 'modals.uploadDoc.connectors.googleDrive.selectFiles',\n )}\n \n
\n\n {selectedFiles.length === 0 && selectedFolders.length === 0 ? (\n

\n {t(\n 'modals.uploadDoc.connectors.googleDrive.noFilesSelected',\n )}\n

\n ) : (\n
\n {selectedFolders.length > 0 && (\n
\n

\n {t('modals.uploadDoc.connectors.googleDrive.folders')}\n

\n {selectedFolders.map((folder) => (\n \n \n \n {folder.name}\n \n {\n const newSelectedFolders =\n selectedFolders.filter(\n (f) => f.id !== folder.id,\n );\n setSelectedFolders(newSelectedFolders);\n onSelectionChange(\n selectedFiles.map((f) => f.id),\n newSelectedFolders.map((f) => f.id),\n );\n }}\n className=\"ml-2 text-sm text-red-500 hover:text-red-700\"\n >\n {t(\n 'modals.uploadDoc.connectors.googleDrive.remove',\n )}\n \n
\n ))}\n
\n )}\n\n {selectedFiles.length > 0 && (\n
\n

\n {t('modals.uploadDoc.connectors.googleDrive.files')}\n

\n {selectedFiles.map((file) => (\n \n \n \n {file.name}\n \n {\n const newSelectedFiles = selectedFiles.filter(\n (f) => f.id !== file.id,\n );\n setSelectedFiles(newSelectedFiles);\n onSelectionChange(\n newSelectedFiles.map((f) => f.id),\n selectedFolders.map((f) => f.id),\n );\n }}\n className=\"ml-2 text-sm text-red-500 hover:text-red-700\"\n >\n {t(\n 'modals.uploadDoc.connectors.googleDrive.remove',\n )}\n \n
\n ))}\n
\n )}\n
\n )}\n
\n \n )}\n \n )}\n \n );\n};\n\nexport default GoogleDrivePicker;\n" }, { "path": "frontend/src/components/Head.tsx", "content": "import React from 'react';\n\ninterface HeadProps {\n title?: string;\n description?: string;\n keywords?: string;\n ogTitle?: string;\n ogDescription?: string;\n ogImage?: string;\n twitterCard?: string;\n twitterTitle?: string;\n twitterDescription?: string;\n children?: React.ReactNode;\n}\n\nexport function Head({\n title,\n description,\n keywords,\n ogTitle,\n ogDescription,\n ogImage,\n twitterCard,\n twitterTitle,\n twitterDescription,\n children,\n}: HeadProps) {\n return (\n <>\n {title && {title}}\n {description && }\n {keywords && }\n\n {/* Open Graph */}\n {ogTitle && }\n {ogDescription && (\n \n )}\n {ogImage && }\n\n {/* Twitter */}\n {twitterCard && }\n {twitterTitle && }\n {twitterDescription && (\n \n )}\n\n {/* Additional elements */}\n {children}\n \n );\n}\n" }, { "path": "frontend/src/components/Help.tsx", "content": "import { useState, useRef, useEffect } from 'react';\nimport Info from '../assets/info.svg';\nimport PageIcon from '../assets/documentation.svg';\nimport EmailIcon from '../assets/envelope.svg';\nimport { useTranslation } from 'react-i18next';\nconst Help = () => {\n const [isOpen, setIsOpen] = useState(false);\n const dropdownRef = useRef(null);\n const buttonRef = useRef(null);\n const { t } = useTranslation();\n\n const toggleDropdown = () => {\n setIsOpen((prev) => !prev);\n };\n\n const handleClickOutside = (event: MouseEvent) => {\n if (\n dropdownRef.current &&\n !dropdownRef.current.contains(event.target as Node) &&\n buttonRef.current &&\n !buttonRef.current.contains(event.target as Node)\n ) {\n setIsOpen(false);\n }\n };\n\n useEffect(() => {\n document.addEventListener('mousedown', handleClickOutside);\n return () => {\n document.removeEventListener('mousedown', handleClickOutside);\n };\n }, []);\n\n return (\n
\n \n \"info\"\n {t('help')}\n \n {isOpen && (\n \n \n \n {t('documentation')}\n \n \n \n {t('emailUs')}\n \n
\n )}\n \n );\n};\n\nexport default Help;\n" }, { "path": "frontend/src/components/Input.tsx", "content": "import { InputProps } from './types';\nimport { useRef } from 'react';\n\nconst Input = ({\n id,\n name,\n type,\n value,\n isAutoFocused = false,\n placeholder,\n required = false,\n maxLength,\n className = '',\n colorVariant = 'silver',\n borderVariant = 'thick',\n textSize = 'medium',\n children,\n labelBgClassName = 'bg-white dark:bg-raisin-black',\n leftIcon,\n onChange,\n onPaste,\n onKeyDown,\n edgeRoundness = 'rounded-full',\n}: InputProps) => {\n const colorStyles = {\n silver: 'border-silver dark:border-silver/40',\n jet: 'border-jet',\n gray: 'border-gray-5000 dark:text-silver',\n };\n const borderStyles = {\n thin: 'border',\n thick: 'border-2',\n };\n const textSizeStyles = {\n small: 'text-sm',\n medium: 'text-base',\n };\n\n const inputRef = useRef(null);\n\n const hasValue = value !== undefined && value !== null && value !== '';\n\n return (\n
\n \n {children}\n \n {leftIcon && (\n
\n {leftIcon}\n
\n )}\n {placeholder && (\n \n {placeholder}\n {required && (\n *\n )}\n \n )}\n
\n );\n};\n\nexport default Input;\n" }, { "path": "frontend/src/components/MermaidRenderer.tsx", "content": "import React, { useEffect, useRef, useState } from 'react';\nimport { useTranslation } from 'react-i18next';\nimport mermaid from 'mermaid';\nimport CopyButton from './CopyButton';\nimport { Prism as SyntaxHighlighter } from 'react-syntax-highlighter';\nimport {\n oneLight,\n vscDarkPlus,\n} from 'react-syntax-highlighter/dist/cjs/styles/prism';\nimport { MermaidRendererProps } from './types';\nimport { useSelector } from 'react-redux';\nimport { selectStatus } from '../conversation/conversationSlice';\nimport { useDarkTheme } from '../hooks';\n\nconst MermaidRenderer: React.FC = ({\n code,\n isLoading,\n}) => {\n const { t } = useTranslation();\n const [isDarkTheme] = useDarkTheme();\n const diagramId = useRef(\n `mermaid-${Date.now()}-${Math.random().toString(36).substring(2)}`,\n );\n const status = useSelector(selectStatus);\n const [error, setError] = useState(null);\n const [showCode, setShowCode] = useState(false);\n const [showDownloadMenu, setShowDownloadMenu] = useState(false);\n const downloadMenuRef = useRef(null);\n const containerRef = useRef(null);\n const [hoverPosition, setHoverPosition] = useState<{\n x: number;\n y: number;\n } | null>(null);\n const [isHovering, setIsHovering] = useState(false);\n const [zoomFactor, setZoomFactor] = useState(2);\n\n const handleMouseMove = (event: React.MouseEvent) => {\n if (!containerRef.current) return;\n\n const rect = containerRef.current.getBoundingClientRect();\n const x = (event.clientX - rect.left) / rect.width;\n const y = (event.clientY - rect.top) / rect.height;\n\n setHoverPosition({ x, y });\n };\n\n const handleMouseEnter = () => setIsHovering(true);\n const handleMouseLeave = () => {\n setIsHovering(false);\n setHoverPosition(null);\n };\n\n const handleKeyDown = (event: React.KeyboardEvent) => {\n if (!isHovering) return;\n\n if (event.key === '+' || event.key === '=') {\n setZoomFactor((prev) => Math.min(6, prev + 0.5)); // Cap at 6x\n event.preventDefault();\n } else if (event.key === '-') {\n setZoomFactor((prev) => Math.max(1, prev - 0.5)); // Minimum 1x\n event.preventDefault();\n }\n };\n\n const handleWheel = (event: React.WheelEvent) => {\n if (!isHovering) return;\n\n if (event.ctrlKey || event.metaKey) {\n event.preventDefault();\n\n if (event.deltaY < 0) {\n setZoomFactor((prev) => Math.min(6, prev + 0.25));\n } else {\n setZoomFactor((prev) => Math.max(1, prev - 0.25));\n }\n }\n };\n\n const getTransformOrigin = () => {\n if (!hoverPosition) return 'center center';\n return `${hoverPosition.x * 100}% ${hoverPosition.y * 100}%`;\n };\n\n useEffect(() => {\n const renderDiagram = async () => {\n mermaid.initialize({\n startOnLoad: true,\n theme: isDarkTheme ? 'dark' : 'default',\n securityLevel: 'loose',\n suppressErrorRendering: true,\n });\n\n const isCurrentlyLoading =\n isLoading !== undefined ? isLoading : status === 'loading';\n if (!isCurrentlyLoading && code) {\n try {\n const element = document.getElementById(diagramId.current);\n if (element) {\n element.removeAttribute('data-processed');\n await mermaid.parse(code); //syntax check\n mermaid.contentLoaded();\n }\n } catch (err) {\n console.error('Error rendering mermaid diagram:', err);\n setError(\n `Failed to render diagram: ${err instanceof Error ? err.message : String(err)}`,\n );\n }\n }\n };\n\n renderDiagram();\n }, [code, isDarkTheme, isLoading]);\n\n useEffect(() => {\n const handleClickOutside = (event: MouseEvent) => {\n if (\n downloadMenuRef.current &&\n !downloadMenuRef.current.contains(event.target as Node)\n ) {\n setShowDownloadMenu(false);\n }\n };\n\n document.addEventListener('mousedown', handleClickOutside);\n return () => {\n document.removeEventListener('mousedown', handleClickOutside);\n };\n }, [showDownloadMenu]);\n\n const downloadSvg = (): void => {\n const element = document.getElementById(diagramId.current);\n if (!element) return;\n const svgElement = element.querySelector('svg');\n if (!svgElement) return;\n\n const svgClone = svgElement.cloneNode(true) as SVGElement;\n\n if (!svgClone.hasAttribute('xmlns')) {\n svgClone.setAttribute('xmlns', 'http://www.w3.org/2000/svg');\n }\n\n if (!svgClone.hasAttribute('width') || !svgClone.hasAttribute('height')) {\n const viewBox = svgClone.getAttribute('viewBox')?.split(' ') || [];\n if (viewBox.length === 4) {\n svgClone.setAttribute('width', viewBox[2]);\n svgClone.setAttribute('height', viewBox[3]);\n }\n }\n\n const serializer = new XMLSerializer();\n const svgString = serializer.serializeToString(svgClone);\n\n const svgBlob = new Blob(\n [`\\n${svgString}`],\n { type: 'image/svg+xml' },\n );\n\n const url = URL.createObjectURL(svgBlob);\n const link = document.createElement('a');\n link.href = url;\n link.download = 'diagram.svg';\n document.body.appendChild(link);\n link.click();\n document.body.removeChild(link);\n URL.revokeObjectURL(url);\n };\n\n const downloadPng = (): void => {\n const element = document.getElementById(diagramId.current);\n if (!element) return;\n\n const svgElement = element.querySelector('svg');\n if (!svgElement) return;\n\n const svgClone = svgElement.cloneNode(true) as SVGElement;\n\n if (!svgClone.hasAttribute('xmlns')) {\n svgClone.setAttribute('xmlns', 'http://www.w3.org/2000/svg');\n }\n\n let width = parseInt(svgClone.getAttribute('width') || '0');\n let height = parseInt(svgClone.getAttribute('height') || '0');\n\n if (!width || !height) {\n const viewBox = svgClone.getAttribute('viewBox')?.split(' ') || [];\n if (viewBox.length === 4) {\n width = parseInt(viewBox[2]);\n height = parseInt(viewBox[3]);\n svgClone.setAttribute('width', width.toString());\n svgClone.setAttribute('height', height.toString());\n } else {\n width = 800;\n height = 600;\n svgClone.setAttribute('width', width.toString());\n svgClone.setAttribute('height', height.toString());\n }\n }\n\n const serializer = new XMLSerializer();\n const svgString = serializer.serializeToString(svgClone);\n const svgBase64 = btoa(unescape(encodeURIComponent(svgString)));\n const dataUrl = `data:image/svg+xml;base64,${svgBase64}`;\n\n const img = new Image();\n img.crossOrigin = 'anonymous';\n\n img.onload = function (): void {\n const canvas = document.createElement('canvas');\n canvas.width = width;\n canvas.height = height;\n\n const ctx = canvas.getContext('2d');\n if (!ctx) {\n console.error('Could not get canvas context');\n return;\n }\n\n ctx.fillRect(0, 0, canvas.width, canvas.height);\n\n ctx.drawImage(img, 0, 0, width, height);\n\n try {\n const pngUrl = canvas.toDataURL('image/png');\n const link = document.createElement('a');\n link.download = 'diagram.png';\n link.href = pngUrl;\n document.body.appendChild(link);\n link.click();\n document.body.removeChild(link);\n } catch (e) {\n console.error('Failed to create PNG:', e);\n // Fallback to SVG download if PNG fails\n downloadSvg();\n }\n };\n\n img.src = dataUrl;\n };\n\n const downloadMmd = (): void => {\n const blob = new Blob([code], { type: 'text/plain' });\n const url = URL.createObjectURL(blob);\n const link = document.createElement('a');\n link.href = url;\n link.download = 'diagram.mmd';\n document.body.appendChild(link);\n link.click();\n document.body.removeChild(link);\n URL.revokeObjectURL(url);\n };\n\n const downloadOptions = [\n { label: 'Download as SVG', action: downloadSvg },\n { label: 'Download as PNG', action: downloadPng },\n { label: 'Download as MMD', action: downloadMmd },\n ];\n\n const isCurrentlyLoading =\n isLoading !== undefined ? isLoading : status === 'loading';\n const showDiagramOptions = !isCurrentlyLoading && !error;\n const errorRender = !isCurrentlyLoading && error;\n\n return (\n
\n
\n \n mermaid\n \n
\n \n\n {showDiagramOptions && (\n
\n setShowDownloadMenu(!showDownloadMenu)}\n className=\"flex h-full items-center rounded-sm bg-gray-100 px-2 py-1 text-xs dark:bg-gray-700\"\n title={t('mermaid.downloadOptions')}\n >\n Download ▼\n \n {showDownloadMenu && (\n
\n
    \n {downloadOptions.map((option, index) => (\n
  • \n {\n option.action();\n setShowDownloadMenu(false);\n }}\n className=\"w-full px-4 py-2 text-left text-xs hover:bg-gray-100 dark:hover:bg-gray-700\"\n >\n {option.label}\n \n
    • \n ))}\n
\n
\n )}\n
\n )}\n\n {showDiagramOptions && (\n setShowCode(!showCode)}\n className={`flex h-full items-center rounded px-2 py-1 text-xs ${\n showCode\n ? 'bg-blue-200 dark:bg-blue-800'\n : 'bg-gray-100 dark:bg-gray-700'\n }`}\n title={t('mermaid.viewCode')}\n >\n Code\n \n )}\n
\n
\n\n {isCurrentlyLoading ? (\n
\n
\n Loading diagram...\n
\n
\n ) : errorRender ? (\n
\n
\n {error}\n
\n
\n ) : (\n <>\n \n {isHovering && (\n <>\n
\n \n setZoomFactor((prev) => Math.max(1, prev - 0.5))\n }\n className=\"rounded px-1 hover:bg-gray-600\"\n title={t('mermaid.decreaseZoom')}\n >\n -\n \n {\n setZoomFactor(2);\n }}\n title={t('mermaid.resetZoom')}\n >\n {zoomFactor.toFixed(1)}x\n \n \n setZoomFactor((prev) => Math.min(6, prev + 0.5))\n }\n className=\"rounded px-1 hover:bg-gray-600\"\n title={t('mermaid.increaseZoom')}\n >\n +\n \n
\n \n )}\n \n {code}\n \n
\n\n {showCode && (\n
\n
\n \n Mermaid Code\n \n
\n \n {code}\n \n
\n )}\n \n )}\n \n );\n};\n\nexport default MermaidRenderer;\n" }, { "path": "frontend/src/components/MessageInput.tsx", "content": "import { useCallback, useEffect, useRef, useState } from 'react';\nimport { createPortal } from 'react-dom';\nimport { LoaderCircle, Mic, Square } from 'lucide-react';\nimport { useDropzone } from 'react-dropzone';\nimport { useTranslation } from 'react-i18next';\nimport { useDispatch, useSelector } from 'react-redux';\n\nimport endpoints from '../api/endpoints';\nimport userService from '../api/services/userService';\nimport AlertIcon from '../assets/alert.svg';\nimport ClipIcon from '../assets/clip.svg';\nimport DragFileUpload from '../assets/DragFileUpload.svg';\nimport ExitIcon from '../assets/exit.svg';\nimport SendArrowIcon from './SendArrowIcon';\nimport SourceIcon from '../assets/source.svg';\nimport DocumentationDark from '../assets/documentation-dark.svg';\nimport ToolIcon from '../assets/tool.svg';\nimport {\n addAttachment,\n removeAttachment,\n selectAttachments,\n updateAttachment,\n reorderAttachments,\n} from '../upload/uploadSlice';\n\nimport { ActiveState, Doc } from '../models/misc';\nimport {\n selectSelectedDocs,\n selectToken,\n} from '../preferences/preferenceSlice';\nimport Upload from '../upload/Upload';\nimport { getOS, isTouchDevice } from '../utils/browserUtils';\nimport SourcesPopup from './SourcesPopup';\nimport ToolsPopup from './ToolsPopup';\nimport { handleAbort } from '../conversation/conversationSlice';\nimport {\n AUDIO_FILE_ACCEPT_ATTR,\n FILE_UPLOAD_ACCEPT,\n FILE_UPLOAD_ACCEPT_ATTR,\n} from '../constants/fileUpload';\n\nconst generateId = (): string =>\n `${Date.now()}-${Math.random().toString(36).substring(2)}`;\n\ntype RecordingState = 'idle' | 'recording' | 'transcribing' | 'error';\n\nconst LIVE_TRANSCRIPTION_TIMESLICE_MS = 1000;\nconst LIVE_CAPTURE_SAMPLE_RATE = 16000;\nconst LIVE_CAPTURE_MAX_BUFFER_SECONDS = 20;\nconst LIVE_SILENCE_RMS_THRESHOLD = 0.015;\nconst ENABLE_VOICE_INPUT = import.meta.env.VITE_ENABLE_VOICE_INPUT === 'true';\n\ntype AudioContextWindow = Window &\n typeof globalThis & {\n webkitAudioContext?: typeof AudioContext;\n };\n\ntype LegacyNavigator = Navigator & {\n getUserMedia?: (\n constraints: MediaStreamConstraints,\n successCallback: (stream: MediaStream) => void,\n errorCallback: (error: DOMException) => void,\n ) => void;\n webkitGetUserMedia?: (\n constraints: MediaStreamConstraints,\n successCallback: (stream: MediaStream) => void,\n errorCallback: (error: DOMException) => void,\n ) => void;\n mozGetUserMedia?: (\n constraints: MediaStreamConstraints,\n successCallback: (stream: MediaStream) => void,\n errorCallback: (error: DOMException) => void,\n ) => void;\n};\n\ntype LiveAudioSnapshot = {\n blob: Blob;\n chunkIndex: number;\n isSilence: boolean;\n};\n\nconst getAudioContextConstructor = (): typeof AudioContext | null => {\n if (typeof window === 'undefined') {\n return null;\n }\n\n const audioWindow = window as AudioContextWindow;\n return audioWindow.AudioContext || audioWindow.webkitAudioContext || null;\n};\n\nconst getLegacyGetUserMedia = () => {\n if (typeof navigator === 'undefined') {\n return null;\n }\n\n const legacyNavigator = navigator as LegacyNavigator;\n return (\n legacyNavigator.getUserMedia ||\n legacyNavigator.webkitGetUserMedia ||\n legacyNavigator.mozGetUserMedia ||\n null\n );\n};\n\nconst getVoiceInputSupportError = (): string | null => {\n if (typeof window === 'undefined' || typeof navigator === 'undefined') {\n return 'Voice input is unavailable right now.';\n }\n\n if (!window.isSecureContext) {\n return 'Voice input requires a secure connection (HTTPS or localhost).';\n }\n\n if (!navigator.mediaDevices?.getUserMedia && !getLegacyGetUserMedia()) {\n return 'Voice input is not available in this browser.';\n }\n\n if (!getAudioContextConstructor()) {\n return 'Voice input requires Web Audio support in this browser.';\n }\n\n return null;\n};\n\nconst getUserMediaStream = (\n constraints: MediaStreamConstraints,\n): Promise => {\n if (navigator.mediaDevices?.getUserMedia) {\n return navigator.mediaDevices.getUserMedia(constraints);\n }\n\n const legacyGetUserMedia = getLegacyGetUserMedia();\n if (!legacyGetUserMedia) {\n return Promise.reject(\n new Error('Voice input is not available in this browser.'),\n );\n }\n\n return new Promise((resolve, reject) => {\n legacyGetUserMedia.call(navigator, constraints, resolve, reject);\n });\n};\n\nconst getVoiceInputErrorMessage = (error: unknown): string => {\n if (typeof window !== 'undefined' && !window.isSecureContext) {\n return 'Voice input requires a secure connection (HTTPS or localhost).';\n }\n\n if (error instanceof DOMException) {\n switch (error.name) {\n case 'NotAllowedError':\n case 'PermissionDeniedError':\n case 'SecurityError':\n return 'Microphone access was blocked. Allow microphone permission and try again.';\n case 'NotFoundError':\n case 'DevicesNotFoundError':\n return 'No microphone was found on this device.';\n case 'NotReadableError':\n case 'TrackStartError':\n return 'The microphone is unavailable or already in use.';\n case 'AbortError':\n return 'Microphone access was interrupted before recording started.';\n default:\n break;\n }\n }\n\n if (error instanceof Error && error.message) {\n return error.message;\n }\n\n return 'Microphone access was denied.';\n};\n\nconst downsampleFloat32Buffer = (\n source: Float32Array,\n inputSampleRate: number,\n outputSampleRate: number,\n): Float32Array => {\n if (\n !source.length ||\n inputSampleRate <= 0 ||\n outputSampleRate <= 0 ||\n inputSampleRate === outputSampleRate\n ) {\n return source;\n }\n\n if (outputSampleRate > inputSampleRate) {\n return source;\n }\n\n const ratio = inputSampleRate / outputSampleRate;\n const outputLength = Math.max(1, Math.round(source.length / ratio));\n const output = new Float32Array(outputLength);\n\n let outputOffset = 0;\n let inputOffset = 0;\n while (outputOffset < output.length) {\n const nextInputOffset = Math.min(\n source.length,\n Math.round((outputOffset + 1) * ratio),\n );\n let accumulator = 0;\n let count = 0;\n for (let index = inputOffset; index < nextInputOffset; index += 1) {\n accumulator += source[index];\n count += 1;\n }\n output[outputOffset] =\n count > 0 ? accumulator / count : source[inputOffset];\n outputOffset += 1;\n inputOffset = nextInputOffset;\n }\n\n return output;\n};\n\nconst concatenateFloat32Chunks = (\n chunks: Float32Array[],\n totalLength: number,\n): Float32Array => {\n const output = new Float32Array(totalLength);\n let offset = 0;\n chunks.forEach((chunk) => {\n output.set(chunk, offset);\n offset += chunk.length;\n });\n return output;\n};\n\nconst encodeWavFromFloat32 = (\n samples: Float32Array,\n sampleRate: number,\n): Blob => {\n const bytesPerSample = 2;\n const blockAlign = bytesPerSample;\n const buffer = new ArrayBuffer(44 + samples.length * bytesPerSample);\n const view = new DataView(buffer);\n let offset = 0;\n\n const writeString = (value: string) => {\n for (let index = 0; index < value.length; index += 1) {\n view.setUint8(offset + index, value.charCodeAt(index));\n }\n offset += value.length;\n };\n\n writeString('RIFF');\n view.setUint32(offset, 36 + samples.length * bytesPerSample, true);\n offset += 4;\n writeString('WAVE');\n writeString('fmt ');\n view.setUint32(offset, 16, true);\n offset += 4;\n view.setUint16(offset, 1, true);\n offset += 2;\n view.setUint16(offset, 1, true);\n offset += 2;\n view.setUint32(offset, sampleRate, true);\n offset += 4;\n view.setUint32(offset, sampleRate * blockAlign, true);\n offset += 4;\n view.setUint16(offset, blockAlign, true);\n offset += 2;\n view.setUint16(offset, 16, true);\n offset += 2;\n writeString('data');\n view.setUint32(offset, samples.length * bytesPerSample, true);\n offset += 4;\n\n for (let index = 0; index < samples.length; index += 1) {\n const clamped = Math.max(-1, Math.min(1, samples[index]));\n view.setInt16(\n offset,\n clamped < 0 ? clamped * 0x8000 : clamped * 0x7fff,\n true,\n );\n offset += 2;\n }\n\n return new Blob([buffer], { type: 'audio/wav' });\n};\n\ntype MessageInputProps = {\n onSubmit: (text: string) => void;\n loading: boolean;\n showSourceButton?: boolean;\n showToolButton?: boolean;\n autoFocus?: boolean;\n};\n\nexport default function MessageInput({\n onSubmit,\n loading,\n showSourceButton = true,\n showToolButton = true,\n autoFocus = true,\n}: MessageInputProps) {\n const { t } = useTranslation();\n const [value, setValue] = useState('');\n const inputRef = useRef(null);\n const voiceFileInputRef = useRef(null);\n const sourceButtonRef = useRef(null);\n const toolButtonRef = useRef(null);\n const [isSourcesPopupOpen, setIsSourcesPopupOpen] = useState(false);\n const [isToolsPopupOpen, setIsToolsPopupOpen] = useState(false);\n const [uploadModalState, setUploadModalState] =\n useState('INACTIVE');\n const [handleDragActive, setHandleDragActive] = useState(false);\n const [recordingState, setRecordingState] = useState('idle');\n const [voiceError, setVoiceError] = useState(null);\n\n const selectedDocs = useSelector(selectSelectedDocs);\n const token = useSelector(selectToken);\n const attachments = useSelector(selectAttachments);\n\n const dispatch = useDispatch();\n const mediaStreamRef = useRef(null);\n const audioContextRef = useRef(null);\n const audioSourceNodeRef = useRef(null);\n const audioProcessorNodeRef = useRef(null);\n const audioSilenceGainRef = useRef(null);\n const snapshotIntervalRef = useRef(null);\n const pcmChunksRef = useRef([]);\n const totalBufferedSamplesRef = useRef(0);\n const totalCapturedSamplesRef = useRef(0);\n const lastSnapshotCapturedSamplesRef = useRef(0);\n const recentWindowRmsRef = useRef({ sumSquares: 0, sampleCount: 0 });\n const liveSessionIdRef = useRef(null);\n const livePendingSnapshotRef = useRef(null);\n const liveChunkIndexRef = useRef(0);\n const liveUploadInFlightRef = useRef(false);\n const liveStopRequestedRef = useRef(false);\n const voiceBaseValueRef = useRef('');\n const liveTranscriptRef = useRef('');\n\n const browserOS = getOS();\n const isTouch = isTouchDevice();\n\n const stopMediaStream = () => {\n mediaStreamRef.current?.getTracks().forEach((track) => track.stop());\n mediaStreamRef.current = null;\n };\n\n const stopAudioProcessing = () => {\n if (snapshotIntervalRef.current !== null) {\n window.clearInterval(snapshotIntervalRef.current);\n snapshotIntervalRef.current = null;\n }\n\n if (audioProcessorNodeRef.current) {\n audioProcessorNodeRef.current.onaudioprocess = null;\n audioProcessorNodeRef.current.disconnect();\n audioProcessorNodeRef.current = null;\n }\n if (audioSourceNodeRef.current) {\n audioSourceNodeRef.current.disconnect();\n audioSourceNodeRef.current = null;\n }\n if (audioSilenceGainRef.current) {\n audioSilenceGainRef.current.disconnect();\n audioSilenceGainRef.current = null;\n }\n if (audioContextRef.current) {\n void audioContextRef.current.close().catch(() => undefined);\n audioContextRef.current = null;\n }\n stopMediaStream();\n };\n\n const resetLiveTranscriptionState = () => {\n pcmChunksRef.current = [];\n totalBufferedSamplesRef.current = 0;\n totalCapturedSamplesRef.current = 0;\n lastSnapshotCapturedSamplesRef.current = 0;\n recentWindowRmsRef.current = { sumSquares: 0, sampleCount: 0 };\n liveSessionIdRef.current = null;\n livePendingSnapshotRef.current = null;\n liveChunkIndexRef.current = 0;\n liveUploadInFlightRef.current = false;\n liveStopRequestedRef.current = false;\n voiceBaseValueRef.current = '';\n liveTranscriptRef.current = '';\n };\n\n useEffect(() => {\n const handleKeyDown = (event: KeyboardEvent) => {\n if (\n ((browserOS === 'win' || browserOS === 'linux') &&\n event.ctrlKey &&\n event.key === 'k') ||\n (browserOS === 'mac' && event.metaKey && event.key === 'k')\n ) {\n event.preventDefault();\n setIsSourcesPopupOpen((s) => !s);\n }\n };\n\n document.addEventListener('keydown', handleKeyDown);\n return () => {\n document.removeEventListener('keydown', handleKeyDown);\n };\n }, [browserOS]);\n\n useEffect(() => {\n return () => {\n stopAudioProcessing();\n resetLiveTranscriptionState();\n };\n }, []);\n\n const uploadFiles = useCallback(\n (files: File[]) => {\n if (!files || files.length === 0) return;\n\n const apiHost = import.meta.env.VITE_API_HOST;\n\n if (files.length > 1) {\n const formData = new FormData();\n const indexToUiId: Record = {};\n\n files.forEach((file, i) => {\n formData.append('file', file);\n const uiId = generateId();\n indexToUiId[i] = uiId;\n dispatch(\n addAttachment({\n id: uiId,\n fileName: file.name,\n progress: 0,\n status: 'uploading' as const,\n taskId: '',\n }),\n );\n });\n\n const xhr = new XMLHttpRequest();\n\n xhr.upload.addEventListener('progress', (event) => {\n if (event.lengthComputable) {\n const progress = Math.round((event.loaded / event.total) * 100);\n Object.values(indexToUiId).forEach((uiId) =>\n dispatch(\n updateAttachment({\n id: uiId,\n updates: { progress },\n }),\n ),\n );\n }\n });\n\n xhr.onload = () => {\n const status = xhr.status;\n if (status === 200) {\n try {\n const response = JSON.parse(xhr.responseText);\n\n if (Array.isArray(response?.tasks)) {\n const tasks = response.tasks as Array<{\n task_id?: string;\n filename?: string;\n attachment_id?: string;\n path?: string;\n upload_index?: number;\n }>;\n const errors = Array.isArray(response?.errors)\n ? (response.errors as Array<{\n filename?: string;\n error?: string;\n upload_index?: number;\n }>)\n : [];\n const hasIndexedResults =\n tasks.some((task) => typeof task.upload_index === 'number') ||\n errors.some(\n (errorItem) => typeof errorItem.upload_index === 'number',\n );\n\n if (hasIndexedResults) {\n const tasksByIndex = new Map<\n number,\n (typeof tasks)[number]\n >();\n const failedIndices = new Set();\n\n tasks.forEach((task, taskOrderIndex) => {\n const uploadIndex =\n typeof task.upload_index === 'number'\n ? task.upload_index\n : taskOrderIndex;\n tasksByIndex.set(uploadIndex, task);\n });\n\n errors.forEach((errorItem) => {\n if (typeof errorItem.upload_index === 'number') {\n failedIndices.add(errorItem.upload_index);\n }\n });\n\n files.forEach((_, index) => {\n const uiId = indexToUiId[index];\n if (!uiId) return;\n\n const task = tasksByIndex.get(index);\n if (task?.task_id) {\n dispatch(\n updateAttachment({\n id: uiId,\n updates: {\n taskId: task.task_id,\n status: 'processing',\n progress: 10,\n },\n }),\n );\n return;\n }\n\n if (failedIndices.has(index)) {\n dispatch(\n updateAttachment({\n id: uiId,\n updates: { status: 'failed' },\n }),\n );\n return;\n }\n\n dispatch(\n updateAttachment({\n id: uiId,\n updates: { status: 'failed' },\n }),\n );\n });\n } else {\n tasks.forEach((t, idx) => {\n const uiId = indexToUiId[idx];\n if (!uiId) return;\n if (t?.task_id) {\n dispatch(\n updateAttachment({\n id: uiId,\n updates: {\n taskId: t.task_id,\n status: 'processing',\n progress: 10,\n },\n }),\n );\n } else {\n dispatch(\n updateAttachment({\n id: uiId,\n updates: { status: 'failed' },\n }),\n );\n }\n });\n\n if (tasks.length < files.length) {\n for (let i = tasks.length; i < files.length; i++) {\n const uiId = indexToUiId[i];\n if (uiId) {\n dispatch(\n updateAttachment({\n id: uiId,\n updates: { status: 'failed' },\n }),\n );\n }\n }\n }\n }\n } else if (response?.task_id) {\n if (files.length === 1) {\n const uiId = indexToUiId[0];\n if (uiId) {\n dispatch(\n updateAttachment({\n id: uiId,\n updates: {\n taskId: response.task_id,\n status: 'processing',\n progress: 10,\n },\n }),\n );\n }\n } else {\n console.warn(\n 'Server returned a single task_id for multiple files. Update backend to return tasks[].',\n );\n const firstUi = indexToUiId[0];\n if (firstUi) {\n dispatch(\n updateAttachment({\n id: firstUi,\n updates: {\n taskId: response.task_id,\n status: 'processing',\n progress: 10,\n },\n }),\n );\n }\n for (let i = 1; i < files.length; i++) {\n const uiId = indexToUiId[i];\n if (uiId) {\n dispatch(\n updateAttachment({\n id: uiId,\n updates: { status: 'failed' },\n }),\n );\n }\n }\n }\n } else {\n console.error('Unexpected upload response shape', response);\n Object.values(indexToUiId).forEach((id) =>\n dispatch(\n updateAttachment({\n id,\n updates: { status: 'failed' },\n }),\n ),\n );\n }\n } catch (err) {\n console.error(\n 'Failed to parse upload response',\n err,\n xhr.responseText,\n );\n Object.values(indexToUiId).forEach((id) =>\n dispatch(\n updateAttachment({\n id,\n updates: { status: 'failed' },\n }),\n ),\n );\n }\n } else {\n console.error('Upload failed', status, xhr.responseText);\n Object.values(indexToUiId).forEach((id) =>\n dispatch(\n updateAttachment({\n id,\n updates: { status: 'failed' },\n }),\n ),\n );\n }\n };\n\n xhr.onerror = () => {\n console.error('Upload network error');\n Object.values(indexToUiId).forEach((id) =>\n dispatch(\n updateAttachment({\n id,\n updates: { status: 'failed' },\n }),\n ),\n );\n };\n\n xhr.open('POST', `${apiHost}${endpoints.USER.STORE_ATTACHMENT}`);\n if (token) xhr.setRequestHeader('Authorization', `Bearer ${token}`);\n xhr.send(formData);\n return;\n }\n\n // Single-file path: upload each file individually (original repo behavior)\n files.forEach((file) => {\n const formData = new FormData();\n formData.append('file', file);\n const xhr = new XMLHttpRequest();\n const uniqueId = generateId();\n\n const newAttachment = {\n id: uniqueId,\n fileName: file.name,\n progress: 0,\n status: 'uploading' as const,\n taskId: '',\n };\n\n dispatch(addAttachment(newAttachment));\n\n xhr.upload.addEventListener('progress', (event) => {\n if (event.lengthComputable) {\n const progress = Math.round((event.loaded / event.total) * 100);\n dispatch(\n updateAttachment({\n id: uniqueId,\n updates: { progress },\n }),\n );\n }\n });\n\n xhr.onload = () => {\n if (xhr.status === 200) {\n try {\n const response = JSON.parse(xhr.responseText);\n if (response.task_id) {\n dispatch(\n updateAttachment({\n id: uniqueId,\n updates: {\n taskId: response.task_id,\n status: 'processing',\n progress: 10,\n },\n }),\n );\n } else {\n // If backend returned tasks[] for single-file, handle gracefully:\n if (\n Array.isArray(response?.tasks) &&\n response.tasks[0]?.task_id\n ) {\n dispatch(\n updateAttachment({\n id: uniqueId,\n updates: {\n taskId: response.tasks[0].task_id,\n status: 'processing',\n progress: 10,\n },\n }),\n );\n } else {\n dispatch(\n updateAttachment({\n id: uniqueId,\n updates: { status: 'failed' },\n }),\n );\n }\n }\n } catch (err) {\n console.error(\n 'Failed to parse upload response',\n err,\n xhr.responseText,\n );\n dispatch(\n updateAttachment({\n id: uniqueId,\n updates: { status: 'failed' },\n }),\n );\n }\n } else {\n dispatch(\n updateAttachment({\n id: uniqueId,\n updates: { status: 'failed' },\n }),\n );\n }\n };\n\n xhr.onerror = () => {\n dispatch(\n updateAttachment({\n id: uniqueId,\n updates: { status: 'failed' },\n }),\n );\n };\n\n xhr.open('POST', `${apiHost}${endpoints.USER.STORE_ATTACHMENT}`);\n if (token) xhr.setRequestHeader('Authorization', `Bearer ${token}`);\n xhr.send(formData);\n });\n },\n [dispatch, token],\n );\n\n const handleFileAttachment = (e: React.ChangeEvent) => {\n if (!e.target.files || e.target.files.length === 0) return;\n const files = Array.from(e.target.files);\n uploadFiles(files);\n // clear input so same file can be selected again\n e.target.value = '';\n };\n\n // Drag & drop via react-dropzone\n const onDrop = useCallback(\n (acceptedFiles: File[]) => {\n uploadFiles(acceptedFiles);\n setHandleDragActive(false);\n },\n [uploadFiles],\n );\n\n const { getRootProps, getInputProps } = useDropzone({\n onDrop,\n noClick: true,\n noKeyboard: true,\n multiple: true,\n onDragEnter: () => {\n setHandleDragActive(true);\n },\n onDragLeave: () => {\n setHandleDragActive(false);\n },\n maxSize: 25000000,\n accept: FILE_UPLOAD_ACCEPT,\n });\n\n useEffect(() => {\n const checkTaskStatus = () => {\n const processingAttachments = attachments.filter(\n (att) => att.status === 'processing' && att.taskId,\n );\n\n processingAttachments.forEach((attachment) => {\n userService\n .getTaskStatus(attachment.taskId!, null)\n .then((data) => data.json())\n .then((data) => {\n if (data.status === 'SUCCESS') {\n dispatch(\n updateAttachment({\n id: attachment.id,\n updates: {\n status: 'completed',\n progress: 100,\n id: data.result?.attachment_id,\n token_count: data.result?.token_count,\n },\n }),\n );\n } else if (data.status === 'FAILURE') {\n dispatch(\n updateAttachment({\n id: attachment.id,\n updates: { status: 'failed' },\n }),\n );\n } else if (data.status === 'PROGRESS' && data.result?.current) {\n dispatch(\n updateAttachment({\n id: attachment.id,\n updates: { progress: data.result.current },\n }),\n );\n }\n })\n .catch(() => {\n dispatch(\n updateAttachment({\n id: attachment.id,\n updates: { status: 'failed' },\n }),\n );\n });\n });\n };\n\n const interval = setInterval(() => {\n if (attachments.some((att) => att.status === 'processing')) {\n checkTaskStatus();\n }\n }, 2000);\n\n return () => clearInterval(interval);\n }, [attachments, dispatch]);\n\n const handleInput = useCallback(() => {\n if (inputRef.current) {\n if (window.innerWidth < 350) inputRef.current.style.height = 'auto';\n else inputRef.current.style.height = '64px';\n inputRef.current.style.height = `${Math.min(\n inputRef.current.scrollHeight,\n 96,\n )}px`;\n }\n }, []);\n\n const buildVoiceDraftValue = (baseText: string, transcript: string) => {\n const normalizedBaseText = baseText ?? '';\n const normalizedTranscript = transcript.trim();\n\n if (!normalizedTranscript) {\n return normalizedBaseText;\n }\n\n return normalizedBaseText.trim()\n ? `${normalizedBaseText}${\n normalizedBaseText.endsWith('\\n') ? '' : '\\n'\n }${normalizedTranscript}`\n : normalizedTranscript;\n };\n\n const applyLiveTranscript = (transcript: string) => {\n const normalizedTranscript = transcript.trim();\n liveTranscriptRef.current = normalizedTranscript;\n setValue(\n buildVoiceDraftValue(voiceBaseValueRef.current, normalizedTranscript),\n );\n setTimeout(() => {\n handleInput();\n }, 0);\n };\n\n const promptVoiceFileFallback = (message: string) => {\n setRecordingState('idle');\n setVoiceError(`${message} Choose or record an audio file instead.`);\n setTimeout(() => {\n voiceFileInputRef.current?.click();\n }, 0);\n };\n\n const transcribeUploadedAudioFile = async (file: File) => {\n try {\n setVoiceError(null);\n setRecordingState('transcribing');\n voiceBaseValueRef.current = value;\n liveTranscriptRef.current = '';\n\n const response = await userService.transcribeAudio(file, token);\n const data = await response.json();\n\n if (!response.ok || !data?.success) {\n throw new Error(data?.message || 'Failed to transcribe audio.');\n }\n\n if (typeof data.text !== 'string' || !data.text.trim()) {\n throw new Error('No transcript was returned for this audio file.');\n }\n\n applyLiveTranscript(data.text);\n setRecordingState('idle');\n if (autoFocus) {\n setTimeout(() => {\n inputRef.current?.focus();\n }, 0);\n }\n } catch (error) {\n console.error('Uploaded audio transcription failed', error);\n setRecordingState('error');\n setVoiceError(\n error instanceof Error ? error.message : 'Failed to transcribe audio.',\n );\n }\n };\n\n const trimLivePcmBuffer = () => {\n const maxBufferedSamples =\n LIVE_CAPTURE_SAMPLE_RATE * LIVE_CAPTURE_MAX_BUFFER_SECONDS;\n\n while (\n totalBufferedSamplesRef.current > maxBufferedSamples &&\n pcmChunksRef.current.length > 1\n ) {\n const removedChunk = pcmChunksRef.current.shift();\n if (!removedChunk) {\n break;\n }\n totalBufferedSamplesRef.current -= removedChunk.length;\n }\n\n if (\n totalBufferedSamplesRef.current > maxBufferedSamples &&\n pcmChunksRef.current.length === 1\n ) {\n const onlyChunk = pcmChunksRef.current[0];\n if (!onlyChunk || onlyChunk.length <= maxBufferedSamples) {\n return;\n }\n\n const trimmedChunk = onlyChunk.slice(\n onlyChunk.length - maxBufferedSamples,\n );\n pcmChunksRef.current = [trimmedChunk];\n totalBufferedSamplesRef.current = trimmedChunk.length;\n }\n };\n\n const cleanupLiveSession = async () => {\n const sessionId = liveSessionIdRef.current;\n if (!sessionId) {\n return;\n }\n\n liveSessionIdRef.current = null;\n try {\n await userService.finishLiveTranscription(sessionId, token);\n } catch {\n // Best-effort cleanup only.\n }\n };\n\n const failLiveTranscription = async (message: string) => {\n console.error('Live audio transcription failed', message);\n stopAudioProcessing();\n await cleanupLiveSession();\n resetLiveTranscriptionState();\n setRecordingState('error');\n setVoiceError(message);\n };\n\n const finalizeLiveTranscription = async () => {\n const sessionId = liveSessionIdRef.current;\n if (!sessionId) {\n resetLiveTranscriptionState();\n setRecordingState('idle');\n return;\n }\n\n try {\n const response = await userService.finishLiveTranscription(\n sessionId,\n token,\n );\n const data = await response.json();\n\n if (!response.ok || !data?.success) {\n throw new Error(\n data?.message || 'Failed to finalize live transcription.',\n );\n }\n\n if (typeof data.text === 'string') {\n applyLiveTranscript(data.text);\n }\n\n setRecordingState('idle');\n if (autoFocus) {\n setTimeout(() => {\n inputRef.current?.focus();\n }, 0);\n }\n } catch (error) {\n console.error('Finalizing live audio transcription failed', error);\n setRecordingState('error');\n setVoiceError(\n error instanceof Error\n ? error.message\n : 'Failed to finalize live transcription.',\n );\n } finally {\n resetLiveTranscriptionState();\n }\n };\n\n const maybeFinalizeLiveTranscription = async () => {\n if (\n !liveStopRequestedRef.current ||\n liveUploadInFlightRef.current ||\n livePendingSnapshotRef.current\n ) {\n return;\n }\n\n await finalizeLiveTranscription();\n };\n\n const processPendingLiveSnapshot = async () => {\n if (liveUploadInFlightRef.current) {\n return;\n }\n\n const nextSnapshot = livePendingSnapshotRef.current;\n const sessionId = liveSessionIdRef.current;\n if (!nextSnapshot || !sessionId) {\n await maybeFinalizeLiveTranscription();\n return;\n }\n\n livePendingSnapshotRef.current = null;\n liveUploadInFlightRef.current = true;\n\n try {\n const file = new File(\n [nextSnapshot.blob],\n `voice-live-${nextSnapshot.chunkIndex}.wav`,\n {\n type: 'audio/wav',\n },\n );\n const response = await userService.transcribeLiveAudioChunk(\n sessionId,\n nextSnapshot.chunkIndex,\n file,\n token,\n nextSnapshot.isSilence,\n );\n const data = await response.json();\n\n if (!response.ok || !data?.success) {\n throw new Error(data?.message || 'Failed to transcribe audio.');\n }\n\n if (typeof data.transcript_text === 'string') {\n applyLiveTranscript(data.transcript_text);\n }\n } catch (error) {\n await failLiveTranscription(\n error instanceof Error ? error.message : 'Failed to transcribe audio.',\n );\n return;\n } finally {\n liveUploadInFlightRef.current = false;\n }\n\n if (livePendingSnapshotRef.current) {\n void processPendingLiveSnapshot();\n return;\n }\n\n void maybeFinalizeLiveTranscription();\n };\n\n const queueCurrentLiveSnapshot = (forceSilence = false) => {\n if (\n totalCapturedSamplesRef.current === lastSnapshotCapturedSamplesRef.current\n ) {\n return;\n }\n\n if (!pcmChunksRef.current.length || totalBufferedSamplesRef.current <= 0) {\n return;\n }\n\n const pcmSnapshot = concatenateFloat32Chunks(\n pcmChunksRef.current,\n totalBufferedSamplesRef.current,\n );\n if (!pcmSnapshot.length) {\n return;\n }\n\n const { sumSquares, sampleCount } = recentWindowRmsRef.current;\n const averageRms =\n sampleCount > 0 ? Math.sqrt(sumSquares / sampleCount) : 0;\n const isSilence = forceSilence || averageRms < LIVE_SILENCE_RMS_THRESHOLD;\n\n recentWindowRmsRef.current = { sumSquares: 0, sampleCount: 0 };\n lastSnapshotCapturedSamplesRef.current = totalCapturedSamplesRef.current;\n livePendingSnapshotRef.current = {\n blob: encodeWavFromFloat32(pcmSnapshot, LIVE_CAPTURE_SAMPLE_RATE),\n chunkIndex: liveChunkIndexRef.current,\n isSilence,\n };\n liveChunkIndexRef.current += 1;\n void processPendingLiveSnapshot();\n };\n\n const handleVoiceInput = async () => {\n if (recordingState === 'transcribing') {\n return;\n }\n\n if (recordingState === 'recording') {\n setRecordingState('transcribing');\n liveStopRequestedRef.current = true;\n stopAudioProcessing();\n queueCurrentLiveSnapshot();\n void maybeFinalizeLiveTranscription();\n return;\n }\n\n const voiceInputSupportError = getVoiceInputSupportError();\n if (voiceInputSupportError) {\n promptVoiceFileFallback(voiceInputSupportError);\n return;\n }\n\n const AudioContextConstructor = getAudioContextConstructor();\n if (!AudioContextConstructor) {\n setRecordingState('error');\n setVoiceError('Voice input requires Web Audio support in this browser.');\n return;\n }\n\n let stream: MediaStream | null = null;\n try {\n setVoiceError(null);\n stream = await getUserMediaStream({ audio: true });\n } catch (error) {\n promptVoiceFileFallback(getVoiceInputErrorMessage(error));\n return;\n }\n\n try {\n const liveStartResponse = await userService.startLiveTranscription(token);\n const liveStartData = await liveStartResponse.json();\n if (!liveStartResponse.ok || !liveStartData?.success) {\n throw new Error(\n liveStartData?.message || 'Failed to start live transcription.',\n );\n }\n\n const audioContext = new AudioContextConstructor();\n await audioContext.resume().catch(() => undefined);\n const sourceNode = audioContext.createMediaStreamSource(stream);\n const processorNode = audioContext.createScriptProcessor(4096, 1, 1);\n const silenceGain = audioContext.createGain();\n silenceGain.gain.value = 0;\n\n pcmChunksRef.current = [];\n totalBufferedSamplesRef.current = 0;\n totalCapturedSamplesRef.current = 0;\n lastSnapshotCapturedSamplesRef.current = 0;\n recentWindowRmsRef.current = { sumSquares: 0, sampleCount: 0 };\n liveSessionIdRef.current = liveStartData.session_id;\n livePendingSnapshotRef.current = null;\n liveChunkIndexRef.current = 0;\n liveUploadInFlightRef.current = false;\n liveStopRequestedRef.current = false;\n voiceBaseValueRef.current = value;\n liveTranscriptRef.current = '';\n applyLiveTranscript('');\n\n processorNode.onaudioprocess = (event: AudioProcessingEvent) => {\n const inputData = event.inputBuffer.getChannelData(0);\n if (!inputData.length) {\n return;\n }\n\n const capturedChunk = new Float32Array(inputData.length);\n capturedChunk.set(inputData);\n\n const downsampledChunk = downsampleFloat32Buffer(\n capturedChunk,\n audioContext.sampleRate,\n LIVE_CAPTURE_SAMPLE_RATE,\n );\n if (!downsampledChunk.length) {\n return;\n }\n\n pcmChunksRef.current.push(downsampledChunk);\n totalBufferedSamplesRef.current += downsampledChunk.length;\n totalCapturedSamplesRef.current += downsampledChunk.length;\n\n let sumSquares = 0;\n for (let index = 0; index < downsampledChunk.length; index += 1) {\n const sample = downsampledChunk[index];\n sumSquares += sample * sample;\n }\n\n recentWindowRmsRef.current.sumSquares += sumSquares;\n recentWindowRmsRef.current.sampleCount += downsampledChunk.length;\n trimLivePcmBuffer();\n };\n\n sourceNode.connect(processorNode);\n processorNode.connect(silenceGain);\n silenceGain.connect(audioContext.destination);\n\n mediaStreamRef.current = stream;\n audioContextRef.current = audioContext;\n audioSourceNodeRef.current = sourceNode;\n audioProcessorNodeRef.current = processorNode;\n audioSilenceGainRef.current = silenceGain;\n snapshotIntervalRef.current = window.setInterval(() => {\n if (!liveStopRequestedRef.current) {\n queueCurrentLiveSnapshot();\n }\n }, LIVE_TRANSCRIPTION_TIMESLICE_MS);\n\n setRecordingState('recording');\n } catch (error) {\n console.error('Live voice transcription failed', error);\n stream?.getTracks().forEach((track) => track.stop());\n stopAudioProcessing();\n await cleanupLiveSession();\n resetLiveTranscriptionState();\n setRecordingState('error');\n setVoiceError(\n error instanceof Error\n ? error.message\n : 'Failed to start live transcription.',\n );\n }\n };\n\n const isMountedRef = useRef(true);\n\n useEffect(() => {\n isMountedRef.current = true;\n return () => {\n isMountedRef.current = false;\n };\n }, []);\n\n useEffect(() => {\n if (autoFocus) inputRef.current?.focus();\n handleInput();\n }, [autoFocus, handleInput]);\n\n const handleChange = (e: React.ChangeEvent) => {\n setValue(e.target.value);\n handleInput();\n };\n\n const handleKeyDown = (e: React.KeyboardEvent) => {\n if (e.key === 'Enter' && !e.shiftKey) {\n e.preventDefault();\n handleSubmit();\n handleInput();\n }\n };\n\n const handlePaste = (e: React.ClipboardEvent) => {\n const clipboardItems = e.clipboardData?.items;\n const files: File[] = [];\n\n if (!clipboardItems) return;\n\n for (let i = 0; i < clipboardItems.length; i++) {\n const item = clipboardItems[i];\n\n if (item.kind === 'file') {\n const file = item.getAsFile();\n if (file) {\n files.push(file);\n }\n }\n }\n\n if (files.length > 0) {\n // Prevent weird binary stuff from being pasted as text\n e.preventDefault();\n uploadFiles(files);\n }\n };\n\n const handleVoiceFileAttachment = (\n e: React.ChangeEvent,\n ) => {\n const file = e.target.files?.[0];\n e.target.value = '';\n\n if (!file) {\n return;\n }\n\n void transcribeUploadedAudioFile(file);\n };\n\n const handlePostDocumentSelect = (_docs: Doc[] | null) => {\n // SourcesPopup updates Redux selection directly; this preserves the prop contract.\n void _docs;\n };\n\n const handleSubmit = () => {\n if (\n value.trim() &&\n !loading &&\n recordingState !== 'recording' &&\n recordingState !== 'transcribing'\n ) {\n onSubmit(value);\n setValue('');\n // Refocus input after submission if autoFocus is enabled\n if (autoFocus) {\n setTimeout(() => {\n if (isMountedRef.current) {\n inputRef.current?.focus();\n }\n }, 0);\n }\n }\n };\n\n const handleCancel = () => {\n handleAbort();\n };\n\n const [draggingId, setDraggingId] = useState(null);\n\n const findIndexById = (id: string) =>\n attachments.findIndex((a) => a.id === id);\n\n const handleDragStart = (e: React.DragEvent, id: string) => {\n setDraggingId(id);\n try {\n e.dataTransfer.setData('text/plain', id);\n e.dataTransfer.effectAllowed = 'move';\n } catch {\n // ignore\n }\n };\n\n const handleDragOver = (e: React.DragEvent) => {\n e.preventDefault();\n e.dataTransfer.dropEffect = 'move';\n };\n\n const handleDropOn = (e: React.DragEvent, targetId: string) => {\n e.preventDefault();\n const sourceId = e.dataTransfer.getData('text/plain');\n if (!sourceId || sourceId === targetId) return;\n\n const sourceIndex = findIndexById(sourceId);\n const destIndex = findIndexById(targetId);\n if (sourceIndex === -1 || destIndex === -1) return;\n\n dispatch(reorderAttachments({ sourceIndex, destinationIndex: destIndex }));\n setDraggingId(null);\n };\n\n const voiceButtonLabel =\n recordingState === 'recording'\n ? 'Stop recording'\n : recordingState === 'transcribing'\n ? 'Transcribing audio'\n : 'Voice input';\n const voiceButtonText =\n recordingState === 'recording'\n ? 'Stop'\n : recordingState === 'transcribing'\n ? 'Transcribing'\n : 'Voice';\n\n return (\n
\n {/* react-dropzone input (for drag/drop) */}\n \n \n\n
\n
\n {attachments.map((attachment) => {\n return (\n handleDragStart(e, attachment.id)}\n onDragOver={handleDragOver}\n onDrop={(e) => handleDropOn(e, attachment.id)}\n className={`group dark:text-bright-gray relative flex items-center rounded-xl bg-[#EFF3F4] px-2 py-1 text-[12px] text-[#5D5D5D] sm:px-3 sm:py-1.5 sm:text-[14px] dark:bg-[#393B3D] ${\n attachment.status !== 'completed'\n ? 'opacity-70'\n : 'opacity-100'\n } ${\n draggingId === attachment.id\n ? 'ring-dashed opacity-60 ring-2 ring-purple-200'\n : ''\n }`}\n title={attachment.fileName}\n >\n
\n {attachment.status === 'completed' && (\n \n )}\n\n {attachment.status === 'failed' && (\n \n )}\n\n {(attachment.status === 'uploading' ||\n attachment.status === 'processing') && (\n
\n \n \n \n \n
\n )}\n
\n\n \n {attachment.fileName}\n \n\n {\n dispatch(removeAttachment(attachment.id));\n }}\n aria-label={t('conversation.attachments.remove')}\n >\n \n \n
\n );\n })}\n
\n\n {voiceError && (\n
\n {voiceError}\n
\n )}\n\n
\n \n \n
\n\n
\n
\n {showSourceButton && (\n setIsSourcesPopupOpen(!isSourcesPopupOpen)}\n title={\n selectedDocs && selectedDocs.length > 0\n ? selectedDocs.map((doc) => doc.name).join(', ')\n : t('conversation.sources.title')\n }\n >\n \n \n {selectedDocs && selectedDocs.length > 0\n ? selectedDocs.length === 1\n ? selectedDocs[0].name\n : `${selectedDocs.length} sources selected`\n : t('conversation.sources.title')}\n \n {!isTouch && (\n \n )}\n \n )}\n\n {showToolButton && (\n setIsToolsPopupOpen(!isToolsPopupOpen)}\n >\n \n \n {t('settings.tools.label')}\n \n \n )}\n {ENABLE_VOICE_INPUT && (\n {\n void handleVoiceInput();\n }}\n aria-label={voiceButtonLabel}\n title={voiceButtonLabel}\n disabled={loading || recordingState === 'transcribing'}\n className={`xs:px-3 xs:py-1.5 dark:border-purple-taupe flex items-center rounded-[32px] border px-2 py-1 transition-colors ${\n recordingState === 'recording'\n ? 'border-[#B42318] bg-[#FEE4E2] text-[#B42318] dark:bg-[#4A2323]'\n : 'border-[#AAAAAA] hover:bg-gray-100 dark:hover:bg-[#2C2E3C]'\n } ${\n loading || recordingState === 'transcribing'\n ? 'cursor-not-allowed opacity-60'\n : ''\n }`}\n >\n {recordingState === 'transcribing' ? (\n \n ) : recordingState === 'recording' ? (\n \n ) : (\n \n )}\n \n {voiceButtonText}\n \n \n )}\n \n {/* Additional badges can be added here in the future */}\n
\n\n {loading ? (\n \n
\n \n ) : (\n \n \n \n )}\n
\n
\n\n setIsSourcesPopupOpen(false)}\n anchorRef={sourceButtonRef}\n handlePostDocumentSelect={handlePostDocumentSelect}\n setUploadModalState={setUploadModalState}\n />\n\n setIsToolsPopupOpen(false)}\n anchorRef={toolButtonRef}\n />\n\n {uploadModalState === 'ACTIVE' && (\n setUploadModalState('INACTIVE')}\n />\n )}\n\n {handleDragActive &&\n createPortal(\n
\n \n \n {t('modals.uploadDoc.drag.title')}\n \n \n {t('modals.uploadDoc.drag.description')}\n \n
,\n document.body,\n )}\n
\n );\n}\n" }, { "path": "frontend/src/components/MultiSelectPopup.tsx", "content": "import React, { useEffect, useLayoutEffect, useRef, useState } from 'react';\nimport { useTranslation } from 'react-i18next';\n\nimport CheckmarkIcon from '../assets/checkmark.svg';\nimport NoFilesDarkIcon from '../assets/no-files-dark.svg';\nimport NoFilesIcon from '../assets/no-files.svg';\nimport { useDarkTheme } from '../hooks';\nimport Input from './Input';\n\nexport type OptionType = {\n id: string | number;\n label: string;\n icon?: string | React.ReactNode;\n [key: string]: any;\n};\n\ntype MultiSelectPopupProps = {\n isOpen: boolean;\n onClose: () => void;\n anchorRef: React.RefObject;\n options: OptionType[];\n selectedIds: Set;\n onSelectionChange: (newSelectedIds: Set) => void;\n title?: string;\n searchPlaceholder?: string;\n noOptionsMessage?: string;\n loading?: boolean;\n footerContent?: React.ReactNode;\n showSearch?: boolean;\n singleSelect?: boolean;\n};\n\nexport default function MultiSelectPopup({\n isOpen,\n onClose,\n anchorRef,\n options,\n selectedIds,\n onSelectionChange,\n title,\n searchPlaceholder,\n noOptionsMessage,\n loading = false,\n footerContent,\n showSearch = true,\n singleSelect = false,\n}: MultiSelectPopupProps) {\n const { t } = useTranslation();\n const [isDarkTheme] = useDarkTheme();\n\n const popupRef = useRef(null);\n\n const [searchTerm, setSearchTerm] = useState('');\n const [popupPosition, setPopupPosition] = useState({\n top: 0,\n left: 0,\n maxHeight: 0,\n showAbove: false,\n });\n\n const filteredOptions = options.filter((option) =>\n option.label.toLowerCase().includes(searchTerm.toLowerCase()),\n );\n\n const handleOptionClick = (optionId: string | number) => {\n let newSelectedIds: Set;\n if (singleSelect) newSelectedIds = new Set();\n else newSelectedIds = new Set(selectedIds);\n if (newSelectedIds.has(optionId)) {\n newSelectedIds.delete(optionId);\n } else newSelectedIds.add(optionId);\n onSelectionChange(newSelectedIds);\n };\n\n const renderIcon = (icon: string | React.ReactNode) => {\n if (typeof icon === 'string') {\n if (icon.startsWith('/') || icon.startsWith('http')) {\n return (\n \n );\n }\n return (\n \n {icon}\n \n );\n }\n return {icon};\n };\n\n useLayoutEffect(() => {\n if (!isOpen || !anchorRef.current) return;\n\n const updatePosition = () => {\n if (!anchorRef.current) return;\n\n const rect = anchorRef.current.getBoundingClientRect();\n const viewportHeight = window.innerHeight;\n const viewportWidth = window.innerWidth;\n const popupPadding = 16;\n const popupMinWidth = 300;\n const popupMaxWidth = 462;\n const popupDefaultHeight = 300;\n\n const spaceAbove = rect.top;\n const spaceBelow = viewportHeight - rect.bottom;\n const showAbove =\n spaceBelow < popupDefaultHeight && spaceAbove >= popupDefaultHeight;\n\n const maxHeight = Math.max(\n 150,\n showAbove ? spaceAbove - popupPadding : spaceBelow - popupPadding,\n );\n\n const availableWidth = viewportWidth - 20;\n const calculatedWidth = Math.min(popupMaxWidth, availableWidth);\n\n let left = rect.left;\n if (left + calculatedWidth > viewportWidth - 10) {\n left = viewportWidth - calculatedWidth - 10;\n }\n left = Math.max(10, left);\n\n setPopupPosition({\n top: showAbove ? rect.top - 8 : rect.bottom + 8,\n left: left,\n maxHeight: Math.min(600, maxHeight),\n showAbove,\n });\n };\n\n updatePosition();\n window.addEventListener('resize', updatePosition);\n window.addEventListener('scroll', updatePosition, true);\n\n return () => {\n window.removeEventListener('resize', updatePosition);\n window.removeEventListener('scroll', updatePosition, true);\n };\n }, [isOpen, anchorRef]);\n\n useEffect(() => {\n const handleClickOutside = (event: MouseEvent) => {\n if (\n popupRef.current &&\n !popupRef.current.contains(event.target as Node) &&\n anchorRef.current &&\n !anchorRef.current.contains(event.target as Node)\n )\n onClose();\n };\n if (isOpen) document.addEventListener('mousedown', handleClickOutside);\n return () => {\n document.removeEventListener('mousedown', handleClickOutside);\n };\n }, [onClose, anchorRef, isOpen]);\n\n useEffect(() => {\n if (!isOpen) setSearchTerm('');\n }, [isOpen]);\n\n if (!isOpen) return null;\n return (\n \n {(title || showSearch) && (\n
\n {title && (\n

\n {title}\n

\n )}\n {showSearch && (\n setSearchTerm(e.target.value)}\n placeholder={\n searchPlaceholder ||\n t('settings.tools.searchPlaceholder', 'Search...')\n }\n labelBgClassName=\"bg-lotion dark:bg-charleston-green-2\"\n borderVariant=\"thin\"\n className=\"mb-4\"\n textSize=\"small\"\n />\n )}\n
\n )}\n
\n {loading ? (\n
\n
\n
\n ) : (\n
\n {filteredOptions.length === 0 ? (\n
\n \n

\n {searchTerm\n ? 'No results found'\n : noOptionsMessage || 'No options available'}\n

\n
\n ) : (\n filteredOptions.map((option) => {\n const isSelected = selectedIds.has(option.id);\n return (\n handleOptionClick(option.id)}\n className=\"dark:border-dim-gray dark:hover:bg-charleston-green-3 flex cursor-pointer items-center justify-between border-b border-[#D9D9D9] p-3 last:border-b-0 hover:bg-gray-100\"\n role=\"option\"\n aria-selected={isSelected}\n >\n
\n {option.icon && renderIcon(option.icon)}\n \n {option.label}\n

\n
\n
\n \n {isSelected && (\n \n )}\n
\n
\n
\n );\n })\n )}\n \n )}\n \n {footerContent && (\n
\n {footerContent}\n
\n )}\n \n );\n}\n" }, { "path": "frontend/src/components/Notification.tsx", "content": "import { useTranslation } from 'react-i18next';\nimport close from '../assets/cross.svg';\n\ninterface NotificationProps {\n notificationText: string;\n notificationLink: string;\n handleCloseNotification: () => void;\n}\n\nconst stars = Array.from({ length: 12 }, (_, i) => ({\n id: i,\n size: Math.random() * 2 + 1, // 1-3px\n left: Math.random() * 100, // 0-100%\n top: Math.random() * 100, // 0-100%\n animationDuration: Math.random() * 3 + 2, // 2-5s\n animationDelay: Math.random() * 2, // 0-2s\n opacity: Math.random() * 0.5 + 0.3, // 0.3-0.8\n}));\n\nexport default function Notification({\n notificationText,\n notificationLink,\n handleCloseNotification,\n}: NotificationProps) {\n const { t } = useTranslation();\n return (\n <>\n \n \n {/* Animated stars background */}\n
\n {stars.map((star) => (\n \n {/* 4-pointed Christmas star */}\n \n \n ))}\n
\n\n

\n {notificationText}\n

\n \n \n {/* Arrow tail - grows leftward from arrow head's back point on hover */}\n \n {/* Arrow head - pushed forward by the tail on hover */}\n \n \n \n\n {\n e.stopPropagation();\n e.preventDefault();\n handleCloseNotification();\n }}\n >\n \"Close\n \n \n \n );\n}" }, { "path": "frontend/src/components/RetryIcon.tsx", "content": "import * as React from 'react';\nimport { SVGProps } from 'react';\nconst RetryIcon = (props: SVGProps) => (\n \n \n \n);\nexport default RetryIcon;\n" }, { "path": "frontend/src/components/SearchableDropdown.tsx", "content": "import React from 'react';\n\nimport Arrow2 from '../assets/dropdown-arrow.svg';\nimport Edit from '../assets/edit.svg';\nimport Search from '../assets/search.svg';\nimport Trash from '../assets/trash.svg';\n\n/**\n * SearchableDropdown - A standalone dropdown component with built-in search functionality\n */\n\ntype SearchableDropdownOptionBase = {\n id?: string;\n type?: string;\n};\n\ntype NameIdOption = { name: string; id: string } & SearchableDropdownOptionBase;\n\nexport type SearchableDropdownOption =\n | string\n | NameIdOption\n | ({ label: string; value: string } & SearchableDropdownOptionBase)\n | ({ value: number; description: string } & SearchableDropdownOptionBase);\n\nexport type SearchableDropdownSelectedValue = SearchableDropdownOption | null;\n\nexport interface SearchableDropdownProps<\n T extends SearchableDropdownOption = SearchableDropdownOption,\n> {\n options: T[];\n selectedValue: SearchableDropdownSelectedValue;\n onSelect: (value: T) => void;\n size?: string;\n /** Controls border radius for both button and dropdown menu */\n rounded?: 'xl' | '3xl';\n border?: 'border' | 'border-2';\n showEdit?: boolean;\n onEdit?: (value: NameIdOption) => void;\n showDelete?: boolean | ((option: T) => boolean);\n onDelete?: (id: string) => void;\n placeholder?: string;\n}\n\nfunction SearchableDropdown({\n options,\n selectedValue,\n onSelect,\n size = 'w-32',\n rounded = 'xl',\n border = 'border-2',\n showEdit,\n onEdit,\n showDelete,\n onDelete,\n placeholder,\n}: SearchableDropdownProps) {\n const dropdownRef = React.useRef(null);\n const searchInputRef = React.useRef(null);\n const [isOpen, setIsOpen] = React.useState(false);\n const [searchQuery, setSearchQuery] = React.useState('');\n\n const borderRadius = rounded === 'xl' ? 'rounded-xl' : 'rounded-3xl';\n\n React.useEffect(() => {\n const handleClickOutside = (event: MouseEvent) => {\n if (\n dropdownRef.current &&\n !dropdownRef.current.contains(event.target as Node)\n ) {\n setIsOpen(false);\n setSearchQuery('');\n }\n };\n\n document.addEventListener('mousedown', handleClickOutside);\n return () => document.removeEventListener('mousedown', handleClickOutside);\n }, []);\n\n React.useEffect(() => {\n if (isOpen && searchInputRef.current) {\n searchInputRef.current.focus();\n }\n }, [isOpen]);\n\n const getOptionText = (option: SearchableDropdownOption): string => {\n if (typeof option === 'string') return option;\n if ('name' in option) return option.name;\n if ('label' in option) return option.label;\n if ('description' in option) return option.description;\n return '';\n };\n\n const filteredOptions = React.useMemo(() => {\n if (!searchQuery.trim()) return options;\n const query = searchQuery.toLowerCase();\n return options.filter((option) =>\n getOptionText(option).toLowerCase().includes(query),\n );\n }, [options, searchQuery]);\n\n const getDisplayValue = (): string => {\n if (!selectedValue) return placeholder ?? 'From URL';\n if (typeof selectedValue === 'string') return selectedValue;\n if ('label' in selectedValue) return selectedValue.label;\n if ('name' in selectedValue) return selectedValue.name;\n if ('description' in selectedValue) {\n return selectedValue.value < 1e9\n ? `${selectedValue.value} (${selectedValue.description})`\n : selectedValue.description;\n }\n return placeholder ?? 'From URL';\n };\n\n const isOptionSelected = (option: T): boolean => {\n if (!selectedValue) return false;\n if (typeof selectedValue === 'string')\n return selectedValue === (option as unknown as string);\n if (typeof option === 'string') return false;\n\n const optionObj = option as Record;\n const selectedObj = selectedValue as Record;\n\n if ('name' in optionObj && 'name' in selectedObj)\n return selectedObj.name === optionObj.name;\n if ('label' in optionObj && 'label' in selectedObj)\n return selectedObj.label === optionObj.label;\n if ('value' in optionObj && 'value' in selectedObj)\n return selectedObj.value === optionObj.value;\n return false;\n };\n\n return (\n \n setIsOpen(!isOpen)}\n className={`flex w-full cursor-pointer items-center justify-between ${border} border-silver dark:border-dim-gray bg-white px-5 py-3 dark:bg-transparent ${borderRadius}`}\n >\n \n {getDisplayValue()}\n \n \n \n\n {isOpen && (\n \n \n
\n \n setSearchQuery(e.target.value)}\n placeholder=\"Search...\"\n className=\"dark:text-bright-gray w-full rounded-lg border-0 bg-transparent py-2 pr-3 pl-10 font-['Inter'] text-[14px] leading-[16.5px] font-normal focus:ring-0 focus:outline-none\"\n onClick={(e) => e.stopPropagation()}\n />\n
\n \n\n
\n {filteredOptions.length === 0 ? (\n
\n No results found\n
\n ) : (\n filteredOptions.map((option, index) => {\n const selected = isOptionSelected(option);\n const optionObj =\n typeof option !== 'string'\n ? (option as Record)\n : null;\n const optionType = optionObj?.type as string | undefined;\n const optionId = optionObj?.id as string | undefined;\n const optionName = optionObj?.name as string | undefined;\n\n return (\n \n {\n onSelect(option);\n setIsOpen(false);\n setSearchQuery('');\n }}\n className=\"dark:text-light-gray ml-5 flex-1 overflow-hidden py-3 font-['Inter'] text-[14px] leading-[16.5px] font-normal text-ellipsis whitespace-nowrap\"\n >\n {getOptionText(option)}\n \n {showEdit &&\n onEdit &&\n optionObj &&\n optionType !== 'public' && (\n {\n if (optionName && optionId) {\n onEdit({\n id: optionId,\n name: optionName,\n type: optionType,\n });\n }\n setIsOpen(false);\n setSearchQuery('');\n }}\n />\n )}\n {showDelete && onDelete && (\n {\n e.stopPropagation();\n const id =\n typeof option === 'string'\n ? option\n : (optionId ?? '');\n onDelete(id);\n }}\n className={`mr-2 h-4 w-4 cursor-pointer hover:opacity-50 ${\n typeof showDelete === 'function' &&\n !showDelete(option)\n ? 'hidden'\n : ''\n }`}\n >\n \n \n )}\n
\n );\n })\n )}\n \n \n )}\n \n );\n}\n\nexport default SearchableDropdown;\n" }, { "path": "frontend/src/components/SendArrowIcon.tsx", "content": "import { SVGProps } from 'react';\nconst SendArrowIcon = (props: SVGProps) => (\n \n \n \n);\nexport default SendArrowIcon;\n" }, { "path": "frontend/src/components/SettingsBar.tsx", "content": "import React, { useCallback, useRef, useState } from 'react';\nimport { useTranslation } from 'react-i18next';\n\nimport ArrowLeft from '../assets/arrow-left.svg';\nimport ArrowRight from '../assets/arrow-right.svg';\n\ntype HiddenGradientType = 'left' | 'right' | undefined;\n\nconst useTabs = () => {\n const { t } = useTranslation();\n const tabs = [\n t('settings.general.label'),\n t('settings.sources.label'),\n t('settings.analytics.label'),\n t('settings.logs.label'),\n t('settings.tools.label'),\n ];\n return tabs;\n};\n\ninterface SettingsBarProps {\n setActiveTab: React.Dispatch>;\n activeTab: string;\n}\n\nconst SettingsBar = ({ setActiveTab, activeTab }: SettingsBarProps) => {\n const { t } = useTranslation();\n const [hiddenGradient, setHiddenGradient] =\n useState('left');\n const containerRef = useRef(null);\n const tabs = useTabs();\n const scrollTabs = useCallback(\n (direction: number) => {\n if (containerRef.current) {\n const container = containerRef.current;\n container.scrollLeft += direction * 100; // Adjust the scroll amount as needed\n if (container.scrollLeft === 0) {\n setHiddenGradient('left');\n } else if (\n container.scrollLeft + container.offsetWidth ===\n container.scrollWidth\n ) {\n setHiddenGradient('right');\n } else {\n setHiddenGradient(undefined);\n }\n }\n },\n [containerRef.current],\n );\n return (\n
\n
\n \n\n
\n scrollTabs(-1)}\n className=\"flex h-6 w-6 items-center justify-center rounded-full transition-all hover:bg-gray-200 dark:hover:bg-gray-700\"\n aria-label={t('settings.scrollTabsLeft')}\n >\n \"left-arrow\"\n \n
\n \n {tabs.map((tab, index) => (\n setActiveTab(tab)}\n className={`h-9 snap-start rounded-3xl px-4 font-bold transition-colors ${\n activeTab === tab\n ? 'dark:bg-dark-charcoal bg-[#F4F4F5] text-neutral-900 dark:text-white'\n : 'text-neutral-700 hover:text-neutral-900 dark:text-neutral-300 dark:hover:text-white'\n }`}\n role=\"tab\"\n aria-selected={activeTab === tab}\n aria-controls={`${tab.toLowerCase()}-panel`}\n id={`${tab.toLowerCase()}-tab`}\n >\n {tab}\n \n ))}\n \n
\n scrollTabs(1)}\n className=\"flex h-6 w-6 items-center justify-center rounded-full hover:bg-gray-200 dark:hover:bg-gray-700\"\n aria-label={t('settings.scrollTabsRight')}\n >\n \"right-arrow\"\n \n
\n \n );\n};\n\nexport default SettingsBar;\n" }, { "path": "frontend/src/components/Sidebar.tsx", "content": "import React from 'react';\n\nimport Exit from '../assets/exit.svg';\n\ntype SidebarProps = {\n isOpen: boolean;\n toggleState: (arg0: boolean) => void;\n children: React.ReactNode;\n};\n\nexport default function Sidebar({\n isOpen,\n toggleState,\n children,\n}: SidebarProps) {\n const sidebarRef = React.useRef(null);\n\n const handleClickOutside = (event: MouseEvent) => {\n if (\n sidebarRef.current &&\n !sidebarRef.current.contains(event.target as Node)\n ) {\n toggleState(false);\n }\n };\n\n React.useEffect(() => {\n document.addEventListener('mousedown', handleClickOutside);\n return () => {\n document.removeEventListener('mousedown', handleClickOutside);\n };\n }, []);\n return (\n
\n \n
\n toggleState(!isOpen)}\n >\n \n \n
\n
\n {children}\n
\n
\n \n );\n}\n" }, { "path": "frontend/src/components/SkeletonLoader.tsx", "content": "import { useState, useEffect } from 'react';\n\ninterface SkeletonLoaderProps {\n count?: number;\n component?:\n | 'default'\n | 'analysis'\n | 'logs'\n | 'fileTable'\n | 'chatbot'\n | 'dropdown'\n | 'chunkCards'\n | 'sourceCards';\n}\n\nconst SkeletonLoader: React.FC = ({\n count = 1,\n component = 'default',\n}) => {\n const [skeletonCount, setSkeletonCount] = useState(count);\n\n useEffect(() => {\n const handleResize = () => {\n const windowWidth = window.innerWidth;\n\n if (windowWidth > 1024) {\n setSkeletonCount(1);\n } else if (windowWidth > 768) {\n setSkeletonCount(count);\n } else {\n setSkeletonCount(Math.min(count, 2));\n }\n };\n\n handleResize();\n window.addEventListener('resize', handleResize);\n\n return () => {\n window.removeEventListener('resize', handleResize);\n };\n }, [count]);\n\n const renderTable = () => (\n <>\n {[...Array(4)].map((_, idx) => (\n \n \n
\n \n \n
\n \n \n
\n \n \n
\n \n \n ))}\n \n );\n\n const renderChatbot = () => (\n <>\n {[...Array(4)].map((_, idx) => (\n \n \n
\n \n \n
\n \n \n
\n \n \n
\n \n \n ))}\n \n );\n\n const renderDropdown = () => (\n
\n
\n
\n
\n
\n
\n
\n );\n\n const renderLogs = () => (\n
\n {[...Array(8)].map((_, idx) => (\n \n
\n
\n
\n
\n
\n
\n
\n
\n
\n ))}\n \n );\n\n const renderDefault = () => (\n <>\n {[...Array(skeletonCount)].map((_, idx) => (\n \n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n \n ))}\n \n );\n\n const renderAnalysis = () => (\n <>\n {[...Array(skeletonCount)].map((_, idx) => (\n \n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n \n ))}\n \n );\n\n const renderChunkCards = () => (\n <>\n {Array.from({ length: count }).map((_, index) => (\n \n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n \n ))}\n \n );\n\n const renderSourceCards = () => (\n <>\n {Array.from({ length: count }).map((_, idx) => (\n \n
\n
\n
\n
\n
\n
\n
\n
\n\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n \n ))}\n \n );\n\n const componentMap = {\n fileTable: renderTable,\n chatbot: renderChatbot,\n dropdown: renderDropdown,\n logs: renderLogs,\n default: renderDefault,\n analysis: renderAnalysis,\n chunkCards: renderChunkCards,\n sourceCards: renderSourceCards,\n };\n\n const render = componentMap[component] || componentMap.default;\n\n return <>{render()};\n};\n\nexport default SkeletonLoader;\n" }, { "path": "frontend/src/components/SourcesPopup.tsx", "content": "import React, { useRef, useEffect, useState, useLayoutEffect } from 'react';\nimport { createPortal } from 'react-dom';\nimport { useTranslation } from 'react-i18next';\nimport { useDispatch, useSelector } from 'react-redux';\nimport { Doc } from '../models/misc';\nimport SourceIcon from '../assets/source.svg';\nimport CheckIcon from '../assets/checkmark.svg';\nimport RedirectIcon from '../assets/redirect.svg';\nimport Input from './Input';\nimport {\n selectSourceDocs,\n selectSelectedDocs,\n setSelectedDocs,\n} from '../preferences/preferenceSlice';\nimport { ActiveState } from '../models/misc';\n\ntype SourcesPopupProps = {\n isOpen: boolean;\n onClose: () => void;\n anchorRef: React.RefObject;\n handlePostDocumentSelect: (doc: Doc[] | null) => void;\n setUploadModalState: React.Dispatch>;\n};\n\nexport default function SourcesPopup({\n isOpen,\n onClose,\n anchorRef,\n handlePostDocumentSelect,\n setUploadModalState,\n}: SourcesPopupProps) {\n const dispatch = useDispatch();\n const { t } = useTranslation();\n const popupRef = useRef(null);\n const [searchTerm, setSearchTerm] = useState('');\n const [popupPosition, setPopupPosition] = useState({\n top: 0,\n left: 0,\n maxHeight: 0,\n showAbove: false,\n });\n\n const options = useSelector(selectSourceDocs);\n const selectedDocs = useSelector(selectSelectedDocs);\n\n const filteredOptions = options?.filter((option) =>\n option.name.toLowerCase().includes(searchTerm.toLowerCase()),\n );\n\n useLayoutEffect(() => {\n if (!isOpen || !anchorRef.current) return;\n\n const updatePosition = () => {\n if (!anchorRef.current) return;\n\n const rect = anchorRef.current.getBoundingClientRect();\n const viewportHeight = window.innerHeight;\n const viewportWidth = window.innerWidth;\n const spaceAbove = rect.top;\n const spaceBelow = viewportHeight - rect.bottom;\n const showAbove = spaceAbove > spaceBelow && spaceAbove >= 300;\n const maxHeight = showAbove ? spaceAbove - 16 : spaceBelow - 16;\n const left = Math.min(\n rect.left,\n viewportWidth - Math.min(480, viewportWidth * 0.95) - 10,\n );\n\n setPopupPosition({\n top: showAbove ? rect.top - 8 : rect.bottom + 8,\n left,\n maxHeight: Math.min(600, maxHeight),\n showAbove,\n });\n };\n\n updatePosition();\n window.addEventListener('resize', updatePosition);\n return () => window.removeEventListener('resize', updatePosition);\n }, [isOpen, anchorRef]);\n\n const handleClickOutside = (event: MouseEvent) => {\n if (\n popupRef.current &&\n !popupRef.current.contains(event.target as Node) &&\n !anchorRef.current?.contains(event.target as Node)\n ) {\n onClose();\n }\n };\n\n useEffect(() => {\n if (isOpen) {\n document.addEventListener('mousedown', handleClickOutside);\n }\n return () => {\n document.removeEventListener('mousedown', handleClickOutside);\n };\n }, [isOpen]);\n\n if (!isOpen) return null;\n\n const handleUploadClick = () => {\n setUploadModalState('ACTIVE');\n onClose();\n };\n\n const popupContent = (\n \n
\n
\n

\n {t('conversation.sources.text')}\n

\n\n setSearchTerm(e.target.value)}\n placeholder={t('settings.sources.searchPlaceholder')}\n borderVariant=\"thin\"\n className=\"mb-4\"\n labelBgClassName=\"bg-lotion dark:bg-charleston-green-2\"\n />\n
\n\n
\n {options ? (\n <>\n {filteredOptions?.map((option: any, index: number) => {\n const isSelected =\n selectedDocs &&\n Array.isArray(selectedDocs) &&\n selectedDocs.length > 0 &&\n selectedDocs.some((doc) =>\n option.id ? doc.id === option.id : doc.date === option.date,\n );\n\n return (\n {\n if (isSelected) {\n const updatedDocs =\n selectedDocs && Array.isArray(selectedDocs)\n ? selectedDocs.filter((doc) =>\n option.id\n ? doc.id !== option.id\n : doc.date !== option.date,\n )\n : [];\n dispatch(setSelectedDocs(updatedDocs));\n handlePostDocumentSelect(\n updatedDocs.length > 0 ? updatedDocs : null,\n );\n } else {\n const updatedDocs =\n selectedDocs && Array.isArray(selectedDocs)\n ? [...selectedDocs, option]\n : [option];\n dispatch(setSelectedDocs(updatedDocs));\n handlePostDocumentSelect(updatedDocs);\n }\n }}\n >\n \n \n {option.name}\n \n \n {isSelected && (\n \n )}\n
\n
\n );\n })}\n \n ) : (\n
\n {t('conversation.sources.noSourcesAvailable')}\n
\n )}\n \n\n
\n \n {t('settings.sources.goToSources')}\n \"Redirect\"\n \n
\n\n
\n \n {t('settings.sources.uploadNew')}\n \n
\n \n \n );\n\n return createPortal(popupContent, document.body);\n}\n" }, { "path": "frontend/src/components/Spinner.tsx", "content": "import React from 'react';\n\ntype SpinnerProps = {\n size?: 'small' | 'medium' | 'large';\n color?: string;\n};\n\nexport default function Spinner({\n size = 'medium',\n color = 'grey',\n}: SpinnerProps) {\n const sizeMap = {\n small: '20px',\n medium: '30px',\n large: '40px',\n };\n const spinnerSize = sizeMap[size];\n\n const spinnerStyle = {\n width: spinnerSize,\n height: spinnerSize,\n aspectRatio: '1',\n borderRadius: '50%',\n background: `\n radial-gradient(farthest-side, ${color} 94%, #0000) top/8px 8px no-repeat,\n conic-gradient(#0000 30%, ${color})\n `,\n WebkitMask:\n 'radial-gradient(farthest-side, #0000 calc(100% - 8px), #000 0)',\n animation: 'l13 1s infinite linear',\n } as React.CSSProperties;\n\n const keyframesStyle = `@keyframes l13 {\n 100% { transform: rotate(1turn) }\n }`;\n\n return (\n <>\n \n
\n \n );\n}\n" }, { "path": "frontend/src/components/Table.tsx", "content": "import React from 'react';\n\ninterface TableProps {\n children: React.ReactNode;\n className?: string;\n minWidth?: string;\n}\n\ninterface TableContainerProps {\n children: React.ReactNode;\n className?: string;\n height?: string;\n bordered?: boolean;\n}\n\ninterface TableHeadProps {\n children: React.ReactNode;\n className?: string;\n}\n\ninterface TableRowProps {\n children: React.ReactNode;\n className?: string;\n onClick?: () => void;\n}\n\ninterface TableCellProps {\n children?: React.ReactNode;\n className?: string;\n minWidth?: string;\n width?: string;\n align?: 'left' | 'right' | 'center';\n}\n\nconst TableContainer = React.forwardRef(\n function TableContainer(\n {\n children,\n className = '',\n height = 'auto',\n bordered = true,\n }: TableContainerProps,\n ref: React.ForwardedRef,\n ) {\n return (\n
\n \n {children}\n
\n
\n );\n },\n);\n\nconst Table: React.FC = ({\n children,\n className = '',\n minWidth = 'min-w-[600px]',\n}) => {\n return (\n \n {children}\n \n );\n};\nconst TableHead: React.FC = ({ children, className = '' }) => {\n return (\n \n {children}\n \n );\n};\n\nconst TableBody: React.FC = ({ children, className = '' }) => {\n return (\n tr:last-child]:border-b-0 ${className}`}>\n {children}\n \n );\n};\n\nconst TableRow: React.FC = ({\n children,\n className = '',\n onClick,\n}) => {\n const baseClasses =\n 'border-b border-[#D7D7D7] hover:bg-[#ECEEEF] dark:border-[#6A6A6A] dark:hover:bg-[#27282D]';\n const cursorClass = onClick ? 'cursor-pointer' : '';\n\n return (\n \n {children}\n \n );\n};\n\nconst TableHeader: React.FC = ({\n children,\n className = '',\n minWidth,\n width,\n align = 'left',\n}) => {\n const getAlignmentClass = () => {\n switch (align) {\n case 'right':\n return 'text-right';\n case 'center':\n return 'text-center';\n default:\n return 'text-left';\n }\n };\n\n const baseClasses = `px-2 py-3 text-sm font-medium text-gray-700 lg:px-3 dark:text-[#59636E] border-b border-[#D7D7D7] dark:border-[#6A6A6A] relative box-border ${getAlignmentClass()}`;\n const widthClasses = minWidth ? minWidth : '';\n\n return (\n \n {children}\n \n );\n};\n\nconst TableCell: React.FC = ({\n children,\n className = '',\n minWidth,\n width,\n align = 'left',\n}) => {\n const getAlignmentClass = () => {\n switch (align) {\n case 'right':\n return 'text-right';\n case 'center':\n return 'text-center';\n default:\n return 'text-left';\n }\n };\n\n const baseClasses = `px-2 py-2 text-sm lg:px-3 dark:text-[#E0E0E0] box-border ${getAlignmentClass()}`;\n const widthClasses = minWidth ? minWidth : '';\n\n return (\n \n {children}\n \n );\n};\n\nexport {\n Table,\n TableContainer,\n TableHead,\n TableBody,\n TableRow,\n TableHeader,\n TableCell,\n};\n\nexport default Table;\n" }, { "path": "frontend/src/components/TextToSpeechButton.tsx", "content": "import { useState, useRef, useEffect } from 'react';\nimport Speaker from '../assets/speaker.svg?react';\nimport Stopspeech from '../assets/stopspeech.svg?react';\nimport LoadingIcon from '../assets/Loading.svg?react'; // Add a loading icon SVG here\n\nconst apiHost = import.meta.env.VITE_API_HOST || 'https://docsapi.arc53.com';\n\nlet currentlyPlayingAudio: {\n audio: HTMLAudioElement;\n stopCallback: () => void;\n} | null = null;\n\nlet currentLoadingRequest: {\n abortController: AbortController;\n stopLoadingCallback: () => void;\n} | null = null;\n\n// LRU Cache for audio\nconst audioCache = new Map();\nconst MAX_CACHE_SIZE = 10;\n\nfunction getCachedAudio(text: string): string | undefined {\n const cached = audioCache.get(text);\n if (cached) {\n audioCache.delete(text);\n audioCache.set(text, cached);\n }\n return cached;\n}\n\nfunction setCachedAudio(text: string, audioBase64: string) {\n if (audioCache.has(text)) {\n audioCache.delete(text);\n }\n if (audioCache.size >= MAX_CACHE_SIZE) {\n const firstKey = audioCache.keys().next().value;\n if (firstKey !== undefined) {\n audioCache.delete(firstKey);\n }\n }\n\n audioCache.set(text, audioBase64);\n}\n\nexport default function SpeakButton({ text }: { text: string }) {\n const [isSpeaking, setIsSpeaking] = useState(false);\n const [isLoading, setIsLoading] = useState(false);\n const audioRef = useRef(null);\n const abortControllerRef = useRef(null);\n\n useEffect(() => {\n return () => {\n // Abort any pending fetch request\n if (abortControllerRef.current) {\n abortControllerRef.current.abort();\n abortControllerRef.current = null;\n }\n\n // Stop any playing audio\n if (audioRef.current) {\n audioRef.current.pause();\n if (currentlyPlayingAudio?.audio === audioRef.current) {\n currentlyPlayingAudio = null;\n }\n audioRef.current = null;\n }\n\n // Clear global loading request if it's this component's\n if (currentLoadingRequest) {\n currentLoadingRequest = null;\n }\n };\n }, []);\n\n const handleSpeakClick = async () => {\n if (isSpeaking) {\n audioRef.current?.pause();\n audioRef.current = null;\n currentlyPlayingAudio = null;\n setIsSpeaking(false);\n return;\n }\n\n // Stop any currently playing audio\n if (currentlyPlayingAudio) {\n currentlyPlayingAudio.audio.pause();\n currentlyPlayingAudio.stopCallback();\n currentlyPlayingAudio = null;\n }\n\n // Abort any pending loading request\n if (currentLoadingRequest) {\n currentLoadingRequest.abortController.abort();\n currentLoadingRequest.stopLoadingCallback();\n currentLoadingRequest = null;\n }\n\n try {\n setIsLoading(true);\n const cachedAudio = getCachedAudio(text);\n let audioBase64: string;\n\n if (cachedAudio) {\n audioBase64 = cachedAudio;\n setIsLoading(false);\n } else {\n const abortController = new AbortController();\n abortControllerRef.current = abortController;\n\n currentLoadingRequest = {\n abortController,\n stopLoadingCallback: () => {\n setIsLoading(false);\n },\n };\n\n const response = await fetch(apiHost + '/api/tts', {\n method: 'POST',\n headers: { 'Content-Type': 'application/json' },\n body: JSON.stringify({ text }),\n signal: abortController.signal,\n });\n\n const data = await response.json();\n abortControllerRef.current = null;\n currentLoadingRequest = null;\n\n if (data.success && data.audio_base64) {\n audioBase64 = data.audio_base64;\n // Store in cache\n setCachedAudio(text, audioBase64);\n setIsLoading(false);\n } else {\n console.error('Failed to retrieve audio.');\n setIsLoading(false);\n return;\n }\n }\n\n const audio = new Audio(`data:audio/mp3;base64,${audioBase64}`);\n audioRef.current = audio;\n\n currentlyPlayingAudio = {\n audio,\n stopCallback: () => {\n setIsSpeaking(false);\n audioRef.current = null;\n },\n };\n\n audio.play().then(() => {\n setIsSpeaking(true);\n setIsLoading(false);\n\n audio.onended = () => {\n setIsSpeaking(false);\n audioRef.current = null;\n if (currentlyPlayingAudio?.audio === audio) {\n currentlyPlayingAudio = null;\n }\n };\n });\n } catch (error: any) {\n abortControllerRef.current = null;\n currentLoadingRequest = null;\n\n if (error.name === 'AbortError') {\n return;\n }\n console.error('Error fetching audio from TTS endpoint', error);\n setIsLoading(false);\n }\n };\n\n return (\n \n {isLoading ? (\n \n ) : isSpeaking ? (\n \n ) : (\n \n )}\n \n );\n}\n" }, { "path": "frontend/src/components/ToggleSwitch.tsx", "content": "import React from 'react';\n\ntype ToggleSwitchProps = {\n checked: boolean;\n onChange: (checked: boolean) => void;\n className?: string;\n label?: string;\n disabled?: boolean;\n size?: 'small' | 'medium' | 'large';\n labelPosition?: 'left' | 'right';\n id?: string;\n ariaLabel?: string;\n};\n\nconst ToggleSwitch: React.FC = ({\n checked,\n onChange,\n className = '',\n label,\n disabled = false,\n size = 'medium',\n labelPosition = 'left',\n id,\n ariaLabel,\n}) => {\n // Size configurations\n const sizeConfig = {\n small: {\n box: 'h-5 w-9',\n toggle: 'h-4 w-4 left-0.5 top-0.5',\n translate: 'translate-x-full',\n },\n medium: {\n box: 'h-8 w-14',\n toggle: 'h-6 w-6 left-1 top-1',\n translate: 'translate-x-full',\n },\n large: {\n box: 'h-10 w-16',\n toggle: 'h-8 w-8 left-1 top-1',\n translate: 'translate-x-full',\n },\n };\n\n const { box, toggle, translate } = sizeConfig[size];\n\n return (\n \n {label && (\n \n {label}\n \n )}\n
\n onChange(e.target.checked)}\n className=\"sr-only\"\n disabled={disabled}\n aria-label={ariaLabel}\n />\n
\n \n \n \n );\n};\n\nexport default ToggleSwitch;\n" }, { "path": "frontend/src/components/ToolsPopup.tsx", "content": "import React, { useEffect, useRef, useState, useLayoutEffect } from 'react';\nimport { createPortal } from 'react-dom';\nimport { useTranslation } from 'react-i18next';\nimport { useSelector } from 'react-redux';\nimport { selectToken } from '../preferences/preferenceSlice';\nimport userService from '../api/services/userService';\nimport { UserToolType } from '../settings/types';\nimport Input from './Input';\nimport RedirectIcon from '../assets/redirect.svg';\nimport NoFilesIcon from '../assets/no-files.svg';\nimport NoFilesDarkIcon from '../assets/no-files-dark.svg';\nimport CheckmarkIcon from '../assets/checkmark.svg';\nimport { useDarkTheme } from '../hooks';\n\ninterface ToolsPopupProps {\n isOpen: boolean;\n onClose: () => void;\n anchorRef: React.RefObject;\n}\n\nexport default function ToolsPopup({\n isOpen,\n onClose,\n anchorRef,\n}: ToolsPopupProps) {\n const { t } = useTranslation();\n const token = useSelector(selectToken);\n const [userTools, setUserTools] = React.useState([]);\n const [loading, setLoading] = React.useState(false);\n const [searchTerm, setSearchTerm] = useState('');\n const [isDarkTheme] = useDarkTheme();\n const popupRef = useRef(null);\n const [popupPosition, setPopupPosition] = useState({\n top: 0,\n left: 0,\n maxHeight: 0,\n showAbove: false,\n });\n\n useLayoutEffect(() => {\n if (!isOpen || !anchorRef.current) return;\n\n const updatePosition = () => {\n if (!anchorRef.current) return;\n\n const rect = anchorRef.current.getBoundingClientRect();\n const viewportHeight = window.innerHeight;\n const viewportWidth = window.innerWidth;\n\n const spaceAbove = rect.top;\n const spaceBelow = viewportHeight - rect.bottom;\n const showAbove = spaceAbove > spaceBelow && spaceAbove >= 300;\n const maxHeight = showAbove ? spaceAbove - 16 : spaceBelow - 16;\n\n const left = Math.min(\n rect.left,\n viewportWidth - Math.min(462, viewportWidth * 0.95) - 10,\n );\n\n setPopupPosition({\n top: showAbove ? rect.top - 8 : rect.bottom + 8,\n left,\n maxHeight: Math.min(600, maxHeight),\n showAbove,\n });\n };\n\n updatePosition();\n window.addEventListener('resize', updatePosition);\n return () => window.removeEventListener('resize', updatePosition);\n }, [isOpen, anchorRef]);\n\n useEffect(() => {\n const handleClickOutside = (event: MouseEvent) => {\n if (\n popupRef.current &&\n !popupRef.current.contains(event.target as Node) &&\n anchorRef.current &&\n !anchorRef.current.contains(event.target as Node)\n ) {\n onClose();\n }\n };\n\n if (isOpen) {\n document.addEventListener('mousedown', handleClickOutside);\n }\n return () => {\n document.removeEventListener('mousedown', handleClickOutside);\n };\n }, [onClose, anchorRef, isOpen]);\n\n useEffect(() => {\n if (isOpen) {\n getUserTools();\n }\n }, [isOpen, token]);\n\n const getUserTools = () => {\n setLoading(true);\n userService\n .getUserTools(token)\n .then((res) => {\n return res.json();\n })\n .then((data) => {\n setUserTools(data.tools);\n setLoading(false);\n })\n .catch((error) => {\n console.error('Error fetching tools:', error);\n setLoading(false);\n });\n };\n\n const updateToolStatus = (toolId: string, newStatus: boolean) => {\n userService\n .updateToolStatus({ id: toolId, status: newStatus }, token)\n .then(() => {\n setUserTools((prevTools) =>\n prevTools.map((tool) =>\n tool.id === toolId ? { ...tool, status: newStatus } : tool,\n ),\n );\n })\n .catch((error) => {\n console.error('Failed to update tool status:', error);\n });\n };\n\n if (!isOpen) return null;\n\n const filteredTools = userTools.filter((tool) =>\n tool.displayName.toLowerCase().includes(searchTerm.toLowerCase()),\n );\n\n const popupContent = (\n \n
\n
\n

\n {t('settings.tools.label')}\n

\n\n setSearchTerm(e.target.value)}\n placeholder={t('settings.tools.searchPlaceholder')}\n labelBgClassName=\"bg-lotion dark:bg-charleston-green-2\"\n borderVariant=\"thin\"\n className=\"mb-4\"\n />\n
\n\n {loading ? (\n
\n
\n
\n ) : (\n
\n
\n {filteredTools.length === 0 ? (\n
\n \n

\n {t('settings.tools.noToolsFound')}\n

\n
\n ) : (\n filteredTools.map((tool) => (\n updateToolStatus(tool.id, !tool.status)}\n className=\"dark:border-dim-gray dark:hover:bg-charleston-green-3 flex items-center justify-between border-b border-[#D9D9D9] p-3 hover:bg-gray-100\"\n >\n
\n \n
\n

\n {tool.customName || tool.displayName}\n

\n
\n
\n
\n \n {tool.status && (\n \n )}\n
\n
\n
\n ))\n )}\n
\n \n )}\n\n
\n \n {t('settings.tools.manageTools')}\n \n \n
\n \n \n );\n\n return createPortal(popupContent, document.body);\n}\n" }, { "path": "frontend/src/components/UploadToast.tsx", "content": "import { useState } from 'react';\n\nimport { useDispatch, useSelector } from 'react-redux';\nimport { useTranslation } from 'react-i18next';\nimport { selectUploadTasks, dismissUploadTask } from '../upload/uploadSlice';\nimport ChevronDown from '../assets/chevron-down.svg';\nimport CheckCircleFilled from '../assets/check-circle-filled.svg';\nimport WarnIcon from '../assets/warn.svg';\n\nconst PROGRESS_RADIUS = 10;\nconst PROGRESS_CIRCUMFERENCE = 2 * Math.PI * PROGRESS_RADIUS;\n\nexport default function UploadToast() {\n const [collapsedTasks, setCollapsedTasks] = useState>(\n {},\n );\n\n const toggleTaskCollapse = (taskId: string) => {\n setCollapsedTasks((prev) => ({\n ...prev,\n [taskId]: !prev[taskId],\n }));\n };\n\n const { t } = useTranslation();\n const dispatch = useDispatch();\n const uploadTasks = useSelector(selectUploadTasks);\n\n const getStatusHeading = (status: string) => {\n switch (status) {\n case 'preparing':\n return t('modals.uploadDoc.progress.wait');\n case 'uploading':\n return t('modals.uploadDoc.progress.upload');\n case 'training':\n return t('modals.uploadDoc.progress.upload');\n case 'completed':\n return t('modals.uploadDoc.progress.completed');\n case 'failed':\n return t('modals.uploadDoc.progress.failed');\n default:\n return t('modals.uploadDoc.progress.preparing');\n }\n };\n\n return (\n e.stopPropagation()}\n >\n {uploadTasks\n .filter((task) => !task.dismissed)\n .map((task) => {\n const shouldShowProgress = [\n 'preparing',\n 'uploading',\n 'training',\n ].includes(task.status);\n const rawProgress = Math.min(Math.max(task.progress ?? 0, 0), 100);\n const formattedProgress = Math.round(rawProgress);\n const progressOffset =\n PROGRESS_CIRCUMFERENCE * (1 - rawProgress / 100);\n const isCollapsed = collapsedTasks[task.id] ?? false;\n\n return (\n \n
\n \n

\n {getStatusHeading(task.status)}\n

\n
\n toggleTaskCollapse(task.id)}\n aria-label={\n isCollapsed\n ? t('modals.uploadDoc.progress.expandDetails')\n : t('modals.uploadDoc.progress.collapseDetails')\n }\n className=\"flex h-8 items-center justify-center p-0 text-black opacity-70 transition-opacity hover:opacity-100 dark:text-white\"\n >\n \n \n dispatch(dismissUploadTask(task.id))}\n className=\"flex h-8 items-center justify-center p-0 text-black opacity-70 transition-opacity hover:opacity-100 dark:text-white\"\n aria-label={t('modals.uploadDoc.progress.dismiss')}\n >\n \n \n \n \n \n
\n
\n\n \n \n
\n \n {task.fileName}\n

\n\n
\n {shouldShowProgress && (\n \n \n \n \n )}\n\n {task.status === 'completed' && (\n \n )}\n\n {task.status === 'failed' && (\n \n )}\n
\n
\n\n {task.status === 'failed' && task.errorMessage && (\n \n {task.errorMessage}\n \n )}\n \n \n \n \n );\n })}\n \n );\n}\n" }, { "path": "frontend/src/components/types/Dropdown.types.ts", "content": "export type DropdownOptionBase = {\n id?: string;\n type?: string;\n};\n\nexport type StringOption = string;\nexport type NameIdOption = { name: string; id: string } & DropdownOptionBase;\nexport type LabelValueOption = {\n label: string;\n value: string;\n} & DropdownOptionBase;\nexport type ValueDescriptionOption = {\n value: number;\n description: string;\n} & DropdownOptionBase;\n\nexport type DropdownOption =\n | StringOption\n | NameIdOption\n | LabelValueOption\n | ValueDescriptionOption;\n\nexport type DropdownSelectedValue = DropdownOption | null;\n\nexport type OnSelectHandler = (\n value: T,\n) => void;\n\nexport interface DropdownProps {\n options: T[];\n selectedValue: DropdownSelectedValue;\n onSelect: OnSelectHandler;\n size?: string;\n rounded?: 'xl' | '3xl';\n buttonClassName?: string;\n optionsClassName?: string;\n border?: 'border' | 'border-2';\n showEdit?: boolean;\n onEdit?: (value: NameIdOption) => void;\n showDelete?: boolean | ((option: T) => boolean);\n onDelete?: (id: string) => void;\n placeholder?: string;\n placeholderClassName?: string;\n contentSize?: string;\n}\n" }, { "path": "frontend/src/components/types/index.ts", "content": "export type InputProps = {\n type: 'text' | 'number';\n value: string | string[] | number;\n colorVariant?: 'silver' | 'jet' | 'gray';\n borderVariant?: 'thin' | 'thick';\n textSize?: 'small' | 'medium';\n isAutoFocused?: boolean;\n id?: string;\n maxLength?: number;\n name?: string;\n placeholder?: string;\n required?: boolean;\n className?: string;\n children?: React.ReactElement;\n labelBgClassName?: string;\n onChange: (\n e: React.ChangeEvent,\n ) => void;\n onPaste?: (\n e: React.ClipboardEvent,\n ) => void;\n onKeyDown?: (\n e: React.KeyboardEvent,\n ) => void;\n leftIcon?: React.ReactNode;\n edgeRoundness?: string;\n};\n\nexport type MermaidRendererProps = {\n code: string;\n isLoading?: boolean;\n};\n" }, { "path": "frontend/src/components/ui/alert.tsx", "content": "import * as React from 'react';\nimport { cva, type VariantProps } from 'class-variance-authority';\n\nimport { cn } from '@/lib/utils';\n\nconst alertVariants = cva(\n 'relative w-full rounded-lg border px-4 py-3 text-sm [&>svg+div]:translate-y-[-3px] [&>svg]:absolute [&>svg]:left-4 [&>svg]:top-4 [&>svg]:text-foreground [&>svg~*]:pl-7',\n {\n variants: {\n variant: {\n default: 'bg-background text-foreground',\n destructive:\n 'border-destructive/50 text-destructive dark:border-destructive [&>svg]:text-destructive',\n },\n },\n defaultVariants: {\n variant: 'default',\n },\n },\n);\n\nconst Alert = React.forwardRef<\n HTMLDivElement,\n React.HTMLAttributes & VariantProps\n>(({ className, variant, ...props }, ref) => (\n \n));\nAlert.displayName = 'Alert';\n\nconst AlertTitle = React.forwardRef<\n HTMLParagraphElement,\n React.HTMLAttributes\n>(({ className, ...props }, ref) => (\n \n));\nAlertTitle.displayName = 'AlertTitle';\n\nconst AlertDescription = React.forwardRef<\n HTMLParagraphElement,\n React.HTMLAttributes\n>(({ className, ...props }, ref) => (\n \n));\nAlertDescription.displayName = 'AlertDescription';\n\nexport { Alert, AlertTitle, AlertDescription };\n" }, { "path": "frontend/src/components/ui/button.tsx", "content": "import * as React from 'react';\nimport { Slot } from '@radix-ui/react-slot';\nimport { cva, type VariantProps } from 'class-variance-authority';\n\nimport { cn } from '@/lib/utils';\n\nconst buttonVariants = cva(\n \"inline-flex items-center justify-center gap-2 whitespace-nowrap rounded-md text-sm font-medium transition-all disabled:pointer-events-none disabled:opacity-50 [&_svg]:pointer-events-none [&_svg:not([class*='size-'])]:size-4 shrink-0 [&_svg]:shrink-0 outline-none focus-visible:border-ring focus-visible:ring-ring/50 focus-visible:ring-[3px] aria-invalid:ring-destructive/20 dark:aria-invalid:ring-destructive/40 aria-invalid:border-destructive\",\n {\n variants: {\n variant: {\n default: 'bg-primary text-primary-foreground hover:bg-primary/90',\n destructive:\n 'bg-destructive text-white hover:bg-destructive/90 focus-visible:ring-destructive/20 dark:focus-visible:ring-destructive/40 dark:bg-destructive/60',\n outline:\n 'border bg-background shadow-xs hover:bg-accent hover:text-accent-foreground dark:bg-input/30 dark:border-input dark:hover:bg-input/50',\n secondary:\n 'bg-secondary text-secondary-foreground hover:bg-secondary/80',\n ghost:\n 'hover:bg-accent hover:text-accent-foreground dark:hover:bg-accent/50',\n link: 'text-primary underline-offset-4 hover:underline',\n },\n size: {\n default: 'h-9 px-4 py-2 has-[>svg]:px-3',\n sm: 'h-8 rounded-md gap-1.5 px-3 has-[>svg]:px-2.5',\n lg: 'h-10 rounded-md px-6 has-[>svg]:px-4',\n icon: 'size-9',\n 'icon-sm': 'size-8',\n 'icon-lg': 'size-10',\n },\n },\n defaultVariants: {\n variant: 'default',\n size: 'default',\n },\n },\n);\n\nfunction Button({\n className,\n variant = 'default',\n size = 'default',\n asChild = false,\n ...props\n}: React.ComponentProps<'button'> &\n VariantProps & {\n asChild?: boolean;\n }) {\n const Comp = asChild ? Slot : 'button';\n\n return (\n \n );\n}\n\nexport { Button, buttonVariants };\n" }, { "path": "frontend/src/components/ui/command.tsx", "content": "import { Command as CommandPrimitive } from 'cmdk';\nimport { SearchIcon } from 'lucide-react';\nimport * as React from 'react';\n\nimport {\n Dialog,\n DialogContent,\n DialogDescription,\n DialogHeader,\n DialogTitle,\n} from '@/components/ui/dialog';\nimport { cn } from '@/lib/utils';\n\nfunction Command({\n className,\n ...props\n}: React.ComponentProps) {\n return (\n \n );\n}\n\nfunction CommandDialog({\n title = 'Command Palette',\n description = 'Search for a command to run...',\n children,\n className,\n showCloseButton = true,\n ...props\n}: React.ComponentProps & {\n title?: string;\n description?: string;\n className?: string;\n showCloseButton?: boolean;\n}) {\n return (\n \n \n {title}\n {description}\n \n \n \n {children}\n \n \n \n );\n}\n\nfunction CommandInput({\n className,\n ...props\n}: React.ComponentProps) {\n return (\n \n \n \n \n );\n}\n\nfunction CommandList({\n className,\n ...props\n}: React.ComponentProps) {\n return (\n \n );\n}\n\nfunction CommandEmpty({\n ...props\n}: React.ComponentProps) {\n return (\n \n );\n}\n\nfunction CommandGroup({\n className,\n ...props\n}: React.ComponentProps) {\n return (\n \n );\n}\n\nfunction CommandSeparator({\n className,\n ...props\n}: React.ComponentProps) {\n return (\n \n );\n}\n\nfunction CommandItem({\n className,\n ...props\n}: React.ComponentProps) {\n return (\n \n );\n}\n\nfunction CommandShortcut({\n className,\n ...props\n}: React.ComponentProps<'span'>) {\n return (\n \n );\n}\n\nexport {\n Command,\n CommandDialog,\n CommandInput,\n CommandList,\n CommandEmpty,\n CommandGroup,\n CommandItem,\n CommandShortcut,\n CommandSeparator,\n};\n" }, { "path": "frontend/src/components/ui/dialog.tsx", "content": "'use client';\n\nimport { XIcon } from 'lucide-react';\nimport * as React from 'react';\n\nimport { cn } from '@/lib/utils';\nimport * as DialogPrimitive from '@radix-ui/react-dialog';\n\nfunction Dialog({\n ...props\n}: React.ComponentProps) {\n return ;\n}\n\nfunction DialogTrigger({\n ...props\n}: React.ComponentProps) {\n return ;\n}\n\nfunction DialogPortal({\n ...props\n}: React.ComponentProps) {\n return ;\n}\n\nfunction DialogClose({\n ...props\n}: React.ComponentProps) {\n return ;\n}\n\nfunction DialogOverlay({\n className,\n ...props\n}: React.ComponentProps) {\n return (\n \n );\n}\n\nfunction DialogContent({\n className,\n children,\n showCloseButton = true,\n ...props\n}: React.ComponentProps & {\n showCloseButton?: boolean;\n}) {\n return (\n \n \n \n {children}\n {showCloseButton && (\n \n \n Close\n \n )}\n \n \n );\n}\n\nfunction DialogHeader({ className, ...props }: React.ComponentProps<'div'>) {\n return (\n \n );\n}\n\nfunction DialogFooter({ className, ...props }: React.ComponentProps<'div'>) {\n return (\n \n );\n}\n\nfunction DialogTitle({\n className,\n ...props\n}: React.ComponentProps) {\n return (\n \n );\n}\n\nfunction DialogDescription({\n className,\n ...props\n}: React.ComponentProps) {\n return (\n \n );\n}\n\nexport {\n Dialog,\n DialogClose,\n DialogContent,\n DialogDescription,\n DialogFooter,\n DialogHeader,\n DialogOverlay,\n DialogPortal,\n DialogTitle,\n DialogTrigger,\n};\n" }, { "path": "frontend/src/components/ui/input.tsx", "content": "import * as React from 'react';\n\nimport { cn } from '@/lib/utils';\n\nfunction Input({ className, type, ...props }: React.ComponentProps<'input'>) {\n return (\n \n );\n}\n\nexport { Input };\n" }, { "path": "frontend/src/components/ui/label.tsx", "content": "import { Label as LabelPrimitive } from 'radix-ui';\nimport * as React from 'react';\n\nimport { cn } from '@/lib/utils';\n\nfunction Label({\n className,\n ...props\n}: React.ComponentProps) {\n return (\n \n );\n}\n\nexport { Label };\n" }, { "path": "frontend/src/components/ui/multi-select.tsx", "content": "'use client';\n\nimport { Check, ChevronsUpDown, X } from 'lucide-react';\nimport * as React from 'react';\n\nimport { Button } from '@/components/ui/button';\nimport {\n Command,\n CommandEmpty,\n CommandGroup,\n CommandInput,\n CommandItem,\n CommandList,\n} from '@/components/ui/command';\nimport {\n Popover,\n PopoverContent,\n PopoverTrigger,\n} from '@/components/ui/popover';\nimport { cn } from '@/lib/utils';\n\nexport interface MultiSelectOption {\n value: string;\n label: string;\n}\n\ninterface MultiSelectProps {\n options: MultiSelectOption[];\n selected: string[];\n onChange: (selected: string[]) => void;\n placeholder?: string;\n emptyText?: string;\n searchPlaceholder?: string;\n className?: string;\n}\n\nexport function MultiSelect({\n options,\n selected,\n onChange,\n placeholder = 'Select items...',\n emptyText = 'No results found.',\n searchPlaceholder = 'Search...',\n className,\n}: MultiSelectProps) {\n const [open, setOpen] = React.useState(false);\n\n const handleSelect = (value: string) => {\n const newSelected = selected.includes(value)\n ? selected.filter((item) => item !== value)\n : [...selected, value];\n onChange(newSelected);\n };\n\n const handleRemove = (value: string, e: React.MouseEvent) => {\n e.preventDefault();\n e.stopPropagation();\n onChange(selected.filter((item) => item !== value));\n };\n\n const selectedLabels = options\n .filter((option) => selected.includes(option.value))\n .map((option) => option.label);\n\n return (\n \n \n \n
\n {selected.length === 0 ? (\n placeholder\n ) : (\n <>\n {selectedLabels.slice(0, 2).map((label) => {\n const option = options.find((o) => o.label === label);\n return (\n \n {label}\n {\n e.preventDefault();\n e.stopPropagation();\n }}\n onClick={(e) => handleRemove(option?.value || '', e)}\n onKeyDown={(e) => {\n if (e.key === 'Enter' || e.key === ' ') {\n e.preventDefault();\n handleRemove(\n option?.value || '',\n e as unknown as React.MouseEvent,\n );\n }\n }}\n >\n \n \n \n );\n })}\n {selected.length > 2 && (\n \n +{selected.length - 2} more\n \n )}\n \n )}\n
\n \n \n \n \n \n \n \n \n {emptyText}\n \n \n {options.map((option) => {\n const isSelected = selected.includes(option.value);\n return (\n handleSelect(option.value)}\n className=\"cursor-pointer dark:hover:bg-[#383838]\"\n >\n \n {isSelected && }\n \n {option.label}\n \n );\n })}\n \n \n \n \n \n );\n}\n" }, { "path": "frontend/src/components/ui/popover.tsx", "content": "import * as React from 'react';\n\nimport { cn } from '@/lib/utils';\nimport * as PopoverPrimitive from '@radix-ui/react-popover';\n\nfunction Popover({\n ...props\n}: React.ComponentProps) {\n return ;\n}\n\nfunction PopoverTrigger({\n ...props\n}: React.ComponentProps) {\n return ;\n}\n\nfunction PopoverContent({\n className,\n align = 'center',\n sideOffset = 4,\n ...props\n}: React.ComponentProps) {\n return (\n \n \n \n );\n}\n\nfunction PopoverAnchor({\n ...props\n}: React.ComponentProps) {\n return ;\n}\n\nexport { Popover, PopoverTrigger, PopoverContent, PopoverAnchor };\n" }, { "path": "frontend/src/components/ui/select.tsx", "content": "import { CheckIcon, ChevronDownIcon, ChevronUpIcon } from 'lucide-react';\nimport * as React from 'react';\n\nimport { cn } from '@/lib/utils';\nimport * as SelectPrimitive from '@radix-ui/react-select';\n\nfunction Select({\n ...props\n}: React.ComponentProps) {\n return ;\n}\n\nfunction SelectGroup({\n ...props\n}: React.ComponentProps) {\n return ;\n}\n\nfunction SelectValue({\n ...props\n}: React.ComponentProps) {\n return ;\n}\n\nfunction SelectTrigger({\n className,\n size = 'default',\n variant = 'default',\n children,\n ...props\n}: React.ComponentProps & {\n size?: 'sm' | 'default' | 'lg';\n variant?: 'default' | 'ghost';\n}) {\n return (\n \n {children}\n \n \n \n \n );\n}\n\nfunction SelectContent({\n className,\n children,\n position = 'item-aligned',\n align = 'center',\n ...props\n}: React.ComponentProps) {\n return (\n \n \n \n \n {children}\n \n \n \n \n );\n}\n\nfunction SelectLabel({\n className,\n ...props\n}: React.ComponentProps) {\n return (\n \n );\n}\n\nfunction SelectItem({\n className,\n children,\n ...props\n}: React.ComponentProps) {\n return (\n \n \n \n \n \n \n {children}\n \n );\n}\n\nfunction SelectSeparator({\n className,\n ...props\n}: React.ComponentProps) {\n return (\n \n );\n}\n\nfunction SelectScrollUpButton({\n className,\n ...props\n}: React.ComponentProps) {\n return (\n \n \n \n );\n}\n\nfunction SelectScrollDownButton({\n className,\n ...props\n}: React.ComponentProps) {\n return (\n \n \n \n );\n}\n\nexport {\n Select,\n SelectContent,\n SelectGroup,\n SelectItem,\n SelectLabel,\n SelectScrollDownButton,\n SelectScrollUpButton,\n SelectSeparator,\n SelectTrigger,\n SelectValue,\n};\n" }, { "path": "frontend/src/components/ui/sheet.tsx", "content": "import * as React from \"react\"\nimport * as SheetPrimitive from \"@radix-ui/react-dialog\"\nimport { XIcon } from \"lucide-react\"\n\nimport { cn } from \"@/lib/utils\"\n\nfunction Sheet({ ...props }: React.ComponentProps) {\n return \n}\n\nfunction SheetTrigger({\n ...props\n}: React.ComponentProps) {\n return \n}\n\nfunction SheetClose({\n ...props\n}: React.ComponentProps) {\n return \n}\n\nfunction SheetPortal({\n ...props\n}: React.ComponentProps) {\n return \n}\n\nfunction SheetOverlay({\n className,\n ...props\n}: React.ComponentProps) {\n return (\n \n )\n}\n\nfunction SheetContent({\n className,\n children,\n side = \"right\",\n showCloseButton = true,\n ...props\n}: React.ComponentProps & {\n side?: \"top\" | \"right\" | \"bottom\" | \"left\"\n showCloseButton?: boolean\n}) {\n return (\n \n \n \n {children}\n {showCloseButton && (\n \n \n Close\n \n )}\n \n \n )\n}\n\nfunction SheetHeader({ className, ...props }: React.ComponentProps<\"div\">) {\n return (\n \n )\n}\n\nfunction SheetFooter({ className, ...props }: React.ComponentProps<\"div\">) {\n return (\n \n )\n}\n\nfunction SheetTitle({\n className,\n ...props\n}: React.ComponentProps) {\n return (\n \n )\n}\n\nfunction SheetDescription({\n className,\n ...props\n}: React.ComponentProps) {\n return (\n \n )\n}\n\nexport {\n Sheet,\n SheetTrigger,\n SheetClose,\n SheetContent,\n SheetHeader,\n SheetFooter,\n SheetTitle,\n SheetDescription,\n}\n" }, { "path": "frontend/src/constants/fileUpload.ts", "content": "export const AUDIO_FILE_ACCEPT: Record = {\n 'audio/mpeg': ['.mp3'],\n 'audio/mp4': ['.m4a'],\n 'audio/ogg': ['.ogg'],\n 'audio/wav': ['.wav'],\n 'audio/webm': ['.webm'],\n 'video/webm': ['.webm'],\n};\n\nexport const FILE_UPLOAD_ACCEPT: Record = {\n 'application/pdf': ['.pdf'],\n 'text/plain': ['.txt'],\n 'text/x-rst': ['.rst'],\n 'text/x-markdown': ['.md'],\n 'application/zip': ['.zip'],\n 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': [\n '.docx',\n ],\n 'application/json': ['.json'],\n 'text/csv': ['.csv'],\n 'text/html': ['.html'],\n 'application/epub+zip': ['.epub'],\n 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': [\n '.xlsx',\n ],\n 'application/vnd.openxmlformats-officedocument.presentationml.presentation': [\n '.pptx',\n ],\n 'image/png': ['.png'],\n 'image/jpeg': ['.jpeg'],\n 'image/jpg': ['.jpg'],\n ...AUDIO_FILE_ACCEPT,\n};\n\nexport const FILE_UPLOAD_ACCEPT_ATTR = [\n '.pdf',\n '.txt',\n '.rst',\n '.md',\n '.zip',\n '.docx',\n '.json',\n '.csv',\n '.html',\n '.epub',\n '.xlsx',\n '.pptx',\n '.png',\n '.jpeg',\n '.jpg',\n '.wav',\n '.mp3',\n '.m4a',\n '.ogg',\n '.webm',\n].join(',');\n\nexport const AUDIO_FILE_ACCEPT_ATTR = [\n '.wav',\n '.mp3',\n '.m4a',\n '.ogg',\n '.webm',\n].join(',');\n\nexport const SOURCE_FILE_TREE_ACCEPT_ATTR = [\n '.rst',\n '.md',\n '.pdf',\n '.txt',\n '.docx',\n '.csv',\n '.epub',\n '.html',\n '.mdx',\n '.json',\n '.xlsx',\n '.pptx',\n '.png',\n '.jpg',\n '.jpeg',\n].join(',');\n" }, { "path": "frontend/src/conversation/Conversation.tsx", "content": "import { useCallback, useEffect, useRef, useState } from 'react';\nimport { useTranslation } from 'react-i18next';\nimport { useDispatch, useSelector } from 'react-redux';\n\nimport SharedAgentCard from '../agents/SharedAgentCard';\nimport ArtifactSidebar from '../components/ArtifactSidebar';\nimport MessageInput from '../components/MessageInput';\nimport { useMediaQuery } from '../hooks';\nimport {\n selectConversationId,\n selectSelectedAgent,\n selectToken,\n} from '../preferences/preferenceSlice';\nimport { AppDispatch } from '../store';\nimport { handleSendFeedback } from './conversationHandlers';\nimport ConversationMessages from './ConversationMessages';\nimport { FEEDBACK, Query } from './conversationModels';\nimport { ToolCallsType } from './types';\nimport {\n addQuery,\n fetchAnswer,\n resendQuery,\n selectQueries,\n selectStatus,\n updateQuery,\n} from './conversationSlice';\nimport { selectCompletedAttachments } from '../upload/uploadSlice';\n\nexport default function Conversation() {\n const { t } = useTranslation();\n const { isMobile } = useMediaQuery();\n const dispatch = useDispatch();\n\n const token = useSelector(selectToken);\n const queries = useSelector(selectQueries);\n const status = useSelector(selectStatus);\n const conversationId = useSelector(selectConversationId);\n const selectedAgent = useSelector(selectSelectedAgent);\n const completedAttachments = useSelector(selectCompletedAttachments);\n\n const [lastQueryReturnedErr, setLastQueryReturnedErr] =\n useState(false);\n\n const lastAutoOpenedArtifactId = useRef(null);\n const didInitArtifactAutoOpen = useRef(false);\n const prevConversationId = useRef(conversationId);\n\n const [openArtifact, setOpenArtifact] = useState<{\n id: string;\n toolName: string;\n } | null>(null);\n\n useEffect(() => {\n const prevId = prevConversationId.current;\n // Don't reset when the backend assigns the conversation id mid-stream (null -> id)\n const isServerAssignedId =\n prevId === null && conversationId !== null && status === 'loading';\n\n if (!isServerAssignedId && prevId !== conversationId) {\n setOpenArtifact(null);\n lastAutoOpenedArtifactId.current = null;\n }\n\n prevConversationId.current = conversationId;\n }, [conversationId, status]);\n\n const handleFetchAnswer = useCallback(\n ({ question, index }: { question: string; index?: number }) => {\n dispatch(fetchAnswer({ question, indx: index }));\n },\n [dispatch, selectedAgent],\n );\n\n const handleQuestion = useCallback(\n ({\n question,\n isRetry = false,\n index = undefined,\n }: {\n question: string;\n isRetry?: boolean;\n index?: number;\n }) => {\n const trimmedQuestion = question.trim();\n if (trimmedQuestion === '') return;\n\n const filesAttached = completedAttachments\n .filter((a) => a.id)\n .map((a) => ({ id: a.id as string, fileName: a.fileName }));\n\n if (index !== undefined) {\n dispatch(resendQuery({ index, prompt: trimmedQuestion }));\n handleFetchAnswer({ question: trimmedQuestion, index });\n } else {\n if (!isRetry)\n dispatch(\n addQuery({\n prompt: trimmedQuestion,\n attachments: filesAttached,\n }),\n );\n handleFetchAnswer({ question: trimmedQuestion, index });\n }\n },\n [dispatch, handleFetchAnswer, completedAttachments],\n );\n\n const handleFeedback = (query: Query, feedback: FEEDBACK, index: number) => {\n const prevFeedback = query.feedback;\n dispatch(updateQuery({ index, query: { feedback } }));\n handleSendFeedback(\n query.prompt,\n query.response!,\n feedback,\n conversationId as string,\n index,\n token,\n ).catch(() =>\n handleSendFeedback(\n query.prompt,\n query.response!,\n feedback,\n conversationId as string,\n index,\n token,\n ).catch(() =>\n dispatch(updateQuery({ index, query: { feedback: prevFeedback } })),\n ),\n );\n };\n\n const handleQuestionSubmission = (\n question?: string,\n updated?: boolean,\n indx?: number,\n ) => {\n if (updated === true) {\n handleQuestion({ question: question as string, index: indx });\n } else if (question && status !== 'loading') {\n if (lastQueryReturnedErr && queries.length > 0) {\n const retryIndex = queries.length - 1;\n dispatch(\n updateQuery({\n index: retryIndex,\n query: {\n prompt: question,\n },\n }),\n );\n handleQuestion({\n question,\n isRetry: true,\n index: retryIndex,\n });\n } else {\n handleQuestion({\n question,\n });\n }\n }\n };\n\n useEffect(() => {\n if (queries.length) {\n const last = queries[queries.length - 1];\n if (last.error) setLastQueryReturnedErr(true);\n if (last.response) setLastQueryReturnedErr(false);\n }\n }, [queries]);\n\n useEffect(() => {\n // Avoid auto-opening an artifact from existing conversation history on first mount.\n if (!didInitArtifactAutoOpen.current) {\n didInitArtifactAutoOpen.current = true;\n return;\n }\n\n const isNotesOrTodoTool = (toolName?: string) => {\n const t = (toolName ?? '').toLowerCase();\n return t === 'notes' || t === 'todo_list' || t === 'todo';\n };\n\n const findLatestCompletedArtifactCall = (\n items: Query[],\n ): ToolCallsType | null => {\n for (let i = items.length - 1; i >= 0; i -= 1) {\n const calls = items[i].tool_calls ?? [];\n for (let j = calls.length - 1; j >= 0; j -= 1) {\n const call = calls[j];\n if (call.artifact_id && call.status === 'completed') return call;\n }\n }\n return null;\n };\n\n const latest = findLatestCompletedArtifactCall(queries);\n if (!latest?.artifact_id) return;\n if (!isNotesOrTodoTool(latest.tool_name)) return;\n if (latest.artifact_id === lastAutoOpenedArtifactId.current) return;\n\n lastAutoOpenedArtifactId.current = latest.artifact_id;\n setOpenArtifact({\n id: latest.artifact_id,\n toolName: latest.tool_name,\n });\n }, [queries]);\n\n const handleOpenArtifact = useCallback(\n (artifact: { id: string; toolName: string }) => {\n lastAutoOpenedArtifactId.current = artifact.id;\n setOpenArtifact(artifact);\n },\n [],\n );\n\n const handleCloseArtifact = useCallback(() => setOpenArtifact(null), []);\n\n const isSplitArtifactOpen = !isMobile && openArtifact !== null;\n\n return (\n
\n \n
\n \n \n
\n ) : undefined\n }\n />\n
\n\n \n
\n {\n handleQuestionSubmission(text);\n }}\n loading={status === 'loading'}\n showSourceButton={selectedAgent ? false : true}\n showToolButton={selectedAgent ? false : true}\n />\n
\n\n \n \n \n\n {isSplitArtifactOpen && (\n
\n \n
\n )}\n\n {isMobile && (\n \n )}\n \n );\n}\n" }, { "path": "frontend/src/conversation/ConversationBubble.module.css", "content": ".list p {\n display: inline;\n}\n\n.list li:not(:first-child) {\n margin-top: 0.5em;\n}\n\n.list li > .list {\n margin-top: 0.5em;\n}\n" }, { "path": "frontend/src/conversation/ConversationBubble.tsx", "content": "import 'katex/dist/katex.min.css';\n\nimport { forwardRef, Fragment, useEffect, useRef, useState } from 'react';\nimport { useTranslation } from 'react-i18next';\nimport ReactMarkdown from 'react-markdown';\nimport { useSelector } from 'react-redux';\nimport { Prism as SyntaxHighlighter } from 'react-syntax-highlighter';\nimport {\n oneLight,\n vscDarkPlus,\n} from 'react-syntax-highlighter/dist/cjs/styles/prism';\nimport rehypeKatex from 'rehype-katex';\nimport remarkGfm from 'remark-gfm';\nimport remarkMath from 'remark-math';\n\nimport ChevronDown from '../assets/chevron-down.svg';\nimport Cloud from '../assets/cloud.svg';\nimport DocsGPT3 from '../assets/cute_docsgpt3.svg';\nimport Dislike from '../assets/dislike.svg?react';\nimport Document from '../assets/document.svg';\nimport DocumentationDark from '../assets/documentation-dark.svg';\nimport Edit from '../assets/edit.svg';\nimport Like from '../assets/like.svg?react';\nimport Link from '../assets/link.svg';\nimport Sources from '../assets/sources.svg';\nimport UserIcon from '../assets/user.svg';\nimport Accordion from '../components/Accordion';\nimport Avatar from '../components/Avatar';\nimport CopyButton from '../components/CopyButton';\nimport MermaidRenderer from '../components/MermaidRenderer';\nimport Sidebar from '../components/Sidebar';\nimport Spinner from '../components/Spinner';\nimport SpeakButton from '../components/TextToSpeechButton';\nimport { useDarkTheme, useOutsideAlerter } from '../hooks';\nimport {\n selectChunks,\n selectSelectedDocs,\n} from '../preferences/preferenceSlice';\nimport classes from './ConversationBubble.module.css';\nimport { FEEDBACK, MESSAGE_TYPE } from './conversationModels';\nimport { ToolCallsType } from './types';\n\nconst DisableSourceFE = import.meta.env.VITE_DISABLE_SOURCE_FE || false;\n\nconst ConversationBubble = forwardRef<\n HTMLDivElement,\n {\n message?: string;\n type: MESSAGE_TYPE;\n className?: string;\n feedback?: FEEDBACK;\n handleFeedback?: (feedback: FEEDBACK) => void;\n thought?: string;\n sources?: { title: string; text: string; link: string }[];\n toolCalls?: ToolCallsType[];\n retryBtn?: React.ReactElement;\n questionNumber?: number;\n isStreaming?: boolean;\n handleUpdatedQuestionSubmission?: (\n updatedquestion?: string,\n updated?: boolean,\n index?: number,\n ) => void;\n filesAttached?: { id: string; fileName: string }[];\n onOpenArtifact?: (artifact: { id: string; toolName: string }) => void;\n }\n>(function ConversationBubble(\n {\n message,\n type,\n className,\n feedback,\n handleFeedback,\n thought,\n sources,\n toolCalls,\n retryBtn,\n questionNumber,\n isStreaming,\n handleUpdatedQuestionSubmission,\n filesAttached,\n onOpenArtifact,\n },\n ref,\n) {\n const { t } = useTranslation();\n const [isDarkTheme] = useDarkTheme();\n // const bubbleRef = useRef(null);\n const chunks = useSelector(selectChunks);\n const selectedDocs = useSelector(selectSelectedDocs);\n const [isEditClicked, setIsEditClicked] = useState(false);\n const [editInputBox, setEditInputBox] = useState('');\n const messageRef = useRef(null);\n const [shouldShowToggle, setShouldShowToggle] = useState(false);\n\n const [activeTooltip, setActiveTooltip] = useState(null);\n const [isSidebarOpen, setIsSidebarOpen] = useState(false);\n const editableQueryRef = useRef(null);\n const [isQuestionCollapsed, setIsQuestionCollapsed] = useState(true);\n\n const completedArtifactCalls = (toolCalls ?? []).filter(\n (toolCall) => toolCall.artifact_id && toolCall.status === 'completed',\n );\n const primaryArtifactCall =\n completedArtifactCalls[completedArtifactCalls.length - 1] ?? null;\n const artifactCount = completedArtifactCalls.length;\n\n const formatToolName = (toolName: string | undefined): string => {\n if (!toolName) return '';\n return toolName\n .split('_')\n .map((word) => word.charAt(0).toUpperCase() + word.slice(1).toLowerCase())\n .join(' ');\n };\n\n useOutsideAlerter(editableQueryRef, () => setIsEditClicked(false), [], true);\n\n useEffect(() => {\n if (messageRef.current) {\n const height = messageRef.current.scrollHeight;\n setShouldShowToggle(height > 84);\n }\n }, [message]);\n\n const handleEditClick = () => {\n setIsEditClicked(false);\n handleUpdatedQuestionSubmission?.(editInputBox, true, questionNumber);\n };\n let bubble;\n if (type === 'QUESTION') {\n bubble = (\n
\n
\n {filesAttached && filesAttached.length > 0 && (\n
\n {filesAttached.map((file, index) => (\n \n
\n \n
\n \n {file.fileName}\n \n
\n ))}\n
\n )}\n \n \n }\n />\n {!isEditClicked && (\n <>\n
\n
\n \n {message}\n
\n {shouldShowToggle && (\n {\n e.stopPropagation();\n setIsQuestionCollapsed(!isQuestionCollapsed);\n }}\n className=\"ml-1 rounded-full p-2 hover:bg-[#D9D9D933]\"\n >\n \n \n )}\n
\n
\n {\n setIsEditClicked(true);\n setEditInputBox(message ?? '');\n }}\n className={`hover:bg-light-silver mt-3 flex h-fit shrink-0 cursor-pointer items-center rounded-full p-2 pt-1.5 pl-1.5 dark:hover:bg-[#35363B] ${isEditClicked ? 'visible' : 'invisible group-hover:visible'}`}\n >\n \"Edit\"\n \n \n )}\n \n {isEditClicked && (\n \n {\n setEditInputBox(e.target.value);\n }}\n onKeyDown={(e) => {\n if (e.key === 'Enter' && !e.shiftKey) {\n e.preventDefault();\n handleEditClick();\n }\n }}\n rows={5}\n value={editInputBox}\n className=\"border-silver text-carbon dark:border-philippine-grey dark:bg-raisin-black dark:text-chinese-white w-full resize-none rounded-3xl border px-4 py-3 text-base leading-relaxed focus:outline-hidden\"\n />\n
\n setIsEditClicked(false)}\n >\n {t('conversation.edit.cancel')}\n \n \n {t('conversation.edit.update')}\n \n
\n \n )}\n \n \n );\n } else {\n const preprocessLaTeX = (content: string) => {\n // Replace block-level LaTeX delimiters \\[ \\] with $$ $$\n const blockProcessedContent = content.replace(\n /\\\\\\[(.*?)\\\\\\]/gs,\n (_, equation) => `$$${equation}$$`,\n );\n\n // Replace inline LaTeX delimiters \\( \\) with $ $\n const inlineProcessedContent = blockProcessedContent.replace(\n /\\\\\\((.*?)\\\\\\)/gs,\n (_, equation) => `$${equation}$`,\n );\n\n return inlineProcessedContent;\n };\n const processMarkdownContent = (content: string) => {\n const processedContent = preprocessLaTeX(content);\n\n const contentSegments: Array<{\n type: 'text' | 'mermaid';\n content: string;\n }> = [];\n\n let lastIndex = 0;\n const regex = /```mermaid\\n([\\s\\S]*?)```/g;\n let match;\n\n while ((match = regex.exec(processedContent)) !== null) {\n const textBefore = processedContent.substring(lastIndex, match.index);\n if (textBefore) {\n contentSegments.push({ type: 'text', content: textBefore });\n }\n\n contentSegments.push({ type: 'mermaid', content: match[1].trim() });\n\n lastIndex = match.index + match[0].length;\n }\n\n const textAfter = processedContent.substring(lastIndex);\n if (textAfter) {\n contentSegments.push({ type: 'text', content: textAfter });\n }\n\n return contentSegments;\n };\n bubble = (\n \n {DisableSourceFE ||\n type === 'ERROR' ||\n sources?.length === 0 ||\n sources?.some((source) => source.link === 'None')\n ? null\n : sources && (\n
\n
\n \n }\n />\n

\n {t('conversation.sources.title')}\n

\n
\n
\n
\n {sources?.slice(0, 3)?.map((source, index) => (\n
\n setActiveTooltip(index)}\n onMouseOut={() => setActiveTooltip(null)}\n >\n

\n {source.text}\n

\n \n source.link && source.link !== 'local'\n ? window.open(\n source.link,\n '_blank',\n 'noopener, noreferrer',\n )\n : null\n }\n >\n \n \n {source.link && source.link !== 'local'\n ? source.link\n : source.title}\n

\n
\n
\n {activeTooltip === index && (\n setActiveTooltip(index)}\n onMouseOut={() => setActiveTooltip(null)}\n >\n

\n {source.text}\n

\n
\n )}\n
\n ))}\n {(sources?.length ?? 0) > 3 && (\n setIsSidebarOpen(true)}\n >\n

\n {t('conversation.sources.view_more', {\n count: sources?.length ? sources.length - 3 : 0,\n })}\n

\n \n )}\n \n \n \n )}\n {toolCalls && toolCalls.length > 0 && (\n \n )}\n {!message && primaryArtifactCall?.artifact_id && onOpenArtifact && (\n
\n \n onOpenArtifact({\n id: primaryArtifactCall.artifact_id!,\n toolName: primaryArtifactCall.tool_name,\n })\n }\n className=\"flex items-center gap-2 rounded-full bg-purple-100 px-3 py-2 text-sm font-medium text-purple-700 transition-colors hover:bg-purple-200 dark:bg-purple-900/30 dark:text-purple-300 dark:hover:bg-purple-900/50\"\n >\n \n \n \n \n {primaryArtifactCall.tool_name\n ? formatToolName(primaryArtifactCall.tool_name)\n : artifactCount > 1\n ? `View artifacts (${artifactCount})`\n : 'View artifact'}\n \n
\n )}\n {thought && (\n \n )}\n {message && (\n
\n
\n \n }\n />\n

\n {t('conversation.answer')}\n

\n
\n \n {(() => {\n const contentSegments = processMarkdownContent(message);\n return (\n <>\n {contentSegments.map((segment, index) => (\n \n {segment.type === 'text' ? (\n \n
\n \n {language}\n \n \n
\n \n {String(children).replace(/\\n$/, '')}\n \n
\n ) : (\n \n {children}\n \n );\n },\n ul({ children }) {\n return (\n \n {children}\n \n );\n },\n ol({ children }) {\n return (\n \n {children}\n \n );\n },\n table({ children }) {\n return (\n
\n \n {children}\n
\n
\n );\n },\n thead({ children }) {\n return (\n \n {children}\n \n );\n },\n tr({ children }) {\n return (\n \n {children}\n \n );\n },\n th({ children }) {\n return (\n {children}\n );\n },\n td({ children }) {\n return (\n {children}\n );\n },\n }}\n >\n {segment.content}\n \n ) : (\n \n \n \n )}\n \n ))}\n \n );\n })()}\n \n \n )}\n {message && (\n
\n {type === 'ERROR' ? (\n
\n
{retryBtn}
\n
\n ) : (\n <>\n {primaryArtifactCall?.artifact_id && onOpenArtifact && (\n
\n \n onOpenArtifact({\n id: primaryArtifactCall.artifact_id!,\n toolName: primaryArtifactCall.tool_name,\n })\n }\n className=\"flex items-center gap-2 rounded-full bg-purple-100 px-3 py-2 text-sm font-medium text-purple-700 transition-colors hover:bg-purple-200 dark:bg-purple-900/30 dark:text-purple-300 dark:hover:bg-purple-900/50\"\n aria-label=\"View artifacts\"\n >\n \n \n \n \n {primaryArtifactCall.tool_name\n ? formatToolName(primaryArtifactCall.tool_name)\n : artifactCount > 1\n ? `Artifacts (${artifactCount})`\n : 'Artifact'}\n \n
\n )}\n {!isStreaming && (\n <>\n
\n \n
\n
\n \n
\n {handleFeedback && (\n <>\n
\n {\n if (feedback === 'LIKE') {\n handleFeedback?.(null);\n } else {\n handleFeedback?.('LIKE');\n }\n }}\n aria-label={\n feedback === 'LIKE' ? 'Remove like' : 'Like'\n }\n >\n \n \n
\n\n
\n {\n if (feedback === 'DISLIKE') {\n handleFeedback?.(null);\n } else {\n handleFeedback?.('DISLIKE');\n }\n }}\n aria-label={\n feedback === 'DISLIKE'\n ? 'Remove dislike'\n : 'Dislike'\n }\n >\n \n \n
\n \n )}\n \n )}\n \n )}\n
\n )}\n {sources && (\n {\n setIsSidebarOpen(state);\n }}\n >\n \n \n )}\n \n );\n }\n return bubble;\n});\n\ntype AllSourcesProps = {\n sources: { title: string; text: string; link?: string }[];\n};\n\nfunction AllSources(sources: AllSourcesProps) {\n const { t } = useTranslation();\n\n const handleCardClick = (link: string) => {\n if (link && link !== 'local') {\n window.open(link, '_blank', 'noopener,noreferrer');\n }\n };\n\n return (\n
\n
\n

{`${sources.sources.length} ${t('conversation.sources.title')}`}

\n
\n
\n
\n {sources.sources.map((source, index) => {\n const isExternalSource = source.link && source.link !== 'local';\n return (\n \n isExternalSource && source.link && handleCardClick(source.link)\n }\n >\n \n {`${index + 1}. ${source.title}`}\n {isExternalSource && (\n \n )}\n

\n

\n {source.text}\n

\n
\n );\n })}\n
\n \n );\n}\nexport default ConversationBubble;\n\nfunction ToolCalls({ toolCalls }: { toolCalls: ToolCallsType[] }) {\n const [isToolCallsOpen, setIsToolCallsOpen] = useState(false);\n\n return (\n\t\t
\n
\n \n }\n />\n setIsToolCallsOpen(!isToolCallsOpen)}\n >\n

Tool Calls

\n \n \n
\n {isToolCallsOpen && (\n
\n
\n {toolCalls.map((toolCall, index) => (\n \n
\n
\n

\n \n Arguments\n {' '}\n \n

\n

\n \n {JSON.stringify(toolCall.arguments, null, 2)}\n \n

\n
\n
\n

\n \n Response\n {' '}\n \n

\n {toolCall.status === 'pending' && (\n \n \n \n )}\n {toolCall.status === 'completed' && (\n

\n \n {JSON.stringify(toolCall.result, null, 2)}\n \n

\n )}\n {toolCall.status === 'error' && (\n

\n \n {toolCall.error}\n \n

\n )}\n
\n
\n \n ))}\n
\n
\n )}\n\t\t
\n );\n}\n\nfunction Thought({\n thought,\n preprocessLaTeX,\n}: {\n thought: string;\n preprocessLaTeX: (content: string) => string;\n}) {\n const { t } = useTranslation();\n const [isDarkTheme] = useDarkTheme();\n const [isThoughtOpen, setIsThoughtOpen] = useState(false);\n\n return (\n
\n
\n \n }\n />\n setIsThoughtOpen(!isThoughtOpen)}\n >\n

\n {t('conversation.reasoning')}\n

\n \n \n
\n {isThoughtOpen && (\n
\n
\n \n
\n \n {language}\n \n \n
\n \n {String(children).replace(/\\n$/, '')}\n \n
\n ) : (\n \n {children}\n \n );\n },\n ul({ children }) {\n return (\n
    \n {children}\n
\n );\n },\n ol({ children }) {\n return (\n
    \n {children}\n
\n );\n },\n table({ children }) {\n return (\n
\n \n {children}\n
\n
\n );\n },\n thead({ children }) {\n return (\n \n {children}\n \n );\n },\n tr({ children }) {\n return (\n \n {children}\n \n );\n },\n th({ children }) {\n return {children};\n },\n td({ children }) {\n return {children};\n },\n }}\n >\n {preprocessLaTeX(thought ?? '')}\n \n
\n
\n )}\n \n );\n}\n" }, { "path": "frontend/src/conversation/ConversationMessages.tsx", "content": "import {\n Fragment,\n ReactNode,\n useCallback,\n useEffect,\n useRef,\n useState,\n} from 'react';\nimport { useTranslation } from 'react-i18next';\n\nimport ArrowDown from '../assets/arrow-down.svg';\nimport DocsGPT3 from '../assets/cute_docsgpt3.svg';\nimport RetryIcon from '../components/RetryIcon';\nimport Hero from '../Hero';\nimport { useDarkTheme } from '../hooks';\nimport ConversationBubble from './ConversationBubble';\nimport { FEEDBACK, Query, Status } from './conversationModels';\n\nconst SCROLL_THRESHOLD = 10;\nconst LAST_BUBBLE_MARGIN = 'mb-32';\nconst DEFAULT_BUBBLE_MARGIN = 'mb-7';\nconst FIRST_QUESTION_BUBBLE_MARGIN_TOP = 'mt-5';\n\ntype ConversationMessagesProps = {\n handleQuestion: (params: {\n question: string;\n isRetry?: boolean;\n index?: number;\n }) => void;\n handleQuestionSubmission: (\n updatedQuestion?: string,\n updated?: boolean,\n index?: number,\n ) => void;\n handleFeedback?: (query: Query, feedback: FEEDBACK, index: number) => void;\n queries: Query[];\n status: Status;\n showHeroOnEmpty?: boolean;\n headerContent?: ReactNode;\n onOpenArtifact?: (artifact: { id: string; toolName: string }) => void;\n isSplitView?: boolean;\n};\n\nexport default function ConversationMessages({\n handleQuestion,\n handleQuestionSubmission,\n queries,\n status,\n handleFeedback,\n showHeroOnEmpty = true,\n headerContent,\n onOpenArtifact,\n isSplitView = false,\n}: ConversationMessagesProps) {\n const [isDarkTheme] = useDarkTheme();\n const { t } = useTranslation();\n\n const conversationRef = useRef(null);\n const [hasScrolledToLast, setHasScrolledToLast] = useState(true);\n const [userInterruptedScroll, setUserInterruptedScroll] = useState(false);\n\n const handleUserScrollInterruption = useCallback(() => {\n if (!userInterruptedScroll && status === 'loading') {\n setUserInterruptedScroll(true);\n }\n }, [userInterruptedScroll, status]);\n\n const scrollConversationToBottom = useCallback(() => {\n if (!conversationRef.current || userInterruptedScroll) return;\n\n requestAnimationFrame(() => {\n if (!conversationRef?.current) return;\n\n if (status === 'idle' || !queries[queries.length - 1]?.response) {\n conversationRef.current.scrollTo({\n behavior: 'smooth',\n top: conversationRef.current.scrollHeight,\n });\n } else {\n conversationRef.current.scrollTop =\n conversationRef.current.scrollHeight;\n }\n });\n }, [userInterruptedScroll, status, queries]);\n\n const checkScrollPosition = useCallback(() => {\n const el = conversationRef.current;\n if (!el) return;\n const isAtBottom =\n el.scrollHeight - el.scrollTop - el.clientHeight < SCROLL_THRESHOLD;\n setHasScrolledToLast(isAtBottom);\n }, [setHasScrolledToLast]);\n\n const lastQuery = queries[queries.length - 1];\n const lastQueryResponse = lastQuery?.response;\n const lastQueryError = lastQuery?.error;\n const lastQueryThought = lastQuery?.thought;\n\n useEffect(() => {\n if (!userInterruptedScroll) {\n scrollConversationToBottom();\n }\n }, [\n queries.length,\n lastQueryResponse,\n lastQueryError,\n lastQueryThought,\n userInterruptedScroll,\n scrollConversationToBottom,\n ]);\n\n useEffect(() => {\n if (status === 'idle') {\n setUserInterruptedScroll(false);\n }\n }, [status]);\n\n useEffect(() => {\n const currentConversationRef = conversationRef.current;\n currentConversationRef?.addEventListener('scroll', checkScrollPosition);\n return () => {\n currentConversationRef?.removeEventListener(\n 'scroll',\n checkScrollPosition,\n );\n };\n }, [checkScrollPosition]);\n\n const retryIconProps = {\n width: 12,\n height: 12,\n fill: isDarkTheme ? 'rgb(236 236 241)' : 'rgb(107 114 120)',\n stroke: isDarkTheme ? 'rgb(236 236 241)' : 'rgb(107 114 120)',\n strokeWidth: 10,\n };\n\n const renderResponseView = (query: Query, index: number) => {\n const isLastMessage = index === queries.length - 1;\n const bubbleMargin = isLastMessage\n ? LAST_BUBBLE_MARGIN\n : DEFAULT_BUBBLE_MARGIN;\n\n if (query.thought || query.response || query.tool_calls) {\n const isCurrentlyStreaming =\n status === 'loading' && index === queries.length - 1;\n return (\n handleFeedback(query, feedback, index)\n : undefined\n }\n />\n );\n }\n\n if (query.error) {\n const retryButton = (\n {\n const questionToRetry = queries[index].prompt;\n handleQuestion({\n question: questionToRetry,\n isRetry: true,\n index,\n });\n }}\n aria-label={t('Retry') || 'Retry'}\n >\n \n \n );\n return (\n \n );\n }\n\n if (status === 'loading' && isLastMessage) {\n return (\n \n
\n
\n
\n \n
\n

\n {t('conversation.answer')}\n

\n
\n
\n
\n \n \n \n
\n
\n
\n \n );\n }\n\n return null;\n };\n\n return (\n \n {queries.length > 0 && !hasScrolledToLast && (\n {\n setUserInterruptedScroll(false);\n scrollConversationToBottom();\n }}\n aria-label={t('Scroll to bottom') || 'Scroll to bottom'}\n className=\"border-gray-alpha bg-opacity-50 dark:bg-gunmetal md:bg-opacity-100 fixed right-14 bottom-40 z-10 flex h-7 w-7 items-center justify-center rounded-full border-[0.5px] bg-gray-100 md:h-9 md:w-9\"\n >\n \n \n )}\n\n \n {headerContent}\n\n {queries.length > 0 ? (\n queries.map((query, index) => (\n \n \n {renderResponseView(query, index)}\n \n ))\n ) : showHeroOnEmpty ? (\n \n ) : null}\n \n \n );\n}\n" }, { "path": "frontend/src/conversation/ConversationTile.tsx", "content": "import {\n SyntheticEvent,\n useEffect,\n useRef,\n useState,\n useCallback,\n} from 'react';\nimport { useSelector } from 'react-redux';\nimport Edit from '../assets/edit.svg';\nimport Exit from '../assets/exit.svg';\nimport { useDarkTheme } from '../hooks';\nimport ConfirmationModal from '../modals/ConfirmationModal';\nimport CheckMark2 from '../assets/checkMark2.svg';\nimport Trash from '../assets/red-trash.svg';\nimport Share from '../assets/share.svg';\nimport threeDots from '../assets/three-dots.svg';\nimport { selectConversationId } from '../preferences/preferenceSlice';\nimport { ActiveState } from '../models/misc';\nimport { ShareConversationModal } from '../modals/ShareConversationModal';\nimport { useTranslation } from 'react-i18next';\nimport ContextMenu from '../components/ContextMenu';\nimport { MenuOption } from '../components/ContextMenu';\nimport { useOutsideAlerter } from '../hooks';\n\ninterface ConversationProps {\n name: string;\n id: string;\n}\ninterface ConversationTileProps {\n conversation: ConversationProps;\n selectConversation: (arg1: string) => void;\n onConversationClick: () => void; //Callback to handle click on conversation tile regardless of selected or not\n onDeleteConversation: (arg1: string) => void;\n onSave: ({ name, id }: ConversationProps) => void;\n}\n\nexport default function ConversationTile({\n conversation,\n selectConversation,\n onConversationClick,\n onDeleteConversation,\n onSave,\n}: ConversationTileProps) {\n const conversationId = useSelector(selectConversationId);\n const tileRef = useRef(null);\n const [isDarkTheme] = useDarkTheme();\n const [isEdit, setIsEdit] = useState(false);\n const [conversationName, setConversationsName] = useState('');\n const [isOpen, setOpen] = useState(false);\n const [isShareModalOpen, setShareModalState] = useState(false);\n const [isHovered, setIsHovered] = useState(false);\n const [deleteModalState, setDeleteModalState] =\n useState('INACTIVE');\n const menuRef = useRef(null);\n const { t } = useTranslation();\n useEffect(() => {\n setConversationsName(conversation.name);\n }, [conversation.name]);\n\n function handleEditConversation(event: SyntheticEvent) {\n event.stopPropagation();\n setIsEdit(true);\n setOpen(false);\n }\n\n function handleSaveConversation(changedConversation: ConversationProps) {\n if (changedConversation.name.trim().length) {\n onSave(changedConversation);\n setIsEdit(false);\n } else {\n onClear();\n }\n }\n\n const handleClickOutside = (event: MouseEvent) => {\n if (menuRef.current && !menuRef.current.contains(event.target as Node)) {\n setOpen(false);\n }\n };\n\n useEffect(() => {\n document.addEventListener('mousedown', handleClickOutside);\n return () => {\n document.removeEventListener('mousedown', handleClickOutside);\n };\n }, []);\n\n const preventScroll = useCallback((event: WheelEvent | TouchEvent) => {\n event.preventDefault();\n }, []);\n\n useEffect(() => {\n const conversationsMainDiv = document.getElementById(\n 'conversationsMainDiv',\n );\n\n if (conversationsMainDiv) {\n if (isOpen) {\n conversationsMainDiv.addEventListener('wheel', preventScroll, {\n passive: false,\n });\n conversationsMainDiv.addEventListener('touchmove', preventScroll, {\n passive: false,\n });\n } else {\n conversationsMainDiv.removeEventListener('wheel', preventScroll);\n conversationsMainDiv.removeEventListener('touchmove', preventScroll);\n }\n\n return () => {\n conversationsMainDiv.removeEventListener('wheel', preventScroll);\n conversationsMainDiv.removeEventListener('touchmove', preventScroll);\n };\n }\n }, [isOpen]);\n\n function onClear() {\n setConversationsName(conversation.name);\n setIsEdit(false);\n }\n\n const handleRenameKeyDown = (e: React.KeyboardEvent) => {\n e.stopPropagation();\n if (e.key === 'Enter') {\n handleSaveConversation({\n id: conversation.id,\n name: conversationName,\n });\n } else if (e.key === 'Escape') {\n onClear();\n }\n };\n\n const menuOptions: MenuOption[] = [\n {\n icon: Share,\n label: t('convTile.share'),\n onClick: (event: SyntheticEvent) => {\n event.stopPropagation();\n setShareModalState(true);\n setOpen(false);\n },\n variant: 'primary',\n iconWidth: 14,\n iconHeight: 14,\n },\n {\n icon: Edit,\n label: t('convTile.rename'),\n onClick: handleEditConversation,\n variant: 'primary',\n },\n {\n icon: Trash,\n label: t('convTile.delete'),\n onClick: (event: SyntheticEvent) => {\n event.stopPropagation();\n setDeleteModalState('ACTIVE');\n setOpen(false);\n },\n iconWidth: 18,\n iconHeight: 18,\n variant: 'danger',\n },\n ];\n\n useOutsideAlerter(\n tileRef,\n () => {\n if (isEdit) {\n onClear();\n }\n },\n [isEdit],\n true,\n );\n\n return (\n <>\n {\n setIsHovered(true);\n }}\n onMouseLeave={() => {\n if (!isEdit) {\n setIsHovered(false);\n }\n }}\n onClick={() => {\n onConversationClick();\n conversationId !== conversation.id &&\n selectConversation(conversation.id);\n }}\n className={`hover:bg-bright-gray dark:hover:bg-dark-charcoal mx-4 my-auto mt-4 flex h-9 cursor-pointer items-center justify-between gap-4 rounded-3xl pl-4 ${\n conversationId === conversation.id || isOpen || isHovered || isEdit\n ? 'bg-bright-gray dark:bg-dark-charcoal'\n : ''\n }`}\n >\n
\n {isEdit ? (\n setConversationsName(e.target.value)}\n onKeyDown={handleRenameKeyDown}\n />\n ) : (\n

\n {conversationName}\n

\n )}\n
\n {(conversationId === conversation.id || isHovered || isOpen) && (\n
\n {isEdit ? (\n
\n {\n event.stopPropagation();\n handleSaveConversation({\n id: conversation.id,\n name: conversationName,\n });\n }}\n />\n {\n event.stopPropagation();\n onClear();\n }}\n />\n
\n ) : (\n {\n event.stopPropagation();\n setOpen(!isOpen);\n }}\n className=\"mr-2 flex h-6 w-6 items-center justify-center rounded-full transition-colors duration-200 hover:bg-gray-200 dark:hover:bg-gray-700\"\n >\n \"menu\"\n \n )}\n \n
\n )}\n \n onDeleteConversation(conversation.id)}\n submitLabel={t('convTile.delete')}\n />\n {isShareModalOpen && (\n {\n setShareModalState(false);\n isHovered && setIsHovered(false);\n }}\n conversationId={conversation.id}\n />\n )}\n \n );\n}\n" }, { "path": "frontend/src/conversation/SharedConversation.tsx", "content": "import { useEffect, useState } from 'react';\nimport { useTranslation } from 'react-i18next';\nimport { useDispatch, useSelector } from 'react-redux';\nimport { useNavigate, useParams } from 'react-router-dom';\n\nimport conversationService from '../api/services/conversationService';\nimport MessageInput from '../components/MessageInput';\nimport { selectToken } from '../preferences/preferenceSlice';\nimport { AppDispatch } from '../store';\nimport { formatDate } from '../utils/dateTimeUtils';\nimport ConversationMessages from './ConversationMessages';\nimport {\n addQuery,\n fetchSharedAnswer,\n selectClientAPIKey,\n selectDate,\n selectQueries,\n selectStatus,\n selectTitle,\n setClientApiKey,\n setFetchedData,\n setIdentifier,\n updateQuery,\n} from './sharedConversationSlice';\nimport { selectCompletedAttachments } from '../upload/uploadSlice';\nimport { Head as DocumentHead } from '../components/Head';\n\nexport const SharedConversation = () => {\n const navigate = useNavigate();\n const { identifier } = useParams(); //identifier is a uuid, not conversationId\n\n const token = useSelector(selectToken);\n const queries = useSelector(selectQueries);\n const title = useSelector(selectTitle);\n const date = useSelector(selectDate);\n const apiKey = useSelector(selectClientAPIKey);\n const status = useSelector(selectStatus);\n const completedAttachments = useSelector(selectCompletedAttachments);\n\n const { t } = useTranslation();\n const dispatch = useDispatch();\n\n const [lastQueryReturnedErr, setLastQueryReturnedErr] = useState(false);\n\n useEffect(() => {\n identifier && dispatch(setIdentifier(identifier));\n }, []);\n\n useEffect(() => {\n if (queries.length) {\n queries[queries.length - 1].error && setLastQueryReturnedErr(true);\n queries[queries.length - 1].response && setLastQueryReturnedErr(false); //considering a query that initially returned error can later include a response property on retry\n }\n }, [queries[queries.length - 1]]);\n\n const fetchQueries = () => {\n identifier &&\n conversationService\n .getSharedConversation(identifier || '', token)\n .then((res) => {\n if (res.status === 404 || res.status === 400)\n navigate('/pagenotfound');\n return res.json();\n })\n .then((data) => {\n if (data.success) {\n dispatch(\n setFetchedData({\n queries: data.queries,\n title: data.title,\n date: formatDate(data.timestamp),\n identifier,\n }),\n );\n data.api_key && dispatch(setClientApiKey(data.api_key));\n }\n });\n };\n\n const handleQuestionSubmission = (question?: string) => {\n if (question && status !== 'loading') {\n if (lastQueryReturnedErr) {\n // update last failed query with new prompt\n dispatch(\n updateQuery({\n index: queries.length - 1,\n query: {\n prompt: question,\n },\n }),\n );\n handleQuestion({\n question: queries[queries.length - 1].prompt,\n isRetry: true,\n });\n } else {\n handleQuestion({ question });\n }\n }\n };\n\n const handleQuestion = ({\n question,\n isRetry = false,\n }: {\n question: string;\n isRetry?: boolean;\n }) => {\n question = question.trim();\n if (question === '') return;\n\n const filesAttached = completedAttachments\n .filter((a) => a.id)\n .map((a) => ({ id: a.id as string, fileName: a.fileName }));\n\n !isRetry &&\n dispatch(\n addQuery({\n prompt: question,\n attachments: filesAttached,\n }),\n ); //dispatch only new queries\n\n dispatch(fetchSharedAnswer({ question }));\n };\n useEffect(() => {\n fetchQueries();\n }, []);\n\n return (\n <>\n \n
\n
\n

\n {title}\n

\n

\n {t('sharedConv.subtitle')}{' '}\n \n DocsGPT\n \n

\n

\n {date}\n

\n
\n \n
\n {apiKey ? (\n
\n {\n handleQuestionSubmission(text);\n }}\n loading={status === 'loading'}\n showSourceButton={false}\n showToolButton={false}\n />\n
\n ) : (\n navigate('/')}\n className=\"bg-purple-30 hover:bg-violets-are-blue mb-14 w-fit rounded-full px-5 py-3 text-white shadow-xl transition-colors duration-200 sm:mb-0\"\n >\n {t('sharedConv.button')}\n \n )}\n\n \n
\n
\n \n );\n};\n" }, { "path": "frontend/src/conversation/conversationHandlers.ts", "content": "import conversationService from '../api/services/conversationService';\nimport { Doc } from '../models/misc';\nimport { Answer, FEEDBACK, RetrievalPayload } from './conversationModels';\nimport { ToolCallsType } from './types';\n\nexport function handleFetchAnswer(\n question: string,\n signal: AbortSignal,\n token: string | null,\n selectedDocs: Doc[],\n conversationId: string | null,\n promptId: string | null,\n chunks: string,\n agentId?: string,\n attachments?: string[],\n save_conversation = true,\n modelId?: string,\n): Promise<\n | {\n result: any;\n answer: any;\n thought: any;\n sources: any;\n toolCalls: ToolCallsType[];\n conversationId: any;\n query: string;\n }\n | {\n result: any;\n answer: any;\n thought: any;\n sources: any;\n toolCalls: ToolCallsType[];\n query: string;\n conversationId: any;\n title: any;\n }\n> {\n const payload: RetrievalPayload = {\n question: question,\n conversation_id: conversationId,\n prompt_id: promptId,\n chunks: chunks,\n isNoneDoc: selectedDocs.length === 0,\n agent_id: agentId,\n save_conversation: save_conversation,\n };\n\n if (modelId) {\n payload.model_id = modelId;\n }\n\n // Add attachments to payload if they exist\n if (attachments && attachments.length > 0) {\n payload.attachments = attachments;\n }\n\n if (selectedDocs.length > 0) {\n if (selectedDocs.length > 1) {\n // Handle multiple documents\n payload.active_docs = selectedDocs.map((doc) => doc.id!);\n payload.retriever = selectedDocs[0]?.retriever as string;\n } else if (selectedDocs.length === 1 && 'id' in selectedDocs[0]) {\n // Handle single document (backward compatibility)\n payload.active_docs = selectedDocs[0].id as string;\n payload.retriever = selectedDocs[0].retriever as string;\n }\n }\n return conversationService\n .answer(payload, token, signal)\n .then((response) => {\n if (response.ok) {\n return response.json();\n } else {\n return Promise.reject(new Error(response.statusText));\n }\n })\n .then((data) => {\n const result = data.answer;\n return {\n answer: result,\n query: question,\n result,\n thought: data.thought,\n sources: data.sources,\n toolCalls: data.tool_calls,\n conversationId: data.conversation_id,\n title: data.title || null,\n };\n });\n}\n\nexport function handleFetchAnswerSteaming(\n question: string,\n signal: AbortSignal,\n token: string | null,\n selectedDocs: Doc[],\n conversationId: string | null,\n promptId: string | null,\n chunks: string,\n onEvent: (event: MessageEvent) => void,\n indx?: number,\n agentId?: string,\n attachments?: string[],\n save_conversation = true,\n modelId?: string,\n): Promise {\n const payload: RetrievalPayload = {\n question: question,\n conversation_id: conversationId,\n prompt_id: promptId,\n chunks: chunks,\n isNoneDoc: selectedDocs.length === 0,\n index: indx,\n agent_id: agentId,\n save_conversation: save_conversation,\n };\n\n if (modelId) {\n payload.model_id = modelId;\n }\n\n // Add attachments to payload if they exist\n if (attachments && attachments.length > 0) {\n payload.attachments = attachments;\n }\n\n if (selectedDocs.length > 0) {\n if (selectedDocs.length > 1) {\n // Handle multiple documents\n payload.active_docs = selectedDocs.map((doc) => doc.id!);\n payload.retriever = selectedDocs[0]?.retriever as string;\n } else if (selectedDocs.length === 1 && 'id' in selectedDocs[0]) {\n // Handle single document (backward compatibility)\n payload.active_docs = selectedDocs[0].id as string;\n payload.retriever = selectedDocs[0].retriever as string;\n }\n }\n\n return new Promise((resolve, reject) => {\n conversationService\n .answerStream(payload, token, signal)\n .then((response) => {\n if (!response.body) throw Error('No response body');\n\n let buffer = '';\n const reader = response.body.getReader();\n const decoder = new TextDecoder('utf-8');\n let counterrr = 0;\n const processStream = ({\n done,\n value,\n }: ReadableStreamReadResult) => {\n if (done) return;\n\n counterrr += 1;\n\n const chunk = decoder.decode(value);\n buffer += chunk;\n\n const events = buffer.split('\\n\\n');\n buffer = events.pop() ?? '';\n\n for (const event of events) {\n if (event.trim().startsWith('data:')) {\n const dataLine: string = event\n .split('\\n')\n .map((line: string) => line.replace(/^data:\\s?/, ''))\n .join('');\n\n const messageEvent = new MessageEvent('message', {\n data: dataLine.trim(),\n });\n\n onEvent(messageEvent);\n }\n }\n\n reader.read().then(processStream).catch(reject);\n };\n\n reader.read().then(processStream).catch(reject);\n })\n .catch((error) => {\n console.error('Connection failed:', error);\n reject(error);\n });\n });\n}\n\nexport function handleSearch(\n question: string,\n token: string | null,\n selectedDocs: Doc[],\n conversation_id: string | null,\n chunks: string,\n) {\n const payload: RetrievalPayload = {\n question: question,\n conversation_id: conversation_id,\n chunks: chunks,\n isNoneDoc: selectedDocs.length === 0,\n };\n if (selectedDocs.length > 0) {\n if (selectedDocs.length > 1) {\n // Handle multiple documents\n payload.active_docs = selectedDocs.map((doc) => doc.id!);\n payload.retriever = selectedDocs[0]?.retriever as string;\n } else if (selectedDocs.length === 1 && 'id' in selectedDocs[0]) {\n // Handle single document (backward compatibility)\n payload.active_docs = selectedDocs[0].id as string;\n payload.retriever = selectedDocs[0].retriever as string;\n }\n }\n return conversationService\n .search(payload, token)\n .then((response) => response.json())\n .then((data) => {\n return data;\n })\n .catch((err) => console.log(err));\n}\n\nexport function handleSearchViaApiKey(\n question: string,\n api_key: string,\n history: Array = [],\n) {\n history = history.map((item) => {\n return {\n prompt: item.prompt,\n response: item.response,\n tool_calls: item.tool_calls,\n };\n });\n return conversationService\n .search(\n {\n question: question,\n history: JSON.stringify(history),\n api_key: api_key,\n },\n null,\n )\n .then((response) => response.json())\n .then((data) => {\n return data;\n })\n .catch((err) => console.log(err));\n}\n\nexport function handleSendFeedback(\n prompt: string,\n response: string,\n feedback: FEEDBACK,\n conversation_id: string,\n prompt_index: number,\n token: string | null,\n) {\n return conversationService\n .feedback(\n {\n question: prompt,\n answer: response,\n feedback: feedback,\n conversation_id: conversation_id,\n question_index: prompt_index,\n },\n token,\n )\n .then((response) => {\n if (response.ok) {\n return Promise.resolve();\n } else {\n return Promise.reject();\n }\n });\n}\n\nexport function handleFetchSharedAnswerStreaming(\n question: string,\n signal: AbortSignal,\n apiKey: string,\n history: Array = [],\n attachments: string[] = [],\n onEvent: (event: MessageEvent) => void,\n): Promise {\n history = history.map((item) => {\n return {\n prompt: item.prompt,\n response: item.response,\n tool_calls: item.tool_calls,\n };\n });\n\n return new Promise((resolve, reject) => {\n const payload = {\n question: question,\n history: JSON.stringify(history),\n api_key: apiKey,\n save_conversation: false,\n attachments: attachments.length > 0 ? attachments : undefined,\n };\n conversationService\n .answerStream(payload, null, signal)\n .then((response) => {\n if (!response.body) throw Error('No response body');\n\n const reader = response.body.getReader();\n const decoder = new TextDecoder('utf-8');\n let counterrr = 0;\n const processStream = ({\n done,\n value,\n }: ReadableStreamReadResult) => {\n if (done) {\n console.log(counterrr);\n return;\n }\n\n counterrr += 1;\n\n const chunk = decoder.decode(value);\n\n const lines = chunk.split('\\n');\n\n for (let line of lines) {\n if (line.trim() == '') {\n continue;\n }\n if (line.startsWith('data:')) {\n line = line.substring(5);\n }\n\n const messageEvent: MessageEvent = new MessageEvent('message', {\n data: line,\n });\n\n onEvent(messageEvent); // handle each message\n }\n\n reader.read().then(processStream).catch(reject);\n };\n\n reader.read().then(processStream).catch(reject);\n })\n .catch((error) => {\n console.error('Connection failed:', error);\n reject(error);\n });\n });\n}\n\nexport function handleFetchSharedAnswer(\n question: string,\n signal: AbortSignal,\n apiKey: string,\n attachments?: string[],\n): Promise<\n | {\n result: any;\n answer: any;\n sources: any;\n query: string;\n }\n | {\n result: any;\n answer: any;\n sources: any;\n query: string;\n title: any;\n }\n> {\n const payload = {\n question: question,\n api_key: apiKey,\n attachments:\n attachments && attachments.length > 0 ? attachments : undefined,\n };\n\n return conversationService\n .answer(payload, null, signal)\n .then((response) => {\n if (response.ok) {\n return response.json();\n } else {\n return Promise.reject(new Error(response.statusText));\n }\n })\n .then((data) => {\n const result = data.answer;\n return {\n answer: result,\n query: question,\n result,\n sources: data.sources,\n toolCalls: data.tool_calls,\n };\n });\n}\n" }, { "path": "frontend/src/conversation/conversationModels.ts", "content": "import { ToolCallsType } from './types';\n\nexport type MESSAGE_TYPE = 'QUESTION' | 'ANSWER' | 'ERROR';\nexport type Status = 'idle' | 'loading' | 'failed';\nexport type FEEDBACK = 'LIKE' | 'DISLIKE' | null;\n\nexport interface Message {\n text: string;\n type: MESSAGE_TYPE;\n}\n\nexport interface Attachment {\n id?: string;\n fileName: string;\n status: 'uploading' | 'processing' | 'completed' | 'failed';\n progress: number;\n taskId?: string;\n token_count?: number;\n}\n\nexport interface ConversationState {\n queries: Query[];\n status: Status;\n conversationId: string | null;\n}\n\nexport interface Answer {\n answer: string;\n query: string;\n result: string;\n conversationId: string | null;\n title: string | null;\n thought: string;\n sources: { title: string; text: string; source: string }[];\n tool_calls: ToolCallsType[];\n structured?: boolean;\n schema?: object;\n}\n\nexport interface Query {\n prompt: string;\n response?: string;\n feedback?: FEEDBACK;\n conversationId?: string | null;\n title?: string | null;\n thought?: string;\n sources?: { title: string; text: string; link: string }[];\n tool_calls?: ToolCallsType[];\n error?: string;\n attachments?: { id: string; fileName: string }[];\n structured?: boolean;\n schema?: object;\n}\n\nexport interface RetrievalPayload {\n question: string;\n active_docs?: string | string[];\n retriever?: string;\n conversation_id: string | null;\n prompt_id?: string | null;\n chunks: string;\n isNoneDoc: boolean;\n index?: number;\n agent_id?: string;\n attachments?: string[];\n save_conversation?: boolean;\n model_id?: string;\n}\n" }, { "path": "frontend/src/conversation/conversationSlice.ts", "content": "import { createAsyncThunk, createSlice, PayloadAction } from '@reduxjs/toolkit';\n\nimport { getConversations } from '../preferences/preferenceApi';\nimport { setConversations } from '../preferences/preferenceSlice';\nimport store from '../store';\nimport {\n clearAttachments,\n selectCompletedAttachments,\n} from '../upload/uploadSlice';\nimport {\n handleFetchAnswer,\n handleFetchAnswerSteaming,\n} from './conversationHandlers';\nimport { Answer, ConversationState, Query, Status } from './conversationModels';\nimport { ToolCallsType } from './types';\n\nconst initialState: ConversationState = {\n queries: [],\n status: 'idle',\n conversationId: null,\n};\n\nconst API_STREAMING = import.meta.env.VITE_API_STREAMING === 'true';\n\nlet abortController: AbortController | null = null;\nexport function handleAbort() {\n if (abortController) {\n abortController.abort();\n abortController = null;\n }\n}\n\nexport const fetchAnswer = createAsyncThunk<\n Answer,\n { question: string; indx?: number }\n>('fetchAnswer', async ({ question, indx }, { dispatch, getState }) => {\n if (abortController) abortController.abort();\n abortController = new AbortController();\n const { signal } = abortController;\n\n let isSourceUpdated = false;\n const state = getState() as RootState;\n const attachmentIds = selectCompletedAttachments(state)\n .filter((a) => a.id)\n .map((a) => a.id) as string[];\n\n if (attachmentIds.length > 0) {\n dispatch(clearAttachments());\n }\n\n const currentConversationId = state.conversation.conversationId;\n const modelId =\n state.preference.selectedAgent?.default_model_id ||\n state.preference.selectedModel?.id;\n\n if (state.preference) {\n if (API_STREAMING) {\n await handleFetchAnswerSteaming(\n question,\n signal,\n state.preference.token,\n state.preference.selectedDocs || [],\n currentConversationId,\n state.preference.prompt.id,\n state.preference.chunks,\n (event) => {\n const data = JSON.parse(event.data);\n const targetIndex = indx ?? state.conversation.queries.length - 1;\n\n // Only process events if they match the current conversation\n if (currentConversationId === state.conversation.conversationId) {\n if (data.type === 'end') {\n dispatch(conversationSlice.actions.setStatus('idle'));\n getConversations(state.preference.token)\n .then((fetchedConversations) => {\n dispatch(setConversations(fetchedConversations));\n })\n .catch((error) => {\n console.error('Failed to fetch conversations: ', error);\n });\n if (!isSourceUpdated) {\n dispatch(\n updateStreamingSource({\n conversationId: currentConversationId,\n index: targetIndex,\n query: { sources: [] },\n }),\n );\n }\n } else if (data.type === 'id') {\n // Only update the conversationId if it's currently null\n const currentState = getState() as RootState;\n if (currentState.conversation.conversationId === null) {\n dispatch(\n updateConversationId({\n query: { conversationId: data.id },\n }),\n );\n }\n } else if (data.type === 'thought') {\n const result = data.thought;\n dispatch(\n updateThought({\n conversationId: currentConversationId,\n index: targetIndex,\n query: { thought: result },\n }),\n );\n } else if (data.type === 'source') {\n isSourceUpdated = true;\n dispatch(\n updateStreamingSource({\n conversationId: currentConversationId,\n index: targetIndex,\n query: { sources: data.source ?? [] },\n }),\n );\n } else if (data.type === 'tool_call') {\n dispatch(\n updateToolCall({\n index: targetIndex,\n tool_call: data.data as ToolCallsType,\n }),\n );\n } else if (data.type === 'error') {\n // set status to 'failed'\n dispatch(conversationSlice.actions.setStatus('failed'));\n dispatch(\n conversationSlice.actions.raiseError({\n conversationId: currentConversationId,\n index: targetIndex,\n message: data.error,\n }),\n );\n } else if (data.type === 'structured_answer') {\n dispatch(\n updateStreamingQuery({\n conversationId: currentConversationId,\n index: targetIndex,\n query: {\n response: data.answer,\n structured: data.structured,\n schema: data.schema,\n },\n }),\n );\n } else {\n dispatch(\n updateStreamingQuery({\n conversationId: currentConversationId,\n index: targetIndex,\n query: { response: data.answer },\n }),\n );\n }\n }\n },\n indx,\n state.preference.selectedAgent?.id,\n attachmentIds,\n true,\n modelId,\n );\n } else {\n const answer = await handleFetchAnswer(\n question,\n signal,\n state.preference.token,\n state.preference.selectedDocs || [],\n state.conversation.conversationId,\n state.preference.prompt.id,\n state.preference.chunks,\n state.preference.selectedAgent?.id,\n attachmentIds,\n true,\n modelId,\n );\n if (answer) {\n let sourcesPrepped = [];\n sourcesPrepped = answer.sources.map((source: { title: string }) => {\n if (source && source.title) {\n const titleParts = source.title.split('/');\n return {\n ...source,\n title: titleParts[titleParts.length - 1],\n };\n }\n return source;\n });\n\n const targetIndex = indx ?? state.conversation.queries.length - 1;\n\n dispatch(\n updateQuery({\n index: targetIndex,\n query: {\n response: answer.answer,\n thought: answer.thought,\n sources: sourcesPrepped,\n tool_calls: answer.toolCalls,\n },\n }),\n );\n dispatch(\n updateConversationId({\n query: { conversationId: answer.conversationId },\n }),\n );\n getConversations(state.preference.token)\n .then((fetchedConversations) => {\n dispatch(setConversations(fetchedConversations));\n })\n .catch((error) => {\n console.error('Failed to fetch conversations: ', error);\n });\n dispatch(conversationSlice.actions.setStatus('idle'));\n }\n }\n }\n return {\n conversationId: null,\n title: null,\n answer: '',\n query: question,\n result: '',\n thought: '',\n sources: [],\n tool_calls: [],\n };\n});\n\nexport const conversationSlice = createSlice({\n name: 'conversation',\n initialState,\n reducers: {\n addQuery(state, action: PayloadAction) {\n state.queries.push(action.payload);\n },\n setConversation(state, action: PayloadAction) {\n state.queries = action.payload;\n },\n resendQuery(\n state,\n action: PayloadAction<{ index: number; prompt: string }>,\n ) {\n const { index, prompt } = action.payload;\n if (index < 0 || index >= state.queries.length) return;\n\n state.queries.splice(index + 1);\n state.queries[index].prompt = prompt;\n delete state.queries[index].response;\n delete state.queries[index].thought;\n delete state.queries[index].sources;\n delete state.queries[index].tool_calls;\n delete state.queries[index].error;\n delete state.queries[index].structured;\n delete state.queries[index].schema;\n delete state.queries[index].feedback;\n },\n updateStreamingQuery(\n state,\n action: PayloadAction<{\n conversationId: string | null;\n index: number;\n query: Partial;\n }>,\n ) {\n const { conversationId, index, query } = action.payload;\n // Only update if this update is for the current conversation\n if (state.status === 'idle' || state.conversationId !== conversationId)\n return;\n\n if (query.response != undefined) {\n state.queries[index].response =\n (state.queries[index].response || '') + query.response;\n }\n\n if (query.structured !== undefined) {\n state.queries[index].structured = query.structured;\n }\n\n if (query.schema !== undefined) {\n state.queries[index].schema = query.schema;\n }\n },\n updateConversationId(\n state,\n action: PayloadAction<{ query: Partial }>,\n ) {\n state.conversationId = action.payload.query.conversationId ?? null;\n state.status = 'idle';\n },\n updateThought(\n state,\n action: PayloadAction<{\n conversationId: string | null;\n index: number;\n query: Partial;\n }>,\n ) {\n const { conversationId, index, query } = action.payload;\n if (state.conversationId !== conversationId) return;\n\n if (query.thought != undefined) {\n state.queries[index].thought =\n (state.queries[index].thought || '') + query.thought;\n }\n },\n updateStreamingSource(\n state,\n action: PayloadAction<{\n conversationId: string | null;\n index: number;\n query: Partial;\n }>,\n ) {\n const { index, query } = action.payload;\n if (query.sources !== undefined)\n state.queries[index].sources = query.sources;\n },\n updateToolCall(state, action) {\n const { index, tool_call } = action.payload;\n\n if (!state.queries[index].tool_calls) {\n state.queries[index].tool_calls = [];\n }\n\n const existingIndex = state.queries[index].tool_calls.findIndex(\n (call) => call.call_id === tool_call.call_id,\n );\n\n if (existingIndex !== -1) {\n const existingCall = state.queries[index].tool_calls[existingIndex];\n state.queries[index].tool_calls[existingIndex] = {\n ...existingCall,\n ...tool_call,\n };\n } else state.queries[index].tool_calls.push(tool_call);\n },\n updateQuery(\n state,\n action: PayloadAction<{ index: number; query: Partial }>,\n ) {\n const { index, query } = action.payload;\n state.queries[index] = {\n ...state.queries[index],\n ...query,\n };\n },\n setStatus(state, action: PayloadAction) {\n state.status = action.payload;\n },\n raiseError(\n state,\n action: PayloadAction<{\n conversationId: string | null;\n index: number;\n message: string;\n }>,\n ) {\n const { conversationId, index, message } = action.payload;\n if (state.conversationId !== conversationId) return;\n\n state.queries[index].error = message;\n },\n\n resetConversation: (state) => {\n state.queries = initialState.queries;\n state.status = initialState.status;\n state.conversationId = initialState.conversationId;\n handleAbort();\n },\n },\n extraReducers(builder) {\n builder\n .addCase(fetchAnswer.pending, (state) => {\n state.status = 'loading';\n })\n .addCase(fetchAnswer.rejected, (state, action) => {\n if (action.meta.aborted) {\n state.status = 'idle';\n return;\n }\n state.status = 'failed';\n if (state.queries.length > 0) {\n state.queries[state.queries.length - 1].error =\n 'Something went wrong';\n }\n });\n },\n});\n\ntype RootState = ReturnType;\n\nexport const selectQueries = (state: RootState) => state.conversation.queries;\n\nexport const selectStatus = (state: RootState) => state.conversation.status;\n\nexport const {\n addQuery,\n updateQuery,\n resendQuery,\n updateStreamingQuery,\n updateConversationId,\n updateThought,\n updateStreamingSource,\n updateToolCall,\n setConversation,\n setStatus,\n raiseError,\n resetConversation,\n} = conversationSlice.actions;\nexport default conversationSlice.reducer;\n" }, { "path": "frontend/src/conversation/sharedConversationSlice.ts", "content": "import { createSlice } from '@reduxjs/toolkit';\nimport type { PayloadAction } from '@reduxjs/toolkit';\nimport store from '../store';\nimport { Query, Status, Answer } from '../conversation/conversationModels';\nimport { createAsyncThunk } from '@reduxjs/toolkit';\nimport {\n handleFetchSharedAnswer,\n handleFetchSharedAnswerStreaming,\n} from './conversationHandlers';\nimport {\n selectCompletedAttachments,\n clearAttachments,\n} from '../upload/uploadSlice';\n\nconst API_STREAMING = import.meta.env.VITE_API_STREAMING === 'true';\ninterface SharedConversationsType {\n queries: Query[];\n apiKey?: string;\n identifier: string;\n status: Status;\n date?: string;\n title?: string;\n}\n\nconst initialState: SharedConversationsType = {\n queries: [],\n identifier: '',\n status: 'idle',\n};\n\nexport const fetchSharedAnswer = createAsyncThunk(\n 'shared/fetchAnswer',\n async ({ question }, { dispatch, getState, signal }) => {\n const state = getState() as RootState;\n\n const attachmentIds = selectCompletedAttachments(state)\n .filter((a) => a.id)\n .map((a) => a.id) as string[];\n\n if (attachmentIds.length > 0) {\n dispatch(clearAttachments());\n }\n\n if (state.preference && state.sharedConversation.apiKey) {\n if (API_STREAMING) {\n await handleFetchSharedAnswerStreaming(\n question,\n signal,\n state.sharedConversation.apiKey,\n state.sharedConversation.queries,\n attachmentIds,\n (event) => {\n const data = JSON.parse(event.data);\n // check if the 'end' event has been received\n if (data.type === 'end') {\n // set status to 'idle'\n dispatch(sharedConversationSlice.actions.setStatus('idle'));\n dispatch(saveToLocalStorage());\n } else if (data.type === 'thought') {\n const result = data.thought;\n console.log('thought', result);\n dispatch(\n updateThought({\n index: state.sharedConversation.queries.length - 1,\n query: { thought: result },\n }),\n );\n } else if (data.type === 'source') {\n dispatch(\n updateStreamingSource({\n index: state.sharedConversation.queries.length - 1,\n query: { sources: data.source ?? [] },\n }),\n );\n } else if (data.type === 'tool_calls') {\n dispatch(\n updateToolCalls({\n index: state.sharedConversation.queries.length - 1,\n query: { tool_calls: data.tool_calls },\n }),\n );\n } else if (data.type === 'error') {\n // set status to 'failed'\n dispatch(sharedConversationSlice.actions.setStatus('failed'));\n dispatch(\n sharedConversationSlice.actions.raiseError({\n index: state.sharedConversation.queries.length - 1,\n message: data.error,\n }),\n );\n } else {\n const result = data.answer;\n dispatch(\n updateStreamingQuery({\n index: state.sharedConversation.queries.length - 1,\n query: { response: result },\n }),\n );\n }\n },\n );\n } else {\n const answer = await handleFetchSharedAnswer(\n question,\n signal,\n state.sharedConversation.apiKey,\n attachmentIds,\n );\n if (answer) {\n let sourcesPrepped = [];\n sourcesPrepped = answer.sources.map((source: { title: string }) => {\n if (source && source.title) {\n const titleParts = source.title.split('/');\n return {\n ...source,\n title: titleParts[titleParts.length - 1],\n };\n }\n return source;\n });\n\n dispatch(\n updateQuery({\n index: state.sharedConversation.queries.length - 1,\n query: { response: answer.answer, sources: sourcesPrepped },\n }),\n );\n dispatch(sharedConversationSlice.actions.setStatus('idle'));\n }\n }\n }\n return {\n conversationId: null,\n title: null,\n answer: '',\n query: question,\n result: '',\n thought: '',\n sources: [],\n tool_calls: [],\n };\n },\n);\n\nexport const sharedConversationSlice = createSlice({\n name: 'sharedConversation',\n initialState,\n reducers: {\n setStatus(state, action: PayloadAction) {\n state.status = action.payload;\n },\n setIdentifier(state, action: PayloadAction) {\n state.identifier = action.payload;\n },\n setFetchedData(\n state,\n action: PayloadAction<{\n queries: Query[];\n title: string;\n date: string;\n identifier: string;\n }>,\n ) {\n const { queries, title, identifier, date } = action.payload;\n const previousQueriesStr = localStorage.getItem(identifier);\n const localySavedQueries: Query[] = previousQueriesStr\n ? JSON.parse(previousQueriesStr)\n : [];\n state.queries = [...queries, ...localySavedQueries];\n state.title = title;\n state.date = date;\n state.identifier = identifier;\n },\n setClientApiKey(state, action: PayloadAction) {\n state.apiKey = action.payload;\n },\n addQuery(state, action: PayloadAction) {\n state.queries.push(action.payload);\n },\n updateStreamingQuery(\n state,\n action: PayloadAction<{ index: number; query: Partial }>,\n ) {\n const { index, query } = action.payload;\n if (query.response !== undefined) {\n state.queries[index].response =\n (state.queries[index].response || '') + query.response;\n }\n },\n updateToolCalls(\n state,\n action: PayloadAction<{ index: number; query: Partial }>,\n ) {\n const { index, query } = action.payload;\n if (!state.queries[index].tool_calls) {\n state.queries[index].tool_calls = query?.tool_calls;\n }\n },\n updateQuery(\n state,\n action: PayloadAction<{ index: number; query: Partial }>,\n ) {\n const { index, query } = action.payload;\n state.queries[index] = {\n ...state.queries[index],\n ...query,\n };\n },\n updateThought(\n state,\n action: PayloadAction<{ index: number; query: Partial }>,\n ) {\n const { index, query } = action.payload;\n if (query.thought != undefined) {\n state.queries[index].thought =\n (state.queries[index].thought || '') + query.thought;\n } else {\n state.queries[index] = {\n ...state.queries[index],\n ...query,\n };\n }\n },\n updateStreamingSource(\n state,\n action: PayloadAction<{ index: number; query: Partial }>,\n ) {\n const { index, query } = action.payload;\n if (!state.queries[index].sources) {\n state.queries[index].sources = query.sources ?? [];\n } else if (query.sources && query.sources.length > 0) {\n state.queries[index].sources = [\n ...(state.queries[index].sources ?? []),\n ...query.sources,\n ];\n }\n },\n raiseError(\n state,\n action: PayloadAction<{ index: number; message: string }>,\n ) {\n const { index, message } = action.payload;\n state.queries[index].error = message;\n },\n saveToLocalStorage(state) {\n const previousQueriesStr = localStorage.getItem(state.identifier);\n previousQueriesStr\n ? localStorage.setItem(\n state.identifier,\n JSON.stringify([\n ...JSON.parse(previousQueriesStr),\n state.queries[state.queries.length - 1],\n ]),\n )\n : localStorage.setItem(\n state.identifier,\n JSON.stringify([state.queries[state.queries.length - 1]]),\n );\n },\n },\n extraReducers(builder) {\n builder\n .addCase(fetchSharedAnswer.pending, (state) => {\n state.status = 'loading';\n })\n .addCase(fetchSharedAnswer.rejected, (state, action) => {\n if (action.meta.aborted) {\n state.status = 'idle';\n return;\n }\n state.status = 'failed';\n if (state.queries.length > 0) {\n state.queries[state.queries.length - 1].error =\n 'Something went wrong';\n }\n });\n },\n});\n\nexport const {\n setStatus,\n setIdentifier,\n setFetchedData,\n setClientApiKey,\n updateQuery,\n updateStreamingQuery,\n updateThought,\n updateToolCalls,\n addQuery,\n saveToLocalStorage,\n updateStreamingSource,\n} = sharedConversationSlice.actions;\n\nexport const selectStatus = (state: RootState) =>\n state.sharedConversation.status;\nexport const selectClientAPIKey = (state: RootState) =>\n state.sharedConversation.apiKey;\nexport const selectQueries = (state: RootState) =>\n state.sharedConversation.queries;\nexport const selectTitle = (state: RootState) => state.sharedConversation.title;\nexport const selectDate = (state: RootState) => state.sharedConversation.date;\n\ntype RootState = ReturnType;\n\nsharedConversationSlice;\nexport default sharedConversationSlice.reducer;\n" }, { "path": "frontend/src/conversation/types/index.ts", "content": "export type ToolCallsType = {\n tool_name: string;\n action_name: string;\n call_id: string;\n arguments: Record;\n result?: Record;\n error?: string;\n status?: 'pending' | 'completed' | 'error';\n artifact_id?: string;\n};\n" }, { "path": "frontend/src/hooks/index.ts", "content": "import { useEffect, RefObject, useState } from 'react';\n\nexport function useOutsideAlerter(\n ref: RefObject,\n handler: () => void,\n additionalDeps: unknown[],\n handleEscapeKey?: boolean,\n) {\n useEffect(() => {\n function handleClickOutside(event: MouseEvent) {\n if (ref.current && !ref.current.contains(event.target as Node)) {\n handler();\n }\n }\n\n function handleEscape(event: KeyboardEvent) {\n if (event.key === 'Escape') {\n handler();\n }\n }\n\n document.addEventListener('mousedown', handleClickOutside);\n if (handleEscapeKey) {\n document.addEventListener('keydown', handleEscape);\n }\n\n return () => {\n document.removeEventListener('mousedown', handleClickOutside);\n if (handleEscapeKey) {\n document.removeEventListener('keydown', handleEscape);\n }\n };\n }, [ref, handler, handleEscapeKey, ...additionalDeps]);\n}\n\nexport function useMediaQuery() {\n const mobileQuery = '(max-width: 768px)';\n const tabletQuery = '(max-width: 1023px)';\n const desktopQuery = '(min-width: 1024px)';\n const [isMobile, setIsMobile] = useState(false);\n const [isTablet, setIsTablet] = useState(false);\n const [isDesktop, setIsDesktop] = useState(false);\n\n useEffect(() => {\n const mobileMedia = window.matchMedia(mobileQuery);\n const tabletMedia = window.matchMedia(tabletQuery);\n const desktopMedia = window.matchMedia(desktopQuery);\n\n const updateMediaQueries = () => {\n setIsMobile(mobileMedia.matches);\n setIsTablet(tabletMedia.matches && !mobileMedia.matches); // Tablet but not mobile\n setIsDesktop(desktopMedia.matches);\n };\n\n updateMediaQueries();\n\n const listener = () => updateMediaQueries();\n window.addEventListener('resize', listener);\n\n return () => {\n window.removeEventListener('resize', listener);\n };\n }, [mobileQuery, tabletQuery, desktopQuery]);\n\n return { isMobile, isTablet, isDesktop };\n}\n\nexport function useDarkTheme() {\n const getSystemThemePreference = () => {\n return (\n window.matchMedia &&\n window.matchMedia('(prefers-color-scheme: dark)').matches\n );\n };\n\n const getInitialTheme = () => {\n const storedTheme = localStorage.getItem('selectedTheme');\n if (storedTheme === 'Dark' || storedTheme === 'Light') {\n return storedTheme === 'Dark';\n }\n return getSystemThemePreference();\n };\n\n const [isDarkTheme, setIsDarkTheme] = useState(getInitialTheme());\n const [componentMounted, setComponentMounted] = useState(false);\n\n useEffect(() => {\n const mediaQuery = window.matchMedia('(prefers-color-scheme: dark)');\n const handleChange = () => {\n if (localStorage.getItem('selectedTheme') === null) {\n setIsDarkTheme(mediaQuery.matches);\n }\n };\n\n mediaQuery.addListener(handleChange);\n return () => mediaQuery.removeListener(handleChange);\n }, []);\n\n useEffect(() => {\n localStorage.setItem('selectedTheme', isDarkTheme ? 'Dark' : 'Light');\n if (isDarkTheme) {\n document.body?.classList.add('dark');\n } else {\n document.body?.classList.remove('dark');\n }\n setComponentMounted(true);\n }, [isDarkTheme]);\n\n const toggleTheme = () => {\n setIsDarkTheme(!isDarkTheme);\n };\n\n return [isDarkTheme, toggleTheme, componentMounted] as const;\n}\n\nexport function useLoaderState(\n initialState = false,\n delay = 250,\n): [boolean, (value: boolean) => void] {\n const [state, setState] = useState(initialState);\n\n const setLoaderState = (value: boolean) => {\n if (value) {\n setState(true);\n } else {\n // Only add delay when changing from true to false\n setTimeout(() => {\n setState(false);\n }, delay);\n }\n };\n\n return [state, setLoaderState];\n}\n" }, { "path": "frontend/src/hooks/useDataInitializer.ts", "content": "import { useEffect } from 'react';\nimport { useDispatch, useSelector } from 'react-redux';\n\nimport { Doc } from '../models/misc';\nimport {\n getDocs,\n getConversations,\n getPrompts,\n} from '../preferences/preferenceApi';\nimport {\n selectConversations,\n selectSelectedDocs,\n selectToken,\n setConversations,\n setPrompts,\n setSelectedDocs,\n setSourceDocs,\n} from '../preferences/preferenceSlice';\n\n/**\n * useDataInitializer Hook\n *\n * Custom hook responsible for initializing all application data on mount.\n * This hook handles:\n * - Fetching and setting up documents (source docs and selected docs)\n * - Fetching and setting up prompts\n * - Fetching and setting up conversations\n *\n * @param isAuthLoading -\n */\nexport default function useDataInitializer(isAuthLoading: boolean) {\n const dispatch = useDispatch();\n const token = useSelector(selectToken);\n const selectedDoc = useSelector(selectSelectedDocs);\n const conversations = useSelector(selectConversations);\n\n // Initialize documents\n useEffect(() => {\n // Skip if auth is still loading\n if (isAuthLoading) {\n return;\n }\n\n const fetchDocs = async () => {\n try {\n const data = await getDocs(token);\n dispatch(setSourceDocs(data));\n\n // Auto-select default document if none selected\n if (\n !selectedDoc ||\n (Array.isArray(selectedDoc) && selectedDoc.length === 0)\n ) {\n if (Array.isArray(data)) {\n data.forEach((doc: Doc) => {\n if (doc.model && doc.name === 'default') {\n dispatch(setSelectedDocs([doc]));\n }\n });\n }\n }\n } catch (error) {\n console.error('Failed to fetch documents:', error);\n }\n };\n\n fetchDocs();\n }, [isAuthLoading, token]);\n\n // Initialize prompts\n useEffect(() => {\n // Skip if auth is still loading\n if (isAuthLoading) {\n return;\n }\n\n const fetchPromptsData = async () => {\n try {\n const data = await getPrompts(token);\n dispatch(setPrompts(data));\n } catch (error) {\n console.error('Failed to fetch prompts:', error);\n }\n };\n\n fetchPromptsData();\n }, [isAuthLoading, token]);\n\n // Initialize conversations\n useEffect(() => {\n // Skip if auth is still loading\n if (isAuthLoading) {\n return;\n }\n\n const fetchConversationsData = async () => {\n if (!conversations?.data) {\n dispatch(setConversations({ ...conversations, loading: true }));\n try {\n const fetchedConversations = await getConversations(token);\n dispatch(setConversations(fetchedConversations));\n } catch (error) {\n console.error('Failed to fetch conversations:', error);\n dispatch(setConversations({ data: null, loading: false }));\n }\n }\n };\n\n fetchConversationsData();\n }, [isAuthLoading, conversations?.data, token]);\n}\n" }, { "path": "frontend/src/hooks/useTokenAuth.ts", "content": "import { useEffect, useRef, useState } from 'react';\nimport { useDispatch, useSelector } from 'react-redux';\n\nimport userService from '../api/services/userService';\nimport { selectToken, setToken } from '../preferences/preferenceSlice';\n\nexport default function useAuth() {\n const dispatch = useDispatch();\n const token = useSelector(selectToken);\n const [authType, setAuthType] = useState(null);\n const [showTokenModal, setShowTokenModal] = useState(false);\n const [isAuthLoading, setIsAuthLoading] = useState(true);\n const isGeneratingToken = useRef(false);\n\n const generateNewToken = async () => {\n if (isGeneratingToken.current) return;\n isGeneratingToken.current = true;\n const response = await userService.getNewToken();\n const { token: newToken } = await response.json();\n localStorage.setItem('authToken', newToken);\n dispatch(setToken(newToken));\n setIsAuthLoading(false);\n return newToken;\n };\n\n useEffect(() => {\n const initializeAuth = async () => {\n try {\n const configRes = await userService.getConfig();\n const config = await configRes.json();\n setAuthType(config.auth_type);\n\n if (config.auth_type === 'session_jwt' && !token) {\n await generateNewToken();\n } else if (config.auth_type === 'simple_jwt' && !token) {\n setShowTokenModal(true);\n setIsAuthLoading(false);\n } else {\n setIsAuthLoading(false);\n }\n } catch (error) {\n console.error('Auth initialization failed:', error);\n setIsAuthLoading(false);\n }\n };\n initializeAuth();\n }, []);\n\n const handleTokenSubmit = (enteredToken: string) => {\n localStorage.setItem('authToken', enteredToken);\n dispatch(setToken(enteredToken));\n setShowTokenModal(false);\n };\n return { authType, showTokenModal, isAuthLoading, token, handleTokenSubmit };\n}\n" }, { "path": "frontend/src/index.css", "content": "@import url('https://fonts.googleapis.com/css2?family=Roboto:wght@300;400;500;700&display=swap')\nlayer(base);\n\n@import 'tailwindcss';\n\n@import \"tw-animate-css\";\n\n@custom-variant dark (&:is(.dark *));\n\n@theme {\n --font-roboto: Roboto, sans-serif;\n\n --color-eerie-black: #212121;\n --color-black-1000: #343541;\n --color-jet: #343541;\n --color-gray-alpha: rgba(0, 0, 0, 0.64);\n --color-gray-1000: #f6f6f6;\n --color-gray-2000: rgba(0, 0, 0, 0.5);\n --color-gray-3000: rgba(243, 243, 243, 1);\n --color-gray-4000: #949494;\n --color-gray-5000: #bbbbbb;\n --color-gray-6000: #757575;\n --color-red-1000: rgb(254, 202, 202);\n --color-red-2000: #f44336;\n --color-red-3000: #621b16;\n --color-blue-1000: #7d54d1;\n --color-blue-2000: #002b49;\n --color-blue-3000: #4b02e2;\n --color-purple-30: #7d54d1;\n --color-purple-3000: rgb(230, 222, 247);\n --color-blue-4000: rgba(0, 125, 255, 0.36);\n --color-blue-5000: rgba(0, 125, 255);\n --color-green-2000: #0fff50;\n --color-light-gray: #edeef0;\n --color-white-3000: #ffffff;\n --color-just-black: #00000;\n --color-purple-taupe: #464152;\n --color-dove-gray: #6c6c6c;\n --color-silver: #c4c4c4;\n --color-rainy-gray: #a4a4a4;\n --color-raisin-black: #222327;\n --color-chinese-black: #161616;\n --color-chinese-silver: #cdcdcd;\n --color-dark-charcoal: #2f3036;\n --color-bright-gray: #ebebeb;\n --color-outer-space: #444654;\n --color-gun-metal: #2e303e;\n --color-sonic-silver: #747474;\n --color-soap: #d8ccf1;\n --color-independence: #54546d;\n --color-philippine-yellow: #ffc700;\n --color-chinese-white: #e0e0e0;\n --color-dark-gray: #aaaaaa;\n --color-dim-gray: #6a6a6a;\n --color-cultured: #f4f4f4;\n --color-charleston-green: #2b2c31;\n --color-charleston-green-2: #26272e;\n --color-charleston-green-3: #26272a;\n --color-grey: #7e7e7e;\n --color-lotion: #fbfbfb;\n --color-platinum: #e6e6e6;\n --color-eerie-black-2: #191919;\n --color-light-silver: #d9d9d9;\n --color-carbon: #2e2e2e;\n --color-onyx: #35363b;\n --color-royal-purple: #6c4ab0;\n --color-chinese-black-2: #0f1419;\n --color-gainsboro: #d9dcde;\n --color-onyx-2: #35383c;\n --color-philippine-grey: #929292;\n --color-charcoal-grey: #53545d;\n --color-rosso-corsa: #d30000;\n --color-north-texas-green: #0c9d35;\n --color-medium-purple: #8d66dd;\n --color-slate-blue: #6f5fca;\n --color-old-silver: #848484;\n --color-arsenic: #4d4e58;\n --color-light-gainsboro: #d7d7d7;\n --color-raisin-black-light: #18181b;\n --color-gunmetal: #32333b;\n --color-sonic-silver-light: #7f7f82;\n --color-violets-are-blue: #976af3;\n}\n\n/*\n The default border color has changed to `currentcolor` in Tailwind CSS v4,\n so we've added these compatibility styles to make sure everything still\n looks the same as it did with Tailwind CSS v3.\n\n If we ever want to remove these styles, we need to add an explicit border\n color utility to any element that depends on these defaults.\n*/\n@layer base {\n *,\n ::after,\n ::before,\n ::backdrop,\n ::file-selector-button {\n border-color: var(--color-gray-200, currentcolor);\n }\n}\n\n@utility no-scrollbar {\n /* Chrome, Safari and Opera */\n &::-webkit-scrollbar {\n display: none;\n }\n -ms-overflow-style: none; /* IE and Edge */\n scrollbar-width: none; /* Firefox */\n}\n\n@utility scrollbar-thin {\n &::-webkit-scrollbar {\n width: 6px;\n height: 6px;\n }\n\n &::-webkit-scrollbar-track {\n background: transparent;\n }\n\n /* Light theme scrollbar */\n &::-webkit-scrollbar-thumb {\n background: #E2E8F0;\n border-radius: 9999px;\n }\n\n &::-webkit-scrollbar-thumb:hover {\n background: #8C9198;\n }\n\n /* Dark theme scrollbar */\n .dark &::-webkit-scrollbar-thumb {\n background: #949494;\n border-radius: 9999px;\n }\n\n .dark &::-webkit-scrollbar-thumb:hover {\n background: #F0F0F0;\n }\n\n scrollbar-width: thin;\n scrollbar-color: #E2E8F0 transparent;\n\n .dark & {\n scrollbar-color: #949494 transparent;\n }\n}\n\n@utility scrollbar-overlay {\n &::-webkit-scrollbar {\n width: 6px;\n height: 6px;\n }\n\n &::-webkit-scrollbar-track {\n background: transparent;\n }\n\n &::-webkit-scrollbar-thumb {\n background: transparent;\n border-radius: 9999px;\n }\n\n &:hover::-webkit-scrollbar-thumb {\n background: #E2E8F0;\n }\n\n &:hover::-webkit-scrollbar-thumb:hover {\n background: #8C9198;\n }\n\n .dark &::-webkit-scrollbar-thumb {\n background: transparent;\n border-radius: 9999px;\n }\n\n .dark &:hover::-webkit-scrollbar-thumb {\n background: #949494;\n }\n\n .dark &:hover::-webkit-scrollbar-thumb:hover {\n background: #F0F0F0;\n }\n\n /* Standard scrollbar properties (Chrome 121+, Firefox) */\n scrollbar-width: thin;\n scrollbar-color: transparent transparent;\n\n &:hover {\n scrollbar-color: #E2E8F0 transparent;\n }\n\n .dark & {\n scrollbar-color: transparent transparent;\n }\n\n .dark &:hover {\n scrollbar-color: #949494 transparent;\n }\n}\n\n@utility table-default {\n @apply border-silver dark:border-silver/40 dark:text-bright-gray block w-full table-auto justify-center overflow-auto rounded-xl border text-center;\n\n & th {\n @apply p-4 font-normal text-nowrap text-gray-400;\n }\n\n & th {\n flex: 1;\n }\n\n & th:last-child {\n flex: 0;\n }\n\n & td {\n @apply border-silver dark:border-silver/40 w-full border-t px-4 py-2;\n }\n\n & td:last-child {\n @apply border-r-0;\n }\n\n & th {\n min-width: 150px;\n max-width: 320px;\n overflow: auto;\n }\n\n & td {\n min-width: 150px;\n max-width: 320px;\n overflow: auto;\n }\n\n @supports (-moz-appearance: none) {\n & th,\n & td {\n scrollbar-width: thin;\n scrollbar-color: #E2E8F0 transparent;\n }\n\n .dark & th,\n .dark & td {\n scrollbar-color: #949494 transparent;\n }\n }\n}\n\n@layer utilities {\n :root {\n --viewport-height: 100vh;\n font-synthesis: none !important;\n }\n\n @supports (height: 100dvh) {\n :root {\n --viewport-height: 100dvh; /* Use dvh where supported */\n }\n }\n\n body.dark {\n background-color: #202124; /* raisin-black */\n }\n ::-webkit-scrollbar {\n width: 6px;\n height: 6px;\n }\n ::-webkit-scrollbar-track {\n background: transparent;\n }\n ::-webkit-scrollbar-thumb {\n background: #E2E8F0;\n border-radius: 9999px;\n }\n ::-webkit-scrollbar-thumb:hover {\n background: #8C9198;\n }\n .dark ::-webkit-scrollbar-thumb {\n background: #949494;\n }\n .dark ::-webkit-scrollbar-thumb:hover {\n background: #F0F0F0;\n }\n\n /* Firefox: base scrollbar styles (Firefox ignores ::-webkit-scrollbar) */\n @supports (-moz-appearance: none) {\n * {\n scrollbar-width: thin;\n scrollbar-color: #E2E8F0 transparent;\n }\n .dark * {\n scrollbar-color: #949494 transparent;\n }\n }\n}\n\n@layer base {\n .prompt-variable-highlight {\n background-color: rgba(106, 77, 244, 0.18);\n border-radius: 0.375rem;\n padding: 0 0.25rem;\n }\n\n .dark .prompt-variable-highlight {\n background-color: rgba(106, 77, 244, 0.32);\n }\n\n /*! normalize.css v8.0.1 | MIT License | github.com/necolas/normalize.css */\n\n /* Document\n ========================================================================== */\n\n /**\n * 1. Correct the line height in all browsers.\n * 2. Prevent adjustments of font size after orientation changes in iOS.\n */\n\n html {\n line-height: 1.15; /* 1 */\n -webkit-text-size-adjust: 100%; /* 2 */\n min-height: 100vh;\n overflow-x: hidden;\n }\n\n /* Sections\n ========================================================================== */\n\n /**\n * Remove the margin in all browsers.\n */\n\n body {\n margin: 0;\n min-height: var(--viewport-height);\n overflow-x: hidden;\n font-family: 'Inter', sans-serif;\n }\n /*\nAvoid over-scrolling in mobile browsers\n*/\n @media only screen and (max-width: 500px) {\n body,\n html {\n min-height: var(--viewport-height);\n position: fixed;\n width: 100%;\n }\n }\n /**\n * Render the `main` element consistently in IE.\n */\n\n main {\n display: block;\n }\n\n /**\n * Correct the font size and margin on `h1` elements within `section` and\n * `article` contexts in Chrome, Firefox, and Safari.\n */\n\n h1 {\n font-size: 2em;\n margin: 0.67em 0;\n }\n\n /* Grouping content\n ========================================================================== */\n\n /**\n * 1. Add the correct box sizing in Firefox.\n * 2. Show the overflow in Edge and IE.\n */\n\n hr {\n box-sizing: content-box; /* 1 */\n height: 0; /* 1 */\n overflow: visible; /* 2 */\n }\n\n /**\n * 1. Correct the inheritance and scaling of font size in all browsers.\n * 2. Correct the odd `em` font sizing in all browsers.\n */\n\n pre {\n font-family: monospace, monospace; /* 1 */\n font-size: 1em; /* 2 */\n }\n\n /* Text-level semantics\n ========================================================================== */\n\n /**\n * Remove the gray background on active links in IE 10.\n */\n\n a {\n background-color: transparent;\n }\n\n /**\n * 1. Remove the bottom border in Chrome 57-\n * 2. Add the correct text decoration in Chrome, Edge, IE, Opera, and Safari.\n */\n\n abbr[title] {\n border-bottom: none; /* 1 */\n text-decoration: underline; /* 2 */\n text-decoration: underline dotted; /* 2 */\n }\n\n /**\n * Add the correct font weight in Chrome, Edge, and Safari.\n */\n\n b,\n strong {\n font-weight: bolder;\n }\n\n /**\n * 1. Correct the inheritance and scaling of font size in all browsers.\n * 2. Correct the odd `em` font sizing in all browsers.\n */\n\n code,\n kbd,\n samp {\n font-family: monospace, monospace; /* 1 */\n font-size: 1em; /* 2 */\n }\n\n /**\n * Add the correct font size in all browsers.\n */\n\n small {\n font-size: 80%;\n }\n\n /**\n * Prevent `sub` and `sup` elements from affecting the line height in\n * all browsers.\n */\n\n sub,\n sup {\n font-size: 75%;\n line-height: 0;\n position: relative;\n vertical-align: baseline;\n }\n\n sub {\n bottom: -0.25em;\n }\n\n sup {\n top: -0.5em;\n }\n\n /* Embedded content\n ========================================================================== */\n\n /**\n * Remove the border on images inside links in IE 10.\n */\n\n img {\n border-style: none;\n }\n\n /* Forms\n ========================================================================== */\n\n /**\n * 1. Change the font styles in all browsers.\n * 2. Remove the margin in Firefox and Safari.\n */\n\n button,\n input,\n optgroup,\n select,\n textarea {\n font-family: inherit; /* 1 */\n font-size: 100%; /* 1 */\n line-height: 1.15; /* 1 */\n margin: 0; /* 2 */\n }\n\n /**\n * Show the overflow in IE.\n * 1. Show the overflow in Edge.\n */\n\n button,\n input {\n /* 1 */\n overflow: visible;\n }\n\n /**\n * Remove the inheritance of text transform in Edge, Firefox, and IE.\n * 1. Remove the inheritance of text transform in Firefox.\n */\n\n button,\n select {\n /* 1 */\n text-transform: none;\n }\n\n /**\n * Correct the inability to style clickable types in iOS and Safari.\n */\n\n button,\n [type='button'],\n [type='reset'],\n [type='submit'] {\n -webkit-appearance: button;\n cursor: pointer;\n }\n\n /**\n * Remove the inner border and padding in Firefox.\n */\n\n button::-moz-focus-inner,\n [type='button']::-moz-focus-inner,\n [type='reset']::-moz-focus-inner,\n [type='submit']::-moz-focus-inner {\n border-style: none;\n padding: 0;\n }\n\n /**\n * Restore the focus styles unset by the previous rule.\n */\n\n button:-moz-focusring,\n [type='button']:-moz-focusring,\n [type='reset']:-moz-focusring,\n [type='submit']:-moz-focusring {\n outline: 1px dotted ButtonText;\n }\n\n /**\n * Correct the padding in Firefox.\n */\n\n fieldset {\n padding: 0.35em 0.75em 0.625em;\n }\n\n /**\n * 1. Correct the text wrapping in Edge and IE.\n * 2. Correct the color inheritance from `fieldset` elements in IE.\n * 3. Remove the padding so developers are not caught out when they zero out\n * `fieldset` elements in all browsers.\n */\n\n legend {\n box-sizing: border-box; /* 1 */\n color: inherit; /* 2 */\n display: table; /* 1 */\n max-width: 100%; /* 1 */\n padding: 0; /* 3 */\n white-space: normal; /* 1 */\n }\n\n /**\n * Add the correct vertical alignment in Chrome, Firefox, and Opera.\n */\n\n progress {\n vertical-align: baseline;\n }\n\n /**\n * Remove the default vertical scrollbar in IE 10+.\n */\n\n textarea {\n overflow: auto;\n }\n\n /**\n * 1. Add the correct box sizing in IE 10.\n * 2. Remove the padding in IE 10.\n */\n\n [type='checkbox'],\n [type='radio'] {\n box-sizing: border-box; /* 1 */\n padding: 0; /* 2 */\n }\n\n /**\n * Correct the cursor style of increment and decrement buttons in Chrome.\n */\n\n [type='number']::-webkit-inner-spin-button,\n [type='number']::-webkit-outer-spin-button {\n height: auto;\n }\n\n /**\n * 1. Correct the odd appearance in Chrome and Safari.\n * 2. Correct the outline style in Safari.\n */\n\n [type='search'] {\n -webkit-appearance: textfield; /* 1 */\n outline-offset: -2px; /* 2 */\n }\n\n /**\n * Remove the inner padding in Chrome and Safari on macOS.\n */\n\n [type='search']::-webkit-search-decoration {\n -webkit-appearance: none;\n }\n\n /**\n * 1. Correct the inability to style clickable types in iOS and Safari.\n * 2. Change font properties to `inherit` in Safari.\n */\n\n ::-webkit-file-upload-button {\n -webkit-appearance: button; /* 1 */\n font: inherit; /* 2 */\n }\n\n /* Interactive\n ========================================================================== */\n\n /*\n * Add the correct display in Edge, IE 10+, and Firefox.\n */\n\n details {\n display: block;\n }\n\n /*\n * Add the correct display in all browsers.\n */\n\n summary {\n display: list-item;\n }\n\n /* Misc\n ========================================================================== */\n\n /**\n * Add the correct display in IE 10+.\n */\n\n template {\n display: none;\n }\n\n /**\n * Add the correct display in IE 10.\n */\n\n [hidden] {\n display: none;\n }\n\n [contentEditable]:empty:before {\n content: attr(placeholder);\n color: #9ca3af;\n opacity: 1;\n }\n\n /* third container laylout for Firefox */\n @-moz-document url-prefix() {\n .firefox {\n padding: 32px;\n }\n }\n\n /* For screens with a maximum width of 768px (mobile) */\n @media (max-width: 768px) {\n .firefox {\n padding: 16px;\n }\n }\n\n @font-face {\n font-family: 'Inter';\n font-weight: 100 200 300 400 500 600 700 800 900;\n src: url('/fonts/Inter-Variable.ttf');\n }\n\n @font-face {\n font-family: 'IBMPlexMono-Medium';\n font-weight: 500;\n src: url('/fonts/IBMPlexMono-Medium.ttf');\n }\n\n /* Light mode specific autofill styles */\n input:-webkit-autofill,\n input:-webkit-autofill:hover,\n input:-webkit-autofill:focus,\n input:-webkit-autofill:active {\n -webkit-text-fill-color: #343541 !important;\n -webkit-box-shadow: 0 0 0 30px transparent inset !important;\n transition: background-color 5000s ease-in-out 0s;\n caret-color: #343541;\n }\n\n /* Dark mode specific autofill styles */\n .dark input:-webkit-autofill,\n .dark input:-webkit-autofill:hover,\n .dark input:-webkit-autofill:focus,\n .dark input:-webkit-autofill:active {\n -webkit-text-fill-color: #e5e7eb !important;\n -webkit-box-shadow: 0 0 0 30px transparent inset !important;\n background-color: transparent !important;\n caret-color: #e5e7eb;\n }\n\n /* Additional autocomplete dropdown styles for dark mode */\n .dark input:-webkit-autofill::first-line {\n color: #e5e7eb;\n }\n\n .inputbox-style {\n resize: none;\n padding-left: 36px;\n padding-right: 36px;\n }\n\n .bottom-safe {\n bottom: env(safe-area-inset-bottom, 0);\n }\n\n .ellipsis-text {\n overflow: hidden;\n display: -webkit-box;\n -webkit-line-clamp: 3;\n -webkit-box-orient: vertical;\n text-overflow: ellipsis;\n }\n\n .logs-table {\n font-family: 'IBMPlexMono-Medium', system-ui;\n }\n\n .fade-in {\n animation: fadeIn 0.5s ease-in-out;\n }\n\n @keyframes fadeIn {\n 0% {\n opacity: 0;\n }\n 100% {\n opacity: 1;\n }\n }\n\n .fade-in-bubble {\n opacity: 0;\n transform: translateY(10px);\n animation: fadeInUp 0.5s forwards;\n }\n\n .thinking-dots {\n display: inline-flex;\n gap: 4px;\n align-items: center;\n }\n\n .thinking-dots span {\n width: 6px;\n height: 6px;\n border-radius: 50%;\n background-color: #7d54d1;\n animation: thinkingBounce 1.4s infinite ease-in-out both;\n }\n\n .dark .thinking-dots span {\n background-color: #976af3;\n }\n\n .thinking-dots span:nth-child(1) {\n animation-delay: -0.32s;\n }\n\n .thinking-dots span:nth-child(2) {\n animation-delay: -0.16s;\n }\n\n .thinking-dots span:nth-child(3) {\n animation-delay: 0s;\n }\n\n @keyframes thinkingBounce {\n 0%, 80%, 100% {\n transform: scale(0.6);\n opacity: 0.4;\n }\n 40% {\n transform: scale(1);\n opacity: 1;\n }\n }\n\n @keyframes fadeInUp {\n to {\n opacity: 1;\n transform: translateY(0);\n }\n }\n}\n\n/*\n ---break---\n*/\n\n@theme inline {\n --radius-sm: calc(var(--radius) - 4px);\n --radius-md: calc(var(--radius) - 2px);\n --radius-lg: var(--radius);\n --radius-xl: calc(var(--radius) + 4px);\n --radius-2xl: calc(var(--radius) + 8px);\n --radius-3xl: calc(var(--radius) + 12px);\n --radius-4xl: calc(var(--radius) + 16px);\n --color-background: var(--background);\n --color-foreground: var(--foreground);\n --color-card: var(--card);\n --color-card-foreground: var(--card-foreground);\n --color-popover: var(--popover);\n --color-popover-foreground: var(--popover-foreground);\n --color-primary: var(--primary);\n --color-primary-foreground: var(--primary-foreground);\n --color-secondary: var(--secondary);\n --color-secondary-foreground: var(--secondary-foreground);\n --color-muted: var(--muted);\n --color-muted-foreground: var(--muted-foreground);\n --color-accent: var(--accent);\n --color-accent-foreground: var(--accent-foreground);\n --color-destructive: var(--destructive);\n --color-border: var(--border);\n --color-input: var(--input);\n --color-ring: var(--ring);\n --color-chart-1: var(--chart-1);\n --color-chart-2: var(--chart-2);\n --color-chart-3: var(--chart-3);\n --color-chart-4: var(--chart-4);\n --color-chart-5: var(--chart-5);\n --color-sidebar: var(--sidebar);\n --color-sidebar-foreground: var(--sidebar-foreground);\n --color-sidebar-primary: var(--sidebar-primary);\n --color-sidebar-primary-foreground: var(--sidebar-primary-foreground);\n --color-sidebar-accent: var(--sidebar-accent);\n --color-sidebar-accent-foreground: var(--sidebar-accent-foreground);\n --color-sidebar-border: var(--sidebar-border);\n --color-sidebar-ring: var(--sidebar-ring);\n}\n\n/*\n ---break---\n*/\n\n:root {\n --radius: 0.625rem;\n --background: oklch(1 0 0);\n --foreground: oklch(0.145 0 0);\n --card: oklch(1 0 0);\n --card-foreground: oklch(0.145 0 0);\n --popover: oklch(1 0 0);\n --popover-foreground: oklch(0.145 0 0);\n --primary: oklch(0.554 0.185 294.8); /* purple-30 #7d54d1 */\n --primary-foreground: oklch(0.985 0 0);\n --secondary: oklch(0.914 0.035 300.2); /* purple-3000 - light purple */\n --secondary-foreground: oklch(0.554 0.185 294.8);\n --muted: oklch(0.97 0 0);\n --muted-foreground: oklch(0.556 0 0);\n --accent: oklch(0.914 0.035 300.2); /* purple-3000 - light purple hover */\n --accent-foreground: oklch(0.205 0 0);\n --destructive: oklch(0.577 0.245 27.325);\n --border: oklch(0.922 0 0);\n --input: oklch(0.870 0 0); /* neutral gray border */\n --ring: oklch(0.554 0.185 294.8); /* purple-30 focus ring */\n --chart-1: oklch(0.646 0.222 41.116);\n --chart-2: oklch(0.6 0.118 184.704);\n --chart-3: oklch(0.398 0.07 227.392);\n --chart-4: oklch(0.828 0.189 84.429);\n --chart-5: oklch(0.769 0.188 70.08);\n --sidebar: oklch(0.985 0 0);\n --sidebar-foreground: oklch(0.145 0 0);\n --sidebar-primary: oklch(0.554 0.185 294.8); /* purple-30 */\n --sidebar-primary-foreground: oklch(0.985 0 0);\n --sidebar-accent: oklch(0.914 0.035 300.2); /* purple-3000 */\n --sidebar-accent-foreground: oklch(0.205 0 0);\n --sidebar-border: oklch(0.922 0 0);\n --sidebar-ring: oklch(0.554 0.185 294.8); /* purple-30 */\n}\n\n/*\n ---break---\n*/\n\n.dark {\n --background: oklch(0.145 0 0);\n --foreground: oklch(0.985 0 0);\n --card: oklch(0.205 0 0);\n --card-foreground: oklch(0.985 0 0);\n --popover: oklch(0.205 0 0);\n --popover-foreground: oklch(0.985 0 0);\n --primary: oklch(0.636 0.197 295.4); /* violets-are-blue #976af3 */\n --primary-foreground: oklch(0.985 0 0);\n --secondary: oklch(0.269 0.03 295.0); /* dark muted purple */\n --secondary-foreground: oklch(0.867 0.052 300.1);\n --muted: oklch(0.269 0 0);\n --muted-foreground: oklch(0.708 0 0);\n --accent: oklch(0.269 0.04 295.0); /* dark purple hover */\n --accent-foreground: oklch(0.985 0 0);\n --destructive: oklch(0.704 0.191 22.216);\n --border: oklch(1 0 0 / 10%);\n --input: oklch(1 0 0 / 15%);\n --ring: oklch(0.636 0.197 295.4); /* violets-are-blue focus ring */\n --chart-1: oklch(0.488 0.243 264.376);\n --chart-2: oklch(0.696 0.17 162.48);\n --chart-3: oklch(0.769 0.188 70.08);\n --chart-4: oklch(0.627 0.265 303.9);\n --chart-5: oklch(0.645 0.246 16.439);\n --sidebar: oklch(0.205 0 0);\n --sidebar-foreground: oklch(0.985 0 0);\n --sidebar-primary: oklch(0.636 0.197 295.4); /* violets-are-blue */\n --sidebar-primary-foreground: oklch(0.985 0 0);\n --sidebar-accent: oklch(0.269 0.04 295.0); /* dark purple */\n --sidebar-accent-foreground: oklch(0.985 0 0);\n --sidebar-border: oklch(1 0 0 / 10%);\n --sidebar-ring: oklch(0.636 0.197 295.4); /* violets-are-blue */\n}\n\n/*\n ---break---\n*/\n\n@layer base {\n * {\n @apply border-border outline-ring/50;\n }\n body {\n @apply bg-background text-foreground;\n }\n}\n" }, { "path": "frontend/src/lib/utils.ts", "content": "import { type ClassValue, clsx } from 'clsx';\nimport { twMerge } from 'tailwind-merge';\n\nexport function cn(...inputs: ClassValue[]) {\n return twMerge(clsx(inputs));\n}\n" }, { "path": "frontend/src/locale/de.json", "content": "{\n \"language\": \"Deutsch\",\n \"chat\": \"Chat\",\n \"chats\": \"Chats\",\n \"newChat\": \"Neuer Chat\",\n \"inputPlaceholder\": \"Wie kann DocsGPT dir helfen?\",\n \"tagline\": \"DocsGPT verwendet GenAI, bitte überprüfe kritische Informationen anhand der Quellen.\",\n \"sourceDocs\": \"Quelle\",\n \"none\": \"Keine\",\n \"cancel\": \"Abbrechen\",\n \"help\": \"Hilfe\",\n \"emailUs\": \"E-Mail senden\",\n \"documentation\": \"Dokumentation\",\n \"manageAgents\": \"Agenten verwalten\",\n \"demo\": [\n {\n \"header\": \"Über DocsGPT lernen\",\n \"query\": \"Was ist DocsGPT?\"\n },\n {\n \"header\": \"Dokumentation zusammenfassen\",\n \"query\": \"Fasse den aktuellen Kontext zusammen\"\n },\n {\n \"header\": \"Code schreiben\",\n \"query\": \"Schreibe Code für eine API-Anfrage an /api/answer\"\n },\n {\n \"header\": \"Lernunterstützung\",\n \"query\": \"Schreibe mögliche Fragen zum Kontext\"\n }\n ],\n \"settings\": {\n \"label\": \"Einstellungen\",\n \"general\": {\n \"label\": \"Allgemein\",\n \"selectTheme\": \"Design auswählen\",\n \"light\": \"Hell\",\n \"dark\": \"Dunkel\",\n \"selectLanguage\": \"Sprache auswählen\",\n \"chunks\": \"Chunks pro Anfrage\",\n \"prompt\": \"Aktiver Prompt\",\n \"deleteAllLabel\": \"Alle Konversationen löschen\",\n \"deleteAllBtn\": \"Alle löschen\",\n \"addNew\": \"Neu hinzufügen\",\n \"convHistory\": \"Konversationsverlauf\",\n \"none\": \"Keine\",\n \"low\": \"Niedrig\",\n \"medium\": \"Mittel\",\n \"high\": \"Hoch\",\n \"unlimited\": \"Unbegrenzt\",\n \"default\": \"Standard\",\n \"add\": \"Hinzufügen\"\n },\n \"sources\": {\n \"title\": \"Hier kannst du alle verfügbaren Quelldateien verwalten, die dir zur Verfügung stehen und die du hochgeladen hast.\",\n \"label\": \"Quellen\",\n \"name\": \"Quellenname\",\n \"date\": \"Vektor-Datum\",\n \"type\": \"Typ\",\n \"tokenUsage\": \"Token-Verbrauch\",\n \"noData\": \"Keine vorhandenen Quellen\",\n \"searchPlaceholder\": \"Suchen...\",\n \"addNew\": \"Neu hinzufügen\",\n \"addSource\": \"Quelle hinzufügen\",\n \"addChunk\": \"Chunk hinzufügen\",\n \"preLoaded\": \"Vorgeladen\",\n \"private\": \"Privat\",\n \"sync\": \"Synchronisieren\",\n \"syncNow\": \"Jetzt synchronisieren\",\n \"syncing\": \"Synchronisiere...\",\n \"syncConfirmation\": \"Bist du sicher, dass du \\\"{{sourceName}}\\\" synchronisieren möchtest? Dies aktualisiert den Inhalt mit deinem Cloud-Speicher und kann Änderungen an einzelnen Chunks überschreiben.\",\n \"syncFrequency\": {\n \"never\": \"Nie\",\n \"daily\": \"Täglich\",\n \"weekly\": \"Wöchentlich\",\n \"monthly\": \"Monatlich\"\n },\n \"actions\": \"Aktionen\",\n \"view\": \"Anzeigen\",\n \"deleteWarning\": \"Bist du sicher, dass du \\\"{{name}}\\\" löschen möchtest?\",\n \"confirmDelete\": \"Bist du sicher, dass du diese Datei löschen möchtest? Diese Aktion kann nicht rückgängig gemacht werden.\",\n \"backToAll\": \"Zurück zu allen Quellen\",\n \"chunks\": \"Chunks\",\n \"noChunks\": \"Keine Chunks gefunden\",\n \"noChunksAlt\": \"Keine Chunks gefunden\",\n \"goToSources\": \"Zu den Quellen\",\n \"uploadNew\": \"Neu hochladen\",\n \"searchFiles\": \"Dateien suchen...\",\n \"noResults\": \"Keine Ergebnisse gefunden\",\n \"fileName\": \"Name\",\n \"tokens\": \"Tokens\",\n \"size\": \"Größe\",\n \"fileAlt\": \"Datei\",\n \"folderAlt\": \"Ordner\",\n \"parentFolderAlt\": \"Übergeordneter Ordner\",\n \"menuAlt\": \"Menü\",\n \"tokensUnit\": \"Tokens\",\n \"editAlt\": \"Bearbeiten\",\n \"uploading\": \"Wird hochgeladen…\",\n \"deleting\": \"Wird gelöscht…\",\n \"queued\": \"In Warteschlange: {{count}}\",\n \"addFile\": \"Datei hinzufügen\",\n \"uploadingFilesTitle\": \"Dateien werden hochgeladen...\",\n \"deletingTitle\": \"Wird gelöscht...\",\n \"deleteDirectoryWarning\": \"Bist du sicher, dass du das Verzeichnis \\\"{{name}}\\\" und seinen gesamten Inhalt löschen möchtest? Diese Aktion kann nicht rückgängig gemacht werden.\",\n \"searchAlt\": \"Suchen\"\n },\n \"apiKeys\": {\n \"label\": \"Chatbots\",\n \"name\": \"Name\",\n \"key\": \"API-Schlüssel\",\n \"sourceDoc\": \"Quelldokument\",\n \"createNew\": \"Neu erstellen\",\n \"noData\": \"Keine vorhandenen Chatbots\",\n \"deleteConfirmation\": \"Bist du sicher, dass du den API-Schlüssel '{{name}}' löschen möchtest?\",\n \"description\": \"Hier kannst du deine Chatbots erstellen und verwalten. Chatbots können als Widgets auf Websites eingebunden oder in deinen Anwendungen verwendet werden.\"\n },\n \"analytics\": {\n \"label\": \"Analytik\",\n \"filterByChatbot\": \"Nach Chatbot filtern\",\n \"selectChatbot\": \"Chatbot auswählen\",\n \"filterOptions\": {\n \"hour\": \"Stunde\",\n \"last24Hours\": \"24 Stunden\",\n \"last7Days\": \"7 Tage\",\n \"last15Days\": \"15 Tage\",\n \"last30Days\": \"30 Tage\"\n },\n \"messages\": \"Nachrichten\",\n \"tokenUsage\": \"Token-Verbrauch\",\n \"userFeedback\": \"Benutzer-Feedback\",\n \"filterPlaceholder\": \"Filter\",\n \"none\": \"Keine\",\n \"positiveFeedback\": \"Positives Feedback\",\n \"negativeFeedback\": \"Negatives Feedback\"\n },\n \"logs\": {\n \"label\": \"Protokolle\",\n \"filterByChatbot\": \"Nach Chatbot filtern\",\n \"selectChatbot\": \"Chatbot auswählen\",\n \"none\": \"Keine\",\n \"tableHeader\": \"API-generierte / Chatbot-Konversationen\"\n },\n \"tools\": {\n \"label\": \"Werkzeuge\",\n \"searchPlaceholder\": \"Werkzeuge suchen...\",\n \"addTool\": \"Werkzeug hinzufügen\",\n \"noToolsFound\": \"Keine Werkzeuge gefunden\",\n \"selectToolSetup\": \"Wähle ein Werkzeug zur Einrichtung\",\n \"settingsIconAlt\": \"Einstellungssymbol\",\n \"configureToolAria\": \"{{toolName}} konfigurieren\",\n \"toggleToolAria\": \"{{toolName}} umschalten\",\n \"manageTools\": \"Zu den Werkzeugen\",\n \"edit\": \"Bearbeiten\",\n \"delete\": \"Löschen\",\n \"deleteWarning\": \"Bist du sicher, dass du das Werkzeug \\\"{{toolName}}\\\" löschen möchtest?\",\n \"unsavedChanges\": \"Du hast ungespeicherte Änderungen, die verloren gehen, wenn du ohne Speichern verlässt.\",\n \"leaveWithoutSaving\": \"Ohne Speichern verlassen\",\n \"saveAndLeave\": \"Speichern und verlassen\",\n \"customName\": \"Benutzerdefinierter Name\",\n \"customNamePlaceholder\": \"Gib einen benutzerdefinierten Namen ein (optional)\",\n \"authentication\": \"Authentifizierung\",\n \"actions\": \"Aktionen\",\n \"addAction\": \"Aktion hinzufügen\",\n \"importSpec\": \"Spezifikation importieren\",\n \"searchActions\": \"Aktionen suchen...\",\n \"noActionsMatch\": \"Keine Aktionen passen zu deiner Suche\",\n \"actionAlreadyExists\": \"Eine Aktion mit diesem Namen existiert bereits\",\n \"noActionsFound\": \"Keine Aktionen gefunden\",\n \"url\": \"URL\",\n \"urlPlaceholder\": \"URL eingeben\",\n \"method\": \"Methode\",\n \"description\": \"Beschreibung\",\n \"descriptionPlaceholder\": \"Beschreibung eingeben\",\n \"bodyContentType\": \"Body-Inhaltstyp\",\n \"headers\": \"Header\",\n \"queryParameters\": \"Abfrageparameter\",\n \"body\": \"Body\",\n \"deleteActionWarning\": \"Bist du sicher, dass du die Aktion \\\"{{name}}\\\" löschen möchtest?\",\n \"backToAllTools\": \"Zurück zu allen Werkzeugen\",\n \"save\": \"Speichern\",\n \"fieldName\": \"Feldname\",\n \"fieldType\": \"Feldtyp\",\n \"filledByLLM\": \"Vom LLM ausgefüllt\",\n \"fieldDescription\": \"Feldbeschreibung\",\n \"value\": \"Wert\",\n \"addProperty\": \"Eigenschaft hinzufügen\",\n \"propertyName\": \"Neuer Eigenschaftsschlüssel\",\n \"add\": \"Hinzufügen\",\n \"cancel\": \"Abbrechen\",\n \"addNew\": \"Neu hinzufügen\",\n \"name\": \"Name\",\n \"type\": \"Typ\",\n \"mcp\": {\n \"addServer\": \"MCP-Server hinzufügen\",\n \"editServer\": \"Server bearbeiten\",\n \"serverName\": \"Servername\",\n \"serverUrl\": \"Server-URL\",\n \"headerName\": \"Header-Name\",\n \"timeout\": \"Timeout (Sekunden)\",\n \"testConnection\": \"Verbindung testen\",\n \"testing\": \"Teste...\",\n \"saving\": \"Speichere...\",\n \"save\": \"Speichern\",\n \"cancel\": \"Abbrechen\",\n \"noAuth\": \"Keine Authentifizierung\",\n \"oauthInProgress\": \"Warte auf OAuth-Abschluss...\",\n \"oauthCompleted\": \"OAuth erfolgreich abgeschlossen\",\n \"authType\": \"Authentifizierungstyp\",\n \"defaultServerName\": \"Mein MCP-Server\",\n \"authTypes\": {\n \"none\": \"Keine Authentifizierung\",\n \"apiKey\": \"API-Schlüssel\",\n \"bearer\": \"Bearer-Token\",\n \"oauth\": \"OAuth\",\n \"basic\": \"Basis-Authentifizierung\"\n },\n \"placeholders\": {\n \"serverUrl\": \"https://api.beispiel.com\",\n \"apiKey\": \"Dein geheimer API-Schlüssel\",\n \"bearerToken\": \"Dein geheimes Token\",\n \"username\": \"Dein Benutzername\",\n \"password\": \"Dein Passwort\",\n \"oauthScopes\": \"OAuth-Bereiche (kommagetrennt)\"\n },\n \"errors\": {\n \"nameRequired\": \"Servername ist erforderlich\",\n \"urlRequired\": \"Server-URL ist erforderlich\",\n \"invalidUrl\": \"Bitte gib eine gültige URL ein\",\n \"apiKeyRequired\": \"API-Schlüssel ist erforderlich\",\n \"tokenRequired\": \"Bearer-Token ist erforderlich\",\n \"usernameRequired\": \"Benutzername ist erforderlich\",\n \"passwordRequired\": \"Passwort ist erforderlich\",\n \"testFailed\": \"Verbindungstest fehlgeschlagen\",\n \"saveFailed\": \"MCP-Server konnte nicht gespeichert werden\",\n \"oauthFailed\": \"OAuth-Prozess fehlgeschlagen oder abgebrochen\",\n \"oauthTimeout\": \"OAuth-Prozess abgelaufen, bitte erneut versuchen\",\n \"timeoutRange\": \"Timeout muss zwischen 1 und 300 Sekunden liegen\"\n }\n }\n },\n \"scrollTabsLeft\": \"Tabs nach links scrollen\",\n \"tabsAriaLabel\": \"Einstellungs-Tabs\",\n \"scrollTabsRight\": \"Tabs nach rechts scrollen\"\n },\n \"modals\": {\n \"uploadDoc\": {\n \"label\": \"Neues Dokument hochladen\",\n \"select\": \"Wähle, wie du dein Dokument zu DocsGPT hochladen möchtest\",\n \"selectSource\": \"Wähle die Art, wie du deine Quelle hinzufügen möchtest\",\n \"selectedFiles\": \"Ausgewählte Dateien\",\n \"noFilesSelected\": \"Keine Dateien ausgewählt\",\n \"file\": \"Vom Gerät hochladen\",\n \"back\": \"Zurück\",\n \"wait\": \"Bitte warten ...\",\n \"remote\": \"Von Website sammeln\",\n \"start\": \"Chat starten\",\n \"name\": \"Name\",\n \"choose\": \"Dateien auswählen\",\n \"info\": \"Bitte lade .pdf, .txt, .rst, .csv, .xlsx, .docx, .md, .html, .epub, .json, .pptx, .zip hoch (max. 25 MB)\",\n \"uploadedFiles\": \"Hochgeladene Dateien\",\n \"cancel\": \"Abbrechen\",\n \"train\": \"Trainieren\",\n \"link\": \"Link\",\n \"urlLink\": \"URL-Link\",\n \"repoUrl\": \"Repository-URL\",\n \"reddit\": {\n \"id\": \"Client-ID\",\n \"secret\": \"Client-Secret\",\n \"agent\": \"User-Agent\",\n \"searchQueries\": \"Suchanfragen\",\n \"numberOfPosts\": \"Anzahl der Beiträge\",\n \"addQuery\": \"Anfrage hinzufügen\"\n },\n \"drag\": {\n \"title\": \"Anhänge hier ablegen\",\n \"description\": \"Loslassen, um deine Anhänge hochzuladen\"\n },\n \"progress\": {\n \"upload\": \"Upload läuft\",\n \"training\": \"Upload läuft\",\n \"completed\": \"Upload abgeschlossen\",\n \"failed\": \"Upload fehlgeschlagen\",\n \"wait\": \"Dies kann einige Minuten dauern\",\n \"preparing\": \"Upload wird vorbereitet\",\n \"tokenLimit\": \"Token-Limit überschritten, bitte lade ein kleineres Dokument hoch\",\n \"expandDetails\": \"Upload-Details erweitern\",\n \"collapseDetails\": \"Upload-Details einklappen\",\n \"dismiss\": \"Upload-Benachrichtigung schließen\",\n \"uploadProgress\": \"Upload-Fortschritt {{progress}}%\",\n \"clear\": \"Löschen\"\n },\n \"showAdvanced\": \"Erweiterte Optionen anzeigen\",\n \"hideAdvanced\": \"Erweiterte Optionen ausblenden\",\n \"ingestors\": {\n \"local_file\": {\n \"label\": \"Datei hochladen\",\n \"heading\": \"Neues Dokument hochladen\"\n },\n \"crawler\": {\n \"label\": \"Crawler\",\n \"heading\": \"Inhalt mit Web-Crawler hinzufügen\"\n },\n \"url\": {\n \"label\": \"Link\",\n \"heading\": \"Inhalt von URL hinzufügen\"\n },\n \"github\": {\n \"label\": \"GitHub\",\n \"heading\": \"Inhalt von GitHub hinzufügen\"\n },\n \"reddit\": {\n \"label\": \"Reddit\",\n \"heading\": \"Inhalt von Reddit hinzufügen\"\n },\n \"google_drive\": {\n \"label\": \"Google Drive\",\n \"heading\": \"Von Google Drive hochladen\"\n },\n \"s3\": {\n \"label\": \"Amazon S3\",\n \"heading\": \"Inhalt von Amazon S3 hinzufügen\"\n }\n },\n \"connectors\": {\n \"auth\": {\n \"connectedUser\": \"Verbundener Benutzer\",\n \"authFailed\": \"Authentifizierung fehlgeschlagen\",\n \"authUrlFailed\": \"Autorisierungs-URL konnte nicht abgerufen werden\",\n \"popupBlocked\": \"Authentifizierungsfenster konnte nicht geöffnet werden. Bitte erlaube Popups.\",\n \"authCancelled\": \"Authentifizierung wurde abgebrochen\",\n \"connectedAs\": \"Verbunden als {{email}}\",\n \"disconnect\": \"Trennen\"\n },\n \"googleDrive\": {\n \"connect\": \"Mit Google Drive verbinden\",\n \"sessionExpired\": \"Sitzung abgelaufen. Bitte verbinde dich erneut mit Google Drive.\",\n \"sessionExpiredGeneric\": \"Sitzung abgelaufen. Bitte verbinde dein Konto erneut.\",\n \"validateFailed\": \"Sitzung konnte nicht validiert werden. Bitte verbinde dich erneut.\",\n \"noSession\": \"Keine gültige Sitzung gefunden. Bitte verbinde dich erneut mit Google Drive.\",\n \"noAccessToken\": \"Kein Zugriffstoken verfügbar. Bitte verbinde dich erneut mit Google Drive.\",\n \"pickerFailed\": \"Dateiauswahl konnte nicht geöffnet werden. Bitte versuche es erneut.\",\n \"selectedFiles\": \"Ausgewählte Dateien\",\n \"selectFiles\": \"Dateien auswählen\",\n \"loading\": \"Laden...\",\n \"noFilesSelected\": \"Keine Dateien oder Ordner ausgewählt\",\n \"folders\": \"Ordner\",\n \"files\": \"Dateien\",\n \"remove\": \"Entfernen\",\n \"folderAlt\": \"Ordner\",\n \"fileAlt\": \"Datei\"\n }\n }\n },\n \"createAPIKey\": {\n \"label\": \"Neuen API-Schlüssel erstellen\",\n \"apiKeyName\": \"API-Schlüssel-Name\",\n \"chunks\": \"Chunks pro Anfrage\",\n \"prompt\": \"Aktiven Prompt auswählen\",\n \"sourceDoc\": \"Quelldokument\",\n \"create\": \"Erstellen\"\n },\n \"saveKey\": {\n \"note\": \"Bitte speichere deinen Schlüssel\",\n \"disclaimer\": \"Dies ist das einzige Mal, dass dein Schlüssel angezeigt wird.\",\n \"copy\": \"Kopieren\",\n \"copied\": \"Kopiert\",\n \"confirm\": \"Ich habe den Schlüssel gespeichert\",\n \"apiKeyLabel\": \"API-Schlüssel\"\n },\n \"deleteConv\": {\n \"confirm\": \"Bist du sicher, dass du alle Konversationen löschen möchtest?\",\n \"delete\": \"Löschen\"\n },\n \"shareConv\": {\n \"label\": \"Öffentliche Seite zum Teilen erstellen\",\n \"note\": \"Quelldokument, persönliche Informationen und weitere Konversationen bleiben privat\",\n \"create\": \"Erstellen\",\n \"option\": \"Benutzern weitere Eingaben erlauben\"\n },\n \"configTool\": {\n \"title\": \"Werkzeug-Konfiguration\",\n \"type\": \"Typ\",\n \"apiKeyLabel\": \"API-Schlüssel / OAuth\",\n \"apiKeyPlaceholder\": \"API-Schlüssel / OAuth eingeben\",\n \"addButton\": \"Werkzeug hinzufügen\",\n \"closeButton\": \"Schließen\",\n \"customNamePlaceholder\": \"Benutzerdefinierten Namen eingeben (optional)\"\n },\n \"prompts\": {\n \"addPrompt\": \"Prompt hinzufügen\",\n \"addDescription\": \"Füge deinen benutzerdefinierten Prompt hinzu und speichere ihn in DocsGPT\",\n \"editPrompt\": \"Prompt bearbeiten\",\n \"editDescription\": \"Bearbeite deinen benutzerdefinierten Prompt und speichere ihn in DocsGPT\",\n \"promptName\": \"Prompt-Name\",\n \"promptText\": \"Prompt-Text\",\n \"save\": \"Speichern\",\n \"cancel\": \"Abbrechen\",\n \"nameExists\": \"Name existiert bereits\",\n \"deleteConfirmation\": \"Bist du sicher, dass du den Prompt '{{name}}' löschen möchtest?\",\n \"placeholderText\": \"Gib hier deinen Prompt-Text ein...\",\n \"addExamplePlaceholder\": \"Bitte fasse diesen Text zusammen:\",\n \"variablesLabel\": \"Variablen\",\n \"variablesSubtext\": \"Klicken zum Einfügen in den Prompt\",\n \"variablesDescription\": \"Klicken zum Einfügen in den Prompt\",\n \"systemVariables\": \"Klicken zum Einfügen in den Prompt\",\n \"toolVariables\": \"Werkzeug-Variablen\",\n \"systemVariablesDropdownLabel\": \"System-Variablen\",\n \"systemVariableOptions\": {\n \"sourceContent\": \"Quelleninhalte\",\n \"sourceSummaries\": \"Alias für Inhalte (abwärtskompatibel)\",\n \"sourceDocuments\": \"Dokumentenobjekte-Liste\",\n \"sourceCount\": \"Anzahl der abgerufenen Dokumente\",\n \"systemDate\": \"Aktuelles Datum (JJJJ-MM-TT)\",\n \"systemTime\": \"Aktuelle Uhrzeit (HH:MM:SS)\",\n \"systemTimestamp\": \"ISO 8601 Zeitstempel\",\n \"systemRequestId\": \"Eindeutige Anfrage-ID\",\n \"systemUserId\": \"Aktuelle Benutzer-ID\"\n },\n \"learnAboutPrompts\": \"Mehr über Prompts erfahren →\",\n \"publicPromptEditDisabled\": \"Öffentliche Prompts können nicht bearbeitet werden\",\n \"promptTypePublic\": \"öffentlich\",\n \"promptTypePrivate\": \"privat\"\n },\n \"chunk\": {\n \"add\": \"Chunk hinzufügen\",\n \"edit\": \"Bearbeiten\",\n \"title\": \"Titel\",\n \"enterTitle\": \"Titel eingeben\",\n \"bodyText\": \"Textkörper\",\n \"promptText\": \"Prompt-Text\",\n \"save\": \"Speichern\",\n \"close\": \"Schließen\",\n \"cancel\": \"Abbrechen\",\n \"delete\": \"Löschen\",\n \"deleteConfirmation\": \"Bist du sicher, dass du diesen Chunk löschen möchtest?\"\n },\n \"addAction\": {\n \"title\": \"Neue Aktion\",\n \"actionNamePlaceholder\": \"Aktionsname\",\n \"invalidFormat\": \"Ungültiges Funktionsnamenformat. Verwende nur Buchstaben, Zahlen, Unterstriche und Bindestriche.\",\n \"formatHelp\": \"Verwende nur Buchstaben, Zahlen, Unterstriche und Bindestriche (z.B. `get_data`, `send_report`, etc.)\",\n \"addButton\": \"Hinzufügen\"\n },\n \"agentDetails\": {\n \"title\": \"Zugangsdaten\",\n \"publicLink\": \"Öffentlicher Link\",\n \"apiKey\": \"API-Schlüssel\",\n \"webhookUrl\": \"Webhook-URL\",\n \"generate\": \"Generieren\",\n \"test\": \"Testen\",\n \"learnMore\": \"Mehr erfahren\"\n },\n \"importSpec\": {\n \"title\": \"API-Spezifikation importieren\",\n \"description\": \"Lade eine OpenAPI 3.x- oder Swagger 2.0-Spezifikationsdatei hoch, um automatisch Aktionen zu generieren.\",\n \"dropzoneText\": \"Zum Hochladen klicken oder per Drag & Drop\",\n \"supportedFormats\": \"JSON- oder YAML-Format\",\n \"invalidFileType\": \"Ungültiger Dateityp. Bitte eine JSON- oder YAML-Datei hochladen.\",\n \"parseError\": \"Spezifikation konnte nicht geparst werden. Bitte Dateiformat prüfen.\",\n \"version\": \"Version\",\n \"baseUrl\": \"Basis-URL\",\n \"actionsFound\": \"{{count}} Aktionen gefunden\",\n \"selectAll\": \"Alle auswählen\",\n \"deselectAll\": \"Alle abwählen\",\n \"cancel\": \"Abbrechen\",\n \"parse\": \"Parsen\",\n \"import\": \"Importieren ({{count}})\"\n }\n },\n \"sharedConv\": {\n \"subtitle\": \"Erstellt mit\",\n \"button\": \"Mit DocsGPT starten\",\n \"meta\": \"DocsGPT verwendet GenAI, bitte überprüfe kritische Informationen anhand der Quellen.\"\n },\n \"convTile\": {\n \"share\": \"Teilen\",\n \"delete\": \"Löschen\",\n \"rename\": \"Umbenennen\",\n \"deleteWarning\": \"Bist du sicher, dass du diese Konversation löschen möchtest?\"\n },\n \"pagination\": {\n \"rowsPerPage\": \"Zeilen pro Seite\",\n \"pageOf\": \"Seite {{currentPage}} von {{totalPages}}\",\n \"firstPage\": \"Erste Seite\",\n \"previousPage\": \"Vorherige Seite\",\n \"nextPage\": \"Nächste Seite\",\n \"lastPage\": \"Letzte Seite\"\n },\n \"conversation\": {\n \"copy\": \"Kopieren\",\n \"copied\": \"Kopiert\",\n \"speak\": \"Vorlesen\",\n \"answer\": \"Antwort\",\n \"edit\": {\n \"update\": \"Aktualisieren\",\n \"cancel\": \"Abbrechen\",\n \"placeholder\": \"Aktualisierte Anfrage eingeben...\"\n },\n \"sources\": {\n \"title\": \"Quellen\",\n \"text\": \"Wähle deine Quellen\",\n \"link\": \"Quellen-Link\",\n \"view_more\": \"{{count}} weitere Quellen\",\n \"noSourcesAvailable\": \"Keine Quellen verfügbar\"\n },\n \"attachments\": {\n \"attach\": \"Anhängen\",\n \"remove\": \"Anhang entfernen\"\n },\n \"retry\": \"Erneut versuchen\",\n \"reasoning\": \"Begründung\"\n },\n \"agents\": {\n \"title\": \"Agenten\",\n \"description\": \"Entdecke und erstelle benutzerdefinierte Versionen von DocsGPT, die Anweisungen, zusätzliches Wissen und beliebige Kombinationen von Fähigkeiten kombinieren\",\n \"newAgent\": \"Neuer Agent\",\n \"backToAll\": \"Zurück zu allen Agenten\",\n \"searchPlaceholder\": \"Suchen...\",\n \"noSearchResults\": \"Keine Agenten gefunden\",\n \"tryDifferentSearch\": \"Versuche einen anderen Suchbegriff\",\n \"filters\": {\n \"all\": \"Alle\",\n \"byDocsGPT\": \"Von DocsGPT\",\n \"byMe\": \"Von mir\",\n \"shared\": \"Mit mir geteilt\"\n },\n \"sections\": {\n \"template\": {\n \"title\": \"Von DocsGPT\",\n \"description\": \"Von DocsGPT bereitgestellte Agenten\",\n \"emptyState\": \"Keine Vorlagen-Agenten gefunden.\"\n },\n \"user\": {\n \"title\": \"Von mir\",\n \"description\": \"Von dir erstellte oder veröffentlichte Agenten\",\n \"emptyState\": \"Du hast noch keine Agenten erstellt.\"\n },\n \"shared\": {\n \"title\": \"Mit mir geteilt\",\n \"description\": \"Über einen öffentlichen Link importierte Agenten\",\n \"emptyState\": \"Keine geteilten Agenten gefunden.\"\n }\n },\n \"form\": {\n \"headings\": {\n \"new\": \"Neuer Agent\",\n \"edit\": \"Agent bearbeiten\",\n \"draft\": \"Neuer Agent (Entwurf)\"\n },\n \"buttons\": {\n \"publish\": \"Veröffentlichen\",\n \"save\": \"Speichern\",\n \"saveDraft\": \"Entwurf speichern\",\n \"cancel\": \"Abbrechen\",\n \"delete\": \"Löschen\",\n \"logs\": \"Protokolle\",\n \"accessDetails\": \"Zugangsdaten\",\n \"add\": \"Hinzufügen\"\n },\n \"sections\": {\n \"meta\": \"Meta\",\n \"source\": \"Quelle\",\n \"prompt\": \"Prompt\",\n \"tools\": \"Werkzeuge\",\n \"agentType\": \"Agententyp\",\n \"models\": \"Modelle\",\n \"advanced\": \"Erweitert\",\n \"preview\": \"Vorschau\"\n },\n \"placeholders\": {\n \"agentName\": \"Agentenname\",\n \"describeAgent\": \"Beschreibe deinen Agenten\",\n \"selectSources\": \"Quellen auswählen\",\n \"chunksPerQuery\": \"Chunks pro Anfrage\",\n \"selectType\": \"Typ auswählen\",\n \"selectTools\": \"Werkzeuge auswählen\",\n \"selectModels\": \"Modelle für diesen Agenten auswählen\",\n \"selectDefaultModel\": \"Standardmodell auswählen\",\n \"enterTokenLimit\": \"Token-Limit eingeben\",\n \"enterRequestLimit\": \"Anfrage-Limit eingeben\"\n },\n \"sourcePopup\": {\n \"title\": \"Quellen auswählen\",\n \"searchPlaceholder\": \"Quellen suchen...\",\n \"noOptionsMessage\": \"Keine Quellen verfügbar\"\n },\n \"toolsPopup\": {\n \"title\": \"Werkzeuge auswählen\",\n \"searchPlaceholder\": \"Werkzeuge suchen...\",\n \"noOptionsMessage\": \"Keine Werkzeuge verfügbar\"\n },\n \"modelsPopup\": {\n \"title\": \"Modelle auswählen\",\n \"searchPlaceholder\": \"Modelle suchen...\",\n \"noOptionsMessage\": \"Keine Modelle verfügbar\"\n },\n \"upload\": {\n \"clickToUpload\": \"Klicken zum Hochladen\",\n \"dragAndDrop\": \" oder per Drag & Drop\"\n },\n \"agentTypes\": {\n \"classic\": \"Klassisch\",\n \"react\": \"ReAct\"\n },\n \"labels\": {\n \"defaultModel\": \"Standardmodell\"\n },\n \"advanced\": {\n \"jsonSchema\": \"JSON-Antwortschema\",\n \"jsonSchemaDescription\": \"Definiere ein JSON-Schema, um ein strukturiertes Ausgabeformat zu erzwingen\",\n \"validJson\": \"Gültiges JSON\",\n \"invalidJson\": \"Ungültiges JSON - zur Aktivierung des Speicherns beheben\",\n \"tokenLimiting\": \"Token-Limitierung\",\n \"tokenLimitingDescription\": \"Begrenze die täglich von diesem Agenten verwendbaren Tokens\",\n \"requestLimiting\": \"Anfrage-Limitierung\",\n \"requestLimitingDescription\": \"Begrenze die täglich an diesen Agenten gestellten Anfragen\"\n },\n \"preview\": {\n \"publishedPreview\": \"Veröffentlichte Agenten können hier in der Vorschau angezeigt werden\"\n },\n \"externalKb\": \"Externe KB\"\n },\n \"logs\": {\n \"title\": \"Agenten-Protokolle\",\n \"lastUsedAt\": \"Zuletzt verwendet am\",\n \"noUsageHistory\": \"Kein Nutzungsverlauf\",\n \"tableHeader\": \"Agenten-Endpunkt-Protokolle\"\n },\n \"shared\": {\n \"notFound\": \"Kein Agent gefunden. Bitte stelle sicher, dass der Agent geteilt ist.\"\n },\n \"preview\": {\n \"testMessage\": \"Teste deinen Agenten hier. Veröffentlichte Agenten können in Konversationen verwendet werden.\"\n },\n \"deleteConfirmation\": \"Bist du sicher, dass du diesen Agenten löschen möchtest?\",\n \"folders\": {\n \"newFolder\": \"Neuer Ordner\",\n \"createFolder\": \"Ordner erstellen\",\n \"folderName\": \"Ordnername\",\n \"rename\": \"Umbenennen\",\n \"delete\": \"Löschen\",\n \"deleteConfirm\": \"Bist du sicher, dass du diesen Ordner löschen möchtest? Agenten im Ordner werden verschoben.\",\n \"empty\": \"Dieser Ordner ist leer\",\n \"moveToFolder\": \"In Ordner verschieben\",\n \"moveTo\": \"Verschieben\",\n \"move\": \"Verschieben\",\n \"noFolder\": \"Kein Ordner (Stammverzeichnis)\",\n \"backToRoot\": \"Zurück\",\n \"noSubfolders\": \"Keine Unterordner\",\n \"noFolders\": \"Noch keine Ordner\"\n }\n },\n \"components\": {\n \"fileUpload\": {\n \"clickToUpload\": \"Klicken zum Hochladen oder per Drag & Drop\",\n \"dropFiles\": \"Dateien hier ablegen\",\n \"fileTypes\": \"PNG, JPG, JPEG bis zu\",\n \"sizeLimitUnit\": \"MB\",\n \"fileSizeError\": \"Datei überschreitet {{size}}MB-Limit\"\n }\n },\n \"pageNotFound\": {\n \"title\": \"404\",\n \"message\": \"Die gesuchte Seite existiert nicht.\",\n \"goHome\": \"Zur Startseite\"\n },\n \"filePicker\": {\n \"searchPlaceholder\": \"Dateien und Ordner suchen...\",\n \"itemsSelected\": \"{{count}} ausgewählt\",\n \"name\": \"Name\",\n \"lastModified\": \"Zuletzt geändert\",\n \"size\": \"Größe\",\n \"myFiles\": \"Meine Dateien\",\n \"sharedWithMe\": \"Mit mir geteilt\",\n \"loadingMore\": \"Weitere Dateien laden...\"\n },\n \"actionButtons\": {\n \"openNewChat\": \"Neuen Chat öffnen\",\n \"share\": \"Teilen\"\n },\n \"mermaid\": {\n \"downloadOptions\": \"Download-Optionen\",\n \"viewCode\": \"Code anzeigen\",\n \"decreaseZoom\": \"Verkleinern\",\n \"resetZoom\": \"Zoom zurücksetzen\",\n \"increaseZoom\": \"Vergrößern\"\n },\n \"navigation\": {\n \"agents\": \"Agenten\"\n },\n \"notification\": {\n \"ariaLabel\": \"Benachrichtigung\",\n \"closeAriaLabel\": \"Benachrichtigung schließen\"\n },\n \"prompts\": {\n \"textAriaLabel\": \"Prompt-Text\"\n }\n}\n" }, { "path": "frontend/src/locale/en.json", "content": "{\n \"language\": \"English\",\n \"chat\": \"Chat\",\n \"chats\": \"Chats\",\n \"newChat\": \"New Chat\",\n \"inputPlaceholder\": \"How can DocsGPT help you?\",\n \"tagline\": \"DocsGPT uses GenAI, please review critical information using sources.\",\n \"sourceDocs\": \"Source\",\n \"none\": \"None\",\n \"cancel\": \"Cancel\",\n \"help\": \"Help\",\n \"emailUs\": \"Email us\",\n \"documentation\": \"Documentation\",\n \"manageAgents\": \"Manage Agents\",\n \"demo\": [\n {\n \"header\": \"Learn about DocsGPT\",\n \"query\": \"What is DocsGPT?\"\n },\n {\n \"header\": \"Summarize documentation\",\n \"query\": \"Summarize current context\"\n },\n {\n \"header\": \"Write Code\",\n \"query\": \"Write code for api request to /api/answer\"\n },\n {\n \"header\": \"Learning Assistance\",\n \"query\": \"Write potential questions for context\"\n }\n ],\n \"settings\": {\n \"label\": \"Settings\",\n \"general\": {\n \"label\": \"General\",\n \"selectTheme\": \"Select Theme\",\n \"light\": \"Light\",\n \"dark\": \"Dark\",\n \"selectLanguage\": \"Select Language\",\n \"chunks\": \"Chunks processed per query\",\n \"prompt\": \"Active Prompt\",\n \"deleteAllLabel\": \"Delete All Conversations\",\n \"deleteAllBtn\": \"Delete All\",\n \"addNew\": \"Add New\",\n \"convHistory\": \"Conversation History\",\n \"none\": \"None\",\n \"low\": \"Low\",\n \"medium\": \"Medium\",\n \"high\": \"High\",\n \"unlimited\": \"Unlimited\",\n \"default\": \"Default\",\n \"add\": \"Add\"\n },\n \"sources\": {\n \"title\": \"Here you can manage all of the source file that are available to you and those you have uploaded.\",\n \"label\": \"Sources\",\n \"name\": \"Source Name\",\n \"date\": \"Vector Date\",\n \"type\": \"Type\",\n \"tokenUsage\": \"Token Usage\",\n \"noData\": \"No existing Sources\",\n \"searchPlaceholder\": \"Search...\",\n \"addNew\": \"Add New\",\n \"addSource\": \"Add Source\",\n \"addChunk\": \"Add Chunk\",\n \"preLoaded\": \"Pre-loaded\",\n \"private\": \"Private\",\n \"sync\": \"Sync\",\n \"syncNow\": \"Sync now\",\n \"syncing\": \"Syncing...\",\n \"syncConfirmation\": \"Are you sure you want to sync \\\"{{sourceName}}\\\"? This will update the content with your cloud storage and may override any edits you made to individual chunks.\",\n \"syncFrequency\": {\n \"never\": \"Never\",\n \"daily\": \"Daily\",\n \"weekly\": \"Weekly\",\n \"monthly\": \"Monthly\"\n },\n \"actions\": \"Actions\",\n \"view\": \"View\",\n \"deleteWarning\": \"Are you sure you want to delete \\\"{{name}}\\\"?\",\n \"confirmDelete\": \"Are you sure you want to delete this file? This action cannot be undone.\",\n \"backToAll\": \"Back to all sources\",\n \"chunks\": \"Chunks\",\n \"noChunks\": \"No chunks found\",\n \"noChunksAlt\": \"No chunks found\",\n \"goToSources\": \"Go to Sources\",\n \"uploadNew\": \"Upload new\",\n \"searchFiles\": \"Search files...\",\n \"noResults\": \"No results found\",\n \"fileName\": \"Name\",\n \"tokens\": \"Tokens\",\n \"size\": \"Size\",\n \"fileAlt\": \"File\",\n \"folderAlt\": \"Folder\",\n \"parentFolderAlt\": \"Parent folder\",\n \"menuAlt\": \"Menu\",\n \"tokensUnit\": \"tokens\",\n \"editAlt\": \"Edit\",\n \"uploading\": \"Uploading…\",\n \"deleting\": \"Deleting…\",\n \"queued\": \"Queued: {{count}}\",\n \"addFile\": \"Add file\",\n \"uploadingFilesTitle\": \"Uploading files...\",\n \"deletingTitle\": \"Deleting...\",\n \"deleteDirectoryWarning\": \"Are you sure you want to delete the directory \\\"{{name}}\\\" and all its contents? This action cannot be undone.\",\n \"searchAlt\": \"Search\"\n },\n \"apiKeys\": {\n \"label\": \"Chatbots\",\n \"name\": \"Name\",\n \"key\": \"API Key\",\n \"sourceDoc\": \"Source Document\",\n \"createNew\": \"Create New\",\n \"noData\": \"No existing Chatbots\",\n \"deleteConfirmation\": \"Are you sure you want to delete the API key '{{name}}'?\",\n \"description\": \"Here you can create and manage your chatbots. Chatbots can be deployed to websites as widgets or used inside your applications.\"\n },\n \"analytics\": {\n \"label\": \"Analytics\",\n \"filterByChatbot\": \"Filter by chatbot\",\n \"selectChatbot\": \"Select chatbot\",\n \"filterOptions\": {\n \"hour\": \"Hour\",\n \"last24Hours\": \"24 Hours\",\n \"last7Days\": \"7 Days\",\n \"last15Days\": \"15 Days\",\n \"last30Days\": \"30 Days\"\n },\n \"messages\": \"Messages\",\n \"tokenUsage\": \"Token Usage\",\n \"userFeedback\": \"User Feedback\",\n \"filterPlaceholder\": \"Filter\",\n \"none\": \"None\",\n \"positiveFeedback\": \"Positive Feedback\",\n \"negativeFeedback\": \"Negative Feedback\"\n },\n \"logs\": {\n \"label\": \"Logs\",\n \"filterByChatbot\": \"Filter by chatbot\",\n \"selectChatbot\": \"Select chatbot\",\n \"none\": \"None\",\n \"tableHeader\": \"API generated / chatbot conversations\"\n },\n \"tools\": {\n \"label\": \"Tools\",\n \"searchPlaceholder\": \"Search tools...\",\n \"addTool\": \"Add Tool\",\n \"noToolsFound\": \"No tools found\",\n \"selectToolSetup\": \"Select a tool to set up\",\n \"settingsIconAlt\": \"Settings icon\",\n \"configureToolAria\": \"Configure {{toolName}}\",\n \"toggleToolAria\": \"Toggle {{toolName}}\",\n \"manageTools\": \"Go to Tools\",\n \"edit\": \"Edit\",\n \"delete\": \"Delete\",\n \"reconnect\": \"Reconnect\",\n \"authStatus\": {\n \"connected\": \"Connected\",\n \"needsAuth\": \"Needs Auth\",\n \"configured\": \"Configured\"\n },\n \"deleteWarning\": \"Are you sure you want to delete the tool \\\"{{toolName}}\\\" ?\",\n \"unsavedChanges\": \"You have unsaved changes that will be lost if you leave without saving.\",\n \"leaveWithoutSaving\": \"Leave without Saving\",\n \"saveAndLeave\": \"Save and Leave\",\n \"customName\": \"Custom Name\",\n \"customNamePlaceholder\": \"Enter a custom name (optional)\",\n \"authentication\": \"Authentication\",\n \"actions\": \"Actions\",\n \"addAction\": \"Add action\",\n \"importSpec\": \"Import Spec\",\n \"searchActions\": \"Search actions...\",\n \"noActionsMatch\": \"No actions match your search\",\n \"actionAlreadyExists\": \"An action with this name already exists\",\n \"noActionsFound\": \"No actions found\",\n \"url\": \"URL\",\n \"urlPlaceholder\": \"Enter URL\",\n \"method\": \"Method\",\n \"description\": \"Description\",\n \"descriptionPlaceholder\": \"Enter description\",\n \"bodyContentType\": \"Body Content Type\",\n \"headers\": \"Headers\",\n \"queryParameters\": \"Query Parameters\",\n \"body\": \"Body\",\n \"deleteActionWarning\": \"Are you sure you want to delete the action \\\"{{name}}\\\"?\",\n \"backToAllTools\": \"Back to all tools\",\n \"save\": \"Save\",\n \"saving\": \"Saving...\",\n \"saveFailed\": \"Failed to save tool configuration\",\n \"fieldName\": \"Field Name\",\n \"fieldType\": \"Field Type\",\n \"filledByLLM\": \"Filled by LLM\",\n \"fieldDescription\": \"Field description\",\n \"value\": \"Value\",\n \"addProperty\": \"Add property\",\n \"propertyName\": \"New property key\",\n \"add\": \"Add\",\n \"cancel\": \"Cancel\",\n \"addNew\": \"Add New\",\n \"name\": \"Name\",\n \"type\": \"Type\",\n \"mcp\": {\n \"addServer\": \"Add MCP Server\",\n \"editServer\": \"Edit Server\",\n \"reconnectServer\": \"Reconnect Server\",\n \"reenterCredentials\": \"Re-enter your credentials to test and update the connection.\",\n \"serverName\": \"Server Name\",\n \"serverUrl\": \"Server URL\",\n \"headerName\": \"Header Name\",\n \"timeout\": \"Timeout (seconds)\",\n \"testConnection\": \"Test Connection\",\n \"testing\": \"Testing\",\n \"saving\": \"Saving\",\n \"save\": \"Save\",\n \"cancel\": \"Cancel\",\n \"noAuth\": \"No Authentication\",\n \"oauthInProgress\": \"Waiting for OAuth completion...\",\n \"oauthCompleted\": \"OAuth completed successfully\",\n \"oauthPopupBlocked\": \"Popup blocked by browser. Click below to authorize:\",\n \"openAuthPage\": \"Open authorization page\",\n \"authType\": \"Authentication Type\",\n \"defaultServerName\": \"My MCP Server\",\n \"authTypes\": {\n \"none\": \"No Authentication\",\n \"apiKey\": \"API Key\",\n \"bearer\": \"Bearer Token\",\n \"oauth\": \"OAuth\",\n \"basic\": \"Basic Authentication\"\n },\n \"placeholders\": {\n \"serverUrl\": \"https://api.example.com\",\n \"apiKey\": \"Your secret API key\",\n \"bearerToken\": \"Your secret token\",\n \"username\": \"Your username\",\n \"password\": \"Your password\",\n \"oauthScopes\": \"OAuth scopes (comma separated)\"\n },\n \"errors\": {\n \"nameRequired\": \"Server name is required\",\n \"urlRequired\": \"Server URL is required\",\n \"invalidUrl\": \"Please enter a valid URL\",\n \"apiKeyRequired\": \"API key is required\",\n \"tokenRequired\": \"Bearer token is required\",\n \"usernameRequired\": \"Username is required\",\n \"passwordRequired\": \"Password is required\",\n \"testFailed\": \"Connection test failed\",\n \"saveFailed\": \"Failed to save MCP server\",\n \"oauthFailed\": \"OAuth process failed or was cancelled\",\n \"oauthTimeout\": \"OAuth process timed out, please try again\",\n \"timeoutRange\": \"Timeout must be between 1 and 300 seconds\"\n }\n }\n },\n \"scrollTabsLeft\": \"Scroll tabs left\",\n \"tabsAriaLabel\": \"Settings tabs\",\n \"scrollTabsRight\": \"Scroll tabs right\"\n },\n \"modals\": {\n \"uploadDoc\": {\n \"label\": \"Upload new document\",\n \"select\": \"Choose how to upload your document to DocsGPT\",\n \"selectSource\": \"Select the way to add your source\",\n \"selectedFiles\": \"Selected Files\",\n \"noFilesSelected\": \"No files selected\",\n \"file\": \"Upload from device\",\n \"back\": \"Back\",\n \"wait\": \"Please wait ...\",\n \"remote\": \"Collect from website\",\n \"start\": \"Start Chatting\",\n \"name\": \"Name\",\n \"choose\": \"Choose Files\",\n \"info\": \"Please upload .pdf, .txt, .rst, .csv, .xlsx, .docx, .md, .html, .epub, .json, .pptx, .zip limited to 25mb\",\n \"uploadedFiles\": \"Uploaded Files\",\n \"cancel\": \"Cancel\",\n \"train\": \"Train\",\n \"link\": \"Link\",\n \"urlLink\": \"URL Link\",\n \"repoUrl\": \"Repository URL\",\n \"reddit\": {\n \"id\": \"Client ID\",\n \"secret\": \"Client Secret\",\n \"agent\": \"User agent\",\n \"searchQueries\": \"Search queries\",\n \"numberOfPosts\": \"Number of posts\",\n \"addQuery\": \"Add Query\"\n },\n \"drag\": {\n \"title\": \"Drop attachments here\",\n \"description\": \"Release to upload your attachments\"\n },\n \"progress\": {\n \"upload\": \"Upload is in progress\",\n \"training\": \"Upload is in progress\",\n \"completed\": \"Upload completed\",\n \"failed\": \"Upload failed\",\n \"wait\": \"This may take several minutes\",\n \"preparing\": \"Preparing upload\",\n \"tokenLimit\": \"Over the token limit, please consider uploading smaller document\",\n \"expandDetails\": \"Expand upload details\",\n \"collapseDetails\": \"Collapse upload details\",\n \"dismiss\": \"Dismiss upload toast\",\n \"uploadProgress\": \"Upload progress {{progress}}%\",\n \"clear\": \"Clear\"\n },\n \"showAdvanced\": \"Show advanced options\",\n \"hideAdvanced\": \"Hide advanced options\",\n \"ingestors\": {\n \"local_file\": {\n \"label\": \"Upload File\",\n \"heading\": \"Upload new document\"\n },\n \"crawler\": {\n \"label\": \"Crawler\",\n \"heading\": \"Add content with Web Crawler\"\n },\n \"url\": {\n \"label\": \"Link\",\n \"heading\": \"Add content from URL\"\n },\n \"github\": {\n \"label\": \"GitHub\",\n \"heading\": \"Add content from GitHub\"\n },\n \"reddit\": {\n \"label\": \"Reddit\",\n \"heading\": \"Add content from Reddit\"\n },\n \"google_drive\": {\n \"label\": \"Google Drive\",\n \"heading\": \"Upload from Google Drive\"\n },\n \"s3\": {\n \"label\": \"Amazon S3\",\n \"heading\": \"Add content from Amazon S3\"\n },\n \"share_point\": {\n \"label\": \"SharePoint\",\n \"heading\": \"Upload from SharePoint\"\n }\n },\n \"connectors\": {\n \"auth\": {\n \"connectedUser\": \"Connected User\",\n \"authFailed\": \"Authentication failed\",\n \"authUrlFailed\": \"Failed to get authorization URL\",\n \"popupBlocked\": \"Failed to open authentication window. Please allow popups.\",\n \"authCancelled\": \"Authentication was cancelled\",\n \"connectedAs\": \"Connected as {{email}}\",\n \"disconnect\": \"Disconnect\"\n },\n \"googleDrive\": {\n \"connect\": \"Connect to Google Drive\",\n \"sessionExpired\": \"Session expired. Please reconnect to Google Drive.\",\n \"sessionExpiredGeneric\": \"Session expired. Please reconnect your account.\",\n \"validateFailed\": \"Failed to validate session. Please reconnect.\",\n \"noSession\": \"No valid session found. Please reconnect to Google Drive.\",\n \"noAccessToken\": \"No access token available. Please reconnect to Google Drive.\",\n \"pickerFailed\": \"Failed to open file picker. Please try again.\",\n \"selectedFiles\": \"Selected Files\",\n \"selectFiles\": \"Select Files\",\n \"loading\": \"Loading...\",\n \"noFilesSelected\": \"No files or folders selected\",\n \"folders\": \"Folders\",\n \"files\": \"Files\",\n \"remove\": \"Remove\",\n \"folderAlt\": \"Folder\",\n \"fileAlt\": \"File\"\n },\n \"sharePoint\": {\n \"connect\": \"Connect to SharePoint\",\n \"sessionExpired\": \"Session expired. Please reconnect to SharePoint.\",\n \"sessionExpiredGeneric\": \"Session expired. Please reconnect your account.\",\n \"validateFailed\": \"Failed to validate session. Please reconnect.\",\n \"noSession\": \"No valid session found. Please reconnect to SharePoint.\",\n \"noAccessToken\": \"No access token available. Please reconnect to SharePoint.\",\n \"pickerFailed\": \"Failed to open file picker. Please try again.\",\n \"selectedFiles\": \"Selected Files\",\n \"selectFiles\": \"Select Files\",\n \"loading\": \"Loading...\",\n \"noFilesSelected\": \"No files or folders selected\",\n \"folders\": \"Folders\",\n \"files\": \"Files\",\n \"remove\": \"Remove\",\n \"folderAlt\": \"Folder\",\n \"fileAlt\": \"File\"\n }\n }\n },\n \"createAPIKey\": {\n \"label\": \"Create New API Key\",\n \"apiKeyName\": \"API Key Name\",\n \"chunks\": \"Chunks processed per query\",\n \"prompt\": \"Select active prompt\",\n \"sourceDoc\": \"Source document\",\n \"create\": \"Create\"\n },\n \"saveKey\": {\n \"note\": \"Please save your Key\",\n \"disclaimer\": \"This is the only time your key will be shown.\",\n \"copy\": \"Copy\",\n \"copied\": \"Copied\",\n \"confirm\": \"I saved the Key\",\n \"apiKeyLabel\": \"API Key\"\n },\n \"deleteConv\": {\n \"confirm\": \"Are you sure you want to delete all the conversations?\",\n \"delete\": \"Delete\"\n },\n \"shareConv\": {\n \"label\": \"Create a public page to share\",\n \"note\": \"Source document, personal information and further conversation will remain private\",\n \"create\": \"Create\",\n \"option\": \"Allow users to prompt further\"\n },\n \"configTool\": {\n \"title\": \"Tool Config\",\n \"type\": \"Type\",\n \"apiKeyLabel\": \"API Key / OAuth\",\n \"apiKeyPlaceholder\": \"Enter API Key / OAuth\",\n \"addButton\": \"Add Tool\",\n \"closeButton\": \"Close\",\n \"customNamePlaceholder\": \"Enter custom name (optional)\"\n },\n \"prompts\": {\n \"addPrompt\": \"Add Prompt\",\n \"addDescription\": \"Add your custom prompt and save it to DocsGPT\",\n \"editPrompt\": \"Edit Prompt\",\n \"editDescription\": \"Edit your custom prompt and save it to DocsGPT\",\n \"promptName\": \"Prompt Name\",\n \"promptText\": \"Prompt Text\",\n \"save\": \"Save\",\n \"cancel\": \"Cancel\",\n \"nameExists\": \"Name already exists\",\n \"deleteConfirmation\": \"Are you sure you want to delete the prompt '{{name}}'?\",\n \"placeholderText\": \"Type your prompt text here...\",\n \"addExamplePlaceholder\": \"Please summarize this text:\",\n \"variablesLabel\": \"Variables\",\n \"variablesSubtext\": \"Click To Insert Into Prompt\",\n \"variablesDescription\": \"Click to insert into prompt\",\n \"systemVariables\": \"Click to insert into prompt\",\n \"toolVariables\": \"Tool Variables\",\n \"systemVariablesDropdownLabel\": \"System Variables\",\n \"systemVariableOptions\": {\n \"sourceContent\": \"Sources content\",\n \"sourceSummaries\": \"Alias for content (backward compatible)\",\n \"sourceDocuments\": \"Document objects list\",\n \"sourceCount\": \"Number of retrieved documents\",\n \"systemDate\": \"Current date (YYYY-MM-DD)\",\n \"systemTime\": \"Current time (HH:MM:SS)\",\n \"systemTimestamp\": \"ISO 8601 timestamp\",\n \"systemRequestId\": \"Unique request identifier\",\n \"systemUserId\": \"Current user ID\"\n },\n \"learnAboutPrompts\": \"Learn about Prompts →\",\n \"publicPromptEditDisabled\": \"Public prompts cannot be edited\",\n \"promptTypePublic\": \"public\",\n \"promptTypePrivate\": \"private\"\n },\n \"chunk\": {\n \"add\": \"Add Chunk\",\n \"edit\": \"Edit\",\n \"title\": \"Title\",\n \"enterTitle\": \"Enter title\",\n \"bodyText\": \"Body text\",\n \"promptText\": \"Prompt Text\",\n \"save\": \"Save\",\n \"close\": \"Close\",\n \"cancel\": \"Cancel\",\n \"delete\": \"Delete\",\n \"deleteConfirmation\": \"Are you sure you want to delete this chunk?\"\n },\n \"addAction\": {\n \"title\": \"New Action\",\n \"actionNamePlaceholder\": \"Action Name\",\n \"invalidFormat\": \"Invalid function name format. Use only letters, numbers, underscores, and hyphens.\",\n \"formatHelp\": \"Use only letters, numbers, underscores, and hyphens (e.g., `get_data`, `send_report`, etc.)\",\n \"addButton\": \"Add\"\n },\n \"agentDetails\": {\n \"title\": \"Access Details\",\n \"publicLink\": \"Public Link\",\n \"apiKey\": \"API Key\",\n \"webhookUrl\": \"Webhook URL\",\n \"generate\": \"Generate\",\n \"test\": \"Test\",\n \"learnMore\": \"Learn more\"\n },\n \"importSpec\": {\n \"title\": \"Import API Specification\",\n \"description\": \"Upload an OpenAPI 3.x or Swagger 2.0 specification file to automatically generate actions.\",\n \"dropzoneText\": \"Click to upload or drag and drop\",\n \"supportedFormats\": \"JSON or YAML format\",\n \"invalidFileType\": \"Invalid file type. Please upload a JSON or YAML file.\",\n \"parseError\": \"Failed to parse the specification. Please check the file format.\",\n \"version\": \"Version\",\n \"baseUrl\": \"Base URL\",\n \"actionsFound\": \"{{count}} actions found\",\n \"selectAll\": \"Select all\",\n \"deselectAll\": \"Deselect all\",\n \"cancel\": \"Cancel\",\n \"parse\": \"Parse\",\n \"import\": \"Import ({{count}})\"\n }\n },\n \"sharedConv\": {\n \"subtitle\": \"Created with\",\n \"button\": \"Get Started with DocsGPT\",\n \"meta\": \"DocsGPT uses GenAI, please review critical information using sources.\"\n },\n \"convTile\": {\n \"share\": \"Share\",\n \"delete\": \"Delete\",\n \"rename\": \"Rename\",\n \"deleteWarning\": \"Are you sure you want to delete this conversation?\"\n },\n \"pagination\": {\n \"rowsPerPage\": \"Rows per page\",\n \"pageOf\": \"Page {{currentPage}} of {{totalPages}}\",\n \"firstPage\": \"First page\",\n \"previousPage\": \"Previous page\",\n \"nextPage\": \"Next page\",\n \"lastPage\": \"Last page\"\n },\n \"conversation\": {\n \"copy\": \"Copy\",\n \"copied\": \"Copied\",\n \"speak\": \"Speak\",\n \"answer\": \"Answer\",\n \"edit\": {\n \"update\": \"Update\",\n \"cancel\": \"Cancel\",\n \"placeholder\": \"Type the updated query...\"\n },\n \"sources\": {\n \"title\": \"Sources\",\n \"text\": \"Choose Your Sources\",\n \"link\": \"Source link\",\n \"view_more\": \"{{count}} more sources\",\n \"noSourcesAvailable\": \"No sources available\"\n },\n \"attachments\": {\n \"attach\": \"Attach\",\n \"remove\": \"Remove attachment\"\n },\n \"retry\": \"Retry\",\n \"reasoning\": \"Reasoning\"\n },\n \"agents\": {\n \"title\": \"Agents\",\n \"description\": \"Discover and create custom versions of DocsGPT that combine instructions, extra knowledge, and any combination of skills\",\n \"newAgent\": \"New Agent\",\n \"backToAll\": \"Back to all agents\",\n \"searchPlaceholder\": \"Search...\",\n \"noSearchResults\": \"No agents found\",\n \"tryDifferentSearch\": \"Try a different search term\",\n \"filters\": {\n \"all\": \"All\",\n \"byDocsGPT\": \"By DocsGPT\",\n \"byMe\": \"By Me\",\n \"shared\": \"Shared With Me\"\n },\n \"sections\": {\n \"template\": {\n \"title\": \"By DocsGPT\",\n \"description\": \"Agents provided by DocsGPT\",\n \"emptyState\": \"No template agents found.\"\n },\n \"user\": {\n \"title\": \"By me\",\n \"description\": \"Agents created or published by you\",\n \"emptyState\": \"You don't have any created agents yet.\"\n },\n \"shared\": {\n \"title\": \"Shared with me\",\n \"description\": \"Agents imported by using a public link\",\n \"emptyState\": \"No shared agents found.\"\n }\n },\n \"form\": {\n \"headings\": {\n \"new\": \"New Agent\",\n \"edit\": \"Edit Agent\",\n \"draft\": \"New Agent (Draft)\"\n },\n \"buttons\": {\n \"publish\": \"Publish\",\n \"save\": \"Save\",\n \"saveDraft\": \"Save Draft\",\n \"cancel\": \"Cancel\",\n \"delete\": \"Delete\",\n \"logs\": \"Logs\",\n \"accessDetails\": \"Access Details\",\n \"add\": \"Add\"\n },\n \"sections\": {\n \"meta\": \"Meta\",\n \"source\": \"Source\",\n \"prompt\": \"Prompt\",\n \"tools\": \"Tools\",\n \"agentType\": \"Agent type\",\n \"models\": \"Models\",\n \"advanced\": \"Advanced\",\n \"preview\": \"Preview\"\n },\n \"placeholders\": {\n \"agentName\": \"Agent name\",\n \"describeAgent\": \"Describe your agent\",\n \"selectSources\": \"Select sources\",\n \"chunksPerQuery\": \"Chunks per query\",\n \"selectType\": \"Select type\",\n \"selectTools\": \"Select tools\",\n \"selectModels\": \"Select models for this agent\",\n \"selectDefaultModel\": \"Select default model\",\n \"enterTokenLimit\": \"Enter token limit\",\n \"enterRequestLimit\": \"Enter request limit\"\n },\n \"sourcePopup\": {\n \"title\": \"Select Sources\",\n \"searchPlaceholder\": \"Search sources...\",\n \"noOptionsMessage\": \"No sources available\"\n },\n \"toolsPopup\": {\n \"title\": \"Select Tools\",\n \"searchPlaceholder\": \"Search tools...\",\n \"noOptionsMessage\": \"No tools available\"\n },\n \"modelsPopup\": {\n \"title\": \"Select Models\",\n \"searchPlaceholder\": \"Search models...\",\n \"noOptionsMessage\": \"No models available\"\n },\n \"upload\": {\n \"clickToUpload\": \"Click to upload\",\n \"dragAndDrop\": \" or drag and drop\"\n },\n \"agentTypes\": {\n \"classic\": \"Classic\",\n \"react\": \"ReAct\"\n },\n \"labels\": {\n \"defaultModel\": \"Default Model\"\n },\n \"advanced\": {\n \"jsonSchema\": \"JSON response schema\",\n \"jsonSchemaDescription\": \"Define a JSON schema to enforce structured output format\",\n \"validJson\": \"Valid JSON\",\n \"invalidJson\": \"Invalid JSON - fix to enable saving\",\n \"tokenLimiting\": \"Token limiting\",\n \"tokenLimitingDescription\": \"Limit daily total tokens that can be used by this agent\",\n \"requestLimiting\": \"Request limiting\",\n \"requestLimitingDescription\": \"Limit daily total requests that can be made to this agent\"\n },\n \"preview\": {\n \"publishedPreview\": \"Published agents can be previewed here\"\n },\n \"externalKb\": \"External KB\"\n },\n \"logs\": {\n \"title\": \"Agent Logs\",\n \"lastUsedAt\": \"Last used at\",\n \"noUsageHistory\": \"No usage history\",\n \"tableHeader\": \"Agent endpoint logs\"\n },\n \"shared\": {\n \"notFound\": \"No agent found. Please ensure the agent is shared.\"\n },\n \"preview\": {\n \"testMessage\": \"Test your agent here. Published agents can be used in conversations.\"\n },\n \"deleteConfirmation\": \"Are you sure you want to delete this agent?\",\n \"folders\": {\n \"newFolder\": \"New Folder\",\n \"createFolder\": \"Create Folder\",\n \"folderName\": \"Folder name\",\n \"rename\": \"Rename\",\n \"delete\": \"Delete\",\n \"deleteConfirm\": \"Are you sure you want to delete this folder? Agents inside will be moved out of the folder.\",\n \"empty\": \"This folder is empty\",\n \"moveToFolder\": \"Move to folder\",\n \"moveTo\": \"Move\",\n \"move\": \"Move\",\n \"noFolder\": \"No folder (root)\",\n \"backToRoot\": \"Back\",\n \"currentFolder\": \"This folder\",\n \"noSubfolders\": \"No subfolders\",\n \"noFolders\": \"No folders yet\"\n }\n },\n \"components\": {\n \"fileUpload\": {\n \"clickToUpload\": \"Click to upload or drag and drop\",\n \"dropFiles\": \"Drop the files here\",\n \"fileTypes\": \"PNG, JPG, JPEG up to\",\n \"sizeLimitUnit\": \"MB\",\n \"fileSizeError\": \"File exceeds {{size}}MB limit\"\n }\n },\n \"pageNotFound\": {\n \"title\": \"404\",\n \"message\": \"The page you are looking for does not exist.\",\n \"goHome\": \"Go Back Home\"\n },\n \"filePicker\": {\n \"searchPlaceholder\": \"Search files and folders...\",\n \"itemsSelected\": \"{{count}} selected\",\n \"name\": \"Name\",\n \"lastModified\": \"Last Modified\",\n \"size\": \"Size\",\n \"myFiles\": \"My Files\",\n \"sharedWithMe\": \"Shared with Me\",\n \"loadingMore\": \"Loading more files...\"\n },\n \"actionButtons\": {\n \"openNewChat\": \"Open New Chat\",\n \"share\": \"Share\"\n },\n \"mermaid\": {\n \"downloadOptions\": \"Download options\",\n \"viewCode\": \"View Code\",\n \"decreaseZoom\": \"Decrease zoom\",\n \"resetZoom\": \"Reset zoom\",\n \"increaseZoom\": \"Increase zoom\"\n },\n \"navigation\": {\n \"agents\": \"Agents\"\n },\n \"notification\": {\n \"ariaLabel\": \"Notification\",\n \"closeAriaLabel\": \"Close notification\"\n },\n \"prompts\": {\n \"textAriaLabel\": \"Prompt Text\"\n }\n}\n" }, { "path": "frontend/src/locale/es.json", "content": "{\n \"language\": \"Español\",\n \"chat\": \"Chat\",\n \"chats\": \"Chats\",\n \"newChat\": \"Nuevo Chat\",\n \"inputPlaceholder\": \"¿Cómo puede DocsGPT ayudarte?\",\n \"tagline\": \"DocsGPT utiliza GenAI, por favor revisa información crítica utilizando fuentes.\",\n \"sourceDocs\": \"Fuente\",\n \"none\": \"Ninguno\",\n \"cancel\": \"Cancelar\",\n \"help\": \"Asistencia\",\n \"emailUs\": \"Envíanos un correo\",\n \"documentation\": \"Documentación\",\n \"manageAgents\": \"Administrar Agentes\",\n \"demo\": [\n {\n \"header\": \"Aprende sobre DocsGPT\",\n \"query\": \"¿Qué es DocsGPT?\"\n },\n {\n \"header\": \"Resumir documentación\",\n \"query\": \"Resumir contexto actual\"\n },\n {\n \"header\": \"Escribir Código\",\n \"query\": \"Escribir código para solicitud de API a /api/answer\"\n },\n {\n \"header\": \"Asistencia de Aprendizaje\",\n \"query\": \"Escribe posibles preguntas para el contexto\"\n }\n ],\n \"settings\": {\n \"label\": \"Configuración\",\n \"general\": {\n \"label\": \"General\",\n \"selectTheme\": \"Seleccionar Tema\",\n \"light\": \"Claro\",\n \"dark\": \"Oscuro\",\n \"selectLanguage\": \"Seleccionar Idioma\",\n \"chunks\": \"Fragmentos procesados por consulta\",\n \"prompt\": \"Prompt Activo\",\n \"deleteAllLabel\": \"Eliminar todas las conversaciones\",\n \"deleteAllBtn\": \"Eliminar todo\",\n \"add\": \"Añadir\",\n \"convHistory\": \"Historial de conversaciones\",\n \"none\": \"Ninguno\",\n \"low\": \"Bajo\",\n \"medium\": \"Medio\",\n \"high\": \"Alto\",\n \"unlimited\": \"Ilimitado\",\n \"default\": \"Predeterminado\",\n \"addNew\": \"Añadir Nuevo\"\n },\n \"sources\": {\n \"title\": \"Aquí puedes gestionar todos los archivos fuente que están disponibles para ti y los que has subido.\",\n \"label\": \"Fuentes\",\n \"name\": \"Nombre de la Fuente\",\n \"date\": \"Fecha de Vector\",\n \"type\": \"Tipo\",\n \"tokenUsage\": \"Uso de Tokens\",\n \"noData\": \"No hay fuentes existentes\",\n \"searchPlaceholder\": \"Buscar...\",\n \"addNew\": \"Agregar Nuevo\",\n \"addSource\": \"Agregar Fuente\",\n \"addChunk\": \"Agregar Fragmento\",\n \"preLoaded\": \"Precargado\",\n \"private\": \"Privado\",\n \"sync\": \"Sincronizar\",\n \"syncNow\": \"Sincronizar ahora\",\n \"syncing\": \"Sincronizando...\",\n \"syncConfirmation\": \"¿Estás seguro de que deseas sincronizar \\\"{{sourceName}}\\\"? Esto actualizará el contenido con tu almacenamiento en la nube y puede anular cualquier edición que hayas realizado en fragmentos individuales.\",\n \"syncFrequency\": {\n \"never\": \"Nunca\",\n \"daily\": \"Diario\",\n \"weekly\": \"Semanal\",\n \"monthly\": \"Mensual\"\n },\n \"actions\": \"Acciones\",\n \"view\": \"Ver\",\n \"deleteWarning\": \"¿Estás seguro de que deseas eliminar \\\"{{name}}\\\"?\",\n \"confirmDelete\": \"¿Estás seguro de que deseas eliminar este archivo? Esta acción no se puede deshacer.\",\n \"backToAll\": \"Volver a todas las fuentes\",\n \"chunks\": \"Fragmentos\",\n \"noChunks\": \"No se encontraron fragmentos\",\n \"noChunksAlt\": \"No se encontraron fragmentos\",\n \"goToSources\": \"Ir a Fuentes\",\n \"uploadNew\": \"Subir nuevo\",\n \"searchFiles\": \"Buscar archivos...\",\n \"noResults\": \"No se encontraron resultados\",\n \"fileName\": \"Nombre\",\n \"tokens\": \"Tokens\",\n \"size\": \"Tamaño\",\n \"fileAlt\": \"Archivo\",\n \"folderAlt\": \"Carpeta\",\n \"parentFolderAlt\": \"Carpeta padre\",\n \"menuAlt\": \"Menú\",\n \"tokensUnit\": \"tokens\",\n \"editAlt\": \"Editar\",\n \"uploading\": \"Subiendo…\",\n \"deleting\": \"Eliminando…\",\n \"queued\": \"En cola: {{count}}\",\n \"addFile\": \"Añadir archivo\",\n \"uploadingFilesTitle\": \"Subiendo archivos...\",\n \"deletingTitle\": \"Eliminando...\",\n \"deleteDirectoryWarning\": \"¿Está seguro de que desea eliminar el directorio \\\"{{name}}\\\" y todo su contenido? Esta acción no se puede deshacer.\",\n \"searchAlt\": \"Buscar\"\n },\n \"apiKeys\": {\n \"label\": \"Chatbots\",\n \"name\": \"Nombre\",\n \"key\": \"Clave de API\",\n \"sourceDoc\": \"Documento Fuente\",\n \"createNew\": \"Crear Nuevo\",\n \"noData\": \"No hay chatbots existentes\",\n \"deleteConfirmation\": \"¿Estás seguro de que quieres eliminar la clave API '{{name}}'?\",\n \"description\": \"Aquí puede crear y gestionar sus chatbots. Los chatbots se pueden implementar en sitios web como widgets o utilizarse dentro de sus aplicaciones.\"\n },\n \"analytics\": {\n \"label\": \"Analítica\",\n \"filterByChatbot\": \"Filtrar por chatbot\",\n \"selectChatbot\": \"Seleccionar chatbot\",\n \"filterOptions\": {\n \"hour\": \"Hora\",\n \"last24Hours\": \"24 Horas\",\n \"last7Days\": \"7 Días\",\n \"last15Days\": \"15 Días\",\n \"last30Days\": \"30 Días\"\n },\n \"messages\": \"Mensajes\",\n \"tokenUsage\": \"Uso de Tokens\",\n \"userFeedback\": \"Retroalimentación del Usuario\",\n \"filterPlaceholder\": \"Filtrar\",\n \"none\": \"Ninguno\",\n \"positiveFeedback\": \"Retroalimentación Positiva\",\n \"negativeFeedback\": \"Retroalimentación Negativa\"\n },\n \"logs\": {\n \"label\": \"Registros\",\n \"filterByChatbot\": \"Filtrar por chatbot\",\n \"selectChatbot\": \"Seleccionar chatbot\",\n \"none\": \"Ninguno\",\n \"tableHeader\": \"Conversaciones generadas por API / chatbot\"\n },\n \"tools\": {\n \"label\": \"Herramientas\",\n \"searchPlaceholder\": \"Buscar...\",\n \"addTool\": \"Agregar Herramienta\",\n \"noToolsFound\": \"No se encontraron herramientas\",\n \"selectToolSetup\": \"Seleccione una herramienta para configurar\",\n \"settingsIconAlt\": \"Icono de configuración\",\n \"configureToolAria\": \"Configurar {{toolName}}\",\n \"toggleToolAria\": \"Alternar {{toolName}}\",\n \"manageTools\": \"Ir a Herramientas\",\n \"edit\": \"Editar\",\n \"delete\": \"Eliminar\",\n \"deleteWarning\": \"¿Estás seguro de que deseas eliminar la herramienta \\\"{{toolName}}\\\"?\",\n \"unsavedChanges\": \"Tienes cambios sin guardar que se perderán si sales sin guardar.\",\n \"leaveWithoutSaving\": \"Salir sin Guardar\",\n \"saveAndLeave\": \"Guardar y Salir\",\n \"customName\": \"Nombre Personalizado\",\n \"customNamePlaceholder\": \"Ingresa un nombre personalizado (opcional)\",\n \"authentication\": \"Autenticación\",\n \"actions\": \"Acciones\",\n \"addAction\": \"Agregar acción\",\n \"importSpec\": \"Importar especificación\",\n \"searchActions\": \"Buscar acciones...\",\n \"noActionsMatch\": \"No hay acciones que coincidan con tu búsqueda\",\n \"actionAlreadyExists\": \"Ya existe una acción con este nombre\",\n \"noActionsFound\": \"No se encontraron acciones\",\n \"url\": \"URL\",\n \"urlPlaceholder\": \"Ingresa url\",\n \"method\": \"Método\",\n \"description\": \"Descripción\",\n \"descriptionPlaceholder\": \"Ingresa descripción\",\n \"bodyContentType\": \"Tipo de contenido del cuerpo\",\n \"headers\": \"Encabezados\",\n \"queryParameters\": \"Parámetros de Consulta\",\n \"body\": \"Cuerpo\",\n \"deleteActionWarning\": \"¿Estás seguro de que deseas eliminar la acción \\\"{{name}}\\\"?\",\n \"save\": \"Guardar\",\n \"name\": \"Nombre\",\n \"type\": \"Tipo\",\n \"filledByLLM\": \"Completado por LLM\",\n \"value\": \"Valor\",\n \"addProperty\": \"Agregar propiedad\",\n \"propertyName\": \"Nueva clave de propiedad\",\n \"backToAllTools\": \"Volver a todas las herramientas\",\n \"fieldName\": \"Nombre del campo\",\n \"fieldType\": \"Tipo de campo\",\n \"fieldDescription\": \"Descripción del campo\",\n \"add\": \"Añadir\",\n \"cancel\": \"Cancelar\",\n \"addNew\": \"Añadir Nuevo\",\n \"mcp\": {\n \"addServer\": \"Add MCP Server\",\n \"editServer\": \"Edit Server\",\n \"serverName\": \"Server Name\",\n \"serverUrl\": \"Server URL\",\n \"headerName\": \"Header Name\",\n \"timeout\": \"Timeout (seconds)\",\n \"testConnection\": \"Test Connection\",\n \"testing\": \"Testing\",\n \"saving\": \"Saving\",\n \"save\": \"Save\",\n \"cancel\": \"Cancel\",\n \"noAuth\": \"No Authentication\",\n \"oauthInProgress\": \"Waiting for OAuth completion...\",\n \"oauthCompleted\": \"OAuth completed successfully\",\n \"authType\": \"Authentication Type\",\n \"defaultServerName\": \"My MCP Server\",\n \"authTypes\": {\n \"none\": \"No Authentication\",\n \"apiKey\": \"API Key\",\n \"bearer\": \"Bearer Token\",\n \"oauth\": \"OAuth\",\n \"basic\": \"Basic Authentication\"\n },\n \"placeholders\": {\n \"serverUrl\": \"https://api.example.com\",\n \"apiKey\": \"Your secret API key\",\n \"bearerToken\": \"Your secret token\",\n \"username\": \"Your username\",\n \"password\": \"Your password\",\n \"oauthScopes\": \"OAuth scopes (comma separated)\"\n },\n \"errors\": {\n \"nameRequired\": \"Server name is required\",\n \"urlRequired\": \"Server URL is required\",\n \"invalidUrl\": \"Please enter a valid URL\",\n \"apiKeyRequired\": \"API key is required\",\n \"tokenRequired\": \"Bearer token is required\",\n \"usernameRequired\": \"Username is required\",\n \"passwordRequired\": \"Password is required\",\n \"testFailed\": \"Connection test failed\",\n \"saveFailed\": \"Failed to save MCP server\",\n \"oauthFailed\": \"OAuth process failed or was cancelled\",\n \"oauthTimeout\": \"OAuth process timed out, please try again\",\n \"timeoutRange\": \"Timeout must be between 1 and 300 seconds\"\n }\n }\n },\n \"scrollTabsLeft\": \"Desplazar pestañas a la izquierda\",\n \"tabsAriaLabel\": \"Pestañas de configuración\",\n \"scrollTabsRight\": \"Desplazar pestañas a la derecha\"\n },\n \"modals\": {\n \"uploadDoc\": {\n \"label\": \"Subir nuevo documento\",\n \"select\": \"Elige cómo cargar tu documento en DocsGPT\",\n \"selectSource\": \"Selecciona la forma de agregar tu fuente\",\n \"selectedFiles\": \"Archivos Seleccionados\",\n \"noFilesSelected\": \"No hay archivos seleccionados\",\n \"file\": \"Subir desde el dispositivo\",\n \"back\": \"Atrás\",\n \"wait\": \"Por favor espera ...\",\n \"remote\": \"Recoger desde un sitio web\",\n \"start\": \"Comenzar a chatear\",\n \"name\": \"Nombre\",\n \"choose\": \"Seleccionar Archivos\",\n \"info\": \"Por favor, sube archivos .pdf, .txt, .rst, .csv, .xlsx, .docx, .md, .html, .epub, .json, .pptx, .zip limitados a 25MB\",\n \"uploadedFiles\": \"Archivos Subidos\",\n \"cancel\": \"Cancelar\",\n \"train\": \"Entrenar\",\n \"link\": \"Enlace\",\n \"urlLink\": \"Enlace URL\",\n \"repoUrl\": \"URL del Repositorio\",\n \"reddit\": {\n \"id\": \"ID del Cliente\",\n \"secret\": \"Secreto del Cliente\",\n \"agent\": \"Agente de usuario\",\n \"searchQueries\": \"Consultas de búsqueda\",\n \"numberOfPosts\": \"Número de publicaciones\",\n \"addQuery\": \"Agregar Consulta\"\n },\n \"drag\": {\n \"title\": \"Suelta los archivos adjuntos aquí\",\n \"description\": \"Suelta para subir tus archivos adjuntos\"\n },\n \"progress\": {\n \"upload\": \"Subida en progreso\",\n \"training\": \"Subida en progreso\",\n \"completed\": \"Subida completada\",\n \"failed\": \"Error al subir\",\n \"wait\": \"Esto puede tardar varios minutos\",\n \"preparing\": \"Preparando subida\",\n \"tokenLimit\": \"Excede el límite de tokens, considere cargar un documento más pequeño\",\n \"expandDetails\": \"Expandir detalles de subida\",\n \"collapseDetails\": \"Contraer detalles de subida\",\n \"dismiss\": \"Descartar notificación de subida\",\n \"uploadProgress\": \"Progreso de subida {{progress}}%\",\n \"clear\": \"Limpiar\"\n },\n \"showAdvanced\": \"Mostrar opciones avanzadas\",\n \"hideAdvanced\": \"Ocultar opciones avanzadas\",\n \"ingestors\": {\n \"local_file\": {\n \"label\": \"Subir archivo\",\n \"heading\": \"Subir nuevo documento\"\n },\n \"crawler\": {\n \"label\": \"Rastreador\",\n \"heading\": \"Agregar contenido con rastreador web\"\n },\n \"url\": {\n \"label\": \"Enlace\",\n \"heading\": \"Agregar contenido desde URL\"\n },\n \"github\": {\n \"label\": \"GitHub\",\n \"heading\": \"Agregar contenido desde GitHub\"\n },\n \"reddit\": {\n \"label\": \"Reddit\",\n \"heading\": \"Agregar contenido desde Reddit\"\n },\n \"google_drive\": {\n \"label\": \"Google Drive\",\n \"heading\": \"Subir desde Google Drive\"\n },\n \"s3\": {\n \"label\": \"Amazon S3\",\n \"heading\": \"Agregar contenido desde Amazon S3\"\n },\n \"share_point\": {\n \"label\": \"SharePoint\",\n \"heading\": \"Subir desde SharePoint\"\n }\n },\n \"connectors\": {\n \"auth\": {\n \"connectedUser\": \"Usuario Conectado\",\n \"authFailed\": \"Autenticación fallida\",\n \"authUrlFailed\": \"Error al obtener la URL de autorización\",\n \"popupBlocked\": \"Error al abrir la ventana de autenticación. Por favor, permita ventanas emergentes.\",\n \"authCancelled\": \"Autenticación cancelada\",\n \"connectedAs\": \"Conectado como {{email}}\",\n \"disconnect\": \"Desconectar\"\n },\n \"googleDrive\": {\n \"connect\": \"Conectar a Google Drive\",\n \"sessionExpired\": \"Sesión expirada. Por favor, reconecte a Google Drive.\",\n \"sessionExpiredGeneric\": \"Sesión expirada. Por favor, reconecte su cuenta.\",\n \"validateFailed\": \"Error al validar la sesión. Por favor, reconecte.\",\n \"noSession\": \"No se encontró una sesión válida. Por favor, reconecte a Google Drive.\",\n \"noAccessToken\": \"No hay token de acceso disponible. Por favor, reconecte a Google Drive.\",\n \"pickerFailed\": \"Error al abrir el selector de archivos. Por favor, inténtelo de nuevo.\",\n \"selectedFiles\": \"Archivos Seleccionados\",\n \"selectFiles\": \"Seleccionar Archivos\",\n \"loading\": \"Cargando...\",\n \"noFilesSelected\": \"No hay archivos o carpetas seleccionados\",\n \"folders\": \"Carpetas\",\n \"files\": \"Archivos\",\n \"remove\": \"Eliminar\",\n \"folderAlt\": \"Carpeta\",\n \"fileAlt\": \"Archivo\"\n },\n \"sharePoint\": {\n \"connect\": \"Conectar a SharePoint\",\n \"sessionExpired\": \"Sesión expirada. Por favor, reconecte a SharePoint.\",\n \"sessionExpiredGeneric\": \"Sesión expirada. Por favor, reconecte su cuenta.\",\n \"validateFailed\": \"Error al validar la sesión. Por favor, reconecte.\",\n \"noSession\": \"No se encontró una sesión válida. Por favor, reconecte a SharePoint.\",\n \"noAccessToken\": \"No hay token de acceso disponible. Por favor, reconecte a SharePoint.\",\n \"pickerFailed\": \"Error al abrir el selector de archivos. Por favor, inténtelo de nuevo.\",\n \"selectedFiles\": \"Archivos Seleccionados\",\n \"selectFiles\": \"Seleccionar Archivos\",\n \"loading\": \"Cargando...\",\n \"noFilesSelected\": \"No hay archivos o carpetas seleccionados\",\n \"folders\": \"Carpetas\",\n \"files\": \"Archivos\",\n \"remove\": \"Eliminar\",\n \"folderAlt\": \"Carpeta\",\n \"fileAlt\": \"Archivo\"\n }\n }\n },\n \"createAPIKey\": {\n \"label\": \"Crear Nueva Clave de API\",\n \"apiKeyName\": \"Nombre de la Clave de API\",\n \"chunks\": \"Fragmentos procesados por consulta\",\n \"prompt\": \"Selecciona el prompt activo\",\n \"sourceDoc\": \"Documento Fuente\",\n \"create\": \"Crear\"\n },\n \"saveKey\": {\n \"note\": \"Por favor, guarda tu Clave\",\n \"disclaimer\": \"Esta es la única vez que se mostrará tu clave.\",\n \"copy\": \"Copiar\",\n \"copied\": \"Copiado\",\n \"confirm\": \"He guardado la Clave\",\n \"apiKeyLabel\": \"API Key\"\n },\n \"deleteConv\": {\n \"confirm\": \"¿Estás seguro de que deseas eliminar todas las conversaciones?\",\n \"delete\": \"Eliminar\"\n },\n \"shareConv\": {\n \"label\": \"Crear una página pública para compartir\",\n \"note\": \"El documento fuente, información personal y conversaciones posteriores permanecerán privadas\",\n \"create\": \"Crear\",\n \"option\": \"Permitir a los usuarios realizar más consultas\"\n },\n \"configTool\": {\n \"title\": \"Configuración de la Herramienta\",\n \"type\": \"Tipo\",\n \"apiKeyLabel\": \"Clave API / OAuth\",\n \"apiKeyPlaceholder\": \"Ingrese la Clave API / OAuth\",\n \"addButton\": \"Agregar Herramienta\",\n \"closeButton\": \"Cerrar\",\n \"customNamePlaceholder\": \"Enter custom name (optional)\"\n },\n \"prompts\": {\n \"addPrompt\": \"Agregar Prompt\",\n \"addDescription\": \"Agrega tu prompt personalizado y guárdalo en DocsGPT\",\n \"editPrompt\": \"Editar Prompt\",\n \"editDescription\": \"Edita tu prompt personalizado y guárdalo en DocsGPT\",\n \"promptName\": \"Nombre del Prompt\",\n \"promptText\": \"Texto del Prompt\",\n \"save\": \"Guardar\",\n \"cancel\": \"Cancelar\",\n \"nameExists\": \"El nombre ya existe\",\n \"deleteConfirmation\": \"¿Estás seguro de que deseas eliminar el prompt '{{name}}'?\",\n \"placeholderText\": \"Escribe tu texto de prompt aquí...\",\n \"addExamplePlaceholder\": \"Por favor, resume este texto:\",\n \"variablesLabel\": \"Variables\",\n \"variablesSubtext\": \"Haz clic para insertar en el prompt\",\n \"variablesDescription\": \"Haz clic para insertar en el prompt\",\n \"systemVariables\": \"Variables del sistema\",\n \"toolVariables\": \"Variables de herramientas\",\n \"systemVariablesDropdownLabel\": \"Variables del sistema\",\n \"systemVariableOptions\": {\n \"sourceContent\": \"Contenido de las fuentes\",\n \"sourceSummaries\": \"Alias del contenido (compatibilidad retroactiva)\",\n \"sourceDocuments\": \"Lista de objetos de documentos\",\n \"sourceCount\": \"Número de documentos recuperados\",\n \"systemDate\": \"Fecha actual (YYYY-MM-DD)\",\n \"systemTime\": \"Hora actual (HH:MM:SS)\",\n \"systemTimestamp\": \"Marca de tiempo ISO 8601\",\n \"systemRequestId\": \"Identificador único de solicitud\",\n \"systemUserId\": \"ID del usuario actual\"\n },\n \"learnAboutPrompts\": \"Aprende sobre los Prompts →\",\n \"publicPromptEditDisabled\": \"Los prompts públicos no se pueden editar\",\n \"promptTypePublic\": \"público\",\n \"promptTypePrivate\": \"privado\"\n },\n \"chunk\": {\n \"add\": \"Agregar Fragmento\",\n \"edit\": \"Editar\",\n \"title\": \"Título\",\n \"enterTitle\": \"Ingresar título\",\n \"bodyText\": \"Texto del cuerpo\",\n \"promptText\": \"Texto del prompt\",\n \"save\": \"Guardar\",\n \"close\": \"Cerrar\",\n \"cancel\": \"Cancelar\",\n \"delete\": \"Eliminar\",\n \"deleteConfirmation\": \"¿Estás seguro de que deseas eliminar este fragmento?\"\n },\n \"addAction\": {\n \"title\": \"New Action\",\n \"actionNamePlaceholder\": \"Action Name\",\n \"invalidFormat\": \"Invalid function name format. Use only letters, numbers, underscores, and hyphens.\",\n \"formatHelp\": \"Use only letters, numbers, underscores, and hyphens (e.g., `get_data`, `send_report`, etc.)\",\n \"addButton\": \"Add\"\n },\n \"agentDetails\": {\n \"title\": \"Access Details\",\n \"publicLink\": \"Public Link\",\n \"apiKey\": \"API Key\",\n \"webhookUrl\": \"Webhook URL\",\n \"generate\": \"Generate\",\n \"test\": \"Test\",\n \"learnMore\": \"Learn more\"\n },\n \"importSpec\": {\n \"title\": \"Importar especificación de API\",\n \"description\": \"Sube un archivo de especificación OpenAPI 3.x o Swagger 2.0 para generar acciones automáticamente.\",\n \"dropzoneText\": \"Haz clic para subir o arrastra y suelta\",\n \"supportedFormats\": \"Formato JSON o YAML\",\n \"invalidFileType\": \"Tipo de archivo no válido. Sube un archivo JSON o YAML.\",\n \"parseError\": \"No se pudo analizar la especificación. Verifica el formato del archivo.\",\n \"version\": \"Versión\",\n \"baseUrl\": \"URL base\",\n \"actionsFound\": \"{{count}} acciones encontradas\",\n \"selectAll\": \"Seleccionar todo\",\n \"deselectAll\": \"Deseleccionar todo\",\n \"cancel\": \"Cancelar\",\n \"parse\": \"Analizar\",\n \"import\": \"Importar ({{count}})\"\n }\n },\n \"sharedConv\": {\n \"subtitle\": \"Creado con\",\n \"button\": \"Comienza con DocsGPT\",\n \"meta\": \"DocsGPT utiliza GenAI, por favor revisa la información crítica utilizando fuentes.\"\n },\n \"convTile\": {\n \"share\": \"Compartir\",\n \"delete\": \"Eliminar\",\n \"rename\": \"Renombrar\",\n \"deleteWarning\": \"¿Está seguro de que desea eliminar esta conversación?\"\n },\n \"pagination\": {\n \"rowsPerPage\": \"Filas por página\",\n \"pageOf\": \"Página {{currentPage}} de {{totalPages}}\",\n \"firstPage\": \"Primera página\",\n \"previousPage\": \"Página anterior\",\n \"nextPage\": \"Página siguiente\",\n \"lastPage\": \"Última página\"\n },\n \"conversation\": {\n \"copy\": \"Copiar\",\n \"copied\": \"Copiado\",\n \"speak\": \"Hablar\",\n \"answer\": \"Respuesta\",\n \"edit\": {\n \"update\": \"Actualizar\",\n \"cancel\": \"Cancelar\",\n \"placeholder\": \"Ingrese la consulta actualizada...\"\n },\n \"sources\": {\n \"title\": \"Fuentes\",\n \"link\": \"Enlace fuente\",\n \"view_more\": \"Ver {{count}} más fuentes\",\n \"text\": \"Elegir tus fuentes\",\n \"noSourcesAvailable\": \"No hay fuentes disponibles\"\n },\n \"attachments\": {\n \"attach\": \"Adjuntar\",\n \"remove\": \"Eliminar adjunto\"\n },\n \"retry\": \"Reintentar\",\n \"reasoning\": \"Razonamiento\"\n },\n \"agents\": {\n \"title\": \"Agentes\",\n \"description\": \"Descubre y crea versiones personalizadas de DocsGPT que combinan instrucciones, conocimiento adicional y cualquier combinación de habilidades\",\n \"newAgent\": \"Nuevo Agente\",\n \"backToAll\": \"Volver a todos los agentes\",\n \"searchPlaceholder\": \"Buscar...\",\n \"noSearchResults\": \"No se encontraron agentes\",\n \"tryDifferentSearch\": \"Prueba con un término de búsqueda diferente\",\n \"filters\": {\n \"all\": \"Todos\",\n \"byDocsGPT\": \"Por DocsGPT\",\n \"byMe\": \"Por mí\",\n \"shared\": \"Compartidos conmigo\"\n },\n \"sections\": {\n \"template\": {\n \"title\": \"Por DocsGPT\",\n \"description\": \"Agentes proporcionados por DocsGPT\",\n \"emptyState\": \"No se encontraron agentes de plantilla.\"\n },\n \"user\": {\n \"title\": \"Por mí\",\n \"description\": \"Agentes creados o publicados por ti\",\n \"emptyState\": \"Aún no tienes agentes creados.\"\n },\n \"shared\": {\n \"title\": \"Compartidos conmigo\",\n \"description\": \"Agentes importados mediante un enlace público\",\n \"emptyState\": \"No se encontraron agentes compartidos.\"\n }\n },\n \"form\": {\n \"headings\": {\n \"new\": \"Nuevo Agente\",\n \"edit\": \"Editar Agente\",\n \"draft\": \"Nuevo Agente (Borrador)\"\n },\n \"buttons\": {\n \"publish\": \"Publicar\",\n \"save\": \"Guardar\",\n \"saveDraft\": \"Guardar Borrador\",\n \"cancel\": \"Cancelar\",\n \"delete\": \"Eliminar\",\n \"logs\": \"Registros\",\n \"accessDetails\": \"Detalles de Acceso\",\n \"add\": \"Agregar\"\n },\n \"sections\": {\n \"meta\": \"Meta\",\n \"source\": \"Fuente\",\n \"prompt\": \"Prompt\",\n \"tools\": \"Herramientas\",\n \"agentType\": \"Tipo de agente\",\n \"models\": \"Modelos\",\n \"advanced\": \"Avanzado\",\n \"preview\": \"Vista previa\"\n },\n \"placeholders\": {\n \"agentName\": \"Nombre del agente\",\n \"describeAgent\": \"Describe tu agente\",\n \"selectSources\": \"Seleccionar fuentes\",\n \"chunksPerQuery\": \"Fragmentos por consulta\",\n \"selectType\": \"Seleccionar tipo\",\n \"selectTools\": \"Seleccionar herramientas\",\n \"selectModels\": \"Seleccionar modelos para este agente\",\n \"selectDefaultModel\": \"Seleccionar modelo predeterminado\",\n \"enterTokenLimit\": \"Ingresar límite de tokens\",\n \"enterRequestLimit\": \"Ingresar límite de solicitudes\"\n },\n \"sourcePopup\": {\n \"title\": \"Seleccionar Fuentes\",\n \"searchPlaceholder\": \"Buscar fuentes...\",\n \"noOptionsMessage\": \"No hay fuentes disponibles\"\n },\n \"toolsPopup\": {\n \"title\": \"Seleccionar Herramientas\",\n \"searchPlaceholder\": \"Buscar herramientas...\",\n \"noOptionsMessage\": \"No hay herramientas disponibles\"\n },\n \"modelsPopup\": {\n \"title\": \"Seleccionar Modelos\",\n \"searchPlaceholder\": \"Buscar modelos...\",\n \"noOptionsMessage\": \"No hay modelos disponibles\"\n },\n \"upload\": {\n \"clickToUpload\": \"Haz clic para subir\",\n \"dragAndDrop\": \" o arrastra y suelta\"\n },\n \"agentTypes\": {\n \"classic\": \"Clásico\",\n \"react\": \"ReAct\"\n },\n \"labels\": {\n \"defaultModel\": \"Modelo Predeterminado\"\n },\n \"advanced\": {\n \"jsonSchema\": \"Esquema de respuesta JSON\",\n \"jsonSchemaDescription\": \"Define un esquema JSON para aplicar formato de salida estructurado\",\n \"validJson\": \"JSON válido\",\n \"invalidJson\": \"JSON inválido - corrige para habilitar el guardado\",\n \"tokenLimiting\": \"Límite de tokens\",\n \"tokenLimitingDescription\": \"Limita el total diario de tokens que puede usar este agente\",\n \"requestLimiting\": \"Límite de solicitudes\",\n \"requestLimitingDescription\": \"Limita el total diario de solicitudes que se pueden hacer a este agente\"\n },\n \"preview\": {\n \"publishedPreview\": \"Los agentes publicados se pueden previsualizar aquí\"\n },\n \"externalKb\": \"KB Externa\"\n },\n \"logs\": {\n \"title\": \"Registros del Agente\",\n \"lastUsedAt\": \"Último uso\",\n \"noUsageHistory\": \"Sin historial de uso\",\n \"tableHeader\": \"Registros del endpoint del agente\"\n },\n \"shared\": {\n \"notFound\": \"No se encontró el agente. Asegúrate de que el agente esté compartido.\"\n },\n \"preview\": {\n \"testMessage\": \"Prueba tu agente aquí. Los agentes publicados se pueden usar en conversaciones.\"\n },\n \"deleteConfirmation\": \"¿Estás seguro de que quieres eliminar este agente?\",\n \"folders\": {\n \"newFolder\": \"Nueva carpeta\",\n \"createFolder\": \"Crear carpeta\",\n \"folderName\": \"Nombre de la carpeta\",\n \"rename\": \"Renombrar\",\n \"delete\": \"Eliminar\",\n \"deleteConfirm\": \"¿Estás seguro de que quieres eliminar esta carpeta? Los agentes serán movidos fuera de la carpeta.\",\n \"empty\": \"Esta carpeta está vacía\",\n \"moveToFolder\": \"Mover a carpeta\",\n \"moveTo\": \"Mover\",\n \"move\": \"Mover\",\n \"noFolder\": \"Sin carpeta (raíz)\",\n \"backToRoot\": \"Volver\",\n \"noSubfolders\": \"Sin subcarpetas\",\n \"noFolders\": \"No hay carpetas todavía\"\n }\n },\n \"components\": {\n \"fileUpload\": {\n \"clickToUpload\": \"Click to upload or drag and drop\",\n \"dropFiles\": \"Drop the files here\",\n \"fileTypes\": \"PNG, JPG, JPEG up to\",\n \"sizeLimitUnit\": \"MB\",\n \"fileSizeError\": \"File exceeds {{size}}MB limit\"\n }\n },\n \"pageNotFound\": {\n \"title\": \"404\",\n \"message\": \"The page you are looking for does not exist.\",\n \"goHome\": \"Go Back Home\"\n },\n \"filePicker\": {\n \"searchPlaceholder\": \"Buscar archivos y carpetas...\",\n \"itemsSelected\": \"{{count}} seleccionados\",\n \"name\": \"Nombre\",\n \"lastModified\": \"Última modificación\",\n \"size\": \"Tamaño\",\n \"myFiles\": \"Mis archivos\",\n \"sharedWithMe\": \"Compartido conmigo\",\n \"loadingMore\": \"Cargando más archivos...\"\n },\n \"actionButtons\": {\n \"openNewChat\": \"Abrir nuevo chat\",\n \"share\": \"Compartir\"\n },\n \"mermaid\": {\n \"downloadOptions\": \"Opciones de descarga\",\n \"viewCode\": \"Ver código\",\n \"decreaseZoom\": \"Reducir zoom\",\n \"resetZoom\": \"Restablecer zoom\",\n \"increaseZoom\": \"Aumentar zoom\"\n },\n \"navigation\": {\n \"agents\": \"Agentes\"\n },\n \"notification\": {\n \"ariaLabel\": \"Notificación\",\n \"closeAriaLabel\": \"Cerrar notificación\"\n },\n \"prompts\": {\n \"textAriaLabel\": \"Texto del prompt\"\n }\n}\n" }, { "path": "frontend/src/locale/i18n.ts", "content": "import i18n from 'i18next';\nimport { initReactI18next } from 'react-i18next';\nimport LanguageDetector from 'i18next-browser-languagedetector';\n\nimport en from './en.json'; //English\nimport es from './es.json'; //Spanish\nimport jp from './jp.json'; //Japanese\nimport zh from './zh.json'; //Mandarin\nimport zhTW from './zh-TW.json'; //Traditional Chinese\nimport ru from './ru.json'; //Russian\nimport de from './de.json'; //German\n\ni18n\n .use(LanguageDetector)\n .use(initReactI18next)\n .init({\n resources: {\n en: {\n translation: en,\n },\n es: {\n translation: es,\n },\n jp: {\n translation: jp,\n },\n zh: {\n translation: zh,\n },\n zhTW: {\n translation: zhTW,\n },\n ru: {\n translation: ru,\n },\n de: {\n translation: de,\n },\n },\n fallbackLng: 'en',\n detection: {\n order: ['localStorage', 'navigator'],\n caches: ['localStorage'],\n lookupLocalStorage: 'docsgpt-locale',\n },\n });\n\ni18n.changeLanguage(i18n.language);\n\nexport default i18n;\n" }, { "path": "frontend/src/locale/jp.json", "content": "{\n \"language\": \"日本語\",\n \"chat\": \"チャット\",\n \"chats\": \"チャット\",\n \"newChat\": \"新しいチャット\",\n \"inputPlaceholder\": \"DocsGPTはどのようにお手伝いできますか？\",\n \"tagline\": \"DocsGPTはGenAIを使用しています。重要な情報はソースで確認してください。\",\n \"sourceDocs\": \"ソース\",\n \"none\": \"なし\",\n \"cancel\": \"キャンセル\",\n \"help\": \"ヘルプ\",\n \"emailUs\": \"メールを送る\",\n \"documentation\": \"ドキュメント\",\n \"manageAgents\": \"エージェント管理\",\n \"demo\": [\n {\n \"header\": \"DocsGPTについて学ぶ\",\n \"query\": \"DocsGPTとは何ですか?\"\n },\n {\n \"header\": \"ドキュメントを要約する\",\n \"query\": \"現在のコンテキストを要約してください\"\n },\n {\n \"header\": \"コードを書く\",\n \"query\": \"APIリクエストのコードを/api/answerに書いてください。\"\n },\n {\n \"header\": \"学習支援\",\n \"query\": \"このコンテンツに対する可能な質問を書いてください\"\n }\n ],\n \"settings\": {\n \"label\": \"設定\",\n \"general\": {\n \"label\": \"一般\",\n \"selectTheme\": \"テーマを選択\",\n \"light\": \"ライト\",\n \"dark\": \"ダーク\",\n \"selectLanguage\": \"言語を選択\",\n \"chunks\": \"クエリごとに処理されるチャンク\",\n \"prompt\": \"アクティブプロンプト\",\n \"deleteAllLabel\": \"すべての会話を削除\",\n \"deleteAllBtn\": \"すべて削除\",\n \"addNew\": \"新規追加\",\n \"convHistory\": \"会話履歴\",\n \"none\": \"なし\",\n \"low\": \"低\",\n \"medium\": \"中\",\n \"high\": \"高\",\n \"unlimited\": \"無制限\",\n \"default\": \"デフォルト\",\n \"add\": \"追加\"\n },\n \"sources\": {\n \"title\": \"ここでは、利用可能なすべてのソースファイルとアップロードしたファイルを管理できます。\",\n \"label\": \"ソース\",\n \"name\": \"ソース名\",\n \"date\": \"ベクトル日付\",\n \"type\": \"タイプ\",\n \"tokenUsage\": \"トークン使用量\",\n \"noData\": \"既存のソースがありません\",\n \"searchPlaceholder\": \"検索...\",\n \"addNew\": \"新規追加\",\n \"addSource\": \"ソースを追加\",\n \"addChunk\": \"チャンクを追加\",\n \"preLoaded\": \"プリロード済み\",\n \"private\": \"プライベート\",\n \"sync\": \"同期\",\n \"syncNow\": \"今すぐ同期\",\n \"syncing\": \"同期中...\",\n \"syncConfirmation\": \"\\\"{{sourceName}}\\\"を同期してもよろしいですか？これにより、コンテンツがクラウドストレージで更新され、個々のチャンクに加えた編集が上書きされる可能性があります。\",\n \"syncFrequency\": {\n \"never\": \"なし\",\n \"daily\": \"毎日\",\n \"weekly\": \"毎週\",\n \"monthly\": \"毎月\"\n },\n \"actions\": \"アクション\",\n \"view\": \"表示\",\n \"deleteWarning\": \"\\\"{{name}}\\\"を削除してもよろしいですか？\",\n \"confirmDelete\": \"このファイルを削除してもよろしいですか？この操作は元に戻せません。\",\n \"backToAll\": \"すべてのソースに戻る\",\n \"chunks\": \"チャンク\",\n \"noChunks\": \"チャンクが見つかりません\",\n \"noChunksAlt\": \"チャンクが見つかりません\",\n \"goToSources\": \"ソースへ移動\",\n \"uploadNew\": \"新規アップロード\",\n \"searchFiles\": \"ファイルを検索...\",\n \"noResults\": \"結果が見つかりません\",\n \"fileName\": \"名前\",\n \"tokens\": \"トークン\",\n \"size\": \"サイズ\",\n \"fileAlt\": \"ファイル\",\n \"folderAlt\": \"フォルダ\",\n \"parentFolderAlt\": \"親フォルダ\",\n \"menuAlt\": \"メニュー\",\n \"tokensUnit\": \"トークン\",\n \"editAlt\": \"編集\",\n \"uploading\": \"アップロード中…\",\n \"deleting\": \"削除中…\",\n \"queued\": \"キュー: {{count}}\",\n \"addFile\": \"ファイルを追加\",\n \"uploadingFilesTitle\": \"ファイルをアップロード中...\",\n \"deletingTitle\": \"削除中...\",\n \"deleteDirectoryWarning\": \"ディレクトリ \\\"{{name}}\\\" とその内容をすべて削除してもよろしいですか？この操作は元に戻せません。\",\n \"searchAlt\": \"検索\"\n },\n \"apiKeys\": {\n \"label\": \"チャットボット\",\n \"name\": \"名前\",\n \"key\": \"APIキー\",\n \"sourceDoc\": \"ソースドキュメント\",\n \"createNew\": \"新規作成\",\n \"noData\": \"既存のチャットボットはありません\",\n \"deleteConfirmation\": \"APIキー '{{name}}' を削除してもよろしいですか？\",\n \"description\": \"ここでチャットボットを作成・管理できます。チャットボットはウィジェットとしてウェブサイトに導入したり、アプリケーション内で使用したりすることができます。\"\n },\n \"analytics\": {\n \"label\": \"分析\",\n \"filterByChatbot\": \"チャットボットでフィルター\",\n \"selectChatbot\": \"チャットボットを選択\",\n \"filterOptions\": {\n \"hour\": \"時間\",\n \"last24Hours\": \"過去24時間\",\n \"last7Days\": \"過去7日間\",\n \"last15Days\": \"過去15日間\",\n \"last30Days\": \"過去30日間\"\n },\n \"messages\": \"メッセージ\",\n \"tokenUsage\": \"トークン使用量\",\n \"filterPlaceholder\": \"フィルター\",\n \"none\": \"なし\",\n \"positiveFeedback\": \"肯定的なフィードバック\",\n \"negativeFeedback\": \"否定的なフィードバック\",\n \"userFeedback\": \"ユーザーフィードバック\"\n },\n \"logs\": {\n \"label\": \"ログ\",\n \"filterByChatbot\": \"チャットボットでフィルター\",\n \"selectChatbot\": \"チャットボットを選択\",\n \"none\": \"なし\",\n \"tableHeader\": \"API生成 / チャットボットの会話\"\n },\n \"tools\": {\n \"label\": \"ツール\",\n \"searchPlaceholder\": \"検索...\",\n \"addTool\": \"ツールを追加\",\n \"noToolsFound\": \"ツールが見つかりません\",\n \"selectToolSetup\": \"設定するツールを選択してください\",\n \"settingsIconAlt\": \"設定アイコン\",\n \"configureToolAria\": \"{{toolName}}を設定\",\n \"toggleToolAria\": \"{{toolName}}を切り替え\",\n \"manageTools\": \"ツールへ移動\",\n \"edit\": \"編集\",\n \"delete\": \"削除\",\n \"deleteWarning\": \"ツール \\\"{{toolName}}\\\" を削除してもよろしいですか？\",\n \"unsavedChanges\": \"保存されていない変更があります。保存せずに離れると失われます。\",\n \"leaveWithoutSaving\": \"保存せずに離れる\",\n \"saveAndLeave\": \"保存して離れる\",\n \"customName\": \"カスタム名\",\n \"customNamePlaceholder\": \"カスタム名を入力（任意）\",\n \"authentication\": \"認証\",\n \"actions\": \"アクション\",\n \"addAction\": \"アクションを追加\",\n \"importSpec\": \"仕様をインポート\",\n \"searchActions\": \"アクションを検索...\",\n \"noActionsMatch\": \"検索に一致するアクションがありません\",\n \"actionAlreadyExists\": \"この名前のアクションは既に存在します\",\n \"noActionsFound\": \"アクションが見つかりません\",\n \"url\": \"URL\",\n \"urlPlaceholder\": \"URLを入力\",\n \"method\": \"メソッド\",\n \"description\": \"説明\",\n \"descriptionPlaceholder\": \"説明を入力\",\n \"bodyContentType\": \"ボディのコンテンツタイプ\",\n \"headers\": \"ヘッダー\",\n \"queryParameters\": \"クエリパラメータ\",\n \"body\": \"ボディ\",\n \"deleteActionWarning\": \"アクション \\\"{{name}}\\\" を削除してもよろしいですか？\",\n \"backToAllTools\": \"すべてのツールに戻る\",\n \"save\": \"保存\",\n \"fieldName\": \"フィールド名\",\n \"fieldType\": \"フィールドタイプ\",\n \"filledByLLM\": \"LLMによる入力\",\n \"fieldDescription\": \"フィールドの説明\",\n \"value\": \"値\",\n \"addProperty\": \"プロパティを追加\",\n \"propertyName\": \"新しいプロパティキー\",\n \"add\": \"追加\",\n \"cancel\": \"キャンセル\",\n \"addNew\": \"新規追加\",\n \"name\": \"名前\",\n \"type\": \"タイプ\",\n \"mcp\": {\n \"addServer\": \"Add MCP Server\",\n \"editServer\": \"Edit Server\",\n \"serverName\": \"Server Name\",\n \"serverUrl\": \"Server URL\",\n \"headerName\": \"Header Name\",\n \"timeout\": \"Timeout (seconds)\",\n \"testConnection\": \"Test Connection\",\n \"testing\": \"Testing\",\n \"saving\": \"Saving\",\n \"save\": \"Save\",\n \"cancel\": \"Cancel\",\n \"noAuth\": \"No Authentication\",\n \"oauthInProgress\": \"Waiting for OAuth completion...\",\n \"oauthCompleted\": \"OAuth completed successfully\",\n \"authType\": \"Authentication Type\",\n \"defaultServerName\": \"My MCP Server\",\n \"authTypes\": {\n \"none\": \"No Authentication\",\n \"apiKey\": \"API Key\",\n \"bearer\": \"Bearer Token\",\n \"oauth\": \"OAuth\",\n \"basic\": \"Basic Authentication\"\n },\n \"placeholders\": {\n \"serverUrl\": \"https://api.example.com\",\n \"apiKey\": \"Your secret API key\",\n \"bearerToken\": \"Your secret token\",\n \"username\": \"Your username\",\n \"password\": \"Your password\",\n \"oauthScopes\": \"OAuth scopes (comma separated)\"\n },\n \"errors\": {\n \"nameRequired\": \"Server name is required\",\n \"urlRequired\": \"Server URL is required\",\n \"invalidUrl\": \"Please enter a valid URL\",\n \"apiKeyRequired\": \"API key is required\",\n \"tokenRequired\": \"Bearer token is required\",\n \"usernameRequired\": \"Username is required\",\n \"passwordRequired\": \"Password is required\",\n \"testFailed\": \"Connection test failed\",\n \"saveFailed\": \"Failed to save MCP server\",\n \"oauthFailed\": \"OAuth process failed or was cancelled\",\n \"oauthTimeout\": \"OAuth process timed out, please try again\",\n \"timeoutRange\": \"Timeout must be between 1 and 300 seconds\"\n }\n }\n },\n \"scrollTabsLeft\": \"タブを左にスクロール\",\n \"tabsAriaLabel\": \"設定タブ\",\n \"scrollTabsRight\": \"タブを右にスクロール\"\n },\n \"modals\": {\n \"uploadDoc\": {\n \"label\": \"新しい文書をアップロードする\",\n \"select\": \"ドキュメントを DocsGPT にアップロードする方法を選択します\",\n \"selectSource\": \"ソースを追加する方法を選択してください\",\n \"selectedFiles\": \"選択されたファイル\",\n \"noFilesSelected\": \"ファイルが選択されていません\",\n \"file\": \"デバイスからアップロード\",\n \"back\": \"戻る\",\n \"wait\": \"お待ちください ...\",\n \"remote\": \"ウェブサイトから収集する\",\n \"start\": \"チャットを開始する\",\n \"name\": \"名前\",\n \"choose\": \"ファイルを選択\",\n \"info\": \"25MBまでの.pdf、.txt、.rst、.csv、.xlsx、.docx、.md、.html、.epub、.json、.pptx、.zipファイルをアップロードしてください\",\n \"uploadedFiles\": \"アップロードされたファイル\",\n \"cancel\": \"キャンセル\",\n \"train\": \"トレーニング\",\n \"link\": \"リンク\",\n \"urlLink\": \"URLリンク\",\n \"repoUrl\": \"リポジトリURL\",\n \"reddit\": {\n \"id\": \"クライアントID\",\n \"secret\": \"クライアントシークレット\",\n \"agent\": \"ユーザーエージェント\",\n \"searchQueries\": \"検索クエリ\",\n \"numberOfPosts\": \"投稿数\",\n \"addQuery\": \"クエリを追加\"\n },\n \"drag\": {\n \"title\": \"添付ファイルをここにドロップ\",\n \"description\": \"リリースして添付ファイルをアップロード\"\n },\n \"progress\": {\n \"upload\": \"アップロード中\",\n \"training\": \"アップロード中\",\n \"completed\": \"アップロード完了\",\n \"failed\": \"アップロード失敗\",\n \"wait\": \"数分かかる場合があります\",\n \"preparing\": \"アップロードを準備中\",\n \"tokenLimit\": \"トークン制限を超えています。より小さいドキュメントをアップロードしてください\",\n \"expandDetails\": \"アップロードの詳細を展開\",\n \"collapseDetails\": \"アップロードの詳細を折りたたむ\",\n \"dismiss\": \"アップロード通知を閉じる\",\n \"uploadProgress\": \"アップロード進行状況 {{progress}}%\",\n \"clear\": \"クリア\"\n },\n \"showAdvanced\": \"詳細オプションを表示\",\n \"hideAdvanced\": \"詳細オプションを非表示\",\n \"ingestors\": {\n \"local_file\": {\n \"label\": \"ファイルをアップロード\",\n \"heading\": \"新しいドキュメントをアップロード\"\n },\n \"crawler\": {\n \"label\": \"クローラー\",\n \"heading\": \"Webクローラーでコンテンツを追加\"\n },\n \"url\": {\n \"label\": \"リンク\",\n \"heading\": \"URLからコンテンツを追加\"\n },\n \"github\": {\n \"label\": \"GitHub\",\n \"heading\": \"GitHubからコンテンツを追加\"\n },\n \"reddit\": {\n \"label\": \"Reddit\",\n \"heading\": \"Redditからコンテンツを追加\"\n },\n \"google_drive\": {\n \"label\": \"Google Drive\",\n \"heading\": \"Google Driveからアップロード\"\n },\n \"s3\": {\n \"label\": \"Amazon S3\",\n \"heading\": \"Amazon S3からコンテンツを追加\"\n },\n \"share_point\": {\n \"label\": \"SharePoint\",\n \"heading\": \"SharePointからアップロード\"\n }\n },\n \"connectors\": {\n \"auth\": {\n \"connectedUser\": \"接続されたユーザー\",\n \"authFailed\": \"認証に失敗しました\",\n \"authUrlFailed\": \"認証URLの取得に失敗しました\",\n \"popupBlocked\": \"認証ウィンドウを開けませんでした。ポップアップを許可してください。\",\n \"authCancelled\": \"認証がキャンセルされました\",\n \"connectedAs\": \"{{email}}として接続\",\n \"disconnect\": \"切断\"\n },\n \"googleDrive\": {\n \"connect\": \"Google Driveに接続\",\n \"sessionExpired\": \"セッションが期限切れです。Google Driveに再接続してください。\",\n \"sessionExpiredGeneric\": \"セッションが期限切れです。アカウントに再接続してください。\",\n \"validateFailed\": \"セッションの検証に失敗しました。再接続してください。\",\n \"noSession\": \"有効なセッションが見つかりません。Google Driveに再接続してください。\",\n \"noAccessToken\": \"アクセストークンが利用できません。Google Driveに再接続してください。\",\n \"pickerFailed\": \"ファイルピッカーを開けませんでした。もう一度お試しください。\",\n \"selectedFiles\": \"選択されたファイル\",\n \"selectFiles\": \"ファイルを選択\",\n \"loading\": \"読み込み中...\",\n \"noFilesSelected\": \"ファイルまたはフォルダが選択されていません\",\n \"folders\": \"フォルダ\",\n \"files\": \"ファイル\",\n \"remove\": \"削除\",\n \"folderAlt\": \"フォルダ\",\n \"fileAlt\": \"ファイル\"\n },\n \"sharePoint\": {\n \"connect\": \"SharePointに接続\",\n \"sessionExpired\": \"セッションが期限切れです。SharePointに再接続してください。\",\n \"sessionExpiredGeneric\": \"セッションが期限切れです。アカウントに再接続してください。\",\n \"validateFailed\": \"セッションの検証に失敗しました。再接続してください。\",\n \"noSession\": \"有効なセッションが見つかりません。SharePointに再接続してください。\",\n \"noAccessToken\": \"アクセストークンが利用できません。SharePointに再接続してください。\",\n \"pickerFailed\": \"ファイルピッカーを開けませんでした。もう一度お試しください。\",\n \"selectedFiles\": \"選択されたファイル\",\n \"selectFiles\": \"ファイルを選択\",\n \"loading\": \"読み込み中...\",\n \"noFilesSelected\": \"ファイルまたはフォルダが選択されていません\",\n \"folders\": \"フォルダ\",\n \"files\": \"ファイル\",\n \"remove\": \"削除\",\n \"folderAlt\": \"フォルダ\",\n \"fileAlt\": \"ファイル\"\n }\n }\n },\n \"createAPIKey\": {\n \"label\": \"新しいAPIキーを作成\",\n \"apiKeyName\": \"APIキー名\",\n \"chunks\": \"クエリごとに処理されるチャンク\",\n \"prompt\": \"アクティブプロンプトを選択\",\n \"sourceDoc\": \"ソースドキュメント\",\n \"create\": \"作成\"\n },\n \"saveKey\": {\n \"note\": \"キーを保存してください\",\n \"disclaimer\": \"キーが表示されるのはこのときだけです。\",\n \"copy\": \"コピー\",\n \"copied\": \"コピーしました\",\n \"confirm\": \"キーを保存しました\",\n \"apiKeyLabel\": \"API Key\"\n },\n \"deleteConv\": {\n \"confirm\": \"すべての会話を削除してもよろしいですか?\",\n \"delete\": \"削除\"\n },\n \"shareConv\": {\n \"label\": \"共有ページを作成して共有する\",\n \"note\": \"ソースドキュメント、個人情報、および以降の会話は非公開のままになります\",\n \"create\": \"作成\",\n \"option\": \"ユーザーがより多くのクエリを実行できるようにします。\"\n },\n \"configTool\": {\n \"title\": \"ツール設定\",\n \"type\": \"タイプ\",\n \"apiKeyLabel\": \"APIキー / OAuth\",\n \"apiKeyPlaceholder\": \"APIキー / OAuthを入力してください\",\n \"addButton\": \"ツールを追加\",\n \"closeButton\": \"閉じる\",\n \"customNamePlaceholder\": \"Enter custom name (optional)\"\n },\n \"prompts\": {\n \"addPrompt\": \"プロンプトを追加\",\n \"addDescription\": \"カスタムプロンプトを追加して DocsGPT に保存します\",\n \"editPrompt\": \"プロンプトを編集\",\n \"editDescription\": \"カスタムプロンプトを編集して DocsGPT に保存します\",\n \"promptName\": \"プロンプト名\",\n \"promptText\": \"プロンプトのテキスト\",\n \"save\": \"保存\",\n \"cancel\": \"キャンセル\",\n \"nameExists\": \"この名前はすでに存在します\",\n \"deleteConfirmation\": \"プロンプト『{{name}}』を削除してもよろしいですか？\",\n \"placeholderText\": \"ここにプロンプトのテキストを入力してください…\",\n \"addExamplePlaceholder\": \"このテキストを要約してください：\",\n \"variablesLabel\": \"変数\",\n \"variablesSubtext\": \"クリックしてプロンプトに挿入\",\n \"variablesDescription\": \"クリックしてプロンプトに挿入\",\n \"systemVariables\": \"システム変数\",\n \"toolVariables\": \"ツール変数\",\n \"systemVariablesDropdownLabel\": \"System Variables\",\n \"systemVariableOptions\": {\n \"sourceContent\": \"Sources content\",\n \"sourceSummaries\": \"Alias for content (backward compatible)\",\n \"sourceDocuments\": \"Document objects list\",\n \"sourceCount\": \"Number of retrieved documents\",\n \"systemDate\": \"Current date (YYYY-MM-DD)\",\n \"systemTime\": \"Current time (HH:MM:SS)\",\n \"systemTimestamp\": \"ISO 8601 timestamp\",\n \"systemRequestId\": \"Unique request identifier\",\n \"systemUserId\": \"Current user ID\"\n },\n \"learnAboutPrompts\": \"プロンプトについて学ぶ →\",\n \"publicPromptEditDisabled\": \"公開プロンプトは編集できません\",\n \"promptTypePublic\": \"公開\",\n \"promptTypePrivate\": \"非公開\"\n },\n \"chunk\": {\n \"add\": \"チャンクを追加\",\n \"edit\": \"編集\",\n \"title\": \"タイトル\",\n \"enterTitle\": \"タイトルを入力\",\n \"bodyText\": \"本文\",\n \"promptText\": \"プロンプトテキスト\",\n \"save\": \"保存\",\n \"close\": \"閉じる\",\n \"cancel\": \"キャンセル\",\n \"delete\": \"削除\",\n \"deleteConfirmation\": \"このチャンクを削除してもよろしいですか？\"\n },\n \"addAction\": {\n \"title\": \"New Action\",\n \"actionNamePlaceholder\": \"Action Name\",\n \"invalidFormat\": \"Invalid function name format. Use only letters, numbers, underscores, and hyphens.\",\n \"formatHelp\": \"Use only letters, numbers, underscores, and hyphens (e.g., `get_data`, `send_report`, etc.)\",\n \"addButton\": \"Add\"\n },\n \"agentDetails\": {\n \"title\": \"Access Details\",\n \"publicLink\": \"Public Link\",\n \"apiKey\": \"API Key\",\n \"webhookUrl\": \"Webhook URL\",\n \"generate\": \"Generate\",\n \"test\": \"Test\",\n \"learnMore\": \"Learn more\"\n },\n \"importSpec\": {\n \"title\": \"API仕様のインポート\",\n \"description\": \"OpenAPI 3.x または Swagger 2.0 の仕様ファイルをアップロードして、アクションを自動生成します。\",\n \"dropzoneText\": \"クリックしてアップロード、またはドラッグ＆ドロップ\",\n \"supportedFormats\": \"JSON または YAML 形式\",\n \"invalidFileType\": \"無効なファイル形式です。JSON または YAML ファイルをアップロードしてください。\",\n \"parseError\": \"仕様の解析に失敗しました。ファイル形式を確認してください。\",\n \"version\": \"バージョン\",\n \"baseUrl\": \"ベースURL\",\n \"actionsFound\": \"{{count}} 件のアクションが見つかりました\",\n \"selectAll\": \"すべて選択\",\n \"deselectAll\": \"すべて解除\",\n \"cancel\": \"キャンセル\",\n \"parse\": \"解析\",\n \"import\": \"インポート ({{count}})\"\n }\n },\n \"sharedConv\": {\n \"subtitle\": \"作成者\",\n \"button\": \"DocsGPT を始める\",\n \"meta\": \"DocsGPT は GenAI を使用しています、情報源を使用して重要情報を確認してください。\"\n },\n \"convTile\": {\n \"share\": \"共有\",\n \"delete\": \"削除\",\n \"rename\": \"名前変更\",\n \"deleteWarning\": \"この会話を削除してもよろしいですか？\"\n },\n \"pagination\": {\n \"rowsPerPage\": \"1ページあたりの行数\",\n \"pageOf\": \"ページ {{currentPage}} / {{totalPages}}\",\n \"firstPage\": \"最初のページ\",\n \"previousPage\": \"前のページ\",\n \"nextPage\": \"次のページ\",\n \"lastPage\": \"最後のページ\"\n },\n \"conversation\": {\n \"copy\": \"コピー\",\n \"copied\": \"コピー済み\",\n \"speak\": \"読み上げ\",\n \"answer\": \"回答\",\n \"edit\": {\n \"update\": \"更新\",\n \"cancel\": \"キャンセル\",\n \"placeholder\": \"更新されたクエリを入力...\"\n },\n \"sources\": {\n \"title\": \"ソース\",\n \"text\": \"ソーステキスト\",\n \"link\": \"ソースリンク\",\n \"view_more\": \"さらに{{count}}個のソース\",\n \"noSourcesAvailable\": \"利用可能なソースがありません\"\n },\n \"attachments\": {\n \"attach\": \"添付\",\n \"remove\": \"添付ファイルを削除\"\n },\n \"retry\": \"再試行\",\n \"reasoning\": \"推論\"\n },\n \"agents\": {\n \"title\": \"エージェント\",\n \"description\": \"指示、追加知識、スキルの組み合わせを含むDocsGPTのカスタムバージョンを発見して作成します\",\n \"newAgent\": \"新しいエージェント\",\n \"backToAll\": \"すべてのエージェントに戻る\",\n \"searchPlaceholder\": \"検索...\",\n \"noSearchResults\": \"エージェントが見つかりません\",\n \"tryDifferentSearch\": \"別の検索語をお試しください\",\n \"filters\": {\n \"all\": \"すべて\",\n \"byDocsGPT\": \"DocsGPT提供\",\n \"byMe\": \"自分の\",\n \"shared\": \"共有された\"\n },\n \"sections\": {\n \"template\": {\n \"title\": \"DocsGPT提供\",\n \"description\": \"DocsGPTが提供するエージェント\",\n \"emptyState\": \"テンプレートエージェントが見つかりません。\"\n },\n \"user\": {\n \"title\": \"自分のエージェント\",\n \"description\": \"あなたが作成または公開したエージェント\",\n \"emptyState\": \"まだ作成されたエージェントがありません。\"\n },\n \"shared\": {\n \"title\": \"共有されたエージェント\",\n \"description\": \"公開リンクを使用してインポートされたエージェント\",\n \"emptyState\": \"共有エージェントが見つかりません。\"\n }\n },\n \"form\": {\n \"headings\": {\n \"new\": \"新しいエージェント\",\n \"edit\": \"エージェントを編集\",\n \"draft\": \"新しいエージェント（下書き）\"\n },\n \"buttons\": {\n \"publish\": \"公開\",\n \"save\": \"保存\",\n \"saveDraft\": \"下書きを保存\",\n \"cancel\": \"キャンセル\",\n \"delete\": \"削除\",\n \"logs\": \"ログ\",\n \"accessDetails\": \"アクセス詳細\",\n \"add\": \"追加\"\n },\n \"sections\": {\n \"meta\": \"メタ\",\n \"source\": \"ソース\",\n \"prompt\": \"プロンプト\",\n \"tools\": \"ツール\",\n \"agentType\": \"エージェントタイプ\",\n \"models\": \"モデル\",\n \"advanced\": \"詳細設定\",\n \"preview\": \"プレビュー\"\n },\n \"placeholders\": {\n \"agentName\": \"エージェント名\",\n \"describeAgent\": \"エージェントを説明してください\",\n \"selectSources\": \"ソースを選択\",\n \"chunksPerQuery\": \"クエリごとのチャンク数\",\n \"selectType\": \"タイプを選択\",\n \"selectTools\": \"ツールを選択\",\n \"selectModels\": \"このエージェントのモデルを選択\",\n \"selectDefaultModel\": \"デフォルトモデルを選択\",\n \"enterTokenLimit\": \"トークン制限を入力\",\n \"enterRequestLimit\": \"リクエスト制限を入力\"\n },\n \"sourcePopup\": {\n \"title\": \"ソースを選択\",\n \"searchPlaceholder\": \"ソースを検索...\",\n \"noOptionsMessage\": \"利用可能なソースがありません\"\n },\n \"toolsPopup\": {\n \"title\": \"ツールを選択\",\n \"searchPlaceholder\": \"ツールを検索...\",\n \"noOptionsMessage\": \"利用可能なツールがありません\"\n },\n \"modelsPopup\": {\n \"title\": \"モデルを選択\",\n \"searchPlaceholder\": \"モデルを検索...\",\n \"noOptionsMessage\": \"利用可能なモデルがありません\"\n },\n \"upload\": {\n \"clickToUpload\": \"クリックしてアップロード\",\n \"dragAndDrop\": \" またはドラッグ＆ドロップ\"\n },\n \"agentTypes\": {\n \"classic\": \"クラシック\",\n \"react\": \"ReAct\"\n },\n \"labels\": {\n \"defaultModel\": \"デフォルトモデル\"\n },\n \"advanced\": {\n \"jsonSchema\": \"JSON応答スキーマ\",\n \"jsonSchemaDescription\": \"構造化された出力形式を適用するためのJSONスキーマを定義します\",\n \"validJson\": \"有効なJSON\",\n \"invalidJson\": \"無効なJSON - 保存を有効にするには修正してください\",\n \"tokenLimiting\": \"トークン制限\",\n \"tokenLimitingDescription\": \"このエージェントが使用できる1日の合計トークン数を制限します\",\n \"requestLimiting\": \"リクエスト制限\",\n \"requestLimitingDescription\": \"このエージェントに対して行える1日の合計リクエスト数を制限します\"\n },\n \"preview\": {\n \"publishedPreview\": \"公開されたエージェントはここでプレビューできます\"\n },\n \"externalKb\": \"外部KB\"\n },\n \"logs\": {\n \"title\": \"エージェントログ\",\n \"lastUsedAt\": \"最終使用日時\",\n \"noUsageHistory\": \"使用履歴がありません\",\n \"tableHeader\": \"エージェントエンドポイントログ\"\n },\n \"shared\": {\n \"notFound\": \"エージェントが見つかりません。エージェントが共有されていることを確認してください。\"\n },\n \"preview\": {\n \"testMessage\": \"ここでエージェントをテストできます。公開されたエージェントは会話で使用できます。\"\n },\n \"deleteConfirmation\": \"このエージェントを削除してもよろしいですか？\",\n \"folders\": {\n \"newFolder\": \"新しいフォルダ\",\n \"createFolder\": \"フォルダを作成\",\n \"folderName\": \"フォルダ名\",\n \"rename\": \"名前を変更\",\n \"delete\": \"削除\",\n \"deleteConfirm\": \"このフォルダを削除してもよろしいですか？フォルダ内のエージェントは移動されます。\",\n \"empty\": \"このフォルダは空です\",\n \"moveToFolder\": \"フォルダに移動\",\n \"moveTo\": \"移動\",\n \"move\": \"移動\",\n \"noFolder\": \"フォルダなし (ルート)\",\n \"backToRoot\": \"戻る\",\n \"noSubfolders\": \"サブフォルダなし\",\n \"noFolders\": \"フォルダがありません\"\n }\n },\n \"components\": {\n \"fileUpload\": {\n \"clickToUpload\": \"Click to upload or drag and drop\",\n \"dropFiles\": \"Drop the files here\",\n \"fileTypes\": \"PNG, JPG, JPEG up to\",\n \"sizeLimitUnit\": \"MB\",\n \"fileSizeError\": \"File exceeds {{size}}MB limit\"\n }\n },\n \"pageNotFound\": {\n \"title\": \"404\",\n \"message\": \"The page you are looking for does not exist.\",\n \"goHome\": \"Go Back Home\"\n },\n \"filePicker\": {\n \"searchPlaceholder\": \"ファイルとフォルダを検索...\",\n \"itemsSelected\": \"{{count}} 件選択済み\",\n \"name\": \"名前\",\n \"lastModified\": \"最終更新日\",\n \"size\": \"サイズ\",\n \"myFiles\": \"マイファイル\",\n \"sharedWithMe\": \"共有アイテム\",\n \"loadingMore\": \"さらに読み込み中...\"\n },\n \"actionButtons\": {\n \"openNewChat\": \"新しいチャットを開く\",\n \"share\": \"共有\"\n },\n \"mermaid\": {\n \"downloadOptions\": \"ダウンロードオプション\",\n \"viewCode\": \"コードを表示\",\n \"decreaseZoom\": \"ズームアウト\",\n \"resetZoom\": \"ズームをリセット\",\n \"increaseZoom\": \"ズームイン\"\n },\n \"navigation\": {\n \"agents\": \"エージェント\"\n },\n \"notification\": {\n \"ariaLabel\": \"通知\",\n \"closeAriaLabel\": \"通知を閉じる\"\n },\n \"prompts\": {\n \"textAriaLabel\": \"プロンプトテキスト\"\n }\n}\n" }, { "path": "frontend/src/locale/ru.json", "content": "{\n \"language\": \"Русский\",\n \"chat\": \"Чат\",\n \"chats\": \"Чаты\",\n \"newChat\": \"Новый чат\",\n \"inputPlaceholder\": \"Как DocsGPT может вам помочь?\",\n \"tagline\": \"DocsGPT использует GenAI, пожалуйста, проверьте важную информацию, используя источники.\",\n \"sourceDocs\": \"Источник\",\n \"none\": \"Нет\",\n \"cancel\": \"Отмена\",\n \"help\": \"Помощь\",\n \"emailUs\": \"Напишите нам\",\n \"documentation\": \"Документация\",\n \"manageAgents\": \"Управление агентами\",\n \"demo\": [\n {\n \"header\": \"Узнайте о DocsGPT\",\n \"query\": \"Что такое DocsGPT?\"\n },\n {\n \"header\": \"Обобщить документацию\",\n \"query\": \"Обобщить текущий контекст\"\n },\n {\n \"header\": \"Написать код\",\n \"query\": \"Написать код для запроса API к /api/answer\"\n },\n {\n \"header\": \"Помощь в обучении\",\n \"query\": \"Написать возможные вопросы для этого контента\"\n }\n ],\n \"settings\": {\n \"label\": \"Настройки\",\n \"general\": {\n \"label\": \"Общие\",\n \"selectTheme\": \"Выбрать тему\",\n \"light\": \"Светлая\",\n \"dark\": \"Тёмная\",\n \"selectLanguage\": \"Выбрать язык\",\n \"chunks\": \"Обработанные фрагменты на запрос\",\n \"prompt\": \"Активная подсказка\",\n \"deleteAllLabel\": \"Удалить все беседы\",\n \"deleteAllBtn\": \"Удалить все\",\n \"addNew\": \"Добавить новый\",\n \"convHistory\": \"История разговоров\",\n \"none\": \"Нет\",\n \"low\": \"Низкий\",\n \"medium\": \"Средний\",\n \"high\": \"Высокий\",\n \"unlimited\": \"Без ограничений\",\n \"default\": \"По умолчанию\",\n \"add\": \"Добавить\"\n },\n \"sources\": {\n \"title\": \"Здесь вы можете управлять всеми исходными файлами, которые доступны вам и которые вы загрузили.\",\n \"label\": \"Источники\",\n \"name\": \"Название источника\",\n \"date\": \"Дата вектора\",\n \"type\": \"Тип\",\n \"tokenUsage\": \"Использование токена\",\n \"noData\": \"Нет существующих источников\",\n \"searchPlaceholder\": \"Поиск...\",\n \"addNew\": \"добавить новый\",\n \"addSource\": \"Добавить источник\",\n \"addChunk\": \"Добавить фрагмент\",\n \"preLoaded\": \"Предзагруженный\",\n \"private\": \"Частный\",\n \"sync\": \"Синхронизация\",\n \"syncNow\": \"Синхронизировать сейчас\",\n \"syncing\": \"Синхронизация...\",\n \"syncConfirmation\": \"Вы уверены, что хотите синхронизировать \\\"{{sourceName}}\\\"? Это обновит содержимое с вашим облачным хранилищем и может перезаписать любые изменения, внесенные вами в отдельные фрагменты.\",\n \"syncFrequency\": {\n \"never\": \"Никогда\",\n \"daily\": \"Ежедневно\",\n \"weekly\": \"Еженедельно\",\n \"monthly\": \"Ежемесячно\"\n },\n \"actions\": \"Действия\",\n \"view\": \"Просмотр\",\n \"deleteWarning\": \"Вы уверены, что хотите удалить \\\"{{name}}\\\"?\",\n \"confirmDelete\": \"Вы уверены, что хотите удалить этот файл? Это действие нельзя отменить.\",\n \"backToAll\": \"Вернуться ко всем источникам\",\n \"chunks\": \"Фрагменты\",\n \"noChunks\": \"Фрагменты не найдены\",\n \"noChunksAlt\": \"Фрагменты не найдены\",\n \"goToSources\": \"Перейти к источникам\",\n \"uploadNew\": \"Загрузить новый\",\n \"searchFiles\": \"Поиск файлов...\",\n \"noResults\": \"Результаты не найдены\",\n \"fileName\": \"Имя\",\n \"tokens\": \"Токены\",\n \"size\": \"Размер\",\n \"fileAlt\": \"Файл\",\n \"folderAlt\": \"Папка\",\n \"parentFolderAlt\": \"Родительская папка\",\n \"menuAlt\": \"Меню\",\n \"tokensUnit\": \"токенов\",\n \"editAlt\": \"Редактировать\",\n \"uploading\": \"Загрузка…\",\n \"deleting\": \"Удаление…\",\n \"queued\": \"В очереди: {{count}}\",\n \"addFile\": \"Добавить файл\",\n \"uploadingFilesTitle\": \"Загрузка файлов...\",\n \"deletingTitle\": \"Удаление...\",\n \"deleteDirectoryWarning\": \"Вы уверены, что хотите удалить каталог \\\"{{name}}\\\" и все его содержимое? Это действие нельзя отменить.\",\n \"searchAlt\": \"Поиск\"\n },\n \"apiKeys\": {\n \"label\": \"API ключи\",\n \"name\": \"Название\",\n \"key\": \"API ключ\",\n \"sourceDoc\": \"Источник документа\",\n \"createNew\": \"Создать новый\",\n \"noData\": \"Нет существующих чатботов\",\n \"deleteConfirmation\": \"Вы уверены, что хотите удалить API ключ '{{name}}'?\",\n \"description\": \"Здесь вы можете создавать и управлять чат-ботами. Чат-боты могут быть развернуты на веб-сайтах в виде виджетов или использоваться внутри ваших приложений.\"\n },\n \"analytics\": {\n \"label\": \"Аналитика\",\n \"filterByChatbot\": \"Фильтровать по чат-боту\",\n \"selectChatbot\": \"Выбрать чат-бота\",\n \"filterOptions\": {\n \"hour\": \"Час\",\n \"last24Hours\": \"Последние 24 часа\",\n \"last7Days\": \"Последние 7 дней\",\n \"last15Days\": \"Последние 15 дней\",\n \"last30Days\": \"Последние 30 дней\"\n },\n \"messages\": \"Сообщения\",\n \"tokenUsage\": \"Использование токена\",\n \"userFeedback\": \"Обратная Связь Пользователя\",\n \"filterPlaceholder\": \"Фильтр\",\n \"none\": \"Нет\",\n \"positiveFeedback\": \"Положительная обратная связь\",\n \"negativeFeedback\": \"Отрицательная обратная связь\"\n },\n \"logs\": {\n \"label\": \"Журналы\",\n \"filterByChatbot\": \"Фильтровать по чат-боту\",\n \"selectChatbot\": \"Выбрать чат-бота\",\n \"none\": \"Нет\",\n \"tableHeader\": \"API сгенерировано / разговоры с чат-ботом\"\n },\n \"tools\": {\n \"label\": \"Инструменты\",\n \"searchPlaceholder\": \"Поиск...\",\n \"addTool\": \"Добавить инструмент\",\n \"noToolsFound\": \"Инструменты не найдены\",\n \"selectToolSetup\": \"Выберите инструмент для настройки\",\n \"settingsIconAlt\": \"Значок настроек\",\n \"configureToolAria\": \"Настроить {{toolName}}\",\n \"toggleToolAria\": \"Переключить {{toolName}}\",\n \"manageTools\": \"Перейти к инструментам\",\n \"edit\": \"Редактировать\",\n \"delete\": \"Удалить\",\n \"deleteWarning\": \"Вы уверены, что хотите удалить инструмент \\\"{{toolName}}\\\"?\",\n \"unsavedChanges\": \"У вас есть несохраненные изменения, которые будут потеряны, если вы уйдете без сохранения.\",\n \"leaveWithoutSaving\": \"Уйти без сохранения\",\n \"saveAndLeave\": \"Сохранить и уйти\",\n \"customName\": \"Пользовательское имя\",\n \"customNamePlaceholder\": \"Введите пользовательское имя (необязательно)\",\n \"authentication\": \"Аутентификация\",\n \"actions\": \"Действия\",\n \"addAction\": \"Добавить действие\",\n \"importSpec\": \"Импорт спецификации\",\n \"searchActions\": \"Поиск действий...\",\n \"noActionsMatch\": \"Нет действий, соответствующих вашему поиску\",\n \"actionAlreadyExists\": \"Действие с таким именем уже существует\",\n \"noActionsFound\": \"Действия не найдены\",\n \"url\": \"URL\",\n \"urlPlaceholder\": \"Введите URL\",\n \"method\": \"Метод\",\n \"description\": \"Описание\",\n \"descriptionPlaceholder\": \"Введите описание\",\n \"bodyContentType\": \"Тип содержимого тела\",\n \"headers\": \"Заголовки\",\n \"queryParameters\": \"Параметры запроса\",\n \"body\": \"Тело запроса\",\n \"deleteActionWarning\": \"Вы уверены, что хотите удалить действие \\\"{{name}}\\\"?\",\n \"backToAllTools\": \"Вернуться ко всем инструментам\",\n \"save\": \"Сохранить\",\n \"fieldName\": \"Имя поля\",\n \"fieldType\": \"Тип поля\",\n \"filledByLLM\": \"Заполняется LLM\",\n \"fieldDescription\": \"Описание поля\",\n \"value\": \"Значение\",\n \"addProperty\": \"Добавить свойство\",\n \"propertyName\": \"Новый ключ свойства\",\n \"add\": \"Добавить\",\n \"cancel\": \"Отмена\",\n \"addNew\": \"Добавить новое\",\n \"name\": \"Имя\",\n \"type\": \"Тип\",\n \"mcp\": {\n \"addServer\": \"Add MCP Server\",\n \"editServer\": \"Edit Server\",\n \"serverName\": \"Server Name\",\n \"serverUrl\": \"Server URL\",\n \"headerName\": \"Header Name\",\n \"timeout\": \"Timeout (seconds)\",\n \"testConnection\": \"Test Connection\",\n \"testing\": \"Testing\",\n \"saving\": \"Saving\",\n \"save\": \"Save\",\n \"cancel\": \"Cancel\",\n \"noAuth\": \"No Authentication\",\n \"oauthInProgress\": \"Waiting for OAuth completion...\",\n \"oauthCompleted\": \"OAuth completed successfully\",\n \"authType\": \"Authentication Type\",\n \"defaultServerName\": \"My MCP Server\",\n \"authTypes\": {\n \"none\": \"No Authentication\",\n \"apiKey\": \"API Key\",\n \"bearer\": \"Bearer Token\",\n \"oauth\": \"OAuth\",\n \"basic\": \"Basic Authentication\"\n },\n \"placeholders\": {\n \"serverUrl\": \"https://api.example.com\",\n \"apiKey\": \"Your secret API key\",\n \"bearerToken\": \"Your secret token\",\n \"username\": \"Your username\",\n \"password\": \"Your password\",\n \"oauthScopes\": \"OAuth scopes (comma separated)\"\n },\n \"errors\": {\n \"nameRequired\": \"Server name is required\",\n \"urlRequired\": \"Server URL is required\",\n \"invalidUrl\": \"Please enter a valid URL\",\n \"apiKeyRequired\": \"API key is required\",\n \"tokenRequired\": \"Bearer token is required\",\n \"usernameRequired\": \"Username is required\",\n \"passwordRequired\": \"Password is required\",\n \"testFailed\": \"Connection test failed\",\n \"saveFailed\": \"Failed to save MCP server\",\n \"oauthFailed\": \"OAuth process failed or was cancelled\",\n \"oauthTimeout\": \"OAuth process timed out, please try again\",\n \"timeoutRange\": \"Timeout must be between 1 and 300 seconds\"\n }\n }\n },\n \"scrollTabsLeft\": \"Прокрутить вкладки влево\",\n \"tabsAriaLabel\": \"Вкладки настроек\",\n \"scrollTabsRight\": \"Прокрутить вкладки вправо\"\n },\n \"modals\": {\n \"uploadDoc\": {\n \"label\": \"Загрузить новый документ\",\n \"select\": \"Выберите способ загрузки документа в DocsGPT\",\n \"selectSource\": \"Выберите способ добавления источника\",\n \"selectedFiles\": \"Выбранные файлы\",\n \"noFilesSelected\": \"Файлы не выбраны\",\n \"file\": \"Загрузить с устройства\",\n \"back\": \"Назад\",\n \"wait\": \"Пожалуйста, подождите...\",\n \"remote\": \"Собрать с веб-сайта\",\n \"start\": \"Начать чат\",\n \"name\": \"Имя\",\n \"choose\": \"Выбрать файлы\",\n \"info\": \"Пожалуйста, загрузите файлы .pdf, .txt, .rst, .csv, .xlsx, .docx, .md, .html, .epub, .json, .pptx, .zip размером до 25 МБ\",\n \"uploadedFiles\": \"Загруженные файлы\",\n \"cancel\": \"Отмена\",\n \"train\": \"Тренировка\",\n \"link\": \"Ссылка\",\n \"urlLink\": \"URL ссылка\",\n \"repoUrl\": \"URL репозитория\",\n \"reddit\": {\n \"id\": \"ID клиента\",\n \"secret\": \"Секрет клиента\",\n \"agent\": \"Пользовательский агент\",\n \"searchQueries\": \"Поисковые запросы\",\n \"numberOfPosts\": \"Количество сообщений\",\n \"addQuery\": \"Добавить запрос\"\n },\n \"drag\": {\n \"title\": \"Перетащите вложения сюда\",\n \"description\": \"Отпустите, чтобы загрузить ваши вложения\"\n },\n \"progress\": {\n \"upload\": \"Идет загрузка\",\n \"training\": \"Идет загрузка\",\n \"completed\": \"Загрузка завершена\",\n \"failed\": \"Ошибка загрузки\",\n \"wait\": \"Это может занять несколько минут\",\n \"preparing\": \"Подготовка загрузки\",\n \"tokenLimit\": \"Превышен лимит токенов, рассмотрите возможность загрузки документа меньшего размера\",\n \"expandDetails\": \"Развернуть детали загрузки\",\n \"collapseDetails\": \"Свернуть детали загрузки\",\n \"dismiss\": \"Закрыть уведомление о загрузке\",\n \"uploadProgress\": \"Прогресс загрузки {{progress}}%\",\n \"clear\": \"Очистить\"\n },\n \"showAdvanced\": \"Показать расширенные настройки\",\n \"hideAdvanced\": \"Скрыть расширенные настройки\",\n \"ingestors\": {\n \"local_file\": {\n \"label\": \"Загрузить файл\",\n \"heading\": \"Загрузить новый документ\"\n },\n \"crawler\": {\n \"label\": \"Краулер\",\n \"heading\": \"Добавить контент с помощью веб-краулера\"\n },\n \"url\": {\n \"label\": \"Ссылка\",\n \"heading\": \"Добавить контент из URL\"\n },\n \"github\": {\n \"label\": \"GitHub\",\n \"heading\": \"Добавить контент из GitHub\"\n },\n \"reddit\": {\n \"label\": \"Reddit\",\n \"heading\": \"Добавить контент из Reddit\"\n },\n \"google_drive\": {\n \"label\": \"Google Drive\",\n \"heading\": \"Загрузить из Google Drive\"\n },\n \"s3\": {\n \"label\": \"Amazon S3\",\n \"heading\": \"Добавить контент из Amazon S3\"\n },\n \"share_point\": {\n \"label\": \"SharePoint\",\n \"heading\": \"Загрузить из SharePoint\"\n }\n },\n \"connectors\": {\n \"auth\": {\n \"connectedUser\": \"Подключенный пользователь\",\n \"authFailed\": \"Ошибка аутентификации\",\n \"authUrlFailed\": \"Не удалось получить URL авторизации\",\n \"popupBlocked\": \"Не удалось открыть окно аутентификации. Пожалуйста, разрешите всплывающие окна.\",\n \"authCancelled\": \"Аутентификация отменена\",\n \"connectedAs\": \"Подключен как {{email}}\",\n \"disconnect\": \"Отключить\"\n },\n \"googleDrive\": {\n \"connect\": \"Подключиться к Google Drive\",\n \"sessionExpired\": \"Сеанс истек. Пожалуйста, переподключитесь к Google Drive.\",\n \"sessionExpiredGeneric\": \"Сеанс истек. Пожалуйста, переподключите свою учетную запись.\",\n \"validateFailed\": \"Не удалось проверить сеанс. Пожалуйста, переподключитесь.\",\n \"noSession\": \"Действительный сеанс не найден. Пожалуйста, переподключитесь к Google Drive.\",\n \"noAccessToken\": \"Токен доступа недоступен. Пожалуйста, переподключитесь к Google Drive.\",\n \"pickerFailed\": \"Не удалось открыть средство выбора файлов. Пожалуйста, попробуйте еще раз.\",\n \"selectedFiles\": \"Выбранные файлы\",\n \"selectFiles\": \"Выбрать файлы\",\n \"loading\": \"Загрузка...\",\n \"noFilesSelected\": \"Файлы или папки не выбраны\",\n \"folders\": \"Папки\",\n \"files\": \"Файлы\",\n \"remove\": \"Удалить\",\n \"folderAlt\": \"Папка\",\n \"fileAlt\": \"Файл\"\n },\n \"sharePoint\": {\n \"connect\": \"Подключиться к SharePoint\",\n \"sessionExpired\": \"Сеанс истек. Пожалуйста, переподключитесь к SharePoint.\",\n \"sessionExpiredGeneric\": \"Сеанс истек. Пожалуйста, переподключите свою учетную запись.\",\n \"validateFailed\": \"Не удалось проверить сеанс. Пожалуйста, переподключитесь.\",\n \"noSession\": \"Действительный сеанс не найден. Пожалуйста, переподключитесь к SharePoint.\",\n \"noAccessToken\": \"Токен доступа недоступен. Пожалуйста, переподключитесь к SharePoint.\",\n \"pickerFailed\": \"Не удалось открыть средство выбора файлов. Пожалуйста, попробуйте еще раз.\",\n \"selectedFiles\": \"Выбранные файлы\",\n \"selectFiles\": \"Выбрать файлы\",\n \"loading\": \"Загрузка...\",\n \"noFilesSelected\": \"Файлы или папки не выбраны\",\n \"folders\": \"Папки\",\n \"files\": \"Файлы\",\n \"remove\": \"Удалить\",\n \"folderAlt\": \"Папка\",\n \"fileAlt\": \"Файл\"\n }\n }\n },\n \"createAPIKey\": {\n \"label\": \"Создать новый API ключ\",\n \"apiKeyName\": \"Название API ключа\",\n \"chunks\": \"Обработанные фрагменты на запрос\",\n \"prompt\": \"Выбрать активную подсказку\",\n \"sourceDoc\": \"Источник документа\",\n \"create\": \"Создать\"\n },\n \"saveKey\": {\n \"note\": \"Пожалуйста, сохраните ваш ключ\",\n \"disclaimer\": \"Ваш ключ будет показан только один раз.\",\n \"copy\": \"Копировать\",\n \"copied\": \"Скопировано\",\n \"confirm\": \"Я сохранил ключ\",\n \"apiKeyLabel\": \"API Key\"\n },\n \"deleteConv\": {\n \"confirm\": \"Вы уверены, что хотите удалить все разговоры?\",\n \"delete\": \"Удалить\"\n },\n \"shareConv\": {\n \"label\": \"Создать публичную страницу для совместного использования\",\n \"note\": \"Исходный документ, личная информация и последующие разговоры останутся приватными\",\n \"create\": \"Создать\",\n \"option\": \"Позволить пользователям делать дополнительные запросы.\"\n },\n \"configTool\": {\n \"title\": \"Настройка инструмента\",\n \"type\": \"Тип\",\n \"apiKeyLabel\": \"API ключ / OAuth\",\n \"apiKeyPlaceholder\": \"Введите API ключ / OAuth\",\n \"addButton\": \"Добавить инструмент\",\n \"closeButton\": \"Закрыть\",\n \"customNamePlaceholder\": \"Enter custom name (optional)\"\n },\n \"prompts\": {\n \"addPrompt\": \"Добавить промпт\",\n \"addDescription\": \"Добавьте свой собственный промпт и сохраните его в DocsGPT\",\n \"editPrompt\": \"Редактировать промпт\",\n \"editDescription\": \"Отредактируйте свой промпт и сохраните его в DocsGPT\",\n \"promptName\": \"Название промпта\",\n \"promptText\": \"Текст промпта\",\n \"save\": \"Сохранить\",\n \"cancel\": \"Отмена\",\n \"nameExists\": \"Имя уже существует\",\n \"deleteConfirmation\": \"Вы уверены, что хотите удалить промпт «{{name}}»?\",\n \"placeholderText\": \"Введите текст вашего промпта здесь...\",\n \"addExamplePlaceholder\": \"Пожалуйста, кратко изложите этот текст:\",\n \"variablesLabel\": \"Переменные\",\n \"variablesSubtext\": \"Нажмите, чтобы вставить в промпт\",\n \"variablesDescription\": \"Нажмите, чтобы вставить в промпт\",\n \"systemVariables\": \"Системные переменные\",\n \"toolVariables\": \"Переменные инструментов\",\n \"systemVariablesDropdownLabel\": \"Системные переменные\",\n \"systemVariableOptions\": {\n \"sourceContent\": \"Содержимое источников\",\n \"sourceSummaries\": \"Псевдоним содержимого (обратная совместимость)\",\n \"sourceDocuments\": \"Список объектов документов\",\n \"sourceCount\": \"Количество полученных документов\",\n \"systemDate\": \"Текущая дата (ГГГГ-ММ-ДД)\",\n \"systemTime\": \"Текущее время (ЧЧ:ММ:СС)\",\n \"systemTimestamp\": \"Отметка времени ISO 8601\",\n \"systemRequestId\": \"Уникальный идентификатор запроса\",\n \"systemUserId\": \"Идентификатор текущего пользователя\"\n },\n \"learnAboutPrompts\": \"Узнать о промптах →\",\n \"publicPromptEditDisabled\": \"Публичные промпты нельзя редактировать\",\n \"promptTypePublic\": \"публичный\",\n \"promptTypePrivate\": \"приватный\"\n },\n \"chunk\": {\n \"add\": \"Добавить фрагмент\",\n \"edit\": \"Редактировать\",\n \"title\": \"Заголовок\",\n \"enterTitle\": \"Введите заголовок\",\n \"bodyText\": \"Текст\",\n \"promptText\": \"Текст подсказки\",\n \"save\": \"Сохранить\",\n \"close\": \"Закрыть\",\n \"cancel\": \"Отмена\",\n \"delete\": \"Удалить\",\n \"deleteConfirmation\": \"Вы уверены, что хотите удалить этот фрагмент?\"\n },\n \"addAction\": {\n \"title\": \"New Action\",\n \"actionNamePlaceholder\": \"Action Name\",\n \"invalidFormat\": \"Invalid function name format. Use only letters, numbers, underscores, and hyphens.\",\n \"formatHelp\": \"Use only letters, numbers, underscores, and hyphens (e.g., `get_data`, `send_report`, etc.)\",\n \"addButton\": \"Add\"\n },\n \"agentDetails\": {\n \"title\": \"Access Details\",\n \"publicLink\": \"Public Link\",\n \"apiKey\": \"API Key\",\n \"webhookUrl\": \"Webhook URL\",\n \"generate\": \"Generate\",\n \"test\": \"Test\",\n \"learnMore\": \"Learn more\"\n },\n \"importSpec\": {\n \"title\": \"Импорт спецификации API\",\n \"description\": \"Загрузите файл спецификации OpenAPI 3.x или Swagger 2.0 для автоматического создания действий.\",\n \"dropzoneText\": \"Нажмите для загрузки или перетащите файл\",\n \"supportedFormats\": \"Формат JSON или YAML\",\n \"invalidFileType\": \"Неверный тип файла. Пожалуйста, загрузите файл JSON или YAML.\",\n \"parseError\": \"Не удалось разобрать спецификацию. Проверьте формат файла.\",\n \"version\": \"Версия\",\n \"baseUrl\": \"Базовый URL\",\n \"actionsFound\": \"{{count}} действий найдено\",\n \"selectAll\": \"Выбрать все\",\n \"deselectAll\": \"Снять выделение со всех\",\n \"cancel\": \"Отмена\",\n \"parse\": \"Разобрать\",\n \"import\": \"Импорт ({{count}})\"\n }\n },\n \"sharedConv\": {\n \"subtitle\": \"Создано с помощью\",\n \"button\": \"Начать работу с DocsGPT\",\n \"meta\": \"DocsGPT использует GenAI, пожалуйста, проверьте важную информацию, используя источники.\"\n },\n \"convTile\": {\n \"share\": \"Поделиться\",\n \"delete\": \"Удалить\",\n \"rename\": \"Переименовать\",\n \"deleteWarning\": \"Вы уверены, что хотите удалить этот разговор?\"\n },\n \"pagination\": {\n \"rowsPerPage\": \"Строк на странице\",\n \"pageOf\": \"Страница {{currentPage}} из {{totalPages}}\",\n \"firstPage\": \"Первая страница\",\n \"previousPage\": \"Предыдущая страница\",\n \"nextPage\": \"Следующая страница\",\n \"lastPage\": \"Последняя страница\"\n },\n \"conversation\": {\n \"copy\": \"Копировать\",\n \"copied\": \"Скопировано\",\n \"speak\": \"Озвучить\",\n \"answer\": \"Ответ\",\n \"edit\": {\n \"update\": \"Обновить\",\n \"cancel\": \"Отмена\",\n \"placeholder\": \"Введите обновленный запрос...\"\n },\n \"sources\": {\n \"title\": \"Источники\",\n \"text\": \"Выберите ваши источники\",\n \"link\": \"Ссылка на источник\",\n \"view_more\": \"ещё {{count}} источников\",\n \"noSourcesAvailable\": \"Нет доступных источников\"\n },\n \"attachments\": {\n \"attach\": \"Прикрепить\",\n \"remove\": \"Удалить вложение\"\n },\n \"retry\": \"Повторить\",\n \"reasoning\": \"Рассуждение\"\n },\n \"agents\": {\n \"title\": \"Агенты\",\n \"description\": \"Откройте и создайте пользовательские версии DocsGPT, которые объединяют инструкции, дополнительные знания и любую комбинацию навыков\",\n \"newAgent\": \"Новый Агент\",\n \"backToAll\": \"Вернуться ко всем агентам\",\n \"searchPlaceholder\": \"Поиск...\",\n \"noSearchResults\": \"Агенты не найдены\",\n \"tryDifferentSearch\": \"Попробуйте другой поисковый запрос\",\n \"filters\": {\n \"all\": \"Все\",\n \"byDocsGPT\": \"От DocsGPT\",\n \"byMe\": \"Мои\",\n \"shared\": \"Поделились со мной\"\n },\n \"sections\": {\n \"template\": {\n \"title\": \"От DocsGPT\",\n \"description\": \"Агенты, предоставленные DocsGPT\",\n \"emptyState\": \"Шаблонные агенты не найдены.\"\n },\n \"user\": {\n \"title\": \"Мои\",\n \"description\": \"Агенты, созданные или опубликованные вами\",\n \"emptyState\": \"У вас пока нет созданных агентов.\"\n },\n \"shared\": {\n \"title\": \"Поделились со мной\",\n \"description\": \"Агенты, импортированные по публичной ссылке\",\n \"emptyState\": \"Общие агенты не найдены.\"\n }\n },\n \"form\": {\n \"headings\": {\n \"new\": \"Новый Агент\",\n \"edit\": \"Редактировать Агента\",\n \"draft\": \"Новый Агент (Черновик)\"\n },\n \"buttons\": {\n \"publish\": \"Опубликовать\",\n \"save\": \"Сохранить\",\n \"saveDraft\": \"Сохранить Черновик\",\n \"cancel\": \"Отмена\",\n \"delete\": \"Удалить\",\n \"logs\": \"Логи\",\n \"accessDetails\": \"Детали Доступа\",\n \"add\": \"Добавить\"\n },\n \"sections\": {\n \"meta\": \"Мета\",\n \"source\": \"Источник\",\n \"prompt\": \"Промпт\",\n \"tools\": \"Инструменты\",\n \"agentType\": \"Тип агента\",\n \"models\": \"Модели\",\n \"advanced\": \"Расширенные\",\n \"preview\": \"Предпросмотр\"\n },\n \"placeholders\": {\n \"agentName\": \"Имя агента\",\n \"describeAgent\": \"Опишите вашего агента\",\n \"selectSources\": \"Выберите источники\",\n \"chunksPerQuery\": \"Фрагментов на запрос\",\n \"selectType\": \"Выберите тип\",\n \"selectTools\": \"Выберите инструменты\",\n \"selectModels\": \"Выберите модели для этого агента\",\n \"selectDefaultModel\": \"Выберите модель по умолчанию\",\n \"enterTokenLimit\": \"Введите лимит токенов\",\n \"enterRequestLimit\": \"Введите лимит запросов\"\n },\n \"sourcePopup\": {\n \"title\": \"Выберите Источники\",\n \"searchPlaceholder\": \"Поиск источников...\",\n \"noOptionsMessage\": \"Нет доступных источников\"\n },\n \"toolsPopup\": {\n \"title\": \"Выберите Инструменты\",\n \"searchPlaceholder\": \"Поиск инструментов...\",\n \"noOptionsMessage\": \"Нет доступных инструментов\"\n },\n \"modelsPopup\": {\n \"title\": \"Выберите Модели\",\n \"searchPlaceholder\": \"Поиск моделей...\",\n \"noOptionsMessage\": \"Нет доступных моделей\"\n },\n \"upload\": {\n \"clickToUpload\": \"Нажмите для загрузки\",\n \"dragAndDrop\": \" или перетащите\"\n },\n \"agentTypes\": {\n \"classic\": \"Классический\",\n \"react\": \"ReAct\"\n },\n \"labels\": {\n \"defaultModel\": \"Модель по умолчанию\"\n },\n \"advanced\": {\n \"jsonSchema\": \"Схема ответа JSON\",\n \"jsonSchemaDescription\": \"Определите схему JSON для применения структурированного формата вывода\",\n \"validJson\": \"Валидный JSON\",\n \"invalidJson\": \"Невалидный JSON - исправьте для сохранения\",\n \"tokenLimiting\": \"Лимит токенов\",\n \"tokenLimitingDescription\": \"Ограничить ежедневное общее количество токенов, которые может использовать этот агент\",\n \"requestLimiting\": \"Лимит запросов\",\n \"requestLimitingDescription\": \"Ограничить ежедневное общее количество запросов, которые можно сделать к этому агенту\"\n },\n \"preview\": {\n \"publishedPreview\": \"Опубликованные агенты можно просмотреть здесь\"\n },\n \"externalKb\": \"Внешняя БЗ\"\n },\n \"logs\": {\n \"title\": \"Логи Агента\",\n \"lastUsedAt\": \"Последнее использование\",\n \"noUsageHistory\": \"Нет истории использования\",\n \"tableHeader\": \"Логи конечной точки агента\"\n },\n \"shared\": {\n \"notFound\": \"Агент не найден. Убедитесь, что агент является общим.\"\n },\n \"preview\": {\n \"testMessage\": \"Протестируйте своего агента здесь. Опубликованные агенты можно использовать в разговорах.\"\n },\n \"deleteConfirmation\": \"Вы уверены, что хотите удалить этого агента?\",\n \"folders\": {\n \"newFolder\": \"Новая папка\",\n \"createFolder\": \"Создать папку\",\n \"folderName\": \"Имя папки\",\n \"rename\": \"Переименовать\",\n \"delete\": \"Удалить\",\n \"deleteConfirm\": \"Вы уверены, что хотите удалить эту папку? Агенты в папке будут перемещены.\",\n \"empty\": \"Эта папка пуста\",\n \"moveToFolder\": \"Переместить в папку\",\n \"moveTo\": \"Переместить\",\n \"move\": \"Переместить\",\n \"noFolder\": \"Без папки (корень)\",\n \"backToRoot\": \"Назад\",\n \"noSubfolders\": \"Нет подпапок\",\n \"noFolders\": \"Пока нет папок\"\n }\n },\n \"components\": {\n \"fileUpload\": {\n \"clickToUpload\": \"Click to upload or drag and drop\",\n \"dropFiles\": \"Drop the files here\",\n \"fileTypes\": \"PNG, JPG, JPEG up to\",\n \"sizeLimitUnit\": \"MB\",\n \"fileSizeError\": \"File exceeds {{size}}MB limit\"\n }\n },\n \"pageNotFound\": {\n \"title\": \"404\",\n \"message\": \"The page you are looking for does not exist.\",\n \"goHome\": \"Go Back Home\"\n },\n \"filePicker\": {\n \"searchPlaceholder\": \"Поиск файлов и папок...\",\n \"itemsSelected\": \"{{count}} выбрано\",\n \"name\": \"Имя\",\n \"lastModified\": \"Последнее изменение\",\n \"size\": \"Размер\",\n \"myFiles\": \"Мои файлы\",\n \"sharedWithMe\": \"Доступные мне\",\n \"loadingMore\": \"Загрузка файлов...\"\n },\n \"actionButtons\": {\n \"openNewChat\": \"Открыть новый чат\",\n \"share\": \"Поделиться\"\n },\n \"mermaid\": {\n \"downloadOptions\": \"Параметры загрузки\",\n \"viewCode\": \"Просмотр кода\",\n \"decreaseZoom\": \"Уменьшить масштаб\",\n \"resetZoom\": \"Сбросить масштаб\",\n \"increaseZoom\": \"Увеличить масштаб\"\n },\n \"navigation\": {\n \"agents\": \"Агенты\"\n },\n \"notification\": {\n \"ariaLabel\": \"Уведомление\",\n \"closeAriaLabel\": \"Закрыть уведомление\"\n },\n \"prompts\": {\n \"textAriaLabel\": \"Текст подсказки\"\n }\n}\n" }, { "path": "frontend/src/locale/zh-TW.json", "content": "{\n \"language\": \"繁體中文（臺灣）\",\n \"chat\": \"對話\",\n \"chats\": \"對話\",\n \"newChat\": \"新對話\",\n \"inputPlaceholder\": \"DocsGPT 如何幫助您？\",\n \"tagline\": \"DocsGPT 使用生成式 AI，請使用原始資料來源審閱重要資訊。\",\n \"sourceDocs\": \"原始文件\",\n \"none\": \"無\",\n \"cancel\": \"取消\",\n \"help\": \"幫助\",\n \"emailUs\": \"給我們發電郵\",\n \"documentation\": \"文件\",\n \"manageAgents\": \"管理代理\",\n \"demo\": [\n {\n \"header\": \"了解 DocsGPT\",\n \"query\": \"什麼是 DocsGPT？\"\n },\n {\n \"header\": \"摘要文件\",\n \"query\": \"摘要目前的內容\"\n },\n {\n \"header\": \"撰寫程式碼\",\n \"query\": \"為 /api/answer 撰寫 API 請求程式碼\"\n },\n {\n \"header\": \"學習輔助\",\n \"query\": \"為此內容撰寫可能的問題\"\n }\n ],\n \"settings\": {\n \"label\": \"設定\",\n \"general\": {\n \"label\": \"一般\",\n \"selectTheme\": \"選擇主題\",\n \"light\": \"淺色\",\n \"dark\": \"深色\",\n \"selectLanguage\": \"選擇語言\",\n \"chunks\": \"每次查詢處理的區塊數\",\n \"prompt\": \"使用中的提示\",\n \"deleteAllLabel\": \"刪除所有對話\",\n \"deleteAllBtn\": \"全部刪除\",\n \"addNew\": \"新增\",\n \"convHistory\": \"對話歷史記錄\",\n \"none\": \"無\",\n \"low\": \"低\",\n \"medium\": \"中\",\n \"high\": \"高\",\n \"unlimited\": \"無限制\",\n \"default\": \"預設\",\n \"add\": \"添加\"\n },\n \"sources\": {\n \"title\": \"在這裡您可以管理所有可用的來源檔案以及您上傳的檔案。\",\n \"label\": \"來源\",\n \"name\": \"來源名稱\",\n \"date\": \"向量日期\",\n \"type\": \"類型\",\n \"tokenUsage\": \"Token 使用量\",\n \"noData\": \"沒有現有的來源\",\n \"searchPlaceholder\": \"搜尋...\",\n \"addNew\": \"新增文件\",\n \"addSource\": \"新增來源\",\n \"addChunk\": \"新增區塊\",\n \"preLoaded\": \"預載入\",\n \"private\": \"私人\",\n \"sync\": \"同步\",\n \"syncNow\": \"立即同步\",\n \"syncing\": \"同步中...\",\n \"syncConfirmation\": \"您確定要同步 \\\"{{sourceName}}\\\" 嗎？這將使用您的雲端儲存更新內容，並可能覆蓋您對個別文本塊所做的任何編輯。\",\n \"syncFrequency\": {\n \"never\": \"從不\",\n \"daily\": \"每天\",\n \"weekly\": \"每週\",\n \"monthly\": \"每月\"\n },\n \"actions\": \"操作\",\n \"view\": \"查看\",\n \"deleteWarning\": \"您確定要刪除 \\\"{{name}}\\\" 嗎？\",\n \"confirmDelete\": \"您確定要刪除此檔案嗎？此操作無法復原。\",\n \"backToAll\": \"返回所有來源\",\n \"chunks\": \"文本塊\",\n \"noChunks\": \"未找到文本塊\",\n \"noChunksAlt\": \"未找到文本塊\",\n \"goToSources\": \"前往來源\",\n \"uploadNew\": \"上傳新文件\",\n \"searchFiles\": \"搜尋檔案...\",\n \"noResults\": \"未找到結果\",\n \"fileName\": \"名稱\",\n \"tokens\": \"Token\",\n \"size\": \"大小\",\n \"fileAlt\": \"檔案\",\n \"folderAlt\": \"資料夾\",\n \"parentFolderAlt\": \"上層資料夾\",\n \"menuAlt\": \"選單\",\n \"tokensUnit\": \"Token\",\n \"editAlt\": \"編輯\",\n \"uploading\": \"正在上傳…\",\n \"deleting\": \"正在刪除…\",\n \"queued\": \"已排隊：{{count}}\",\n \"addFile\": \"新增檔案\",\n \"uploadingFilesTitle\": \"正在上傳檔案...\",\n \"deletingTitle\": \"正在刪除...\",\n \"deleteDirectoryWarning\": \"您確定要刪除目錄 \\\"{{name}}\\\" 及其所有內容嗎？此操作無法復原。\",\n \"searchAlt\": \"搜尋\"\n },\n \"apiKeys\": {\n \"label\": \"聊天機器人\",\n \"name\": \"名稱\",\n \"key\": \"API 金鑰\",\n \"sourceDoc\": \"來源文件\",\n \"createNew\": \"建立新的\",\n \"noData\": \"沒有現有的聊天機器人\",\n \"deleteConfirmation\": \"您確定要刪除 API 金鑰 '{{name}}' 嗎？\",\n \"description\": \"在這裡，您可以創建和管理您的聊天機器人。聊天機器人可以作為小部件部署到網站上，或在您的應用程序中使用。\"\n },\n \"analytics\": {\n \"label\": \"分析\",\n \"filterByChatbot\": \"按聊天機器人篩選\",\n \"selectChatbot\": \"選擇聊天機器人\",\n \"filterOptions\": {\n \"hour\": \"小時\",\n \"last24Hours\": \"24 小時\",\n \"last7Days\": \"7 天\",\n \"last15Days\": \"15 天\",\n \"last30Days\": \"30 天\"\n },\n \"messages\": \"訊息\",\n \"tokenUsage\": \"Token 使用量\",\n \"userFeedback\": \"使用者反饋\",\n \"filterPlaceholder\": \"篩選\",\n \"none\": \"無\",\n \"positiveFeedback\": \"正向回饋\",\n \"negativeFeedback\": \"負向回饋\"\n },\n \"logs\": {\n \"label\": \"日誌\",\n \"filterByChatbot\": \"按聊天機器人篩選\",\n \"selectChatbot\": \"選擇聊天機器人\",\n \"none\": \"無\",\n \"tableHeader\": \"API 生成 / 聊天機器人會話\"\n },\n \"tools\": {\n \"label\": \"工具\",\n \"searchPlaceholder\": \"搜尋工具...\",\n \"addTool\": \"新增工具\",\n \"noToolsFound\": \"找不到工具\",\n \"selectToolSetup\": \"選擇要設定的工具\",\n \"settingsIconAlt\": \"設定圖示\",\n \"configureToolAria\": \"設定 {{toolName}}\",\n \"toggleToolAria\": \"切換 {{toolName}}\",\n \"manageTools\": \"前往工具\",\n \"edit\": \"編輯\",\n \"delete\": \"刪除\",\n \"deleteWarning\": \"您確定要刪除工具 \\\"{{toolName}}\\\" 嗎？\",\n \"unsavedChanges\": \"您有未儲存的變更，如果不儲存就離開將會遺失。\",\n \"leaveWithoutSaving\": \"不儲存離開\",\n \"saveAndLeave\": \"儲存並離開\",\n \"customName\": \"自訂名稱\",\n \"customNamePlaceholder\": \"輸入自訂名稱（選填）\",\n \"authentication\": \"認證\",\n \"actions\": \"操作\",\n \"addAction\": \"新增操作\",\n \"importSpec\": \"匯入規格\",\n \"searchActions\": \"搜尋操作...\",\n \"noActionsMatch\": \"沒有符合搜尋的操作\",\n \"actionAlreadyExists\": \"已存在同名操作\",\n \"noActionsFound\": \"找不到操作\",\n \"url\": \"URL\",\n \"urlPlaceholder\": \"輸入url\",\n \"method\": \"方法\",\n \"description\": \"描述\",\n \"descriptionPlaceholder\": \"輸入描述\",\n \"bodyContentType\": \"主體內容類型\",\n \"headers\": \"標頭\",\n \"queryParameters\": \"查詢參數\",\n \"body\": \"主體\",\n \"deleteActionWarning\": \"您確定要刪除操作 \\\"{{name}}\\\" 嗎？\",\n \"backToAllTools\": \"返回所有工具\",\n \"save\": \"儲存\",\n \"fieldName\": \"欄位名稱\",\n \"fieldType\": \"欄位類型\",\n \"filledByLLM\": \"由LLM填入\",\n \"fieldDescription\": \"欄位描述\",\n \"value\": \"值\",\n \"addProperty\": \"新增屬性\",\n \"propertyName\": \"新屬性鍵\",\n \"add\": \"新增\",\n \"cancel\": \"取消\",\n \"addNew\": \"新增\",\n \"name\": \"名稱\",\n \"type\": \"類型\",\n \"mcp\": {\n \"addServer\": \"Add MCP Server\",\n \"editServer\": \"Edit Server\",\n \"serverName\": \"Server Name\",\n \"serverUrl\": \"Server URL\",\n \"headerName\": \"Header Name\",\n \"timeout\": \"Timeout (seconds)\",\n \"testConnection\": \"Test Connection\",\n \"testing\": \"Testing\",\n \"saving\": \"Saving\",\n \"save\": \"Save\",\n \"cancel\": \"Cancel\",\n \"noAuth\": \"No Authentication\",\n \"oauthInProgress\": \"Waiting for OAuth completion...\",\n \"oauthCompleted\": \"OAuth completed successfully\",\n \"authType\": \"Authentication Type\",\n \"defaultServerName\": \"My MCP Server\",\n \"authTypes\": {\n \"none\": \"No Authentication\",\n \"apiKey\": \"API Key\",\n \"bearer\": \"Bearer Token\",\n \"oauth\": \"OAuth\",\n \"basic\": \"Basic Authentication\"\n },\n \"placeholders\": {\n \"serverUrl\": \"https://api.example.com\",\n \"apiKey\": \"Your secret API key\",\n \"bearerToken\": \"Your secret token\",\n \"username\": \"Your username\",\n \"password\": \"Your password\",\n \"oauthScopes\": \"OAuth scopes (comma separated)\"\n },\n \"errors\": {\n \"nameRequired\": \"Server name is required\",\n \"urlRequired\": \"Server URL is required\",\n \"invalidUrl\": \"Please enter a valid URL\",\n \"apiKeyRequired\": \"API key is required\",\n \"tokenRequired\": \"Bearer token is required\",\n \"usernameRequired\": \"Username is required\",\n \"passwordRequired\": \"Password is required\",\n \"testFailed\": \"Connection test failed\",\n \"saveFailed\": \"Failed to save MCP server\",\n \"oauthFailed\": \"OAuth process failed or was cancelled\",\n \"oauthTimeout\": \"OAuth process timed out, please try again\",\n \"timeoutRange\": \"Timeout must be between 1 and 300 seconds\"\n }\n }\n },\n \"scrollTabsLeft\": \"向左捲動標籤\",\n \"tabsAriaLabel\": \"設定標籤\",\n \"scrollTabsRight\": \"向右捲動標籤\"\n },\n \"modals\": {\n \"uploadDoc\": {\n \"label\": \"上傳新文件\",\n \"select\": \"選擇如何將文件上傳到 DocsGPT\",\n \"selectSource\": \"選擇新增來源的方式\",\n \"selectedFiles\": \"已選擇的檔案\",\n \"noFilesSelected\": \"未選擇檔案\",\n \"file\": \"從檔案\",\n \"remote\": \"遠端\",\n \"back\": \"返回\",\n \"wait\": \"請稍候...\",\n \"start\": \"開始對話\",\n \"name\": \"名稱\",\n \"choose\": \"選擇檔案\",\n \"info\": \"請上傳限制為25MB的.pdf、.txt、.rst、.csv、.xlsx、.docx、.md、.html、.epub、.json、.pptx、.zip檔案\",\n \"uploadedFiles\": \"已上傳檔案\",\n \"cancel\": \"取消\",\n \"train\": \"訓練\",\n \"link\": \"連結\",\n \"urlLink\": \"URL 連結\",\n \"repoUrl\": \"儲存庫 URL\",\n \"reddit\": {\n \"id\": \"客戶端ID\",\n \"secret\": \"客戶端密鑰\",\n \"agent\": \"使用者代理\",\n \"searchQueries\": \"搜尋查詢\",\n \"numberOfPosts\": \"貼文數量\",\n \"addQuery\": \"新增查詢\"\n },\n \"drag\": {\n \"title\": \"將附件拖放到此處\",\n \"description\": \"釋放以上傳您的附件\"\n },\n \"progress\": {\n \"upload\": \"正在上傳\",\n \"training\": \"正在上傳\",\n \"completed\": \"上傳完成\",\n \"failed\": \"上傳失敗\",\n \"wait\": \"這可能需要幾分鐘\",\n \"preparing\": \"準備上傳\",\n \"tokenLimit\": \"超出令牌限制，請考慮上傳較小的文檔\",\n \"expandDetails\": \"展開上傳詳情\",\n \"collapseDetails\": \"摺疊上傳詳情\",\n \"dismiss\": \"關閉上傳通知\",\n \"uploadProgress\": \"上傳進度 {{progress}}%\",\n \"clear\": \"清除\"\n },\n \"showAdvanced\": \"顯示進階選項\",\n \"hideAdvanced\": \"隱藏進階選項\",\n \"ingestors\": {\n \"local_file\": {\n \"label\": \"上傳檔案\",\n \"heading\": \"上傳新文檔\"\n },\n \"crawler\": {\n \"label\": \"爬蟲\",\n \"heading\": \"使用網路爬蟲新增內容\"\n },\n \"url\": {\n \"label\": \"連結\",\n \"heading\": \"從URL新增內容\"\n },\n \"github\": {\n \"label\": \"GitHub\",\n \"heading\": \"從GitHub新增內容\"\n },\n \"reddit\": {\n \"label\": \"Reddit\",\n \"heading\": \"從Reddit新增內容\"\n },\n \"google_drive\": {\n \"label\": \"Google Drive\",\n \"heading\": \"從Google Drive上傳\"\n },\n \"s3\": {\n \"label\": \"Amazon S3\",\n \"heading\": \"從Amazon S3新增內容\"\n },\n \"share_point\": {\n \"label\": \"SharePoint\",\n \"heading\": \"從SharePoint上傳\"\n }\n },\n \"connectors\": {\n \"auth\": {\n \"connectedUser\": \"已連接使用者\",\n \"authFailed\": \"驗證失敗\",\n \"authUrlFailed\": \"取得授權URL失敗\",\n \"popupBlocked\": \"無法開啟驗證視窗。請允許彈出視窗。\",\n \"authCancelled\": \"驗證已取消\",\n \"connectedAs\": \"已連接為 {{email}}\",\n \"disconnect\": \"中斷連接\"\n },\n \"googleDrive\": {\n \"connect\": \"連接到 Google Drive\",\n \"sessionExpired\": \"工作階段已過期。請重新連接到 Google Drive。\",\n \"sessionExpiredGeneric\": \"工作階段已過期。請重新連接您的帳戶。\",\n \"validateFailed\": \"驗證工作階段失敗。請重新連接。\",\n \"noSession\": \"未找到有效工作階段。請重新連接到 Google Drive。\",\n \"noAccessToken\": \"存取權杖不可用。請重新連接到 Google Drive。\",\n \"pickerFailed\": \"無法開啟檔案選擇器。請重試。\",\n \"selectedFiles\": \"已選擇的檔案\",\n \"selectFiles\": \"選擇檔案\",\n \"loading\": \"載入中...\",\n \"noFilesSelected\": \"未選擇檔案或資料夾\",\n \"folders\": \"資料夾\",\n \"files\": \"檔案\",\n \"remove\": \"移除\",\n \"folderAlt\": \"資料夾\",\n \"fileAlt\": \"檔案\"\n },\n \"sharePoint\": {\n \"connect\": \"連接到 SharePoint\",\n \"sessionExpired\": \"工作階段已過期。請重新連接到 SharePoint。\",\n \"sessionExpiredGeneric\": \"工作階段已過期。請重新連接您的帳戶。\",\n \"validateFailed\": \"驗證工作階段失敗。請重新連接。\",\n \"noSession\": \"未找到有效工作階段。請重新連接到 SharePoint。\",\n \"noAccessToken\": \"存取權杖不可用。請重新連接到 SharePoint。\",\n \"pickerFailed\": \"無法開啟檔案選擇器。請重試。\",\n \"selectedFiles\": \"已選擇的檔案\",\n \"selectFiles\": \"選擇檔案\",\n \"loading\": \"載入中...\",\n \"noFilesSelected\": \"未選擇檔案或資料夾\",\n \"folders\": \"資料夾\",\n \"files\": \"檔案\",\n \"remove\": \"移除\",\n \"folderAlt\": \"資料夾\",\n \"fileAlt\": \"檔案\"\n }\n }\n },\n \"createAPIKey\": {\n \"label\": \"建立新的 API 金鑰\",\n \"apiKeyName\": \"API 金鑰名稱\",\n \"chunks\": \"每次查詢處理的區塊數\",\n \"prompt\": \"選擇使用中的提示\",\n \"sourceDoc\": \"來源文件\",\n \"create\": \"建立\"\n },\n \"saveKey\": {\n \"note\": \"請儲存您的金鑰\",\n \"disclaimer\": \"這是唯一一次顯示您的金鑰。\",\n \"copy\": \"複製\",\n \"copied\": \"已複製\",\n \"confirm\": \"我已儲存金鑰\",\n \"apiKeyLabel\": \"API Key\"\n },\n \"deleteConv\": {\n \"confirm\": \"您確定要刪除所有對話嗎？\",\n \"delete\": \"刪除\"\n },\n \"shareConv\": {\n \"label\": \"建立公開頁面以分享\",\n \"note\": \"來源文件、個人資訊和後續對話將保持私密\",\n \"create\": \"建立\",\n \"option\": \"允許使用者進行更多查詢\"\n },\n \"configTool\": {\n \"title\": \"工具設定\",\n \"type\": \"類型\",\n \"apiKeyLabel\": \"API 金鑰 / OAuth\",\n \"apiKeyPlaceholder\": \"輸入 API 金鑰 / OAuth\",\n \"addButton\": \"新增工具\",\n \"closeButton\": \"關閉\",\n \"customNamePlaceholder\": \"Enter custom name (optional)\"\n },\n \"prompts\": {\n \"addPrompt\": \"新增提示\",\n \"addDescription\": \"新增自定義提示並儲存到 DocsGPT\",\n \"editPrompt\": \"編輯提示\",\n \"editDescription\": \"編輯自定義提示並儲存到 DocsGPT\",\n \"promptName\": \"提示名稱\",\n \"promptText\": \"提示文字\",\n \"save\": \"儲存\",\n \"cancel\": \"取消\",\n \"nameExists\": \"名稱已存在\",\n \"deleteConfirmation\": \"您確定要刪除提示「{{name}}」嗎？\",\n \"placeholderText\": \"在此輸入提示內容...\",\n \"addExamplePlaceholder\": \"請總結此文本：\",\n \"variablesLabel\": \"變數\",\n \"variablesSubtext\": \"點擊以插入提示中\",\n \"variablesDescription\": \"點擊以插入到提示中\",\n \"systemVariables\": \"點擊以插入提示中\",\n \"toolVariables\": \"工具變數\",\n \"systemVariablesDropdownLabel\": \"系統變數\",\n \"systemVariableOptions\": {\n \"sourceContent\": \"來源內容\",\n \"sourceSummaries\": \"內容別名（向後相容）\",\n \"sourceDocuments\": \"文件物件列表\",\n \"sourceCount\": \"擷取的文件數量\",\n \"systemDate\": \"目前日期 (YYYY-MM-DD)\",\n \"systemTime\": \"目前時間 (HH:MM:SS)\",\n \"systemTimestamp\": \"ISO 8601 時間戳記\",\n \"systemRequestId\": \"唯一請求識別碼\",\n \"systemUserId\": \"目前使用者 ID\"\n },\n \"learnAboutPrompts\": \"了解提示 →\",\n \"publicPromptEditDisabled\": \"公共提示無法編輯\",\n \"promptTypePublic\": \"公共\",\n \"promptTypePrivate\": \"私人\"\n },\n \"chunk\": {\n \"add\": \"新增區塊\",\n \"edit\": \"編輯\",\n \"title\": \"標題\",\n \"enterTitle\": \"輸入標題\",\n \"bodyText\": \"內文\",\n \"promptText\": \"提示文字\",\n \"save\": \"儲存\",\n \"close\": \"關閉\",\n \"cancel\": \"取消\",\n \"delete\": \"刪除\",\n \"deleteConfirmation\": \"您確定要刪除此區塊嗎？\"\n },\n \"addAction\": {\n \"title\": \"New Action\",\n \"actionNamePlaceholder\": \"Action Name\",\n \"invalidFormat\": \"Invalid function name format. Use only letters, numbers, underscores, and hyphens.\",\n \"formatHelp\": \"Use only letters, numbers, underscores, and hyphens (e.g., `get_data`, `send_report`, etc.)\",\n \"addButton\": \"Add\"\n },\n \"agentDetails\": {\n \"title\": \"Access Details\",\n \"publicLink\": \"Public Link\",\n \"apiKey\": \"API Key\",\n \"webhookUrl\": \"Webhook URL\",\n \"generate\": \"Generate\",\n \"test\": \"Test\",\n \"learnMore\": \"Learn more\"\n },\n \"importSpec\": {\n \"title\": \"匯入 API 規格\",\n \"description\": \"上傳 OpenAPI 3.x 或 Swagger 2.0 規格檔以自動產生操作。\",\n \"dropzoneText\": \"點擊上傳或拖放\",\n \"supportedFormats\": \"JSON 或 YAML 格式\",\n \"invalidFileType\": \"無效的檔案類型。請上傳 JSON 或 YAML 檔案。\",\n \"parseError\": \"解析規格失敗。請檢查檔案格式。\",\n \"version\": \"版本\",\n \"baseUrl\": \"基礎 URL\",\n \"actionsFound\": \"找到 {{count}} 個操作\",\n \"selectAll\": \"全選\",\n \"deselectAll\": \"取消全選\",\n \"cancel\": \"取消\",\n \"parse\": \"解析\",\n \"import\": \"匯入 ({{count}})\"\n }\n },\n \"sharedConv\": {\n \"subtitle\": \"使用以下工具建立\",\n \"button\": \"開始使用 DocsGPT\",\n \"meta\": \"DocsGPT 使用生成式 AI，請使用原始資料來源審閱重要資訊。\"\n },\n \"convTile\": {\n \"share\": \"分享\",\n \"delete\": \"刪除\",\n \"rename\": \"重新命名\",\n \"deleteWarning\": \"您確定要刪除這個對話嗎？\"\n },\n \"pagination\": {\n \"rowsPerPage\": \"每頁行數\",\n \"pageOf\": \"第 {{currentPage}} 頁，共 {{totalPages}} 頁\",\n \"firstPage\": \"第一頁\",\n \"previousPage\": \"上一頁\",\n \"nextPage\": \"下一頁\",\n \"lastPage\": \"最後一頁\"\n },\n \"conversation\": {\n \"copy\": \"複製\",\n \"copied\": \"已複製\",\n \"speak\": \"朗讀\",\n \"answer\": \"回答\",\n \"edit\": {\n \"update\": \"更新\",\n \"cancel\": \"取消\",\n \"placeholder\": \"輸入更新的查詢...\"\n },\n \"sources\": {\n \"title\": \"來源\",\n \"text\": \"來源文字\",\n \"link\": \"來源連結\",\n \"view_more\": \"查看更多 {{count}} 個來源\",\n \"noSourcesAvailable\": \"沒有可用的來源\"\n },\n \"attachments\": {\n \"attach\": \"附件\",\n \"remove\": \"刪除附件\"\n },\n \"retry\": \"重試\",\n \"reasoning\": \"推理\"\n },\n \"agents\": {\n \"title\": \"代理\",\n \"description\": \"探索並創建結合指令、額外知識和任意技能組合的DocsGPT自訂版本\",\n \"newAgent\": \"新建代理\",\n \"backToAll\": \"返回所有代理\",\n \"searchPlaceholder\": \"搜尋...\",\n \"noSearchResults\": \"未找到代理\",\n \"tryDifferentSearch\": \"請嘗試不同的搜尋詞\",\n \"filters\": {\n \"all\": \"全部\",\n \"byDocsGPT\": \"由DocsGPT提供\",\n \"byMe\": \"我的\",\n \"shared\": \"與我共享\"\n },\n \"sections\": {\n \"template\": {\n \"title\": \"由DocsGPT提供\",\n \"description\": \"DocsGPT提供的代理\",\n \"emptyState\": \"未找到範本代理。\"\n },\n \"user\": {\n \"title\": \"我的代理\",\n \"description\": \"您創建或發佈的代理\",\n \"emptyState\": \"您還沒有創建任何代理。\"\n },\n \"shared\": {\n \"title\": \"與我共享\",\n \"description\": \"透過公共連結匯入的代理\",\n \"emptyState\": \"未找到共享代理。\"\n }\n },\n \"form\": {\n \"headings\": {\n \"new\": \"新建代理\",\n \"edit\": \"編輯代理\",\n \"draft\": \"新建代理（草稿）\"\n },\n \"buttons\": {\n \"publish\": \"發佈\",\n \"save\": \"儲存\",\n \"saveDraft\": \"儲存草稿\",\n \"cancel\": \"取消\",\n \"delete\": \"刪除\",\n \"logs\": \"日誌\",\n \"accessDetails\": \"存取詳情\",\n \"add\": \"新增\"\n },\n \"sections\": {\n \"meta\": \"中繼資料\",\n \"source\": \"來源\",\n \"prompt\": \"提示詞\",\n \"tools\": \"工具\",\n \"agentType\": \"代理類型\",\n \"models\": \"模型\",\n \"advanced\": \"進階\",\n \"preview\": \"預覽\"\n },\n \"placeholders\": {\n \"agentName\": \"代理名稱\",\n \"describeAgent\": \"描述您的代理\",\n \"selectSources\": \"選擇來源\",\n \"chunksPerQuery\": \"每次查詢的區塊數\",\n \"selectType\": \"選擇類型\",\n \"selectTools\": \"選擇工具\",\n \"selectModels\": \"為此代理選擇模型\",\n \"selectDefaultModel\": \"選擇預設模型\",\n \"enterTokenLimit\": \"輸入權杖限制\",\n \"enterRequestLimit\": \"輸入請求限制\"\n },\n \"sourcePopup\": {\n \"title\": \"選擇來源\",\n \"searchPlaceholder\": \"搜尋來源...\",\n \"noOptionsMessage\": \"沒有可用的來源\"\n },\n \"toolsPopup\": {\n \"title\": \"選擇工具\",\n \"searchPlaceholder\": \"搜尋工具...\",\n \"noOptionsMessage\": \"沒有可用的工具\"\n },\n \"modelsPopup\": {\n \"title\": \"選擇模型\",\n \"searchPlaceholder\": \"搜尋模型...\",\n \"noOptionsMessage\": \"沒有可用的模型\"\n },\n \"upload\": {\n \"clickToUpload\": \"點擊上傳\",\n \"dragAndDrop\": \" 或拖放\"\n },\n \"agentTypes\": {\n \"classic\": \"經典\",\n \"react\": \"ReAct\"\n },\n \"labels\": {\n \"defaultModel\": \"預設模型\"\n },\n \"advanced\": {\n \"jsonSchema\": \"JSON回應架構\",\n \"jsonSchemaDescription\": \"定義JSON架構以強制執行結構化輸出格式\",\n \"validJson\": \"有效的JSON\",\n \"invalidJson\": \"無效的JSON - 修復後才能儲存\",\n \"tokenLimiting\": \"權杖限制\",\n \"tokenLimitingDescription\": \"限制此代理每天可使用的總權杖數\",\n \"requestLimiting\": \"請求限制\",\n \"requestLimitingDescription\": \"限制每天可向此代理發出的總請求數\"\n },\n \"preview\": {\n \"publishedPreview\": \"已發佈的代理可以在此處預覽\"\n },\n \"externalKb\": \"外部知識庫\"\n },\n \"logs\": {\n \"title\": \"代理日誌\",\n \"lastUsedAt\": \"最後使用時間\",\n \"noUsageHistory\": \"無使用歷史\",\n \"tableHeader\": \"代理端點日誌\"\n },\n \"shared\": {\n \"notFound\": \"未找到代理。請確保代理已共享。\"\n },\n \"preview\": {\n \"testMessage\": \"在此測試您的代理。已發佈的代理可以在對話中使用。\"\n },\n \"deleteConfirmation\": \"您確定要刪除此代理嗎？\",\n \"folders\": {\n \"newFolder\": \"新建資料夾\",\n \"createFolder\": \"建立資料夾\",\n \"folderName\": \"資料夾名稱\",\n \"rename\": \"重新命名\",\n \"delete\": \"刪除\",\n \"deleteConfirm\": \"確定要刪除此資料夾嗎？資料夾中的代理將被移出。\",\n \"empty\": \"此資料夾為空\",\n \"moveToFolder\": \"移動到資料夾\",\n \"moveTo\": \"移動\",\n \"move\": \"移動\",\n \"noFolder\": \"無資料夾 (根目錄)\",\n \"backToRoot\": \"返回\",\n \"noSubfolders\": \"沒有子資料夾\",\n \"noFolders\": \"暫無資料夾\"\n }\n },\n \"components\": {\n \"fileUpload\": {\n \"clickToUpload\": \"Click to upload or drag and drop\",\n \"dropFiles\": \"Drop the files here\",\n \"fileTypes\": \"PNG, JPG, JPEG up to\",\n \"sizeLimitUnit\": \"MB\",\n \"fileSizeError\": \"File exceeds {{size}}MB limit\"\n }\n },\n \"pageNotFound\": {\n \"title\": \"404\",\n \"message\": \"The page you are looking for does not exist.\",\n \"goHome\": \"Go Back Home\"\n },\n \"filePicker\": {\n \"searchPlaceholder\": \"搜尋檔案和資料夾...\",\n \"itemsSelected\": \"已選擇 {{count}} 項\",\n \"name\": \"名稱\",\n \"lastModified\": \"最後修改\",\n \"size\": \"大小\",\n \"myFiles\": \"我的檔案\",\n \"sharedWithMe\": \"與我共用\",\n \"loadingMore\": \"載入更多檔案...\"\n },\n \"actionButtons\": {\n \"openNewChat\": \"開啟新聊天\",\n \"share\": \"分享\"\n },\n \"mermaid\": {\n \"downloadOptions\": \"下載選項\",\n \"viewCode\": \"查看程式碼\",\n \"decreaseZoom\": \"縮小\",\n \"resetZoom\": \"重設縮放\",\n \"increaseZoom\": \"放大\"\n },\n \"navigation\": {\n \"agents\": \"代理\"\n },\n \"notification\": {\n \"ariaLabel\": \"通知\",\n \"closeAriaLabel\": \"關閉通知\"\n },\n \"prompts\": {\n \"textAriaLabel\": \"提示文字\"\n }\n}\n" }, { "path": "frontend/src/locale/zh.json", "content": "{\n \"language\": \"普通话\",\n \"chat\": \"聊天\",\n \"chats\": \"聊天\",\n \"newChat\": \"新聊天\",\n \"inputPlaceholder\": \"DocsGPT 如何帮助您？\",\n \"tagline\": \"DocsGPT 使用 GenAI, 请使用来源审核关键信息.\",\n \"sourceDocs\": \"源\",\n \"none\": \"无\",\n \"cancel\": \"取消\",\n \"help\": \"帮助\",\n \"emailUs\": \"给我们发邮件\",\n \"documentation\": \"文档\",\n \"manageAgents\": \"管理代理\",\n \"demo\": [\n {\n \"header\": \"了解 DocsGPT\",\n \"query\": \"DocsGPT 是什么\"\n },\n {\n \"header\": \"总结文档\",\n \"query\": \"总结当前情况\"\n },\n {\n \"header\": \"编写代码\",\n \"query\": \"为 /api/answer API 请求编写代码\"\n },\n {\n \"header\": \"学习帮助\",\n \"query\": \"为背景写出潜在问题\"\n }\n ],\n \"settings\": {\n \"label\": \"设置\",\n \"general\": {\n \"label\": \"一般\",\n \"selectTheme\": \"选择主题\",\n \"light\": \"浅色\",\n \"dark\": \"暗色\",\n \"selectLanguage\": \"选择语言\",\n \"chunks\": \"每个查询处理的块\",\n \"prompt\": \"提示\",\n \"deleteAllLabel\": \"删除所有对话\",\n \"deleteAllBtn\": \"删除所有\",\n \"addNew\": \"添加新的\",\n \"convHistory\": \"对话历史\",\n \"none\": \"无\",\n \"low\": \"低\",\n \"medium\": \"中\",\n \"high\": \"高\",\n \"unlimited\": \"无限\",\n \"default\": \"默认\",\n \"add\": \"添加\"\n },\n \"sources\": {\n \"title\": \"在这里您可以管理所有可用的源文件以及您上传的文件。\",\n \"label\": \"来源\",\n \"name\": \"来源名称\",\n \"date\": \"向量日期\",\n \"type\": \"类型\",\n \"tokenUsage\": \"令牌使用\",\n \"noData\": \"没有现有的来源\",\n \"searchPlaceholder\": \"搜索...\",\n \"addNew\": \"添加新文档\",\n \"addSource\": \"添加来源\",\n \"addChunk\": \"添加块\",\n \"preLoaded\": \"预加载\",\n \"private\": \"私有\",\n \"sync\": \"同步\",\n \"syncNow\": \"立即同步\",\n \"syncing\": \"同步中...\",\n \"syncConfirmation\": \"您确定要同步 \\\"{{sourceName}}\\\" 吗？这将使用您的云存储更新内容，并可能覆盖您对单个文本块所做的任何编辑。\",\n \"syncFrequency\": {\n \"never\": \"从不\",\n \"daily\": \"每天\",\n \"weekly\": \"每周\",\n \"monthly\": \"每月\"\n },\n \"actions\": \"操作\",\n \"view\": \"查看\",\n \"deleteWarning\": \"您确定要删除 \\\"{{name}}\\\" 吗？\",\n \"confirmDelete\": \"您确定要删除此文件吗？此操作无法撤销。\",\n \"backToAll\": \"返回所有来源\",\n \"chunks\": \"文本块\",\n \"noChunks\": \"未找到文本块\",\n \"noChunksAlt\": \"未找到文本块\",\n \"goToSources\": \"前往来源\",\n \"uploadNew\": \"上传新文档\",\n \"searchFiles\": \"搜索文件...\",\n \"noResults\": \"未找到结果\",\n \"fileName\": \"名称\",\n \"tokens\": \"令牌\",\n \"size\": \"大小\",\n \"fileAlt\": \"文件\",\n \"folderAlt\": \"文件夹\",\n \"parentFolderAlt\": \"父文件夹\",\n \"menuAlt\": \"菜单\",\n \"tokensUnit\": \"令牌\",\n \"editAlt\": \"编辑\",\n \"uploading\": \"正在上传…\",\n \"deleting\": \"正在删除…\",\n \"queued\": \"已排队：{{count}}\",\n \"addFile\": \"添加文件\",\n \"uploadingFilesTitle\": \"正在上传文件...\",\n \"deletingTitle\": \"正在删除...\",\n \"deleteDirectoryWarning\": \"确定要删除目录 \\\"{{name}}\\\" 及其所有内容吗？此操作无法撤销。\",\n \"searchAlt\": \"搜索\"\n },\n \"apiKeys\": {\n \"label\": \"聊天机器人\",\n \"name\": \"名称\",\n \"key\": \"API 密钥\",\n \"sourceDoc\": \"源文档\",\n \"createNew\": \"创建新的\",\n \"noData\": \"没有现有的聊天机器人\",\n \"deleteConfirmation\": \"您确定要删除 API 密钥 '{{name}}' 吗？\",\n \"description\": \"在这里，您可以创建和管理您的聊天机器人。聊天机器人可以作为小部件部署到网站上，或在您的应用程序中使用。\"\n },\n \"analytics\": {\n \"label\": \"分析\",\n \"filterByChatbot\": \"按聊天机器人筛选\",\n \"selectChatbot\": \"选择聊天机器人\",\n \"filterOptions\": {\n \"hour\": \"小时\",\n \"last24Hours\": \"24 小时\",\n \"last7Days\": \"7 天\",\n \"last15Days\": \"15 天\",\n \"last30Days\": \"30 天\"\n },\n \"messages\": \"消息\",\n \"tokenUsage\": \"令牌使用\",\n \"userFeedback\": \"用户反馈\",\n \"filterPlaceholder\": \"筛选\",\n \"none\": \"无\",\n \"positiveFeedback\": \"正向反馈\",\n \"negativeFeedback\": \"负向反馈\"\n },\n \"logs\": {\n \"label\": \"日志\",\n \"filterByChatbot\": \"按聊天机器人筛选\",\n \"selectChatbot\": \"选择聊天机器人\",\n \"none\": \"无\",\n \"tableHeader\": \"API 生成 / 聊天机器人会话\"\n },\n \"tools\": {\n \"label\": \"工具\",\n \"searchPlaceholder\": \"搜索工具...\",\n \"addTool\": \"添加工具\",\n \"noToolsFound\": \"未找到工具\",\n \"selectToolSetup\": \"选择要设置的工具\",\n \"settingsIconAlt\": \"设置图标\",\n \"configureToolAria\": \"配置 {{toolName}}\",\n \"toggleToolAria\": \"切换 {{toolName}}\",\n \"manageTools\": \"前往工具\",\n \"edit\": \"编辑\",\n \"delete\": \"删除\",\n \"deleteWarning\": \"您确定要删除工具 \\\"{{toolName}}\\\" 吗？\",\n \"unsavedChanges\": \"您有未保存的更改，如果不保存就离开将会丢失。\",\n \"leaveWithoutSaving\": \"不保存离开\",\n \"saveAndLeave\": \"保存并离开\",\n \"customName\": \"自定义名称\",\n \"customNamePlaceholder\": \"输入自定义名称（可选）\",\n \"authentication\": \"认证\",\n \"actions\": \"操作\",\n \"addAction\": \"添加操作\",\n \"importSpec\": \"导入规范\",\n \"searchActions\": \"搜索操作...\",\n \"noActionsMatch\": \"没有与搜索匹配的操作\",\n \"actionAlreadyExists\": \"已存在同名操作\",\n \"noActionsFound\": \"未找到操作\",\n \"url\": \"URL\",\n \"urlPlaceholder\": \"输入url\",\n \"method\": \"方法\",\n \"description\": \"描述\",\n \"descriptionPlaceholder\": \"输入描述\",\n \"bodyContentType\": \"请求体内容类型\",\n \"headers\": \"请求头\",\n \"queryParameters\": \"查询参数\",\n \"body\": \"请求体\",\n \"deleteActionWarning\": \"您确定要删除操作 \\\"{{name}}\\\" 吗？\",\n \"backToAllTools\": \"返回所有工具\",\n \"save\": \"保存\",\n \"fieldName\": \"字段名称\",\n \"fieldType\": \"字段类型\",\n \"filledByLLM\": \"由LLM填充\",\n \"fieldDescription\": \"字段描述\",\n \"value\": \"值\",\n \"addProperty\": \"添加属性\",\n \"propertyName\": \"新属性键\",\n \"add\": \"添加\",\n \"cancel\": \"取消\",\n \"addNew\": \"添加新的\",\n \"name\": \"名称\",\n \"type\": \"类型\",\n \"mcp\": {\n \"addServer\": \"Add MCP Server\",\n \"editServer\": \"Edit Server\",\n \"serverName\": \"Server Name\",\n \"serverUrl\": \"Server URL\",\n \"headerName\": \"Header Name\",\n \"timeout\": \"Timeout (seconds)\",\n \"testConnection\": \"Test Connection\",\n \"testing\": \"Testing\",\n \"saving\": \"Saving\",\n \"save\": \"Save\",\n \"cancel\": \"Cancel\",\n \"noAuth\": \"No Authentication\",\n \"oauthInProgress\": \"Waiting for OAuth completion...\",\n \"oauthCompleted\": \"OAuth completed successfully\",\n \"authType\": \"Authentication Type\",\n \"defaultServerName\": \"My MCP Server\",\n \"authTypes\": {\n \"none\": \"No Authentication\",\n \"apiKey\": \"API Key\",\n \"bearer\": \"Bearer Token\",\n \"oauth\": \"OAuth\",\n \"basic\": \"Basic Authentication\"\n },\n \"placeholders\": {\n \"serverUrl\": \"https://api.example.com\",\n \"apiKey\": \"Your secret API key\",\n \"bearerToken\": \"Your secret token\",\n \"username\": \"Your username\",\n \"password\": \"Your password\",\n \"oauthScopes\": \"OAuth scopes (comma separated)\"\n },\n \"errors\": {\n \"nameRequired\": \"Server name is required\",\n \"urlRequired\": \"Server URL is required\",\n \"invalidUrl\": \"Please enter a valid URL\",\n \"apiKeyRequired\": \"API key is required\",\n \"tokenRequired\": \"Bearer token is required\",\n \"usernameRequired\": \"Username is required\",\n \"passwordRequired\": \"Password is required\",\n \"testFailed\": \"Connection test failed\",\n \"saveFailed\": \"Failed to save MCP server\",\n \"oauthFailed\": \"OAuth process failed or was cancelled\",\n \"oauthTimeout\": \"OAuth process timed out, please try again\",\n \"timeoutRange\": \"Timeout must be between 1 and 300 seconds\"\n }\n }\n },\n \"scrollTabsLeft\": \"向左滚动标签\",\n \"tabsAriaLabel\": \"设置标签\",\n \"scrollTabsRight\": \"向右滚动标签\"\n },\n \"modals\": {\n \"uploadDoc\": {\n \"label\": \"上传新文档\",\n \"select\": \"选择如何将文档上传到 DocsGPT\",\n \"selectSource\": \"选择添加源的方式\",\n \"selectedFiles\": \"已选择的文件\",\n \"noFilesSelected\": \"未选择文件\",\n \"file\": \"从设备上传\",\n \"back\": \"后退\",\n \"wait\": \"请稍等 ...\",\n \"remote\": \"从网站收集\",\n \"start\": \"开始聊天\",\n \"name\": \"名称\",\n \"choose\": \"选择文件\",\n \"info\": \"请上传限制为25MB的.pdf、.txt、.rst、.csv、.xlsx、.docx、.md、.html、.epub、.json、.pptx、.zip文件\",\n \"uploadedFiles\": \"已上传文件\",\n \"cancel\": \"取消\",\n \"train\": \"训练\",\n \"link\": \"链接\",\n \"urlLink\": \"URL 链接\",\n \"repoUrl\": \"存储库 URL\",\n \"reddit\": {\n \"id\": \"客户端ID\",\n \"secret\": \"客户端密钥\",\n \"agent\": \"用户代理\",\n \"searchQueries\": \"搜索查询\",\n \"numberOfPosts\": \"帖子数量\",\n \"addQuery\": \"添加查询\"\n },\n \"drag\": {\n \"title\": \"将附件拖放到此处\",\n \"description\": \"释放以上传您的附件\"\n },\n \"progress\": {\n \"upload\": \"正在上传\",\n \"training\": \"正在上传\",\n \"completed\": \"上传完成\",\n \"failed\": \"上传失败\",\n \"wait\": \"这可能需要几分钟\",\n \"preparing\": \"准备上传\",\n \"tokenLimit\": \"超出令牌限制，请考虑上传较小的文档\",\n \"expandDetails\": \"展开上传详情\",\n \"collapseDetails\": \"折叠上传详情\",\n \"dismiss\": \"关闭上传通知\",\n \"uploadProgress\": \"上传进度 {{progress}}%\",\n \"clear\": \"清除\"\n },\n \"showAdvanced\": \"显示高级选项\",\n \"hideAdvanced\": \"隐藏高级选项\",\n \"ingestors\": {\n \"local_file\": {\n \"label\": \"上传文件\",\n \"heading\": \"上传新文档\"\n },\n \"crawler\": {\n \"label\": \"爬虫\",\n \"heading\": \"使用网络爬虫添加内容\"\n },\n \"url\": {\n \"label\": \"链接\",\n \"heading\": \"从URL添加内容\"\n },\n \"github\": {\n \"label\": \"GitHub\",\n \"heading\": \"从GitHub添加内容\"\n },\n \"reddit\": {\n \"label\": \"Reddit\",\n \"heading\": \"从Reddit添加内容\"\n },\n \"google_drive\": {\n \"label\": \"Google Drive\",\n \"heading\": \"从Google Drive上传\"\n },\n \"s3\": {\n \"label\": \"Amazon S3\",\n \"heading\": \"从Amazon S3添加内容\"\n },\n \"share_point\": {\n \"label\": \"SharePoint\",\n \"heading\": \"从SharePoint上传\"\n }\n },\n \"connectors\": {\n \"auth\": {\n \"connectedUser\": \"已连接用户\",\n \"authFailed\": \"身份验证失败\",\n \"authUrlFailed\": \"获取授权URL失败\",\n \"popupBlocked\": \"无法打开身份验证窗口。请允许弹出窗口。\",\n \"authCancelled\": \"身份验证已取消\",\n \"connectedAs\": \"已连接为 {{email}}\",\n \"disconnect\": \"断开连接\"\n },\n \"googleDrive\": {\n \"connect\": \"连接到 Google Drive\",\n \"sessionExpired\": \"会话已过期。请重新连接到 Google Drive。\",\n \"sessionExpiredGeneric\": \"会话已过期。请重新连接您的账户。\",\n \"validateFailed\": \"验证会话失败。请重新连接。\",\n \"noSession\": \"未找到有效会话。请重新连接到 Google Drive。\",\n \"noAccessToken\": \"访问令牌不可用。请重新连接到 Google Drive。\",\n \"pickerFailed\": \"无法打开文件选择器。请重试。\",\n \"selectedFiles\": \"已选择的文件\",\n \"selectFiles\": \"选择文件\",\n \"loading\": \"加载中...\",\n \"noFilesSelected\": \"未选择文件或文件夹\",\n \"folders\": \"文件夹\",\n \"files\": \"文件\",\n \"remove\": \"删除\",\n \"folderAlt\": \"文件夹\",\n \"fileAlt\": \"文件\"\n },\n \"sharePoint\": {\n \"connect\": \"连接到 SharePoint\",\n \"sessionExpired\": \"会话已过期。请重新连接到 SharePoint。\",\n \"sessionExpiredGeneric\": \"会话已过期。请重新连接您的账户。\",\n \"validateFailed\": \"验证会话失败。请重新连接。\",\n \"noSession\": \"未找到有效会话。请重新连接到 SharePoint。\",\n \"noAccessToken\": \"访问令牌不可用。请重新连接到 SharePoint。\",\n \"pickerFailed\": \"无法打开文件选择器。请重试。\",\n \"selectedFiles\": \"已选择的文件\",\n \"selectFiles\": \"选择文件\",\n \"loading\": \"加载中...\",\n \"noFilesSelected\": \"未选择文件或文件夹\",\n \"folders\": \"文件夹\",\n \"files\": \"文件\",\n \"remove\": \"删除\",\n \"folderAlt\": \"文件夹\",\n \"fileAlt\": \"文件\"\n }\n }\n },\n \"createAPIKey\": {\n \"label\": \"创建新的 API 密钥\",\n \"apiKeyName\": \"API 密钥名称\",\n \"chunks\": \"每个查询处理的块\",\n \"prompt\": \"选择活动提示\",\n \"sourceDoc\": \"源文档\",\n \"create\": \"创建\"\n },\n \"saveKey\": {\n \"note\": \"请保存您的密钥\",\n \"disclaimer\": \"这是您的密钥唯一一次展示机会。\",\n \"copy\": \"复制\",\n \"copied\": \"已复制\",\n \"confirm\": \"我已保存密钥\",\n \"apiKeyLabel\": \"API Key\"\n },\n \"deleteConv\": {\n \"confirm\": \"您确定要删除所有对话吗？\",\n \"delete\": \"删除\"\n },\n \"shareConv\": {\n \"label\": \"创建用于分享的公共页面\",\n \"note\": \"源文档、个人信息和后续对话将保持私密\",\n \"create\": \"创建\",\n \"option\": \"允许用户进行更多查询。\"\n },\n \"configTool\": {\n \"title\": \"工具配置\",\n \"type\": \"类型\",\n \"apiKeyLabel\": \"API 密钥 / OAuth\",\n \"apiKeyPlaceholder\": \"输入 API 密钥 / OAuth\",\n \"addButton\": \"添加工具\",\n \"closeButton\": \"关闭\",\n \"customNamePlaceholder\": \"Enter custom name (optional)\"\n },\n \"prompts\": {\n \"addPrompt\": \"添加提示\",\n \"addDescription\": \"添加自定义提示并保存到 DocsGPT\",\n \"editPrompt\": \"编辑提示\",\n \"editDescription\": \"编辑自定义提示并保存到 DocsGPT\",\n \"promptName\": \"提示名称\",\n \"promptText\": \"提示文本\",\n \"save\": \"保存\",\n \"nameExists\": \"名称已存在\",\n \"deleteConfirmation\": \"您确定要删除提示'{{name}}'吗？\",\n \"cancel\": \"取消\",\n \"placeholderText\": \"在此輸入提示內容...\",\n \"addExamplePlaceholder\": \"請總結此文本：\",\n \"variablesLabel\": \"變數\",\n \"variablesSubtext\": \"點擊以插入提示中\",\n \"variablesDescription\": \"點擊以插入到提示中\",\n \"systemVariables\": \"點擊以插入提示中\",\n \"toolVariables\": \"工具變數\",\n \"systemVariablesDropdownLabel\": \"系統變數\",\n \"systemVariableOptions\": {\n \"sourceContent\": \"來源內容\",\n \"sourceSummaries\": \"內容別名（向後相容）\",\n \"sourceDocuments\": \"文件物件列表\",\n \"sourceCount\": \"擷取的文件數量\",\n \"systemDate\": \"目前日期 (YYYY-MM-DD)\",\n \"systemTime\": \"目前時間 (HH:MM:SS)\",\n \"systemTimestamp\": \"ISO 8601 時間戳記\",\n \"systemRequestId\": \"唯一請求識別碼\",\n \"systemUserId\": \"目前使用者 ID\"\n },\n \"learnAboutPrompts\": \"了解提示 →\",\n \"publicPromptEditDisabled\": \"公共提示無法編輯\",\n \"promptTypePublic\": \"公共\",\n \"promptTypePrivate\": \"私人\"\n },\n \"chunk\": {\n \"add\": \"添加块\",\n \"edit\": \"编辑\",\n \"title\": \"标题\",\n \"enterTitle\": \"输入标题\",\n \"bodyText\": \"正文\",\n \"promptText\": \"提示文本\",\n \"save\": \"保存\",\n \"close\": \"关闭\",\n \"cancel\": \"取消\",\n \"delete\": \"删除\",\n \"deleteConfirmation\": \"您确定要删除此块吗？\"\n },\n \"addAction\": {\n \"title\": \"New Action\",\n \"actionNamePlaceholder\": \"Action Name\",\n \"invalidFormat\": \"Invalid function name format. Use only letters, numbers, underscores, and hyphens.\",\n \"formatHelp\": \"Use only letters, numbers, underscores, and hyphens (e.g., `get_data`, `send_report`, etc.)\",\n \"addButton\": \"Add\"\n },\n \"agentDetails\": {\n \"title\": \"Access Details\",\n \"publicLink\": \"Public Link\",\n \"apiKey\": \"API Key\",\n \"webhookUrl\": \"Webhook URL\",\n \"generate\": \"Generate\",\n \"test\": \"Test\",\n \"learnMore\": \"Learn more\"\n },\n \"importSpec\": {\n \"title\": \"导入 API 规范\",\n \"description\": \"上传 OpenAPI 3.x 或 Swagger 2.0 规范文件以自动生成操作。\",\n \"dropzoneText\": \"点击上传或拖拽到此处\",\n \"supportedFormats\": \"JSON 或 YAML 格式\",\n \"invalidFileType\": \"文件类型无效。请上传 JSON 或 YAML 文件。\",\n \"parseError\": \"解析规范失败。请检查文件格式。\",\n \"version\": \"版本\",\n \"baseUrl\": \"基础 URL\",\n \"actionsFound\": \"找到 {{count}} 个操作\",\n \"selectAll\": \"全选\",\n \"deselectAll\": \"取消全选\",\n \"cancel\": \"取消\",\n \"parse\": \"解析\",\n \"import\": \"导入 ({{count}})\"\n }\n },\n \"sharedConv\": {\n \"subtitle\": \"使用创建\",\n \"button\": \"开始使用 DocsGPT\",\n \"meta\": \"DocsGPT 使用 GenAI，请使用资源查看关键信息。\"\n },\n \"convTile\": {\n \"share\": \"分享\",\n \"delete\": \"删除\",\n \"rename\": \"重命名\",\n \"deleteWarning\": \"您确定要删除此对话吗？\"\n },\n \"pagination\": {\n \"rowsPerPage\": \"每页行数\",\n \"pageOf\": \"第 {{currentPage}} 页，共 {{totalPages}} 页\",\n \"firstPage\": \"第一页\",\n \"previousPage\": \"上一页\",\n \"nextPage\": \"下一页\",\n \"lastPage\": \"最后一页\"\n },\n \"conversation\": {\n \"copy\": \"复制\",\n \"copied\": \"已复制\",\n \"speak\": \"朗读\",\n \"answer\": \"回答\",\n \"edit\": {\n \"update\": \"更新\",\n \"cancel\": \"取消\",\n \"placeholder\": \"输入更新的查询...\"\n },\n \"sources\": {\n \"title\": \"来源\",\n \"text\": \"来源文本\",\n \"link\": \"来源链接\",\n \"view_more\": \"还有{{count}}个来源\",\n \"noSourcesAvailable\": \"没有可用的来源\"\n },\n \"attachments\": {\n \"attach\": \"附件\",\n \"remove\": \"删除附件\"\n },\n \"retry\": \"重试\",\n \"reasoning\": \"推理\"\n },\n \"agents\": {\n \"title\": \"代理\",\n \"description\": \"发现并创建结合指令、额外知识和任意技能组合的DocsGPT自定义版本\",\n \"newAgent\": \"新建代理\",\n \"backToAll\": \"返回所有代理\",\n \"searchPlaceholder\": \"搜索...\",\n \"noSearchResults\": \"未找到代理\",\n \"tryDifferentSearch\": \"请尝试不同的搜索词\",\n \"filters\": {\n \"all\": \"全部\",\n \"byDocsGPT\": \"由DocsGPT提供\",\n \"byMe\": \"我的\",\n \"shared\": \"与我共享\"\n },\n \"sections\": {\n \"template\": {\n \"title\": \"由DocsGPT提供\",\n \"description\": \"DocsGPT提供的代理\",\n \"emptyState\": \"未找到模板代理。\"\n },\n \"user\": {\n \"title\": \"我的代理\",\n \"description\": \"您创建或发布的代理\",\n \"emptyState\": \"您还没有创建任何代理。\"\n },\n \"shared\": {\n \"title\": \"与我共享\",\n \"description\": \"通过公共链接导入的代理\",\n \"emptyState\": \"未找到共享代理。\"\n }\n },\n \"form\": {\n \"headings\": {\n \"new\": \"新建代理\",\n \"edit\": \"编辑代理\",\n \"draft\": \"新建代理（草稿）\"\n },\n \"buttons\": {\n \"publish\": \"发布\",\n \"save\": \"保存\",\n \"saveDraft\": \"保存草稿\",\n \"cancel\": \"取消\",\n \"delete\": \"删除\",\n \"logs\": \"日志\",\n \"accessDetails\": \"访问详情\",\n \"add\": \"添加\"\n },\n \"sections\": {\n \"meta\": \"元数据\",\n \"source\": \"来源\",\n \"prompt\": \"提示词\",\n \"tools\": \"工具\",\n \"agentType\": \"代理类型\",\n \"models\": \"模型\",\n \"advanced\": \"高级\",\n \"preview\": \"预览\"\n },\n \"placeholders\": {\n \"agentName\": \"代理名称\",\n \"describeAgent\": \"描述您的代理\",\n \"selectSources\": \"选择来源\",\n \"chunksPerQuery\": \"每次查询的块数\",\n \"selectType\": \"选择类型\",\n \"selectTools\": \"选择工具\",\n \"selectModels\": \"为此代理选择模型\",\n \"selectDefaultModel\": \"选择默认模型\",\n \"enterTokenLimit\": \"输入令牌限制\",\n \"enterRequestLimit\": \"输入请求限制\"\n },\n \"sourcePopup\": {\n \"title\": \"选择来源\",\n \"searchPlaceholder\": \"搜索来源...\",\n \"noOptionsMessage\": \"没有可用的来源\"\n },\n \"toolsPopup\": {\n \"title\": \"选择工具\",\n \"searchPlaceholder\": \"搜索工具...\",\n \"noOptionsMessage\": \"没有可用的工具\"\n },\n \"modelsPopup\": {\n \"title\": \"选择模型\",\n \"searchPlaceholder\": \"搜索模型...\",\n \"noOptionsMessage\": \"没有可用的模型\"\n },\n \"upload\": {\n \"clickToUpload\": \"点击上传\",\n \"dragAndDrop\": \" 或拖放\"\n },\n \"agentTypes\": {\n \"classic\": \"经典\",\n \"react\": \"ReAct\"\n },\n \"labels\": {\n \"defaultModel\": \"默认模型\"\n },\n \"advanced\": {\n \"jsonSchema\": \"JSON响应架构\",\n \"jsonSchemaDescription\": \"定义JSON架构以强制执行结构化输出格式\",\n \"validJson\": \"有效的JSON\",\n \"invalidJson\": \"无效的JSON - 修复后才能保存\",\n \"tokenLimiting\": \"令牌限制\",\n \"tokenLimitingDescription\": \"限制此代理每天可使用的总令牌数\",\n \"requestLimiting\": \"请求限制\",\n \"requestLimitingDescription\": \"限制每天可向此代理发出的总请求数\"\n },\n \"preview\": {\n \"publishedPreview\": \"已发布的代理可以在此处预览\"\n },\n \"externalKb\": \"外部知识库\"\n },\n \"logs\": {\n \"title\": \"代理日志\",\n \"lastUsedAt\": \"最后使用时间\",\n \"noUsageHistory\": \"无使用历史\",\n \"tableHeader\": \"代理端点日志\"\n },\n \"shared\": {\n \"notFound\": \"未找到代理。请确保代理已共享。\"\n },\n \"preview\": {\n \"testMessage\": \"在此测试您的代理。已发布的代理可以在对话中使用。\"\n },\n \"deleteConfirmation\": \"您确定要删除此代理吗？\",\n \"folders\": {\n \"newFolder\": \"新建文件夹\",\n \"createFolder\": \"创建文件夹\",\n \"folderName\": \"文件夹名称\",\n \"rename\": \"重命名\",\n \"delete\": \"删除\",\n \"deleteConfirm\": \"确定要删除此文件夹吗？文件夹中的代理将被移出。\",\n \"empty\": \"此文件夹为空\",\n \"moveToFolder\": \"移动到文件夹\",\n \"moveTo\": \"移动\",\n \"move\": \"移动\",\n \"noFolder\": \"无文件夹 (根目录)\",\n \"backToRoot\": \"返回\",\n \"noSubfolders\": \"没有子文件夹\",\n \"noFolders\": \"暂无文件夹\"\n }\n },\n \"components\": {\n \"fileUpload\": {\n \"clickToUpload\": \"Click to upload or drag and drop\",\n \"dropFiles\": \"Drop the files here\",\n \"fileTypes\": \"PNG, JPG, JPEG up to\",\n \"sizeLimitUnit\": \"MB\",\n \"fileSizeError\": \"File exceeds {{size}}MB limit\"\n }\n },\n \"pageNotFound\": {\n \"title\": \"404\",\n \"message\": \"The page you are looking for does not exist.\",\n \"goHome\": \"Go Back Home\"\n },\n \"filePicker\": {\n \"searchPlaceholder\": \"搜索文件和文件夹...\",\n \"itemsSelected\": \"已选择 {{count}} 项\",\n \"name\": \"名称\",\n \"lastModified\": \"最后修改\",\n \"size\": \"大小\",\n \"myFiles\": \"我的文件\",\n \"sharedWithMe\": \"与我共享\",\n \"loadingMore\": \"加载更多文件...\"\n },\n \"actionButtons\": {\n \"openNewChat\": \"打开新聊天\",\n \"share\": \"分享\"\n },\n \"mermaid\": {\n \"downloadOptions\": \"下载选项\",\n \"viewCode\": \"查看代码\",\n \"decreaseZoom\": \"缩小\",\n \"resetZoom\": \"重置缩放\",\n \"increaseZoom\": \"放大\"\n },\n \"navigation\": {\n \"agents\": \"代理\"\n },\n \"notification\": {\n \"ariaLabel\": \"通知\",\n \"closeAriaLabel\": \"关闭通知\"\n },\n \"prompts\": {\n \"textAriaLabel\": \"提示文本\"\n }\n}\n" }, { "path": "frontend/src/main.tsx", "content": "import React from 'react';\nimport ReactDOM from 'react-dom/client';\nimport App from './App';\nimport { BrowserRouter } from 'react-router-dom';\nimport { Provider } from 'react-redux';\nimport store from './store';\nimport './index.css';\n\n// Show scrollbar on scroll for scrollbar-overlay elements, hide after 1s idle\nconst scrollTimers = new WeakMap>();\nlet sbIdCounter = 0;\nconst sbStyleEl = document.createElement('style');\ndocument.head.appendChild(sbStyleEl);\nconst activeSbs = new Map();\nfunction rebuildSbStyles() {\n sbStyleEl.textContent = Array.from(activeSbs.values()).join('');\n}\nfunction showOverlayScrollbar(el: HTMLElement) {\n if (!el.dataset.sbId) el.dataset.sbId = String(++sbIdCounter);\n const sbId = el.dataset.sbId;\n const isDark = document.body.classList.contains('dark');\n const thumb = isDark ? '#949494' : '#E2E8F0';\n const thumbHover = isDark ? '#F0F0F0' : '#8C9198';\n // Webkit: inject