Repository: Mintplex-Labs/anything-llm Branch: master Commit: 02ee24baac8d Files: 997 Total size: 5.8 MB Directory structure: gitextract_5urqiqdy/ ├── .devcontainer/ │ ├── README.md │ └── devcontainer.json ├── .dockerignore ├── .editorconfig ├── .gitattributes ├── .github/ │ ├── FUNDING.yml │ ├── ISSUE_TEMPLATE/ │ │ ├── 01_bug.yml │ │ ├── 02_feature.yml │ │ ├── 03_documentation.yml │ │ └── config.yml │ └── workflows/ │ ├── build-and-push-image-semver.yaml │ ├── build-and-push-image.yaml │ ├── build-qa-tag.yaml │ ├── check-package-versions.yaml │ ├── check-translations.yaml │ ├── cleanup-qa-tag.yaml │ ├── lint.yaml │ ├── run-tests.yaml │ └── sponsors.yaml ├── .gitignore ├── .gitmodules ├── .hadolint.yaml ├── .nvmrc ├── .prettierignore ├── .prettierrc ├── .vscode/ │ ├── launch.json │ ├── settings.json │ └── tasks.json ├── BARE_METAL.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── SECURITY.md ├── TERMS_SELF_HOSTED.md ├── cloud-deployments/ │ ├── aws/ │ │ └── cloudformation/ │ │ ├── DEPLOY.md │ │ ├── aws_https_instructions.md │ │ └── cloudformation_create_anythingllm.json │ ├── digitalocean/ │ │ └── terraform/ │ │ ├── DEPLOY.md │ │ ├── main.tf │ │ ├── outputs.tf │ │ └── user_data.tp1 │ ├── gcp/ │ │ └── deployment/ │ │ ├── DEPLOY.md │ │ └── gcp_deploy_anything_llm.yaml │ ├── helm/ │ │ └── charts/ │ │ └── anythingllm/ │ │ ├── .helmignore │ │ ├── Chart.yaml │ │ ├── README.md │ │ ├── README.md.gotmpl │ │ ├── templates/ │ │ │ ├── NOTES.txt │ │ │ ├── _helpers.tpl │ │ │ ├── configmap.yaml │ │ │ ├── deployment.yaml │ │ │ ├── extra-objects.yaml │ │ │ ├── ingress.yaml │ │ │ ├── pvc.yaml │ │ │ ├── service.yaml │ │ │ ├── serviceaccount.yaml │ │ │ └── tests/ │ │ │ └── test-connection.yaml │ │ └── values.yaml │ ├── huggingface-spaces/ │ │ └── Dockerfile │ └── k8/ │ └── manifest.yaml ├── collector/ │ ├── .env.example │ ├── .gitignore │ ├── .nvmrc │ ├── __tests__/ │ │ └── utils/ │ │ ├── WhisperProviders/ │ │ │ └── ffmpeg/ │ │ │ └── index.test.js │ │ └── url/ │ │ └── index.test.js │ ├── eslint.config.mjs │ ├── extensions/ │ │ ├── index.js │ │ └── resync/ │ │ └── index.js │ ├── hotdir/ │ │ └── __HOTDIR__.md │ ├── index.js │ ├── middleware/ │ │ ├── httpLogger.js │ │ ├── setDataSigner.js │ │ └── verifyIntegrity.js │ ├── nodemon.json │ ├── package.json │ ├── processLink/ │ │ ├── convert/ │ │ │ └── generic.js │ │ ├── helpers/ │ │ │ └── index.js │ │ └── index.js │ ├── processRawText/ │ │ └── index.js │ ├── processSingleFile/ │ │ ├── convert/ │ │ │ ├── asAudio.js │ │ │ ├── asDocx.js │ │ │ ├── asEPub.js │ │ │ ├── asImage.js │ │ │ ├── asMbox.js │ │ │ ├── asOfficeMime.js │ │ │ ├── asPDF/ │ │ │ │ ├── PDFLoader/ │ │ │ │ │ └── index.js │ │ │ │ └── index.js │ │ │ ├── asTxt.js │ │ │ └── asXlsx.js │ │ └── index.js │ ├── storage/ │ │ ├── .gitignore │ │ └── tmp/ │ │ └── .placeholder │ ├── utils/ │ │ ├── EncryptionWorker/ │ │ │ └── index.js │ │ ├── OCRLoader/ │ │ │ ├── index.js │ │ │ └── validLangs.js │ │ ├── WhisperProviders/ │ │ │ ├── OpenAiWhisper.js │ │ │ ├── ffmpeg/ │ │ │ │ └── index.js │ │ │ └── localWhisper.js │ │ ├── comKey/ │ │ │ └── index.js │ │ ├── constants.js │ │ ├── downloadURIToFile/ │ │ │ └── index.js │ │ ├── extensions/ │ │ │ ├── Confluence/ │ │ │ │ ├── ConfluenceLoader/ │ │ │ │ │ └── index.js │ │ │ │ └── index.js │ │ │ ├── DrupalWiki/ │ │ │ │ ├── DrupalWiki/ │ │ │ │ │ └── index.js │ │ │ │ └── index.js │ │ │ ├── ObsidianVault/ │ │ │ │ └── index.js │ │ │ ├── PaperlessNgx/ │ │ │ │ ├── PaperlessNgxLoader/ │ │ │ │ │ └── index.js │ │ │ │ └── index.js │ │ │ ├── RepoLoader/ │ │ │ │ ├── GithubRepo/ │ │ │ │ │ ├── RepoLoader/ │ │ │ │ │ │ └── index.js │ │ │ │ │ └── index.js │ │ │ │ ├── GitlabRepo/ │ │ │ │ │ ├── RepoLoader/ │ │ │ │ │ │ └── index.js │ │ │ │ │ └── index.js │ │ │ │ └── index.js │ │ │ ├── WebsiteDepth/ │ │ │ │ └── index.js │ │ │ └── YoutubeTranscript/ │ │ │ ├── YoutubeLoader/ │ │ │ │ ├── index.js │ │ │ │ └── youtube-transcript.js │ │ │ └── index.js │ │ ├── files/ │ │ │ ├── index.js │ │ │ └── mime.js │ │ ├── http/ │ │ │ └── index.js │ │ ├── logger/ │ │ │ └── index.js │ │ ├── runtimeSettings/ │ │ │ └── index.js │ │ ├── shell.js │ │ ├── tokenizer/ │ │ │ └── index.js │ │ └── url/ │ │ └── index.js │ └── yarn.lock ├── docker/ │ ├── .env.example │ ├── Dockerfile │ ├── HOW_TO_USE_DOCKER.md │ ├── docker-compose.yml │ ├── docker-entrypoint.sh │ ├── docker-healthcheck.sh │ └── vex/ │ ├── CVE-2019-10790.vex.json │ ├── CVE-2024-29415.vex.json │ ├── CVE-2024-37890.vex.json │ └── CVE-2024-4068.vex.json ├── eslint.config.js ├── extras/ │ ├── scripts/ │ │ └── verifyPackageVersions.mjs │ ├── support/ │ │ └── announcements/ │ │ ├── 2025-04-08.json │ │ ├── 2025-07-08.json │ │ ├── 2026-01-12.json │ │ └── list.txt │ └── translator/ │ ├── .env.example │ ├── README.md │ └── index.mjs ├── frontend/ │ ├── .env.example │ ├── .gitignore │ ├── .nvmrc │ ├── eslint.config.js │ ├── index.html │ ├── jsconfig.json │ ├── package.json │ ├── postcss.config.js │ ├── public/ │ │ ├── manifest.json │ │ ├── robots.txt │ │ └── service-workers/ │ │ └── push-notifications.js │ ├── scripts/ │ │ └── postbuild.js │ ├── src/ │ │ ├── App.jsx │ │ ├── AuthContext.jsx │ │ ├── LogoContext.jsx │ │ ├── PWAContext.jsx │ │ ├── PfpContext.jsx │ │ ├── ThemeContext.jsx │ │ ├── components/ │ │ │ ├── CanViewChatHistory/ │ │ │ │ └── index.jsx │ │ │ ├── ChangeWarning/ │ │ │ │ └── index.jsx │ │ │ ├── ChatBubble/ │ │ │ │ └── index.jsx │ │ │ ├── CommunityHub/ │ │ │ │ ├── PublishEntityModal/ │ │ │ │ │ ├── AgentFlows/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── SlashCommands/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── SystemPrompts/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ └── index.jsx │ │ │ │ └── UnauthenticatedHubModal/ │ │ │ │ └── index.jsx │ │ │ ├── ContextualSaveBar/ │ │ │ │ └── index.jsx │ │ │ ├── DataConnectorOption/ │ │ │ │ └── media/ │ │ │ │ └── index.js │ │ │ ├── DefaultChat/ │ │ │ │ └── index.jsx │ │ │ ├── EmbeddingSelection/ │ │ │ │ ├── AzureAiOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── CohereOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── EmbedderItem/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── GeminiOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── GenericOpenAiOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── LMStudioOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── LemonadeOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── LiteLLMOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── LocalAiOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── MistralAiOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── NativeEmbeddingOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── OllamaOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── OpenAiOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── OpenRouterOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ └── VoyageAiOptions/ │ │ │ │ └── index.jsx │ │ │ ├── ErrorBoundaryFallback/ │ │ │ │ └── index.jsx │ │ │ ├── Footer/ │ │ │ │ └── index.jsx │ │ │ ├── KeyboardShortcutsHelp/ │ │ │ │ └── index.jsx │ │ │ ├── LLMSelection/ │ │ │ │ ├── AnthropicAiOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── ApiPieOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── AwsBedrockLLMOptions/ │ │ │ │ │ ├── index.jsx │ │ │ │ │ └── regions.js │ │ │ │ ├── AzureAiOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── CohereAiOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── CometApiLLMOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── DPAISOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── DeepSeekOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── DockerModelRunnerOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── FireworksAiOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── FoundryOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── GeminiLLMOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── GenericOpenAiOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── GiteeAIOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── GroqAiOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── HuggingFaceOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── KoboldCPPOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── LLMItem/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── LLMProviderOption/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── LMStudioOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── LemonadeOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── LiteLLMOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── LocalAiOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── MistralOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── MoonshotAiOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── NovitaLLMOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── NvidiaNimOptions/ │ │ │ │ │ ├── index.jsx │ │ │ │ │ ├── managed.jsx │ │ │ │ │ └── remote.jsx │ │ │ │ ├── OllamaLLMOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── OpenAiOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── OpenRouterOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── PPIOLLMOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── PerplexityOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── PrivateModeOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── SambaNovaOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── TextGenWebUIOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── TogetherAiOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── XAiLLMOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ └── ZAiLLMOptions/ │ │ │ │ └── index.jsx │ │ │ ├── ModalWrapper/ │ │ │ │ └── index.jsx │ │ │ ├── Modals/ │ │ │ │ ├── DisplayRecoveryCodeModal/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── ManageWorkspace/ │ │ │ │ │ ├── DataConnectors/ │ │ │ │ │ │ ├── ConnectorOption/ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ ├── Connectors/ │ │ │ │ │ │ │ ├── Confluence/ │ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ │ ├── DrupalWiki/ │ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ │ ├── Github/ │ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ │ ├── Gitlab/ │ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ │ ├── Obsidian/ │ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ │ ├── PaperlessNgx/ │ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ │ ├── WebsiteDepth/ │ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ │ └── Youtube/ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── Documents/ │ │ │ │ │ │ ├── Directory/ │ │ │ │ │ │ │ ├── ContextMenu/ │ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ │ ├── FileRow/ │ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ │ ├── FolderRow/ │ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ │ ├── FolderSelectionPopup/ │ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ │ ├── MoveToFolderIcon.jsx │ │ │ │ │ │ │ ├── NewFolderModal/ │ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ │ ├── index.jsx │ │ │ │ │ │ │ └── utils.js │ │ │ │ │ │ ├── UploadFile/ │ │ │ │ │ │ │ ├── FileUploadProgress/ │ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ ├── WorkspaceDirectory/ │ │ │ │ │ │ │ ├── WorkspaceFileRow/ │ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ └── index.jsx │ │ │ │ │ └── index.jsx │ │ │ │ ├── NewWorkspace.jsx │ │ │ │ └── Password/ │ │ │ │ ├── MultiUserAuth.jsx │ │ │ │ ├── SingleUserAuth.jsx │ │ │ │ └── index.jsx │ │ │ ├── Preloader.jsx │ │ │ ├── PrivateRoute/ │ │ │ │ └── index.jsx │ │ │ ├── ProviderPrivacy/ │ │ │ │ ├── constants.js │ │ │ │ └── index.jsx │ │ │ ├── SettingsButton/ │ │ │ │ └── index.jsx │ │ │ ├── SettingsSidebar/ │ │ │ │ ├── MenuOption/ │ │ │ │ │ └── index.jsx │ │ │ │ └── index.jsx │ │ │ ├── Sidebar/ │ │ │ │ ├── ActiveWorkspaces/ │ │ │ │ │ ├── ThreadContainer/ │ │ │ │ │ │ ├── ThreadItem/ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ └── index.jsx │ │ │ │ │ └── index.jsx │ │ │ │ ├── SearchBox/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── SidebarToggle/ │ │ │ │ │ └── index.jsx │ │ │ │ └── index.jsx │ │ │ ├── SpeechToText/ │ │ │ │ └── BrowserNative/ │ │ │ │ └── index.jsx │ │ │ ├── TextToSpeech/ │ │ │ │ ├── BrowserNative/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── ElevenLabsOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── OpenAiGenericOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── OpenAiOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ └── PiperTTSOptions/ │ │ │ │ └── index.jsx │ │ │ ├── TranscriptionSelection/ │ │ │ │ ├── NativeTranscriptionOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ └── OpenAiOptions/ │ │ │ │ └── index.jsx │ │ │ ├── UserIcon/ │ │ │ │ └── index.jsx │ │ │ ├── UserMenu/ │ │ │ │ ├── AccountModal/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── UserButton/ │ │ │ │ │ └── index.jsx │ │ │ │ └── index.jsx │ │ │ ├── VectorDBSelection/ │ │ │ │ ├── AstraDBOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── ChromaCloudOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── ChromaDBOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── LanceDBOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── MilvusDBOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── PGVectorOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── PineconeDBOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── QDrantDBOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── VectorDBItem/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── WeaviateDBOptions/ │ │ │ │ │ └── index.jsx │ │ │ │ └── ZillizCloudOptions/ │ │ │ │ └── index.jsx │ │ │ ├── WorkspaceChat/ │ │ │ │ ├── ChatContainer/ │ │ │ │ │ ├── ChatHistory/ │ │ │ │ │ │ ├── Chartable/ │ │ │ │ │ │ │ ├── CustomCell.jsx │ │ │ │ │ │ │ ├── CustomTooltip.jsx │ │ │ │ │ │ │ ├── chart-utils.js │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ ├── Citation/ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ ├── HistoricalMessage/ │ │ │ │ │ │ │ ├── Actions/ │ │ │ │ │ │ │ │ ├── ActionMenu/ │ │ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ │ │ ├── DeleteMessage/ │ │ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ │ │ ├── EditMessage/ │ │ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ │ │ ├── RenderMetrics/ │ │ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ │ │ ├── TTSButton/ │ │ │ │ │ │ │ │ │ ├── asyncTts.jsx │ │ │ │ │ │ │ │ │ ├── index.jsx │ │ │ │ │ │ │ │ │ ├── native.jsx │ │ │ │ │ │ │ │ │ └── piperTTS.jsx │ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ ├── PromptReply/ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ ├── StatusResponse/ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ ├── ThoughtContainer/ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── ChatTooltips/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── DnDWrapper/ │ │ │ │ │ │ ├── FileUploadWarningModal/ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── PromptInput/ │ │ │ │ │ │ ├── AgentMenu/ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ ├── AttachItem/ │ │ │ │ │ │ │ ├── ParsedFilesMenu/ │ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ ├── Attachments/ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ ├── LLMSelector/ │ │ │ │ │ │ │ ├── ChatModelSelection/ │ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ │ ├── LLMSelector/ │ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ │ ├── SetupProvider/ │ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ │ ├── action.jsx │ │ │ │ │ │ │ ├── index.jsx │ │ │ │ │ │ │ └── utils.js │ │ │ │ │ │ ├── SpeechToText/ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ ├── StopGenerationButton/ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ ├── TextSizeMenu/ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ ├── ToolsMenu/ │ │ │ │ │ │ │ ├── Tabs/ │ │ │ │ │ │ │ │ ├── AgentSkills/ │ │ │ │ │ │ │ │ │ ├── SkillRow/ │ │ │ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ │ │ └── SlashCommands/ │ │ │ │ │ │ │ │ ├── SlashCommandRow/ │ │ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ │ │ ├── SlashPresets/ │ │ │ │ │ │ │ │ │ ├── AddPresetModal.jsx │ │ │ │ │ │ │ │ │ ├── EditPresetModal.jsx │ │ │ │ │ │ │ │ │ └── constants.js │ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ │ ├── index.jsx │ │ │ │ │ │ │ └── useToolsMenuItems.js │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── SourcesSidebar/ │ │ │ │ │ │ ├── MobileCitationModal/ │ │ │ │ │ │ │ ├── SourceDetailView/ │ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ ├── SourceItem/ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── TextSizeMenu/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── WorkspaceModelPicker/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ └── index.jsx │ │ │ │ ├── LoadingChat/ │ │ │ │ │ └── index.jsx │ │ │ │ └── index.jsx │ │ │ ├── contexts/ │ │ │ │ └── TTSProvider.jsx │ │ │ └── lib/ │ │ │ ├── CTAButton/ │ │ │ │ └── index.jsx │ │ │ ├── ModelTable/ │ │ │ │ ├── index.jsx │ │ │ │ ├── layout.jsx │ │ │ │ └── loading.jsx │ │ │ ├── MonoProviderIcon/ │ │ │ │ └── index.jsx │ │ │ ├── QuickActions/ │ │ │ │ └── index.jsx │ │ │ ├── SuggestedMessages/ │ │ │ │ └── index.jsx │ │ │ └── Toggle/ │ │ │ └── index.jsx │ │ ├── hooks/ │ │ │ ├── useAppVersion.js │ │ │ ├── useChatContainerQuickScroll.js │ │ │ ├── useChatHistoryScrollHandle.js │ │ │ ├── useCommunityHubAuth.js │ │ │ ├── useCopyText.js │ │ │ ├── useGetProvidersModels.js │ │ │ ├── useLanguageOptions.js │ │ │ ├── useLoginMode.js │ │ │ ├── useLogo.js │ │ │ ├── useModal.js │ │ │ ├── useOnboardingComplete.js │ │ │ ├── usePfp.js │ │ │ ├── usePrefersDarkMode.js │ │ │ ├── usePromptInputStorage.js │ │ │ ├── useProviderEndpointAutoDiscovery.js │ │ │ ├── useQuery.js │ │ │ ├── useScrollActiveItemIntoView.js │ │ │ ├── useSimpleSSO.js │ │ │ ├── useTextSize.js │ │ │ ├── useTheme.js │ │ │ ├── useUser.js │ │ │ └── useWebPushNotifications.js │ │ ├── i18n.js │ │ ├── index.css │ │ ├── locales/ │ │ │ ├── ar/ │ │ │ │ └── common.js │ │ │ ├── cs/ │ │ │ │ └── common.js │ │ │ ├── da/ │ │ │ │ └── common.js │ │ │ ├── de/ │ │ │ │ └── common.js │ │ │ ├── dynamicKeyAllowlist.js │ │ │ ├── en/ │ │ │ │ └── common.js │ │ │ ├── es/ │ │ │ │ └── common.js │ │ │ ├── et/ │ │ │ │ └── common.js │ │ │ ├── fa/ │ │ │ │ └── common.js │ │ │ ├── findUnusedTranslations.mjs │ │ │ ├── fr/ │ │ │ │ └── common.js │ │ │ ├── he/ │ │ │ │ └── common.js │ │ │ ├── it/ │ │ │ │ └── common.js │ │ │ ├── ja/ │ │ │ │ └── common.js │ │ │ ├── ko/ │ │ │ │ └── common.js │ │ │ ├── lv/ │ │ │ │ └── common.js │ │ │ ├── nl/ │ │ │ │ └── common.js │ │ │ ├── normalizeEn.mjs │ │ │ ├── pl/ │ │ │ │ └── common.js │ │ │ ├── pt_BR/ │ │ │ │ └── common.js │ │ │ ├── resources.js │ │ │ ├── ro/ │ │ │ │ └── common.js │ │ │ ├── ru/ │ │ │ │ └── common.js │ │ │ ├── tr/ │ │ │ │ └── common.js │ │ │ ├── verifyTranslations.mjs │ │ │ ├── vn/ │ │ │ │ └── common.js │ │ │ ├── zh/ │ │ │ │ └── common.js │ │ │ └── zh_TW/ │ │ │ └── common.js │ │ ├── main.jsx │ │ ├── media/ │ │ │ └── animations/ │ │ │ ├── agent-animation.webm │ │ │ └── thinking-animation.webm │ │ ├── models/ │ │ │ ├── admin.js │ │ │ ├── agentFlows.js │ │ │ ├── appearance.js │ │ │ ├── browserExtensionApiKey.js │ │ │ ├── communityHub.js │ │ │ ├── dataConnector.js │ │ │ ├── document.js │ │ │ ├── embed.js │ │ │ ├── experimental/ │ │ │ │ ├── agentPlugins.js │ │ │ │ └── liveSync.js │ │ │ ├── invite.js │ │ │ ├── mcpServers.js │ │ │ ├── mobile.js │ │ │ ├── promptHistory.js │ │ │ ├── system.js │ │ │ ├── systemPromptVariable.js │ │ │ ├── utils/ │ │ │ │ ├── dmrUtils.js │ │ │ │ └── lemonadeUtils.js │ │ │ ├── workspace.js │ │ │ └── workspaceThread.js │ │ ├── pages/ │ │ │ ├── 404.jsx │ │ │ ├── Admin/ │ │ │ │ ├── AgentBuilder/ │ │ │ │ │ ├── AddBlockMenu/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── BlockList/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── HeaderMenu/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── index.jsx │ │ │ │ │ └── nodes/ │ │ │ │ │ ├── ApiCallNode/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── CodeNode/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── FileNode/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── FinishNode/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── FlowInfoNode/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── LLMInstructionNode/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── StartNode/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── WebScrapingNode/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ └── WebsiteNode/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── Agents/ │ │ │ │ │ ├── AgentFlows/ │ │ │ │ │ │ ├── FlowPanel.jsx │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── AgentSkillSettings/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── Badges/ │ │ │ │ │ │ └── default.jsx │ │ │ │ │ ├── DefaultSkillPanel/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── GenericSkillPanel/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── Imported/ │ │ │ │ │ │ ├── ImportedSkillConfig/ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ └── SkillList/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── MCPServers/ │ │ │ │ │ │ ├── ServerPanel.jsx │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── SQLConnectorSelection/ │ │ │ │ │ │ ├── DBConnection.jsx │ │ │ │ │ │ ├── SQLConnectionModal.jsx │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── WebSearchSelection/ │ │ │ │ │ │ ├── SearchProviderItem/ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ ├── SearchProviderOptions/ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── index.jsx │ │ │ │ │ └── skills.js │ │ │ │ ├── DefaultSystemPrompt/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── ExperimentalFeatures/ │ │ │ │ │ ├── Features/ │ │ │ │ │ │ └── LiveSync/ │ │ │ │ │ │ ├── manage/ │ │ │ │ │ │ │ ├── DocumentSyncQueueRow/ │ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ └── toggle.jsx │ │ │ │ │ ├── features.js │ │ │ │ │ └── index.jsx │ │ │ │ ├── Invitations/ │ │ │ │ │ ├── InviteRow/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── NewInviteModal/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ └── index.jsx │ │ │ │ ├── Logging/ │ │ │ │ │ ├── LogRow/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ └── index.jsx │ │ │ │ ├── SystemPromptVariables/ │ │ │ │ │ ├── AddVariableModal/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── VariableRow/ │ │ │ │ │ │ ├── EditVariableModal/ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ └── index.jsx │ │ │ │ │ └── index.jsx │ │ │ │ ├── Users/ │ │ │ │ │ ├── NewUserModal/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── UserRow/ │ │ │ │ │ │ ├── EditUserModal/ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ └── index.jsx │ │ │ │ │ └── index.jsx │ │ │ │ └── Workspaces/ │ │ │ │ ├── NewWorkspaceModal/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── WorkspaceRow/ │ │ │ │ │ └── index.jsx │ │ │ │ └── index.jsx │ │ │ ├── GeneralSettings/ │ │ │ │ ├── ApiKeys/ │ │ │ │ │ ├── ApiKeyRow/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── NewApiKeyModal/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ └── index.jsx │ │ │ │ ├── AudioPreference/ │ │ │ │ │ ├── index.jsx │ │ │ │ │ ├── stt.jsx │ │ │ │ │ └── tts.jsx │ │ │ │ ├── BrowserExtensionApiKey/ │ │ │ │ │ ├── BrowserExtensionApiKeyRow/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── NewBrowserExtensionApiKeyModal/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ └── index.jsx │ │ │ │ ├── ChatEmbedWidgets/ │ │ │ │ │ ├── EmbedChats/ │ │ │ │ │ │ ├── ChatRow/ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ ├── MarkdownRenderer.jsx │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── EmbedConfigs/ │ │ │ │ │ │ ├── EmbedRow/ │ │ │ │ │ │ │ ├── CodeSnippetModal/ │ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ │ ├── EditEmbedModal/ │ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ ├── NewEmbedModal/ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ └── index.jsx │ │ │ │ │ └── index.jsx │ │ │ │ ├── Chats/ │ │ │ │ │ ├── ChatRow/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── MarkdownRenderer.jsx │ │ │ │ │ └── index.jsx │ │ │ │ ├── CommunityHub/ │ │ │ │ │ ├── Authentication/ │ │ │ │ │ │ ├── UserItems/ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ ├── index.jsx │ │ │ │ │ │ └── useUserItems.js │ │ │ │ │ ├── ImportItem/ │ │ │ │ │ │ ├── Steps/ │ │ │ │ │ │ │ ├── Completed/ │ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ │ ├── Introduction/ │ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ │ ├── PullAndReview/ │ │ │ │ │ │ │ │ ├── HubItem/ │ │ │ │ │ │ │ │ │ ├── AgentFlow.jsx │ │ │ │ │ │ │ │ │ ├── AgentSkill.jsx │ │ │ │ │ │ │ │ │ ├── SlashCommand.jsx │ │ │ │ │ │ │ │ │ ├── SystemPrompt.jsx │ │ │ │ │ │ │ │ │ ├── Unknown.jsx │ │ │ │ │ │ │ │ │ └── index.js │ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── Trending/ │ │ │ │ │ │ ├── HubItems/ │ │ │ │ │ │ │ ├── HubItemCard/ │ │ │ │ │ │ │ │ ├── agentFlow.jsx │ │ │ │ │ │ │ │ ├── agentSkill.jsx │ │ │ │ │ │ │ │ ├── generic.jsx │ │ │ │ │ │ │ │ ├── index.jsx │ │ │ │ │ │ │ │ ├── slashCommand.jsx │ │ │ │ │ │ │ │ └── systemPrompt.jsx │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ └── index.jsx │ │ │ │ │ └── utils.js │ │ │ │ ├── EmbeddingPreference/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── EmbeddingTextSplitterPreference/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── LLMPreference/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── MobileConnections/ │ │ │ │ │ ├── ConnectionModal/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── DeviceRow/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ └── index.jsx │ │ │ │ ├── PrivacyAndData/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── Security/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── Settings/ │ │ │ │ │ ├── Branding/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── Chat/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── Interface/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ └── components/ │ │ │ │ │ ├── AutoSpeak/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── AutoSubmit/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── ChatRenderHTML/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── CustomAppName/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── CustomLogo/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── CustomSiteSettings/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── FooterCustomization/ │ │ │ │ │ │ ├── NewIconForm/ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── LanguagePreference/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── ShowScrollbar/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── SpellCheck/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── SupportEmail/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ └── ThemePreference/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── TranscriptionPreference/ │ │ │ │ │ └── index.jsx │ │ │ │ └── VectorDatabase/ │ │ │ │ └── index.jsx │ │ │ ├── Invite/ │ │ │ │ ├── NewUserModal/ │ │ │ │ │ └── index.jsx │ │ │ │ └── index.jsx │ │ │ ├── Login/ │ │ │ │ ├── SSO/ │ │ │ │ │ └── simple.jsx │ │ │ │ └── index.jsx │ │ │ ├── Main/ │ │ │ │ ├── Home/ │ │ │ │ │ └── index.jsx │ │ │ │ └── index.jsx │ │ │ ├── OnboardingFlow/ │ │ │ │ ├── Steps/ │ │ │ │ │ ├── DataHandling/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── Home/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── LLMPreference/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── Survey/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── UserSetup/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ └── index.jsx │ │ │ │ └── index.jsx │ │ │ ├── WorkspaceChat/ │ │ │ │ └── index.jsx │ │ │ └── WorkspaceSettings/ │ │ │ ├── AgentConfig/ │ │ │ │ ├── AgentLLMSelection/ │ │ │ │ │ ├── AgentLLMItem/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ └── index.jsx │ │ │ │ ├── AgentModelSelection/ │ │ │ │ │ └── index.jsx │ │ │ │ └── index.jsx │ │ │ ├── ChatSettings/ │ │ │ │ ├── ChatHistorySettings/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── ChatModeSelection/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── ChatPromptSettings/ │ │ │ │ │ ├── ChatPromptHistory/ │ │ │ │ │ │ ├── PromptHistoryItem/ │ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ │ └── index.jsx │ │ │ │ │ └── index.jsx │ │ │ │ ├── ChatQueryRefusalResponse/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── ChatTemperatureSettings/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── WorkspaceLLMSelection/ │ │ │ │ │ ├── ChatModelSelection/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ ├── WorkspaceLLMItem/ │ │ │ │ │ │ └── index.jsx │ │ │ │ │ └── index.jsx │ │ │ │ └── index.jsx │ │ │ ├── GeneralAppearance/ │ │ │ │ ├── DeleteWorkspace/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── SuggestedChatMessages/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── WorkspaceName/ │ │ │ │ │ └── index.jsx │ │ │ │ └── index.jsx │ │ │ ├── Members/ │ │ │ │ ├── AddMemberModal/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── WorkspaceMemberRow/ │ │ │ │ │ └── index.jsx │ │ │ │ └── index.jsx │ │ │ ├── VectorDatabase/ │ │ │ │ ├── DocumentSimilarityThreshold/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── MaxContextSnippets/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── ResetDatabase/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── VectorCount/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── VectorDBIdentifier/ │ │ │ │ │ └── index.jsx │ │ │ │ ├── VectorSearchMode/ │ │ │ │ │ └── index.jsx │ │ │ │ └── index.jsx │ │ │ └── index.jsx │ │ └── utils/ │ │ ├── chat/ │ │ │ ├── agent.js │ │ │ ├── hljs-libraries/ │ │ │ │ └── svelte.js │ │ │ ├── index.js │ │ │ ├── markdown.js │ │ │ ├── plugins/ │ │ │ │ └── markdown-katex.js │ │ │ ├── purify.js │ │ │ └── themes/ │ │ │ ├── github-dark.css │ │ │ └── github.css │ │ ├── constants.js │ │ ├── directories.js │ │ ├── keyboardShortcuts.js │ │ ├── numbers.js │ │ ├── paths.js │ │ ├── piperTTS/ │ │ │ ├── index.js │ │ │ └── worker.js │ │ ├── request.js │ │ ├── session.js │ │ ├── toast.js │ │ ├── types.js │ │ └── username.js │ ├── tailwind.config.js │ └── vite.config.js ├── locales/ │ ├── README.fa-IR.md │ ├── README.ja-JP.md │ ├── README.tr-TR.md │ └── README.zh-CN.md ├── package.json ├── pull_request_template.md └── server/ ├── .env.example ├── .flowconfig ├── .gitignore ├── .nvmrc ├── __tests__/ │ ├── models/ │ │ ├── systemPromptVariables.test.js │ │ └── user.test.js │ └── utils/ │ ├── SQLConnectors/ │ │ └── connectionParser.test.js │ ├── TextSplitter/ │ │ └── index.test.js │ ├── agentFlows/ │ │ └── executor.test.js │ ├── agents/ │ │ ├── aibitat/ │ │ │ └── providers/ │ │ │ └── helpers/ │ │ │ └── untooled.test.js │ │ └── defaults.test.js │ ├── chats/ │ │ ├── openaiCompatible.test.js │ │ └── openaiHelpers.test.js │ ├── helpers/ │ │ ├── azureOpenAiModelPref.test.js │ │ └── convertTo.test.js │ ├── safeJSONStringify/ │ │ └── safeJSONStringify.test.js │ └── vectorDbProviders/ │ └── pgvector/ │ └── index.test.js ├── endpoints/ │ ├── admin.js │ ├── agentFlows.js │ ├── agentWebsocket.js │ ├── api/ │ │ ├── admin/ │ │ │ └── index.js │ │ ├── auth/ │ │ │ └── index.js │ │ ├── document/ │ │ │ └── index.js │ │ ├── embed/ │ │ │ └── index.js │ │ ├── index.js │ │ ├── openai/ │ │ │ ├── compatibility-test-script.cjs │ │ │ ├── helpers.js │ │ │ └── index.js │ │ ├── system/ │ │ │ └── index.js │ │ ├── userManagement/ │ │ │ └── index.js │ │ ├── workspace/ │ │ │ └── index.js │ │ └── workspaceThread/ │ │ └── index.js │ ├── browserExtension.js │ ├── chat.js │ ├── communityHub.js │ ├── document.js │ ├── embed/ │ │ └── index.js │ ├── embedManagement.js │ ├── experimental/ │ │ ├── imported-agent-plugins.js │ │ ├── index.js │ │ └── liveSync.js │ ├── extensions/ │ │ └── index.js │ ├── invite.js │ ├── mcpServers.js │ ├── mobile/ │ │ ├── index.js │ │ ├── middleware/ │ │ │ └── index.js │ │ └── utils/ │ │ └── index.js │ ├── system.js │ ├── utils/ │ │ ├── dockerModelRunnerUtils.js │ │ └── lemonadeUtilsEndpoints.js │ ├── utils.js │ ├── webPush.js │ ├── workspaceThreads.js │ ├── workspaces.js │ └── workspacesParsedFiles.js ├── eslint.config.mjs ├── index.js ├── jobs/ │ ├── cleanup-orphan-documents.js │ ├── helpers/ │ │ └── index.js │ └── sync-watched-documents.js ├── jsconfig.json ├── middleware/ │ └── httpLogger.js ├── models/ │ ├── apiKeys.js │ ├── browserExtensionApiKey.js │ ├── cacheData.js │ ├── communityHub.js │ ├── documentSyncQueue.js │ ├── documentSyncRun.js │ ├── documents.js │ ├── embedChats.js │ ├── embedConfig.js │ ├── eventLogs.js │ ├── invite.js │ ├── mobileDevice.js │ ├── passwordRecovery.js │ ├── promptHistory.js │ ├── slashCommandsPresets.js │ ├── systemPromptVariables.js │ ├── systemSettings.js │ ├── telemetry.js │ ├── temporaryAuthToken.js │ ├── user.js │ ├── vectors.js │ ├── workspace.js │ ├── workspaceAgentInvocation.js │ ├── workspaceChats.js │ ├── workspaceParsedFiles.js │ ├── workspaceThread.js │ ├── workspaceUsers.js │ └── workspacesSuggestedMessages.js ├── nodemon.json ├── package.json ├── prisma/ │ ├── migrations/ │ │ ├── 20230921191814_init/ │ │ │ └── migration.sql │ │ ├── 20231101001441_init/ │ │ │ └── migration.sql │ │ ├── 20231101195421_init/ │ │ │ └── migration.sql │ │ ├── 20231129012019_add/ │ │ │ └── migration.sql │ │ ├── 20240113013409_init/ │ │ │ └── migration.sql │ │ ├── 20240118201333_init/ │ │ │ └── migration.sql │ │ ├── 20240202002020_init/ │ │ │ └── migration.sql │ │ ├── 20240206181106_init/ │ │ │ └── migration.sql │ │ ├── 20240206211916_init/ │ │ │ └── migration.sql │ │ ├── 20240208224848_init/ │ │ │ └── migration.sql │ │ ├── 20240210004405_init/ │ │ │ └── migration.sql │ │ ├── 20240216214639_init/ │ │ │ └── migration.sql │ │ ├── 20240219211018_init/ │ │ │ └── migration.sql │ │ ├── 20240301002308_init/ │ │ │ └── migration.sql │ │ ├── 20240326231053_init/ │ │ │ └── migration.sql │ │ ├── 20240405015034_init/ │ │ │ └── migration.sql │ │ ├── 20240412183346_init/ │ │ │ └── migration.sql │ │ ├── 20240425004220_init/ │ │ │ └── migration.sql │ │ ├── 20240430230707_init/ │ │ │ └── migration.sql │ │ ├── 20240510032311_init/ │ │ │ └── migration.sql │ │ ├── 20240618224346_init/ │ │ │ └── migration.sql │ │ ├── 20240821215625_init/ │ │ │ └── migration.sql │ │ ├── 20240824005054_init/ │ │ │ └── migration.sql │ │ ├── 20241003192954_init/ │ │ │ └── migration.sql │ │ ├── 20241029203722_init/ │ │ │ └── migration.sql │ │ ├── 20241029233509_init/ │ │ │ └── migration.sql │ │ ├── 20250102204948_init/ │ │ │ └── migration.sql │ │ ├── 20250226005538_init/ │ │ │ └── migration.sql │ │ ├── 20250318154720_init/ │ │ │ └── migration.sql │ │ ├── 20250506214129_init/ │ │ │ └── migration.sql │ │ ├── 20250709230835_init/ │ │ │ └── migration.sql │ │ ├── 20250725194841_init/ │ │ │ └── migration.sql │ │ ├── 20250808171557_init/ │ │ │ └── migration.sql │ │ ├── 20260130040204_init/ │ │ │ └── migration.sql │ │ ├── 20260313192859_init/ │ │ │ └── migration.sql │ │ └── migration_lock.toml │ ├── schema.prisma │ └── seed.js ├── storage/ │ ├── README.md │ └── models/ │ ├── .gitignore │ ├── README.md │ └── downloaded/ │ └── .placeholder ├── swagger/ │ ├── dark-swagger.css │ ├── index.css │ ├── index.js │ ├── init.js │ ├── openapi.json │ └── utils.js └── utils/ ├── AiProviders/ │ ├── anthropic/ │ │ └── index.js │ ├── apipie/ │ │ └── index.js │ ├── azureOpenAi/ │ │ └── index.js │ ├── bedrock/ │ │ ├── index.js │ │ └── utils.js │ ├── cohere/ │ │ └── index.js │ ├── cometapi/ │ │ ├── constants.js │ │ └── index.js │ ├── deepseek/ │ │ └── index.js │ ├── dellProAiStudio/ │ │ └── index.js │ ├── dockerModelRunner/ │ │ └── index.js │ ├── fireworksAi/ │ │ └── index.js │ ├── foundry/ │ │ └── index.js │ ├── gemini/ │ │ ├── defaultModels.js │ │ ├── index.js │ │ └── syncStaticLists.mjs │ ├── genericOpenAi/ │ │ └── index.js │ ├── giteeai/ │ │ └── index.js │ ├── groq/ │ │ └── index.js │ ├── huggingface/ │ │ └── index.js │ ├── koboldCPP/ │ │ └── index.js │ ├── lemonade/ │ │ └── index.js │ ├── liteLLM/ │ │ └── index.js │ ├── lmStudio/ │ │ └── index.js │ ├── localAi/ │ │ └── index.js │ ├── mistral/ │ │ └── index.js │ ├── modelMap/ │ │ ├── index.js │ │ └── legacy.js │ ├── moonshotAi/ │ │ └── index.js │ ├── novita/ │ │ └── index.js │ ├── nvidiaNim/ │ │ └── index.js │ ├── ollama/ │ │ └── index.js │ ├── openAi/ │ │ └── index.js │ ├── openRouter/ │ │ └── index.js │ ├── perplexity/ │ │ ├── index.js │ │ ├── models.js │ │ └── scripts/ │ │ ├── .gitignore │ │ ├── chat_models.txt │ │ └── parse.mjs │ ├── ppio/ │ │ └── index.js │ ├── privatemode/ │ │ └── index.js │ ├── sambanova/ │ │ └── index.js │ ├── textGenWebUI/ │ │ └── index.js │ ├── togetherAi/ │ │ └── index.js │ ├── xai/ │ │ └── index.js │ └── zai/ │ └── index.js ├── BackgroundWorkers/ │ └── index.js ├── DocumentManager/ │ └── index.js ├── EmbeddingEngines/ │ ├── azureOpenAi/ │ │ └── index.js │ ├── cohere/ │ │ └── index.js │ ├── gemini/ │ │ └── index.js │ ├── genericOpenAi/ │ │ └── index.js │ ├── lemonade/ │ │ └── index.js │ ├── liteLLM/ │ │ └── index.js │ ├── lmstudio/ │ │ └── index.js │ ├── localAi/ │ │ └── index.js │ ├── mistral/ │ │ └── index.js │ ├── native/ │ │ ├── constants.js │ │ └── index.js │ ├── ollama/ │ │ └── index.js │ ├── openAi/ │ │ └── index.js │ ├── openRouter/ │ │ └── index.js │ └── voyageAi/ │ └── index.js ├── EmbeddingRerankers/ │ └── native/ │ └── index.js ├── EncryptionManager/ │ └── index.js ├── MCP/ │ ├── hypervisor/ │ │ └── index.js │ └── index.js ├── PasswordRecovery/ │ └── index.js ├── PushNotifications/ │ └── index.js ├── TextSplitter/ │ └── index.js ├── TextToSpeech/ │ ├── elevenLabs/ │ │ └── index.js │ ├── index.js │ ├── openAi/ │ │ └── index.js │ └── openAiGeneric/ │ └── index.js ├── agentFlows/ │ ├── executor.js │ ├── executors/ │ │ ├── api-call.js │ │ ├── llm-instruction.js │ │ └── web-scraping.js │ ├── flowTypes.js │ └── index.js ├── agents/ │ ├── aibitat/ │ │ ├── error.js │ │ ├── example/ │ │ │ ├── .gitignore │ │ │ ├── beginner-chat.js │ │ │ ├── blog-post-coding.js │ │ │ └── websocket/ │ │ │ ├── index.html │ │ │ ├── websock-branding-collab.js │ │ │ └── websock-multi-turn-chat.js │ │ ├── index.js │ │ ├── plugins/ │ │ │ ├── chat-history.js │ │ │ ├── cli.js │ │ │ ├── file-history.js │ │ │ ├── http-socket.js │ │ │ ├── index.js │ │ │ ├── memory.js │ │ │ ├── rechart.js │ │ │ ├── save-file-browser.js │ │ │ ├── sql-agent/ │ │ │ │ ├── SQLConnectors/ │ │ │ │ │ ├── MSSQL.js │ │ │ │ │ ├── MySQL.js │ │ │ │ │ ├── Postgresql.js │ │ │ │ │ ├── index.js │ │ │ │ │ └── utils.js │ │ │ │ ├── get-table-schema.js │ │ │ │ ├── index.js │ │ │ │ ├── list-database.js │ │ │ │ ├── list-table.js │ │ │ │ └── query.js │ │ │ ├── summarize.js │ │ │ ├── web-browsing.js │ │ │ ├── web-scraping.js │ │ │ └── websocket.js │ │ ├── providers/ │ │ │ ├── ai-provider.js │ │ │ ├── anthropic.js │ │ │ ├── apipie.js │ │ │ ├── azure.js │ │ │ ├── bedrock.js │ │ │ ├── cohere.js │ │ │ ├── cometapi.js │ │ │ ├── deepseek.js │ │ │ ├── dellProAiStudio.js │ │ │ ├── dockerModelRunner.js │ │ │ ├── fireworksai.js │ │ │ ├── foundry.js │ │ │ ├── gemini.js │ │ │ ├── genericOpenAi.js │ │ │ ├── giteeai.js │ │ │ ├── groq.js │ │ │ ├── helpers/ │ │ │ │ ├── classes.js │ │ │ │ ├── tooled.js │ │ │ │ └── untooled.js │ │ │ ├── index.js │ │ │ ├── koboldcpp.js │ │ │ ├── lemonade.js │ │ │ ├── litellm.js │ │ │ ├── lmstudio.js │ │ │ ├── localai.js │ │ │ ├── mistral.js │ │ │ ├── moonshotAi.js │ │ │ ├── novita.js │ │ │ ├── nvidiaNim.js │ │ │ ├── ollama.js │ │ │ ├── openai.js │ │ │ ├── openrouter.js │ │ │ ├── perplexity.js │ │ │ ├── ppio.js │ │ │ ├── privatemode.js │ │ │ ├── sambanova.js │ │ │ ├── textgenwebui.js │ │ │ ├── togetherai.js │ │ │ ├── xai.js │ │ │ └── zai.js │ │ └── utils/ │ │ ├── dedupe.js │ │ ├── summarize.js │ │ └── toolReranker.js │ ├── defaults.js │ ├── ephemeral.js │ ├── imported-manifest.schema.json │ ├── imported.js │ └── index.js ├── boot/ │ ├── MetaGenerator.js │ ├── eagerLoadContextWindows.js │ ├── index.js │ └── markOnboarded.js ├── chats/ │ ├── agents.js │ ├── apiChatHandler.js │ ├── commands/ │ │ └── reset.js │ ├── embed.js │ ├── index.js │ ├── openaiCompatible.js │ └── stream.js ├── collectorApi/ │ └── index.js ├── comKey/ │ └── index.js ├── database/ │ └── index.js ├── files/ │ ├── index.js │ ├── logo.js │ ├── multer.js │ ├── pfp.js │ └── purgeDocument.js ├── helpers/ │ ├── admin/ │ │ └── index.js │ ├── camelcase.js │ ├── chat/ │ │ ├── LLMPerformanceMonitor.js │ │ ├── convertTo.js │ │ ├── index.js │ │ └── responses.js │ ├── customModels.js │ ├── index.js │ ├── portAvailabilityChecker.js │ ├── search.js │ ├── shell.js │ ├── tiktoken.js │ └── updateENV.js ├── http/ │ └── index.js ├── logger/ │ └── index.js ├── middleware/ │ ├── chatHistoryViewable.js │ ├── communityHubDownloadsEnabled.js │ ├── embedMiddleware.js │ ├── featureFlagEnabled.js │ ├── isSupportedRepoProviders.js │ ├── multiUserProtected.js │ ├── simpleSSOEnabled.js │ ├── validApiKey.js │ ├── validBrowserExtensionApiKey.js │ ├── validWorkspace.js │ └── validatedRequest.js ├── prisma/ │ ├── PRISMA.md │ └── index.js ├── telemetry/ │ └── index.js ├── vectorDbProviders/ │ ├── astra/ │ │ ├── ASTRA_SETUP.md │ │ └── index.js │ ├── base.js │ ├── chroma/ │ │ └── index.js │ ├── chromacloud/ │ │ └── index.js │ ├── lance/ │ │ └── index.js │ ├── milvus/ │ │ ├── MILVUS_SETUP.md │ │ └── index.js │ ├── pgvector/ │ │ ├── SETUP.md │ │ └── index.js │ ├── pinecone/ │ │ ├── PINECONE_SETUP.md │ │ └── index.js │ ├── qdrant/ │ │ ├── QDRANT_SETUP.md │ │ └── index.js │ ├── weaviate/ │ │ ├── WEAVIATE_SETUP.md │ │ └── index.js │ └── zilliz/ │ └── index.js └── vectorStore/ └── resetAllVectorStores.js ================================================ FILE CONTENTS ================================================ ================================================ FILE: .devcontainer/README.md ================================================ # AnythingLLM Development Container Setup Welcome to the AnythingLLM development container configuration, designed to create a seamless and feature-rich development environment for this project.

PLEASE READ THIS

## Prerequisites - [Docker](https://www.docker.com/get-started) - [Visual Studio Code](https://code.visualstudio.com/) - [Remote - Containers](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) VS Code extension ## Features - **Base Image**: Built on `mcr.microsoft.com/devcontainers/javascript-node:1-18-bookworm`, thus Node.JS LTS v18. - **Additional Tools**: Includes `hadolint`, and essential apt-packages such as `curl`, `gnupg`, and more. - **Ports**: Configured to auto-forward ports `3000` (Frontend) and `3001` (Backend). - **Environment Variables**: Sets `NODE_ENV` to `development` and `ESLINT_USE_FLAT_CONFIG` to `true`. - **VS Code Extensions**: A suite of extensions such as `Prettier`, `Docker`, `ESLint`, and more are automatically installed. Please revise if you do not agree with any of these extensions. AI-powered extensions and time trackers are (for now) not included to avoid any privacy concerns, but you can install them later in your own environment. ## Getting Started 1. Using GitHub Codespaces. Just select to create a new workspace, and the devcontainer will be created for you. 2. Using your Local VSCode (Release or Insiders). We suggest you first make a fork of the repo and then clone it to your local machine using VSCode tools. Then open the project folder in VSCode, which will prompt you to open the project in a devcontainer. Select yes, and the devcontainer will be created for you. If this does not happen, you can open the command palette and select "Remote-Containers: Reopen in Container". ## On Creation: When the container is built for the first time, it will automatically run `yarn setup` to ensure everything is in place for the Collector, Server and Frontend. This command is expected to be automatically re-run if there is a content change on next reboot. ## Work in the Container: Once the container is up, be patient. Some extensions may complain because dependencies are still being installed, and in the Extensions tab, some may ask you to "Reload" the project. Don't do that yet. First, wait until all settle down for the first time. We suggest you create a new VSCode profile for this devcontainer, so any configuration and extensions you change, won't affect your default profile. Checklist: - [ ] The usual message asking you to start the Server and Frontend in different windows are now "hidden" in the building process of the devcontainer. Don't forget to do as suggested. - [ ] Open a JavaScript file, for example "server/index.js" and check if `eslint` is working. It will complain that `'err' is defined but never used.`. This means it is working. - [ ] Open a React File, for example, "frontend/src/main.jsx," and check if `eslint` complains about `Fast refresh only works when a file has exports. Move your component(s) to a separate file.`. Again, it means `eslint` is working. Now check at the status bar if the `Prettier` has a double checkmark :heavy_check_mark: (double). It means Prettier is working. You will see a nice extension `Formatting:`:heavy_check_mark: that can be used to disable the `Format on Save` feature temporarily. - [ ] Check if, on the left pane, you have the NPM Scripts (this may be disabled; look at the "Explorer" tree-dots up-right). There will be scripts inside the `package.json` files. You will basically need to run the `dev:collector`, `dev:server` and the `dev:frontend` in this order. When the frontend finishes starting, a window browser will open **inside** the VSCode. Still, you can open it outside. :warning: **Important for all developers** :warning: - [ ] When you are using the `NODE_ENV=development` the server will not store the configurations you set for security reasons. Please set the proper config on file `.env.development`. The side-effect if you don't, everytime you restart the server, you will be sent to the "Onboarding" page again. **Note when using GitHub Codespaces** - [ ] When running the "Server" for the first time, it will automatically configure its port to be publicly accessible by default, as this is required for the front end to reach the server backend. To know more, read the content of the `.env` file on the frontend folder about this, and if any issues occur, make sure to manually set the port "Visibility" of the "Server" is set to "Public" if needed. Again, this is only needed for developing on GitHub Codespaces. **For the Collector:** - [x] In the past, the Collector dwelled within the Python domain, but now it has journeyed to the splendid realm of Node.JS. Consequently, the configuration complexities of bygone versions are no longer a concern. ### Now it is ready to start In the status bar you will see three shortcuts names `Collector`, `Server` and `Frontend`. Just click-and-wait on that order (don't forget to set the Server port 3001 to Public if you are using GH Codespaces **_before_** starting the Frontend). Now you can enjoy your time developing instead of reconfiguring everything. ## Debugging with the devcontainers ### For debugging the collector, server and frontend First, make sure the built-in extension (ms-vscode.js-debug) is active (I don't know why it would not be, but just in case). If you want, you can install the nightly version (ms-vscode.js-debug-nightly) Then, in the "Run and Debug" tab (Ctrl+shift+D), you can select on the menu: - Collector debug. This will start the collector in debug mode and attach the debugger. Works very well. - Server debug. This will start the server in debug mode and attach the debugger. Works very well. - Frontend debug. This will start the frontend in debug mode and attach the debugger. I am still struggling with this one. I don't know if VSCode can handle the .jsx files seamlessly as the pure .js on the server. Maybe there is a need for a particular configuration for Vite or React. Anyway, it starts. Another two configurations launch Chrome and Edge, and I think we could add breakpoints on .jsx files somehow. The best scenario would be always to use the embedded browser. WIP. Please leave comments on the Issues tab or the [![](https://img.shields.io/discord/1114740394715004990?logo=Discord&logoColor=white&label=Discord&labelColor=%235568ee&color=%2355A2DD&link=https%3A%2F%2Fdiscord.gg%2F6UyHPeGZAC)]("https://discord.gg/6UyHPeGZAC") ================================================ FILE: .devcontainer/devcontainer.json ================================================ // For format details, see https://aka.ms/devcontainer.json. For config options, see the // README at: https://github.com/devcontainers/templates/tree/main/src/javascript-node { "name": "Node.js", // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile // "build": { // "args": { // "ARG_UID": "1000", // "ARG_GID": "1000" // }, // "dockerfile": "Dockerfile" // }, // "containerUser": "anythingllm", // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile "image": "mcr.microsoft.com/devcontainers/javascript-node:1-18-bookworm", "forwardPorts": [3001, 3000], // Features to add to the dev container. More info: https://containers.dev/features. "features": { // Docker very useful linter "ghcr.io/dhoeric/features/hadolint:1": { "version": "latest" }, // Terraform support "ghcr.io/devcontainers/features/terraform:1": {}, // Just a wrap to install needed packages "ghcr.io/devcontainers-contrib/features/apt-packages:1": { // Dependencies copied from ../docker/Dockerfile plus some dev stuff "packages": [ "build-essential", "ca-certificates", "curl", "ffmpeg", "fonts-liberation", "git", "gnupg", "htop", "less", "libappindicator1", "libasound2", "libatk-bridge2.0-0", "libatk1.0-0", "libc6", "libcairo2", "libcups2", "libdbus-1-3", "libexpat1", "libfontconfig1", "libgbm1", "libgcc1", "libgfortran5", "libglib2.0-0", "libgtk-3-0", "libnspr4", "libnss3", "libpango-1.0-0", "libpangocairo-1.0-0", "libstdc++6", "libx11-6", "libx11-xcb1", "libxcb1", "libxcomposite1", "libxcursor1", "libxdamage1", "libxext6", "libxfixes3", "libxi6", "libxrandr2", "libxrender1", "libxss1", "libxtst6", "locales", "lsb-release", "procps", "tzdata", "wget", "xdg-utils" ] } }, "updateContentCommand": "cd server && yarn && cd ../collector && PUPPETEER_DOWNLOAD_BASE_URL=https://storage.googleapis.com/chrome-for-testing-public yarn && cd ../frontend && yarn && cd .. && yarn setup:envs && yarn prisma:setup && echo \"Please run yarn dev:server, yarn dev:collector, and yarn dev:frontend in separate terminal tabs.\"", // Use 'postCreateCommand' to run commands after the container is created. // This configures VITE for github codespaces and installs gh cli "postCreateCommand": "if [ \"${CODESPACES}\" = \"true\" ]; then echo 'VITE_API_BASE=\"https://$CODESPACE_NAME-3001.$GITHUB_CODESPACES_PORT_FORWARDING_DOMAIN/api\"' > ./frontend/.env && (type -p wget >/dev/null || (sudo apt update && sudo apt-get install wget -y)) && sudo mkdir -p -m 755 /etc/apt/keyrings && wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null && sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg && echo \"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main\" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null && sudo apt update && sudo apt install gh -y; fi", "portsAttributes": { "3001": { "label": "Backend", "onAutoForward": "notify" }, "3000": { "label": "Frontend", "onAutoForward": "openPreview" } }, "capAdd": [ "SYS_ADMIN" // needed for puppeteer using headless chrome in sandbox ], "remoteEnv": { "NODE_ENV": "development", "ESLINT_USE_FLAT_CONFIG": "true", "ANYTHING_LLM_RUNTIME": "docker" }, // "initializeCommand": "echo Initialize....", "shutdownAction": "stopContainer", // Configure tool-specific properties. "customizations": { "codespaces": { "openFiles": [ "README.md", ".devcontainer/README.md" ] }, "vscode": { "openFiles": [ "README.md", ".devcontainer/README.md" ], "extensions": [ "bierner.github-markdown-preview", "bradlc.vscode-tailwindcss", "dbaeumer.vscode-eslint", "editorconfig.editorconfig", "esbenp.prettier-vscode", "exiasr.hadolint", "flowtype.flow-for-vscode", "gamunu.vscode-yarn", "hashicorp.terraform", "mariusschulz.yarn-lock-syntax", "ms-azuretools.vscode-docker", "streetsidesoftware.code-spell-checker", "actboy168.tasks", "tombonnike.vscode-status-bar-format-toggle", "ms-vscode.js-debug" ], "settings": { "[css]": { "editor.defaultFormatter": "esbenp.prettier-vscode" }, "[dockercompose]": { "editor.defaultFormatter": "esbenp.prettier-vscode" }, "[dockerfile]": { "editor.defaultFormatter": "ms-azuretools.vscode-docker" }, "[html]": { "editor.defaultFormatter": "esbenp.prettier-vscode" }, "[javascript]": { "editor.defaultFormatter": "esbenp.prettier-vscode" }, "[javascriptreact]": { "editor.defaultFormatter": "esbenp.prettier-vscode" }, "[json]": { "editor.defaultFormatter": "esbenp.prettier-vscode" }, "[jsonc]": { "editor.defaultFormatter": "esbenp.prettier-vscode" }, "[markdown]": { "editor.defaultFormatter": "esbenp.prettier-vscode" }, "[postcss]": { "editor.defaultFormatter": "esbenp.prettier-vscode" }, "[toml]": { "editor.defaultFormatter": "tamasfe.even-better-toml" }, "eslint.debug": true, "eslint.enable": true, "eslint.experimental.useFlatConfig": true, "eslint.run": "onSave", "files.associations": { ".*ignore": "ignore", ".editorconfig": "editorconfig", ".env*": "properties", ".flowconfig": "ini", ".prettierrc": "json", "*.css": "tailwindcss", "*.md": "markdown", "*.sh": "shellscript", "docker-compose.*": "dockercompose", "Dockerfile*": "dockerfile", "yarn.lock": "yarnlock" }, "javascript.format.enable": false, "javascript.inlayHints.enumMemberValues.enabled": true, "javascript.inlayHints.functionLikeReturnTypes.enabled": true, "javascript.inlayHints.parameterTypes.enabled": true, "javascript.inlayHints.variableTypes.enabled": true, "js/ts.implicitProjectConfig.module": "CommonJS", "json.format.enable": false, "json.schemaDownload.enable": true, "npm.autoDetect": "on", "npm.packageManager": "yarn", "prettier.useEditorConfig": false, "tailwindCSS.files.exclude": [ "**/.git/**", "**/node_modules/**", "**/.hg/**", "**/.svn/**", "**/dist/**" ], "typescript.validate.enable": false, "workbench.editorAssociations": { "*.md": "vscode.markdown.preview.editor" } } } } // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root. // "remoteUser": "root" } ================================================ FILE: .dockerignore ================================================ **/server/utils/agents/aibitat/example/** **/server/storage/documents/** **/server/storage/vector-cache/** **/server/storage/*.db **/server/storage/lancedb **/collector/hotdir/** **/collector/outputs/** **/node_modules/ **/dist/ **/v-env/ **/__pycache__/ **/.env **/.env.* **/bundleinspector.html **/tmp/** **/.log !docker/.env.example !frontend/.env.production ================================================ FILE: .editorconfig ================================================ # EditorConfig is awesome: https://EditorConfig.org # top-most EditorConfig file root = true [*] # Non-configurable Prettier behaviors charset = utf-8 insert_final_newline = true trim_trailing_whitespace = true # Configurable Prettier behaviors # (change these if your Prettier config differs) end_of_line = lf indent_style = space indent_size = 2 max_line_length = 80 ================================================ FILE: .gitattributes ================================================ * text=auto eol=lf ================================================ FILE: .github/FUNDING.yml ================================================ github: Mintplex-Labs ================================================ FILE: .github/ISSUE_TEMPLATE/01_bug.yml ================================================ name: 🐛 Bug Report description: File a bug report for AnythingLLM title: "[BUG]: " labels: [possible bug] body: - type: markdown attributes: value: | Use this template to file a bug report for AnythingLLM. Please be as descriptive as possible to allow everyone to replicate and solve your issue. - type: dropdown id: runtime attributes: label: How are you running AnythingLLM? description: AnythingLLM can be run in many environments, pick the one that best represents where you encounter the bug. options: - Docker (local) - Docker (remote machine) - Local development - AnythingLLM desktop app - All versions - Not listed default: 0 validations: required: true - type: textarea id: what-happened attributes: label: What happened? description: Also tell us, what did you expect to happen? validations: required: true - type: textarea id: reproduction attributes: label: Are there known steps to reproduce? description: | Let us know how to reproduce the bug and we may be able to fix it more quickly. This is not required, but it is helpful. validations: required: false ================================================ FILE: .github/ISSUE_TEMPLATE/02_feature.yml ================================================ name: ✨ New Feature suggestion description: Suggest a new feature for AnythingLLM! title: "[FEAT]: " labels: [enhancement, feature request] body: - type: markdown attributes: value: | Share a new idea for a feature or improvement. Be sure to search existing issues first to avoid duplicates. - type: textarea id: description attributes: label: What would you like to see? description: | Describe the feature and why it would be useful to your use-case as well as others. validations: required: true ================================================ FILE: .github/ISSUE_TEMPLATE/03_documentation.yml ================================================ name: 📚 Documentation improvement title: "[DOCS]: " description: Report an issue or problem with the documentation. labels: [documentation] body: - type: textarea id: description attributes: label: Description description: Describe the issue with the documentation that is giving you trouble or causing confusion. validations: required: true ================================================ FILE: .github/ISSUE_TEMPLATE/config.yml ================================================ blank_issues_enabled: true contact_links: - name: 🧑‍🤝‍🧑 Community Discord url: https://discord.gg/6UyHPeGZAC about: Interact with the Mintplex Labs community here by asking for help, discussing and more! ================================================ FILE: .github/workflows/build-and-push-image-semver.yaml ================================================ name: Publish AnythingLLM Docker image on Release (amd64 & arm64) concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true on: release: types: [published] jobs: push_multi_platform_to_registries: name: Push Docker multi-platform image to multiple registries runs-on: ubuntu-22.04-arm permissions: packages: write contents: read steps: - name: Check out the repo uses: actions/checkout@v4 - name: Check if DockerHub build needed shell: bash run: | # Check if the secret for USERNAME is set (don't even check for the password) if [[ -z "${{ secrets.DOCKER_USERNAME }}" ]]; then echo "DockerHub build not needed" echo "enabled=false" >> $GITHUB_OUTPUT else echo "DockerHub build needed" echo "enabled=true" >> $GITHUB_OUTPUT fi id: dockerhub - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 with: version: v0.22.0 - name: Log in to Docker Hub uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # Only login to the Docker Hub if the repo is mintplex/anythingllm, to allow for forks to build on GHCR if: steps.dockerhub.outputs.enabled == 'true' with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Log in to the Container registry uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Extract metadata (tags, labels) for Docker id: meta uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 with: images: | ${{ steps.dockerhub.outputs.enabled == 'true' && 'mintplexlabs/anythingllm' || '' }} ghcr.io/${{ github.repository }} tags: | type=semver,pattern={{version}} type=semver,pattern={{major}}.{{minor}} - name: Build and push multi-platform Docker image uses: docker/build-push-action@v6 with: context: . file: ./docker/Dockerfile push: true sbom: true provenance: mode=max platforms: linux/amd64,linux/arm64 tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha cache-to: type=gha,mode=max # For Docker scout there are some intermediary reported CVEs which exists outside # of execution content or are unreachable by an attacker but exist in image. # We create VEX files for these so they don't show in scout summary. - name: Collect known and verified CVE exceptions id: cve-list run: | # Collect CVEs from filenames in vex folder CVE_NAMES="" for file in ./docker/vex/*.vex.json; do [ -e "$file" ] || continue filename=$(basename "$file") stripped_filename=${filename%.vex.json} CVE_NAMES+=" $stripped_filename" done echo "CVE_EXCEPTIONS=$CVE_NAMES" >> $GITHUB_OUTPUT shell: bash # About VEX attestations https://docs.docker.com/scout/explore/exceptions/ # Justifications https://github.com/openvex/spec/blob/main/OPENVEX-SPEC.md#status-justifications - name: Add VEX attestations env: CVE_EXCEPTIONS: ${{ steps.cve-list.outputs.CVE_EXCEPTIONS }} run: | echo $CVE_EXCEPTIONS curl -sSfL https://raw.githubusercontent.com/docker/scout-cli/main/install.sh | sh -s -- for cve in $CVE_EXCEPTIONS; do for tag in "${{ join(fromJSON(steps.meta.outputs.json).tags, ' ') }}"; do echo "Attaching VEX exception $cve to $tag" docker scout attestation add \ --file "./docker/vex/$cve.vex.json" \ --predicate-type https://openvex.dev/ns/v0.2.0 \ $tag done done shell: bash ================================================ FILE: .github/workflows/build-and-push-image.yaml ================================================ # This GitHub action is for publishing of the primary image for AnythingLLM # It will publish a linux/amd64 and linux/arm64 image at the same time # This file should ONLY BY USED FOR `master` BRANCH. # TODO: GitHub now has an ubuntu-24.04-arm64 runner, but we still need # to use QEMU to build the arm64 image because Chromium is not available for Linux arm64 # so builds will still fail, or fail much more often. Its inconsistent and frustrating. name: Publish AnythingLLM Primary Docker image (amd64/arm64) concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true on: push: branches: ['master'] # master branch only. Do not modify. paths-ignore: - '**.md' - '.gitmodules' - 'cloud-deployments/**/*' - 'images/**/*' - '.vscode/**/*' - '**/.env.example' - '.github/ISSUE_TEMPLATE/**/*' - '.devcontainer/**/*' - 'embed/**/*' # Embed is submodule - 'browser-extension/**/*' # Chrome extension is submodule - 'server/utils/agents/aibitat/example/**/*' # Do not push new image for local dev testing of new aibitat images. - 'extras/**/*' # Extra is just for news and other local content. jobs: push_multi_platform_to_registries: name: Push Docker multi-platform image to multiple registries runs-on: ubuntu-22.04-arm permissions: packages: write contents: read steps: - name: Check out the repo uses: actions/checkout@v4 - name: Check if DockerHub build needed shell: bash run: | # Check if the secret for USERNAME is set (don't even check for the password) if [[ -z "${{ secrets.DOCKER_USERNAME }}" ]]; then echo "DockerHub build not needed" echo "enabled=false" >> $GITHUB_OUTPUT else echo "DockerHub build needed" echo "enabled=true" >> $GITHUB_OUTPUT fi id: dockerhub - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 with: version: v0.22.0 - name: Log in to Docker Hub uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # Only login to the Docker Hub if the repo is mintplex/anythingllm, to allow for forks to build on GHCR if: steps.dockerhub.outputs.enabled == 'true' with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Log in to the Container registry uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Extract metadata (tags, labels) for Docker id: meta uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 with: images: | ${{ steps.dockerhub.outputs.enabled == 'true' && 'mintplexlabs/anythingllm' || '' }} ghcr.io/${{ github.repository }} tags: | type=raw,value=latest,enable={{is_default_branch}} type=ref,event=branch type=ref,event=tag type=ref,event=pr - name: Build and push multi-platform Docker image uses: docker/build-push-action@v6 with: context: . file: ./docker/Dockerfile push: true sbom: true provenance: mode=max platforms: linux/amd64,linux/arm64 tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha cache-to: type=gha,mode=max # For Docker scout there are some intermediary reported CVEs which exists outside # of execution content or are unreachable by an attacker but exist in image. # We create VEX files for these so they don't show in scout summary. - name: Collect known and verified CVE exceptions id: cve-list run: | # Collect CVEs from filenames in vex folder CVE_NAMES="" for file in ./docker/vex/*.vex.json; do [ -e "$file" ] || continue filename=$(basename "$file") stripped_filename=${filename%.vex.json} CVE_NAMES+=" $stripped_filename" done echo "CVE_EXCEPTIONS=$CVE_NAMES" >> $GITHUB_OUTPUT shell: bash # About VEX attestations https://docs.docker.com/scout/explore/exceptions/ # Justifications https://github.com/openvex/spec/blob/main/OPENVEX-SPEC.md#status-justifications - name: Add VEX attestations env: CVE_EXCEPTIONS: ${{ steps.cve-list.outputs.CVE_EXCEPTIONS }} run: | echo $CVE_EXCEPTIONS curl -sSfL https://raw.githubusercontent.com/docker/scout-cli/main/install.sh | sh -s -- for cve in $CVE_EXCEPTIONS; do for tag in "${{ join(fromJSON(steps.meta.outputs.json).tags, ' ') }}"; do echo "Attaching VEX exception $cve to $tag" docker scout attestation add \ --file "./docker/vex/$cve.vex.json" \ --predicate-type https://openvex.dev/ns/v0.2.0 \ $tag done done shell: bash ================================================ FILE: .github/workflows/build-qa-tag.yaml ================================================ # Builds a QA GHCR image for a PR when the "PR: Ready for QA" label is present. # Triggers on: # - "PR: Ready for QA" label added to a PR # - New commits pushed to a PR that already has the label will trigger a new build name: Build QA GHCR Image on: pull_request: types: [labeled, synchronize] paths-ignore: - "**.md" - ".gitmodules" - "cloud-deployments/**/*" - "images/**/*" - ".vscode/**/*" - "**/.env.example" - ".github/ISSUE_TEMPLATE/**/*" - ".devcontainer/**/*" - "embed/**/*" - "browser-extension/**/*" - "extras/**/*" concurrency: group: qa-build-pr-${{ github.event.pull_request.number }} cancel-in-progress: true jobs: build: name: Build and push QA image for PR runs-on: ubuntu-22.04-arm # Run when labeled with "PR: Ready for QA" if: >- ${{ contains(github.event.pull_request.labels.*.name, 'PR: Ready for QA') }} permissions: packages: write contents: read pull-requests: write steps: - name: Check out the repo uses: actions/checkout@v4 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 with: version: v0.22.0 - name: Log in to GitHub Container Registry uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Set lowercase repository owner run: echo "REPO_OWNER_LC=${GITHUB_REPOSITORY_OWNER,,}" >> $GITHUB_ENV - name: Build and push uses: docker/build-push-action@v6 with: context: . file: ./docker/Dockerfile push: true sbom: true provenance: mode=max platforms: linux/arm64 tags: ghcr.io/${{ env.REPO_OWNER_LC }}/${{ github.event.repository.name }}:pr-${{ github.event.pull_request.number }} cache-from: type=gha cache-to: type=gha,mode=max ================================================ FILE: .github/workflows/check-package-versions.yaml ================================================ # This GitHub action is for checking the versions of the packages in the project. # Any package that is present in both the `server` and `collector` package.json file # is checked to ensure that they are the same version. name: Check package versions concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true on: pull_request: types: [opened, synchronize, reopened] paths: - "server/package.json" - "collector/package.json" jobs: run-script: runs-on: ubuntu-latest steps: - name: Checkout repository uses: actions/checkout@v2 - name: Set up Node.js uses: actions/setup-node@v3 with: node-version: '18' - name: Run verifyPackageVersions.mjs script run: | cd extras/scripts node verifyPackageVersions.mjs - name: Fail job on error if: failure() run: exit 1 ================================================ FILE: .github/workflows/check-translations.yaml ================================================ # This GitHub action is for validation of all languages which translations are offered for # in the locales folder in `frontend/src`. All languages are compared to the EN translation # schema since that is the fallback language setting. This workflow will run on all PRs that # modify any files in the translation directory name: Verify translations files concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true on: pull_request: types: [opened, synchronize, reopened] paths: - "frontend/src/locales/**.js" jobs: run-script: runs-on: ubuntu-latest steps: - name: Checkout repository uses: actions/checkout@v2 - name: Set up Node.js uses: actions/setup-node@v3 with: node-version: '18' - name: Run verifyTranslations.mjs script run: | cd frontend/src/locales node verifyTranslations.mjs - name: Fail job on error if: failure() run: exit 1 ================================================ FILE: .github/workflows/cleanup-qa-tag.yaml ================================================ # Cleans up the GHCR image tag when the PR is closed or the "PR: Ready for QA" label is removed. name: Cleanup QA GHCR Image on: pull_request: types: [closed, unlabeled] workflow_dispatch: inputs: pr_number: description: 'PR number to clean up (e.g., 123)' required: true jobs: cleanup-manual: name: Delete QA GHCR image tag (manual) runs-on: ubuntu-latest if: github.event_name == 'workflow_dispatch' permissions: packages: write steps: - name: Delete PR tag from GHCR env: GH_TOKEN: ${{ secrets.ALLM_RW_PACKAGES }} PR_NUMBER: ${{ inputs.pr_number }} run: | # Must use lowercase - packages are published with lowercase owner ORG_LC="${GITHUB_REPOSITORY_OWNER,,}" REPO_LC="${GITHUB_REPOSITORY#*/}" REPO_LC="${REPO_LC,,}" echo "Looking for tag: pr-${PR_NUMBER}" echo "Package: /orgs/${ORG_LC}/packages/container/${REPO_LC}/versions" VERSION_ID=$(gh api \ -H "Accept: application/vnd.github+json" \ --paginate \ "/orgs/${ORG_LC}/packages/container/${REPO_LC}/versions" \ --jq ".[] | select(.metadata.container.tags[] == \"pr-${PR_NUMBER}\") | .id") if [ -n "$VERSION_ID" ]; then echo "Deleting package version $VERSION_ID (tag: pr-${PR_NUMBER})" gh api \ --method DELETE \ -H "Accept: application/vnd.github+json" \ "/orgs/${ORG_LC}/packages/container/${REPO_LC}/versions/$VERSION_ID" else echo "No package found with tag pr-${PR_NUMBER}, skipping cleanup" fi cleanup-auto: name: Delete QA GHCR image tag (auto) runs-on: ubuntu-latest if: >- (github.event.action == 'closed' && contains(github.event.pull_request.labels.*.name, 'PR: Ready for QA')) || (github.event.action == 'unlabeled' && github.event.label.name == 'PR: Ready for QA') permissions: packages: write steps: - name: Delete PR tag from GHCR env: GH_TOKEN: ${{ secrets.ALLM_RW_PACKAGES }} PR_NUMBER: ${{ github.event.pull_request.number }} run: | # Must use lowercase - packages are published with lowercase owner ORG_LC="${GITHUB_REPOSITORY_OWNER,,}" REPO_LC="${GITHUB_REPOSITORY#*/}" REPO_LC="${REPO_LC,,}" VERSION_ID=$(gh api \ -H "Accept: application/vnd.github+json" \ --paginate \ "/orgs/${ORG_LC}/packages/container/${REPO_LC}/versions" \ --jq ".[] | select(.metadata.container.tags[] == \"pr-${PR_NUMBER}\") | .id" \ 2>/dev/null || true) if [ -n "$VERSION_ID" ]; then echo "Deleting package version $VERSION_ID (tag: pr-${PR_NUMBER})" gh api \ --method DELETE \ -H "Accept: application/vnd.github+json" \ "/orgs/${ORG_LC}/packages/container/${REPO_LC}/versions/$VERSION_ID" else echo "No package found with tag pr-${PR_NUMBER}, skipping cleanup" fi ================================================ FILE: .github/workflows/lint.yaml ================================================ name: Lint concurrency: group: lint-${{ github.ref }} cancel-in-progress: true on: pull_request: types: [opened, synchronize, reopened] paths: - "server/**/*.js" - "server/eslint.config.mjs" - "collector/**/*.js" - "collector/eslint.config.mjs" - "frontend/src/**/*.js" - "frontend/src/**/*.jsx" - "frontend/eslint.config.js" jobs: lint: runs-on: ubuntu-latest steps: - name: Checkout repository uses: actions/checkout@v4 - name: Set up Node.js uses: actions/setup-node@v4 with: node-version: "18" - name: Cache server dependencies uses: actions/cache@v4 with: path: server/node_modules key: ${{ runner.os }}-yarn-server-${{ hashFiles('server/yarn.lock') }} restore-keys: | ${{ runner.os }}-yarn-server- - name: Cache frontend dependencies uses: actions/cache@v4 with: path: frontend/node_modules key: ${{ runner.os }}-yarn-frontend-${{ hashFiles('frontend/yarn.lock') }} restore-keys: | ${{ runner.os }}-yarn-frontend- - name: Cache collector dependencies uses: actions/cache@v4 with: path: collector/node_modules key: ${{ runner.os }}-yarn-collector-${{ hashFiles('collector/yarn.lock') }} restore-keys: | ${{ runner.os }}-yarn-collector- - name: Install server dependencies run: cd server && yarn install --frozen-lockfile - name: Install frontend dependencies run: cd frontend && yarn install --frozen-lockfile - name: Install collector dependencies run: cd collector && yarn install --frozen-lockfile env: PUPPETEER_SKIP_DOWNLOAD: "true" SHARP_IGNORE_GLOBAL_LIBVIPS: "true" - name: Lint server run: cd server && yarn lint:check - name: Lint frontend run: cd frontend && yarn lint:check - name: Lint collector run: cd collector && yarn lint:check ================================================ FILE: .github/workflows/run-tests.yaml ================================================ name: Run backend tests concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true on: pull_request: types: [opened, synchronize, reopened] paths: - "server/**.js" - "collector/**.js" jobs: run-script: runs-on: ubuntu-latest steps: - name: Checkout repository uses: actions/checkout@v2 - name: Set up Node.js uses: actions/setup-node@v3 with: node-version: '18' - name: Cache root dependencies uses: actions/cache@v3 with: path: | node_modules ~/.cache/yarn key: ${{ runner.os }}-yarn-root-${{ hashFiles('**/yarn.lock') }} restore-keys: | ${{ runner.os }}-yarn-root- - name: Cache server dependencies uses: actions/cache@v3 with: path: | server/node_modules ~/.cache/yarn key: ${{ runner.os }}-yarn-server-${{ hashFiles('server/yarn.lock') }} restore-keys: | ${{ runner.os }}-yarn-server- - name: Cache collector dependencies uses: actions/cache@v3 with: path: | collector/node_modules ~/.cache/yarn key: ${{ runner.os }}-yarn-collector-${{ hashFiles('collector/yarn.lock') }} restore-keys: | ${{ runner.os }}-yarn-collector- - name: Install root dependencies if: steps.cache-root.outputs.cache-hit != 'true' run: yarn install --frozen-lockfile - name: Install server dependencies if: steps.cache-server.outputs.cache-hit != 'true' run: cd server && yarn install --frozen-lockfile - name: Install collector dependencies if: steps.cache-collector.outputs.cache-hit != 'true' run: cd collector && yarn install --frozen-lockfile env: PUPPETEER_SKIP_DOWNLOAD: "true" SHARP_IGNORE_GLOBAL_LIBVIPS: "true" - name: Setup environment and Prisma run: yarn setup:envs && yarn prisma:setup - name: Run test suites run: yarn test - name: Fail job on error if: failure() run: exit 1 ================================================ FILE: .github/workflows/sponsors.yaml ================================================ name: Generate Sponsors README on: schedule: - cron: "0 12 * * 3" # Run every Wednesday at 12:00 PM UTC permissions: contents: write jobs: deploy: runs-on: ubuntu-latest steps: - name: Checkout 🛎️ uses: actions/checkout@v2 - name: Generate All Sponsors README id: generate-all-sponsors uses: JamesIves/github-sponsors-readme-action@v1 with: token: ${{ secrets.SPONSOR_PAT }} file: 'README.md' organization: true active-only: false marker: 'all-sponsors' - name: Commit and Push 🚀 uses: stefanzweifel/git-auto-commit-action@v5 id: auto-commit-action with: commit_message: 'Update Sponsors README' file_pattern: 'README.md' - name: Generate PR if changes detected uses: peter-evans/create-pull-request@v7 if: steps.auto-commit-action.outputs.files_changed == 'true' with: token: ${{ secrets.GITHUB_TOKEN }} title: 'Update Sponsors README' branch: 'chore/update-sponsors' base: 'master' draft: false reviewers: 'timothycarambat' assignees: 'timothycarambat' maintainer-can-modify: true ================================================ FILE: .gitignore ================================================ v-env .env !.env.example node_modules __pycache__ v-env .DS_Store aws_cf_deploy_anything_llm.json yarn.lock *.bak .idea ================================================ FILE: .gitmodules ================================================ [submodule "browser-extension"] path = browser-extension url = https://github.com/Mintplex-Labs/anythingllm-extension.git [submodule "embed"] path = embed url = https://github.com/Mintplex-Labs/anythingllm-embed.git branch = main ================================================ FILE: .hadolint.yaml ================================================ failure-threshold: warning ignored: - DL3008 - DL3013 format: tty trustedRegistries: - docker.io - gcr.io ================================================ FILE: .nvmrc ================================================ v18.18.0 ================================================ FILE: .prettierignore ================================================ # defaults **/.git **/.svn **/.hg **/node_modules #frontend frontend/bundleinspector.html **/dist #server server/swagger/openapi.json server/**/*.mjs #embed **/static/** embed/src/utils/chat/hljs.js ================================================ FILE: .prettierrc ================================================ { "tabWidth": 2, "useTabs": false, "endOfLine": "lf", "semi": true, "singleQuote": false, "printWidth": 80, "trailingComma": "es5", "bracketSpacing": true, "bracketSameLine": false, "overrides": [ { "files": ["*.js", "*.mjs", "*.jsx"], "options": { "parser": "flow", "arrowParens": "always" } }, { "files": ["*.config.js"], "options": { "semi": false, "parser": "flow", "trailingComma": "none" } }, { "files": "*.html", "options": { "bracketSameLine": true } }, { "files": ".prettierrc", "options": { "parser": "json" } } ] } ================================================ FILE: .vscode/launch.json ================================================ { // Use IntelliSense to learn about possible attributes. // Hover to view descriptions of existing attributes. // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 "version": "0.2.0", "configurations": [ { "name": "Collector debug", "request": "launch", "cwd": "${workspaceFolder}/collector", "env": { "NODE_ENV": "development" }, "runtimeArgs": [ "index.js" ], // not using yarn/nodemon because it doesn't work with breakpoints // "runtimeExecutable": "yarn", "skipFiles": [ "/**" ], "type": "node" }, { "name": "Server debug", "request": "launch", "cwd": "${workspaceFolder}/server", "env": { "NODE_ENV": "development" }, "runtimeArgs": [ "index.js" ], // not using yarn/nodemon because it doesn't work with breakpoints // "runtimeExecutable": "yarn", "skipFiles": [ "/**" ], "type": "node" }, { "name": "Frontend debug", "request": "launch", "cwd": "${workspaceFolder}/frontend", "env": { "NODE_ENV": "development", }, "runtimeExecutable": "${workspaceFolder}/frontend/node_modules/.bin/vite", "runtimeArgs": [ "--debug", "--host=0.0.0.0" ], // "runtimeExecutable": "yarn", "skipFiles": [ "/**" ], "type": "node" }, { "name": "Launch Edge", "request": "launch", "type": "msedge", "url": "http://localhost:3000", "webRoot": "${workspaceFolder}" }, { "type": "chrome", "request": "launch", "name": "Launch Chrome against localhost", "url": "http://localhost:3000", "webRoot": "${workspaceFolder}" } ] } ================================================ FILE: .vscode/settings.json ================================================ { "cSpell.words": [ "adoc", "aibitat", "AIbitat", "allm", "anythingllm", "Apipie", "Astra", "Chartable", "cleancss", "comkey", "cooldown", "cooldowns", "datafile", "Deduplicator", "Dockerized", "docpath", "elevenlabs", "Embeddable", "epub", "fireworksai", "GROQ", "hljs", "huggingface", "inferencing", "koboldcpp", "Langchain", "lmstudio", "localai", "mbox", "Milvus", "Mintplex", "mixtral", "moderations", "novita", "numpages", "Ollama", "Oobabooga", "openai", "opendocument", "openrouter", "pagerender", "ppio", "Qdrant", "royalblue", "SearchApi", "searxng", "SerpApi", "Serper", "Serply", "streamable", "textgenwebui", "togetherai", "Unembed", "uuidv", "vectordbs", "Weaviate", "XAILLM", "Zilliz" ], "eslint.experimental.useFlatConfig": true, "docker.languageserver.formatter.ignoreMultilineInstructions": true } ================================================ FILE: .vscode/tasks.json ================================================ { // See https://go.microsoft.com/fwlink/?LinkId=733558 // for the documentation about the tasks.json format "version": "2.0.0", "tasks": [ { "type": "shell", "options": { "cwd": "${workspaceFolder}/collector", "statusbar": { "color": "#ffea00", "detail": "Runs the collector", "label": "Collector: $(play) run", "running": { "color": "#ffea00", "label": "Collector: $(gear~spin) running" } } }, "command": "cd ${workspaceFolder}/collector/ && yarn dev", "runOptions": { "instanceLimit": 1, "reevaluateOnRerun": true }, "presentation": { "echo": true, "reveal": "always", "focus": false, "panel": "shared", "showReuseMessage": true, "clear": false }, "label": "Collector: run" }, { "type": "shell", "options": { "cwd": "${workspaceFolder}/server", "statusbar": { "color": "#ffea00", "detail": "Runs the server", "label": "Server: $(play) run", "running": { "color": "#ffea00", "label": "Server: $(gear~spin) running" } } }, "command": "if [ \"${CODESPACES}\" = \"true\" ]; then while ! gh codespace ports -c $CODESPACE_NAME | grep 3001; do sleep 1; done; gh codespace ports visibility 3001:public -c $CODESPACE_NAME; fi & cd ${workspaceFolder}/server/ && yarn dev", "runOptions": { "instanceLimit": 1, "reevaluateOnRerun": true }, "presentation": { "echo": true, "reveal": "always", "focus": false, "panel": "shared", "showReuseMessage": true, "clear": false }, "label": "Server: run" }, { "type": "shell", "options": { "cwd": "${workspaceFolder}/frontend", "statusbar": { "color": "#ffea00", "detail": "Runs the frontend", "label": "Frontend: $(play) run", "running": { "color": "#ffea00", "label": "Frontend: $(gear~spin) running" } } }, "command": "cd ${workspaceFolder}/frontend/ && yarn dev", "runOptions": { "instanceLimit": 1, "reevaluateOnRerun": true }, "presentation": { "echo": true, "reveal": "always", "focus": false, "panel": "shared", "showReuseMessage": true, "clear": false }, "label": "Frontend: run" } ] } ================================================ FILE: BARE_METAL.md ================================================ # Run AnythingLLM in production without Docker > [!WARNING] > This method of deployment is **not supported** by the core-team and is to be used as a reference for your deployment. > You are fully responsible for securing your deployment and data in this mode. > **Any issues** experienced from bare-metal or non-containerized deployments will be **not** answered or supported. Here you can find the scripts and known working process to run AnythingLLM outside of a Docker container. ### Minimum Requirements > [!TIP] > You should aim for at least 2GB of RAM. Disk storage is proportional to however much data > you will be storing (documents, vectors, models, etc). Minimum 10GB recommended. - NodeJS v18 - Yarn ## Getting started 1. Clone the repo into your server as the user who the application will run as. `git clone git@github.com:Mintplex-Labs/anything-llm.git` 2. `cd anything-llm` and run `yarn setup`. This will install all dependencies to run in production as well as debug the application. 3. `cp server/.env.example server/.env` to create the basic ENV file for where instance settings will be read from on service start. 4. Ensure that the `server/.env` file has _at least_ these keys to start. These values will persist and this file will be automatically written and managed after your first successful boot. ``` STORAGE_DIR="/your/absolute/path/to/server/storage" ``` 5. Edit the `frontend/.env` file for the `VITE_BASE_API` to now be set to `/api`. This is documented in the .env for which one you should use. ``` # VITE_API_BASE='http://localhost:3001/api' # Use this URL when developing locally # VITE_API_BASE="https://$CODESPACE_NAME-3001.$GITHUB_CODESPACES_PORT_FORWARDING_DOMAIN/api" # for GitHub Codespaces VITE_API_BASE='/api' # Use this URL deploying on non-localhost address OR in docker. ``` ## To start the application AnythingLLM is comprised of three main sections. The `frontend`, `server`, and `collector`. When running in production you will be running `server` and `collector` on two different processes, with a build step for compilation of the frontend. 1. Build the frontend application. `cd frontend && yarn build` - this will produce a `frontend/dist` folder that will be used later. 2. Copy `frontend/dist` to `server/public` - `cp -R frontend/dist server/public`. This should create a folder in `server` named `public` which contains a top level `index.html` file and various other files/folders. 3. Migrate and prepare your database file. ``` cd server && npx prisma generate --schema=./prisma/schema.prisma cd server && npx prisma migrate deploy --schema=./prisma/schema.prisma ``` 4. Boot the server in production `cd server && NODE_ENV=production node index.js &` 5. Boot the collection in another process `cd collector && NODE_ENV=production node index.js &` AnythingLLM should now be running on `http://localhost:3001`! ## Updating AnythingLLM To update AnythingLLM with future updates you can `git pull origin master` to pull in the latest code and then repeat steps 2 - 5 to deploy with all changes fully. _note_ You should ensure that each folder runs `yarn` again to ensure packages are up to date in case any dependencies were added, changed, or removed. _note_ You should `pkill node` before running an update so that you are not running multiple AnythingLLM processes on the same instance as this can cause conflicts. ### Example update script ```shell #!/bin/bash cd $HOME/anything-llm &&\ git checkout . &&\ git pull origin master &&\ echo "HEAD pulled to commit $(git log -1 --pretty=format:"%h" | tail -n 1)" echo "Freezing current ENVs" curl -I "http://localhost:3001/api/env-dump" | head -n 1|cut -d$' ' -f2 echo "Rebuilding Frontend" cd $HOME/anything-llm/frontend && yarn && yarn build && cd $HOME/anything-llm echo "Copying to Server Public" rm -rf server/public cp -r frontend/dist server/public echo "Killing node processes" pkill node echo "Installing collector dependencies" cd $HOME/anything-llm/collector && yarn echo "Installing server dependencies & running migrations" cd $HOME/anything-llm/server && yarn cd $HOME/anything-llm/server && npx prisma migrate deploy --schema=./prisma/schema.prisma cd $HOME/anything-llm/server && npx prisma generate echo "Booting up services." truncate -s 0 /logs/server.log # Or any other log file location. truncate -s 0 /logs/collector.log cd $HOME/anything-llm/server (NODE_ENV=production node index.js) &> /logs/server.log & cd $HOME/anything-llm/collector (NODE_ENV=production node index.js) &> /logs/collector.log & ``` ## Using Nginx? If you are using Nginx, you can use the following example configuration to proxy the requests to the server. Chats for streaming require **websocket** connections, so you need to ensure that the Nginx configuration is set up to support websockets. You can do this with a simple reverse proxy configuration. ```nginx server { # Enable websocket connections for agent protocol. location ~* ^/api/agent-invocation/(.*) { proxy_pass http://0.0.0.0:3001; proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "Upgrade"; } listen 80; server_name [insert FQDN here]; location / { # Prevent timeouts on long-running requests. proxy_connect_timeout 605; proxy_send_timeout 605; proxy_read_timeout 605; send_timeout 605; keepalive_timeout 605; # Enable readable HTTP Streaming for LLM streamed responses proxy_buffering off; proxy_cache off; # Proxy your locally running service proxy_pass http://0.0.0.0:3001; } } ``` ================================================ FILE: CONTRIBUTING.md ================================================ # Contributing to AnythingLLM AnythingLLM is an open-source project and we welcome contributions from the community. ## Reporting Issues If you encounter a bug or have a feature request, please open an issue on the [GitHub issue tracker](https://github.com/mintplex-labs/anything-llm). ## Picking an issue We track issues on the GitHub issue tracker. If you are looking for something to work on, check the [good first issue](https://github.com/mintplex-labs/anything-llm/contribute) label. These issues are typically the best described and have the smallest scope. There may be issues that are not labeled as good first issue, but are still a good starting point. If there's an issue you are interested in working on, please leave a comment on the issue. This will help us avoid duplicate work. Additionally, if you have questions about the issue, please ask them in the issue comments. We are happy to provide guidance on how to approach the issue. ## Before you start Keep in mind that we are a small team and have limited resources. We will do our best to review and merge your PRs, but please be patient. Ultimately, **we become the maintainer** of your changes. It is our responsibility to make sure that the changes are working as expected and are of high quality as well as being compatible with the rest of the project both for existing users and for future users & features. Before you start working on an issue, please read the following so that you don't waste time on something that is not a good fit for the project or is more suitable for a personal fork. We would rather answer a comment on an issue than close a PR after you've spent time on it. Your time is valuable and we appreciate your time and effort to make AnythingLLM better. 0. (most important) If you are making a PR that does not have a corresponding issue, **it will not be merged.** _The only exception to this is language translations._ 1. If you are modifying the permission system for a new role or something custom, you are likely better off forking the project and building your own version since this is a core part of the project and is only to be maintained by the AnythingLLM team. 2. Integrations (LLM, Vector DB, etc.) are reviewed at our discretion. We will eventually get to them. Do not expect us to merge your integration PR instantly since there are often many moving parts and we want to make sure we get it right. We will get to it! 3. It is our discretion to merge or not merge a PR. We value every contribution, but we also value the quality of the code and the user experience we envision for the project. It is a fine line to walk when running a project like this and please understand that merging or not merging a PR is not a reflection of the quality of the contribution and is not personal. We will do our best to provide feedback on the PR and help you make the changes necessary to get it merged. 4. **Security** is always important. If you have a security concern, please do not open an issue. Instead, please open a CVE on our designated reporting platform [Huntr](https://huntr.com) or contact us at [team@mintplexlabs.com](mailto:team@mintplexlabs.com). ## Configuring Git First, fork the repository on GitHub, then clone your fork: ```bash git clone https://github.com//anything-llm.git cd anything-llm ``` Then add the main repository as a remote: ```bash git remote add upstream https://github.com/mintplex-labs/anything-llm.git git fetch upstream ``` ## Setting up your development environment In the root of the repository, run: ```bash yarn setup ``` This will install the dependencies, set up the proper and expected ENV files for the project, and run the prisma setup script. Next, run: ```bash yarn dev:all ``` This will start the server, frontend, and collector in development mode. Changes to the code will be hot reloaded. ## Best practices for pull requests For the best chance of having your pull request accepted, please follow these guidelines: 1. Unit test all bug fixes and new features. Your code will not be merged if it doesn't have tests. 1. If you change the public API, update the documentation in the `anythingllm-docs` repository. 1. Aim to minimize the number of changes in each pull request. Keep to solving one problem at a time, when possible. 1. Before marking a pull request ready-for-review, do a self review of your code. Is it clear why you are making the changes? Are the changes easy to understand? 1. Use [conventional commit messages](https://www.conventionalcommits.org/en/) as pull request titles. Examples: * New feature: `feat: adding foo API` * Bug fix: `fix: issue with foo API` * Documentation change: `docs: adding foo API documentation` 1. If your pull request is a work in progress, leave the pull request as a draft. We will assume the pull request is ready for review when it is opened. 1. When writing tests, test the error cases. Make sure they have understandable error messages. ## Project structure The core library is written in Node.js. There are additional sub-repositories for the embed widget and browser extension. These are not part of the core AnythingLLM project, but are maintained by the AnythingLLM team. * `server`: Node.js server source code * `frontend`: React frontend source code * `collector`: Python collector source code ## Release process Changes to the core AnythingLLM project are released through the `master` branch. When a PR is merged into `master`, a new version of the package is published to Docker and GitHub Container Registry under the `latest` tag. When a new version is released, the following steps are taken a new image is built and pushed to Docker Hub and GitHub Container Registry under the associated version tag. Version tags are of the format `v..` and are pinned code, while `latest` is the latest version of the code at any point in time. ### Desktop propagation Changes to the desktop app are downstream of the core AnythingLLM project. Releases of the desktop app are published at the same time as the core AnythingLLM project. Code from the core AnythingLLM project is copied into the desktop app into an Electron wrapper. The Electron wrapper that wraps around the core AnythingLLM project is **not** part of the core AnythingLLM project, but is maintained by the AnythingLLM team. ## License By contributing to AnythingLLM (this repository), you agree to license your contributions under the MIT license. ================================================ FILE: LICENSE ================================================ The MIT License Copyright (c) Mintplex Labs Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: README.md ================================================

AnythingLLM logo

Mintplex-Labs%2Fanything-llm | Trendshift

AnythingLLM: The all-in-one AI app you were looking for.
Chat with your docs, use AI Agents, hyper-configurable, multi-user, & no frustrating setup required.

Discord | License | Docs | Hosted Instance

English · 简体中文 · 日本語

👉 AnythingLLM for desktop (Mac, Windows, & Linux)! Download Now

Chat with your docs. Automate complex workflows with AI Agents. Hyper-configurable, multi-user ready, battle-tested—and runs locally by default with zero setup friction. ![Chatting](https://github.com/Mintplex-Labs/anything-llm/releases/download/v1.11.2/AnythingLLM720p.gif)
Watch the demo! [![Watch the video](/images/youtube.png)](https://youtu.be/f95rGD9trL0)
### Product Overview AnythingLLM is the all-in-one AI application that lets you build a private, fully-featured ChatGPT—without compromises. Connect your favorite local or cloud LLM, ingest your documents, and start chatting in minutes. Out of the box you get built-in agents, multi-user support, vector databases, and document pipelines — no extra configuration required. AnythingLLM supports multiple users as well where you can control the access and experience per user without compromising the security or privacy of the instance or your intellectual property. ## Cool features of AnythingLLM - [Intelligent Skill Selection](https://docs.anythingllm.com/agent/intelligent-tool-selection) Enable **unlimited** tools for your models while reducing token usage by up to 80% per query - [**No-code AI Agent builder**](https://docs.anythingllm.com/agent-flows/overview) - [**Full MCP-compatibility**](https://docs.anythingllm.com/mcp-compatibility/overview) - **Multi-modal support (both closed and open-source LLMs!)** - [**Custom AI Agents**](https://docs.anythingllm.com/agent/custom/introduction) - 👤 Multi-user instance support and permissioning _Docker version only_ - 🦾 Agents inside your workspace (browse the web, etc) - 💬 [Custom Embeddable Chat widget for your website](https://github.com/Mintplex-Labs/anythingllm-embed/blob/main/README.md) _Docker version only_ - 📖 Multiple document type support (PDF, TXT, DOCX, etc) - Intuitive chat UI with drag-and-drop uploads and source citations. - Production-ready for any cloud deployment. - Works with all popular [closed and open-source LLM providers](#supported-llms-embedder-models-speech-models-and-vector-databases). - Built-in optimizations for large document sets—lower costs and faster responses than other chat UIs. - Full Developer API for custom integrations! - ...and much more—install in minutes and see for yourself. ### Supported LLMs, Embedder Models, Speech models, and Vector Databases **Large Language Models (LLMs):** - [Any open-source llama.cpp compatible model](/server/storage/models/README.md#text-generation-llm-selection) - [OpenAI](https://openai.com) - [OpenAI (Generic)](https://openai.com) - [Azure OpenAI](https://azure.microsoft.com/en-us/products/ai-services/openai-service) - [AWS Bedrock](https://aws.amazon.com/bedrock/) - [Anthropic](https://www.anthropic.com/) - [NVIDIA NIM (chat models)](https://build.nvidia.com/explore/discover) - [Google Gemini Pro](https://ai.google.dev/) - [Hugging Face (chat models)](https://huggingface.co/) - [Ollama (chat models)](https://ollama.ai/) - [LM Studio (all models)](https://lmstudio.ai) - [LocalAI (all models)](https://localai.io/) - [Together AI (chat models)](https://www.together.ai/) - [Fireworks AI (chat models)](https://fireworks.ai/) - [Perplexity (chat models)](https://www.perplexity.ai/) - [OpenRouter (chat models)](https://openrouter.ai/) - [DeepSeek (chat models)](https://deepseek.com/) - [Mistral](https://mistral.ai/) - [Groq](https://groq.com/) - [Cohere](https://cohere.com/) - [KoboldCPP](https://github.com/LostRuins/koboldcpp) - [LiteLLM](https://github.com/BerriAI/litellm) - [Text Generation Web UI](https://github.com/oobabooga/text-generation-webui) - [Apipie](https://apipie.ai/) - [xAI](https://x.ai/) - [Z.AI (chat models)](https://z.ai/model-api) - [Novita AI (chat models)](https://novita.ai/model-api/product/llm-api?utm_source=github_anything-llm&utm_medium=github_readme&utm_campaign=link) - [PPIO](https://ppinfra.com?utm_source=github_anything-llm) - [Gitee AI](https://ai.gitee.com/) - [Moonshot AI](https://www.moonshot.ai/) - [Microsoft Foundry Local](https://github.com/microsoft/Foundry-Local) - [CometAPI (chat models)](https://api.cometapi.com/) - [Docker Model Runner](https://docs.docker.com/ai/model-runner/) - [PrivateModeAI (chat models)](https://privatemode.ai/) - [SambaNova Cloud (chat models)](https://cloud.sambanova.ai/) - [Lemonade by AMD](https://lemonade-server.ai) **Embedder models:** - [AnythingLLM Native Embedder](/server/storage/models/README.md) (default) - [OpenAI](https://openai.com) - [Azure OpenAI](https://azure.microsoft.com/en-us/products/ai-services/openai-service) - [LocalAI (all)](https://localai.io/) - [Ollama (all)](https://ollama.ai/) - [LM Studio (all)](https://lmstudio.ai) - [Cohere](https://cohere.com/) **Audio Transcription models:** - [AnythingLLM Built-in](https://github.com/Mintplex-Labs/anything-llm/tree/master/server/storage/models#audiovideo-transcription) (default) - [OpenAI](https://openai.com/) **TTS (text-to-speech) support:** - Native Browser Built-in (default) - [PiperTTSLocal - runs in browser](https://github.com/rhasspy/piper) - [OpenAI TTS](https://platform.openai.com/docs/guides/text-to-speech/voice-options) - [ElevenLabs](https://elevenlabs.io/) - Any OpenAI Compatible TTS service. **STT (speech-to-text) support:** - Native Browser Built-in (default) **Vector Databases:** - [LanceDB](https://github.com/lancedb/lancedb) (default) - [PGVector](https://github.com/pgvector/pgvector) - [Astra DB](https://www.datastax.com/products/datastax-astra) - [Pinecone](https://pinecone.io) - [Chroma & ChromaCloud](https://trychroma.com) - [Weaviate](https://weaviate.io) - [Qdrant](https://qdrant.tech) - [Milvus](https://milvus.io) - [Zilliz](https://zilliz.com) ### Technical Overview This monorepo consists of six main sections: - `frontend`: A viteJS + React frontend that you can run to easily create and manage all your content the LLM can use. - `server`: A NodeJS express server to handle all the interactions and do all the vectorDB management and LLM interactions. - `collector`: NodeJS express server that processes and parses documents from the UI. - `docker`: Docker instructions and build process + information for building from source. - `embed`: Submodule for generation & creation of the [web embed widget](https://github.com/Mintplex-Labs/anythingllm-embed). - `browser-extension`: Submodule for the [chrome browser extension](https://github.com/Mintplex-Labs/anythingllm-extension). ## 🛳 Self-Hosting Mintplex Labs & the community maintain a number of deployment methods, scripts, and templates that you can use to run AnythingLLM locally. Refer to the table below to read how to deploy on your preferred environment or to automatically deploy. | Docker | AWS | GCP | Digital Ocean | Render.com | |----------------------------------------|----|-----|---------------|------------| | [![Deploy on Docker][docker-btn]][docker-deploy] | [![Deploy on AWS][aws-btn]][aws-deploy] | [![Deploy on GCP][gcp-btn]][gcp-deploy] | [![Deploy on DigitalOcean][do-btn]][do-deploy] | [![Deploy on Render.com][render-btn]][render-deploy] | | Railway | RepoCloud | Elestio | Northflank | | --------------------------------------------------- | --------------------------------------------------------- | --------------------------------------------------- | ------------------------------------------------------------ | | [![Deploy on Railway][railway-btn]][railway-deploy] | [![Deploy on RepoCloud][repocloud-btn]][repocloud-deploy] | [![Deploy on Elestio][elestio-btn]][elestio-deploy] | [![Deploy on Northflank][northflank-btn]][northflank-deploy] | [or set up a production AnythingLLM instance without Docker →](./BARE_METAL.md) ## How to setup for development - `yarn setup` To fill in the required `.env` files you'll need in each of the application sections (from root of repo). - Go fill those out before proceeding. Ensure `server/.env.development` is filled or else things won't work right. - `yarn dev:server` To boot the server locally (from root of repo). - `yarn dev:frontend` To boot the frontend locally (from root of repo). - `yarn dev:collector` To then run the document collector (from root of repo). [Learn about documents](./server/storage/documents/DOCUMENTS.md) ## Telemetry & Privacy AnythingLLM by Mintplex Labs Inc contains a telemetry feature that collects anonymous usage information.
More about Telemetry & Privacy for AnythingLLM ### Why? We use this information to help us understand how AnythingLLM is used, to help us prioritize work on new features and bug fixes, and to help us improve AnythingLLM's performance and stability. ### Opting out Set `DISABLE_TELEMETRY` in your server or docker .env settings to "true" to opt out of telemetry. You can also do this in-app by going to the sidebar > `Privacy` and disabling telemetry. ### What do you explicitly track? We will only track usage details that help us make product and roadmap decisions, specifically: - Type of your installation (Docker or Desktop) - When a document is added or removed. No information _about_ the document. Just that the event occurred. This gives us an idea of use. - Type of vector database in use. This helps us prioritize changes when updates arrive for that provider. - Type of LLM provider & model tag in use. This helps us prioritize changes when updates arrive for that provider or model, or combination thereof. eg: reasoning vs regular, multi-modal models, etc. - When a chat is sent. This is the most regular "event" and gives us an idea of the daily-activity of this project across all installations. Again, only the **event** is sent - we have no information on the nature or content of the chat itself. You can verify these claims by finding all locations `Telemetry.sendTelemetry` is called. Additionally these events are written to the output log so you can also see the specific data which was sent - if enabled. **No IP or other identifying information is collected**. The Telemetry provider is [PostHog](https://posthog.com/) - an open-source telemetry collection service. We take privacy very seriously, and we hope you understand that we want to learn how our tool is used, without using annoying popup surveys, so we can build something worth using. The anonymous data is _never_ shared with third parties, ever. [View all telemetry events in source code](https://github.com/search?q=repo%3AMintplex-Labs%2Fanything-llm%20.sendTelemetry(&type=code)
## 👋 Contributing - [Contributing to AnythingLLM](./CONTRIBUTING.md) - How to contribute to AnythingLLM. ## 💖 Sponsors ### Premium Sponsors User avatar: DCS DIGITAL ### All Sponsors User avatar: JaschaUser avatar: KickAssUser avatar: ShadowArcanistUser avatar: AtlasUser avatar: Predrag StojadinovićUser avatar: Diego SpinolaUser avatar: KyleUser avatar: Giulio De PasqualeUser avatar: User avatar: MacStadiumUser avatar: User avatar: User avatar: User avatar: User avatar: DennisUser avatar: Michael Hamilton, Ph.D.User avatar: User avatar: TernaryLabsUser avatar: Daniel CelaUser avatar: AlessoUser avatar: Rune MathisenUser avatar: User avatar: User avatar: AlanUser avatar: Damien PetersUser avatar: DCS DigitalUser avatar: Paul McilreavyUser avatar: Til WolfUser avatar: Leopoldo Crhistian Riverin GomezUser avatar: AJEsauUser avatar: Steven VanOmmerenUser avatar: Casey BoettcherUser avatar: User avatar: AvineetUser avatar: ChrisUser avatar: mirkoUser avatar: Tim ChampUser avatar: Peter MathisenUser avatar: Ed di GirolamoUser avatar: Wojciech MiłkowskiUser avatar: ADS FundUser avatar: arc46 GmbHUser avatar: Li YinUser avatar: SylphAI ## 🌟 Contributors [![anythingllm contributors](https://contrib.rocks/image?repo=mintplex-labs/anything-llm)](https://github.com/mintplex-labs/anything-llm/graphs/contributors) [![Star History Chart](https://api.star-history.com/svg?repos=mintplex-labs/anything-llm&type=Timeline)](https://star-history.com/#mintplex-labs/anything-llm&Date) ## 🔗 More Products - **[VectorAdmin][vector-admin]:** An all-in-one GUI & tool-suite for managing vector databases. - **[OpenAI Assistant Swarm][assistant-swarm]:** Turn your entire library of OpenAI assistants into one single army commanded from a single agent.
[![][back-to-top]](#readme-top)
--- Copyright © 2026 [Mintplex Labs][profile-link].
This project is [MIT](./LICENSE) licensed. [back-to-top]: https://img.shields.io/badge/-BACK_TO_TOP-222628?style=flat-square [profile-link]: https://github.com/mintplex-labs [vector-admin]: https://github.com/mintplex-labs/vector-admin [assistant-swarm]: https://github.com/Mintplex-Labs/openai-assistant-swarm [docker-btn]: ./images/deployBtns/docker.png [docker-deploy]: ./docker/HOW_TO_USE_DOCKER.md [aws-btn]: ./images/deployBtns/aws.png [aws-deploy]: ./cloud-deployments/aws/cloudformation/DEPLOY.md [gcp-btn]: https://deploy.cloud.run/button.svg [gcp-deploy]: ./cloud-deployments/gcp/deployment/DEPLOY.md [do-btn]: https://www.deploytodo.com/do-btn-blue.svg [do-deploy]: ./cloud-deployments/digitalocean/terraform/DEPLOY.md [render-btn]: https://render.com/images/deploy-to-render-button.svg [render-deploy]: https://render.com/deploy?repo=https://github.com/Mintplex-Labs/anything-llm&branch=render [render-btn]: https://render.com/images/deploy-to-render-button.svg [render-deploy]: https://render.com/deploy?repo=https://github.com/Mintplex-Labs/anything-llm&branch=render [railway-btn]: https://railway.app/button.svg [railway-deploy]: https://railway.app/template/HNSCS1?referralCode=WFgJkn [repocloud-btn]: https://d16t0pc4846x52.cloudfront.net/deploylobe.svg [repocloud-deploy]: https://repocloud.io/details/?app_id=276 [elestio-btn]: https://elest.io/images/logos/deploy-to-elestio-btn.png [elestio-deploy]: https://elest.io/open-source/anythingllm [northflank-btn]: https://assets.northflank.com/deploy_to_northflank_smm_36700fb050.svg [northflank-deploy]: https://northflank.com/stacks/deploy-anythingllm ================================================ FILE: SECURITY.md ================================================ # Security Policy ## Supported Versions Use this section to tell people about which versions of your project are currently being supported with security updates. | Version | Supported | | ------- | ------------------ | | 0.1.x | :white_check_mark: | ## Reporting a Vulnerability If a security concern is found that you would like to disclose you can create a PR for it or if you would like to clear this issue before posting you can email [Core Mintplex Labs Team](mailto:team@mintplexlabs.com). ================================================ FILE: TERMS_SELF_HOSTED.md ================================================ # AnythingLLM Self-Hosted: Data Privacy & Terms of Service This document outlines the privacy standards, data handling procedures, and licensing terms for the self-hosted version of AnythingLLM, developed by Mintplex Labs Inc. ## 1. Data Sovereignty & Local-First Architecture AnythingLLM is designed as a **local-first** application. When utilizing the self-hosted version (Docker, Desktop, or Source): * **No External Access:** Mintplex Labs Inc. does not host, store, or have access to any documents, chat histories, workspace settings, or embeddings created within your instance. * **On-Premise Storage:** All data resides strictly on the infrastructure provisioned and managed by the user or their organization. * **Air-Gap Capability:** AnythingLLM can be operated in a strictly air-gapped environment with no internet connectivity, provided local LLM and Vector database providers (e.g., Ollama, LocalAI, LanceDB) are utilized. ## 2. Telemetry and Analytics To improve software performance and stability, AnythingLLM includes an optional telemetry feature. * **Anonymity:** Collected data is strictly anonymous and contains no Personally Identifiable Information (PII), document content, chat logs, fingerprinting data, or any other sensitive information. Purely usage based data is collected. * **Opt-Out:** Users may disable telemetry at any time via the **Settings** menu within the application. Once disabled, no usage data is transmitted to Mintplex Labs. ## 3. Third-Party Integrations AnythingLLM allows users to connect to external services (e.g., OpenAI, Anthropic, Pinecone). * **Data Transmission:** When these services are enabled, data is transmitted directly from your instance to the third-party provider. * **Governing Terms:** Data handled by third-party providers is subject to their respective Terms of Service and Privacy Policies. Mintplex Labs is not responsible for the data practices of these external entities. _by default, AnythingLLM does **everything on-device first** - so you would have to manually configure and enable these integrations to be subject to third party terms._ ## 4. Security & Network * **No "Phone Home":** Aside from [optional telemetry](https://github.com/Mintplex-Labs/anything-llm?tab=readme-ov-file#telemetry--privacy), the software does not require an external connection to Mintplex Labs servers to function. * **Environment Security:** The user is responsible for securing the host environment, including network firewalls, SSL/TLS encryption, and access control for the AnythingLLM instance. * **CDN Assets:** Out of a convience to international users, we use a hosted CDN to mirror some critical path models (eg: the default embedder and reranking ONNX models) which are not available in all regions. These models are downloaded from our CDN as a fallback, and any air-gapped installations you can either download these models manually or use another provider. Assets of these nature are downloaded once and cached in your associated local storage. ## 5. Licensing and Liability * **License:** The AnythingLLM core is provided under the **MIT License**. * **No Warranty:** As per the license agreement, the software is provided "as is," without warranty of any kind, express or implied, including but not limited to the warranties of merchantability or fitness for a particular purpose. * **Liability:** In no event shall the authors or copyright holders be liable for any claim, damages, or other liability arising from the use of the software. ## 6. Support and Compatibility While Mintplex Labs prioritizes stability and backward compatibility, the self-hosted version is used at the user's discretion. Formal Service Level Agreements (SLAs) are not provided for the standard self-hosted version unless otherwise negotiated via a separate enterprise agreement. --- *Last Updated: March 2026* ================================================ FILE: cloud-deployments/aws/cloudformation/DEPLOY.md ================================================ # How to deploy a private AnythingLLM instance on AWS With an AWS account you can easily deploy a private AnythingLLM instance on AWS. This will create a url that you can access from any browser over HTTP (HTTPS not supported). This single instance will run on your own keys and they will not be exposed - however if you want your instance to be protected it is highly recommend that you set a password once setup is complete. **Quick Launch (EASY)** 1. Log in to your AWS account 2. Open [CloudFormation](https://us-west-1.console.aws.amazon.com/cloudformation/home) 3. Ensure you are deploying in a geographic zone that is nearest to your physical location to reduce latency. 4. Click `Create Stack` ![Create Stack](../../../images/screenshots/create_stack.png) 5. Use the file `cloudformation_create_anythingllm.json` as your JSON template. ![Upload Stack](../../../images/screenshots/upload.png) 6. Click Deploy. 7. Wait for stack events to finish and be marked as `Completed` 8. View `Outputs` tab. ![Stack Output](../../../images/screenshots/cf_outputs.png) 9. Wait for all resources to be built. Now wait until instance is available on `[InstanceIP]:3001`. This process may take up to 10 minutes. See **Note** below on how to visualize this process. The output of this cloudformation stack will be: - 1 EC2 Instance - 1 Security Group with 0.0.0.0/0 access on port 3001 - 1 EC2 Instance Volume `gb2` of 10Gib minimum - customizable pre-deploy. **Requirements** - An AWS account with billing information. ## Please read this notice before submitting issues about your deployment **Note:** Your instance will not be available instantly. Depending on the instance size you launched with it can take 5-10 minutes to fully boot up. If you want to check the instance's progress, navigate to [your deployed EC2 instances](https://us-west-1.console.aws.amazon.com/ec2/home) and connect to your instance via SSH in browser. Once connected run `sudo tail -f /var/log/cloud-init-output.log` and wait for the file to conclude deployment of the docker image. You should see an output like this ``` [+] Running 2/2 ⠿ Network docker_anything-llm Created ⠿ Container anything-llm Started ``` Additionally, your use of this deployment process means you are responsible for any costs of these AWS resources fully. ================================================ FILE: cloud-deployments/aws/cloudformation/aws_https_instructions.md ================================================ # How to Configure HTTPS for Anything LLM AWS private deployment Instructions for manual https configuration after generating and running the aws cloudformation template (aws_build_from_source_no_credentials.json). Tested on following browsers: Firefox version 119, Chrome version 118, Edge 118. **Requirements** - Successful deployment of Amazon Linux 2023 EC2 instance with Docker container running Anything LLM - Admin priv to configure Elastic IP for EC2 instance via AWS Management Console UI - Admin priv to configure DNS services (i.e. AWS Route 53) via AWS Management Console UI - Admin priv to configure EC2 Security Group rules via AWS Management Console UI ## Step 1: Allocate and assign Elastic IP Address to your deployed EC2 instance 1. Follow AWS instructions on allocating EIP here: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/elastic-ip-addresses-eip.html#using-instance-addressing-eips-allocating 2. Follow AWS instructions on assigning EIP to EC2 instance here: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/elastic-ip-addresses-eip.html#using-instance-addressing-eips-associating ## Step 2: Configure DNS A record to resolve to the previously assigned EC2 instance via EIP These instructions assume that you already have a top-level domain configured and are using a subdomain to access AnythingLLM. 1. Follow AWS instructions on routing traffic to EC2 instance here: https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/routing-to-ec2-instance.html ## Step 3: Install and enable nginx These instructions are for CLI configuration and assume you are logged in to EC2 instance as the ec2-user. 1. $sudo yum install nginx -y 2. $sudo systemctl enable nginx && sudo systemctl start nginx ## Step 4: Install certbot These instructions are for CLI configuration and assume you are logged in to EC2 instance as the ec2-user. 1. $sudo yum install -y augeas-libs 2. $sudo python3 -m venv /opt/certbot/ 3. $sudo /opt/certbot/bin/pip install --upgrade pip 4. $sudo /opt/certbot/bin/pip install certbot certbot-nginx 5. $sudo ln -s /opt/certbot/bin/certbot /usr/bin/certbot ## Step 5: Configure temporary Inbound Traffic Rule for Security Group to certbot DNS verification 1. Follow AWS instructions on creating inbound rule (http port 80 0.0.0.0/0) for EC2 security group here: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/working-with-security-groups.html#adding-security-group-rule ## Step 6: Comment out default http NGINX proxy configuration These instructions are for CLI configuration and assume you are logged in to EC2 instance as the ec2-user. 1. $sudo vi /etc/nginx/nginx.conf 2. In the nginx.conf file, comment out the default server block configuration for http/port 80. It should look something like the following: ``` # server { # listen 80; # listen [::]:80; # server_name _; # root /usr/share/nginx/html; # # # Load configuration files for the default server block. # include /etc/nginx/default.d/*.conf; # # error_page 404 /404.html; # location = /404.html { # } # # error_page 500 502 503 504 /50x.html; # location = /50x.html { # } # } ``` 3. Enter ':wq' to save the changes to the nginx default config ## Step 7: Create simple http proxy configuration for AnythingLLM These instructions are for CLI configuration and assume you are logged in to EC2 instance as the ec2-user. 1. $sudo vi /etc/nginx/conf.d/anything.conf 2. Add the following configuration ensuring that you add your FQDN:. ``` server { # Enable websocket connections for agent protocol. location ~* ^/api/agent-invocation/(.*) { proxy_pass http://0.0.0.0:3001; proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "Upgrade"; } listen 80; server_name [insert FQDN here]; location / { # Prevent timeouts on long-running requests. proxy_connect_timeout 605; proxy_send_timeout 605; proxy_read_timeout 605; send_timeout 605; keepalive_timeout 605; # Enable readable HTTP Streaming for LLM streamed responses proxy_buffering off; proxy_cache off; # Proxy your locally running service proxy_pass http://0.0.0.0:3001; } } ``` 3. Enter ':wq' to save the changes to the anything config file ## Step 8: Test nginx http proxy config and restart nginx service These instructions are for CLI configuration and assume you are logged in to EC2 instance as the ec2-user. 1. $sudo nginx -t 2. $sudo systemctl restart nginx 3. Navigate to http://FQDN in a browser and you should be proxied to the AnythingLLM web UI. ## Step 9: Generate/install cert These instructions are for CLI configuration and assume you are logged in to EC2 instance as the ec2-user. 1. $sudo certbot --nginx -d [Insert FQDN here] Example command: $sudo certbot --nginx -d anythingllm.exampleorganization.org This command will generate the appropriate certificate files, write the files to /etc/letsencrypt/live/yourFQDN, and make updates to the nginx configuration file for anythingllm located at /etc/nginx/conf.d/anything.llm 3. Enter the email address you would like to use for updates. 4. Accept the terms of service. 5. Accept or decline to receive communication from LetsEncrypt. ## Step 10: Test Cert installation 1. $sudo cat /etc/nginx/conf.d/anything.conf Your should see a completely updated configuration that includes https/443 and a redirect configuration for http/80. 2. Navigate to https://FQDN in a browser and you should be proxied to the AnythingLLM web UI. ## Step 11: (Optional) Remove temporary Inbound Traffic Rule for Security Group to certbot DNS verification 1. Follow AWS instructions on deleting inbound rule (http port 80 0.0.0.0/0) for EC2 security group here: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/working-with-security-groups.html#deleting-security-group-rule ================================================ FILE: cloud-deployments/aws/cloudformation/cloudformation_create_anythingllm.json ================================================ { "AWSTemplateFormatVersion": "2010-09-09", "Description": "Create a stack that runs AnythingLLM on a single instance", "Parameters": { "InstanceType": { "Description": "EC2 instance type", "Type": "String", "Default": "t3.small" }, "InstanceVolume": { "Description": "Storage size of disk on Instance in GB", "Type": "Number", "Default": 10, "MinValue": 4 } }, "Resources": { "AnythingLLMInstance": { "Type": "AWS::EC2::Instance", "Properties": { "ImageId": { "Fn::FindInMap": [ "Region2AMI", { "Ref": "AWS::Region" }, "AMI" ] }, "InstanceType": { "Ref": "InstanceType" }, "SecurityGroupIds": [ { "Ref": "AnythingLLMInstanceSecurityGroup" } ], "BlockDeviceMappings": [ { "DeviceName": { "Fn::FindInMap": [ "Region2AMI", { "Ref": "AWS::Region" }, "RootDeviceName" ] }, "Ebs": { "VolumeSize": { "Ref": "InstanceVolume" } } } ], "UserData": { "Fn::Base64": { "Fn::Join": [ "", [ "Content-Type: multipart/mixed; boundary=\"//\"\n", "MIME-Version: 1.0\n", "\n", "--//\n", "Content-Type: text/cloud-config; charset=\"us-ascii\"\n", "MIME-Version: 1.0\n", "Content-Transfer-Encoding: 7bit\n", "Content-Disposition: attachment; filename=\"cloud-config.txt\"\n", "\n", "\n", "#cloud-config\n", "cloud_final_modules:\n", "- [scripts-user, once-per-instance]\n", "\n", "\n", "--//\n", "Content-Type: text/x-shellscript; charset=\"us-ascii\"\n", "MIME-Version: 1.0\n", "Content-Transfer-Encoding: 7bit\n", "Content-Disposition: attachment; filename=\"userdata.txt\"\n", "\n", "\n", "#!/bin/bash\n", "# check output of userdata script with sudo tail -f /var/log/cloud-init-output.log\n", "sudo yum install docker iptables -y\n", "sudo iptables -A OUTPUT -m owner ! --uid-owner root -d 169.254.169.254 -j DROP\n", "sudo systemctl enable docker\n", "sudo systemctl start docker\n", "mkdir -p /home/ec2-user/anythingllm\n", "touch /home/ec2-user/anythingllm/.env\n", "sudo chown ec2-user:ec2-user -R /home/ec2-user/anythingllm\n", "docker pull mintplexlabs/anythingllm\n", "docker run -d -p 3001:3001 --cap-add SYS_ADMIN -v /home/ec2-user/anythingllm:/app/server/storage -v /home/ec2-user/anythingllm/.env:/app/server/.env -e STORAGE_DIR=\"/app/server/storage\" mintplexlabs/anythingllm\n", "echo \"Container ID: $(sudo docker ps --latest --quiet)\"\n", "export ONLINE=$(curl -Is http://localhost:3001/api/ping | head -n 1|cut -d$' ' -f2)\n", "echo \"Health check: $ONLINE\"\n", "echo \"Setup complete! AnythingLLM instance is now online!\"\n", "\n", "--//--\n" ] ] } } } }, "AnythingLLMInstanceSecurityGroup": { "Type": "AWS::EC2::SecurityGroup", "Properties": { "GroupDescription": "AnythingLLM Instance Security Group", "SecurityGroupIngress": [ { "IpProtocol": "tcp", "FromPort": "22", "ToPort": "22", "CidrIp": "0.0.0.0/0" }, { "IpProtocol": "tcp", "FromPort": "3001", "ToPort": "3001", "CidrIp": "0.0.0.0/0" }, { "IpProtocol": "tcp", "FromPort": "3001", "ToPort": "3001", "CidrIpv6": "::/0" } ] } } }, "Outputs": { "ServerIp": { "Description": "IP address of the AnythingLLM instance", "Value": { "Fn::GetAtt": [ "AnythingLLMInstance", "PublicIp" ] } }, "ServerURL": { "Description": "URL of the AnythingLLM server", "Value": { "Fn::Join": [ "", [ "http://", { "Fn::GetAtt": [ "AnythingLLMInstance", "PublicIp" ] }, ":3001" ] ] } } }, "Mappings": { "Region2AMI": { "ap-south-1": { "AMI": "ami-0e6329e222e662a52", "RootDeviceName": "/dev/xvda" }, "eu-north-1": { "AMI": "ami-08c308b1bb265e927", "RootDeviceName": "/dev/xvda" }, "eu-west-3": { "AMI": "ami-069d1ea6bc64443f0", "RootDeviceName": "/dev/xvda" }, "eu-west-2": { "AMI": "ami-06a566ca43e14780d", "RootDeviceName": "/dev/xvda" }, "eu-west-1": { "AMI": "ami-0a8dc52684ee2fee2", "RootDeviceName": "/dev/xvda" }, "ap-northeast-3": { "AMI": "ami-0c8a89b455fae8513", "RootDeviceName": "/dev/xvda" }, "ap-northeast-2": { "AMI": "ami-0ff56409a6e8ea2a0", "RootDeviceName": "/dev/xvda" }, "ap-northeast-1": { "AMI": "ami-0ab0bbbd329f565e6", "RootDeviceName": "/dev/xvda" }, "ca-central-1": { "AMI": "ami-033c256a10931f206", "RootDeviceName": "/dev/xvda" }, "sa-east-1": { "AMI": "ami-0dabf4dab6b183eef", "RootDeviceName": "/dev/xvda" }, "ap-southeast-1": { "AMI": "ami-0dc5785603ad4ff54", "RootDeviceName": "/dev/xvda" }, "ap-southeast-2": { "AMI": "ami-0c5d61202c3b9c33e", "RootDeviceName": "/dev/xvda" }, "eu-central-1": { "AMI": "ami-004359656ecac6a95", "RootDeviceName": "/dev/xvda" }, "us-east-1": { "AMI": "ami-0cff7528ff583bf9a", "RootDeviceName": "/dev/xvda" }, "us-east-2": { "AMI": "ami-02238ac43d6385ab3", "RootDeviceName": "/dev/xvda" }, "us-west-1": { "AMI": "ami-01163e76c844a2129", "RootDeviceName": "/dev/xvda" }, "us-west-2": { "AMI": "ami-0ceecbb0f30a902a6", "RootDeviceName": "/dev/xvda" } } } } ================================================ FILE: cloud-deployments/digitalocean/terraform/DEPLOY.md ================================================ # How to deploy a private AnythingLLM instance on DigitalOcean using Terraform With a DigitalOcean account, you can easily deploy a private AnythingLLM instance using Terraform. This will create a URL that you can access from any browser over HTTP (HTTPS not supported). This single instance will run on your own keys, and they will not be exposed. However, if you want your instance to be protected, it is highly recommended that you set a password once setup is complete. The output of this Terraform configuration will be: - 1 DigitalOcean Droplet - An IP address to access your application **Requirements** - An DigitalOcean account with billing information - Terraform installed on your local machine - Follow the instructions in the [official Terraform documentation](https://developer.hashicorp.com/terraform/tutorials/aws-get-started/install-cli) for your operating system. ## How to deploy on DigitalOcean Open your terminal and navigate to the `docker` folder 1. Create a `.env` file by cloning the `.env.example`. 2. Navigate to `digitalocean/terraform` folder. 3. Replace the token value in the provider "digitalocean" block in main.tf with your DigitalOcean API token. 4. Run the following commands to initialize Terraform, review the infrastructure changes, and apply them: ``` terraform init terraform plan terraform apply ``` Confirm the changes by typing yes when prompted. 5. Once the deployment is complete, Terraform will output the public IP address of your droplet. You can access your application using this IP address. ## How to deploy on DigitalOcean To delete the resources created by Terraform, run the following command in the terminal: ` terraform destroy ` ## Please read this notice before submitting issues about your deployment **Note:** Your instance will not be available instantly. Depending on the instance size you launched with it can take anywhere from 5-10 minutes to fully boot up. If you want to check the instances progress, navigate to [your deployed instances](https://cloud.digitalocean.com/droplets) and connect to your instance via SSH in browser. Once connected run `sudo tail -f /var/log/cloud-init-output.log` and wait for the file to conclude deployment of the docker image. Additionally, your use of this deployment process means you are responsible for any costs of these Digital Ocean resources fully. ================================================ FILE: cloud-deployments/digitalocean/terraform/main.tf ================================================ terraform { required_version = ">= 1.0.0" required_providers { digitalocean = { source = "digitalocean/digitalocean" version = "~> 2.0" } } } provider "digitalocean" { # Add your DigitalOcean API token here token = "DigitalOcean API token" } resource "digitalocean_droplet" "anything_llm_instance" { image = "ubuntu-24-04-x64" name = "anything-llm-instance" region = "nyc3" size = "s-2vcpu-2gb" user_data = templatefile("user_data.tp1", { env_content = local.formatted_env_content }) } locals { env_content = file("../../../docker/.env") formatted_env_content = join("\n", [ for line in split("\n", local.env_content) : line if !( ( substr(line, 0, 1) == "#" ) || ( substr(line, 0, 3) == "UID" ) || ( substr(line, 0, 3) == "GID" ) || ( substr(line, 0, 11) == "CLOUD_BUILD" ) || ( line == "" ) ) ]) } ================================================ FILE: cloud-deployments/digitalocean/terraform/outputs.tf ================================================ output "ip_address" { value = digitalocean_droplet.anything_llm_instance.ipv4_address description = "The public IP address of your droplet application." } ================================================ FILE: cloud-deployments/digitalocean/terraform/user_data.tp1 ================================================ #!/bin/bash # check output of userdata script with sudo tail -f /var/log/cloud-init-output.log sudo apt-get update sudo apt-get install -y docker.io sudo usermod -a -G docker ubuntu sudo systemctl enable docker sudo systemctl start docker mkdir -p /home/anythingllm cat </home/anythingllm/.env ${env_content} EOF sudo docker pull mintplexlabs/anythingllm sudo docker run -d -p 3001:3001 --cap-add SYS_ADMIN -v /home/anythingllm:/app/server/storage -v /home/anythingllm/.env:/app/server/.env -e STORAGE_DIR="/app/server/storage" mintplexlabs/anythingllm echo "Container ID: $(sudo docker ps --latest --quiet)" export ONLINE=$(curl -Is http://localhost:3001/api/ping | head -n 1|cut -d$' ' -f2) echo "Health check: $ONLINE" echo "Setup complete! AnythingLLM instance is now online!" ================================================ FILE: cloud-deployments/gcp/deployment/DEPLOY.md ================================================ # How to deploy a private AnythingLLM instance on GCP With a GCP account you can easily deploy a private AnythingLLM instance on GCP. This will create a url that you can access from any browser over HTTP (HTTPS not supported). This single instance will run on your own keys and they will not be exposed - however if you want your instance to be protected it is highly recommend that you set a password once setup is complete. The output of this cloudformation stack will be: - 1 GCP VM - 1 Security Group with 0.0.0.0/0 access on Ports 22 & 3001 - 1 GCP VM Volume `gb2` of 10Gib minimum **Requirements** - An GCP account with billing information. ## How to deploy on GCP Open your terminal 1. Log in to your GCP account using the following command: ``` gcloud auth login ``` 2. After successful login, Run the following command to create a deployment using the Deployment Manager CLI: ``` gcloud deployment-manager deployments create anything-llm-deployment --config gcp/deployment/gcp_deploy_anything_llm.yaml ``` Once you execute these steps, the CLI will initiate the deployment process on GCP based on your configuration file. You can monitor the deployment status and view the outputs using the Google Cloud Console or the Deployment Manager CLI commands. ``` gcloud compute instances get-serial-port-output anything-llm-instance ``` ssh into the instance ``` gcloud compute ssh anything-llm-instance ``` Delete the deployment ``` gcloud deployment-manager deployments delete anything-llm-deployment ``` ## Please read this notice before submitting issues about your deployment **Note:** Your instance will not be available instantly. Depending on the instance size you launched with it can take anywhere from 5-10 minutes to fully boot up. If you want to check the instances progress, navigate to [your deployed instances](https://console.cloud.google.com/compute/instances) and connect to your instance via SSH in browser. Once connected run `sudo tail -f /var/log/cloud-init-output.log` and wait for the file to conclude deployment of the docker image. Additionally, your use of this deployment process means you are responsible for any costs of these GCP resources fully. ================================================ FILE: cloud-deployments/gcp/deployment/gcp_deploy_anything_llm.yaml ================================================ resources: - name: anything-llm-instance type: compute.v1.instance properties: zone: us-central1-a machineType: zones/us-central1-a/machineTypes/n1-standard-1 disks: - deviceName: boot type: PERSISTENT boot: true autoDelete: true initializeParams: sourceImage: projects/ubuntu-os-cloud/global/images/family/ubuntu-2004-lts diskSizeGb: 10 networkInterfaces: - network: global/networks/default accessConfigs: - name: External NAT type: ONE_TO_ONE_NAT metadata: items: - key: startup-script value: | #!/bin/bash # check output of userdata script with sudo tail -f /var/log/cloud-init-output.log sudo apt-get update sudo apt-get install -y docker.io sudo usermod -a -G docker ubuntu sudo systemctl enable docker sudo systemctl start docker mkdir -p /home/anythingllm touch /home/anythingllm/.env sudo chown -R ubuntu:ubuntu /home/anythingllm sudo docker pull mintplexlabs/anythingllm sudo docker run -d -p 3001:3001 --cap-add SYS_ADMIN -v /home/anythingllm:/app/server/storage -v /home/anythingllm/.env:/app/server/.env -e STORAGE_DIR="/app/server/storage" mintplexlabs/anythingllm echo "Container ID: $(sudo docker ps --latest --quiet)" export ONLINE=$(curl -Is http://localhost:3001/api/ping | head -n 1|cut -d$' ' -f2) echo "Health check: $ONLINE" echo "Setup complete! AnythingLLM instance is now online!" ================================================ FILE: cloud-deployments/helm/charts/anythingllm/.helmignore ================================================ # Patterns to ignore when building packages. # This supports shell glob matching, relative path matching, and # negation (prefixed with !). Only one pattern per line. .DS_Store # Common VCS dirs .git/ .gitignore .bzr/ .bzrignore .hg/ .hgignore .svn/ # Common backup files *.swp *.bak *.tmp *.orig *~ # Various IDEs .project .idea/ *.tmproj .vscode/ ================================================ FILE: cloud-deployments/helm/charts/anythingllm/Chart.yaml ================================================ apiVersion: v2 name: anythingllm description: The all-in-one Desktop & Docker AI application with built-in RAG, AI agents, No-code agent builder, MCP compatibility, and more. type: application version: 1.0.0 appVersion: "1.85.0" icon: https://raw.githubusercontent.com/Mintplex-Labs/anything-llm/refs/heads/master/frontend/public/favicon.png ================================================ FILE: cloud-deployments/helm/charts/anythingllm/README.md ================================================ # anythingllm ![Version: 1.0.0](https://img.shields.io/badge/Version-1.0.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.85.0](https://img.shields.io/badge/AppVersion-1.85.0-informational?style=flat-square) ![AnythingLLM](https://raw.githubusercontent.com/Mintplex-Labs/anything-llm/master/images/wordmark.png) [AnythingLLM](https://github.com/Mintplex-Labs/anything-llm) The all-in-one Desktop & Docker AI application with built-in RAG, AI agents, No-code agent builder, MCP compatibility, and more. **Configuration & Usage** - **Config vs Secrets:** This chart exposes application configuration via two mechanisms: - `config` (in `values.yaml`) — rendered into a `ConfigMap` and injected using `envFrom` in the pod. Do NOT place sensitive values (API keys, secrets) in `config` because `ConfigMap`s are not encrypted. - `env` / `envFrom` — the preferred way to inject secrets. Use Kubernetes `Secret` objects and reference them from `env` (with `valueFrom.secretKeyRef`) or `envFrom.secretRef`. - **Storage & STORAGE_DIR mapping:** The chart creates (or mounts) a `PersistentVolumeClaim` using the `persistentVolume.*` settings. The container mount path is set from `persistentVolume.mountPath`. Ensure the container `STORAGE_DIR` config key matches that path (defaults are set in `values.yaml`). **Providing API keys & secrets (recommended)** Use Kubernetes Secrets. Below are example workflows and `values.yaml` snippets. 1) Create a Kubernetes Secret with API keys: ``` kubectl create secret generic openai-secret --from-literal=OPENAI_KEY="sk-..." # or from a file # kubectl create secret generic openai-secret --from-file=OPENAI_KEY=/path/to/keyfile ``` 2) Reference the Secret from `values.yaml` using `envFrom` (recommended when your secret contains multiple env keys): ```yaml envFrom: - secretRef: name: openai-secret ``` This will inject all key/value pairs from the `openai-secret` Secret as environment variables in the container. 3) Or reference a single secret key via `env` (explicit mapping): ```yaml env: - name: OPENAI_KEY valueFrom: secretKeyRef: name: openai-secret key: OPENAI_KEY ``` Notes: - Avoid placing secret values into `config:` (the chart's `ConfigMap`) — `ConfigMap`s are visible to anyone who can read the namespace. Use `Secret` objects for any credentials/tokens. - If you use a GitOps workflow, consider integrating an external secret operator (ExternalSecrets, SealedSecrets, etc.) so you don't store raw secrets in Git. **Example `values-secret.yaml` to pass during `helm install`** ```yaml image: repository: mintplexlabs/anythingllm tag: "1.11.2" service: type: ClusterIP port: 3001 # Reference secret containing API keys envFrom: - secretRef: name: openai-secret # Optionally override other values persistentVolume: size: 16Gi mountPath: /storage ``` Install with: ``` helm install my-anythingllm ./anythingllm -f values-secret.yaml ``` **Best practices & tips** - Use `envFrom` for convenience when many environment variables are stored in a single `Secret` and use `env`/`valueFrom` for explicit single-key mappings. - Use `kubectl create secret generic` or your secrets management solution. If you need to reference multiple different provider keys (OpenAI, Anthropic, etc.), create a single `Secret` with multiple keys or multiple Secrets and add multiple `envFrom` entries. - Keep probe paths and `service.port` aligned. If your probes fail after deployment, check that the probe `port` matches the container port (or named port `http`) and that the `path` is valid. - For storage, if you have a pre-existing PVC set `persistentVolume.existingClaim` to the PVC name; the chart will mount that claim (and will not attempt to create a new PVC). - For production, provide resource `requests` and `limits` in `values.yaml` to prevent scheduler starvation and to control cost. ## Values | Key | Type | Default | Description | |-----|------|---------|-------------| | affinity | object | `{}` | | | config.DISABLE_TELEMETRY | string | `"true"` | | | config.GID | string | `"1000"` | | | config.NODE_ENV | string | `"production"` | | | config.STORAGE_DIR | string | `"/storage"` | | | config.UID | string | `"1000"` | | | env | object | `{}` | | | envFrom | object | `{}` | | | fullnameOverride | string | `""` | | | image.pullPolicy | string | `"IfNotPresent"` | | | image.repository | string | `"mintplexlabs/anythingllm"` | | | image.tag | string | `"1.11.2"` | | | imagePullSecrets | list | `[]` | | | ingress.annotations | object | `{}` | | | ingress.className | string | `""` | | | ingress.enabled | bool | `false` | | | ingress.hosts[0].host | string | `"chart-example.local"` | | | ingress.hosts[0].paths[0].path | string | `"/"` | | | ingress.hosts[0].paths[0].pathType | string | `"ImplementationSpecific"` | | | ingress.tls | list | `[]` | | | initContainers | list | `[]` | | | livenessProbe.failureThreshold | int | `3` | | | livenessProbe.httpGet.path | string | `"/v1/api/health"` | | | livenessProbe.httpGet.port | int | `8888` | | | livenessProbe.initialDelaySeconds | int | `15` | | | livenessProbe.periodSeconds | int | `5` | | | nameOverride | string | `""` | | | nodeSelector | object | `{}` | | | persistentVolume.accessModes[0] | string | `"ReadWriteOnce"` | | | persistentVolume.annotations | object | `{}` | | | persistentVolume.existingClaim | string | `""` | | | persistentVolume.labels | object | `{}` | | | persistentVolume.mountPath | string | `"/storage"` | | | persistentVolume.size | string | `"8Gi"` | | | podAnnotations | object | `{}` | | | podLabels | object | `{}` | | | podSecurityContext.fsGroup | int | `1000` | | | readinessProbe.httpGet.path | string | `"/v1/api/health"` | | | readinessProbe.httpGet.port | int | `8888` | | | readinessProbe.initialDelaySeconds | int | `15` | | | readinessProbe.periodSeconds | int | `5` | | | readinessProbe.successThreshold | int | `2` | | | replicaCount | int | `1` | | | resources | object | `{}` | | | securityContext | object | `{}` | | | service.port | int | `3001` | | | service.type | string | `"ClusterIP"` | | | serviceAccount.annotations | object | `{}` | | | serviceAccount.automount | bool | `true` | | | serviceAccount.create | bool | `true` | | | serviceAccount.name | string | `""` | | | tolerations | list | `[]` | | | volumeMounts | list | `[]` | | | volumes | list | `[]` | | ================================================ FILE: cloud-deployments/helm/charts/anythingllm/README.md.gotmpl ================================================ {{ template "chart.header" . }} {{ template "chart.deprecationWarning" . }} {{ template "chart.badgesSection" . }} ![AnythingLLM](https://raw.githubusercontent.com/Mintplex-Labs/anything-llm/master/images/wordmark.png) [AnythingLLM](https://github.com/Mintplex-Labs/anything-llm) {{ template "chart.description" . }} {{ template "chart.homepageLine" . }} {{ template "chart.maintainersSection" . }} {{ template "chart.sourcesSection" . }} {{ template "chart.requirementsSection" . }} **Configuration & Usage** - **Config vs Secrets:** This chart exposes application configuration via two mechanisms: - `config` (in `values.yaml`) — rendered into a `ConfigMap` and injected using `envFrom` in the pod. Do NOT place sensitive values (API keys, secrets) in `config` because `ConfigMap`s are not encrypted. - `env` / `envFrom` — the preferred way to inject secrets. Use Kubernetes `Secret` objects and reference them from `env` (with `valueFrom.secretKeyRef`) or `envFrom.secretRef`. - **Storage & STORAGE_DIR mapping:** The chart creates (or mounts) a `PersistentVolumeClaim` using the `persistentVolume.*` settings. The container mount path is set from `persistentVolume.mountPath`. Ensure the container `STORAGE_DIR` config key matches that path (defaults are set in `values.yaml`). **Providing API keys & secrets (recommended)** Use Kubernetes Secrets. Below are example workflows and `values.yaml` snippets. 1) Create a Kubernetes Secret with API keys: ``` kubectl create secret generic openai-secret --from-literal=OPENAI_KEY="sk-..." # or from a file # kubectl create secret generic openai-secret --from-file=OPENAI_KEY=/path/to/keyfile ``` 2) Reference the Secret from `values.yaml` using `envFrom` (recommended when your secret contains multiple env keys): ```yaml envFrom: - secretRef: name: openai-secret ``` This will inject all key/value pairs from the `openai-secret` Secret as environment variables in the container. 3) Or reference a single secret key via `env` (explicit mapping): ```yaml env: - name: OPENAI_KEY valueFrom: secretKeyRef: name: openai-secret key: OPENAI_KEY ``` Notes: - Avoid placing secret values into `config:` (the chart's `ConfigMap`) — `ConfigMap`s are visible to anyone who can read the namespace. Use `Secret` objects for any credentials/tokens. - If you use a GitOps workflow, consider integrating an external secret operator (ExternalSecrets, SealedSecrets, etc.) so you don't store raw secrets in Git. **Example `values-secret.yaml` to pass during `helm install`** ```yaml image: repository: mintplexlabs/anythingllm tag: "1.11.2" service: type: ClusterIP port: 3001 # Reference secret containing API keys envFrom: - secretRef: name: openai-secret # Optionally override other values persistentVolume: size: 16Gi mountPath: /storage ``` Install with: ``` helm install my-anythingllm ./anythingllm -f values-secret.yaml ``` **Best practices & tips** - Use `envFrom` for convenience when many environment variables are stored in a single `Secret` and use `env`/`valueFrom` for explicit single-key mappings. - Use `kubectl create secret generic` or your secrets management solution. If you need to reference multiple different provider keys (OpenAI, Anthropic, etc.), create a single `Secret` with multiple keys or multiple Secrets and add multiple `envFrom` entries. - Keep probe paths and `service.port` aligned. If your probes fail after deployment, check that the probe `port` matches the container port (or named port `http`) and that the `path` is valid. - For storage, if you have a pre-existing PVC set `persistentVolume.existingClaim` to the PVC name; the chart will mount that claim (and will not attempt to create a new PVC). - For production, provide resource `requests` and `limits` in `values.yaml` to prevent scheduler starvation and to control cost. {{ template "chart.valuesSection" . }} ================================================ FILE: cloud-deployments/helm/charts/anythingllm/templates/NOTES.txt ================================================ 1. Get the application URL by running these commands: {{- if .Values.ingress.enabled }} {{- range $host := .Values.ingress.hosts }} {{- range .paths }} http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} {{- end }} {{- end }} {{- else if contains "NodePort" .Values.service.type }} export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "anythingllm.fullname" . }}) export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") echo http://$NODE_IP:$NODE_PORT {{- else if contains "LoadBalancer" .Values.service.type }} NOTE: It may take a few minutes for the LoadBalancer IP to be available. You can watch its status by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "anythingllm.fullname" . }}' export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "anythingllm.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") echo http://$SERVICE_IP:{{ .Values.service.port }} {{- else if contains "ClusterIP" .Values.service.type }} export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "anythingllm.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") echo "To access locally, run:" echo " kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT" echo "Then visit http://127.0.0.1:8080" {{- end }} ================================================ FILE: cloud-deployments/helm/charts/anythingllm/templates/_helpers.tpl ================================================ {{/* Expand the name of the chart. */}} {{- define "anythingllm.name" -}} {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} {{- end }} {{/* Create a default fully qualified app name. We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). If release name contains chart name it will be used as a full name. */}} {{- define "anythingllm.fullname" -}} {{- if .Values.fullnameOverride }} {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} {{- else }} {{- $name := default .Chart.Name .Values.nameOverride }} {{- if contains $name .Release.Name }} {{- .Release.Name | trunc 63 | trimSuffix "-" }} {{- else }} {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} {{- end }} {{- end }} {{- end }} {{/* Create chart name and version as used by the chart label. */}} {{- define "anythingllm.chart" -}} {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} {{- end }} {{/* Common labels */}} {{- define "anythingllm.labels" -}} helm.sh/chart: {{ include "anythingllm.chart" . }} {{ include "anythingllm.selectorLabels" . }} {{- if .Chart.AppVersion }} app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} {{- end }} app.kubernetes.io/managed-by: {{ .Release.Service }} {{- end }} {{/* Selector labels */}} {{- define "anythingllm.selectorLabels" -}} app.kubernetes.io/name: {{ include "anythingllm.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} {{/* Create the name of the service account to use */}} {{- define "anythingllm.serviceAccountName" -}} {{- if .Values.serviceAccount.create }} {{- default (include "anythingllm.fullname" .) .Values.serviceAccount.name }} {{- else }} {{- default "default" .Values.serviceAccount.name }} {{- end }} {{- end }} ================================================ FILE: cloud-deployments/helm/charts/anythingllm/templates/configmap.yaml ================================================ apiVersion: v1 kind: ConfigMap metadata: labels: {{- include "anythingllm.labels" . | nindent 4 }} name: {{ include "anythingllm.fullname" . }}-config data: {{- range $key, $value := .Values.config }} {{ $key }}: "{{ $value }}" {{- end }} ================================================ FILE: cloud-deployments/helm/charts/anythingllm/templates/deployment.yaml ================================================ apiVersion: apps/v1 kind: Deployment metadata: name: {{ include "anythingllm.fullname" . }} labels: {{- include "anythingllm.labels" . | nindent 4 }} spec: replicas: {{ .Values.replicaCount }} selector: matchLabels: {{- include "anythingllm.selectorLabels" . | nindent 6 }} template: metadata: {{- with .Values.podAnnotations }} annotations: {{- toYaml . | nindent 8 }} {{- end }} labels: {{- include "anythingllm.labels" . | nindent 8 }} {{- with .Values.podLabels }} {{- toYaml . | nindent 8 }} {{- end }} spec: {{- with .Values.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end }} serviceAccountName: {{ include "anythingllm.serviceAccountName" . }} securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} {{- with .Values.initContainers }} initContainers: {{- toYaml . | nindent 8 }} {{- end }} {{- with .Values.strategy }} strategy: {{- toYaml . | nindent 8 }} {{- end }} containers: - name: {{ .Chart.Name }} securityContext: {{- toYaml .Values.securityContext | nindent 12 }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} {{- with .Values.env }} env: {{- toYaml . | nindent 12 }} {{- end }} envFrom: - configMapRef: name: {{ include "anythingllm.fullname" . }}-config {{- with .Values.envFrom }} {{- toYaml . | nindent 12 }} {{- end }} ports: - name: http containerPort: {{ .Values.service.port }} protocol: TCP livenessProbe: {{- toYaml .Values.livenessProbe | nindent 12 }} readinessProbe: {{- toYaml .Values.readinessProbe | nindent 12 }} resources: {{- toYaml .Values.resources | nindent 12 }} volumeMounts: - name: storage mountPath: {{ .Values.persistentVolume.mountPath }} volumes: - name: storage persistentVolumeClaim: claimName: {{ include "anythingllm.fullname" . }}-storage-claim {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} {{- end }} {{- with .Values.affinity }} affinity: {{- toYaml . | nindent 8 }} {{- end }} {{- with .Values.tolerations }} tolerations: {{- toYaml . | nindent 8 }} {{- end }} ================================================ FILE: cloud-deployments/helm/charts/anythingllm/templates/extra-objects.yaml ================================================ {{ range .Values.extraObjects }} --- {{ tpl (toYaml .) $ }} {{ end }} ================================================ FILE: cloud-deployments/helm/charts/anythingllm/templates/ingress.yaml ================================================ {{- if .Values.ingress.enabled -}} {{- $fullName := include "anythingllm.fullname" . -}} {{- $svcPort := .Values.service.port -}} {{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} {{- end }} {{- end }} {{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} apiVersion: networking.k8s.io/v1 {{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} apiVersion: networking.k8s.io/v1beta1 {{- else -}} apiVersion: extensions/v1beta1 {{- end }} kind: Ingress metadata: name: {{ $fullName }} labels: {{- include "anythingllm.labels" . | nindent 4 }} {{- with .Values.ingress.annotations }} annotations: {{- toYaml . | nindent 4 }} {{- end }} spec: {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} ingressClassName: {{ .Values.ingress.className }} {{- end }} {{- if .Values.ingress.tls }} tls: {{- range .Values.ingress.tls }} - hosts: {{- range .hosts }} - {{ . | quote }} {{- end }} secretName: {{ .secretName }} {{- end }} {{- end }} rules: {{- range .Values.ingress.hosts }} - host: {{ .host | quote }} http: paths: {{- range .paths }} - path: {{ .path }} {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} pathType: {{ .pathType }} {{- end }} backend: {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} service: name: {{ $fullName }} port: number: {{ $svcPort }} {{- else }} serviceName: {{ $fullName }} servicePort: {{ $svcPort }} {{- end }} {{- end }} {{- end }} {{- end }} ================================================ FILE: cloud-deployments/helm/charts/anythingllm/templates/pvc.yaml ================================================ apiVersion: v1 kind: PersistentVolumeClaim metadata: {{- if .Values.persistentVolume.annotations }} annotations: {{ toYaml .Values.persistentVolume.annotations | indent 4 }} {{- end }} labels: {{- include "anythingllm.labels" . | nindent 4 }} {{- with .Values.persistentVolume.labels }} {{- toYaml . | nindent 4 }} {{- end }} name: {{ include "anythingllm.fullname" . }}-storage-claim spec: accessModes: {{- toYaml .Values.persistentVolume.accessModes | nindent 4 }} {{- if .Values.persistentVolume.storageClass }} {{- if (eq "-" .Values.persistentVolume.storageClass) }} storageClassName: "" {{- else }} storageClassName: "{{ .Values.persistentVolume.storageClass }}" {{- end }} {{- end }} resources: requests: storage: {{ .Values.persistentVolume.size }} {{- if .Values.persistentVolume.volumeName }} volumeName: "{{ .Values.persistentVolume.volumeName }}" {{- end -}} {{- with .Values.persistentVolume.selector }} selector: {{- toYaml . | nindent 4 }} {{- end }} ================================================ FILE: cloud-deployments/helm/charts/anythingllm/templates/service.yaml ================================================ apiVersion: v1 kind: Service metadata: name: {{ include "anythingllm.fullname" . }} labels: {{- include "anythingllm.labels" . | nindent 4 }} spec: type: {{ .Values.service.type }} ports: - port: {{ .Values.service.port }} targetPort: http protocol: TCP name: http selector: {{- include "anythingllm.selectorLabels" . | nindent 4 }} ================================================ FILE: cloud-deployments/helm/charts/anythingllm/templates/serviceaccount.yaml ================================================ {{- if .Values.serviceAccount.create -}} apiVersion: v1 kind: ServiceAccount metadata: name: {{ include "anythingllm.serviceAccountName" . }} labels: {{- include "anythingllm.labels" . | nindent 4 }} {{- with .Values.serviceAccount.annotations }} annotations: {{- toYaml . | nindent 4 }} {{- end }} automountServiceAccountToken: {{ .Values.serviceAccount.automount }} {{- end }} ================================================ FILE: cloud-deployments/helm/charts/anythingllm/templates/tests/test-connection.yaml ================================================ apiVersion: v1 kind: Pod metadata: name: "{{ include "anythingllm.fullname" . }}-test-connection" labels: {{- include "anythingllm.labels" . | nindent 4 }} annotations: "helm.sh/hook": test spec: containers: - name: healthcheck image: curlimages/curl:8.1.2 command: ["sh", "-c"] args: - "curl -fsS -o /dev/null http://{{ include "anythingllm.fullname" . }}:{{ .Values.service.port }}|| exit 1" restartPolicy: Never ================================================ FILE: cloud-deployments/helm/charts/anythingllm/values.yaml ================================================ replicaCount: 1 initContainers: [] # - name: init-myservice # image: busybox # command: ['sh', '-c', 'chown -R 1000:1000 /storage'] image: repository: mintplexlabs/anythingllm pullPolicy: IfNotPresent tag: "1.11.2" imagePullSecrets: [] nameOverride: "" fullnameOverride: "" persistentVolume: # AnythingLLM storage data Persistent Volume access modes # Must match those of existing PV or dynamic provisioner # Ref: http://kubernetes.io/docs/user-guide/persistent-volumes/ # accessModes: - ReadWriteOnce # AnythingLLM storage data Persistent Volume labels # labels: {} # AnythingLLM storage data Persistent Volume annotations # annotations: {} # AnythingLLM storage data Persistent Volume existing claim name # If defined, PVC must be created manually before volume will be bound # existingClaim: "" # AnythingLLM storage data Persistent Volume size # size: 8Gi # AnythingLLM storage data Persistent Volume mount path # Must match the STORAGE_DIR config value # mountPath: /app/server/storage # AnythingLLM storage data Persistent Volume Storage Class # If defined, storageClassName: # If set to "-", storageClassName: "", which disables dynamic provisioning # If undefined (the default) or set to null, no storageClassName spec is # set, choosing the default provisioner. (gp2 on AWS, standard on # GKE, AWS & OpenStack) # storageClass: "" # AnythingLLM storage data Persistent Volume Claim Selector # Useful if Persistent Volumes have been provisioned in advance # Ref: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#selector # selector: {} # selector: # matchLabels: # release: "stable" # matchExpressions: # - { key: environment, operator: In, values: [ dev ] } # AnythingLLM storage data Persistent Volume Name # Useful if Persistent Volumes have been provisioned in advance and you want to use a specific one # volumeName: "" serviceAccount: # Specifies whether a service account should be created create: true # Automatically mount a ServiceAccount's API credentials? automount: true # Annotations to add to the service account annotations: {} # The name of the service account to use. # If not set and create is true, a name is generated using the fullname template name: "" # The Anything LLM application deployment strategy # This is set to "Recreate" by default as AnythingLLM support is not yet # production ready. Once it is, this can be changed to "RollingUpdate" # Ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy # strategy: # Type of deployment. Can be "Recreate" or "RollingUpdate". Default is "Recreate" type: Recreate # If type is "RollingUpdate", the following values can be set: # rollingUpdate: # maxUnavailable: 1 # maxSurge: 1 podAnnotations: {} podLabels: {} podSecurityContext: # fsGroup needs to be set as the same as the uid/gid used to run the application # in order to have the right permissions on mounted volumes fsGroup: 1000 securityContext: {} # capabilities: # drop: # - ALL # readOnlyRootFilesystem: true # runAsNonRoot: true # runAsUser: 1000 # AnythingLLM configuration options, these are stored in a ConfigMap and passed # to the container as environment variables. # See https://github.com/Mintplex-Labs/anything-llm/blob/render/docker/.env.example # for all available environment variables to use as configuration options # config: DISABLE_TELEMETRY: "true" NODE_ENV: production STORAGE_DIR: /app/server/storage UID: "1000" GID: "1000" # The preferred method for setting secret environment variables # Ref: https://kubernetes.io/docs/tasks/inject-data-application/distribute-credentials-secure/#define-a-container-environment-variable-with-data-from-a-single-secret # env: {} # - name: OPEN_AI_KEY # valueFrom: # secretKeyRef: # name: openai-secret # key: openai_key # Typically used to reference a pre existing Secret containing multiple environment variables # Ref: https://kubernetes.io/docs/tasks/inject-data-application/distribute-credentials-secure/#define-a-container-environment-variable-with-data-from-a-single-secret # envFrom: {} # - secretRef: # name: mysecret service: type: ClusterIP port: 3001 ingress: enabled: false className: "" annotations: {} # kubernetes.io/ingress.class: nginx # kubernetes.io/tls-acme: "true" hosts: - host: chart-example.local paths: - path: / pathType: ImplementationSpecific tls: [] # - secretName: chart-example-tls # hosts: # - chart-example.local resources: {} # We usually recommend not to specify default resources and to leave this as a conscious # choice for the user. This also increases chances charts run on environments with little # resources, such as Minikube. If you do want to specify resources, uncomment the following # lines, adjust them as necessary, and remove the curly braces after 'resources:'. # limits: # cpu: 100m # memory: 128Mi # requests: # cpu: 100m # memory: 128Mi readinessProbe: httpGet: path: /v1/api/health port: 8888 initialDelaySeconds: 15 periodSeconds: 5 successThreshold: 2 livenessProbe: httpGet: path: /v1/api/health port: 8888 initialDelaySeconds: 15 periodSeconds: 5 failureThreshold: 3 # Additional volumes on the output Deployment definition. # volumes: [] # - name: foo # secret: # secretName: mysecret # optional: false # Additional volumeMounts on the output Deployment definition. # volumeMounts: [] # - name: foo # mountPath: "/etc/foo" # readOnly: true nodeSelector: {} tolerations: [] affinity: {} ## Array of extra manifests/obhects to create # extraObjects: [] # - apiVersion: external-secrets.io/v1beta1 # kind: ExternalSecret # metadata: # name: open-ai-api-key-external-secret # namespace: default # spec: # refreshInterval: 1h # secretStoreRef: # name: vault # kind: ClusterSecretStore # target: # name: open-ai-api-key-secret # template: # type: Opaque # data: # - secretKey: open_ai_key # remoteRef: # key: secret/data/anything-llm # property: open_ai_key ================================================ FILE: cloud-deployments/huggingface-spaces/Dockerfile ================================================ # With this dockerfile in a Huggingface space you will get an entire AnythingLLM instance running # in your space with all features you would normally get from the docker based version of AnythingLLM. # # How to use # - Login to https://huggingface.co/spaces # - Click on "Create new Space" # - Name the space and select "Docker" as the SDK w/ a blank template # - The default 2vCPU/16GB machine is OK. The more the merrier. # - Decide if you want your AnythingLLM Space public or private. # **You might want to stay private until you at least set a password or enable multi-user mode** # - Click "Create Space" # - Click on "Settings" on top of page (https://huggingface.co/spaces///settings) # - Scroll to "Persistent Storage" and select the lowest tier of now - you can upgrade if you run out. # - Confirm and continue storage upgrade # - Go to "Files" Tab (https://huggingface.co/spaces///tree/main) # - Click "Add Files" # - Upload this file or create a file named `Dockerfile` and copy-paste this content into it. "Commit to main" and save. # - Your container will build and boot. You now have AnythingLLM on HuggingFace. Your data is stored in the persistent storage attached. # Have Fun 🤗 # Have issues? Check the logs on HuggingFace for clues. FROM mintplexlabs/anythingllm:render USER root RUN mkdir -p /data/storage RUN ln -s /data/storage /storage USER anythingllm ENV STORAGE_DIR="/data/storage" ENV SERVER_PORT=7860 ENTRYPOINT ["/bin/bash", "/usr/local/bin/render-entrypoint.sh"] ================================================ FILE: cloud-deployments/k8/manifest.yaml ================================================ --- apiVersion: v1 kind: PersistentVolume metadata: name: anything-llm-volume annotations: pv.beta.kubernetes.io/uid: "1000" pv.beta.kubernetes.io/gid: "1000" spec: storageClassName: gp2 capacity: storage: 5Gi accessModes: - ReadWriteOnce awsElasticBlockStore: # This is the volume UUID from AWS EC2 EBS Volumes list. volumeID: "{{ anythingllm_awsElasticBlockStore_volumeID }}" fsType: ext4 nodeAffinity: required: nodeSelectorTerms: - matchExpressions: - key: topology.kubernetes.io/zone operator: In values: - us-east-1c --- apiVersion: v1 kind: PersistentVolumeClaim metadata: name: anything-llm-volume-claim namespace: "{{ namespace }}" spec: accessModes: - ReadWriteOnce resources: requests: storage: 5Gi --- apiVersion: apps/v1 kind: Deployment metadata: name: anything-llm namespace: "{{ namespace }}" labels: anything-llm: "true" spec: selector: matchLabels: k8s-app: anything-llm replicas: 1 strategy: type: RollingUpdate rollingUpdate: maxSurge: 0% maxUnavailable: 100% template: metadata: labels: anything-llm: "true" k8s-app: anything-llm app.kubernetes.io/name: anything-llm app.kubernetes.io/part-of: anything-llm annotations: prometheus.io/scrape: "true" prometheus.io/path: /metrics prometheus.io/port: "9090" spec: serviceAccountName: "default" terminationGracePeriodSeconds: 10 securityContext: fsGroup: 1000 runAsNonRoot: true runAsGroup: 1000 runAsUser: 1000 affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: topology.kubernetes.io/zone operator: In values: - us-east-1c containers: - name: anything-llm resources: limits: memory: "1Gi" cpu: "500m" requests: memory: "512Mi" cpu: "250m" imagePullPolicy: IfNotPresent image: "mintplexlabs/anythingllm:render" securityContext: allowPrivilegeEscalation: true capabilities: add: - SYS_ADMIN runAsNonRoot: true runAsGroup: 1000 runAsUser: 1000 command: # Specify a command to override the Dockerfile's ENTRYPOINT. - /bin/bash - -c - | set -x -e sleep 3 echo "AWS_REGION: $AWS_REGION" echo "SERVER_PORT: $SERVER_PORT" echo "NODE_ENV: $NODE_ENV" echo "STORAGE_DIR: $STORAGE_DIR" { cd /app/server/ && npx prisma generate --schema=./prisma/schema.prisma && npx prisma migrate deploy --schema=./prisma/schema.prisma && node /app/server/index.js echo "Server process exited with status $?" } & { node /app/collector/index.js echo "Collector process exited with status $?" } & wait -n exit $? readinessProbe: httpGet: path: /v1/api/health port: 8888 initialDelaySeconds: 15 periodSeconds: 5 successThreshold: 2 livenessProbe: httpGet: path: /v1/api/health port: 8888 initialDelaySeconds: 15 periodSeconds: 5 failureThreshold: 3 env: - name: AWS_REGION value: "{{ aws_region }}" - name: AWS_ACCESS_KEY_ID value: "{{ aws_access_id }}" - name: AWS_SECRET_ACCESS_KEY value: "{{ aws_access_secret }}" - name: SERVER_PORT value: "3001" - name: JWT_SECRET value: "my-random-string-for-seeding" # Please generate random string at least 12 chars long. - name: STORAGE_DIR value: "/storage" - name: NODE_ENV value: "production" - name: UID value: "1000" - name: GID value: "1000" volumeMounts: - name: anything-llm-server-storage-volume-mount mountPath: /storage volumes: - name: anything-llm-server-storage-volume-mount persistentVolumeClaim: claimName: anything-llm-volume-claim --- # This serves the UI and the backend. apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: anything-llm-ingress namespace: "{{ namespace }}" annotations: external-dns.alpha.kubernetes.io/hostname: "{{ namespace }}-chat.{{ base_domain }}" kubernetes.io/ingress.class: "internal-ingress" nginx.ingress.kubernetes.io/rewrite-target: / ingress.kubernetes.io/ssl-redirect: "false" spec: rules: - host: "{{ namespace }}-chat.{{ base_domain }}" http: paths: - path: / pathType: Prefix backend: service: name: anything-llm-svc port: number: 3001 tls: # < placing a host in the TLS config will indicate a cert should be created - hosts: - "{{ namespace }}-chat.{{ base_domain }}" secretName: letsencrypt-prod --- apiVersion: v1 kind: Service metadata: labels: kubernetes.io/name: anything-llm name: anything-llm-svc namespace: "{{ namespace }}" spec: ports: # "port" is external port, and "targetPort" is internal. - port: 3301 targetPort: 3001 name: traffic - port: 9090 targetPort: 9090 name: metrics selector: k8s-app: anything-llm ================================================ FILE: collector/.env.example ================================================ # Placeholder .env file for collector runtime # This enables HTTP request/response logging in development. Set value to truthy string to enable, leave empty value or comment out to disable # ENABLE_HTTP_LOGGER="" # This enables timestamps for the HTTP Logger. Set value to true to enable, leave empty or comment out to disable # ENABLE_HTTP_LOGGER_TIMESTAMPS="" ================================================ FILE: collector/.gitignore ================================================ hotdir/* !hotdir/__HOTDIR__.md yarn-error.log !yarn.lock outputs scripts .env.development .env.production .env.test ================================================ FILE: collector/.nvmrc ================================================ v18.18.0 ================================================ FILE: collector/__tests__/utils/WhisperProviders/ffmpeg/index.test.js ================================================ process.env.STORAGE_DIR = "test-storage"; const fs = require("fs"); const path = require("path"); // Mock fix-path as a noop to prevent SIGSEGV (segfault) // Returns ESM-style default export for dynamic import() jest.mock("fix-path", () => ({ default: jest.fn() })); const { FFMPEGWrapper } = require("../../../../utils/WhisperProviders/ffmpeg"); const describeRunner = process.env.GITHUB_ACTIONS ? describe.skip : describe; describeRunner("FFMPEGWrapper", () => { /** @type { import("../../../../utils/WhisperProviders/ffmpeg/index").FFMPEGWrapper } */ let ffmpeg; const testDir = path.resolve(__dirname, "../../../../storage/tmp"); const inputPath = path.resolve(testDir, "test-input.wav"); const outputPath = path.resolve(testDir, "test-output.wav"); beforeEach(() => { ffmpeg = new FFMPEGWrapper(); }); afterEach(() => { if (fs.existsSync(inputPath)) fs.rmSync(inputPath); if (fs.existsSync(outputPath)) fs.rmSync(outputPath); }); it("should find ffmpeg executable", async () => { const knownPath = await ffmpeg.ffmpegPath(); expect(knownPath).toBeDefined(); expect(typeof knownPath).toBe("string"); expect(knownPath.length).toBeGreaterThan(0); }); it("should validate ffmpeg executable", async () => { const knownPath = await ffmpeg.ffmpegPath(); expect(ffmpeg.isValidFFMPEG(knownPath)).toBe(true); }); it("should return false for invalid ffmpeg path", () => { expect(ffmpeg.isValidFFMPEG("/invalid/path/to/ffmpeg")).toBe(false); }); it("should convert audio file to wav format", async () => { if (!fs.existsSync(testDir)) fs.mkdirSync(testDir, { recursive: true }); const sampleUrl = "https://github.com/ringcentral/ringcentral-api-docs/blob/main/resources/sample1.wav?raw=true"; const response = await fetch(sampleUrl); if (!response.ok) throw new Error( `Failed to download sample file: ${response.statusText}` ); const buffer = await response.arrayBuffer(); fs.writeFileSync(inputPath, Buffer.from(buffer)); const result = await ffmpeg.convertAudioToWav(inputPath, outputPath); expect(result).toBe(true); expect(fs.existsSync(outputPath)).toBe(true); const stats = fs.statSync(outputPath); expect(stats.size).toBeGreaterThan(0); }, 30000); it("should throw error when conversion fails", () => { const nonExistentFile = path.resolve(testDir, "non-existent-file.wav"); const outputPath = path.resolve(testDir, "test-output-fail.wav"); expect(async () => { return await ffmpeg.convertAudioToWav(nonExistentFile, outputPath); }).rejects.toThrow(`Input file ${nonExistentFile} does not exist.`); }); }); ================================================ FILE: collector/__tests__/utils/url/index.test.js ================================================ process.env.STORAGE_DIR = "test-storage"; // needed for tests to run const { validURL, validateURL, validYoutubeVideoUrl } = require("../../../utils/url"); // Mock the RuntimeSettings module jest.mock("../../../utils/runtimeSettings", () => { const mockInstance = { get: jest.fn(), set: jest.fn(), }; return jest.fn().mockImplementation(() => mockInstance); }); describe("validURL", () => { let mockRuntimeSettings; beforeEach(() => { const RuntimeSettings = require("../../../utils/runtimeSettings"); mockRuntimeSettings = new RuntimeSettings(); jest.clearAllMocks(); }); it("should validate a valid URL", () => { mockRuntimeSettings.get.mockImplementation((key) => { if (key === "allowAnyIp") return false; if (key === "seenAnyIpWarning") return true; // silence the warning for tests return false; }); expect(validURL("https://www.google.com")).toBe(true); expect(validURL("http://www.google.com")).toBe(true); // JS URL does not require extensions, so in theory // these should be valid expect(validURL("https://random")).toBe(true); expect(validURL("http://123")).toBe(true); // missing protocols expect(validURL("www.google.com")).toBe(false); expect(validURL("google.com")).toBe(false); // invalid protocols expect(validURL("ftp://www.google.com")).toBe(false); expect(validURL("mailto://www.google.com")).toBe(false); expect(validURL("tel://www.google.com")).toBe(false); expect(validURL("data://www.google.com")).toBe(false); }); it("should block private/local IPs when allowAnyIp is false (default behavior)", () => { mockRuntimeSettings.get.mockImplementation((key) => { if (key === "allowAnyIp") return false; if (key === "seenAnyIpWarning") return true; // silence the warning for tests return false; }); expect(validURL("http://192.168.1.1")).toBe(false); expect(validURL("http://10.0.0.1")).toBe(false); expect(validURL("http://172.16.0.1")).toBe(false); // But localhost should still be allowed expect(validURL("http://127.0.0.1")).toBe(true); expect(validURL("http://0.0.0.0")).toBe(true); }); it("should allow any IP when allowAnyIp is true", () => { mockRuntimeSettings.get.mockImplementation((key) => { if (key === "allowAnyIp") return true; if (key === "seenAnyIpWarning") return true; // silence the warning for tests return false; }); expect(validURL("http://192.168.1.1")).toBe(true); expect(validURL("http://10.0.0.1")).toBe(true); expect(validURL("http://172.16.0.1")).toBe(true); }); }); describe("validateURL", () => { it("should return the same URL if it's already valid", () => { expect(validateURL("https://www.google.com")).toBe( "https://www.google.com" ); expect(validateURL("http://www.google.com")).toBe("http://www.google.com"); expect(validateURL("https://random")).toBe("https://random"); // With numbers as a url this will turn into an ip expect(validateURL("123")).toBe("https://0.0.0.123"); expect(validateURL("123.123.123.123")).toBe("https://123.123.123.123"); expect(validateURL("http://127.0.123.45")).toBe("http://127.0.123.45"); }); it("should assume https:// if the URL doesn't have a protocol", () => { expect(validateURL("www.google.com")).toBe("https://www.google.com"); expect(validateURL("google.com")).toBe("https://google.com"); expect(validateURL("EXAMPLE.com/ABCDEF/q1=UPPER")).toBe("https://example.com/ABCDEF/q1=UPPER"); expect(validateURL("ftp://www.google.com")).toBe("ftp://www.google.com"); expect(validateURL("mailto://www.google.com")).toBe( "mailto://www.google.com" ); expect(validateURL("tel://www.google.com")).toBe("tel://www.google.com"); expect(validateURL("data://www.google.com")).toBe("data://www.google.com"); }); it("should remove trailing slashes post-validation", () => { expect(validateURL("https://www.google.com/")).toBe( "https://www.google.com" ); expect(validateURL("http://www.google.com/")).toBe("http://www.google.com"); expect(validateURL("https://random/")).toBe("https://random"); expect(validateURL("https://example.com/ABCDEF/")).toBe("https://example.com/ABCDEF"); }); it("should handle edge cases and bad data inputs", () => { expect(validateURL({})).toBe(""); expect(validateURL(null)).toBe(""); expect(validateURL(undefined)).toBe(""); expect(validateURL(124512)).toBe(""); expect(validateURL("")).toBe(""); expect(validateURL(" ")).toBe(""); expect(validateURL(" look here! ")).toBe("look here!"); }); it("should preserve case of characters in URL pathname", () => { expect(validateURL("https://example.com/To/ResOURce?q1=Value&qZ22=UPPE!R")) .toBe("https://example.com/To/ResOURce?q1=Value&qZ22=UPPE!R"); expect(validateURL("https://sample.com/uPeRCaSe")) .toBe("https://sample.com/uPeRCaSe"); expect(validateURL("Example.com/PATH/To/Resource?q2=Value&q1=UPPER")) .toBe("https://example.com/PATH/To/Resource?q2=Value&q1=UPPER"); }); }); describe("validYoutubeVideoUrl", () => { const ID = "dQw4w9WgXcQ"; // 11-char valid video id it("returns true for youtube watch URLs with v param", () => { expect(validYoutubeVideoUrl(`https://www.youtube.com/watch?v=${ID}`)).toBe( true ); expect(validYoutubeVideoUrl(`https://youtube.com/watch?v=${ID}&t=10s`)).toBe( true ); expect(validYoutubeVideoUrl(`https://m.youtube.com/watch?v=${ID}`)).toBe(true); expect(validYoutubeVideoUrl(`youtube.com/watch?v=${ID}`)).toBe(true); }); it("returns true for youtu.be short URLs", () => { expect(validYoutubeVideoUrl(`https://youtu.be/${ID}`)).toBe(true); expect(validYoutubeVideoUrl(`https://youtu.be/${ID}?si=abc`)).toBe(true); // extra path segments after id should still validate the id component expect(validYoutubeVideoUrl(`https://youtu.be/${ID}/extra`)).toBe(true); }); it("returns true for embed and v path formats", () => { expect(validYoutubeVideoUrl(`https://www.youtube.com/embed/${ID}`)).toBe(true); expect(validYoutubeVideoUrl(`https://youtube.com/v/${ID}`)).toBe(true); }); it("returns false for non-YouTube hosts", () => { expect(validYoutubeVideoUrl("https://example.com/watch?v=dQw4w9WgXcQ")).toBe( false ); expect(validYoutubeVideoUrl("https://vimeo.com/123456")).toBe(false); }); it("returns false for unrelated YouTube paths without a video id", () => { expect(validYoutubeVideoUrl("https://www.youtube.com/user/somechannel")).toBe( false ); expect(validYoutubeVideoUrl("https://www.youtube.com/")).toBe(false); }); it("returns false for empty or bad inputs", () => { expect(validYoutubeVideoUrl("")).toBe(false); expect(validYoutubeVideoUrl(null)).toBe(false); expect(validYoutubeVideoUrl(undefined)).toBe(false); }); it("returns the video ID for valid YouTube video URLs", () => { expect(validYoutubeVideoUrl(`https://www.youtube.com/watch?v=${ID}`, true)).toBe(ID); expect(validYoutubeVideoUrl(`https://youtube.com/watch?v=${ID}&t=10s`, true)).toBe(ID); expect(validYoutubeVideoUrl(`https://m.youtube.com/watch?v=${ID}`, true)).toBe(ID); expect(validYoutubeVideoUrl(`youtube.com/watch?v=${ID}`, true)).toBe(ID); expect(validYoutubeVideoUrl(`https://youtu.be/${ID}`, true)).toBe(ID); expect(validYoutubeVideoUrl(`https://youtu.be/${ID}?si=abc`, true)).toBe(ID); expect(validYoutubeVideoUrl(`https://youtu.be/${ID}/extra`, true)).toBe(ID); expect(validYoutubeVideoUrl(`https://www.youtube.com/embed/${ID}`, true)).toBe(ID); expect(validYoutubeVideoUrl(`https://youtube.com/v/${ID}`, true)).toBe(ID); // invalid video IDs expect(validYoutubeVideoUrl(`https://www.youtube.com/watch?v=invalid`, true)).toBe(null); expect(validYoutubeVideoUrl(`https://youtube.com/watch?v=invalid`, true)).toBe(null); expect(validYoutubeVideoUrl(`https://m.youtube.com/watch?v=invalid`, true)).toBe(null); expect(validYoutubeVideoUrl(`youtube.com/watch`, true)).toBe(null); expect(validYoutubeVideoUrl(`https://youtu.be/invalid`, true)).toBe(null); expect(validYoutubeVideoUrl(`https://youtu.be/invalid?si=abc`, true)).toBe(null); }); }); ================================================ FILE: collector/eslint.config.mjs ================================================ import js from "@eslint/js"; import globals from "globals"; import { defineConfig } from "eslint/config"; import pluginPrettier from "eslint-plugin-prettier"; import configPrettier from "eslint-config-prettier"; import unusedImports from "eslint-plugin-unused-imports"; export default defineConfig([ { ignores: ["__tests__/**"] }, { files: ["**/*.{js,mjs,cjs}"], plugins: { js, prettier: pluginPrettier, "unused-imports": unusedImports }, extends: ["js/recommended"], languageOptions: { globals: { ...globals.node, ...globals.browser } }, rules: { ...configPrettier.rules, "prettier/prettier": "error", "no-case-declarations": "off", "no-prototype-builtins": "off", "no-async-promise-executor": "off", "no-extra-boolean-cast": "off", "no-empty": "off", "no-unused-private-class-members": "warn", "no-unused-vars": "off", "unused-imports/no-unused-imports": "error", "unused-imports/no-unused-vars": [ "error", { vars: "all", varsIgnorePattern: "^_", args: "after-used", argsIgnorePattern: "^_", }, ], }, }, { files: ["**/*.js"], languageOptions: { sourceType: "commonjs" } }, ]); ================================================ FILE: collector/extensions/index.js ================================================ const { setDataSigner } = require("../middleware/setDataSigner"); const { verifyPayloadIntegrity } = require("../middleware/verifyIntegrity"); const { resolveRepoLoader, resolveRepoLoaderFunction, } = require("../utils/extensions/RepoLoader"); const { reqBody } = require("../utils/http"); const { validURL, validateURL } = require("../utils/url"); const RESYNC_METHODS = require("./resync"); const { loadObsidianVault } = require("../utils/extensions/ObsidianVault"); function extensions(app) { if (!app) return; app.post( "/ext/resync-source-document", [verifyPayloadIntegrity, setDataSigner], async function (request, response) { try { const { type, options } = reqBody(request); if (!RESYNC_METHODS.hasOwnProperty(type)) throw new Error(`Type "${type}" is not a valid type to sync.`); return await RESYNC_METHODS[type](options, response); } catch (e) { console.error(e); response.status(200).json({ success: false, content: null, reason: e.message || "A processing error occurred.", }); } return; } ); app.post( "/ext/:repo_platform-repo", [verifyPayloadIntegrity, setDataSigner], async function (request, response) { try { const loadRepo = resolveRepoLoaderFunction( request.params.repo_platform ); const { success, reason, data } = await loadRepo( reqBody(request), response ); response.status(200).json({ success, reason, data, }); } catch (e) { console.error(e); response.status(200).json({ success: false, reason: e.message || "A processing error occurred.", data: {}, }); } return; } ); // gets all branches for a specific repo app.post( "/ext/:repo_platform-repo/branches", [verifyPayloadIntegrity], async function (request, response) { try { const RepoLoader = resolveRepoLoader(request.params.repo_platform); const allBranches = await new RepoLoader( reqBody(request) ).getRepoBranches(); response.status(200).json({ success: true, reason: null, data: { branches: allBranches, }, }); } catch (e) { console.error(e); response.status(400).json({ success: false, reason: e.message, data: { branches: [], }, }); } return; } ); app.post( "/ext/youtube-transcript", [verifyPayloadIntegrity], async function (request, response) { try { const { loadYouTubeTranscript, } = require("../utils/extensions/YoutubeTranscript"); const { success, reason, data } = await loadYouTubeTranscript( reqBody(request) ); response.status(200).json({ success, reason, data }); } catch (e) { console.error(e); response.status(400).json({ success: false, reason: e.message, data: { title: null, author: null, }, }); } return; } ); app.post( "/ext/website-depth", [verifyPayloadIntegrity], async function (request, response) { try { const websiteDepth = require("../utils/extensions/WebsiteDepth"); const { url, depth = 1, maxLinks = 20 } = reqBody(request); const validatedUrl = validateURL(url); if (!validURL(validatedUrl)) throw new Error("Not a valid URL."); const scrapedData = await websiteDepth(validatedUrl, depth, maxLinks); response.status(200).json({ success: true, data: scrapedData }); } catch (e) { console.error(e); response.status(400).json({ success: false, reason: e.message }); } return; } ); app.post( "/ext/confluence", [verifyPayloadIntegrity, setDataSigner], async function (request, response) { try { const { loadConfluence } = require("../utils/extensions/Confluence"); const { success, reason, data } = await loadConfluence( reqBody(request), response ); response.status(200).json({ success, reason, data }); } catch (e) { console.error(e); response.status(400).json({ success: false, reason: e.message, data: { title: null, author: null, }, }); } return; } ); app.post( "/ext/drupalwiki", [verifyPayloadIntegrity, setDataSigner], async function (request, response) { try { const { loadAndStoreSpaces, } = require("../utils/extensions/DrupalWiki"); const { success, reason, data } = await loadAndStoreSpaces( reqBody(request), response ); response.status(200).json({ success, reason, data }); } catch (e) { console.error(e); response.status(400).json({ success: false, reason: e.message, data: { title: null, author: null, }, }); } return; } ); app.post( "/ext/obsidian/vault", [verifyPayloadIntegrity, setDataSigner], async function (request, response) { try { const { files } = reqBody(request); const result = await loadObsidianVault({ files }); response.status(200).json(result); } catch (e) { console.error(e); response.status(400).json({ success: false, reason: e.message, data: null, }); } return; } ); app.post( "/ext/paperless-ngx", [verifyPayloadIntegrity, setDataSigner], async function (request, response) { try { const { loadPaperlessNgx, } = require("../utils/extensions/PaperlessNgx"); const result = await loadPaperlessNgx(reqBody(request), response); response.status(200).json(result); } catch (e) { console.error(e); response.status(400).json({ success: false, reason: e.message, data: null, }); } return; } ); } module.exports = extensions; ================================================ FILE: collector/extensions/resync/index.js ================================================ const { getLinkText } = require("../../processLink"); /** * Fetches the content of a raw link. Returns the content as a text string of the link in question. * @param {object} data - metadata from document (eg: link) * @param {import("../../middleware/setDataSigner").ResponseWithSigner} response */ async function resyncLink({ link }, response) { if (!link) throw new Error("Invalid link provided"); try { const { success, content = null, reason } = await getLinkText(link); if (!success) throw new Error(`Failed to sync link content. ${reason}`); response.status(200).json({ success, content }); } catch (e) { console.error(e); response.status(200).json({ success: false, content: null, }); } } /** * Fetches the content of a YouTube link. Returns the content as a text string of the video in question. * We offer this as there may be some videos where a transcription could be manually edited after initial scraping * but in general - transcriptions often never change. * @param {object} data - metadata from document (eg: link) * @param {import("../../middleware/setDataSigner").ResponseWithSigner} response */ async function resyncYouTube({ link }, response) { if (!link) throw new Error("Invalid link provided"); try { const { fetchVideoTranscriptContent, } = require("../../utils/extensions/YoutubeTranscript"); const { success, reason, content } = await fetchVideoTranscriptContent({ url: link, }); if (!success) throw new Error(`Failed to sync YouTube video transcript. ${reason}`); response.status(200).json({ success, content }); } catch (e) { console.error(e); response.status(200).json({ success: false, content: null, }); } } /** * Fetches the content of a specific confluence page via its chunkSource. * Returns the content as a text string of the page in question and only that page. * @param {object} data - metadata from document (eg: chunkSource) * @param {import("../../middleware/setDataSigner").ResponseWithSigner} response */ async function resyncConfluence({ chunkSource }, response) { if (!chunkSource) throw new Error("Invalid source property provided"); try { // Confluence data is `payload` encrypted. So we need to expand its // encrypted payload back into query params so we can reFetch the page with same access token/params. const source = response.locals.encryptionWorker.expandPayload(chunkSource); const { fetchConfluencePage, } = require("../../utils/extensions/Confluence"); const { success, reason, content } = await fetchConfluencePage({ pageUrl: `https:${source.pathname}`, // need to add back the real protocol baseUrl: source.searchParams.get("baseUrl"), spaceKey: source.searchParams.get("spaceKey"), accessToken: source.searchParams.get("token"), username: source.searchParams.get("username"), cloud: source.searchParams.get("cloud") === "true", bypassSSL: source.searchParams.get("bypassSSL") === "true", }); if (!success) throw new Error(`Failed to sync Confluence page content. ${reason}`); response.status(200).json({ success, content }); } catch (e) { console.error(e); response.status(200).json({ success: false, content: null, }); } } /** * Fetches the content of a specific confluence page via its chunkSource. * Returns the content as a text string of the page in question and only that page. * @param {object} data - metadata from document (eg: chunkSource) * @param {import("../../middleware/setDataSigner").ResponseWithSigner} response */ async function resyncGithub({ chunkSource }, response) { if (!chunkSource) throw new Error("Invalid source property provided"); try { // Github file data is `payload` encrypted (might contain PAT). So we need to expand its // encrypted payload back into query params so we can reFetch the page with same access token/params. const source = response.locals.encryptionWorker.expandPayload(chunkSource); const { fetchGithubFile, } = require("../../utils/extensions/RepoLoader/GithubRepo"); const { success, reason, content } = await fetchGithubFile({ repoUrl: `https:${source.pathname}`, // need to add back the real protocol branch: source.searchParams.get("branch"), accessToken: source.searchParams.get("pat"), sourceFilePath: source.searchParams.get("path"), }); if (!success) throw new Error(`Failed to sync GitHub file content. ${reason}`); response.status(200).json({ success, content }); } catch (e) { console.error(e); response.status(200).json({ success: false, content: null, }); } } /** * Fetches the content of a specific DrupalWiki page via its chunkSource. * Returns the content as a text string of the page in question and only that page. * @param {object} data - metadata from document (eg: chunkSource) * @param {import("../../middleware/setDataSigner").ResponseWithSigner} response */ async function resyncDrupalWiki({ chunkSource }, response) { if (!chunkSource) throw new Error("Invalid source property provided"); try { // DrupalWiki data is `payload` encrypted. So we need to expand its // encrypted payload back into query params so we can reFetch the page with same access token/params. const source = response.locals.encryptionWorker.expandPayload(chunkSource); const { loadPage } = require("../../utils/extensions/DrupalWiki"); const { success, reason, content } = await loadPage({ baseUrl: source.searchParams.get("baseUrl"), pageId: source.searchParams.get("pageId"), accessToken: source.searchParams.get("accessToken"), }); if (!success) { console.error(`Failed to sync DrupalWiki page content. ${reason}`); response.status(200).json({ success: false, content: null, }); } else { response.status(200).json({ success, content }); } } catch (e) { console.error(e); response.status(200).json({ success: false, content: null, }); } } /** * Fetches the content of a specific Paperless-ngx document via its chunkSource. * Returns the content as a text string of the document. * @param {object} data - metadata from document (eg: chunkSource) * @param {import("../../middleware/setDataSigner").ResponseWithSigner} response */ async function resyncPaperlessNgx({ chunkSource }, response) { if (!chunkSource) throw new Error("Invalid source property provided"); try { const source = response.locals.encryptionWorker.expandPayload(chunkSource); const { PaperlessNgxLoader, } = require("../../utils/extensions/PaperlessNgx/PaperlessNgxLoader"); const loader = new PaperlessNgxLoader({ baseUrl: source.searchParams.get("baseUrl"), apiToken: source.searchParams.get("token"), }); const documentId = source.pathname.split("//")[1]; const content = await loader.fetchDocumentContent(documentId); if (!content) throw new Error("Failed to fetch document content"); response.status(200).json({ success: true, content }); } catch (e) { console.error(e); response.status(200).json({ success: false, content: null, }); } } module.exports = { link: resyncLink, youtube: resyncYouTube, confluence: resyncConfluence, github: resyncGithub, drupalwiki: resyncDrupalWiki, "paperless-ngx": resyncPaperlessNgx, }; ================================================ FILE: collector/hotdir/__HOTDIR__.md ================================================ ### What is the "Hot directory" This is a pre-set file location that documents will be written to when uploaded by AnythingLLM. There is really no need to touch it. ================================================ FILE: collector/index.js ================================================ process.env.NODE_ENV === "development" ? require("dotenv").config({ path: `.env.${process.env.NODE_ENV}` }) : require("dotenv").config(); require("./utils/logger")(); const express = require("express"); const bodyParser = require("body-parser"); const cors = require("cors"); const path = require("path"); const { ACCEPTED_MIMES } = require("./utils/constants"); const { reqBody } = require("./utils/http"); const { processSingleFile } = require("./processSingleFile"); const { processLink, getLinkText } = require("./processLink"); const { wipeCollectorStorage } = require("./utils/files"); const extensions = require("./extensions"); const { processRawText } = require("./processRawText"); const { verifyPayloadIntegrity } = require("./middleware/verifyIntegrity"); const { httpLogger } = require("./middleware/httpLogger"); const app = express(); const FILE_LIMIT = "3GB"; // Only log HTTP requests in development mode and if the ENABLE_HTTP_LOGGER environment variable is set to true if ( process.env.NODE_ENV === "development" && !!process.env.ENABLE_HTTP_LOGGER ) { app.use( httpLogger({ enableTimestamps: !!process.env.ENABLE_HTTP_LOGGER_TIMESTAMPS, }) ); } app.use(cors({ origin: true })); app.use( bodyParser.text({ limit: FILE_LIMIT }), bodyParser.json({ limit: FILE_LIMIT }), bodyParser.urlencoded({ limit: FILE_LIMIT, extended: true, }) ); app.post( "/process", [verifyPayloadIntegrity], async function (request, response) { const { filename, options = {}, metadata = {} } = reqBody(request); try { const targetFilename = path .normalize(filename) .replace(/^(\.\.(\/|\\|$))+/, ""); const { success, reason, documents = [], } = await processSingleFile(targetFilename, options, metadata); response .status(200) .json({ filename: targetFilename, success, reason, documents }); } catch (e) { console.error(e); response.status(200).json({ filename: filename, success: false, reason: "A processing error occurred.", documents: [], }); } return; } ); app.post( "/parse", [verifyPayloadIntegrity], async function (request, response) { const { filename, options = {} } = reqBody(request); try { const targetFilename = path .normalize(filename) .replace(/^(\.\.(\/|\\|$))+/, ""); const { success, reason, documents = [], } = await processSingleFile(targetFilename, { ...options, parseOnly: true, }); response .status(200) .json({ filename: targetFilename, success, reason, documents }); } catch (e) { console.error(e); response.status(200).json({ filename: filename, success: false, reason: "A processing error occurred.", documents: [], }); } return; } ); app.post( "/process-link", [verifyPayloadIntegrity], async function (request, response) { const { link, scraperHeaders = {}, metadata = {} } = reqBody(request); try { const { success, reason, documents = [], } = await processLink(link, scraperHeaders, metadata); response.status(200).json({ url: link, success, reason, documents }); } catch (e) { console.error(e); response.status(200).json({ url: link, success: false, reason: "A processing error occurred.", documents: [], }); } return; } ); app.post( "/util/get-link", [verifyPayloadIntegrity], async function (request, response) { const { link, captureAs = "text" } = reqBody(request); try { const { success, content = null } = await getLinkText(link, captureAs); response.status(200).json({ url: link, success, content }); } catch (e) { console.error(e); response.status(200).json({ url: link, success: false, content: null, }); } return; } ); app.post( "/process-raw-text", [verifyPayloadIntegrity], async function (request, response) { const { textContent, metadata } = reqBody(request); try { const { success, reason, documents = [], } = await processRawText(textContent, metadata); response .status(200) .json({ filename: metadata.title, success, reason, documents }); } catch (e) { console.error(e); response.status(200).json({ filename: metadata?.title || "Unknown-doc.txt", success: false, reason: "A processing error occurred.", documents: [], }); } return; } ); extensions(app); app.get("/accepts", function (_, response) { response.status(200).json(ACCEPTED_MIMES); }); app.all("*", function (_, response) { response.sendStatus(200); }); app .listen(8888, async () => { await wipeCollectorStorage(); console.log(`Document processor app listening on port 8888`); }) .on("error", function (_) { process.once("SIGUSR2", function () { process.kill(process.pid, "SIGUSR2"); }); process.on("SIGINT", function () { process.kill(process.pid, "SIGINT"); }); }); ================================================ FILE: collector/middleware/httpLogger.js ================================================ const httpLogger = ({ enableTimestamps = false }) => (req, res, next) => { // Capture the original res.end to log response status const originalEnd = res.end; res.end = function (chunk, encoding) { // Log the request method, status code, and path const statusColor = res.statusCode >= 400 ? "\x1b[31m" : "\x1b[32m"; // Red for errors, green for success console.log( `\x1b[32m[HTTP]\x1b[0m ${statusColor}${res.statusCode}\x1b[0m ${ req.method } -> ${req.path} ${ enableTimestamps ? `@ ${new Date().toLocaleTimeString("en-US", { hour12: true })}` : "" }`.trim() ); // Call the original end method return originalEnd.call(this, chunk, encoding); }; next(); }; module.exports = { httpLogger, }; ================================================ FILE: collector/middleware/setDataSigner.js ================================================ const { EncryptionWorker } = require("../utils/EncryptionWorker"); const { CommunicationKey } = require("../utils/comKey"); /** * Express Response Object interface with defined encryptionWorker attached to locals property. * @typedef {import("express").Response & import("express").Response['locals'] & {encryptionWorker: EncryptionWorker} } ResponseWithSigner */ // You can use this middleware to assign the EncryptionWorker to the response locals // property so that if can be used to encrypt/decrypt arbitrary data via response object. // eg: Encrypting API keys in chunk sources. // The way this functions is that the rolling RSA Communication Key is used server-side to private-key encrypt the raw // key of the persistent EncryptionManager credentials. Since EncryptionManager credentials do _not_ roll, we should not send them // even between server<>collector in plaintext because if the user configured the server/collector to be public they could technically // be exposing the key in transit via the X-Payload-Signer header. Even if this risk is minimal we should not do this. // This middleware uses the CommunicationKey public key to first decrypt the base64 representation of the EncryptionManager credentials // and then loads that in to the EncryptionWorker as a buffer so we can use the same credentials across the system. Should we ever break the // collector out into its own service this would still work without SSL/TLS. /** * * @param {import("express").Request} request * @param {import("express").Response} response * @param {import("express").NextFunction} next */ function setDataSigner(request, response, next) { const comKey = new CommunicationKey(); const encryptedPayloadSigner = request.header("X-Payload-Signer"); if (!encryptedPayloadSigner) console.log( "Failed to find signed-payload to set encryption worker! Encryption calls will fail." ); const decryptedPayloadSignerKey = comKey.decrypt(encryptedPayloadSigner); const encryptionWorker = new EncryptionWorker(decryptedPayloadSignerKey); response.locals.encryptionWorker = encryptionWorker; next(); } module.exports = { setDataSigner, }; ================================================ FILE: collector/middleware/verifyIntegrity.js ================================================ const { CommunicationKey } = require("../utils/comKey"); const RuntimeSettings = require("../utils/runtimeSettings"); const runtimeSettings = new RuntimeSettings(); function verifyPayloadIntegrity(request, response, next) { const comKey = new CommunicationKey(); if (process.env.NODE_ENV === "development") { comKey.log("verifyPayloadIntegrity is skipped in development."); runtimeSettings.parseOptionsFromRequest(request); next(); return; } const signature = request.header("X-Integrity"); if (!signature) return response .status(400) .json({ msg: "Failed integrity signature check." }); const validSignedPayload = comKey.verify(signature, request.body); if (!validSignedPayload) return response .status(400) .json({ msg: "Failed integrity signature check." }); runtimeSettings.parseOptionsFromRequest(request); next(); } module.exports = { verifyPayloadIntegrity, }; ================================================ FILE: collector/nodemon.json ================================================ { "events": {} } ================================================ FILE: collector/package.json ================================================ { "name": "anything-llm-document-collector", "version": "1.11.1", "description": "Document collector server endpoints", "main": "index.js", "author": "Timothy Carambat (Mintplex Labs)", "license": "MIT", "private": false, "engines": { "node": ">=18.12.1" }, "scripts": { "dev": "cross-env NODE_ENV=development nodemon --ignore hotdir --ignore storage --trace-warnings index.js", "start": "cross-env NODE_ENV=production node index.js", "lint": "eslint --fix .", "lint:check": "eslint ." }, "dependencies": { "@langchain/community": "^0.2.23", "@xenova/transformers": "^2.14.0", "body-parser": "^1.20.3", "cors": "^2.8.5", "dotenv": "^16.0.3", "epub2": "git+https://github.com/Mintplex-Labs/epub2-static.git#main", "express": "^4.21.2", "fix-path": "^4.0.0", "html-to-text": "^9.0.5", "ignore": "^5.3.0", "js-tiktoken": "^1.0.8", "langchain": "0.1.36", "mammoth": "^1.6.0", "mbox-parser": "^1.0.1", "mime": "^3.0.0", "moment": "^2.29.4", "node-html-parser": "^6.1.13", "node-xlsx": "^0.24.0", "officeparser": "^4.0.5", "openai": "4.95.1", "pdf-parse": "^1.1.1", "puppeteer": "~21.5.2", "sharp": "^0.33.5", "slugify": "^1.6.6", "strip-ansi": "^7.1.2", "tesseract.js": "^6.0.0", "url-pattern": "^1.0.3", "uuid": "^9.0.0", "wavefile": "^11.0.0", "winston": "^3.13.0", "youtube-transcript-plus": "^1.1.2", "youtubei.js": "^9.1.0" }, "devDependencies": { "@eslint/js": "^9.0.0", "cross-env": "^7.0.3", "eslint": "^9.0.0", "eslint-config-prettier": "^9.0.0", "eslint-plugin-prettier": "^5.0.0", "eslint-plugin-unused-imports": "^4.0.0", "globals": "^17.4.0", "nodemon": "^2.0.22", "prettier": "^2.4.1" }, "resolutions": { "string-width": "^4.2.3", "wrap-ansi": "^7.0.0" } } ================================================ FILE: collector/processLink/convert/generic.js ================================================ const { v4 } = require("uuid"); const { PuppeteerWebBaseLoader, } = require("langchain/document_loaders/web/puppeteer"); const { writeToServerDocuments } = require("../../utils/files"); const { tokenizeString } = require("../../utils/tokenizer"); const { default: slugify } = require("slugify"); const { returnResult, determineContentType, processAsFile, } = require("../helpers"); const { loadYouTubeTranscript, } = require("../../utils/extensions/YoutubeTranscript"); const RuntimeSettings = require("../../utils/runtimeSettings"); /** * Scrape a generic URL and return the content in the specified format * @param {Object} config - The configuration object * @param {string} config.link - The URL to scrape * @param {('html' | 'text')} config.captureAs - The format to capture the page content as. Default is 'text' * @param {{[key: string]: string}} config.scraperHeaders - Custom headers to use when making the request * @param {{[key: string]: string}} config.metadata - Metadata to use when creating the document * @param {boolean} config.saveAsDocument - Whether to save the content as a document. Default is true * @returns {Promise} - The content of the page */ async function scrapeGenericUrl({ link, captureAs = "text", scraperHeaders = {}, metadata = {}, saveAsDocument = true, }) { /** @type {'web' | 'file' | 'youtube'} */ console.log(`-- Working URL ${link} => (captureAs: ${captureAs}) --`); let { contentType, processVia } = await determineContentType(link); console.log(`-- URL determined to be ${contentType} (${processVia}) --`); /** * When the content is a file or a YouTube video, we can use the existing processing functions * These are self-contained and will return the correct response based on the saveAsDocument flag already * so we can return the content immediately. */ if (processVia === "file") return await processAsFile({ uri: link, saveAsDocument }); else if (processVia === "youtube") return await loadYouTubeTranscript( { url: link }, { parseOnly: saveAsDocument === false } ); // Otherwise, assume the content is a webpage and scrape the content from the webpage const content = await getPageContent({ link, captureAs, headers: scraperHeaders, }); if (!content || !content.length) { console.error(`Resulting URL content was empty at ${link}.`); return returnResult({ success: false, reason: `No URL content found at ${link}.`, documents: [], content: null, saveAsDocument, }); } // If the captureAs is text, return the content as a string immediately // so that we dont save the content as a document if (!saveAsDocument) return returnResult({ success: true, content, saveAsDocument, }); // Save the content as a document from the URL const url = new URL(link); const decodedPathname = decodeURIComponent(url.pathname); const filename = `${url.hostname}${decodedPathname.replace(/\//g, "_")}`; const data = { id: v4(), url: "file://" + slugify(filename) + ".html", title: metadata.title || slugify(filename) + ".html", docAuthor: metadata.docAuthor || "no author found", description: metadata.description || "No description found.", docSource: metadata.docSource || "URL link uploaded by the user.", chunkSource: `link://${link}`, published: new Date().toLocaleString(), wordCount: content.split(" ").length, pageContent: content, token_count_estimate: tokenizeString(content), }; const document = writeToServerDocuments({ data, filename: `url-${slugify(filename)}-${data.id}`, }); console.log(`[SUCCESS]: URL ${link} converted & ready for embedding.\n`); return { success: true, reason: null, documents: [document] }; } /** * Validate the headers object * - Keys & Values must be strings and not empty * - Assemble a new object with only the valid keys and values * @param {{[key: string]: string}} headers - The headers object to validate * @returns {{[key: string]: string}} - The validated headers object */ function validatedHeaders(headers = {}) { try { if (Object.keys(headers).length === 0) return {}; let validHeaders = {}; for (const key of Object.keys(headers)) { if (!key?.trim()) continue; if (typeof headers[key] !== "string" || !headers[key]?.trim()) continue; validHeaders[key] = headers[key].trim(); } return validHeaders; } catch (error) { console.error("Error validating headers", error); return {}; } } /** * Get the content of a page * @param {Object} config - The configuration object * @param {string} config.link - The URL to get the content of * @param {('html' | 'text')} config.captureAs - The format to capture the page content as. Default is 'text' * @param {{[key: string]: string}} config.headers - Custom headers to use when making the request * @returns {Promise} - The content of the page */ async function getPageContent({ link, captureAs = "text", headers = {} }) { try { let pageContents = []; const runtimeSettings = new RuntimeSettings(); /** @type {import('puppeteer').PuppeteerLaunchOptions} */ let launchConfig = { headless: "new" }; /* On MacOS 15.1, the headless=new option causes the browser to crash immediately. * It is not clear why this is the case, but it is reproducible. Since AnythinglLM * in production runs in a container, we can disable headless mode to workaround the issue for development purposes. * * This may show a popup window when scraping a page in development mode. * This is expected behavior if seen in development mode on MacOS 15+ */ if ( process.platform === "darwin" && process.env.NODE_ENV === "development" ) { console.log( "Darwin Development Mode: Disabling headless mode to prevent Chromium from crashing." ); launchConfig.headless = "false"; } const loader = new PuppeteerWebBaseLoader(link, { launchOptions: { headless: launchConfig.headless, ignoreHTTPSErrors: true, args: runtimeSettings.get("browserLaunchArgs"), }, gotoOptions: { waitUntil: "networkidle2", }, async evaluate(page, browser) { const result = await page.evaluate((captureAs) => { if (captureAs === "text") return document.body.innerText; if (captureAs === "html") return document.documentElement.innerHTML; return document.body.innerText; }, captureAs); await browser.close(); return result; }, }); // Override scrape method if headers are available let overrideHeaders = validatedHeaders(headers); if (Object.keys(overrideHeaders).length > 0) { loader.scrape = async function () { const { launch } = await PuppeteerWebBaseLoader.imports(); const browser = await launch({ headless: "new", defaultViewport: null, ignoreDefaultArgs: ["--disable-extensions"], ...this.options?.launchOptions, }); const page = await browser.newPage(); await page.setExtraHTTPHeaders(overrideHeaders); await page.goto(this.webPath, { timeout: 180000, waitUntil: "networkidle2", ...this.options?.gotoOptions, }); const bodyHTML = this.options?.evaluate ? await this.options.evaluate(page, browser) : await page.evaluate(() => document.body.innerHTML); await browser.close(); return bodyHTML; }; } const docs = await loader.load(); for (const doc of docs) pageContents.push(doc.pageContent); return pageContents.join(" "); } catch (error) { console.error( "getPageContent failed to be fetched by puppeteer - falling back to fetch!", error ); } try { const pageText = await fetch(link, { method: "GET", headers: { "Content-Type": "text/plain", "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.83 Safari/537.36,gzip(gfe)", ...validatedHeaders(headers), }, }).then((res) => res.text()); return pageText; } catch (error) { console.error("getPageContent failed to be fetched by any method.", error); } return null; } module.exports = { scrapeGenericUrl, }; ================================================ FILE: collector/processLink/helpers/index.js ================================================ const path = require("path"); const { validURL } = require("../../utils/url"); const { processSingleFile } = require("../../processSingleFile"); const { downloadURIToFile } = require("../../utils/downloadURIToFile"); const { ACCEPTED_MIMES } = require("../../utils/constants"); const { validYoutubeVideoUrl } = require("../../utils/url"); /** * Get the content type of a resource * - Sends a HEAD request to the URL and returns the Content-Type header with a 5 second timeout * @param {string} url - The URL to get the content type of * @returns {Promise<{success: boolean, reason: string|null, contentType: string|null}>} - The content type of the resource */ async function getContentTypeFromURL(url) { try { if (!url || typeof url !== "string" || !validURL(url)) return { success: false, reason: "Not a valid URL.", contentType: null }; const abortController = new AbortController(); const timeout = setTimeout(() => { abortController.abort(); console.error("Timeout fetching content type for URL:", url.toString()); }, 5_000); const res = await fetch(url, { method: "HEAD", signal: abortController.signal, }).finally(() => clearTimeout(timeout)); if (!res.ok) return { success: false, reason: `HTTP ${res.status}: ${res.statusText}`, contentType: null, }; const contentType = res.headers.get("Content-Type")?.toLowerCase(); const contentTypeWithoutCharset = contentType?.split(";")[0].trim(); if (!contentTypeWithoutCharset) return { success: false, reason: "No Content-Type found.", contentType: null, }; return { success: true, reason: null, contentType: contentTypeWithoutCharset, }; } catch (error) { return { success: false, reason: `Error: ${error.message}`, contentType: null, }; } } /** * Normalize the result object based on the saveAsDocument flag * @param {Object} result - The result object to normalize * @param {boolean} result.success - Whether the result is successful * @param {string|null} result.reason - The reason for the result * @param {Object[]} result.documents - The documents from the result * @param {string|null} result.content - The content of the result * @param {boolean} result.saveAsDocument - Whether to save the content as a document. Default is true * @returns {{success: boolean, reason: string|null, documents: Object[], content: string|null}} - The normalized result object */ function returnResult({ success, reason, documents, content, saveAsDocument = true, } = {}) { if (!saveAsDocument) { return { success, content, }; } else return { success, reason, documents }; } /** * Determine the content type of a link - should be a URL * @param {string} uri - The link to determine the content type of * @returns {Promise<{contentType: string|null, processVia: 'web' | 'file' | 'youtube'}>} - The content type of the link */ async function determineContentType(uri) { let processVia = "web"; // Dont check for content type if it is a YouTube video URL if (validYoutubeVideoUrl(uri)) return { contentType: "text/html", processVia: "youtube" }; return await getContentTypeFromURL(uri) .then((result) => { if (!!result.reason) console.error(result.reason); // If the content type is not text/html or text/plain, and it is in the ACCEPTED_MIMES, // then we can process it as a file if ( !!result.contentType && !["text/html", "text/plain"].includes(result.contentType) && result.contentType in ACCEPTED_MIMES ) processVia = "file"; return { contentType: result.contentType, processVia }; }) .catch((error) => { console.error("Error getting content type from URL", error); return { contentType: null, processVia }; }); } /** * Process a link as a file * @param {string} uri - The link to process as a file * @param {boolean} saveAsDocument - Whether to save the content as a document. Default is true * @returns {Promise<{success: boolean, reason: string|null, documents: Object[], content: string|null, saveAsDocument: boolean}>} - The content of the file */ async function processAsFile({ uri, saveAsDocument = true }) { const fileContentResult = await downloadURIToFile(uri); if (!fileContentResult.success) return returnResult({ success: false, reason: fileContentResult.reason, documents: [], content: null, saveAsDocument, }); const fileFilePath = fileContentResult.fileLocation; const targetFilename = path.basename(fileFilePath); /** * If the saveAsDocument is false, we are only interested in the text content * and can ignore the file as a document by using `parseOnly` in the options. * This will send the file to the Direct Uploads folder instead of the Documents folder. * that will be deleted by the cleanup-orphan-documents job that runs frequently. The trade off * is that since it still is in FS we can debug its output or even potentially reuse it for other purposes. * * TODO: Improve this process via a new option that will instantly delete the file after processing * if we find we dont need this file ever after processing. */ const processSingleFileResult = await processSingleFile(targetFilename, { parseOnly: saveAsDocument === false, }); if (!processSingleFileResult.success) { return returnResult({ success: false, reason: processSingleFileResult.reason, documents: [], content: null, saveAsDocument, }); } // If we intend to return only the text content, return the content from the file // and then delete the file - otherwise it will be saved as a document if (!saveAsDocument) { return returnResult({ success: true, content: processSingleFileResult.documents[0].pageContent, saveAsDocument, }); } return processSingleFileResult; } module.exports = { returnResult, getContentTypeFromURL, determineContentType, processAsFile, }; ================================================ FILE: collector/processLink/index.js ================================================ const { validURL } = require("../utils/url"); const { scrapeGenericUrl } = require("./convert/generic"); const { validateURL } = require("../utils/url"); /** * Process a link and return the text content. This util will save the link as a document * so it can be used for embedding later. * @param {string} link - The link to process * @param {{[key: string]: string}} scraperHeaders - Custom headers to apply when scraping the link * @param {Object} metadata - Optional metadata to attach to the document * @returns {Promise<{success: boolean, content: string}>} - Response from collector */ async function processLink(link, scraperHeaders = {}, metadata = {}) { const validatedLink = validateURL(link); if (!validURL(validatedLink)) return { success: false, reason: "Not a valid URL." }; return await scrapeGenericUrl({ link: validatedLink, captureAs: "text", scraperHeaders, metadata, saveAsDocument: true, }); } /** * Get the text content of a link - does not save the link as a document * Mostly used in agentic flows/tools calls to get the text content of a link * @param {string} link - The link to get the text content of * @param {('html' | 'text' | 'json')} captureAs - The format to capture the page content as * @returns {Promise<{success: boolean, content: string}>} - Response from collector */ async function getLinkText(link, captureAs = "text") { const validatedLink = validateURL(link); if (!validURL(validatedLink)) return { success: false, reason: "Not a valid URL." }; return await scrapeGenericUrl({ link: validatedLink, captureAs, saveAsDocument: false, }); } module.exports = { processLink, getLinkText, }; ================================================ FILE: collector/processRawText/index.js ================================================ const { v4 } = require("uuid"); const { writeToServerDocuments } = require("../utils/files"); const { tokenizeString } = require("../utils/tokenizer"); const { default: slugify } = require("slugify"); // Will remove the last .extension from the input // and stringify the input + move to lowercase. function stripAndSlug(input) { if (!input.includes(".")) return slugify(input, { lower: true }); return slugify(input.split(".").slice(0, -1).join("-"), { lower: true }); } const METADATA_KEYS = { possible: { url: ({ url, title }) => { let validUrl; try { const u = new URL(url); validUrl = ["https:", "http:"].includes(u.protocol); } catch {} if (validUrl) return `web://${url.toLowerCase()}.website`; return `file://${stripAndSlug(title)}.txt`; }, title: ({ title }) => `${stripAndSlug(title)}.txt`, docAuthor: ({ docAuthor }) => { return typeof docAuthor === "string" ? docAuthor : "no author specified"; }, description: ({ description }) => { return typeof description === "string" ? description : "no description found"; }, docSource: ({ docSource }) => { return typeof docSource === "string" ? docSource : "no source set"; }, chunkSource: ({ chunkSource, title }) => { return typeof chunkSource === "string" ? chunkSource : `${stripAndSlug(title)}.txt`; }, published: ({ published }) => { if (isNaN(Number(published))) return new Date().toLocaleString(); return new Date(Number(published)).toLocaleString(); }, }, }; async function processRawText(textContent, metadata) { console.log(`-- Working Raw Text doc ${metadata.title} --`); if (!textContent || textContent.length === 0) { return { success: false, reason: "textContent was empty - nothing to process.", documents: [], }; } const data = { id: v4(), url: METADATA_KEYS.possible.url(metadata), title: METADATA_KEYS.possible.title(metadata), docAuthor: METADATA_KEYS.possible.docAuthor(metadata), description: METADATA_KEYS.possible.description(metadata), docSource: METADATA_KEYS.possible.docSource(metadata), chunkSource: METADATA_KEYS.possible.chunkSource(metadata), published: METADATA_KEYS.possible.published(metadata), wordCount: textContent.split(" ").length, pageContent: textContent, token_count_estimate: tokenizeString(textContent), }; const document = writeToServerDocuments({ data, filename: `raw-${stripAndSlug(metadata.title)}-${data.id}`, }); console.log( `[SUCCESS]: Raw text and metadata saved & ready for embedding.\n` ); return { success: true, reason: null, documents: [document] }; } module.exports = { processRawText }; ================================================ FILE: collector/processSingleFile/convert/asAudio.js ================================================ const { v4 } = require("uuid"); const { createdDate, trashFile, writeToServerDocuments, } = require("../../utils/files"); const { tokenizeString } = require("../../utils/tokenizer"); const { default: slugify } = require("slugify"); const { LocalWhisper } = require("../../utils/WhisperProviders/localWhisper"); const { OpenAiWhisper } = require("../../utils/WhisperProviders/OpenAiWhisper"); const WHISPER_PROVIDERS = { openai: OpenAiWhisper, local: LocalWhisper, }; async function asAudio({ fullFilePath = "", filename = "", options = {}, metadata = {}, }) { const WhisperProvider = WHISPER_PROVIDERS.hasOwnProperty( options?.whisperProvider ) ? WHISPER_PROVIDERS[options?.whisperProvider] : WHISPER_PROVIDERS.local; console.log(`-- Working ${filename} --`); const whisper = new WhisperProvider({ options }); const { content, error } = await whisper.processFile(fullFilePath, filename); if (!!error) { console.error(`Error encountered for parsing of ${filename}.`); trashFile(fullFilePath); return { success: false, reason: error, documents: [], }; } if (!content?.length) { console.error(`Resulting text content was empty for ${filename}.`); trashFile(fullFilePath); return { success: false, reason: `No text content found in ${filename}.`, documents: [], }; } const data = { id: v4(), url: "file://" + fullFilePath, title: metadata.title || filename, docAuthor: metadata.docAuthor || "no author found", description: metadata.description || "No description found.", docSource: metadata.docSource || "audio file uploaded by the user.", chunkSource: metadata.chunkSource || "", published: createdDate(fullFilePath), wordCount: content.split(" ").length, pageContent: content, token_count_estimate: tokenizeString(content), }; const document = writeToServerDocuments({ data, filename: `${slugify(filename)}-${data.id}`, options: { parseOnly: options.parseOnly }, }); trashFile(fullFilePath); console.log( `[SUCCESS]: ${filename} transcribed, converted & ready for embedding.\n` ); return { success: true, reason: null, documents: [document] }; } module.exports = asAudio; ================================================ FILE: collector/processSingleFile/convert/asDocx.js ================================================ const { v4 } = require("uuid"); const { DocxLoader } = require("langchain/document_loaders/fs/docx"); const { createdDate, trashFile, writeToServerDocuments, } = require("../../utils/files"); const { tokenizeString } = require("../../utils/tokenizer"); const { default: slugify } = require("slugify"); async function asDocX({ fullFilePath = "", filename = "", options = {}, metadata = {}, }) { const loader = new DocxLoader(fullFilePath); console.log(`-- Working ${filename} --`); let pageContent = []; const docs = await loader.load(); for (const doc of docs) { console.log(`-- Parsing content from docx page --`); if (!doc.pageContent.length) continue; pageContent.push(doc.pageContent); } if (!pageContent.length) { console.error(`Resulting text content was empty for ${filename}.`); trashFile(fullFilePath); return { success: false, reason: `No text content found in ${filename}.`, documents: [], }; } const content = pageContent.join(""); const data = { id: v4(), url: "file://" + fullFilePath, title: metadata.title || filename, docAuthor: metadata.docAuthor || "no author found", description: metadata.description || "No description found.", docSource: metadata.docSource || "docx file uploaded by the user.", chunkSource: metadata.chunkSource || "", published: createdDate(fullFilePath), wordCount: content.split(" ").length, pageContent: content, token_count_estimate: tokenizeString(content), }; const document = writeToServerDocuments({ data, filename: `${slugify(filename)}-${data.id}`, options: { parseOnly: options.parseOnly }, }); trashFile(fullFilePath); console.log(`[SUCCESS]: ${filename} converted & ready for embedding.\n`); return { success: true, reason: null, documents: [document] }; } module.exports = asDocX; ================================================ FILE: collector/processSingleFile/convert/asEPub.js ================================================ const { v4 } = require("uuid"); const { EPubLoader } = require("langchain/document_loaders/fs/epub"); const { tokenizeString } = require("../../utils/tokenizer"); const { createdDate, trashFile, writeToServerDocuments, } = require("../../utils/files"); const { default: slugify } = require("slugify"); async function asEPub({ fullFilePath = "", filename = "", options = {}, metadata = {}, }) { let content = ""; try { const loader = new EPubLoader(fullFilePath, { splitChapters: false }); const docs = await loader.load(); docs.forEach((doc) => (content += doc.pageContent)); } catch (err) { console.error("Could not read epub file!", err); } if (!content?.length) { console.error(`Resulting text content was empty for ${filename}.`); trashFile(fullFilePath); return { success: false, reason: `No text content found in ${filename}.`, documents: [], }; } console.log(`-- Working ${filename} --`); const data = { id: v4(), url: "file://" + fullFilePath, title: metadata.title || filename, docAuthor: metadata.docAuthor || "Unknown", description: metadata.description || "Unknown", docSource: metadata.docSource || "epub file uploaded by the user.", chunkSource: metadata.chunkSource || "", published: createdDate(fullFilePath), wordCount: content.split(" ").length, pageContent: content, token_count_estimate: tokenizeString(content), }; const document = writeToServerDocuments({ data, filename: `${slugify(filename)}-${data.id}`, options: { parseOnly: options.parseOnly }, }); trashFile(fullFilePath); console.log(`[SUCCESS]: ${filename} converted & ready for embedding.\n`); return { success: true, reason: null, documents: [document] }; } module.exports = asEPub; ================================================ FILE: collector/processSingleFile/convert/asImage.js ================================================ const { v4 } = require("uuid"); const { tokenizeString } = require("../../utils/tokenizer"); const { createdDate, trashFile, writeToServerDocuments, } = require("../../utils/files"); const OCRLoader = require("../../utils/OCRLoader"); const { default: slugify } = require("slugify"); async function asImage({ fullFilePath = "", filename = "", options = {}, metadata = {}, }) { let content = await new OCRLoader({ targetLanguages: options?.ocr?.langList, }).ocrImage(fullFilePath); if (!content?.length) { console.error(`Resulting text content was empty for ${filename}.`); trashFile(fullFilePath); return { success: false, reason: `No text content found in ${filename}.`, documents: [], }; } console.log(`-- Working ${filename} --`); const data = { id: v4(), url: "file://" + fullFilePath, title: metadata.title || filename, docAuthor: metadata.docAuthor || "Unknown", description: metadata.description || "Unknown", docSource: metadata.docSource || "image file uploaded by the user.", chunkSource: metadata.chunkSource || "", published: createdDate(fullFilePath), wordCount: content.split(" ").length, pageContent: content, token_count_estimate: tokenizeString(content), }; const document = writeToServerDocuments({ data, filename: `${slugify(filename)}-${data.id}`, options: { parseOnly: options.parseOnly }, }); trashFile(fullFilePath); console.log(`[SUCCESS]: ${filename} converted & ready for embedding.\n`); return { success: true, reason: null, documents: [document] }; } module.exports = asImage; ================================================ FILE: collector/processSingleFile/convert/asMbox.js ================================================ const { v4 } = require("uuid"); const fs = require("fs"); const { mboxParser } = require("mbox-parser"); const { createdDate, trashFile, writeToServerDocuments, } = require("../../utils/files"); const { tokenizeString } = require("../../utils/tokenizer"); const { default: slugify } = require("slugify"); async function asMbox({ fullFilePath = "", filename = "", options = {}, metadata = {}, }) { console.log(`-- Working ${filename} --`); const mails = await mboxParser(fs.createReadStream(fullFilePath)) .then((mails) => mails) .catch((error) => { console.log(`Could not parse mail items`, error); return []; }); if (!mails.length) { console.error(`Resulting mail items was empty for ${filename}.`); trashFile(fullFilePath); return { success: false, reason: `No mail items found in ${filename}.`, documents: [], }; } let item = 1; const documents = []; for (const mail of mails) { if (!mail.hasOwnProperty("text")) continue; const content = mail.text; if (!content) continue; console.log( `-- Working on message "${mail.subject || "Unknown subject"}" --` ); const data = { id: v4(), url: "file://" + fullFilePath, title: metadata.title || (mail?.subject ? slugify(mail?.subject?.replace(".", "")) + ".mbox" : `msg_${item}-${filename}`), docAuthor: metadata.docAuthor || mail?.from?.text, description: metadata.description || "No description found.", docSource: metadata.docSource || "Mbox message file uploaded by the user.", chunkSource: metadata.chunkSource || "", published: createdDate(fullFilePath), wordCount: content.split(" ").length, pageContent: content, token_count_estimate: tokenizeString(content), }; item++; const document = writeToServerDocuments({ data, filename: `${slugify(filename)}-${data.id}-msg-${item}`, options: { parseOnly: options.parseOnly }, }); documents.push(document); } trashFile(fullFilePath); console.log( `[SUCCESS]: ${filename} messages converted & ready for embedding.\n` ); return { success: true, reason: null, documents }; } module.exports = asMbox; ================================================ FILE: collector/processSingleFile/convert/asOfficeMime.js ================================================ const { v4 } = require("uuid"); const officeParser = require("officeparser"); const { createdDate, trashFile, writeToServerDocuments, } = require("../../utils/files"); const { tokenizeString } = require("../../utils/tokenizer"); const { default: slugify } = require("slugify"); async function asOfficeMime({ fullFilePath = "", filename = "", options = {}, metadata = {}, }) { console.log(`-- Working ${filename} --`); let content = ""; try { content = await officeParser.parseOfficeAsync(fullFilePath); } catch (error) { console.error(`Could not parse office or office-like file`, error); } if (!content.length) { console.error(`Resulting text content was empty for ${filename}.`); trashFile(fullFilePath); return { success: false, reason: `No text content found in ${filename}.`, documents: [], }; } const data = { id: v4(), url: "file://" + fullFilePath, title: metadata.title || filename, docAuthor: metadata.docAuthor || "no author found", description: metadata.description || "No description found.", docSource: metadata.docSource || "Office file uploaded by the user.", chunkSource: metadata.chunkSource || "", published: createdDate(fullFilePath), wordCount: content.split(" ").length, pageContent: content, token_count_estimate: tokenizeString(content), }; const document = writeToServerDocuments({ data, filename: `${slugify(filename)}-${data.id}`, options: { parseOnly: options.parseOnly }, }); trashFile(fullFilePath); console.log(`[SUCCESS]: ${filename} converted & ready for embedding.\n`); return { success: true, reason: null, documents: [document] }; } module.exports = asOfficeMime; ================================================ FILE: collector/processSingleFile/convert/asPDF/PDFLoader/index.js ================================================ const fs = require("fs").promises; class PDFLoader { constructor(filePath, { splitPages = true } = {}) { this.filePath = filePath; this.splitPages = splitPages; } async load() { const buffer = await fs.readFile(this.filePath); const { getDocument, version } = await this.getPdfJS(); const pdf = await getDocument({ data: new Uint8Array(buffer), useWorkerFetch: false, isEvalSupported: false, useSystemFonts: true, }).promise; const meta = await pdf.getMetadata().catch(() => null); const documents = []; for (let i = 1; i <= pdf.numPages; i += 1) { const page = await pdf.getPage(i); const content = await page.getTextContent(); if (content.items.length === 0) { continue; } let lastY; const textItems = []; for (const item of content.items) { if ("str" in item) { if (lastY === item.transform[5] || !lastY) { textItems.push(item.str); } else { textItems.push(`\n${item.str}`); } lastY = item.transform[5]; } } const text = textItems.join(""); documents.push({ pageContent: text.trim(), metadata: { source: this.filePath, pdf: { version, info: meta?.info, metadata: meta?.metadata, totalPages: pdf.numPages, }, loc: { pageNumber: i }, }, }); } if (this.splitPages) { return documents; } if (documents.length === 0) { return []; } return [ { pageContent: documents.map((doc) => doc.pageContent).join("\n\n"), metadata: { source: this.filePath, pdf: { version, info: meta?.info, metadata: meta?.metadata, totalPages: pdf.numPages, }, }, }, ]; } async getPdfJS() { try { const pdfjs = await import("pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js"); return { getDocument: pdfjs.getDocument, version: pdfjs.version }; } catch (e) { console.error(e); throw new Error( "Failed to load pdf-parse. Please install it with eg. `npm install pdf-parse`." ); } } } module.exports = PDFLoader; ================================================ FILE: collector/processSingleFile/convert/asPDF/index.js ================================================ const { v4 } = require("uuid"); const { createdDate, trashFile, writeToServerDocuments, } = require("../../../utils/files"); const { tokenizeString } = require("../../../utils/tokenizer"); const { default: slugify } = require("slugify"); const PDFLoader = require("./PDFLoader"); const OCRLoader = require("../../../utils/OCRLoader"); async function asPdf({ fullFilePath = "", filename = "", options = {}, metadata = {}, }) { const pdfLoader = new PDFLoader(fullFilePath, { splitPages: true, }); console.log(`-- Working ${filename} --`); const pageContent = []; let docs = await pdfLoader.load(); if (docs.length === 0) { console.log( `[asPDF] No text content found for ${filename}. Will attempt OCR parse.` ); docs = await new OCRLoader({ targetLanguages: options?.ocr?.langList, }).ocrPDF(fullFilePath); } for (const doc of docs) { console.log( `-- Parsing content from pg ${ doc.metadata?.loc?.pageNumber || "unknown" } --` ); if (!doc.pageContent || !doc.pageContent.length) continue; pageContent.push(doc.pageContent); } if (!pageContent.length) { console.error(`[asPDF] Resulting text content was empty for ${filename}.`); trashFile(fullFilePath); return { success: false, reason: `No text content found in ${filename}.`, documents: [], }; } const content = pageContent.join(""); const data = { id: v4(), url: "file://" + fullFilePath, title: metadata.title || filename, docAuthor: metadata.docAuthor || docs[0]?.metadata?.pdf?.info?.Creator || "no author found", description: metadata.description || docs[0]?.metadata?.pdf?.info?.Title || "No description found.", docSource: metadata.docSource || "pdf file uploaded by the user.", chunkSource: metadata.chunkSource || "", published: createdDate(fullFilePath), wordCount: content.split(" ").length, pageContent: content, token_count_estimate: tokenizeString(content), }; const document = writeToServerDocuments({ data, filename: `${slugify(filename)}-${data.id}`, options: { parseOnly: options.parseOnly }, }); trashFile(fullFilePath); console.log(`[SUCCESS]: ${filename} converted & ready for embedding.\n`); return { success: true, reason: null, documents: [document] }; } module.exports = asPdf; ================================================ FILE: collector/processSingleFile/convert/asTxt.js ================================================ const { v4 } = require("uuid"); const fs = require("fs"); const { tokenizeString } = require("../../utils/tokenizer"); const { createdDate, trashFile, writeToServerDocuments, } = require("../../utils/files"); const { default: slugify } = require("slugify"); async function asTxt({ fullFilePath = "", filename = "", options = {}, metadata = {}, }) { let content = ""; try { content = fs.readFileSync(fullFilePath, "utf8"); } catch (err) { console.error("Could not read file!", err); } if (!content?.length) { console.error(`Resulting text content was empty for ${filename}.`); trashFile(fullFilePath); return { success: false, reason: `No text content found in ${filename}.`, documents: [], }; } console.log(`-- Working ${filename} --`); const data = { id: v4(), url: "file://" + fullFilePath, title: metadata.title || filename, docAuthor: metadata.docAuthor || "Unknown", description: metadata.description || "Unknown", docSource: metadata.docSource || "a text file uploaded by the user.", chunkSource: metadata.chunkSource || "", published: createdDate(fullFilePath), wordCount: content.split(" ").length, pageContent: content, token_count_estimate: tokenizeString(content), }; const document = writeToServerDocuments({ data, filename: `${slugify(filename)}-${data.id}`, options: { parseOnly: options.parseOnly }, }); trashFile(fullFilePath); console.log(`[SUCCESS]: ${filename} converted & ready for embedding.\n`); return { success: true, reason: null, documents: [document] }; } module.exports = asTxt; ================================================ FILE: collector/processSingleFile/convert/asXlsx.js ================================================ const { v4 } = require("uuid"); const xlsx = require("node-xlsx").default; const path = require("path"); const fs = require("fs"); const { createdDate, trashFile, writeToServerDocuments, documentsFolder, } = require("../../utils/files"); const { tokenizeString } = require("../../utils/tokenizer"); const { default: slugify } = require("slugify"); function convertToCSV(data) { return data .map((row) => row .map((cell) => { if (cell === null || cell === undefined) return ""; if (typeof cell === "string" && cell.includes(",")) return `"${cell}"`; return cell; }) .join(",") ) .join("\n"); } async function asXlsx({ fullFilePath = "", filename = "", options = {}, metadata = {}, }) { const documents = []; try { const workSheetsFromFile = xlsx.parse(fullFilePath); if (options.parseOnly) { const allSheetContents = []; let totalWordCount = 0; const sheetNames = []; for (const sheet of workSheetsFromFile) { const processed = processSheet(sheet); if (!processed) continue; const { name, content, wordCount } = processed; sheetNames.push(name); allSheetContents.push(`\nSheet: ${name}\n${content}`); totalWordCount += wordCount; } if (allSheetContents.length === 0) { console.log(`No valid sheets found in ${filename}.`); return { success: false, reason: `No valid sheets found in ${filename}.`, documents: [], }; } const combinedContent = allSheetContents.join("\n"); const sheetListText = sheetNames.length > 1 ? ` (Sheets: ${sheetNames.join(", ")})` : ` (Sheet: ${sheetNames[0]})`; const combinedData = { id: v4(), url: `file://${fullFilePath}`, title: metadata.title || `${filename}${sheetListText}`, docAuthor: metadata.docAuthor || "Unknown", description: metadata.description || `Spreadsheet data from ${filename} containing ${sheetNames.length} ${ sheetNames.length === 1 ? "sheet" : "sheets" }`, docSource: metadata.docSource || "an xlsx file uploaded by the user.", chunkSource: metadata.chunkSource || "", published: createdDate(fullFilePath), wordCount: totalWordCount, pageContent: combinedContent, token_count_estimate: tokenizeString(combinedContent), }; const document = writeToServerDocuments({ data: combinedData, filename: `${slugify(path.basename(filename))}-${combinedData.id}`, destinationOverride: null, options: { parseOnly: true }, }); documents.push(document); console.log(`[SUCCESS]: ${filename} converted & ready for embedding.`); } else { const folderName = slugify( `${path.basename(filename)}-${v4().slice(0, 4)}`, { lower: true, trim: true, } ); const outFolderPath = path.resolve(documentsFolder, folderName); if (!fs.existsSync(outFolderPath)) fs.mkdirSync(outFolderPath, { recursive: true }); for (const sheet of workSheetsFromFile) { const processed = processSheet(sheet); if (!processed) continue; const { name, content, wordCount } = processed; const sheetData = { id: v4(), url: `file://${path.join(outFolderPath, `${slugify(name)}.csv`)}`, title: metadata.title || `${filename} - Sheet:${name}`, docAuthor: metadata.docAuthor || "Unknown", description: metadata.description || `Spreadsheet data from sheet: ${name}`, docSource: metadata.docSource || "an xlsx file uploaded by the user.", chunkSource: metadata.chunkSource || "", published: createdDate(fullFilePath), wordCount: wordCount, pageContent: content, token_count_estimate: tokenizeString(content), }; const document = writeToServerDocuments({ data: sheetData, filename: `sheet-${slugify(name)}`, destinationOverride: outFolderPath, options: { parseOnly: options.parseOnly }, }); documents.push(document); console.log( `[SUCCESS]: Sheet "${name}" converted & ready for embedding.` ); } } } catch (err) { console.error("Could not process xlsx file!", err); return { success: false, reason: `Error processing ${filename}: ${err.message}`, documents: [], }; } finally { trashFile(fullFilePath); } if (documents.length === 0) { console.error(`No valid sheets found in ${filename}.`); return { success: false, reason: `No valid sheets found in ${filename}.`, documents: [], }; } console.log( `[SUCCESS]: ${filename} fully processed. Created ${documents.length} document(s).\n` ); return { success: true, reason: null, documents }; } /** * Processes a single sheet and returns its content and metadata * @param {{name: string, data: Array>}} sheet - Parsed sheet with name and 2D array of cell values * @returns {{name: string, content: string, wordCount: number}|null} - Object with name, CSV content, and word count, or null if sheet is empty */ function processSheet(sheet) { try { const { name, data } = sheet; const content = convertToCSV(data); if (!content?.length) { console.log(`Sheet "${name}" is empty. Skipping.`); return null; } console.log(`-- Processing sheet: ${name} --`); return { name, content, wordCount: content.split(/\s+/).length, }; } catch (err) { console.error(`Error processing sheet "${sheet.name}":`, err); return null; } } module.exports = asXlsx; ================================================ FILE: collector/processSingleFile/index.js ================================================ const path = require("path"); const fs = require("fs"); const { WATCH_DIRECTORY, SUPPORTED_FILETYPE_CONVERTERS, } = require("../utils/constants"); const { trashFile, isTextType, normalizePath, isWithin, } = require("../utils/files"); const RESERVED_FILES = ["__HOTDIR__.md"]; /** * Process a single file and return the documents * @param {string} targetFilename - The filename to process * @param {Object} options - The options for the file processing * @param {boolean} options.parseOnly - If true, the file will not be saved as a document even when `writeToServerDocuments` is called in the handler. Must be explicitly set to true to use. * @param {Object} metadata - The metadata for the file processing * @returns {Promise<{success: boolean, reason: string, documents: Object[]}>} - The documents from the file processing */ async function processSingleFile(targetFilename, options = {}, metadata = {}) { const fullFilePath = path.resolve( WATCH_DIRECTORY, normalizePath(targetFilename) ); if (!isWithin(path.resolve(WATCH_DIRECTORY), fullFilePath)) return { success: false, reason: "Filename is a not a valid path to process.", documents: [], }; if (RESERVED_FILES.includes(targetFilename)) return { success: false, reason: "Filename is a reserved filename and cannot be processed.", documents: [], }; if (!fs.existsSync(fullFilePath)) return { success: false, reason: "File does not exist in upload directory.", documents: [], }; const fileExtension = path.extname(fullFilePath).toLowerCase(); if (fullFilePath.includes(".") && !fileExtension) { return { success: false, reason: `No file extension found. This file cannot be processed.`, documents: [], }; } let processFileAs = fileExtension; if (!SUPPORTED_FILETYPE_CONVERTERS.hasOwnProperty(fileExtension)) { if (isTextType(fullFilePath)) { console.log( `\x1b[33m[Collector]\x1b[0m The provided filetype of ${fileExtension} does not have a preset and will be processed as .txt.` ); processFileAs = ".txt"; } else { trashFile(fullFilePath); return { success: false, reason: `File extension ${fileExtension} not supported for parsing and cannot be assumed as text file type.`, documents: [], }; } } const FileTypeProcessor = require(SUPPORTED_FILETYPE_CONVERTERS[ processFileAs ]); return await FileTypeProcessor({ fullFilePath, filename: targetFilename, options, metadata, }); } module.exports = { processSingleFile, }; ================================================ FILE: collector/storage/.gitignore ================================================ tmp/* !tmp/.placeholder ================================================ FILE: collector/storage/tmp/.placeholder ================================================ ================================================ FILE: collector/utils/EncryptionWorker/index.js ================================================ const crypto = require("crypto"); // Differs from EncryptionManager in that is does not set or define the keys that will be used // to encrypt or read data and it must be told the key (as base64 string) explicitly that will be used and is provided to // the class on creation. This key should be the same `key` that is used by the EncryptionManager class. class EncryptionWorker { constructor(presetKeyBase64 = "") { this.key = Buffer.from(presetKeyBase64, "base64"); this.algorithm = "aes-256-cbc"; this.separator = ":"; } log(text, ...args) { console.log(`\x1b[36m[EncryptionManager]\x1b[0m ${text}`, ...args); } /** * Give a chunk source, parse its payload query param and expand that object back into the URL * as additional query params * @param {string} chunkSource * @returns {URL} Javascript URL object with query params decrypted from payload query param. */ expandPayload(chunkSource = "") { try { const url = new URL(chunkSource); if (!url.searchParams.has("payload")) return url; const decryptedPayload = this.decrypt(url.searchParams.get("payload")); const encodedParams = JSON.parse(decryptedPayload); url.searchParams.delete("payload"); // remove payload prop // Add all query params needed to replay as query params Object.entries(encodedParams).forEach(([key, value]) => url.searchParams.append(key, value) ); return url; } catch (e) { console.error(e); } return new URL(chunkSource); } encrypt(plainTextString = null) { try { if (!plainTextString) throw new Error("Empty string is not valid for this method."); const iv = crypto.randomBytes(16); const cipher = crypto.createCipheriv(this.algorithm, this.key, iv); const encrypted = cipher.update(plainTextString, "utf8", "hex"); return [ encrypted + cipher.final("hex"), Buffer.from(iv).toString("hex"), ].join(this.separator); } catch (e) { this.log(e); return null; } } decrypt(encryptedString) { try { const [encrypted, iv] = encryptedString.split(this.separator); if (!iv) throw new Error("IV not found"); const decipher = crypto.createDecipheriv( this.algorithm, this.key, Buffer.from(iv, "hex") ); return decipher.update(encrypted, "hex", "utf8") + decipher.final("utf8"); } catch (e) { this.log(e); return null; } } } module.exports = { EncryptionWorker }; ================================================ FILE: collector/utils/OCRLoader/index.js ================================================ const fs = require("fs"); const os = require("os"); const path = require("path"); const { VALID_LANGUAGE_CODES } = require("./validLangs"); class OCRLoader { /** * The language code(s) to use for the OCR. * @type {string[]} */ language; /** * The cache directory for the OCR. * @type {string} */ cacheDir; /** * The constructor for the OCRLoader. * @param {Object} options - The options for the OCRLoader. * @param {string} options.targetLanguages - The target languages to use for the OCR as a comma separated string. eg: "eng,deu,..." */ constructor({ targetLanguages = "eng" } = {}) { this.language = this.parseLanguages(targetLanguages); this.cacheDir = path.resolve( process.env.STORAGE_DIR ? path.resolve(process.env.STORAGE_DIR, `models`, `tesseract`) : path.resolve(__dirname, `../../../server/storage/models/tesseract`) ); // Ensure the cache directory exists or else Tesseract will persist the cache in the default location. if (!fs.existsSync(this.cacheDir)) fs.mkdirSync(this.cacheDir, { recursive: true }); this.log( `OCRLoader initialized with language support for:`, this.language.map((lang) => VALID_LANGUAGE_CODES[lang]).join(", ") ); } /** * Parses the language code from a provided comma separated string of language codes. * @param {string} language - The language code to parse. * @returns {string[]} The parsed language code. */ parseLanguages(language = null) { try { if (!language || typeof language !== "string") return ["eng"]; const langList = language .split(",") .map((lang) => (lang.trim() !== "" ? lang.trim() : null)) .filter(Boolean) .filter((lang) => VALID_LANGUAGE_CODES.hasOwnProperty(lang)); if (langList.length === 0) return ["eng"]; return langList; } catch (e) { this.log(`Error parsing languages: ${e.message}`, e.stack); return ["eng"]; } } log(text, ...args) { console.log(`\x1b[36m[OCRLoader]\x1b[0m ${text}`, ...args); } /** * Loads a PDF file and returns an array of documents. * This function is reserved to parsing for SCANNED documents - digital documents are not supported in this function * @returns {Promise<{pageContent: string, metadata: object}[]>} An array of documents with page content and metadata. */ async ocrPDF( filePath, { maxExecutionTime = 300_000, batchSize = 10, maxWorkers = null } = {} ) { if ( !filePath || !fs.existsSync(filePath) || !fs.statSync(filePath).isFile() ) { this.log(`File ${filePath} does not exist. Skipping OCR.`); return []; } const documentTitle = path.basename(filePath); this.log(`Starting OCR of ${documentTitle}`); const pdfjs = await import("pdf-parse/lib/pdf.js/v2.0.550/build/pdf.js"); let buffer = fs.readFileSync(filePath); const pdfDocument = await pdfjs.getDocument({ data: buffer }); const documents = []; const meta = await pdfDocument.getMetadata().catch(() => null); const metadata = { source: filePath, pdf: { version: "v2.0.550", info: meta?.info, metadata: meta?.metadata, totalPages: pdfDocument.numPages, }, }; const pdfSharp = new PDFSharp({ validOps: [ pdfjs.OPS.paintJpegXObject, pdfjs.OPS.paintImageXObject, pdfjs.OPS.paintInlineImageXObject, ], }); await pdfSharp.init(); const { createWorker, OEM } = require("tesseract.js"); const BATCH_SIZE = batchSize; const MAX_EXECUTION_TIME = maxExecutionTime; const NUM_WORKERS = maxWorkers ?? Math.min(os.cpus().length, 4); const totalPages = pdfDocument.numPages; const workerPool = await Promise.all( Array(NUM_WORKERS) .fill(0) .map(() => createWorker(this.language, OEM.LSTM_ONLY, { cachePath: this.cacheDir, }) ) ); const startTime = Date.now(); try { this.log("Bootstrapping OCR completed successfully!", { MAX_EXECUTION_TIME_MS: MAX_EXECUTION_TIME, BATCH_SIZE, MAX_CONCURRENT_WORKERS: NUM_WORKERS, TOTAL_PAGES: totalPages, }); const timeoutPromise = new Promise((_, reject) => { setTimeout(() => { reject( new Error( `OCR job took too long to complete (${ MAX_EXECUTION_TIME / 1000 } seconds)` ) ); }, MAX_EXECUTION_TIME); }); const processPages = async () => { for ( let startPage = 1; startPage <= totalPages; startPage += BATCH_SIZE ) { const endPage = Math.min(startPage + BATCH_SIZE - 1, totalPages); const pageNumbers = Array.from( { length: endPage - startPage + 1 }, (_, i) => startPage + i ); this.log(`Working on pages ${startPage} - ${endPage}`); const pageQueue = [...pageNumbers]; const results = []; const workerPromises = workerPool.map(async (worker, workerIndex) => { while (pageQueue.length > 0) { const pageNum = pageQueue.shift(); this.log( `\x1b[34m[Worker ${ workerIndex + 1 }]\x1b[0m assigned pg${pageNum}` ); const page = await pdfDocument.getPage(pageNum); const imageBuffer = await pdfSharp.pageToBuffer({ page }); if (!imageBuffer) continue; const { data } = await worker.recognize(imageBuffer, {}, "text"); this.log( `✅ \x1b[34m[Worker ${ workerIndex + 1 }]\x1b[0m completed pg${pageNum}` ); results.push({ pageContent: data.text, metadata: { ...metadata, loc: { pageNumber: pageNum }, }, }); } }); await Promise.all(workerPromises); documents.push( ...results.sort( (a, b) => a.metadata.loc.pageNumber - b.metadata.loc.pageNumber ) ); } return documents; }; await Promise.race([timeoutPromise, processPages()]); } catch (e) { this.log(`Error: ${e.message}`, e.stack); } finally { global.Image = undefined; await Promise.all(workerPool.map((worker) => worker.terminate())); } this.log(`Completed OCR of ${documentTitle}!`, { documentsParsed: documents.length, totalPages: totalPages, executionTime: `${((Date.now() - startTime) / 1000).toFixed(2)}s`, }); return documents; } /** * Loads an image file and returns the OCRed text. * @param {string} filePath - The path to the image file. * @param {Object} options - The options for the OCR. * @param {number} options.maxExecutionTime - The maximum execution time of the OCR in milliseconds. * @returns {Promise} The OCRed text. */ async ocrImage(filePath, { maxExecutionTime = 300_000 } = {}) { let content = ""; let worker = null; if ( !filePath || !fs.existsSync(filePath) || !fs.statSync(filePath).isFile() ) { this.log(`File ${filePath} does not exist. Skipping OCR.`); return null; } const documentTitle = path.basename(filePath); try { this.log(`Starting OCR of ${documentTitle}`); const startTime = Date.now(); const { createWorker, OEM } = require("tesseract.js"); worker = await createWorker(this.language, OEM.LSTM_ONLY, { cachePath: this.cacheDir, }); // Race the timeout with the OCR const timeoutPromise = new Promise((_, reject) => { setTimeout(() => { reject( new Error( `OCR job took too long to complete (${ maxExecutionTime / 1000 } seconds)` ) ); }, maxExecutionTime); }); const processImage = async () => { const { data } = await worker.recognize(filePath, {}, "text"); content = data.text; }; await Promise.race([timeoutPromise, processImage()]); this.log(`Completed OCR of ${documentTitle}!`, { executionTime: `${((Date.now() - startTime) / 1000).toFixed(2)}s`, }); return content; } catch (e) { this.log(`Error: ${e.message}`); return null; } finally { //eslint-disable-next-line if (!worker) return; await worker.terminate(); } } } /** * Converts a PDF page to a buffer using Sharp. * @param {Object} options - The options for the Sharp PDF page object. * @param {Object} options.page - The PDFJS page proxy object. * @returns {Promise} The buffer of the page. */ class PDFSharp { constructor({ validOps = [] } = {}) { this.sharp = null; this.validOps = validOps; } log(text, ...args) { console.log(`\x1b[36m[PDFSharp]\x1b[0m ${text}`, ...args); } async init() { this.sharp = (await import("sharp")).default; } /** * Converts a PDF page to a buffer. * @param {Object} options - The options for the Sharp PDF page object. * @param {Object} options.page - The PDFJS page proxy object. * @returns {Promise} The buffer of the page. */ async pageToBuffer({ page }) { if (!this.sharp) await this.init(); try { this.log(`Converting page ${page.pageNumber} to image...`); const ops = await page.getOperatorList(); const pageImages = ops.fnArray.length; for (let i = 0; i < pageImages; i++) { try { if (!this.validOps.includes(ops.fnArray[i])) continue; const name = ops.argsArray[i][0]; const img = await page.objs.get(name); const { width, height } = img; const size = img.data.length; const channels = size / width / height; const targetDPI = 70; const targetWidth = Math.floor(width * (targetDPI / 72)); const targetHeight = Math.floor(height * (targetDPI / 72)); const image = this.sharp(img.data, { raw: { width, height, channels }, density: targetDPI, }) .resize({ width: targetWidth, height: targetHeight, fit: "fill", }) .withMetadata({ density: targetDPI, resolution: targetDPI, }) .png(); // For debugging purposes // await image.toFile(path.resolve(__dirname, `../../storage/`, `pg${page.pageNumber}.png`)); return await image.toBuffer(); } catch (error) { this.log(`Iteration error: ${error.message}`, error.stack); continue; } } this.log(`No valid images found on page ${page.pageNumber}`); return null; } catch (error) { this.log(`Error: ${error.message}`, error.stack); return null; } } } module.exports = OCRLoader; ================================================ FILE: collector/utils/OCRLoader/validLangs.js ================================================ /* To get the list of valid language codes - do the following: Open the following URL in your browser: https://tesseract-ocr.github.io/tessdoc/Data-Files-in-different-versions.html Check this element is the proper table tbody with all the codes via console: document.getElementsByTagName('table').item(0).children.item(1) Now, copy the following code and paste it into the console: function parseLangs() { let langs = {}; Array.from(document.getElementsByTagName('table').item(0).children.item(1).children).forEach((el) => { const [codeEl, languageEl, ...rest] = el.children const code = codeEl.innerText.trim() const language = languageEl.innerText.trim() if (!!code && !!language) langs[code] = language }) return langs; } now, run the function: copy(parseLangs()) */ const VALID_LANGUAGE_CODES = { afr: "Afrikaans", amh: "Amharic", ara: "Arabic", asm: "Assamese", aze: "Azerbaijani", aze_cyrl: "Azerbaijani - Cyrilic", bel: "Belarusian", ben: "Bengali", bod: "Tibetan", bos: "Bosnian", bre: "Breton", bul: "Bulgarian", cat: "Catalan; Valencian", ceb: "Cebuano", ces: "Czech", chi_sim: "Chinese - Simplified", chi_tra: "Chinese - Traditional", chr: "Cherokee", cos: "Corsican", cym: "Welsh", dan: "Danish", dan_frak: "Danish - Fraktur (contrib)", deu: "German", deu_frak: "German - Fraktur (contrib)", deu_latf: "German (Fraktur Latin)", dzo: "Dzongkha", ell: "Greek, Modern (1453-)", eng: "English", enm: "English, Middle (1100-1500)", epo: "Esperanto", equ: "Math / equation detection module", est: "Estonian", eus: "Basque", fao: "Faroese", fas: "Persian", fil: "Filipino (old - Tagalog)", fin: "Finnish", fra: "French", frk: "German - Fraktur (now deu_latf)", frm: "French, Middle (ca.1400-1600)", fry: "Western Frisian", gla: "Scottish Gaelic", gle: "Irish", glg: "Galician", grc: "Greek, Ancient (to 1453) (contrib)", guj: "Gujarati", hat: "Haitian; Haitian Creole", heb: "Hebrew", hin: "Hindi", hrv: "Croatian", hun: "Hungarian", hye: "Armenian", iku: "Inuktitut", ind: "Indonesian", isl: "Icelandic", ita: "Italian", ita_old: "Italian - Old", jav: "Javanese", jpn: "Japanese", kan: "Kannada", kat: "Georgian", kat_old: "Georgian - Old", kaz: "Kazakh", khm: "Central Khmer", kir: "Kirghiz; Kyrgyz", kmr: "Kurmanji (Kurdish - Latin Script)", kor: "Korean", kor_vert: "Korean (vertical)", kur: "Kurdish (Arabic Script)", lao: "Lao", lat: "Latin", lav: "Latvian", lit: "Lithuanian", ltz: "Luxembourgish", mal: "Malayalam", mar: "Marathi", mkd: "Macedonian", mlt: "Maltese", mon: "Mongolian", mri: "Maori", msa: "Malay", mya: "Burmese", nep: "Nepali", nld: "Dutch; Flemish", nor: "Norwegian", oci: "Occitan (post 1500)", ori: "Oriya", osd: "Orientation and script detection module", pan: "Panjabi; Punjabi", pol: "Polish", por: "Portuguese", pus: "Pushto; Pashto", que: "Quechua", ron: "Romanian; Moldavian; Moldovan", rus: "Russian", san: "Sanskrit", sin: "Sinhala; Sinhalese", slk: "Slovak", slk_frak: "Slovak - Fraktur (contrib)", slv: "Slovenian", snd: "Sindhi", spa: "Spanish; Castilian", spa_old: "Spanish; Castilian - Old", sqi: "Albanian", srp: "Serbian", srp_latn: "Serbian - Latin", sun: "Sundanese", swa: "Swahili", swe: "Swedish", syr: "Syriac", tam: "Tamil", tat: "Tatar", tel: "Telugu", tgk: "Tajik", tgl: "Tagalog (new - Filipino)", tha: "Thai", tir: "Tigrinya", ton: "Tonga", tur: "Turkish", uig: "Uighur; Uyghur", ukr: "Ukrainian", urd: "Urdu", uzb: "Uzbek", uzb_cyrl: "Uzbek - Cyrilic", vie: "Vietnamese", yid: "Yiddish", yor: "Yoruba", }; module.exports.VALID_LANGUAGE_CODES = VALID_LANGUAGE_CODES; ================================================ FILE: collector/utils/WhisperProviders/OpenAiWhisper.js ================================================ const fs = require("fs"); class OpenAiWhisper { constructor({ options }) { const { OpenAI: OpenAIApi } = require("openai"); if (!options.openAiKey) throw new Error("No OpenAI API key was set."); this.openai = new OpenAIApi({ apiKey: options.openAiKey, }); this.model = "whisper-1"; this.temperature = 0; this.#log("Initialized."); } #log(text, ...args) { console.log(`\x1b[32m[OpenAiWhisper]\x1b[0m ${text}`, ...args); } async processFile(fullFilePath) { return await this.openai.audio.transcriptions .create({ file: fs.createReadStream(fullFilePath), model: this.model, temperature: this.temperature, }) .then((response) => { if (!response) { return { content: "", error: "No content was able to be transcribed.", }; } return { content: response.text, error: null }; }) .catch((error) => { this.#log( `Could not get any response from openai whisper`, error.message ); return { content: "", error: error.message }; }); } } module.exports = { OpenAiWhisper, }; ================================================ FILE: collector/utils/WhisperProviders/ffmpeg/index.js ================================================ const fs = require("fs"); const path = require("path"); const { execSync, spawnSync } = require("child_process"); const { patchShellEnvironmentPath } = require("../../shell"); /** * Custom FFMPEG wrapper class for audio file conversion. * Replaces deprecated fluent-ffmpeg package. * Locates ffmpeg binary and converts audio files to required * WAV format (16k hz mono 32f) for Whisper transcription. * * @class FFMPEGWrapper */ class FFMPEGWrapper { static _instance; constructor() { if (FFMPEGWrapper._instance) return FFMPEGWrapper._instance; FFMPEGWrapper._instance = this; this._ffmpegPath = null; } log(text, ...args) { console.log(`\x1b[35m[FFMPEG]\x1b[0m ${text}`, ...args); } /** * Locates ffmpeg binary. * Uses fix-path on non-Windows platforms to ensure we can find ffmpeg. * * @returns {Promise} Path to ffmpeg binary * @throws {Error} */ async ffmpegPath() { if (this._ffmpegPath) return this._ffmpegPath; await patchShellEnvironmentPath(); try { const which = process.platform === "win32" ? "where" : "which"; const result = execSync(`${which} ffmpeg`, { encoding: "utf8" }).trim(); const candidatePath = result?.split("\n")?.[0]?.trim(); if (!candidatePath) throw new Error("FFMPEG candidate path not found."); if (!this.isValidFFMPEG(candidatePath)) throw new Error("FFMPEG candidate path is not valid ffmpeg binary."); this.log(`Found FFMPEG binary at ${candidatePath}`); this._ffmpegPath = candidatePath; return this._ffmpegPath; } catch (error) { this.log(error.message); } throw new Error("FFMPEG binary not found."); } /** * Validates that path points to a valid ffmpeg binary. * Runs ffmpeg -version command. * * @param {string} pathToTest - Path of ffmpeg binary * @returns {boolean} */ isValidFFMPEG(pathToTest) { try { if (!pathToTest || !fs.existsSync(pathToTest)) return false; execSync(`"${pathToTest}" -version`, { encoding: "utf8", stdio: "pipe" }); return true; } catch { return false; } } /** * Converts audio file to WAV format with required parameters for Whisper. * Output: 16k hz, mono, 32bit float. * * @param {string} inputPath - Input path for audio file (any format supported by ffmpeg) * @param {string} outputPath - Output path for converted file * @returns {Promise} * @throws {Error} If ffmpeg binary cannot be found or conversion fails */ async convertAudioToWav(inputPath, outputPath) { if (!fs.existsSync(inputPath)) throw new Error(`Input file ${inputPath} does not exist.`); const outputDir = path.dirname(outputPath); if (!fs.existsSync(outputDir)) fs.mkdirSync(outputDir, { recursive: true }); this.log(`Converting ${path.basename(inputPath)} to WAV format...`); // Convert to 16k hz mono 32f const result = spawnSync( await this.ffmpegPath(), [ "-i", inputPath, "-ar", "16000", "-ac", "1", "-acodec", "pcm_f32le", "-y", outputPath, ], { encoding: "utf8" } ); // ffmpeg writes progress to stderr if (result.stderr) this.log(result.stderr.trim()); if (result.status !== 0) throw new Error(`FFMPEG conversion failed`); this.log(`Conversion complete: ${path.basename(outputPath)}`); return true; } } module.exports = { FFMPEGWrapper }; ================================================ FILE: collector/utils/WhisperProviders/localWhisper.js ================================================ const fs = require("fs"); const path = require("path"); const { v4 } = require("uuid"); const defaultWhisper = "Xenova/whisper-small"; // Model Card: https://huggingface.co/Xenova/whisper-small const fileSize = { "Xenova/whisper-small": "250mb", "Xenova/whisper-large": "1.56GB", }; class LocalWhisper { constructor({ options }) { this.model = options?.WhisperModelPref ?? defaultWhisper; this.fileSize = fileSize[this.model]; this.cacheDir = path.resolve( process.env.STORAGE_DIR ? path.resolve(process.env.STORAGE_DIR, `models`) : path.resolve(__dirname, `../../../server/storage/models`) ); this.modelPath = path.resolve(this.cacheDir, ...this.model.split("/")); // Make directory when it does not exist in existing installations if (!fs.existsSync(this.cacheDir)) fs.mkdirSync(this.cacheDir, { recursive: true }); this.#log("Initialized."); } #log(text, ...args) { console.log(`\x1b[32m[LocalWhisper]\x1b[0m ${text}`, ...args); } #validateAudioFile(wavFile) { const sampleRate = wavFile.fmt.sampleRate; const duration = wavFile.data.samples / sampleRate; // Most speech recognition systems expect minimum 8kHz // But we'll set it lower to be safe if (sampleRate < 4000) { // 4kHz minimum throw new Error( "Audio file sample rate is too low for accurate transcription. Minimum required is 4kHz." ); } // Typical audio file duration limits const MAX_DURATION_SECONDS = 4 * 60 * 60; // 4 hours if (duration > MAX_DURATION_SECONDS) { throw new Error("Audio file duration exceeds maximum limit of 4 hours."); } // Check final sample count after upsampling to prevent memory issues const targetSampleRate = 16000; const upsampledSamples = duration * targetSampleRate; const MAX_SAMPLES = 230_400_000; // ~4 hours at 16kHz if (upsampledSamples > MAX_SAMPLES) { throw new Error("Audio file exceeds maximum allowed length."); } return true; } async #convertToWavAudioData(sourcePath) { try { let buffer; const wavefile = require("wavefile"); const { FFMPEGWrapper } = require("./ffmpeg"); const ffmpeg = new FFMPEGWrapper(); const outFolder = path.resolve(__dirname, `../../storage/tmp`); if (!fs.existsSync(outFolder)) fs.mkdirSync(outFolder, { recursive: true }); const outputFile = path.resolve(outFolder, `${v4()}.wav`); const success = await ffmpeg.convertAudioToWav(sourcePath, outputFile); if (!success) throw new Error( "[Conversion Failed]: Could not convert file to .wav format!" ); buffer = fs.readFileSync(outputFile); fs.rmSync(outputFile); const wavFile = new wavefile.WaveFile(buffer); try { this.#validateAudioFile(wavFile); } catch (error) { this.#log(`Audio validation failed: ${error.message}`); throw new Error(`Invalid audio file: ${error.message}`); } // Although we use ffmpeg to convert to the correct format (16k hz 32f), // different versions of ffmpeg produce different results based on the // environment. To ensure consistency, we convert to the correct format again. wavFile.toBitDepth("32f"); wavFile.toSampleRate(16000); let audioData = wavFile.getSamples(); if (Array.isArray(audioData)) { if (audioData.length > 1) { const SCALING_FACTOR = Math.sqrt(2); // Merge channels into first channel to save memory for (let i = 0; i < audioData[0].length; ++i) { audioData[0][i] = (SCALING_FACTOR * (audioData[0][i] + audioData[1][i])) / 2; } } audioData = audioData[0]; } return audioData; } catch (error) { console.error(`convertToWavAudioData`, error); return null; } } async client() { if (!fs.existsSync(this.modelPath)) { this.#log( `The native whisper model has never been run and will be downloaded right now. Subsequent runs will be faster. (~${this.fileSize})` ); } try { // Convert ESM to CommonJS via import so we can load this library. const pipeline = (...args) => import("@xenova/transformers").then(({ pipeline }) => { return pipeline(...args); }); return await pipeline("automatic-speech-recognition", this.model, { cache_dir: this.cacheDir, ...(!fs.existsSync(this.modelPath) ? { // Show download progress if we need to download any files progress_callback: (data) => { if (!data.hasOwnProperty("progress")) return; console.log( `\x1b[34m[ONNXWhisper - Downloading Model Files]\x1b[0m ${ data.file } ${~~data?.progress}%` ); }, } : {}), }); } catch (error) { let errMsg = error.message; if (errMsg.includes("Could not locate file")) { errMsg = "The native whisper model failed to download from the huggingface.co CDN. Your internet connection may be unstable or blocked by Huggingface.co - you will need to download the model manually and place it in the storage/models folder to use local Whisper transcription."; } this.#log( `Failed to load the native whisper model: ${errMsg}`, error.stack ); throw new Error(errMsg); } } async processFile(fullFilePath, filename) { try { const audioDataPromise = new Promise((resolve) => this.#convertToWavAudioData(fullFilePath).then((audioData) => resolve(audioData) ) ); const [audioData, transcriber] = await Promise.all([ audioDataPromise, this.client(), ]); if (!audioData) { this.#log(`Failed to parse content from ${filename}.`); return { content: null, error: `Failed to parse content from ${filename}.`, }; } this.#log(`Transcribing audio data to text...`); const { text } = await transcriber(audioData, { chunk_length_s: 30, stride_length_s: 5, }); return { content: text, error: null }; } catch (error) { return { content: null, error: error.message }; } } } module.exports = { LocalWhisper, }; ================================================ FILE: collector/utils/comKey/index.js ================================================ const crypto = require("crypto"); const fs = require("fs"); const path = require("path"); const keyPath = process.env.NODE_ENV === "development" ? path.resolve(__dirname, `../../../server/storage/comkey`) : path.resolve( process.env.STORAGE_DIR ?? path.resolve(__dirname, `../../../server/storage`), `comkey` ); class CommunicationKey { #pubKeyName = "ipc-pub.pem"; #storageLoc = keyPath; constructor() {} log(text, ...args) { console.log(`\x1b[36m[CommunicationKeyVerify]\x1b[0m ${text}`, ...args); } #readPublicKey() { return fs.readFileSync(path.resolve(this.#storageLoc, this.#pubKeyName)); } // Given a signed payload from private key from /app/server/ this signature should // decode to match the textData provided. This class does verification only in collector. // Note: The textData is typically the JSON stringified body sent to the document processor API. verify(signature = "", textData = "") { try { let data = textData; if (typeof textData !== "string") data = JSON.stringify(data); return crypto.verify( "RSA-SHA256", Buffer.from(data), this.#readPublicKey(), Buffer.from(signature, "hex") ); } catch {} return false; } // Use the rolling public-key to decrypt arbitrary data that was encrypted via the private key on the server side CommunicationKey class // that we know was done with the same key-pair and the given input is in base64 format already. // Returns plaintext string of the data that was encrypted. decrypt(base64String = "") { return crypto .publicDecrypt(this.#readPublicKey(), Buffer.from(base64String, "base64")) .toString(); } } module.exports = { CommunicationKey }; ================================================ FILE: collector/utils/constants.js ================================================ const WATCH_DIRECTORY = require("path").resolve(__dirname, "../hotdir"); const ACCEPTED_MIMES = { "text/plain": [".txt", ".md", ".org", ".adoc", ".rst"], "text/html": [".html"], "text/csv": [".csv"], "application/json": [".json"], // TODO: Create asDoc.js that works for standard MS Word files. // "application/msword": [".doc"], "application/vnd.openxmlformats-officedocument.wordprocessingml.document": [ ".docx", ], "application/vnd.openxmlformats-officedocument.presentationml.presentation": [ ".pptx", ], "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": [ ".xlsx", ], "application/vnd.oasis.opendocument.text": [".odt"], "application/vnd.oasis.opendocument.presentation": [".odp"], "application/pdf": [".pdf"], "application/mbox": [".mbox"], "audio/wav": [".wav"], "audio/mpeg": [".mp3"], "video/mp4": [".mp4"], "video/mpeg": [".mpeg"], "application/epub+zip": [".epub"], "image/png": [".png"], "image/jpeg": [".jpg"], "image/jpg": [".jpg"], "image/webp": [".webp"], }; const SUPPORTED_FILETYPE_CONVERTERS = { ".txt": "./convert/asTxt.js", ".md": "./convert/asTxt.js", ".org": "./convert/asTxt.js", ".adoc": "./convert/asTxt.js", ".rst": "./convert/asTxt.js", ".csv": "./convert/asTxt.js", ".json": "./convert/asTxt.js", ".html": "./convert/asTxt.js", ".pdf": "./convert/asPDF/index.js", ".docx": "./convert/asDocx.js", // TODO: Create asDoc.js that works for standard MS Word files. // ".doc": "./convert/asDoc.js", ".pptx": "./convert/asOfficeMime.js", ".odt": "./convert/asOfficeMime.js", ".odp": "./convert/asOfficeMime.js", ".xlsx": "./convert/asXlsx.js", ".mbox": "./convert/asMbox.js", ".epub": "./convert/asEPub.js", ".mp3": "./convert/asAudio.js", ".wav": "./convert/asAudio.js", ".mp4": "./convert/asAudio.js", ".mpeg": "./convert/asAudio.js", ".png": "./convert/asImage.js", ".jpg": "./convert/asImage.js", ".jpeg": "./convert/asImage.js", ".webp": "./convert/asImage.js", }; module.exports = { SUPPORTED_FILETYPE_CONVERTERS, WATCH_DIRECTORY, ACCEPTED_MIMES, }; ================================================ FILE: collector/utils/downloadURIToFile/index.js ================================================ const { WATCH_DIRECTORY } = require("../constants"); const fs = require("fs"); const path = require("path"); const { pipeline } = require("stream/promises"); const { validURL } = require("../url"); const { default: slugify } = require("slugify"); /** * Download a file to the hotdir * @param {string} url - The URL of the file to download * @param {number} maxTimeout - The maximum timeout in milliseconds * @returns {Promise<{success: boolean, fileLocation: string|null, reason: string|null}>} - The path to the downloaded file */ async function downloadURIToFile(url, maxTimeout = 10_000) { if (!url || typeof url !== "string" || !validURL(url)) return { success: false, reason: "Not a valid URL.", fileLocation: null }; try { const abortController = new AbortController(); const timeout = setTimeout(() => { abortController.abort(); console.error( `Timeout ${maxTimeout}ms reached while downloading file for URL:`, url.toString() ); }, maxTimeout); const res = await fetch(url, { signal: abortController.signal }) .then((res) => { if (!res.ok) throw new Error(`HTTP ${res.status}: ${res.statusText}`); return res; }) .finally(() => clearTimeout(timeout)); const urlObj = new URL(url); const filename = `${urlObj.hostname}-${slugify( urlObj.pathname.replace(/\//g, "-"), { lower: true } )}`; const localFilePath = path.join(WATCH_DIRECTORY, filename); const writeStream = fs.createWriteStream(localFilePath); await pipeline(res.body, writeStream); console.log(`[SUCCESS]: File ${localFilePath} downloaded to hotdir.`); return { success: true, fileLocation: localFilePath, reason: null }; } catch (error) { console.error(`Error writing to hotdir: ${error} for URL: ${url}`); return { success: false, reason: error.message, fileLocation: null }; } } module.exports = { downloadURIToFile, }; ================================================ FILE: collector/utils/extensions/Confluence/ConfluenceLoader/index.js ================================================ /* * This is a custom implementation of the Confluence langchain loader. There was an issue where * code blocks were not being extracted. This is a temporary fix until this issue is resolved.*/ const { htmlToText } = require("html-to-text"); class ConfluencePagesLoader { constructor({ baseUrl, spaceKey, username, accessToken, limit = 25, expand = "body.storage,version", personalAccessToken, cloud = true, bypassSSL = false, }) { this.baseUrl = baseUrl; this.spaceKey = spaceKey; this.username = username; this.accessToken = accessToken; this.limit = limit; this.expand = expand; this.personalAccessToken = personalAccessToken; this.cloud = cloud; this.bypassSSL = bypassSSL; this.log("Initialized Confluence Loader"); if (this.bypassSSL) this.log("!!SSL bypass is enabled!! Use at your own risk!!"); } log(message, ...args) { console.log(`\x1b[36m[Confluence Loader]\x1b[0m ${message}`, ...args); } get authorizationHeader() { if (this.personalAccessToken) { return `Bearer ${this.personalAccessToken}`; } else if (this.username && this.accessToken) { const authToken = Buffer.from( `${this.username}:${this.accessToken}` ).toString("base64"); return `Basic ${authToken}`; } return undefined; } async load(options) { try { const pages = await this.fetchAllPagesInSpace( options?.start, options?.limit ); return pages.map((page) => this.createDocumentFromPage(page)); } catch (error) { this.log("Error:", error); return []; } } async fetchConfluenceData(url) { try { const initialHeaders = { "Content-Type": "application/json", Accept: "application/json", }; const authHeader = this.authorizationHeader; if (authHeader) initialHeaders.Authorization = authHeader; // If SSL bypass is enabled, set the NODE_TLS_REJECT_UNAUTHORIZED environment variable if (this.bypassSSL) process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0"; const response = await fetch(url, { headers: initialHeaders }); if (!response.ok) { throw new Error( `Failed to fetch ${url} from Confluence: ${response.status}` ); } return await response.json(); } catch (error) { this.log("Error:", error); throw new Error(error.message); } finally { if (this.bypassSSL) process.env.NODE_TLS_REJECT_UNAUTHORIZED = "1"; } } // https://developer.atlassian.com/cloud/confluence/rest/v2/intro/#auth async fetchAllPagesInSpace(start = 0, limit = this.limit) { const url = `${this.baseUrl}${ this.cloud ? "/wiki" : "" }/rest/api/content?spaceKey=${ this.spaceKey }&limit=${limit}&start=${start}&expand=${this.expand}`; const data = await this.fetchConfluenceData(url); if (data.size === 0) { return []; } const nextPageStart = start + data.size; const nextPageResults = await this.fetchAllPagesInSpace( nextPageStart, limit ); return data.results.concat(nextPageResults); } createDocumentFromPage(page) { // Function to extract code blocks const extractCodeBlocks = (content) => { const codeBlockRegex = /]*>[\s\S]*?<\/ac:plain-text-body>[\s\S]*?<\/ac:structured-macro>/g; const languageRegex = /(.*?)<\/ac:parameter>/; return content.replace(codeBlockRegex, (match) => { const language = match.match(languageRegex)?.[1] || ""; const code = match.match( /<\/ac:plain-text-body>/ )?.[1] || ""; return `\n\`\`\`${language}\n${code.trim()}\n\`\`\`\n`; }); }; const contentWithCodeBlocks = extractCodeBlocks(page.body.storage.value); const plainTextContent = htmlToText(contentWithCodeBlocks, { wordwrap: false, preserveNewlines: true, }); const textWithPreservedStructure = plainTextContent.replace( /\n{3,}/g, "\n\n" ); const pageUrl = `${this.baseUrl}${this.cloud ? "/wiki" : ""}/spaces/${ this.spaceKey }/pages/${page.id}`; return { pageContent: textWithPreservedStructure, metadata: { id: page.id, status: page.status, title: page.title, type: page.type, url: pageUrl, version: page.version?.number, updated_by: page.version?.by?.displayName, updated_at: page.version?.when, }, }; } } module.exports = { ConfluencePagesLoader }; ================================================ FILE: collector/utils/extensions/Confluence/index.js ================================================ const fs = require("fs"); const path = require("path"); const { default: slugify } = require("slugify"); const { v4 } = require("uuid"); const { writeToServerDocuments, sanitizeFileName } = require("../../files"); const { tokenizeString } = require("../../tokenizer"); const { ConfluencePagesLoader } = require("./ConfluenceLoader"); /** * Load Confluence documents from a spaceID and Confluence credentials * @param {object} args - forwarded request body params * @param {import("../../../middleware/setDataSigner").ResponseWithSigner} response - Express response object with encryptionWorker * @returns */ async function loadConfluence( { baseUrl = null, spaceKey = null, username = null, accessToken = null, cloud = true, personalAccessToken = null, bypassSSL = false, }, response ) { if (!personalAccessToken && (!username || !accessToken)) { return { success: false, reason: "You need either a personal access token (PAT), or a username and access token to use the Confluence connector.", }; } if (!baseUrl || !validBaseUrl(baseUrl)) { return { success: false, reason: "Provided base URL is not a valid URL.", }; } if (!spaceKey) { return { success: false, reason: "You need to provide a Confluence space key.", }; } const { origin, hostname } = new URL(baseUrl); console.log(`-- Working Confluence ${origin} --`); const loader = new ConfluencePagesLoader({ baseUrl: origin, // Use the origin to avoid issues with subdomains, ports, protocols, etc. spaceKey, username, accessToken, cloud, personalAccessToken, bypassSSL, }); const { docs, error } = await loader .load() .then((docs) => { return { docs, error: null }; }) .catch((e) => { return { docs: [], error: e.message?.split("Error:")?.[1] || e.message, }; }); if (!docs.length || !!error) { return { success: false, reason: error ?? "No pages found for that Confluence space.", }; } const outFolder = slugify( `confluence-${hostname}-${v4().slice(0, 4)}` ).toLowerCase(); const outFolderPath = process.env.NODE_ENV === "development" ? path.resolve( __dirname, `../../../../server/storage/documents/${outFolder}` ) : path.resolve(process.env.STORAGE_DIR, `documents/${outFolder}`); if (!fs.existsSync(outFolderPath)) fs.mkdirSync(outFolderPath, { recursive: true }); docs.forEach((doc) => { if (!doc.pageContent) return; const data = { id: v4(), url: doc.metadata.url + ".page", title: doc.metadata.title || doc.metadata.source, docAuthor: origin, description: doc.metadata.title, docSource: `${origin} Confluence`, chunkSource: generateChunkSource( { doc, baseUrl: origin, spaceKey, accessToken, username, cloud, bypassSSL, }, response.locals.encryptionWorker ), published: new Date().toLocaleString(), wordCount: doc.pageContent.split(" ").length, pageContent: doc.pageContent, token_count_estimate: tokenizeString(doc.pageContent), }; console.log( `[Confluence Loader]: Saving ${doc.metadata.title} to ${outFolder}` ); const fileName = sanitizeFileName( `${slugify(doc.metadata.title)}-${data.id}` ); writeToServerDocuments({ data, filename: fileName, destinationOverride: outFolderPath, }); }); return { success: true, reason: null, data: { spaceKey, destination: outFolder, }, }; } /** * Gets the page content from a specific Confluence page, not all pages in a workspace. * @returns */ async function fetchConfluencePage({ pageUrl, baseUrl, spaceKey, username, accessToken, cloud = true, bypassSSL = false, }) { if (!pageUrl || !baseUrl || !spaceKey || !username || !accessToken) { return { success: false, content: null, reason: "You need either a username and access token, or a personal access token (PAT), to use the Confluence connector.", }; } if (!validBaseUrl(baseUrl)) { return { success: false, content: null, reason: "Provided base URL is not a valid URL.", }; } if (!spaceKey) { return { success: false, content: null, reason: "You need to provide a Confluence space key.", }; } console.log(`-- Working Confluence Page ${pageUrl} --`); const loader = new ConfluencePagesLoader({ baseUrl, // Should be the origin of the baseUrl spaceKey, username, accessToken, cloud, bypassSSL, }); const { docs, error } = await loader .load() .then((docs) => { return { docs, error: null }; }) .catch((e) => { return { docs: [], error: e.message?.split("Error:")?.[1] || e.message, }; }); if (!docs.length || !!error) { return { success: false, reason: error ?? "No pages found for that Confluence space.", content: null, }; } const targetDocument = docs.find( (doc) => doc.pageContent && doc.metadata.url === pageUrl ); if (!targetDocument) { return { success: false, reason: "Target page could not be found in Confluence space.", content: null, }; } return { success: true, reason: null, content: targetDocument.pageContent, }; } /** * Validates if the provided baseUrl is a valid URL at all. * @param {string} baseUrl * @returns {boolean} */ function validBaseUrl(baseUrl) { try { new URL(baseUrl); return true; } catch { return false; } } /** * Generate the full chunkSource for a specific Confluence page so that we can resync it later. * This data is encrypted into a single `payload` query param so we can replay credentials later * since this was encrypted with the systems persistent password and salt. * @param {object} chunkSourceInformation * @param {import("../../EncryptionWorker").EncryptionWorker} encryptionWorker * @returns {string} */ function generateChunkSource( { doc, baseUrl, spaceKey, accessToken, username, cloud, bypassSSL }, encryptionWorker ) { const payload = { baseUrl, spaceKey, token: accessToken, username, cloud, bypassSSL, }; return `confluence://${doc.metadata.url}?payload=${encryptionWorker.encrypt( JSON.stringify(payload) )}`; } module.exports = { loadConfluence, fetchConfluencePage, }; ================================================ FILE: collector/utils/extensions/DrupalWiki/DrupalWiki/index.js ================================================ /** * Copyright 2024 * * Authors: * - Eugen Mayer (KontextWork) */ const { htmlToText } = require("html-to-text"); const { tokenizeString } = require("../../../tokenizer"); const { sanitizeFileName, writeToServerDocuments, documentsFolder, normalizePath, isWithin, } = require("../../../files"); const { default: slugify } = require("slugify"); const path = require("path"); const fs = require("fs"); const { processSingleFile } = require("../../../../processSingleFile"); const { WATCH_DIRECTORY, SUPPORTED_FILETYPE_CONVERTERS, } = require("../../../constants"); class Page { /** * * @param {number }id * @param {string }title * @param {string} created * @param {string} type * @param {string} processedBody * @param {string} url * @param {number} spaceId */ constructor({ id, title, created, type, processedBody, url, spaceId }) { this.id = id; this.title = title; this.url = url; this.created = created; this.type = type; this.processedBody = processedBody; this.spaceId = spaceId; } } class DrupalWiki { /** * * @param baseUrl * @param spaceId * @param accessToken */ constructor({ baseUrl, accessToken }) { this.baseUrl = baseUrl; this.accessToken = accessToken; this.storagePath = this.#prepareStoragePath(baseUrl); } /** * Load all pages for the given space, fetching storing each page one by one * to minimize the memory usage * * @param {number} spaceId * @param {import("../../EncryptionWorker").EncryptionWorker} encryptionWorker * @returns {Promise} */ async loadAndStoreAllPagesForSpace(spaceId, encryptionWorker) { const pageIndex = await this.#getPageIndexForSpace(spaceId); for (const pageId of pageIndex) { try { const page = await this.loadPage(pageId); // Pages with an empty body will lead to embedding issues / exceptions if (page.processedBody.trim() !== "") { this.#storePage(page, encryptionWorker); await this.#downloadAndProcessAttachments(page.id); } else { console.log(`Skipping page (${page.id}) since it has no content`); } } catch (e) { console.error( `Could not process DrupalWiki page ${pageId} (skipping and continuing): ` ); console.error(e); } } } /** * @param {number} pageId * @returns {Promise} */ async loadPage(pageId) { return this.#fetchPage(pageId); } /** * Fetches the page ids for the configured space * @param {number} spaceId * @returns{Promise} array of pageIds */ async #getPageIndexForSpace(spaceId) { // errors on fetching the pageIndex is fatal, no error handling let hasNext = true; let pageIds = []; let pageNr = 0; do { let { isLast, pageIdsForPage } = await this.#getPagesForSpacePaginated( spaceId, pageNr ); hasNext = !isLast; pageNr++; if (pageIdsForPage.length) { pageIds = pageIds.concat(pageIdsForPage); } } while (hasNext); return pageIds; } /** * * @param {number} pageNr * @param {number} spaceId * @returns {Promise<{isLast,pageIds}>} */ async #getPagesForSpacePaginated(spaceId, pageNr) { /* * { * content: Page[], * last: boolean, * pageable: { * pageNumber: number * } * } */ const data = await this._doFetch( `${this.baseUrl}/api/rest/scope/api/page?size=100&space=${spaceId}&page=${pageNr}` ); const pageIds = data.content.map((page) => { return Number(page.id); }); return { isLast: data.last, pageIdsForPage: pageIds, }; } /** * @param pageId * @returns {Promise} */ async #fetchPage(pageId) { const data = await this._doFetch( `${this.baseUrl}/api/rest/scope/api/page/${pageId}` ); const url = `${this.baseUrl}/node/${data.id}`; return new Page({ id: data.id, title: data.title, created: data.lastModified, type: data.type, processedBody: this.#processPageBody({ body: data.body, title: data.title, lastModified: data.lastModified, url: url, }), url: url, }); } /** * @param {Page} page * @param {import("../../EncryptionWorker").EncryptionWorker} encryptionWorker */ #storePage(page, encryptionWorker) { const { hostname } = new URL(this.baseUrl); // This UUID will ensure that re-importing the same page without any changes will not // show up (deduplication). const targetUUID = `${hostname}.${page.spaceId}.${page.id}.${page.created}`; const wordCount = page.processedBody.split(" ").length; const data = { id: targetUUID, url: `drupalwiki://${page.url}`, title: page.title, docAuthor: this.baseUrl, description: page.title, docSource: `${this.baseUrl} DrupalWiki`, chunkSource: this.#generateChunkSource(page.id, encryptionWorker), published: new Date().toLocaleString(), wordCount: wordCount, pageContent: page.processedBody, token_count_estimate: tokenizeString(page.processedBody), }; const fileName = sanitizeFileName(`${slugify(page.title)}-${data.id}`); console.log( `[DrupalWiki Loader]: Saving page '${page.title}' (${page.id}) to '${this.storagePath}/${fileName}'` ); writeToServerDocuments({ data, filename: fileName, destinationOverride: this.storagePath, }); } /** * Generate the full chunkSource for a specific Confluence page so that we can resync it later. * This data is encrypted into a single `payload` query param so we can replay credentials later * since this was encrypted with the systems persistent password and salt. * @param {number} pageId * @param {import("../../EncryptionWorker").EncryptionWorker} encryptionWorker * @returns {string} */ #generateChunkSource(pageId, encryptionWorker) { const payload = { baseUrl: this.baseUrl, pageId: pageId, accessToken: this.accessToken, }; return `drupalwiki://${ this.baseUrl }/node/${pageId}?payload=${encryptionWorker.encrypt( JSON.stringify(payload) )}`; } async _doFetch(url) { const response = await fetch(url, { headers: this.#getHeaders(), }); if (!response.ok) { throw new Error(`Failed to fetch ${url}: ${response.status}`); } return response.json(); } #getHeaders() { return { "Content-Type": "application/json", Accept: "application/json", Authorization: `Bearer ${this.accessToken}`, }; } #prepareStoragePath(baseUrl) { const { hostname } = new URL(baseUrl); const subFolder = slugify(`drupalwiki-${hostname}`).toLowerCase(); const outFolder = path.resolve(documentsFolder, subFolder); if (!fs.existsSync(outFolder)) fs.mkdirSync(outFolder, { recursive: true }); return outFolder; } /** * @param {string} body * @param {string} url * @param {string} title * @param {string} lastModified * @returns {string} * @private */ #processPageBody({ body, title }) { const textContent = body.trim() !== "" ? body : title; const plainTextContent = htmlToText(textContent, { wordwrap: false, preserveNewlines: true, selectors: [ { selector: "table", format: "dataTable", options: { colSpacing: 3, rowSpacing: 1, uppercaseHeaderCells: true, maxColumnWidth: Infinity, }, }, ], }); const plainBody = plainTextContent.replace(/\n{3,}/g, "\n\n"); return plainBody; } async #downloadAndProcessAttachments(pageId) { try { const data = await this._doFetch( `${this.baseUrl}/api/rest/scope/api/attachment?pageId=${pageId}&size=2000` ); const extensionsList = Object.keys(SUPPORTED_FILETYPE_CONVERTERS); for (const attachment of data.content || data) { const { fileName, id: attachId } = attachment; const lowerName = fileName.toLowerCase(); if (!extensionsList.some((ext) => lowerName.endsWith(ext))) { continue; } const downloadUrl = `${this.baseUrl}/api/rest/scope/api/attachment/${attachId}/download`; const attachmentResponse = await fetch(downloadUrl, { headers: this.#getHeaders(), }); if (!attachmentResponse.ok) { console.log(`Skipping attachment: ${fileName} - Download failed`); continue; } const buffer = await attachmentResponse.arrayBuffer(); const localFilePath = normalizePath( sanitizeFileName(path.resolve(WATCH_DIRECTORY, fileName)) ); if (!isWithin(path.resolve(WATCH_DIRECTORY), localFilePath)) { console.error( `[DrupalWiki Loader]: File name ${localFilePath} is not within the storage path ${path.resolve( WATCH_DIRECTORY )}` ); continue; } require("fs").writeFileSync(localFilePath, Buffer.from(buffer)); await processSingleFile(localFilePath); } } catch (err) { console.error(`Fetching/processing attachments failed:`, err); } } } module.exports = { DrupalWiki }; ================================================ FILE: collector/utils/extensions/DrupalWiki/index.js ================================================ /** * Copyright 2024 * * Authors: * - Eugen Mayer (KontextWork) */ const { DrupalWiki } = require("./DrupalWiki"); const { validBaseUrl } = require("../../../utils/http"); async function loadAndStoreSpaces( { baseUrl = null, spaceIds = null, accessToken = null }, response ) { if (!baseUrl) { return { success: false, reason: "Please provide your baseUrl like https://mywiki.drupal-wiki.net.", }; } else if (!validBaseUrl(baseUrl)) { return { success: false, reason: "Provided base URL is not a valid URL.", }; } if (!spaceIds) { return { success: false, reason: "Please provide a list of spaceIds like 21,56,67 you want to extract", }; } if (!accessToken) { return { success: false, reason: "Please provide a REST API-Token.", }; } console.log(`-- Working Drupal Wiki ${baseUrl} for spaceIds: ${spaceIds} --`); const drupalWiki = new DrupalWiki({ baseUrl, accessToken }); const encryptionWorker = response.locals.encryptionWorker; const spaceIdsArr = spaceIds.split(",").map((idStr) => { return Number(idStr.trim()); }); for (const spaceId of spaceIdsArr) { try { await drupalWiki.loadAndStoreAllPagesForSpace(spaceId, encryptionWorker); console.log(`--- Finished space ${spaceId} ---`); } catch (e) { console.error(e); return { success: false, reason: e.message, data: {}, }; } } console.log(`-- Finished all spaces--`); return { success: true, reason: null, data: { spaceIds, destination: drupalWiki.storagePath, }, }; } /** * Gets the page content from a specific Confluence page, not all pages in a workspace. * @returns */ async function loadPage({ baseUrl, pageId, accessToken }) { console.log(`-- Working Drupal Wiki Page ${pageId} of ${baseUrl} --`); const drupalWiki = new DrupalWiki({ baseUrl, accessToken }); try { const page = await drupalWiki.loadPage(pageId); return { success: true, reason: null, content: page.processedBody, }; } catch { return { success: false, reason: `Failed (re)-fetching DrupalWiki page ${pageId} form ${baseUrl}}`, content: null, }; } } module.exports = { loadAndStoreSpaces, loadPage, }; ================================================ FILE: collector/utils/extensions/ObsidianVault/index.js ================================================ const { v4 } = require("uuid"); const { default: slugify } = require("slugify"); const path = require("path"); const fs = require("fs"); const { writeToServerDocuments, sanitizeFileName, documentsFolder, } = require("../../files"); function parseObsidianVaultPath(files = []) { const possiblePaths = new Set(); files.forEach( (file) => file?.path && possiblePaths.add(file.path.split("/")[0]) ); switch (possiblePaths.size) { case 0: return null; case 1: // The user specified a vault properly - so all files are in the same folder. return possiblePaths.values().next().value; default: return null; } } async function loadObsidianVault({ files = [] }) { if (!files || files?.length === 0) return { success: false, error: "No files provided" }; const vaultName = parseObsidianVaultPath(files); const folderUUId = v4().slice(0, 4); const outFolder = vaultName ? slugify(`obsidian-vault-${vaultName}-${folderUUId}`).toLowerCase() : slugify(`obsidian-${folderUUId}`).toLowerCase(); const outFolderPath = path.resolve(documentsFolder, outFolder); if (!fs.existsSync(outFolderPath)) fs.mkdirSync(outFolderPath, { recursive: true }); console.log( `Processing ${files.length} files from Obsidian Vault ${ vaultName ? `"${vaultName}"` : "" }` ); const results = []; for (const file of files) { try { const fullPageContent = file?.content; // If the file has no content or is just whitespace, skip it. if (!fullPageContent || fullPageContent.trim() === "") continue; const data = { id: v4(), url: `obsidian://${file.path}`, title: file.name, docAuthor: "Obsidian Vault", description: file.name, docSource: "Obsidian Vault", chunkSource: `obsidian://${file.path}`, published: new Date().toLocaleString(), wordCount: fullPageContent.split(" ").length, pageContent: fullPageContent, token_count_estimate: fullPageContent.length / 4, // rough estimate }; const targetFileName = sanitizeFileName( `${slugify(file.name)}-${data.id}` ); writeToServerDocuments({ data, filename: targetFileName, destinationOverride: outFolderPath, }); results.push({ file: file.path, status: "success" }); } catch (e) { console.error(`Failed to process ${file.path}:`, e); results.push({ file: file.path, status: "failed", reason: e.message }); } } return { success: true, data: { processed: results.filter((r) => r.status === "success").length, failed: results.filter((r) => r.status === "failed").length, total: files.length, results, destination: path.basename(outFolderPath), }, }; } module.exports = { loadObsidianVault, }; ================================================ FILE: collector/utils/extensions/PaperlessNgx/PaperlessNgxLoader/index.js ================================================ const { htmlToText } = require("html-to-text"); const pdf = require("pdf-parse"); class PaperlessNgxLoader { constructor({ baseUrl, apiToken }) { this.baseUrl = new URL(baseUrl).origin; this.apiToken = apiToken; this.baseHeaders = { Authorization: `Token ${this.apiToken}`, }; } async load() { try { const documents = await this.fetchAllDocuments(); return documents.map((doc) => this.createDocumentFromPage(doc)); } catch (error) { console.error("Error:", error); throw error; } } /** * Fetches all documents from Paperless-ngx * @returns {Promise<{{[key: string]: any, content: string}[]}>} The documents with their content */ async fetchAllDocuments() { try { const documents = []; let nextUrl = `${this.baseUrl}/api/documents/`; let page = 1; while (nextUrl) { console.log(`Fetching documents page ${page} from Paperless-ngx`); try { const data = await fetch(nextUrl, { headers: { "Content-Type": "application/json", ...this.baseHeaders, }, }).then((res) => { if (!res.ok) throw new Error( `Failed to fetch documents from Paperless-ngx: ${res.status}` ); return res.json(); }); const validResults = data.results.filter((doc) => doc?.id); if (!validResults.length) break; documents.push(...validResults); if (data.next === nextUrl) break; nextUrl = data.next || null; page++; } catch (error) { console.error( `Error fetching page ${page} from Paperless-ngx:`, error ); break; } } console.log( `Fetched ${documents.length} documents from Paperless-ngx (Pages: ${ page - 1 })` ); const documentsWithContent = await Promise.all( documents.map(async (doc) => { const content = await this.fetchDocumentContent(doc.id); return { ...doc, content }; }) ); return documentsWithContent.filter((doc) => !!doc.content); } catch (error) { throw new Error( `Failed to fetch documents from Paperless-ngx: ${error.message}` ); } } /** * Fetches the content of a document from Paperless-ngx * @param {string} documentId - The ID of the document to fetch * @returns {Promise} The content of the document */ async fetchDocumentContent(documentId) { try { const response = await fetch( `${this.baseUrl}/api/documents/${documentId}/download/`, { headers: this.baseHeaders, } ); if (!response.ok) throw new Error(`Failed to fetch document content: ${response.status}`); const contentType = response.headers.get("content-type"); switch (contentType) { case "text/plain": return await response.text(); case "application/pdf": const buffer = await response.arrayBuffer(); return await this.parsePdfContent(buffer); default: return await response.text(); } } catch (error) { console.error( `Failed to fetch content for document ${documentId}:`, error ); return ""; } } async parsePdfContent(buffer) { try { const data = await pdf(Buffer.from(buffer)); return data.text; } catch (error) { console.error("Failed to parse PDF content:", error); return ""; } } createDocumentFromPage(doc) { const content = doc.content || ""; const plainTextContent = htmlToText(content, { wordwrap: false, preserveNewlines: true, }); return { pageContent: plainTextContent, metadata: { id: doc.id, title: doc.original_file_name, created: doc.created, modified: doc.modified, added: doc.added, tags: doc.tags, correspondent: doc.correspondent, documentType: doc.document_type, url: `${this.baseUrl}/documents/${doc.id}`, }, }; } } module.exports = PaperlessNgxLoader; ================================================ FILE: collector/utils/extensions/PaperlessNgx/index.js ================================================ const fs = require("fs"); const path = require("path"); const { default: slugify } = require("slugify"); const { v4 } = require("uuid"); const { writeToServerDocuments, sanitizeFileName, documentsFolder, } = require("../../files"); const { tokenizeString } = require("../../tokenizer"); const { validBaseUrl } = require("../../http"); const PaperlessNgxLoader = require("./PaperlessNgxLoader"); /** * Load documents from a Paperless-ngx instance * @param {object} args - forwarded request body params * @param {import("../../../middleware/setDataSigner").ResponseWithSigner} response - Express response object with encryptionWorker * @returns */ async function loadPaperlessNgx({ baseUrl = null, apiToken = null }, response) { if (!baseUrl || !validBaseUrl(baseUrl)) { return { success: false, reason: "Provided base URL is not a valid URL.", }; } if (!apiToken) { return { success: false, reason: "You need to provide an API token to use the Paperless-ngx connector.", }; } const { origin, hostname } = new URL(baseUrl); console.log(`-- Working Paperless-ngx ${origin} --`); const loader = new PaperlessNgxLoader({ baseUrl: origin, apiToken, }); const { docs, error } = await loader .load() .then((docs) => ({ docs, error: null })) .catch((e) => ({ docs: [], error: e.message?.split("Error:")?.[1] || e.message, })); if (!docs.length || !!error) { return { success: false, reason: error ?? "No parseable documents found in that Paperless-ngx instance.", data: null, }; } const outFolder = slugify( `paperless-${hostname}-${v4().slice(0, 4)}` ).toLowerCase(); const outFolderPath = path.resolve(documentsFolder, outFolder); if (!fs.existsSync(outFolderPath)) fs.mkdirSync(outFolderPath, { recursive: true }); docs.forEach((doc) => { if (!doc.pageContent) return; const data = { id: v4(), url: doc.metadata.url, title: doc.metadata.title, docAuthor: doc.metadata.correspondent || "Unknown", description: `A document from the Paperless-ngx instance at ${origin}`, docSource: `paperless-ngx`, chunkSource: generateChunkSource( { doc, baseUrl: origin, apiToken }, response.locals.encryptionWorker ), published: doc.metadata.created, wordCount: doc.pageContent.split(" ").length, pageContent: doc.pageContent, token_count_estimate: tokenizeString(doc.pageContent), }; console.log( `[Paperless-ngx Loader]: Saving ${doc.metadata.title} to ${outFolder}` ); const fileName = sanitizeFileName( `${slugify(doc.metadata.title)}-${data.id}` ); writeToServerDocuments({ data, filename: fileName, destinationOverride: outFolderPath, }); }); return { success: true, reason: null, data: { files: docs.length, destination: outFolder, }, }; } /** * Generate the full chunkSource for a specific Paperless-ngx document so that we can resync it later. * @param {object} chunkSourceInformation * @param {import("../../EncryptionWorker").EncryptionWorker} encryptionWorker * @returns {string} */ function generateChunkSource({ doc, baseUrl, apiToken }, encryptionWorker) { const payload = { baseUrl, token: apiToken, }; return `paperless-ngx://${doc.metadata.id}?payload=${encryptionWorker.encrypt( JSON.stringify(payload) )}`; } module.exports = { loadPaperlessNgx, }; ================================================ FILE: collector/utils/extensions/RepoLoader/GithubRepo/RepoLoader/index.js ================================================ /** * @typedef {Object} RepoLoaderArgs * @property {string} repo - The GitHub repository URL. * @property {string} [branch] - The branch to load from (optional). * @property {string} [accessToken] - GitHub access token for authentication (optional). * @property {string[]} [ignorePaths] - Array of paths to ignore when loading (optional). */ /** * @class * @classdesc Loads and manages GitHub repository content. */ class GitHubRepoLoader { /** * Creates an instance of RepoLoader. * @param {RepoLoaderArgs} [args] - The configuration options. * @returns {GitHubRepoLoader} */ constructor(args = {}) { this.ready = false; this.repo = this.#processRepoUrl(args?.repo); this.branch = args?.branch; this.accessToken = args?.accessToken || null; this.ignorePaths = args?.ignorePaths || []; this.author = null; this.project = null; this.branches = []; } /** * Processes a repository URL to ensure it is in the correct format * - remove the .git suffix if present * - ensure the url is valid * @param {string} repoUrl - The repository URL to process. * @returns {string|null} The processed repository URL, or null if the URL is invalid. */ #processRepoUrl(repoUrl) { if (!repoUrl) return repoUrl; try { const url = new URL(repoUrl); if (url.pathname.endsWith(".git")) url.pathname = url.pathname.slice(0, -4); return url.toString(); } catch (e) { console.error( `[GitHub Loader]: Error processing repository URL ${this.repo}: ${e.message}` ); return repoUrl; } } /** * Validates the GitHub URL format. * - ensure the url is valid * - ensure the hostname is github.com * - ensure the pathname is in the format of github.com/{author}/{project} * - sets the author and project properties of class instance * @returns {boolean} True if the URL is valid, false otherwise. */ #validGithubUrl() { try { const url = new URL(this.repo); // Not a github url at all. if (url.hostname !== "github.com") { console.log( `[GitHub Loader]: Invalid GitHub URL provided! Hostname must be 'github.com'. Got ${url.hostname}` ); return false; } // Assume the url is in the format of github.com/{author}/{project} // Remove the first slash from the pathname so we can split it properly. const [author, project, ..._rest] = url.pathname.slice(1).split("/"); if (!author || !project) { console.log( `[GitHub Loader]: Invalid GitHub URL provided! URL must be in the format of 'github.com/{author}/{project}'. Got ${url.pathname}` ); return false; } this.author = author; this.project = project; return true; } catch (e) { console.log( `[GitHub Loader]: Invalid GitHub URL provided! Error: ${e.message}` ); return false; } } // Ensure the branch provided actually exists // and if it does not or has not been set auto-assign to primary branch. async #validBranch() { await this.getRepoBranches(); if (!!this.branch && this.branches.includes(this.branch)) return; console.log( "[GitHub Loader]: Branch not set! Auto-assigning to a default branch." ); this.branch = this.branches.includes("main") ? "main" : "master"; console.log(`[GitHub Loader]: Branch auto-assigned to ${this.branch}.`); return; } async #validateAccessToken() { if (!this.accessToken) return; const valid = await fetch("https://api.github.com/octocat", { method: "GET", headers: { Authorization: `Bearer ${this.accessToken}`, "X-GitHub-Api-Version": "2022-11-28", }, }) .then((res) => { if (!res.ok) throw new Error(res.statusText); return res.ok; }) .catch((e) => { console.error( "Invalid GitHub Access Token provided! Access token will not be used", e.message ); return false; }); if (!valid) this.accessToken = null; return; } /** * Initializes the RepoLoader instance. * @returns {Promise} The initialized RepoLoader instance. */ async init() { if (!this.#validGithubUrl()) return; await this.#validBranch(); await this.#validateAccessToken(); this.ready = true; return this; } /** * Recursively loads the repository content. * @returns {Promise>} An array of loaded documents. * @throws {Error} If the RepoLoader is not in a ready state. */ async recursiveLoader() { if (!this.ready) throw new Error("[GitHub Loader]: not in ready state!"); const { GithubRepoLoader: LCGithubLoader, } = require("@langchain/community/document_loaders/web/github"); if (this.accessToken) console.log( `[GitHub Loader]: Access token set! Recursive loading enabled!` ); const loader = new LCGithubLoader(this.repo, { branch: this.branch, recursive: !!this.accessToken, // Recursive will hit rate limits. maxConcurrency: 5, unknown: "warn", accessToken: this.accessToken, ignorePaths: this.ignorePaths, verbose: true, }); const docs = await loader.load(); return docs; } // Sort branches to always show either main or master at the top of the result. #branchPrefSort(branches = []) { const preferredSort = ["main", "master"]; return branches.reduce((acc, branch) => { if (preferredSort.includes(branch)) return [branch, ...acc]; return [...acc, branch]; }, []); } /** * Retrieves all branches for the repository. * @returns {Promise} An array of branch names. */ async getRepoBranches() { if (!this.#validGithubUrl() || !this.author || !this.project) return []; await this.#validateAccessToken(); // Ensure API access token is valid for pre-flight let page = 0; let polling = true; const branches = []; while (polling) { console.log(`Fetching page ${page} of branches for ${this.project}`); await fetch( `https://api.github.com/repos/${this.author}/${this.project}/branches?per_page=100&page=${page}`, { method: "GET", headers: { ...(this.accessToken ? { Authorization: `Bearer ${this.accessToken}` } : {}), "X-GitHub-Api-Version": "2022-11-28", }, } ) .then((res) => { if (res.ok) return res.json(); throw new Error(`Invalid request to Github API: ${res.statusText}`); }) .then((branchObjects) => { polling = branchObjects.length > 0; branches.push(branchObjects.map((branch) => branch.name)); page++; }) .catch((err) => { polling = false; console.log(`RepoLoader.branches`, err); }); } this.branches = [...new Set(branches.flat())]; return this.#branchPrefSort(this.branches); } /** * Fetches the content of a single file from the repository. * @param {string} sourceFilePath - The path to the file in the repository. * @returns {Promise} The content of the file, or null if fetching fails. */ async fetchSingleFile(sourceFilePath) { try { return fetch( `https://api.github.com/repos/${this.author}/${this.project}/contents/${sourceFilePath}?ref=${this.branch}`, { method: "GET", headers: { Accept: "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28", ...(!!this.accessToken ? { Authorization: `Bearer ${this.accessToken}` } : {}), }, } ) .then((res) => { if (res.ok) return res.json(); throw new Error(`Failed to fetch from Github API: ${res.statusText}`); }) .then((json) => { if (json.hasOwnProperty("status") || !json.hasOwnProperty("content")) throw new Error(json?.message || "missing content"); return atob(json.content); }); } catch (e) { console.error(`RepoLoader.fetchSingleFile`, e); return null; } } } module.exports = GitHubRepoLoader; ================================================ FILE: collector/utils/extensions/RepoLoader/GithubRepo/index.js ================================================ const RepoLoader = require("./RepoLoader"); const fs = require("fs"); const path = require("path"); const { default: slugify } = require("slugify"); const { v4 } = require("uuid"); const { writeToServerDocuments } = require("../../../files"); const { tokenizeString } = require("../../../tokenizer"); /** * Load in a GitHub Repo recursively or just the top level if no PAT is provided * @param {object} args - forwarded request body params * @param {import("../../../middleware/setDataSigner").ResponseWithSigner} response - Express response object with encryptionWorker * @returns */ async function loadGithubRepo(args, response) { const repo = new RepoLoader(args); await repo.init(); if (!repo.ready) return { success: false, reason: "Could not prepare GitHub repo for loading! Check URL", }; console.log( `-- Working GitHub ${repo.author}/${repo.project}:${repo.branch} --` ); const docs = await repo.recursiveLoader(); if (!docs.length) { return { success: false, reason: "No files were found for those settings.", }; } console.log(`[GitHub Loader]: Found ${docs.length} source files. Saving...`); const outFolder = slugify( `${repo.author}-${repo.project}-${repo.branch}-${v4().slice(0, 4)}` ).toLowerCase(); const outFolderPath = process.env.NODE_ENV === "development" ? path.resolve( __dirname, `../../../../../server/storage/documents/${outFolder}` ) : path.resolve(process.env.STORAGE_DIR, `documents/${outFolder}`); if (!fs.existsSync(outFolderPath)) fs.mkdirSync(outFolderPath, { recursive: true }); for (const doc of docs) { if (!doc.pageContent) continue; const data = { id: v4(), url: "github://" + doc.metadata.source, title: doc.metadata.source, docAuthor: repo.author, description: "No description found.", docSource: doc.metadata.source, chunkSource: generateChunkSource( repo, doc, response.locals.encryptionWorker ), published: new Date().toLocaleString(), wordCount: doc.pageContent.split(" ").length, pageContent: doc.pageContent, token_count_estimate: tokenizeString(doc.pageContent), }; console.log( `[GitHub Loader]: Saving ${doc.metadata.source} to ${outFolder}` ); writeToServerDocuments({ data, filename: `${slugify(doc.metadata.source)}-${data.id}`, destinationOverride: outFolderPath, }); } return { success: true, reason: null, data: { author: repo.author, repo: repo.project, branch: repo.branch, files: docs.length, destination: outFolder, }, }; } /** * Gets the page content from a specific source file in a give GitHub Repo, not all items in a repo. * @returns */ async function fetchGithubFile({ repoUrl, branch, accessToken = null, sourceFilePath, }) { const repo = new RepoLoader({ repo: repoUrl, branch, accessToken, }); await repo.init(); if (!repo.ready) return { success: false, content: null, reason: "Could not prepare GitHub repo for loading! Check URL or PAT.", }; console.log( `-- Working GitHub ${repo.author}/${repo.project}:${repo.branch} file:${sourceFilePath} --` ); const fileContent = await repo.fetchSingleFile(sourceFilePath); if (!fileContent) { return { success: false, reason: "Target file returned a null content response.", content: null, }; } return { success: true, reason: null, content: fileContent, }; } /** * Generate the full chunkSource for a specific file so that we can resync it later. * This data is encrypted into a single `payload` query param so we can replay credentials later * since this was encrypted with the systems persistent password and salt. * @param {RepoLoader} repo * @param {import("@langchain/core/documents").Document} doc * @param {import("../../EncryptionWorker").EncryptionWorker} encryptionWorker * @returns {string} */ function generateChunkSource(repo, doc, encryptionWorker) { const payload = { owner: repo.author, project: repo.project, branch: repo.branch, path: doc.metadata.source, pat: !!repo.accessToken ? repo.accessToken : null, }; return `github://${repo.repo}?payload=${encryptionWorker.encrypt( JSON.stringify(payload) )}`; } module.exports = { loadGithubRepo, fetchGithubFile }; ================================================ FILE: collector/utils/extensions/RepoLoader/GitlabRepo/RepoLoader/index.js ================================================ const ignore = require("ignore"); const MAX_RETRIES = 3; /** * @typedef {Object} RepoLoaderArgs * @property {string} repo - The GitLab repository URL. * @property {string} [branch] - The branch to load from (optional). * @property {string} [accessToken] - GitLab access token for authentication (optional). * @property {string[]} [ignorePaths] - Array of paths to ignore when loading (optional). * @property {boolean} [fetchIssues] - Should issues be fetched (optional). * @property {boolean} [fetchWikis] - Should wiki be fetched (optional). */ /** * @typedef {Object} FileTreeObject * @property {string} id - The file object ID. * @property {string} name - name of file. * @property {('blob'|'tree')} type - type of file object. * @property {string} path - path + name of file. * @property {string} mode - Linux permission code. */ /** * @class * @classdesc Loads and manages GitLab repository content. */ class GitLabRepoLoader { /** * Creates an instance of RepoLoader. * @param {RepoLoaderArgs} [args] - The configuration options. * @returns {GitLabRepoLoader} */ constructor(args = {}) { this.ready = false; this.repo = args?.repo; this.branch = args?.branch; this.accessToken = args?.accessToken || null; this.ignorePaths = args?.ignorePaths || []; this.ignoreFilter = ignore().add(this.ignorePaths); this.withIssues = args?.fetchIssues || false; this.withWikis = args?.fetchWikis || false; this.projectId = null; this.apiBase = "https://gitlab.com"; this.author = null; this.project = null; this.branches = []; } #wait(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); } #validGitlabUrl() { const validPatterns = [ //eslint-disable-next-line /https:\/\/gitlab\.com\/(?[^\/]+)\/(?.*)/, // This should even match the regular hosted URL, but we may want to know // if this was a hosted GitLab (above) or a self-hosted (below) instance // since the API interface could be different. //eslint-disable-next-line /(http|https):\/\/[^\/]+\/(?[^\/]+)\/(?.*)/, ]; const match = validPatterns .find((pattern) => this.repo.match(pattern)?.groups) ?.exec(this.repo); if (!match?.groups) return false; const { author, project } = match.groups; this.projectId = encodeURIComponent(`${author}/${project}`); this.apiBase = new URL(this.repo).origin; this.author = author; this.project = project; return true; } async #validBranch() { await this.getRepoBranches(); if (!!this.branch && this.branches.includes(this.branch)) return; console.log( "[Gitlab Loader]: Branch not set! Auto-assigning to a default branch." ); this.branch = this.branches.includes("main") ? "main" : "master"; console.log(`[Gitlab Loader]: Branch auto-assigned to ${this.branch}.`); return; } async #validateAccessToken() { if (!this.accessToken) return; try { await fetch(`${this.apiBase}/api/v4/user`, { method: "GET", headers: this.accessToken ? { "PRIVATE-TOKEN": this.accessToken } : {}, }).then((res) => res.ok); } catch (e) { console.error( "Invalid Gitlab Access Token provided! Access token will not be used", e.message ); this.accessToken = null; } } /** * Initializes the RepoLoader instance. * @returns {Promise} The initialized RepoLoader instance. */ async init() { if (!this.#validGitlabUrl()) return; await this.#validBranch(); await this.#validateAccessToken(); this.ready = true; return this; } /** * Recursively loads the repository content. * @returns {Promise>} An array of loaded documents. * @throws {Error} If the RepoLoader is not in a ready state. */ async recursiveLoader() { if (!this.ready) throw new Error("[Gitlab Loader]: not in ready state!"); if (this.accessToken) console.log( `[Gitlab Loader]: Access token set! Recursive loading enabled for ${this.repo}!` ); const docs = []; console.log(`[Gitlab Loader]: Fetching files.`); const files = await this.fetchFilesRecursive(); console.log(`[Gitlab Loader]: Fetched ${files.length} files.`); for (const file of files) { if (this.ignoreFilter.ignores(file.path)) continue; docs.push({ pageContent: file.content, metadata: { source: file.path, url: `${this.repo}/-/blob/${this.branch}/${file.path}`, }, }); } if (this.withIssues) { console.log(`[Gitlab Loader]: Fetching issues.`); const issues = await this.fetchIssues(); console.log( `[Gitlab Loader]: Fetched ${issues.length} issues with discussions.` ); docs.push( ...issues.map((issue) => ({ issue, metadata: { source: `issue-${this.repo}-${issue.iid}`, url: issue.web_url, }, })) ); } if (this.withWikis) { console.log(`[Gitlab Loader]: Fetching wiki.`); const wiki = await this.fetchWiki(); console.log(`[Gitlab Loader]: Fetched ${wiki.length} wiki pages.`); docs.push( ...wiki.map((wiki) => ({ wiki, metadata: { source: `wiki-${this.repo}-${wiki.slug}`, url: `${this.repo}/-/wikis/${wiki.slug}`, }, })) ); } return docs; } #branchPrefSort(branches = []) { const preferredSort = ["main", "master"]; return branches.reduce((acc, branch) => { if (preferredSort.includes(branch)) return [branch, ...acc]; return [...acc, branch]; }, []); } /** * Retrieves all branches for the repository. * @returns {Promise} An array of branch names. */ async getRepoBranches() { if (!this.#validGitlabUrl() || !this.projectId) return []; await this.#validateAccessToken(); this.branches = []; const branchesRequestData = { endpoint: `/api/v4/projects/${this.projectId}/repository/branches`, }; let branchesPage = []; while ((branchesPage = await this.fetchNextPage(branchesRequestData))) { if (!Array.isArray(branchesPage) || !branchesPage?.length) break; this.branches.push(...branchesPage.map((branch) => branch.name)); } return this.#branchPrefSort(this.branches); } /** * Returns list of all file objects from tree API for GitLab * @returns {Promise} */ async fetchFilesRecursive() { const files = []; const filesRequestData = { endpoint: `/api/v4/projects/${this.projectId}/repository/tree`, queryParams: { ref: this.branch, recursive: true, }, }; let filesPage = null; let pagePromises = []; while ((filesPage = await this.fetchNextPage(filesRequestData))) { if (!Array.isArray(filesPage) || !filesPage?.length) break; // Fetch all the files that are not ignored in parallel. pagePromises = filesPage .filter((file) => { if (file.type !== "blob") return false; return !this.ignoreFilter.ignores(file.path); }) .map(async (file) => { const content = await this.fetchSingleFileContents(file.path); if (!content) return null; return { path: file.path, content, }; }); const pageFiles = await Promise.all(pagePromises); files.push(...pageFiles.filter((item) => item !== null)); console.log(`Fetched ${files.length} files.`); } console.log(`Total files fetched: ${files.length}`); return files; } /** * Fetches all issues from the repository. * @returns {Promise} An array of issue objects. */ async fetchIssues() { const issues = []; const issuesRequestData = { endpoint: `/api/v4/projects/${this.projectId}/issues`, }; let issuesPage = null; let pagePromises = []; while ((issuesPage = await this.fetchNextPage(issuesRequestData))) { if (!Array.isArray(issuesPage) || !issuesPage?.length) break; // Fetch all the issues in parallel. pagePromises = issuesPage.map(async (issue) => { const discussionsRequestData = { endpoint: `/api/v4/projects/${this.projectId}/issues/${issue.iid}/discussions`, }; let discussionPage = null; const discussions = []; while ( (discussionPage = await this.fetchNextPage(discussionsRequestData)) ) { if (!Array.isArray(discussionPage) || !discussionPage?.length) break; discussions.push( ...discussionPage.map(({ notes }) => notes.map( ({ body, author, created_at }) => `${author.username} at ${created_at}: ${body}` ) ) ); } const result = { ...issue, discussions, }; return result; }); const pageIssues = await Promise.all(pagePromises); issues.push(...pageIssues); console.log(`Fetched ${issues.length} issues.`); } console.log(`Total issues fetched: ${issues.length}`); return issues; } /** * Fetches all wiki pages from the repository. * @returns {Promise} An array of wiki page objects. */ async fetchWiki() { const wikiRequestData = { endpoint: `/api/v4/projects/${this.projectId}/wikis`, queryParams: { with_content: "1", }, }; const wikiPages = await this.fetchNextPage(wikiRequestData); if (!Array.isArray(wikiPages)) return []; console.log(`Total wiki pages fetched: ${wikiPages.length}`); return wikiPages; } /** * Fetches the content of a single file from the repository. * @param {string} sourceFilePath - The path to the file in the repository. * @returns {Promise} The content of the file, or null if fetching fails. */ async fetchSingleFileContents(sourceFilePath, retries = 0) { try { const url = `${this.apiBase}/api/v4/projects/${ this.projectId }/repository/files/${encodeURIComponent(sourceFilePath)}/raw?ref=${ this.branch }`; const response = await fetch(url, { method: "GET", headers: this.accessToken ? { "PRIVATE-TOKEN": this.accessToken } : {}, }); if (response.status === 429) { if (retries >= MAX_RETRIES) { console.warn( `[Gitlab Loader]: Rate limit persists for ${sourceFilePath} after ${retries} retries. Skipping.` ); return null; } const retryAfter = Number(response.headers.get("retry-after")) || 60; console.warn( `[Gitlab Loader]: Rate limit hit fetching ${sourceFilePath}. Waiting ${retryAfter}s...` ); await this.#wait(retryAfter * 1000); return this.fetchSingleFileContents(sourceFilePath, retries + 1); } if (!response.ok) throw new Error(`Failed to fetch single file ${sourceFilePath}`); return await response.text(); } catch (e) { console.error(`RepoLoader.fetchSingleFileContents`, e); return null; } } /** * Fetches the next page of data from the API. * @param {Object} requestData - The request data. * @returns {Promise|null>} The next page of data, or null if no more pages. */ async fetchNextPage(requestData, retries = 0) { try { if (requestData.page === -1) return null; if (!requestData.page) requestData.page = 1; const { endpoint, perPage = 100, queryParams = {} } = requestData; const params = new URLSearchParams({ ...queryParams, per_page: perPage, page: requestData.page, }); const url = `${this.apiBase}${endpoint}?${params.toString()}`; const response = await fetch(url, { method: "GET", headers: this.accessToken ? { "PRIVATE-TOKEN": this.accessToken } : {}, }); if (response.status === 429) { if (retries >= MAX_RETRIES) { console.warn( `[Gitlab Loader]: Rate limit persists for ${endpoint} after ${retries} retries. Skipping.` ); return null; } const retryAfter = Number(response.headers.get("retry-after")) || 60; console.warn( `[Gitlab Loader]: Rate limit hit for ${endpoint}. Waiting ${retryAfter}s before retrying...` ); await this.#wait(retryAfter * 1000); return this.fetchNextPage(requestData, retries + 1); } if (response.status === 401) { console.warn( `[Gitlab Loader]: Unauthorized request for ${endpoint}. Skipping.` ); return null; } if (!response.ok) { console.warn( `[Gitlab Loader]: Unexpected status ${response.status} for ${endpoint}. Skipping.` ); return null; } const data = await response.json(); if (!Array.isArray(data)) { console.warn(`Unexpected response format for ${endpoint}:`, data); return []; } // GitLab omits x-total-pages for large repos, so use x-next-page // as the sole pagination signal — it's empty on the last page. const nextPage = response.headers.get("x-next-page"); const totalPages = response.headers.get("x-total-pages"); console.log( `Gitlab RepoLoader: fetched ${endpoint} page ${requestData.page}${ totalPages ? `/${totalPages}` : "" } with ${data.length} records.` ); requestData.page = nextPage?.trim() ? Number(nextPage) : -1; return data; } catch (e) { console.error(`RepoLoader.fetchNextPage`, e); return null; } } } module.exports = GitLabRepoLoader; ================================================ FILE: collector/utils/extensions/RepoLoader/GitlabRepo/index.js ================================================ const RepoLoader = require("./RepoLoader"); const fs = require("fs"); const path = require("path"); const { default: slugify } = require("slugify"); const { v4 } = require("uuid"); const { sanitizeFileName, writeToServerDocuments } = require("../../../files"); const { tokenizeString } = require("../../../tokenizer"); /** * Load in a Gitlab Repo recursively or just the top level if no PAT is provided * @param {object} args - forwarded request body params * @param {import("../../../middleware/setDataSigner").ResponseWithSigner} response - Express response object with encryptionWorker * @returns */ async function loadGitlabRepo(args, response) { const repo = new RepoLoader(args); await repo.init(); if (!repo.ready) return { success: false, reason: "Could not prepare Gitlab repo for loading! Check URL", }; console.log( `-- Working GitLab ${repo.author}/${repo.project}:${repo.branch} --` ); const docs = await repo.recursiveLoader(); if (!docs.length) { return { success: false, reason: "No files were found for those settings.", }; } console.log(`[GitLab Loader]: Found ${docs.length} source files. Saving...`); const outFolder = slugify( `${repo.author}-${repo.project}-${repo.branch}-${v4().slice(0, 4)}` ).toLowerCase(); const outFolderPath = process.env.NODE_ENV === "development" ? path.resolve( __dirname, `../../../../../server/storage/documents/${outFolder}` ) : path.resolve(process.env.STORAGE_DIR, `documents/${outFolder}`); if (!fs.existsSync(outFolderPath)) fs.mkdirSync(outFolderPath, { recursive: true }); for (const doc of docs) { if (!doc.metadata || (!doc.pageContent && !doc.issue && !doc.wiki)) continue; let pageContent = null; const data = { id: v4(), url: "gitlab://" + doc.metadata.source, docSource: doc.metadata.source, chunkSource: generateChunkSource( repo, doc, response.locals.encryptionWorker ), published: new Date().toLocaleString(), }; if (doc.pageContent) { pageContent = doc.pageContent; data.title = doc.metadata.source; data.docAuthor = repo.author; data.description = "No description found."; } else if (doc.issue) { pageContent = issueToMarkdown(doc.issue); data.title = `Issue ${doc.issue.iid}: ${doc.issue.title}`; data.docAuthor = doc.issue.author.username; data.description = doc.issue.description; } else if (doc.wiki) { pageContent = doc.wiki.content; data.title = doc.wiki.title; data.docAuthor = repo.author; data.description = doc.wiki.format === "markdown" ? "GitLab Wiki Page (Markdown)" : "GitLab Wiki Page"; } else { continue; } data.wordCount = pageContent.split(" ").length; data.token_count_estimate = tokenizeString(pageContent); data.pageContent = pageContent; console.log( `[GitLab Loader]: Saving ${doc.metadata.source} to ${outFolder}` ); writeToServerDocuments({ data, filename: sanitizeFileName(`${slugify(doc.metadata.source)}-${data.id}`), destinationOverride: outFolderPath, }); } return { success: true, reason: null, data: { author: repo.author, repo: repo.project, projectId: repo.projectId, branch: repo.branch, files: docs.length, destination: outFolder, }, }; } async function fetchGitlabFile({ repoUrl, branch, accessToken = null, sourceFilePath, }) { const repo = new RepoLoader({ repo: repoUrl, branch, accessToken, }); await repo.init(); if (!repo.ready) return { success: false, content: null, reason: "Could not prepare GitLab repo for loading! Check URL or PAT.", }; console.log( `-- Working GitLab ${repo.author}/${repo.project}:${repo.branch} file:${sourceFilePath} --` ); const fileContent = await repo.fetchSingleFile(sourceFilePath); if (!fileContent) { return { success: false, reason: "Target file returned a null content response.", content: null, }; } return { success: true, reason: null, content: fileContent, }; } function generateChunkSource(repo, doc, encryptionWorker) { const payload = { projectId: decodeURIComponent(repo.projectId), branch: repo.branch, path: doc.metadata.source, pat: !!repo.accessToken ? repo.accessToken : null, }; return `gitlab://${repo.repo}?payload=${encryptionWorker.encrypt( JSON.stringify(payload) )}`; } function issueToMarkdown(issue) { const metadata = {}; const userFields = ["author", "assignees", "closed_by"]; const userToUsername = ({ username }) => username; for (const userField of userFields) { if (issue[userField]) { if (Array.isArray(issue[userField])) { metadata[userField] = issue[userField].map(userToUsername); } else { metadata[userField] = userToUsername(issue[userField]); } } } const singleValueFields = [ "web_url", "state", "created_at", "updated_at", "closed_at", "due_date", "type", "merge_request_count", "upvotes", "downvotes", "labels", "has_tasks", "task_status", "confidential", "severity", ]; for (const singleValueField of singleValueFields) { metadata[singleValueField] = issue[singleValueField]; } if (issue.milestone) { metadata.milestone = `${issue.milestone.title} (${issue.milestone.id})`; } if (issue.time_stats) { const timeFields = ["time_estimate", "total_time_spent"]; for (const timeField of timeFields) { const fieldName = `human_${timeField}`; if (issue?.time_stats[fieldName]) { metadata[timeField] = issue.time_stats[fieldName]; } } } const metadataString = Object.entries(metadata) .map(([name, value]) => { if (!value || value?.length < 1) { return null; } let result = `- ${name.replace("_", " ")}:`; if (!Array.isArray(value)) { result += ` ${value}`; } else { result += "\n" + value.map((s) => ` - ${s}`).join("\n"); } return result; }) .filter((item) => item != null) .join("\n"); let markdown = `# ${issue.title} (${issue.iid}) ${issue.description} ## Metadata ${metadataString}`; if (issue.discussions.length > 0) { markdown += ` ## Activity ${issue.discussions.join("\n\n")} `; } return markdown; } module.exports = { loadGitlabRepo, fetchGitlabFile }; ================================================ FILE: collector/utils/extensions/RepoLoader/index.js ================================================ /** * Dynamically load the correct repository loader from a specific platform * by default will return GitHub. * @param {('github'|'gitlab')} platform * @returns {import("./GithubRepo/RepoLoader")|import("./GitlabRepo/RepoLoader")} the repo loader class for provider */ function resolveRepoLoader(platform = "github") { switch (platform) { case "github": console.log(`Loading GitHub RepoLoader...`); return require("./GithubRepo/RepoLoader"); case "gitlab": console.log(`Loading GitLab RepoLoader...`); return require("./GitlabRepo/RepoLoader"); default: console.log(`Loading GitHub RepoLoader...`); return require("./GithubRepo/RepoLoader"); } } /** * Dynamically load the correct repository loader function from a specific platform * by default will return Github. * @param {('github'|'gitlab')} platform * @returns {import("./GithubRepo")['fetchGithubFile'] | import("./GitlabRepo")['fetchGitlabFile']} the repo loader class for provider */ function resolveRepoLoaderFunction(platform = "github") { switch (platform) { case "github": console.log(`Loading GitHub loader function...`); return require("./GithubRepo").loadGithubRepo; case "gitlab": console.log(`Loading GitLab loader function...`); return require("./GitlabRepo").loadGitlabRepo; default: console.log(`Loading GitHub loader function...`); return require("./GithubRepo").loadGithubRepo; } } module.exports = { resolveRepoLoader, resolveRepoLoaderFunction }; ================================================ FILE: collector/utils/extensions/WebsiteDepth/index.js ================================================ const { v4 } = require("uuid"); const { PuppeteerWebBaseLoader, } = require("langchain/document_loaders/web/puppeteer"); const { default: slugify } = require("slugify"); const { parse } = require("node-html-parser"); const { writeToServerDocuments, documentsFolder } = require("../../files"); const { tokenizeString } = require("../../tokenizer"); const path = require("path"); const fs = require("fs"); const RuntimeSettings = require("../../runtimeSettings"); async function discoverLinks(startUrl, maxDepth = 1, maxLinks = 20) { const baseUrl = new URL(startUrl); const discoveredLinks = new Set([startUrl]); let queue = [[startUrl, 0]]; // [url, currentDepth] const scrapedUrls = new Set(); for (let currentDepth = 0; currentDepth < maxDepth; currentDepth++) { const levelSize = queue.length; const nextQueue = []; for (let i = 0; i < levelSize && discoveredLinks.size < maxLinks; i++) { const [currentUrl, urlDepth] = queue[i]; if (!scrapedUrls.has(currentUrl)) { scrapedUrls.add(currentUrl); const newLinks = await getPageLinks(currentUrl, baseUrl); for (const link of newLinks) { if (!discoveredLinks.has(link) && discoveredLinks.size < maxLinks) { discoveredLinks.add(link); if (urlDepth + 1 < maxDepth) { nextQueue.push([link, urlDepth + 1]); } } } } } queue = nextQueue; if (queue.length === 0 || discoveredLinks.size >= maxLinks) break; } return Array.from(discoveredLinks); } async function getPageLinks(url, baseUrl) { try { const runtimeSettings = new RuntimeSettings(); /** @type {import('puppeteer').PuppeteerLaunchOptions} */ let launchConfig = { headless: "new" }; /* On MacOS 15.1, the headless=new option causes the browser to crash immediately. * It is not clear why this is the case, but it is reproducible. Since AnythinglLM * in production runs in a container, we can disable headless mode to workaround the issue for development purposes. * * This may show a popup window when scraping a page in development mode. * This is expected behavior if seen in development mode on MacOS 15+ */ if ( process.platform === "darwin" && process.env.NODE_ENV === "development" ) { console.log( "Darwin Development Mode: Disabling headless mode to prevent Chromium from crashing." ); launchConfig.headless = "false"; } const loader = new PuppeteerWebBaseLoader(url, { launchOptions: { headless: launchConfig.headless, ignoreHTTPSErrors: true, args: runtimeSettings.get("browserLaunchArgs"), }, gotoOptions: { waitUntil: "networkidle2" }, }); const docs = await loader.load(); const html = docs[0].pageContent; const links = extractLinks(html, baseUrl); return links; } catch (error) { console.error(`Failed to get page links from ${url}.`, error); return []; } } function extractLinks(html, baseUrl) { const root = parse(html); const links = root.querySelectorAll("a"); const extractedLinks = new Set(); for (const link of links) { const href = link.getAttribute("href"); if (href) { const absoluteUrl = new URL(href, baseUrl.href).href; if ( absoluteUrl.startsWith( baseUrl.origin + baseUrl.pathname.split("/").slice(0, -1).join("/") ) ) { extractedLinks.add(absoluteUrl); } } } return Array.from(extractedLinks); } async function bulkScrapePages(links, outFolderPath) { const runtimeSettings = new RuntimeSettings(); /** @type {import('puppeteer').PuppeteerLaunchOptions} */ let launchConfig = { headless: "new" }; /* On MacOS 15.1, the headless=new option causes the browser to crash immediately. * It is not clear why this is the case, but it is reproducible. Since AnythinglLM * in production runs in a container, we can disable headless mode to workaround the issue for development purposes. * * This may show a popup window when scraping a page in development mode. * This is expected behavior if seen in development mode on MacOS 15+ */ if (process.platform === "darwin" && process.env.NODE_ENV === "development") { console.log( "Darwin Development Mode: Disabling headless mode to prevent Chromium from crashing." ); launchConfig.headless = "false"; } const scrapedData = []; for (let i = 0; i < links.length; i++) { const link = links[i]; console.log(`Scraping ${i + 1}/${links.length}: ${link}`); try { const loader = new PuppeteerWebBaseLoader(link, { launchOptions: { headless: launchConfig.headless, ignoreHTTPSErrors: true, args: runtimeSettings.get("browserLaunchArgs"), }, gotoOptions: { waitUntil: "networkidle2" }, async evaluate(page, browser) { const result = await page.evaluate(() => document.body.innerText); await browser.close(); return result; }, }); const docs = await loader.load(); const content = docs[0].pageContent; if (!content.length) { console.warn(`Empty content for ${link}. Skipping.`); continue; } const url = new URL(link); const decodedPathname = decodeURIComponent(url.pathname); const filename = `${url.hostname}${decodedPathname.replace(/\//g, "_")}`; const data = { id: v4(), url: "file://" + slugify(filename) + ".html", title: slugify(filename) + ".html", docAuthor: "no author found", description: "No description found.", docSource: "URL link uploaded by the user.", chunkSource: `link://${link}`, published: new Date().toLocaleString(), wordCount: content.split(" ").length, pageContent: content, token_count_estimate: tokenizeString(content), }; writeToServerDocuments({ data, filename: data.title, destinationOverride: outFolderPath, }); scrapedData.push(data); console.log(`Successfully scraped ${link}.`); } catch (error) { console.error(`Failed to scrape ${link}.`, error); } } return scrapedData; } async function websiteScraper(startUrl, depth = 1, maxLinks = 20) { const websiteName = new URL(startUrl).hostname; const outFolder = slugify( `${slugify(websiteName)}-${v4().slice(0, 4)}` ).toLowerCase(); const outFolderPath = path.resolve(documentsFolder, outFolder); console.log("Discovering links..."); const linksToScrape = await discoverLinks(startUrl, depth, maxLinks); console.log(`Found ${linksToScrape.length} links to scrape.`); if (!fs.existsSync(outFolderPath)) fs.mkdirSync(outFolderPath, { recursive: true }); console.log("Starting bulk scraping..."); const scrapedData = await bulkScrapePages(linksToScrape, outFolderPath); console.log(`Scraped ${scrapedData.length} pages.`); return scrapedData; } module.exports = websiteScraper; ================================================ FILE: collector/utils/extensions/YoutubeTranscript/YoutubeLoader/index.js ================================================ const { validYoutubeVideoUrl } = require("../../../url"); /* * This is just a custom implementation of the Langchain JS YouTubeLoader class * as the dependency for YoutubeTranscript is quite fickle and its a rat race to keep it up * and instead of waiting for patches we can just bring this simple script in-house and at least * be able to patch it since its so flaky. When we have more connectors we can kill this because * it will be a pain to maintain over time. */ class YoutubeLoader { #videoId; #language; #addVideoInfo; constructor({ videoId = null, language = null, addVideoInfo = false } = {}) { if (!videoId) throw new Error("Invalid video id!"); this.#videoId = videoId; this.#language = language; this.#addVideoInfo = addVideoInfo; } /** * Extracts the videoId from a YouTube video URL. * @param url The URL of the YouTube video. * @returns The videoId of the YouTube video. */ static getVideoID(url) { const videoId = validYoutubeVideoUrl(url, true); if (videoId) return videoId; throw new Error("Failed to get youtube video id from the url"); } /** * Creates a new instance of the YoutubeLoader class from a YouTube video * URL. * @param url The URL of the YouTube video. * @param config Optional configuration options for the YoutubeLoader instance, excluding the videoId. * @returns A new instance of the YoutubeLoader class. */ static createFromUrl(url, config = {}) { const videoId = YoutubeLoader.getVideoID(url); return new YoutubeLoader({ ...config, videoId }); } /** * Loads the transcript and video metadata from the specified YouTube * video. It uses the youtube-transcript library to fetch the transcript * and the youtubei.js library to fetch the video metadata. * @returns Langchain like doc that is 1 element with PageContent and */ async load() { let transcript; const metadata = { source: this.#videoId, }; try { const fetchTranscript = await import("youtube-transcript-plus").then( (module) => module.fetchTranscript ); const transcriptSegments = await fetchTranscript(this.#videoId, { lang: this.#language, }); if (!transcriptSegments || transcriptSegments.length === 0) throw new Error("Transcription not found"); transcript = this.#convertTranscriptSegmentsToText(transcriptSegments); if (this.#addVideoInfo) { const { Innertube } = require("youtubei.js"); const youtube = await Innertube.create(); const info = (await youtube.getBasicInfo(this.#videoId)).basic_info; metadata.description = info.short_description; metadata.title = info.title; metadata.view_count = info.view_count; metadata.author = info.author; } } catch (e) { throw new Error( `Failed to get YouTube video transcription: ${e?.message}` ); } return [ { pageContent: transcript, metadata, }, ]; } #convertTranscriptSegmentsToText(transcriptSegments) { return transcriptSegments .map((segment) => typeof segment === "string" ? segment : segment.text || "" ) .join(" ") .replace(/\s+/g, " ") .trim(); } } module.exports.YoutubeLoader = YoutubeLoader; ================================================ FILE: collector/utils/extensions/YoutubeTranscript/YoutubeLoader/youtube-transcript.js ================================================ const { validYoutubeVideoUrl } = require("../../../url"); class YoutubeTranscriptError extends Error { constructor(message) { super(`[YoutubeTranscript] ${message}`); } } /** * Handles fetching and parsing YouTube video transcripts */ class YoutubeTranscript { /** * Encodes a string as a protobuf field * @param {number} fieldNumber - The protobuf field number * @param {string} str - The string to encode * @returns {Buffer} Encoded protobuf field */ static #encodeProtobufString(fieldNumber, str) { const utf8Bytes = Buffer.from(str, "utf8"); const tag = (fieldNumber << 3) | 2; // wire type 2 for string const lengthBytes = this.#encodeVarint(utf8Bytes.length); return Buffer.concat([ Buffer.from([tag]), Buffer.from(lengthBytes), utf8Bytes, ]); } /** * Encodes a number as a protobuf varint * @param {number} value - The number to encode * @returns {number[]} Encoded varint bytes */ static #encodeVarint(value) { const bytes = []; while (value >= 0x80) { bytes.push((value & 0x7f) | 0x80); value >>>= 7; } bytes.push(value); return bytes; } /** * Creates a base64 encoded protobuf message * @param {Object} param - The parameters to encode * @param {string} param.param1 - First parameter * @param {string} param.param2 - Second parameter * @returns {string} Base64 encoded protobuf */ static #getBase64Protobuf({ param1, param2 }) { const field1 = this.#encodeProtobufString(1, param1); const field2 = this.#encodeProtobufString(2, param2); return Buffer.concat([field1, field2]).toString("base64"); } /** * Extracts transcript text from YouTube API response * @param {Object} responseData - The YouTube API response * @returns {string} Combined transcript text */ static #extractTranscriptFromResponse(responseData) { const transcriptRenderer = responseData.actions?.[0]?.updateEngagementPanelAction?.content ?.transcriptRenderer; if (!transcriptRenderer) { throw new Error("No transcript data found in response"); } const segments = transcriptRenderer.content?.transcriptSearchPanelRenderer?.body ?.transcriptSegmentListRenderer?.initialSegments; if (!segments) { throw new Error("Transcript segments not found in response"); } return segments .map((segment) => { const runs = segment.transcriptSegmentRenderer?.snippet?.runs; return runs ? runs.map((run) => run.text).join("") : ""; }) .filter((text) => text) .join(" ") .trim() .replace(/\s+/g, " "); } /** * Calculates a preference score for a caption track to determine the best match * @param {Object} track - The caption track object from YouTube * @param {string} track.languageCode - ISO language code (e.g., 'zh-HK', 'en', 'es') * @param {string} track.kind - Track type ('asr' for auto-generated, "" for human-transcribed) * @param {string[]} preferredLanguages - Array of language codes in preference order (e.g., ['zh-HK', 'en']) * @returns {number} Preference score (lower is better) */ static #calculatePreferenceScore(track, preferredLanguages) { // Language preference: index in preferredLanguages array (0 = most preferred) const languagePreference = preferredLanguages.indexOf(track.languageCode); const languageScore = languagePreference === -1 ? 9999 : languagePreference; // Kind bonus: prefer human-transcribed (undefined) over auto-generated ('asr') const kindBonus = track.kind === "asr" ? 0.5 : 0; return languageScore + kindBonus; } /** * Finds the most suitable caption track based on preferred languages * @param {string} videoBody - The raw HTML response from YouTube * @param {string[]} preferredLanguages - Array of language codes in preference order * @returns {Object|null} The selected caption track or null if none found */ static #findPreferredCaptionTrack(videoBody, preferredLanguages) { const captionsConfigJson = videoBody.match( /"captions":(.*?),"videoDetails":/s ); const captionsConfig = captionsConfigJson?.[1] ? JSON.parse(captionsConfigJson[1]) : null; const captionTracks = captionsConfig ? captionsConfig.playerCaptionsTracklistRenderer.captionTracks : null; if (!captionTracks || captionTracks.length === 0) { return null; } const sortedTracks = [...captionTracks].sort((a, b) => { const scoreA = this.#calculatePreferenceScore(a, preferredLanguages); const scoreB = this.#calculatePreferenceScore(b, preferredLanguages); return scoreA - scoreB; }); return sortedTracks[0]; } /** * Fetches video page content and finds the preferred caption track * @param {string} videoId - YouTube video ID * @param {string[]} preferredLanguages - Array of preferred language codes * @returns {Promise} The preferred caption track * @throws {YoutubeTranscriptError} If no suitable caption track is found */ static async #getPreferredCaptionTrack(videoId, preferredLanguages) { const videoResponse = await fetch( `https://www.youtube.com/watch?v=${videoId}`, { credentials: "omit" } ); const videoBody = await videoResponse.text(); const preferredCaptionTrack = this.#findPreferredCaptionTrack( videoBody, preferredLanguages ); if (!preferredCaptionTrack) { throw new YoutubeTranscriptError( "No suitable caption track found for the video" ); } return preferredCaptionTrack; } /** * Fetch transcript from YouTube video * @param {string} videoId - Video URL or video identifier * @param {Object} config - Configuration options * @param {string} [config.lang='en'] - Language code (e.g., 'en', 'es', 'fr') * @returns {Promise} Video transcript text */ static async fetchTranscript(videoId, config = {}) { const preferredLanguages = config?.lang ? [config?.lang, "en"] : ["en"]; const identifier = this.retrieveVideoId(videoId); try { const preferredCaptionTrack = await this.#getPreferredCaptionTrack( identifier, preferredLanguages ); const innerProto = this.#getBase64Protobuf({ param1: preferredCaptionTrack.kind || "", param2: preferredCaptionTrack.languageCode, }); const params = this.#getBase64Protobuf({ param1: identifier, param2: innerProto, }); const response = await fetch( "https://www.youtube.com/youtubei/v1/get_transcript", { method: "POST", headers: { "Content-Type": "application/json", "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.83 Safari/537.36,gzip(gfe)", }, body: JSON.stringify({ context: { client: { clientName: "WEB", clientVersion: "2.20240826.01.00", }, }, params, }), } ); if (!response.ok) { throw new Error(`HTTP error! status: ${response.status}`); } const responseData = await response.json(); return this.#extractTranscriptFromResponse(responseData); } catch (e) { throw new YoutubeTranscriptError(e.message || e); } } /** * Extract video ID from a YouTube URL or verify an existing ID * @param {string} videoId - Video URL or ID * @returns {string} YouTube video ID */ static retrieveVideoId(videoId) { if (videoId.length === 11) return videoId; // already a valid ID most likely const matchedId = validYoutubeVideoUrl(videoId, true); if (matchedId) return matchedId; throw new YoutubeTranscriptError( "Impossible to retrieve Youtube video ID." ); } } module.exports = { YoutubeTranscript, YoutubeTranscriptError, }; ================================================ FILE: collector/utils/extensions/YoutubeTranscript/index.js ================================================ const fs = require("fs"); const path = require("path"); const { default: slugify } = require("slugify"); const { v4 } = require("uuid"); const { writeToServerDocuments, sanitizeFileName, documentsFolder, isWithin, } = require("../../files"); const { tokenizeString } = require("../../tokenizer"); const { YoutubeLoader } = require("./YoutubeLoader"); const { validYoutubeVideoUrl } = require("../../url"); /** * Fetch the transcript content for a YouTube video * @param {string} url - The URL of the YouTube video * @returns {Promise<{success: boolean, reason: string|null, content: string|null, metadata: TranscriptMetadata}>} - The transcript content for the YouTube video */ async function fetchVideoTranscriptContent({ url }) { if (!validYoutubeVideoUrl(url)) { return { success: false, reason: "Invalid URL. Should be youtu.be or youtube.com/watch.", content: null, metadata: {}, }; } console.log(`-- Working YouTube ${url} --`); const loader = YoutubeLoader.createFromUrl(url, { addVideoInfo: true }); const { docs, error } = await loader .load() .then((docs) => ({ docs, error: null })) .catch((e) => ({ docs: [], error: e.message?.split("Error:")?.[1] || e.message, })); if (!docs.length || !!error) { return { success: false, reason: error ?? "No transcript found for that YouTube video.", content: null, metadata: {}, }; } const metadata = docs[0].metadata; const content = docs[0].pageContent; if (!content.length) { return { success: false, reason: "No transcript could be parsed for that YouTube video.", content: null, metadata: {}, }; } return { success: true, reason: null, content, metadata, }; } /** * @typedef {Object} TranscriptMetadata * @property {string} title - The title of the video * @property {string} author - The author of the video * @property {string} description - The description of the video * @property {string} view_count - The view count of the video * @property {string} source - The source of the video (videoId) */ /** * @typedef {Object} TranscriptAsDocument * @property {boolean} success - Whether the transcript was successful * @property {string|null} reason - The reason for the transcript * @property {TranscriptMetadata} metadata - The metadata from the transcript */ /** * @typedef {Object} TranscriptAsContent * @property {boolean} success - Whether the transcript was successful * @property {string|null} reason - The reason for the transcript * @property {string|null} content - The content of the transcript * @property {Object[]} documents - The documents from the transcript * @property {boolean} saveAsDocument - Whether to save the transcript as a document */ /** * Load the transcript content for a YouTube video as well as save it to the server documents * @param {Object} params - The parameters for the YouTube transcript * @param {string} params.url - The URL of the YouTube video * @param {Object} options - The options for the YouTube transcript * @param {boolean} options.parseOnly - Whether to parse the transcript content only or save it to the server documents * @returns {Promise} - The transcript content for the YouTube video */ async function loadYouTubeTranscript({ url }, options = { parseOnly: false }) { const transcriptResults = await fetchVideoTranscriptContent({ url }); if (!transcriptResults.success) { return { success: false, reason: transcriptResults.reason || "An unknown error occurred during transcription retrieval", documents: [], content: null, saveAsDocument: options.parseOnly, data: {}, }; } const { content, metadata } = transcriptResults; if (options.parseOnly) { return { success: true, reason: null, content: buildTranscriptContentWithMetadata(content, metadata), documents: [], saveAsDocument: options.parseOnly, data: {}, }; } const outFolder = sanitizeFileName( slugify(`${metadata.author} YouTube transcripts`).toLowerCase() ); const outFolderPath = path.resolve(documentsFolder, outFolder); const uuid = v4(); const fileName = sanitizeFileName(`${slugify(metadata.title)}-${uuid}`); if (!isWithin(documentsFolder, path.resolve(outFolderPath, fileName))) { console.error( `[YouTube Loader]: Invalid file path ${path.resolve( outFolderPath, fileName )} is not within the documents folder ${documentsFolder}` ); return { success: false, reason: `[YouTube Loader]: Invalid file path ${path.resolve( outFolderPath, fileName )} is not within the documents folder ${documentsFolder}`, documents: [], data: {}, }; } if (!fs.existsSync(outFolderPath)) fs.mkdirSync(outFolderPath, { recursive: true }); const data = { id: uuid, url: url + ".youtube", title: metadata.title || url, docAuthor: metadata.author, description: metadata.description, docSource: url, chunkSource: `youtube://${url}`, published: new Date().toLocaleString(), wordCount: content.split(" ").length, pageContent: content, token_count_estimate: tokenizeString(content), }; console.log(`[YouTube Loader]: Saving ${metadata.title} to ${outFolder}`); const document = writeToServerDocuments({ data, filename: fileName, destinationOverride: outFolderPath, }); return { success: true, reason: null, documents: [document], data: { title: metadata.title, author: metadata.author, destination: outFolder, }, }; } /** * Generate the transcript content and metadata into a single string * * Why? For ephemeral documents where we just want the content, we want to include the metadata as keys in the content * so that the LLM has context about the video, this gives it a better understanding of the video * and allows it to use the metadata in the conversation if relevant. * Examples: * - How many views does have? * - Checkout and tell me the key points and if it is performing well * - Summarize this video ? -> description could have links and references * @param {string} content - The content of the transcript * @param {TranscriptMetadata} metadata - The metadata from the transcript * @returns {string} - The concatenated transcript content and metadata */ function buildTranscriptContentWithMetadata(content = "", metadata = {}) { const VALID_METADATA_KEYS = ["title", "author", "description", "view_count"]; if (!content || !metadata || Object.keys(metadata).length === 0) return content; let contentWithMetadata = ""; VALID_METADATA_KEYS.forEach((key) => { if (!metadata[key]) return; contentWithMetadata += `<${key}>${metadata[key]}`; }); return `${contentWithMetadata}\nTranscript:\n${content}`; } module.exports = { loadYouTubeTranscript, fetchVideoTranscriptContent, }; ================================================ FILE: collector/utils/files/index.js ================================================ const fs = require("fs"); const path = require("path"); const { MimeDetector } = require("./mime"); /** * The folder where documents are stored to be stored when * processed by the collector. */ const documentsFolder = process.env.NODE_ENV === "development" ? path.resolve(__dirname, `../../../server/storage/documents`) : path.resolve(process.env.STORAGE_DIR, `documents`); /** * The folder where direct uploads are stored to be stored when * processed by the collector. These are files that were DnD'd into UI * and are not to be embedded or selectable from the file picker. */ const directUploadsFolder = process.env.NODE_ENV === "development" ? path.resolve(__dirname, `../../../server/storage/direct-uploads`) : path.resolve(process.env.STORAGE_DIR, `direct-uploads`); /** * Checks if a file is text by checking the mime type and then falling back to buffer inspection. * This way we can capture all the cases where the mime type is not known but still parseable as text * without having to constantly add new mime type overrides. * @param {string} filepath - The path to the file. * @returns {boolean} - Returns true if the file is text, false otherwise. */ function isTextType(filepath) { if (!fs.existsSync(filepath)) return false; const result = isKnownTextMime(filepath); if (result.valid) return true; // Known text type - return true. if (result.reason !== "generic") return false; // If any other reason than generic - return false. return parseableAsText(filepath); // Fallback to parsing as text via buffer inspection. } /** * Checks if a file is known to be text by checking the mime type. * @param {string} filepath - The path to the file. * @returns {boolean} - Returns true if the file is known to be text, false otherwise. */ function isKnownTextMime(filepath) { try { const mimeLib = new MimeDetector(); const mime = mimeLib.getType(filepath); if (mimeLib.badMimes.includes(mime)) return { valid: false, reason: "bad_mime" }; const type = mime.split("/")[0]; if (mimeLib.nonTextTypes.includes(type)) return { valid: false, reason: "non_text_mime" }; return { valid: true, reason: "valid_mime" }; } catch { return { valid: false, reason: "generic" }; } } /** * Checks if a file is parseable as text by forcing it to be read as text in utf8 encoding. * If the file looks too much like a binary file, it will return false. * @param {string} filepath - The path to the file. * @returns {boolean} - Returns true if the file is parseable as text, false otherwise. */ function parseableAsText(filepath) { try { const fd = fs.openSync(filepath, "r"); const buffer = Buffer.alloc(1024); // Read first 1KB of the file synchronously const bytesRead = fs.readSync(fd, buffer, 0, 1024, 0); fs.closeSync(fd); const content = buffer.subarray(0, bytesRead).toString("utf8"); const nullCount = (content.match(/\0/g) || []).length; //eslint-disable-next-line const controlCount = (content.match(/[\x00-\x08\x0B\x0C\x0E-\x1F]/g) || []) .length; const threshold = bytesRead * 0.1; return nullCount + controlCount < threshold; } catch { return false; } } function trashFile(filepath) { if (!fs.existsSync(filepath)) return; try { const isDir = fs.lstatSync(filepath).isDirectory(); if (isDir) return; } catch { return; } fs.rmSync(filepath); return; } function createdDate(filepath) { try { const { birthtimeMs, birthtime } = fs.statSync(filepath); if (birthtimeMs === 0) throw new Error("Invalid stat for file!"); return birthtime.toLocaleString(); } catch { return "unknown"; } } /** * Writes a document to the server documents folder. * @param {Object} params - The parameters for the function. * @param {Object} params.data - The data to write to the file. Must look like a document object. * @param {string} params.filename - The name of the file to write to. * @param {string|null} params.destinationOverride - A forced destination to write to - will be honored if provided. * @param {Object} params.options - The options for the function. * @param {boolean} params.options.parseOnly - If true, the file will be written to the direct uploads folder instead of the documents folder. Will be ignored if destinationOverride is provided. * @returns {Object} - The data with the location added. */ function writeToServerDocuments({ data = {}, filename, destinationOverride = null, options = {}, }) { if (!filename) throw new Error("Filename is required!"); let destination = null; if (destinationOverride) destination = path.resolve(destinationOverride); else if (options.parseOnly) destination = path.resolve(directUploadsFolder); else destination = path.resolve(documentsFolder, "custom-documents"); if (!fs.existsSync(destination)) fs.mkdirSync(destination, { recursive: true }); const destinationFilePath = normalizePath( path.resolve(destination, filename) + ".json" ); fs.writeFileSync(destinationFilePath, JSON.stringify(data, null, 4), { encoding: "utf-8", }); return { ...data, // relative location string that can be passed into the /update-embeddings api // that will work since we know the location exists and since we only allow // 1-level deep folders this will always work. This still works for integrations like GitHub and YouTube. location: destinationFilePath.split("/").slice(-2).join("/"), isDirectUpload: options.parseOnly || false, }; } // When required we can wipe the entire collector hotdir and tmp storage in case // there were some large file failures that we unable to be removed a reboot will // force remove them. async function wipeCollectorStorage() { const cleanHotDir = new Promise((resolve) => { const directory = path.resolve(__dirname, "../../hotdir"); fs.readdir(directory, (err, files) => { if (err) resolve(); for (const file of files) { if (file === "__HOTDIR__.md") continue; try { fs.rmSync(path.join(directory, file)); } catch {} } resolve(); }); }); const cleanTmpDir = new Promise((resolve) => { const directory = path.resolve(__dirname, "../../storage/tmp"); fs.readdir(directory, (err, files) => { if (err) resolve(); for (const file of files) { if (file === ".placeholder") continue; try { fs.rmSync(path.join(directory, file)); } catch {} } resolve(); }); }); await Promise.all([cleanHotDir, cleanTmpDir]); console.log(`Collector hot directory and tmp storage wiped!`); return; } /** * Checks if a given path is within another path. * @param {string} outer - The outer path (should be resolved). * @param {string} inner - The inner path (should be resolved). * @returns {boolean} - Returns true if the inner path is within the outer path, false otherwise. */ function isWithin(outer, inner) { if (outer === inner) return false; const rel = path.relative(outer, inner); return !rel.startsWith("../") && rel !== ".."; } function normalizePath(filepath = "") { const result = path .normalize(filepath.trim()) .replace(/^(\.\.(\/|\\|$))+/, "") .trim(); if (["..", ".", "/"].includes(result)) throw new Error("Invalid path."); return result; } function sanitizeFileName(fileName) { if (!fileName) return fileName; //eslint-disable-next-line return fileName.replace(/[<>:"\/\\|?*]/g, ""); } module.exports = { trashFile, isTextType, createdDate, writeToServerDocuments, wipeCollectorStorage, normalizePath, isWithin, sanitizeFileName, documentsFolder, directUploadsFolder, }; ================================================ FILE: collector/utils/files/mime.js ================================================ const MimeLib = require("mime"); class MimeDetector { nonTextTypes = ["multipart", "model", "audio", "video", "font"]; badMimes = [ "application/octet-stream", "application/zip", "application/pkcs8", "application/vnd.microsoft.portable-executable", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", // XLSX are binaries and need to be handled explicitly. "application/x-msdownload", ]; constructor() { this.lib = MimeLib; this.setOverrides(); } setOverrides() { // the .ts extension maps to video/mp2t because of https://en.wikipedia.org/wiki/MPEG_transport_stream // which has had this extension far before TS was invented. So need to force re-map this MIME map. this.lib.define( { "text/plain": [ "ts", "tsx", "py", "opts", "lock", "jsonl", "qml", "sh", "c", "cs", "h", "js", "lua", "pas", "r", "go", "ino", "hpp", "linq", "cs", ], }, true ); } /** * Returns the MIME type of the file. If the file has no extension found, it will be processed as a text file. * @param {string} filepath * @returns {string} */ getType(filepath) { const parsedMime = this.lib.getType(filepath); if (!!parsedMime) return parsedMime; return null; } } module.exports = { MimeDetector, }; ================================================ FILE: collector/utils/http/index.js ================================================ process.env.NODE_ENV === "development" ? require("dotenv").config({ path: `.env.${process.env.NODE_ENV}` }) : require("dotenv").config(); function reqBody(request) { return typeof request.body === "string" ? JSON.parse(request.body) : request.body; } function queryParams(request) { return request.query; } /** * Validates if the provided baseUrl is a valid URL at all. * - Does not validate if the URL is reachable or accessible. * - Does not do any further validation of the URL like `validURL` in `utils/url/index.js` * @param {string} baseUrl * @returns {boolean} */ function validBaseUrl(baseUrl) { try { new URL(baseUrl); return true; } catch { return false; } } module.exports = { reqBody, queryParams, validBaseUrl, }; ================================================ FILE: collector/utils/logger/index.js ================================================ const winston = require("winston"); class Logger { logger = console; static _instance; constructor() { if (Logger._instance) return Logger._instance; this.logger = process.env.NODE_ENV === "production" ? this.getWinstonLogger() : console; Logger._instance = this; } getWinstonLogger() { const logger = winston.createLogger({ level: "info", defaultMeta: { service: "collector" }, transports: [ new winston.transports.Console({ format: winston.format.combine( winston.format.colorize(), winston.format.printf( ({ level, message, service, origin = "" }) => { return `\x1b[36m[${service}]\x1b[0m${ origin ? `\x1b[33m[${origin}]\x1b[0m` : "" } ${level}: ${message}`; } ) ), }), ], }); function formatArgs(args) { return args .map((arg) => { if (arg instanceof Error) { return arg.stack; // If argument is an Error object, return its stack trace } else if (typeof arg === "object") { return JSON.stringify(arg); // Convert objects to JSON string } else { return arg; // Otherwise, return as-is } }) .join(" "); } console.log = function (...args) { logger.info(formatArgs(args)); }; console.error = function (...args) { logger.error(formatArgs(args)); }; console.info = function (...args) { logger.warn(formatArgs(args)); }; return logger; } } /** * Sets and overrides Console methods for logging when called. * This is a singleton method and will not create multiple loggers. * @returns {winston.Logger | console} - instantiated logger interface. */ function setLogger() { return new Logger().logger; } module.exports = setLogger; ================================================ FILE: collector/utils/runtimeSettings/index.js ================================================ const { reqBody } = require("../http"); /** * Runtime settings are used to configure the collector per-request. * These settings are persisted across requests, but can be overridden per-request. * * The settings are passed in the request body via `options.runtimeSettings` * which is set in the backend #attachOptions function in CollectorApi. * * We do this so that the collector and backend can share the same ENV variables * but only pass the relevant settings to the collector per-request and be able to * access them across the collector via a single instance of RuntimeSettings. * * TODO: We may want to set all options passed from backend to collector here, * but for now - we are only setting the runtime settings specifically for backwards * compatibility with existing CollectorApi usage. */ class RuntimeSettings { static _instance = null; settings = {}; // Any settings here will be persisted across requests // and must be explicitly defined here. settingConfigs = { seenAnyIpWarning: { default: false, validate: (value) => String(value) === "true", }, allowAnyIp: { default: false, // Value must be explicitly "true" or "false" as a string validate: (value) => String(value) === "true", }, browserLaunchArgs: { default: [], validate: (value) => { let args = []; if (Array.isArray(value)) args = value.map((arg) => String(arg.trim())); if (typeof value === "string") args = value.split(",").map((arg) => arg.trim()); return args; }, }, }; constructor() { if (RuntimeSettings._instance) return RuntimeSettings._instance; RuntimeSettings._instance = this; return this; } /** * Parse the runtime settings from the request body options body * see #attachOptions https://github.com/Mintplex-Labs/anything-llm/blob/ebf112007e0d579af3d2b43569db95bdfc59074b/server/utils/collectorApi/index.js#L18 * @param {import('express').Request} request * @returns {void} */ parseOptionsFromRequest(request = {}) { const options = reqBody(request)?.options?.runtimeSettings || {}; for (const [key, value] of Object.entries(options)) { if (!this.settingConfigs.hasOwnProperty(key)) continue; this.set(key, value); } return; } /** * Get a runtime setting * - Will throw an error if the setting requested is not a supported runtime setting key * - Will return the default value if the setting requested is not set at all * @param {string} key * @returns {any} */ get(key) { if (!this.settingConfigs[key]) throw new Error(`Invalid runtime setting: ${key}`); return this.settings.hasOwnProperty(key) ? this.settings[key] : this.settingConfigs[key].default; } /** * Set a runtime setting * - Will throw an error if the setting requested is not a supported runtime setting key * - Will validate the value against the setting's validate function * @param {string} key * @param {any} value * @returns {void} */ set(key, value = null) { if (!this.settingConfigs[key]) throw new Error(`Invalid runtime setting: ${key}`); this.settings[key] = this.settingConfigs[key].validate(value); } } module.exports = RuntimeSettings; ================================================ FILE: collector/utils/shell.js ================================================ /** * Patch the shell environment path to ensure the PATH is properly set for the current platform. * On Docker, we are on Node v18 and cannot support fix-path v5. * So we need to use the ESM-style import() to import the fix-path module + add the strip-ansi call to patch the PATH, which is the only change between v4 and v5. * https://github.com/sindresorhus/fix-path/issues/6 * @returns {Promise<{[key: string]: string}>} - Environment variables from shell */ async function patchShellEnvironmentPath() { try { if (process.platform === "win32") return process.env; const { default: fixPath } = await import("fix-path"); const { default: stripAnsi } = await import("strip-ansi"); fixPath(); if (process.env.PATH) process.env.PATH = stripAnsi(process.env.PATH); console.log("Shell environment path patched successfully."); return process.env; } catch (error) { console.error("Failed to patch shell environment path:", error); return process.env; } } module.exports = { patchShellEnvironmentPath, }; ================================================ FILE: collector/utils/tokenizer/index.js ================================================ const { getEncoding } = require("js-tiktoken"); class TikTokenTokenizer { static MAX_KB_ESTIMATE = 10; static DIVISOR = 8; constructor() { if (TikTokenTokenizer.instance) { this.log( "Singleton instance already exists. Returning existing instance." ); return TikTokenTokenizer.instance; } this.encoder = getEncoding("cl100k_base"); TikTokenTokenizer.instance = this; this.log("Initialized new TikTokenTokenizer instance."); } log(text, ...args) { console.log(`\x1b[35m[TikTokenTokenizer]\x1b[0m ${text}`, ...args); } /** * Check if the input is too long to encode * this is more of a rough estimate and a sanity check to prevent * CPU issues from encoding too large of strings * Assumes 1 character = 2 bytes in JS * @param {string} input * @returns {boolean} */ #isTooLong(input) { const bytesEstimate = input.length * 2; const kbEstimate = Math.floor(bytesEstimate / 1024); return kbEstimate >= TikTokenTokenizer.MAX_KB_ESTIMATE; } /** * Encode a string into tokens for rough token count estimation. * @param {string} input * @returns {number} */ tokenizeString(input = "") { try { if (this.#isTooLong(input)) { this.log("Input will take too long to encode - estimating"); return Math.ceil(input.length / TikTokenTokenizer.DIVISOR); } return this.encoder.encode(input).length; } catch (e) { this.log("Could not tokenize string! Estimating...", e.message, e.stack); return Math.ceil(input?.length / TikTokenTokenizer.DIVISOR) || 0; } } } const tokenizer = new TikTokenTokenizer(); module.exports = { /** * Encode a string into tokens for rough token count estimation. * @param {string} input * @returns {number} */ tokenizeString: (input) => tokenizer.tokenizeString(input), }; ================================================ FILE: collector/utils/url/index.js ================================================ const RuntimeSettings = require("../runtimeSettings"); /** ATTN: SECURITY RESEARCHERS * To Security researchers about to submit an SSRF report CVE - please don't. * We are aware that the code below is does not defend against any of the thousands of ways * you can map a hostname to another IP via tunneling, hosts editing, etc. The code below does not have intention of blocking this * and is simply to prevent the user from accidentally putting in non-valid websites, which is all this protects * since _all urls must be submitted by the user anyway_ and cannot be done with authentication and manager or admin roles. * If an attacker has those roles then the system is already vulnerable and this is not a primary concern. * * We have gotten this report may times, marked them as duplicate or information and continue to get them. We communicate * already that deployment (and security) of an instance is on the deployer and system admin deploying it. This would include * isolation, firewalls, and the general security of the instance. */ const VALID_PROTOCOLS = ["https:", "http:"]; const INVALID_OCTETS = [192, 172, 10, 127]; const runtimeSettings = new RuntimeSettings(); /** * If an ip address is passed in the user is attempting to collector some internal service running on internal/private IP. * This is not a security feature and simply just prevents the user from accidentally entering invalid IP addresses. * Can be bypassed via COLLECTOR_ALLOW_ANY_IP environment variable. * @param {URL} param0 * @param {URL['hostname']} param0.hostname * @returns {boolean} */ function isInvalidIp({ hostname }) { if (runtimeSettings.get("allowAnyIp")) { if (!runtimeSettings.get("seenAnyIpWarning")) { console.log( "\x1b[33mURL IP local address restrictions have been disabled by administrator!\x1b[0m" ); runtimeSettings.set("seenAnyIpWarning", true); } return false; } const IPRegex = new RegExp( /^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$/gi ); // Not an IP address at all - passthrough if (!IPRegex.test(hostname)) return false; const [octetOne, ..._rest] = hostname.split("."); // If fails to validate to number - abort and return as invalid. if (isNaN(Number(octetOne))) return true; // Allow localhost loopback and 0.0.0.0 for scraping convenience // for locally hosted services or websites if (["127.0.0.1", "0.0.0.0"].includes(hostname)) return false; return INVALID_OCTETS.includes(Number(octetOne)); } /** * Validates a URL strictly * - Checks the URL forms a valid URL * - Checks the URL is at least HTTP(S) * - Checks the URL is not an internal IP - can be bypassed via COLLECTOR_ALLOW_ANY_IP * @param {string} url * @returns {boolean} */ function validURL(url) { try { const destination = new URL(url); if (!VALID_PROTOCOLS.includes(destination.protocol)) return false; if (isInvalidIp(destination)) return false; return true; } catch {} return false; } /** * Modifies a URL to be valid: * - Checks the URL is at least HTTP(S) so that protocol exists * - Checks the URL forms a valid URL * @param {string} url * @returns {string} */ function validateURL(url) { try { let destination = url.trim(); // If the URL has a protocol, just pass through // If the URL doesn't have a protocol, assume https:// if (destination.includes("://")) destination = new URL(destination).toString(); else destination = new URL(`https://${destination}`).toString(); // If the URL ends with a slash, remove it return destination.endsWith("/") ? destination.slice(0, -1) : destination; } catch { if (typeof url !== "string") return ""; return url.trim(); } } /** * Validate if a link is a valid YouTube video URL * - Checks youtu.be, youtube.com, m.youtube.com, music.youtube.com * - Embed video URLs * - Short URLs * - Live URLs * - Regular watch URLs * - Optional query parameters (including ?v parameter) * * Can be used to extract the video ID from a YouTube video URL via the returnVideoId parameter. * @param {string} link - The link to validate * @param {boolean} returnVideoId - Whether to return the video ID if the link is a valid YouTube video URL * @returns {boolean|string} - Whether the link is a valid YouTube video URL or the video ID if returnVideoId is true */ function validYoutubeVideoUrl(link, returnVideoId = false) { try { if (!link || typeof link !== "string") return false; let urlToValidate = link; if (!link.startsWith("http://") && !link.startsWith("https://")) { urlToValidate = "https://" + link; urlToValidate = new URL(urlToValidate).toString(); } const regex = /^(?:https?:\/\/)?(?:www\.|m\.|music\.)?(?:youtu\.be\/|youtube\.com\/(?:embed\/|v\/|watch\?(?:.*&)?v=|(?:live\/)?|shorts\/))([\w-]{11})(?:\S+)?$/; const match = urlToValidate.match(regex); if (returnVideoId) return match?.[1] ?? null; return !!match?.[1]; } catch (error) { console.error("Error validating YouTube video URL", error); return returnVideoId ? null : false; } } module.exports = { validURL, validateURL, validYoutubeVideoUrl, }; ================================================ FILE: collector/yarn.lock ================================================ # THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. # yarn lockfile v1 "@anthropic-ai/sdk@^0.9.1": version "0.9.1" resolved "https://registry.yarnpkg.com/@anthropic-ai/sdk/-/sdk-0.9.1.tgz#b2d2b7bf05c90dce502c9a2e869066870f69ba88" integrity sha512-wa1meQ2WSfoY8Uor3EdrJq0jTiZJoKoSii2ZVWRY1oN4Tlr5s59pADg9T79FTbPe1/se5c3pBeZgJL63wmuoBA== dependencies: "@types/node" "^18.11.18" "@types/node-fetch" "^2.6.4" abort-controller "^3.0.0" agentkeepalive "^4.2.1" digest-fetch "^1.3.0" form-data-encoder "1.7.2" formdata-node "^4.3.2" node-fetch "^2.6.7" web-streams-polyfill "^3.2.1" "@babel/code-frame@^7.0.0": version "7.27.1" resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.27.1.tgz#200f715e66d52a23b221a9435534a91cc13ad5be" integrity sha512-cjQ7ZlQ0Mv3b47hABuTevyTuYN4i+loJKGeV9flcCgIK37cCXRh+L1bd3iBHlynerhQ7BhCkn2BPbQUL+rGqFg== dependencies: "@babel/helper-validator-identifier" "^7.27.1" js-tokens "^4.0.0" picocolors "^1.1.1" "@babel/helper-validator-identifier@^7.27.1": version "7.28.5" resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.28.5.tgz#010b6938fab7cb7df74aa2bbc06aa503b8fe5fb4" integrity sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q== "@colors/colors@1.6.0", "@colors/colors@^1.6.0": version "1.6.0" resolved "https://registry.yarnpkg.com/@colors/colors/-/colors-1.6.0.tgz#ec6cd237440700bc23ca23087f513c75508958b0" integrity sha512-Ir+AOibqzrIsL6ajt3Rz3LskB7OiMVHqltZmspbW/TJuTVuyOMirVqAkjfY6JISiLHgyNqicAC8AyHHGzNd/dA== "@dabh/diagnostics@^2.0.8": version "2.0.8" resolved "https://registry.yarnpkg.com/@dabh/diagnostics/-/diagnostics-2.0.8.tgz#ead97e72ca312cf0e6dd7af0d300b58993a31a5e" integrity sha512-R4MSXTVnuMzGD7bzHdW2ZhhdPC/igELENcq5IjEverBvq5hn1SXCWcsi6eSsdWP0/Ur+SItRRjAktmdoX/8R/Q== dependencies: "@so-ric/colorspace" "^1.1.6" enabled "2.0.x" kuler "^2.0.0" "@emnapi/runtime@^1.2.0": version "1.7.1" resolved "https://registry.yarnpkg.com/@emnapi/runtime/-/runtime-1.7.1.tgz#a73784e23f5d57287369c808197288b52276b791" integrity sha512-PVtJr5CmLwYAU9PZDMITZoR5iAOShYREoR45EyyLrbntV50mdePTgUn4AmOw90Ifcj+x2kRjdzr1HP3RrNiHGA== dependencies: tslib "^2.4.0" "@eslint-community/eslint-utils@^4.8.0": version "4.9.1" resolved "https://registry.yarnpkg.com/@eslint-community/eslint-utils/-/eslint-utils-4.9.1.tgz#4e90af67bc51ddee6cdef5284edf572ec376b595" integrity sha512-phrYmNiYppR7znFEdqgfWHXR6NCkZEK7hwWDHZUjit/2/U0r6XvkDl0SYnoM51Hq7FhCGdLDT6zxCCOY1hexsQ== dependencies: eslint-visitor-keys "^3.4.3" "@eslint-community/regexpp@^4.12.1": version "4.12.2" resolved "https://registry.yarnpkg.com/@eslint-community/regexpp/-/regexpp-4.12.2.tgz#bccdf615bcf7b6e8db830ec0b8d21c9a25de597b" integrity sha512-EriSTlt5OC9/7SXkRSCAhfSxxoSUgBm33OH+IkwbdpgoqsSsUg7y3uh+IICI/Qg4BBWr3U2i39RpmycbxMq4ew== "@eslint/config-array@^0.21.1": version "0.21.1" resolved "https://registry.yarnpkg.com/@eslint/config-array/-/config-array-0.21.1.tgz#7d1b0060fea407f8301e932492ba8c18aff29713" integrity sha512-aw1gNayWpdI/jSYVgzN5pL0cfzU02GT3NBpeT/DXbx1/1x7ZKxFPd9bwrzygx/qiwIQiJ1sw/zD8qY/kRvlGHA== dependencies: "@eslint/object-schema" "^2.1.7" debug "^4.3.1" minimatch "^3.1.2" "@eslint/config-helpers@^0.4.2": version "0.4.2" resolved "https://registry.yarnpkg.com/@eslint/config-helpers/-/config-helpers-0.4.2.tgz#1bd006ceeb7e2e55b2b773ab318d300e1a66aeda" integrity sha512-gBrxN88gOIf3R7ja5K9slwNayVcZgK6SOUORm2uBzTeIEfeVaIhOpCtTox3P6R7o2jLFwLFTLnC7kU/RGcYEgw== dependencies: "@eslint/core" "^0.17.0" "@eslint/core@^0.17.0": version "0.17.0" resolved "https://registry.yarnpkg.com/@eslint/core/-/core-0.17.0.tgz#77225820413d9617509da9342190a2019e78761c" integrity sha512-yL/sLrpmtDaFEiUj1osRP4TI2MDz1AddJL+jZ7KSqvBuliN4xqYY54IfdN8qD8Toa6g1iloph1fxQNkjOxrrpQ== dependencies: "@types/json-schema" "^7.0.15" "@eslint/eslintrc@^3.3.1": version "3.3.4" resolved "https://registry.yarnpkg.com/@eslint/eslintrc/-/eslintrc-3.3.4.tgz#e402b1920f7c1f5a15342caa432b1348cacbb641" integrity sha512-4h4MVF8pmBsncB60r0wSJiIeUKTSD4m7FmTFThG8RHlsg9ajqckLm9OraguFGZE4vVdpiI1Q4+hFnisopmG6gQ== dependencies: ajv "^6.14.0" debug "^4.3.2" espree "^10.0.1" globals "^14.0.0" ignore "^5.2.0" import-fresh "^3.2.1" js-yaml "^4.1.1" minimatch "^3.1.3" strip-json-comments "^3.1.1" "@eslint/js@9.39.3", "@eslint/js@^9.0.0": version "9.39.3" resolved "https://registry.yarnpkg.com/@eslint/js/-/js-9.39.3.tgz#c6168736c7e0c43ead49654ed06a4bcb3833363d" integrity sha512-1B1VkCq6FuUNlQvlBYb+1jDu/gV297TIs/OeiaSR9l1H27SVW55ONE1e1Vp16NqP683+xEGzxYtv4XCiDPaQiw== "@eslint/object-schema@^2.1.7": version "2.1.7" resolved "https://registry.yarnpkg.com/@eslint/object-schema/-/object-schema-2.1.7.tgz#6e2126a1347e86a4dedf8706ec67ff8e107ebbad" integrity sha512-VtAOaymWVfZcmZbp6E2mympDIHvyjXs/12LqWYjVw6qjrfF+VK+fyG33kChz3nnK+SU5/NeHOqrTEHS8sXO3OA== "@eslint/plugin-kit@^0.4.1": version "0.4.1" resolved "https://registry.yarnpkg.com/@eslint/plugin-kit/-/plugin-kit-0.4.1.tgz#9779e3fd9b7ee33571a57435cf4335a1794a6cb2" integrity sha512-43/qtrDUokr7LJqoF2c3+RInu/t4zfrpYdoSDfYyhg52rwLV6TnOvdG4fXm7IkSB3wErkcmJS9iEhjVtOSEjjA== dependencies: "@eslint/core" "^0.17.0" levn "^0.4.1" "@fastify/busboy@^2.0.0": version "2.1.1" resolved "https://registry.yarnpkg.com/@fastify/busboy/-/busboy-2.1.1.tgz#b9da6a878a371829a0502c9b6c1c143ef6663f4d" integrity sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA== "@huggingface/jinja@^0.2.2": version "0.2.2" resolved "https://registry.yarnpkg.com/@huggingface/jinja/-/jinja-0.2.2.tgz#faeb205a9d6995089bef52655ddd8245d3190627" integrity sha512-/KPde26khDUIPkTGU82jdtTW9UAuvUTumCAbFs/7giR0SxsvZC4hru51PBvpijH6BVkHcROcvZM/lpy5h1jRRA== "@humanfs/core@^0.19.1": version "0.19.1" resolved "https://registry.yarnpkg.com/@humanfs/core/-/core-0.19.1.tgz#17c55ca7d426733fe3c561906b8173c336b40a77" integrity sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA== "@humanfs/node@^0.16.6": version "0.16.7" resolved "https://registry.yarnpkg.com/@humanfs/node/-/node-0.16.7.tgz#822cb7b3a12c5a240a24f621b5a2413e27a45f26" integrity sha512-/zUx+yOsIrG4Y43Eh2peDeKCxlRt/gET6aHfaKpuq267qXdYDFViVHfMaLyygZOnl0kGWxFIgsBy8QFuTLUXEQ== dependencies: "@humanfs/core" "^0.19.1" "@humanwhocodes/retry" "^0.4.0" "@humanwhocodes/module-importer@^1.0.1": version "1.0.1" resolved "https://registry.yarnpkg.com/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz#af5b2691a22b44be847b0ca81641c5fb6ad0172c" integrity sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA== "@humanwhocodes/retry@^0.4.0", "@humanwhocodes/retry@^0.4.2": version "0.4.3" resolved "https://registry.yarnpkg.com/@humanwhocodes/retry/-/retry-0.4.3.tgz#c2b9d2e374ee62c586d3adbea87199b1d7a7a6ba" integrity sha512-bV0Tgo9K4hfPCek+aMAn81RppFKv2ySDQeMoSZuvTASywNTnVJCArCZE2FWqpvIatKu7VMRLWlR1EazvVhDyhQ== "@img/sharp-darwin-arm64@0.33.5": version "0.33.5" resolved "https://registry.yarnpkg.com/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.33.5.tgz#ef5b5a07862805f1e8145a377c8ba6e98813ca08" integrity sha512-UT4p+iz/2H4twwAoLCqfA9UH5pI6DggwKEGuaPy7nCVQ8ZsiY5PIcrRvD1DzuY3qYL07NtIQcWnBSY/heikIFQ== optionalDependencies: "@img/sharp-libvips-darwin-arm64" "1.0.4" "@img/sharp-darwin-x64@0.33.5": version "0.33.5" resolved "https://registry.yarnpkg.com/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.33.5.tgz#e03d3451cd9e664faa72948cc70a403ea4063d61" integrity sha512-fyHac4jIc1ANYGRDxtiqelIbdWkIuQaI84Mv45KvGRRxSAa7o7d1ZKAOBaYbnepLC1WqxfpimdeWfvqqSGwR2Q== optionalDependencies: "@img/sharp-libvips-darwin-x64" "1.0.4" "@img/sharp-libvips-darwin-arm64@1.0.4": version "1.0.4" resolved "https://registry.yarnpkg.com/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.0.4.tgz#447c5026700c01a993c7804eb8af5f6e9868c07f" integrity sha512-XblONe153h0O2zuFfTAbQYAX2JhYmDHeWikp1LM9Hul9gVPjFY427k6dFEcOL72O01QxQsWi761svJ/ev9xEDg== "@img/sharp-libvips-darwin-x64@1.0.4": version "1.0.4" resolved "https://registry.yarnpkg.com/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.0.4.tgz#e0456f8f7c623f9dbfbdc77383caa72281d86062" integrity sha512-xnGR8YuZYfJGmWPvmlunFaWJsb9T/AO2ykoP3Fz/0X5XV2aoYBPkX6xqCQvUTKKiLddarLaxpzNe+b1hjeWHAQ== "@img/sharp-libvips-linux-arm64@1.0.4": version "1.0.4" resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.0.4.tgz#979b1c66c9a91f7ff2893556ef267f90ebe51704" integrity sha512-9B+taZ8DlyyqzZQnoeIvDVR/2F4EbMepXMc/NdVbkzsJbzkUjhXv/70GQJ7tdLA4YJgNP25zukcxpX2/SueNrA== "@img/sharp-libvips-linux-arm@1.0.5": version "1.0.5" resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.0.5.tgz#99f922d4e15216ec205dcb6891b721bfd2884197" integrity sha512-gvcC4ACAOPRNATg/ov8/MnbxFDJqf/pDePbBnuBDcjsI8PssmjoKMAz4LtLaVi+OnSb5FK/yIOamqDwGmXW32g== "@img/sharp-libvips-linux-s390x@1.0.4": version "1.0.4" resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.0.4.tgz#f8a5eb1f374a082f72b3f45e2fb25b8118a8a5ce" integrity sha512-u7Wz6ntiSSgGSGcjZ55im6uvTrOxSIS8/dgoVMoiGE9I6JAfU50yH5BoDlYA1tcuGS7g/QNtetJnxA6QEsCVTA== "@img/sharp-libvips-linux-x64@1.0.4": version "1.0.4" resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.0.4.tgz#d4c4619cdd157774906e15770ee119931c7ef5e0" integrity sha512-MmWmQ3iPFZr0Iev+BAgVMb3ZyC4KeFc3jFxnNbEPas60e1cIfevbtuyf9nDGIzOaW9PdnDciJm+wFFaTlj5xYw== "@img/sharp-libvips-linuxmusl-arm64@1.0.4": version "1.0.4" resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.0.4.tgz#166778da0f48dd2bded1fa3033cee6b588f0d5d5" integrity sha512-9Ti+BbTYDcsbp4wfYib8Ctm1ilkugkA/uscUn6UXK1ldpC1JjiXbLfFZtRlBhjPZ5o1NCLiDbg8fhUPKStHoTA== "@img/sharp-libvips-linuxmusl-x64@1.0.4": version "1.0.4" resolved "https://registry.yarnpkg.com/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.0.4.tgz#93794e4d7720b077fcad3e02982f2f1c246751ff" integrity sha512-viYN1KX9m+/hGkJtvYYp+CCLgnJXwiQB39damAO7WMdKWlIhmYTfHjwSbQeUK/20vY154mwezd9HflVFM1wVSw== "@img/sharp-linux-arm64@0.33.5": version "0.33.5" resolved "https://registry.yarnpkg.com/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.33.5.tgz#edb0697e7a8279c9fc829a60fc35644c4839bb22" integrity sha512-JMVv+AMRyGOHtO1RFBiJy/MBsgz0x4AWrT6QoEVVTyh1E39TrCUpTRI7mx9VksGX4awWASxqCYLCV4wBZHAYxA== optionalDependencies: "@img/sharp-libvips-linux-arm64" "1.0.4" "@img/sharp-linux-arm@0.33.5": version "0.33.5" resolved "https://registry.yarnpkg.com/@img/sharp-linux-arm/-/sharp-linux-arm-0.33.5.tgz#422c1a352e7b5832842577dc51602bcd5b6f5eff" integrity sha512-JTS1eldqZbJxjvKaAkxhZmBqPRGmxgu+qFKSInv8moZ2AmT5Yib3EQ1c6gp493HvrvV8QgdOXdyaIBrhvFhBMQ== optionalDependencies: "@img/sharp-libvips-linux-arm" "1.0.5" "@img/sharp-linux-s390x@0.33.5": version "0.33.5" resolved "https://registry.yarnpkg.com/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.33.5.tgz#f5c077926b48e97e4a04d004dfaf175972059667" integrity sha512-y/5PCd+mP4CA/sPDKl2961b+C9d+vPAveS33s6Z3zfASk2j5upL6fXVPZi7ztePZ5CuH+1kW8JtvxgbuXHRa4Q== optionalDependencies: "@img/sharp-libvips-linux-s390x" "1.0.4" "@img/sharp-linux-x64@0.33.5": version "0.33.5" resolved "https://registry.yarnpkg.com/@img/sharp-linux-x64/-/sharp-linux-x64-0.33.5.tgz#d806e0afd71ae6775cc87f0da8f2d03a7c2209cb" integrity sha512-opC+Ok5pRNAzuvq1AG0ar+1owsu842/Ab+4qvU879ippJBHvyY5n2mxF1izXqkPYlGuP/M556uh53jRLJmzTWA== optionalDependencies: "@img/sharp-libvips-linux-x64" "1.0.4" "@img/sharp-linuxmusl-arm64@0.33.5": version "0.33.5" resolved "https://registry.yarnpkg.com/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.33.5.tgz#252975b915894fb315af5deea174651e208d3d6b" integrity sha512-XrHMZwGQGvJg2V/oRSUfSAfjfPxO+4DkiRh6p2AFjLQztWUuY/o8Mq0eMQVIY7HJ1CDQUJlxGGZRw1a5bqmd1g== optionalDependencies: "@img/sharp-libvips-linuxmusl-arm64" "1.0.4" "@img/sharp-linuxmusl-x64@0.33.5": version "0.33.5" resolved "https://registry.yarnpkg.com/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.33.5.tgz#3f4609ac5d8ef8ec7dadee80b560961a60fd4f48" integrity sha512-WT+d/cgqKkkKySYmqoZ8y3pxx7lx9vVejxW/W4DOFMYVSkErR+w7mf2u8m/y4+xHe7yY9DAXQMWQhpnMuFfScw== optionalDependencies: "@img/sharp-libvips-linuxmusl-x64" "1.0.4" "@img/sharp-wasm32@0.33.5": version "0.33.5" resolved "https://registry.yarnpkg.com/@img/sharp-wasm32/-/sharp-wasm32-0.33.5.tgz#6f44f3283069d935bb5ca5813153572f3e6f61a1" integrity sha512-ykUW4LVGaMcU9lu9thv85CbRMAwfeadCJHRsg2GmeRa/cJxsVY9Rbd57JcMxBkKHag5U/x7TSBpScF4U8ElVzg== dependencies: "@emnapi/runtime" "^1.2.0" "@img/sharp-win32-ia32@0.33.5": version "0.33.5" resolved "https://registry.yarnpkg.com/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.33.5.tgz#1a0c839a40c5351e9885628c85f2e5dfd02b52a9" integrity sha512-T36PblLaTwuVJ/zw/LaH0PdZkRz5rd3SmMHX8GSmR7vtNSP5Z6bQkExdSK7xGWyxLw4sUknBuugTelgw2faBbQ== "@img/sharp-win32-x64@0.33.5": version "0.33.5" resolved "https://registry.yarnpkg.com/@img/sharp-win32-x64/-/sharp-win32-x64-0.33.5.tgz#56f00962ff0c4e0eb93d34a047d29fa995e3e342" integrity sha512-MpY/o8/8kj+EcnxwvrP4aTJSWw/aZ7JIGR4aBeZkZw5B7/Jn+tY9/VNwtcoGmdT7GfggGIU4kygOMSbYnOrAbg== "@isaacs/cliui@^8.0.2": version "8.0.2" resolved "https://registry.yarnpkg.com/@isaacs/cliui/-/cliui-8.0.2.tgz#b37667b7bc181c168782259bab42474fbf52b550" integrity sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA== dependencies: string-width "^5.1.2" string-width-cjs "npm:string-width@^4.2.0" strip-ansi "^7.0.1" strip-ansi-cjs "npm:strip-ansi@^6.0.1" wrap-ansi "^8.1.0" wrap-ansi-cjs "npm:wrap-ansi@^7.0.0" "@langchain/community@^0.2.23": version "0.2.33" resolved "https://registry.yarnpkg.com/@langchain/community/-/community-0.2.33.tgz#c3c920492470196d0f560552cc95954fccdde0e8" integrity sha512-YsytROnBYoPqtUcV2In+afyLACzxwFrYRn7EBKYL7XWl3XNwrT85U1+nLM5b+MOjXvg9YfJSjrs1Tlbfy4st8g== dependencies: "@langchain/core" ">=0.2.21 <0.3.0" "@langchain/openai" ">=0.2.0 <0.3.0" binary-extensions "^2.2.0" expr-eval "^2.0.2" flat "^5.0.2" js-yaml "^4.1.0" langchain "~0.2.3" langsmith "~0.1.30" uuid "^10.0.0" zod "^3.22.3" zod-to-json-schema "^3.22.5" "@langchain/community@~0.0.47": version "0.0.57" resolved "https://registry.yarnpkg.com/@langchain/community/-/community-0.0.57.tgz#9d77c5acb74a4a8ec01d2cefb71dcd4088701c44" integrity sha512-tib4UJNkyA4TPNsTNChiBtZmThVJBr7X/iooSmKeCr+yUEha2Yxly3A4OAO95Vlpj4Q+od8HAfCbZih/1XqAMw== dependencies: "@langchain/core" "~0.1.60" "@langchain/openai" "~0.0.28" expr-eval "^2.0.2" flat "^5.0.2" langsmith "~0.1.1" uuid "^9.0.0" zod "^3.22.3" zod-to-json-schema "^3.22.5" "@langchain/core@>0.1.56 <0.3.0", "@langchain/core@>0.2.0 <0.3.0", "@langchain/core@>=0.2.21 <0.3.0", "@langchain/core@>=0.2.26 <0.3.0": version "0.2.36" resolved "https://registry.yarnpkg.com/@langchain/core/-/core-0.2.36.tgz#75754c33aa5b9310dcf117047374a1ae011005a4" integrity sha512-qHLvScqERDeH7y2cLuJaSAlMwg3f/3Oc9nayRSXRU2UuaK/SOhI42cxiPLj1FnuHJSmN0rBQFkrLx02gI4mcVg== dependencies: ansi-styles "^5.0.0" camelcase "6" decamelize "1.2.0" js-tiktoken "^1.0.12" langsmith "^0.1.56-rc.1" mustache "^4.2.0" p-queue "^6.6.2" p-retry "4" uuid "^10.0.0" zod "^3.22.4" zod-to-json-schema "^3.22.3" "@langchain/core@~0.1.60": version "0.1.63" resolved "https://registry.yarnpkg.com/@langchain/core/-/core-0.1.63.tgz#33cc48877739e9fdb5885fbd4b16fd08d1597050" integrity sha512-+fjyYi8wy6x1P+Ee1RWfIIEyxd9Ee9jksEwvrggPwwI/p45kIDTdYTblXsM13y4mNWTiACyLSdbwnPaxxdoz+w== dependencies: ansi-styles "^5.0.0" camelcase "6" decamelize "1.2.0" js-tiktoken "^1.0.12" langsmith "~0.1.7" ml-distance "^4.0.0" mustache "^4.2.0" p-queue "^6.6.2" p-retry "4" uuid "^9.0.0" zod "^3.22.4" zod-to-json-schema "^3.22.3" "@langchain/openai@>=0.1.0 <0.3.0", "@langchain/openai@>=0.2.0 <0.3.0": version "0.2.11" resolved "https://registry.yarnpkg.com/@langchain/openai/-/openai-0.2.11.tgz#b1a0403eb5db8133bb4ff41fe0680e727b78ddfc" integrity sha512-Pu8+WfJojCgSf0bAsXb4AjqvcDyAWyoEB1AoCRNACgEnBWZuitz3hLwCo9I+6hAbeg3QJ37g82yKcmvKAg1feg== dependencies: "@langchain/core" ">=0.2.26 <0.3.0" js-tiktoken "^1.0.12" openai "^4.57.3" zod "^3.22.4" zod-to-json-schema "^3.22.3" "@langchain/openai@~0.0.28": version "0.0.34" resolved "https://registry.yarnpkg.com/@langchain/openai/-/openai-0.0.34.tgz#36c9bca0721ab9f7e5d40927e7c0429cacbd5b56" integrity sha512-M+CW4oXle5fdoz2T2SwdOef8pl3/1XmUx1vjn2mXUVM/128aO0l23FMF0SNBsAbRV6P+p/TuzjodchJbi0Ht/A== dependencies: "@langchain/core" ">0.1.56 <0.3.0" js-tiktoken "^1.0.12" openai "^4.41.1" zod "^3.22.4" zod-to-json-schema "^3.22.3" "@langchain/textsplitters@~0.0.0": version "0.0.3" resolved "https://registry.yarnpkg.com/@langchain/textsplitters/-/textsplitters-0.0.3.tgz#1a3cc93dd2ab330edb225400ded190a22fea14e3" integrity sha512-cXWgKE3sdWLSqAa8ykbCcUsUF1Kyr5J3HOWYGuobhPEycXW4WI++d5DhzdpL238mzoEXTi90VqfSCra37l5YqA== dependencies: "@langchain/core" ">0.2.0 <0.3.0" js-tiktoken "^1.0.12" "@pkgjs/parseargs@^0.11.0": version "0.11.0" resolved "https://registry.yarnpkg.com/@pkgjs/parseargs/-/parseargs-0.11.0.tgz#a77ea742fab25775145434eb1d2328cf5013ac33" integrity sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg== "@pkgr/core@^0.2.9": version "0.2.9" resolved "https://registry.yarnpkg.com/@pkgr/core/-/core-0.2.9.tgz#d229a7b7f9dac167a156992ef23c7f023653f53b" integrity sha512-QNqXyfVS2wm9hweSYD2O7F0G06uurj9kZ96TRQE5Y9hU7+tgdZwIkbAKc5Ocy1HxEY2kuDQa6cQ1WRs/O5LFKA== "@protobufjs/aspromise@^1.1.1", "@protobufjs/aspromise@^1.1.2": version "1.1.2" resolved "https://registry.yarnpkg.com/@protobufjs/aspromise/-/aspromise-1.1.2.tgz#9b8b0cc663d669a7d8f6f5d0893a14d348f30fbf" integrity sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ== "@protobufjs/base64@^1.1.2": version "1.1.2" resolved "https://registry.yarnpkg.com/@protobufjs/base64/-/base64-1.1.2.tgz#4c85730e59b9a1f1f349047dbf24296034bb2735" integrity sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg== "@protobufjs/codegen@^2.0.4": version "2.0.4" resolved "https://registry.yarnpkg.com/@protobufjs/codegen/-/codegen-2.0.4.tgz#7ef37f0d010fb028ad1ad59722e506d9262815cb" integrity sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg== "@protobufjs/eventemitter@^1.1.0": version "1.1.0" resolved "https://registry.yarnpkg.com/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz#355cbc98bafad5978f9ed095f397621f1d066b70" integrity sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q== "@protobufjs/fetch@^1.1.0": version "1.1.0" resolved "https://registry.yarnpkg.com/@protobufjs/fetch/-/fetch-1.1.0.tgz#ba99fb598614af65700c1619ff06d454b0d84c45" integrity sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ== dependencies: "@protobufjs/aspromise" "^1.1.1" "@protobufjs/inquire" "^1.1.0" "@protobufjs/float@^1.0.2": version "1.0.2" resolved "https://registry.yarnpkg.com/@protobufjs/float/-/float-1.0.2.tgz#5e9e1abdcb73fc0a7cb8b291df78c8cbd97b87d1" integrity sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ== "@protobufjs/inquire@^1.1.0": version "1.1.0" resolved "https://registry.yarnpkg.com/@protobufjs/inquire/-/inquire-1.1.0.tgz#ff200e3e7cf2429e2dcafc1140828e8cc638f089" integrity sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q== "@protobufjs/path@^1.1.2": version "1.1.2" resolved "https://registry.yarnpkg.com/@protobufjs/path/-/path-1.1.2.tgz#6cc2b20c5c9ad6ad0dccfd21ca7673d8d7fbf68d" integrity sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA== "@protobufjs/pool@^1.1.0": version "1.1.0" resolved "https://registry.yarnpkg.com/@protobufjs/pool/-/pool-1.1.0.tgz#09fd15f2d6d3abfa9b65bc366506d6ad7846ff54" integrity sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw== "@protobufjs/utf8@^1.1.0": version "1.1.0" resolved "https://registry.yarnpkg.com/@protobufjs/utf8/-/utf8-1.1.0.tgz#a777360b5b39a1a2e5106f8e858f2fd2d060c570" integrity sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw== "@puppeteer/browsers@1.8.0": version "1.8.0" resolved "https://registry.yarnpkg.com/@puppeteer/browsers/-/browsers-1.8.0.tgz#fb6ee61de15e7f0e67737aea9f9bab1512dbd7d8" integrity sha512-TkRHIV6k2D8OlUe8RtG+5jgOF/H98Myx0M6AOafC8DdNVOFiBSFa5cpRDtpm8LXOa9sVwe0+e6Q3FC56X/DZfg== dependencies: debug "4.3.4" extract-zip "2.0.1" progress "2.0.3" proxy-agent "6.3.1" tar-fs "3.0.4" unbzip2-stream "1.4.3" yargs "17.7.2" "@selderee/plugin-htmlparser2@^0.11.0": version "0.11.0" resolved "https://registry.yarnpkg.com/@selderee/plugin-htmlparser2/-/plugin-htmlparser2-0.11.0.tgz#d5b5e29a7ba6d3958a1972c7be16f4b2c188c517" integrity sha512-P33hHGdldxGabLFjPPpaTxVolMrzrcegejx+0GxjrIb9Zv48D8yAIA/QTDR2dFl7Uz7urX8aX6+5bCZslr+gWQ== dependencies: domhandler "^5.0.3" selderee "^0.11.0" "@so-ric/colorspace@^1.1.6": version "1.1.6" resolved "https://registry.yarnpkg.com/@so-ric/colorspace/-/colorspace-1.1.6.tgz#62515d8b9f27746b76950a83bde1af812d91923b" integrity sha512-/KiKkpHNOBgkFJwu9sh48LkHSMYGyuTcSFK/qMBdnOAlrRJzRSXAOFB5qwzaVQuDl8wAvHVMkaASQDReTahxuw== dependencies: color "^5.0.2" text-hex "1.0.x" "@tokenizer/token@^0.3.0": version "0.3.0" resolved "https://registry.yarnpkg.com/@tokenizer/token/-/token-0.3.0.tgz#fe98a93fe789247e998c75e74e9c7c63217aa276" integrity sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A== "@tootallnate/quickjs-emscripten@^0.23.0": version "0.23.0" resolved "https://registry.yarnpkg.com/@tootallnate/quickjs-emscripten/-/quickjs-emscripten-0.23.0.tgz#db4ecfd499a9765ab24002c3b696d02e6d32a12c" integrity sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA== "@types/estree@^1.0.6": version "1.0.8" resolved "https://registry.yarnpkg.com/@types/estree/-/estree-1.0.8.tgz#958b91c991b1867ced318bedea0e215ee050726e" integrity sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w== "@types/json-schema@^7.0.15": version "7.0.15" resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.15.tgz#596a1747233694d50f6ad8a7869fcb6f56cf5841" integrity sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA== "@types/long@^4.0.1": version "4.0.2" resolved "https://registry.yarnpkg.com/@types/long/-/long-4.0.2.tgz#b74129719fc8d11c01868010082d483b7545591a" integrity sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA== "@types/mailparser@^3.0.0": version "3.4.6" resolved "https://registry.yarnpkg.com/@types/mailparser/-/mailparser-3.4.6.tgz#fcca99fe9f919f3da691a0bf5e3022c6283c3068" integrity sha512-wVV3cnIKzxTffaPH8iRnddX1zahbYB1ZEoAxyhoBo3TBCBuK6nZ8M8JYO/RhsCuuBVOw/DEN/t/ENbruwlxn6Q== dependencies: "@types/node" "*" iconv-lite "^0.6.3" "@types/node-fetch@^2.6.4": version "2.6.13" resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.13.tgz#e0c9b7b5edbdb1b50ce32c127e85e880872d56ee" integrity sha512-QGpRVpzSaUs30JBSGPjOg4Uveu384erbHBoT1zeONvyCfwQxIkUshLAOqN/k9EjGviPRmWTTe6aH2qySWKTVSw== dependencies: "@types/node" "*" form-data "^4.0.4" "@types/node@*", "@types/node@>=13.7.0": version "24.10.1" resolved "https://registry.yarnpkg.com/@types/node/-/node-24.10.1.tgz#91e92182c93db8bd6224fca031e2370cef9a8f01" integrity sha512-GNWcUTRBgIRJD5zj+Tq0fKOJ5XZajIiBroOF0yvj2bSU1WvNdYS/dn9UxwsujGW4JX06dnHyjV2y9rRaybH0iQ== dependencies: undici-types "~7.16.0" "@types/node@^18.11.18": version "18.19.130" resolved "https://registry.yarnpkg.com/@types/node/-/node-18.19.130.tgz#da4c6324793a79defb7a62cba3947ec5add00d59" integrity sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg== dependencies: undici-types "~5.26.4" "@types/retry@0.12.0": version "0.12.0" resolved "https://registry.yarnpkg.com/@types/retry/-/retry-0.12.0.tgz#2b35eccfcee7d38cd72ad99232fbd58bffb3c84d" integrity sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA== "@types/triple-beam@^1.3.2": version "1.3.5" resolved "https://registry.yarnpkg.com/@types/triple-beam/-/triple-beam-1.3.5.tgz#74fef9ffbaa198eb8b588be029f38b00299caa2c" integrity sha512-6WaYesThRMCl19iryMYP7/x2OVgCtbIVflDGFpWnb9irXI3UjYE4AzmYuiUKY1AJstGijoY+MgUszMgRxIYTYw== "@types/uuid@^10.0.0": version "10.0.0" resolved "https://registry.yarnpkg.com/@types/uuid/-/uuid-10.0.0.tgz#e9c07fe50da0f53dc24970cca94d619ff03f6f6d" integrity sha512-7gqG38EyHgyP1S+7+xomFtL+ZNHcKv6DwNaCZmJmo1vgMugyF3TCnXVg4t1uk89mLNwnLtnY3TpOpCOyp1/xHQ== "@types/yauzl@^2.9.1": version "2.10.3" resolved "https://registry.yarnpkg.com/@types/yauzl/-/yauzl-2.10.3.tgz#e9b2808b4f109504a03cda958259876f61017999" integrity sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q== dependencies: "@types/node" "*" "@xenova/transformers@^2.14.0": version "2.17.2" resolved "https://registry.yarnpkg.com/@xenova/transformers/-/transformers-2.17.2.tgz#7448d73b90f67bced66f39fe2dd656adc891fde5" integrity sha512-lZmHqzrVIkSvZdKZEx7IYY51TK0WDrC8eR0c5IMnBsO8di8are1zzw8BlLhyO2TklZKLN5UffNGs1IJwT6oOqQ== dependencies: "@huggingface/jinja" "^0.2.2" onnxruntime-web "1.14.0" sharp "^0.32.0" optionalDependencies: onnxruntime-node "1.14.0" "@xmldom/xmldom@^0.8.10", "@xmldom/xmldom@^0.8.6": version "0.8.11" resolved "https://registry.yarnpkg.com/@xmldom/xmldom/-/xmldom-0.8.11.tgz#b79de2d67389734c57c52595f7a7305e30c2d608" integrity sha512-cQzWCtO6C8TQiYl1ruKNn2U6Ao4o4WBBcbL61yJl84x+j5sOWWFU9X7DpND8XZG3daDppSsigMdfAIl2upQBRw== "@zone-eu/mailsplit@5.4.7": version "5.4.7" resolved "https://registry.yarnpkg.com/@zone-eu/mailsplit/-/mailsplit-5.4.7.tgz#ad86fe08222883418f33cf02d57025de02d2eb38" integrity sha512-jApX86aDgolMz08pP20/J2zcns02NSK3zSiYouf01QQg4250L+GUAWSWicmS7eRvs+Z7wP7QfXrnkaTBGrIpwQ== dependencies: libbase64 "1.3.0" libmime "5.3.7" libqp "2.1.1" abort-controller@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/abort-controller/-/abort-controller-3.0.0.tgz#eaf54d53b62bae4138e809ca225c8439a6efb392" integrity sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg== dependencies: event-target-shim "^5.0.0" accepts@~1.3.8: version "1.3.8" resolved "https://registry.yarnpkg.com/accepts/-/accepts-1.3.8.tgz#0bf0be125b67014adcb0b0921e62db7bffe16b2e" integrity sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw== dependencies: mime-types "~2.1.34" negotiator "0.6.3" acorn-jsx@^5.3.2: version "5.3.2" resolved "https://registry.yarnpkg.com/acorn-jsx/-/acorn-jsx-5.3.2.tgz#7ed5bb55908b3b2f1bc55c6af1653bada7f07937" integrity sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ== acorn@^8.15.0: version "8.16.0" resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.16.0.tgz#4ce79c89be40afe7afe8f3adb902a1f1ce9ac08a" integrity sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw== acorn@^8.8.0: version "8.15.0" resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.15.0.tgz#a360898bc415edaac46c8241f6383975b930b816" integrity sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg== adm-zip@^0.5.10: version "0.5.16" resolved "https://registry.yarnpkg.com/adm-zip/-/adm-zip-0.5.16.tgz#0b5e4c779f07dedea5805cdccb1147071d94a909" integrity sha512-TGw5yVi4saajsSEgz25grObGHEUaDrniwvA2qwSC060KfqGPdglhvPMA2lPIoxs3PQIItj2iag35fONcQqgUaQ== agent-base@^7.0.2, agent-base@^7.1.0, agent-base@^7.1.2: version "7.1.4" resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-7.1.4.tgz#e3cd76d4c548ee895d3c3fd8dc1f6c5b9032e7a8" integrity sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ== agentkeepalive@^4.2.1: version "4.6.0" resolved "https://registry.yarnpkg.com/agentkeepalive/-/agentkeepalive-4.6.0.tgz#35f73e94b3f40bf65f105219c623ad19c136ea6a" integrity sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ== dependencies: humanize-ms "^1.2.1" ajv@^6.12.4, ajv@^6.14.0: version "6.14.0" resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.14.0.tgz#fd067713e228210636ebb08c60bd3765d6dbe73a" integrity sha512-IWrosm/yrn43eiKqkfkHis7QioDleaXQHdDVPKg0FSwwd/DuvyX79TZnFOnYpB7dcsFAMmtFztZuXPDvSePkFw== dependencies: fast-deep-equal "^3.1.1" fast-json-stable-stringify "^2.0.0" json-schema-traverse "^0.4.1" uri-js "^4.2.2" ansi-regex@^5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.1.tgz#082cb2c89c9fe8659a311a53bd6a4dc5301db304" integrity sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ== ansi-regex@^6.0.1: version "6.2.2" resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-6.2.2.tgz#60216eea464d864597ce2832000738a0589650c1" integrity sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg== ansi-styles@^4.0.0, ansi-styles@^4.1.0: version "4.3.0" resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-4.3.0.tgz#edd803628ae71c04c85ae7a0906edad34b648937" integrity sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg== dependencies: color-convert "^2.0.1" ansi-styles@^5.0.0: version "5.2.0" resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-5.2.0.tgz#07449690ad45777d1924ac2abb2fc8895dba836b" integrity sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA== anymatch@~3.1.2: version "3.1.3" resolved "https://registry.yarnpkg.com/anymatch/-/anymatch-3.1.3.tgz#790c58b19ba1720a84205b57c618d5ad8524973e" integrity sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw== dependencies: normalize-path "^3.0.0" picomatch "^2.0.4" argparse@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/argparse/-/argparse-2.0.1.tgz#246f50f3ca78a3240f6c997e8a9bd1eac49e4b38" integrity sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q== argparse@~1.0.3: version "1.0.10" resolved "https://registry.yarnpkg.com/argparse/-/argparse-1.0.10.tgz#bcd6791ea5ae09725e17e5ad988134cd40b3d911" integrity sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg== dependencies: sprintf-js "~1.0.2" array-flatten@1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/array-flatten/-/array-flatten-1.1.1.tgz#9a5f699051b1e7073328f2a008968b64ea2955d2" integrity sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg== array-hyper-unique@^2.1.4: version "2.1.6" resolved "https://registry.yarnpkg.com/array-hyper-unique/-/array-hyper-unique-2.1.6.tgz#429412fd63b7bd7c920f6cdbf60d1dd292855b2e" integrity sha512-BdlHRqjKSYs88WFaVNVEc6Kv8ln/FdzCKPbcDPuWs4/EXkQFhnjc8TyR7hnPxRjcjo5LKOhUMGUWpAqRgeJvpA== dependencies: deep-eql "= 4.0.0" lodash "^4.17.21" ast-types@^0.13.4: version "0.13.4" resolved "https://registry.yarnpkg.com/ast-types/-/ast-types-0.13.4.tgz#ee0d77b343263965ecc3fb62da16e7222b2b6782" integrity sha512-x1FCFnFifvYDDzTaLII71vG5uvDwgtmDTEVWAxrgeiR8VjMONcCXJx7E+USjDtHlwFmt9MysbqgF9b9Vjr6w+w== dependencies: tslib "^2.0.1" async@^3.2.3: version "3.2.6" resolved "https://registry.yarnpkg.com/async/-/async-3.2.6.tgz#1b0728e14929d51b85b449b7f06e27c1145e38ce" integrity sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA== asynckit@^0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" integrity sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q== available-typed-arrays@^1.0.7: version "1.0.7" resolved "https://registry.yarnpkg.com/available-typed-arrays/-/available-typed-arrays-1.0.7.tgz#a5cc375d6a03c2efc87a553f3e0b1522def14846" integrity sha512-wvUjBtSGN7+7SjNpq/9M2Tg350UZD3q62IFZLbRAR1bSMlCo1ZaeW+BJ+D090e4hIIZLBcTDWe4Mh4jvUDajzQ== dependencies: possible-typed-array-names "^1.0.0" b4a@^1.6.4: version "1.7.3" resolved "https://registry.yarnpkg.com/b4a/-/b4a-1.7.3.tgz#24cf7ccda28f5465b66aec2bac69e32809bf112f" integrity sha512-5Q2mfq2WfGuFp3uS//0s6baOJLMoVduPYVeNmDYxu5OUA1/cBfvr2RIS7vi62LdNj/urk1hfmj867I3qt6uZ7Q== balanced-match@^1.0.0: version "1.0.2" resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee" integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw== bare-events@^2.5.4, bare-events@^2.7.0: version "2.8.2" resolved "https://registry.yarnpkg.com/bare-events/-/bare-events-2.8.2.tgz#7b3e10bd8e1fc80daf38bb516921678f566ab89f" integrity sha512-riJjyv1/mHLIPX4RwiK+oW9/4c3TEUeORHKefKAKnZ5kyslbN+HXowtbaVEqt4IMUB7OXlfixcs6gsFeo/jhiQ== bare-fs@^4.0.1: version "4.5.1" resolved "https://registry.yarnpkg.com/bare-fs/-/bare-fs-4.5.1.tgz#498a20a332d4a7f0b310eb89b8d2319041aa1eef" integrity sha512-zGUCsm3yv/ePt2PHNbVxjjn0nNB1MkIaR4wOCxJ2ig5pCf5cCVAYJXVhQg/3OhhJV6DB1ts7Hv0oUaElc2TPQg== dependencies: bare-events "^2.5.4" bare-path "^3.0.0" bare-stream "^2.6.4" bare-url "^2.2.2" fast-fifo "^1.3.2" bare-os@^3.0.1: version "3.6.2" resolved "https://registry.yarnpkg.com/bare-os/-/bare-os-3.6.2.tgz#b3c4f5ad5e322c0fd0f3c29fc97d19009e2796e5" integrity sha512-T+V1+1srU2qYNBmJCXZkUY5vQ0B4FSlL3QDROnKQYOqeiQR8UbjNHlPa+TIbM4cuidiN9GaTaOZgSEgsvPbh5A== bare-path@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/bare-path/-/bare-path-3.0.0.tgz#b59d18130ba52a6af9276db3e96a2e3d3ea52178" integrity sha512-tyfW2cQcB5NN8Saijrhqn0Zh7AnFNsnczRcuWODH0eYAXBsJ5gVxAUuNr7tsHSC6IZ77cA0SitzT+s47kot8Mw== dependencies: bare-os "^3.0.1" bare-stream@^2.6.4: version "2.7.0" resolved "https://registry.yarnpkg.com/bare-stream/-/bare-stream-2.7.0.tgz#5b9e7dd0a354d06e82d6460c426728536c35d789" integrity sha512-oyXQNicV1y8nc2aKffH+BUHFRXmx6VrPzlnaEvMhram0nPBrKcEdcyBg5r08D0i8VxngHFAiVyn1QKXpSG0B8A== dependencies: streamx "^2.21.0" bare-url@^2.2.2: version "2.3.2" resolved "https://registry.yarnpkg.com/bare-url/-/bare-url-2.3.2.tgz#4aef382efa662b2180a6fe4ca07a71b39bdf7ca3" integrity sha512-ZMq4gd9ngV5aTMa5p9+UfY0b3skwhHELaDkhEHetMdX0LRkW9kzaym4oo/Eh+Ghm0CCDuMTsRIGM/ytUc1ZYmw== dependencies: bare-path "^3.0.0" base-64@^0.1.0: version "0.1.0" resolved "https://registry.yarnpkg.com/base-64/-/base-64-0.1.0.tgz#780a99c84e7d600260361511c4877613bf24f6bb" integrity sha512-Y5gU45svrR5tI2Vt/X9GPd3L0HNIKzGu202EjxrXMpuc2V2CiKgemAbUUsqYmZJvPtCXoUKjNZwBJzsNScUbXA== base64-js@^1.3.1, base64-js@^1.5.1: version "1.5.1" resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA== basic-ftp@^5.0.2: version "5.0.5" resolved "https://registry.yarnpkg.com/basic-ftp/-/basic-ftp-5.0.5.tgz#14a474f5fffecca1f4f406f1c26b18f800225ac0" integrity sha512-4Bcg1P8xhUuqcii/S0Z9wiHIrQVPMermM1any+MX5GeGD7faD3/msQUDGLol9wOcz4/jbg/WJnGqoJF6LiBdtg== binary-extensions@^2.0.0, binary-extensions@^2.2.0: version "2.3.0" resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.3.0.tgz#f6e14a97858d327252200242d4ccfe522c445522" integrity sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw== binary-search@^1.3.5: version "1.3.6" resolved "https://registry.yarnpkg.com/binary-search/-/binary-search-1.3.6.tgz#e32426016a0c5092f0f3598836a1c7da3560565c" integrity sha512-nbE1WxOTTrUWIfsfZ4aHGYu5DOuNkbxGokjV6Z2kxfJK3uaAb8zNK1muzOeipoLHZjInT4Br88BHpzevc681xA== bl@^1.0.0: version "1.2.3" resolved "https://registry.yarnpkg.com/bl/-/bl-1.2.3.tgz#1e8dd80142eac80d7158c9dccc047fb620e035e7" integrity sha512-pvcNpa0UU69UT341rO6AYy4FVAIkUHuZXRIWbq+zHnsVcRzDDjIAhGuuYoi0d//cwIwtt4pkpKycWEfjdV+vww== dependencies: readable-stream "^2.3.5" safe-buffer "^5.1.1" bl@^4.0.3: version "4.1.0" resolved "https://registry.yarnpkg.com/bl/-/bl-4.1.0.tgz#451535264182bec2fbbc83a62ab98cf11d9f7b3a" integrity sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w== dependencies: buffer "^5.5.0" inherits "^2.0.4" readable-stream "^3.4.0" bluebird@^3.7.2: version "3.7.2" resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f" integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg== bluebird@~3.4.0: version "3.4.7" resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.4.7.tgz#f72d760be09b7f76d08ed8fae98b289a8d05fab3" integrity sha512-iD3898SR7sWVRHbiQv+sHUtHnMvC1o3nW5rAcqnq3uOn07DSAppZYUkIGslDz6gXC7HfunPe7YVBgoEJASPcHA== bmp-js@^0.1.0: version "0.1.0" resolved "https://registry.yarnpkg.com/bmp-js/-/bmp-js-0.1.0.tgz#e05a63f796a6c1ff25f4771ec7adadc148c07233" integrity sha512-vHdS19CnY3hwiNdkaqk93DvjVLfbEcI8mys4UjuWrlX1haDmroo8o4xCzh4wD6DGV6HxRCyauwhHRqMTfERtjw== body-parser@^1.20.3, body-parser@~1.20.3: version "1.20.4" resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.20.4.tgz#f8e20f4d06ca8a50a71ed329c15dccad1cdc547f" integrity sha512-ZTgYYLMOXY9qKU/57FAo8F+HA2dGX7bqGc71txDRC1rS4frdFI5R7NhluHxH6M0YItAP0sHB4uqAOcYKxO6uGA== dependencies: bytes "~3.1.2" content-type "~1.0.5" debug "2.6.9" depd "2.0.0" destroy "~1.2.0" http-errors "~2.0.1" iconv-lite "~0.4.24" on-finished "~2.4.1" qs "~6.14.0" raw-body "~2.5.3" type-is "~1.6.18" unpipe "~1.0.0" boolbase@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/boolbase/-/boolbase-1.0.0.tgz#68dff5fbe60c51eb37725ea9e3ed310dcc1e776e" integrity sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww== brace-expansion@^1.1.7: version "1.1.12" resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.12.tgz#ab9b454466e5a8cc3a187beaad580412a9c5b843" integrity sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg== dependencies: balanced-match "^1.0.0" concat-map "0.0.1" brace-expansion@^2.0.1: version "2.0.2" resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-2.0.2.tgz#54fc53237a613d854c7bd37463aad17df87214e7" integrity sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ== dependencies: balanced-match "^1.0.0" braces@~3.0.2: version "3.0.3" resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.3.tgz#490332f40919452272d55a8480adc0c441358789" integrity sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA== dependencies: fill-range "^7.1.1" buffer-alloc-unsafe@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/buffer-alloc-unsafe/-/buffer-alloc-unsafe-1.1.0.tgz#bd7dc26ae2972d0eda253be061dba992349c19f0" integrity sha512-TEM2iMIEQdJ2yjPJoSIsldnleVaAk1oW3DBVUykyOLsEsFmEc9kn+SFFPz+gl54KQNxlDnAwCXosOS9Okx2xAg== buffer-alloc@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/buffer-alloc/-/buffer-alloc-1.2.0.tgz#890dd90d923a873e08e10e5fd51a57e5b7cce0ec" integrity sha512-CFsHQgjtW1UChdXgbyJGtnm+O/uLQeZdtbDo8mfUgYXCHSM1wgrVxXm6bSyrUuErEb+4sYVGCzASBRot7zyrow== dependencies: buffer-alloc-unsafe "^1.1.0" buffer-fill "^1.0.0" buffer-crc32@~0.2.3: version "0.2.13" resolved "https://registry.yarnpkg.com/buffer-crc32/-/buffer-crc32-0.2.13.tgz#0d333e3f00eac50aa1454abd30ef8c2a5d9a7242" integrity sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ== buffer-fill@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/buffer-fill/-/buffer-fill-1.0.0.tgz#f8f78b76789888ef39f205cd637f68e702122b2c" integrity sha512-T7zexNBwiiaCOGDg9xNX9PBmjrubblRkENuptryuI64URkXDFum9il/JGL8Lm8wYfAXpredVXXZz7eMHilimiQ== buffer@^5.2.1, buffer@^5.5.0: version "5.7.1" resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.7.1.tgz#ba62e7c13133053582197160851a8f648e99eed0" integrity sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ== dependencies: base64-js "^1.3.1" ieee754 "^1.1.13" buffer@^6.0.3: version "6.0.3" resolved "https://registry.yarnpkg.com/buffer/-/buffer-6.0.3.tgz#2ace578459cc8fbe2a70aaa8f52ee63b6a74c6c6" integrity sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA== dependencies: base64-js "^1.3.1" ieee754 "^1.2.1" bytes@~3.1.2: version "3.1.2" resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.1.2.tgz#8b0beeb98605adf1b128fa4386403c009e0221a5" integrity sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg== call-bind-apply-helpers@^1.0.0, call-bind-apply-helpers@^1.0.1, call-bind-apply-helpers@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz#4b5428c222be985d79c3d82657479dbe0b59b2d6" integrity sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ== dependencies: es-errors "^1.3.0" function-bind "^1.1.2" call-bind@^1.0.8: version "1.0.8" resolved "https://registry.yarnpkg.com/call-bind/-/call-bind-1.0.8.tgz#0736a9660f537e3388826f440d5ec45f744eaa4c" integrity sha512-oKlSFMcMwpUg2ednkhQ454wfWiU/ul3CkJe/PEHcTKuiX6RpbehUiFMXu13HalGZxfUwCQzZG747YXBn1im9ww== dependencies: call-bind-apply-helpers "^1.0.0" es-define-property "^1.0.0" get-intrinsic "^1.2.4" set-function-length "^1.2.2" call-bound@^1.0.2, call-bound@^1.0.3, call-bound@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/call-bound/-/call-bound-1.0.4.tgz#238de935d2a2a692928c538c7ccfa91067fd062a" integrity sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg== dependencies: call-bind-apply-helpers "^1.0.2" get-intrinsic "^1.3.0" callsites@^3.0.0: version "3.1.0" resolved "https://registry.yarnpkg.com/callsites/-/callsites-3.1.0.tgz#b3630abd8943432f54b3f0519238e33cd7df2f73" integrity sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ== camelcase@6: version "6.3.0" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-6.3.0.tgz#5685b95eb209ac9c0c177467778c9c84df58ba9a" integrity sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA== chalk@^4.0.0: version "4.1.2" resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.1.2.tgz#aac4e2b7734a740867aeb16bf02aad556a1e7a01" integrity sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA== dependencies: ansi-styles "^4.1.0" supports-color "^7.1.0" charenc@0.0.2: version "0.0.2" resolved "https://registry.yarnpkg.com/charenc/-/charenc-0.0.2.tgz#c0a1d2f3a7092e03774bfa83f14c0fc5790a8667" integrity sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA== chokidar@^3.5.2: version "3.6.0" resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-3.6.0.tgz#197c6cc669ef2a8dc5e7b4d97ee4e092c3eb0d5b" integrity sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw== dependencies: anymatch "~3.1.2" braces "~3.0.2" glob-parent "~5.1.2" is-binary-path "~2.1.0" is-glob "~4.0.1" normalize-path "~3.0.0" readdirp "~3.6.0" optionalDependencies: fsevents "~2.3.2" chownr@^1.1.1: version "1.1.4" resolved "https://registry.yarnpkg.com/chownr/-/chownr-1.1.4.tgz#6fc9d7b42d32a583596337666e7d08084da2cc6b" integrity sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg== chromium-bidi@0.4.33: version "0.4.33" resolved "https://registry.yarnpkg.com/chromium-bidi/-/chromium-bidi-0.4.33.tgz#9a9aba5a5b07118c8e7d6405f8ee79f47418dd1d" integrity sha512-IxoFM5WGQOIAd95qrSXzJUv4eXIrh+RvU3rwwqIiwYuvfE7U/Llj4fejbsJnjJMUYCuGtVQsY2gv7oGl4aTNSQ== dependencies: mitt "3.0.1" urlpattern-polyfill "9.0.0" cliui@^8.0.1: version "8.0.1" resolved "https://registry.yarnpkg.com/cliui/-/cliui-8.0.1.tgz#0c04b075db02cbfe60dc8e6cf2f5486b1a3608aa" integrity sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ== dependencies: string-width "^4.2.0" strip-ansi "^6.0.1" wrap-ansi "^7.0.0" color-convert@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-2.0.1.tgz#72d3a68d598c9bdb3af2ad1e84f21d896abd4de3" integrity sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ== dependencies: color-name "~1.1.4" color-convert@^3.1.3: version "3.1.3" resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-3.1.3.tgz#db6627b97181cb8facdfce755ae26f97ab0711f1" integrity sha512-fasDH2ont2GqF5HpyO4w0+BcewlhHEZOFn9c1ckZdHpJ56Qb7MHhH/IcJZbBGgvdtwdwNbLvxiBEdg336iA9Sg== dependencies: color-name "^2.0.0" color-name@^1.0.0, color-name@~1.1.4: version "1.1.4" resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2" integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA== color-name@^2.0.0: version "2.1.0" resolved "https://registry.yarnpkg.com/color-name/-/color-name-2.1.0.tgz#0b677385c1c4b4edfdeaf77e38fa338e3a40b693" integrity sha512-1bPaDNFm0axzE4MEAzKPuqKWeRaT43U/hyxKPBdqTfmPF+d6n7FSoTFxLVULUJOmiLp01KjhIPPH+HrXZJN4Rg== color-string@^1.9.0: version "1.9.1" resolved "https://registry.yarnpkg.com/color-string/-/color-string-1.9.1.tgz#4467f9146f036f855b764dfb5bf8582bf342c7a4" integrity sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg== dependencies: color-name "^1.0.0" simple-swizzle "^0.2.2" color-string@^2.1.3: version "2.1.4" resolved "https://registry.yarnpkg.com/color-string/-/color-string-2.1.4.tgz#9dcf566ff976e23368c8bd673f5c35103ab41058" integrity sha512-Bb6Cq8oq0IjDOe8wJmi4JeNn763Xs9cfrBcaylK1tPypWzyoy2G3l90v9k64kjphl/ZJjPIShFztenRomi8WTg== dependencies: color-name "^2.0.0" color@^4.2.3: version "4.2.3" resolved "https://registry.yarnpkg.com/color/-/color-4.2.3.tgz#d781ecb5e57224ee43ea9627560107c0e0c6463a" integrity sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A== dependencies: color-convert "^2.0.1" color-string "^1.9.0" color@^5.0.2: version "5.0.3" resolved "https://registry.yarnpkg.com/color/-/color-5.0.3.tgz#f79390b1b778e222ffbb54304d3dbeaef633f97f" integrity sha512-ezmVcLR3xAVp8kYOm4GS45ZLLgIE6SPAFoduLr6hTDajwb3KZ2F46gulK3XpcwRFb5KKGCSezCBAY4Dw4HsyXA== dependencies: color-convert "^3.1.3" color-string "^2.1.3" combined-stream@^1.0.8: version "1.0.8" resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.8.tgz#c3d45a8b34fd730631a110a8a2520682b31d5a7f" integrity sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg== dependencies: delayed-stream "~1.0.0" commander@^10.0.1: version "10.0.1" resolved "https://registry.yarnpkg.com/commander/-/commander-10.0.1.tgz#881ee46b4f77d1c1dccc5823433aa39b022cbe06" integrity sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug== commander@^2.8.1: version "2.20.3" resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33" integrity sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ== concat-map@0.0.1: version "0.0.1" resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" integrity sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg== content-disposition@~0.5.4: version "0.5.4" resolved "https://registry.yarnpkg.com/content-disposition/-/content-disposition-0.5.4.tgz#8b82b4efac82512a02bb0b1dcec9d2c5e8eb5bfe" integrity sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ== dependencies: safe-buffer "5.2.1" content-type@~1.0.4, content-type@~1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/content-type/-/content-type-1.0.5.tgz#8b773162656d1d1086784c8f23a54ce6d73d7918" integrity sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA== cookie-signature@~1.0.6: version "1.0.7" resolved "https://registry.yarnpkg.com/cookie-signature/-/cookie-signature-1.0.7.tgz#ab5dd7ab757c54e60f37ef6550f481c426d10454" integrity sha512-NXdYc3dLr47pBkpUCHtKSwIOQXLVn8dZEuywboCOJY/osA0wFSLlSawr3KN8qXJEyX66FcONTH8EIlVuK0yyFA== cookie@~0.7.1: version "0.7.2" resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.7.2.tgz#556369c472a2ba910f2979891b526b3436237ed7" integrity sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w== core-util-is@~1.0.0: version "1.0.3" resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.3.tgz#a6042d3634c2b27e9328f837b965fac83808db85" integrity sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ== cors@^2.8.5: version "2.8.5" resolved "https://registry.yarnpkg.com/cors/-/cors-2.8.5.tgz#eac11da51592dd86b9f06f6e7ac293b3df875d29" integrity sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g== dependencies: object-assign "^4" vary "^1" cosmiconfig@8.3.6: version "8.3.6" resolved "https://registry.yarnpkg.com/cosmiconfig/-/cosmiconfig-8.3.6.tgz#060a2b871d66dba6c8538ea1118ba1ac16f5fae3" integrity sha512-kcZ6+W5QzcJ3P1Mt+83OUv/oHFqZHIx8DuxG6eZ5RGMERoLqp4BuGjhHLYGK+Kf5XVkQvqBSmAy/nGWN3qDgEA== dependencies: import-fresh "^3.3.0" js-yaml "^4.1.0" parse-json "^5.2.0" path-type "^4.0.0" crlf-normalize@^1.0.19: version "1.0.20" resolved "https://registry.yarnpkg.com/crlf-normalize/-/crlf-normalize-1.0.20.tgz#0b3105d3de807bce8a7599113235d725fe9361a8" integrity sha512-h/rBerTd3YHQGfv7tNT25mfhWvRq2BBLCZZ80GFarFxf6HQGbpW6iqDL3N+HBLpjLfAdcBXfWAzVlLfHkRUQBQ== dependencies: ts-type ">=2" cross-env@^7.0.3: version "7.0.3" resolved "https://registry.yarnpkg.com/cross-env/-/cross-env-7.0.3.tgz#865264b29677dc015ba8418918965dd232fc54cf" integrity sha512-+/HKd6EgcQCJGh2PSjZuUitQBQynKor4wrFbRg4DtAgS1aWO+gU52xpH7M9ScGgXSYmAVS9bIJ8EzuaGw0oNAw== dependencies: cross-spawn "^7.0.1" cross-fetch@4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/cross-fetch/-/cross-fetch-4.0.0.tgz#f037aef1580bb3a1a35164ea2a848ba81b445983" integrity sha512-e4a5N8lVvuLgAWgnCrLr2PP0YyDOTHa9H/Rj54dirp61qXnNq46m82bRhNqIA5VccJtWBvPTFRV3TtvHUKPB1g== dependencies: node-fetch "^2.6.12" cross-spawn@^7.0.1, cross-spawn@^7.0.3, cross-spawn@^7.0.6: version "7.0.6" resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.6.tgz#8a58fe78f00dcd70c370451759dfbfaf03e8ee9f" integrity sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA== dependencies: path-key "^3.1.0" shebang-command "^2.0.0" which "^2.0.1" crypt@0.0.2: version "0.0.2" resolved "https://registry.yarnpkg.com/crypt/-/crypt-0.0.2.tgz#88d7ff7ec0dfb86f713dc87bbb42d044d3e6c41b" integrity sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow== css-select@^5.1.0: version "5.2.2" resolved "https://registry.yarnpkg.com/css-select/-/css-select-5.2.2.tgz#01b6e8d163637bb2dd6c982ca4ed65863682786e" integrity sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw== dependencies: boolbase "^1.0.0" css-what "^6.1.0" domhandler "^5.0.2" domutils "^3.0.1" nth-check "^2.0.1" css-what@^6.1.0: version "6.2.2" resolved "https://registry.yarnpkg.com/css-what/-/css-what-6.2.2.tgz#cdcc8f9b6977719fdfbd1de7aec24abf756b9dea" integrity sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA== data-uri-to-buffer@^6.0.2: version "6.0.2" resolved "https://registry.yarnpkg.com/data-uri-to-buffer/-/data-uri-to-buffer-6.0.2.tgz#8a58bb67384b261a38ef18bea1810cb01badd28b" integrity sha512-7hvf7/GW8e86rW0ptuwS3OcBGDjIi6SZva7hCyWC0yYry2cOPmLIjXAUHI6DK2HsnwJd9ifmt57i8eV2n4YNpw== debug@2.6.9: version "2.6.9" resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" integrity sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA== dependencies: ms "2.0.0" debug@4, debug@^4.1.1, debug@^4.3.1, debug@^4.3.2, debug@^4.3.4: version "4.4.3" resolved "https://registry.yarnpkg.com/debug/-/debug-4.4.3.tgz#c6ae432d9bd9662582fce08709b038c58e9e3d6a" integrity sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA== dependencies: ms "^2.1.3" debug@4.3.4: version "4.3.4" resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.4.tgz#1319f6579357f2338d3337d2cdd4914bb5dcc865" integrity sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ== dependencies: ms "2.1.2" debug@^3.2.7: version "3.2.7" resolved "https://registry.yarnpkg.com/debug/-/debug-3.2.7.tgz#72580b7e9145fb39b6676f9c5e5fb100b934179a" integrity sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ== dependencies: ms "^2.1.1" decamelize@1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290" integrity sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA== decompress-response@^6.0.0: version "6.0.0" resolved "https://registry.yarnpkg.com/decompress-response/-/decompress-response-6.0.0.tgz#ca387612ddb7e104bd16d85aab00d5ecf09c66fc" integrity sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ== dependencies: mimic-response "^3.1.0" decompress-tar@^4.0.0, decompress-tar@^4.1.0, decompress-tar@^4.1.1: version "4.1.1" resolved "https://registry.yarnpkg.com/decompress-tar/-/decompress-tar-4.1.1.tgz#718cbd3fcb16209716e70a26b84e7ba4592e5af1" integrity sha512-JdJMaCrGpB5fESVyxwpCx4Jdj2AagLmv3y58Qy4GE6HMVjWz1FeVQk1Ct4Kye7PftcdOo/7U7UKzYBJgqnGeUQ== dependencies: file-type "^5.2.0" is-stream "^1.1.0" tar-stream "^1.5.2" decompress-tarbz2@^4.0.0: version "4.1.1" resolved "https://registry.yarnpkg.com/decompress-tarbz2/-/decompress-tarbz2-4.1.1.tgz#3082a5b880ea4043816349f378b56c516be1a39b" integrity sha512-s88xLzf1r81ICXLAVQVzaN6ZmX4A6U4z2nMbOwobxkLoIIfjVMBg7TeguTUXkKeXni795B6y5rnvDw7rxhAq9A== dependencies: decompress-tar "^4.1.0" file-type "^6.1.0" is-stream "^1.1.0" seek-bzip "^1.0.5" unbzip2-stream "^1.0.9" decompress-targz@^4.0.0: version "4.1.1" resolved "https://registry.yarnpkg.com/decompress-targz/-/decompress-targz-4.1.1.tgz#c09bc35c4d11f3de09f2d2da53e9de23e7ce1eee" integrity sha512-4z81Znfr6chWnRDNfFNqLwPvm4db3WuZkqV+UgXQzSngG3CEKdBkw5jrv3axjjL96glyiiKjsxJG3X6WBZwX3w== dependencies: decompress-tar "^4.1.1" file-type "^5.2.0" is-stream "^1.1.0" decompress-unzip@^4.0.1: version "4.0.1" resolved "https://registry.yarnpkg.com/decompress-unzip/-/decompress-unzip-4.0.1.tgz#deaaccdfd14aeaf85578f733ae8210f9b4848f69" integrity sha512-1fqeluvxgnn86MOh66u8FjbtJpAFv5wgCT9Iw8rcBqQcCo5tO8eiJw7NNTrvt9n4CRBVq7CstiS922oPgyGLrw== dependencies: file-type "^3.8.0" get-stream "^2.2.0" pify "^2.3.0" yauzl "^2.4.2" decompress@^4.2.1: version "4.2.1" resolved "https://registry.yarnpkg.com/decompress/-/decompress-4.2.1.tgz#007f55cc6a62c055afa37c07eb6a4ee1b773f118" integrity sha512-e48kc2IjU+2Zw8cTb6VZcJQ3lgVbS4uuB1TfCHbiZIP/haNXm+SVyhu+87jts5/3ROpd82GSVCoNs/z8l4ZOaQ== dependencies: decompress-tar "^4.0.0" decompress-tarbz2 "^4.0.0" decompress-targz "^4.0.0" decompress-unzip "^4.0.1" graceful-fs "^4.1.10" make-dir "^1.0.0" pify "^2.3.0" strip-dirs "^2.0.0" "deep-eql@= 4.0.0": version "4.0.0" resolved "https://registry.yarnpkg.com/deep-eql/-/deep-eql-4.0.0.tgz#c70af2713a4e18d9c2c1203ff9d11abbd51c8fbd" integrity sha512-GxJC5MOg2KyQlv6WiUF/VAnMj4MWnYiXo4oLgeptOELVoknyErb4Z8+5F/IM/K4g9/80YzzatxmWcyRwUseH0A== dependencies: type-detect "^4.0.0" deep-extend@^0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/deep-extend/-/deep-extend-0.6.0.tgz#c4fa7c95404a17a9c3e8ca7e1537312b736330ac" integrity sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA== deep-is@^0.1.3: version "0.1.4" resolved "https://registry.yarnpkg.com/deep-is/-/deep-is-0.1.4.tgz#a6f2dce612fadd2ef1f519b73551f17e85199831" integrity sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ== deepmerge@^4.3.1: version "4.3.1" resolved "https://registry.yarnpkg.com/deepmerge/-/deepmerge-4.3.1.tgz#44b5f2147cd3b00d4b56137685966f26fd25dd4a" integrity sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A== default-shell@^2.0.0: version "2.2.0" resolved "https://registry.yarnpkg.com/default-shell/-/default-shell-2.2.0.tgz#31481c19747bfe59319b486591643eaf115a1864" integrity sha512-sPpMZcVhRQ0nEMDtuMJ+RtCxt7iHPAMBU+I4tAlo5dU1sjRpNax0crj6nR3qKpvVnckaQ9U38enXcwW9nZJeCw== define-data-property@^1.1.4: version "1.1.4" resolved "https://registry.yarnpkg.com/define-data-property/-/define-data-property-1.1.4.tgz#894dc141bb7d3060ae4366f6a0107e68fbe48c5e" integrity sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A== dependencies: es-define-property "^1.0.0" es-errors "^1.3.0" gopd "^1.0.1" degenerator@^5.0.0: version "5.0.1" resolved "https://registry.yarnpkg.com/degenerator/-/degenerator-5.0.1.tgz#9403bf297c6dad9a1ece409b37db27954f91f2f5" integrity sha512-TllpMR/t0M5sqCXfj85i4XaAzxmS5tVA16dqvdkMwGmzI+dXLXnw3J+3Vdv7VKw+ThlTMboK6i9rnZ6Nntj5CQ== dependencies: ast-types "^0.13.4" escodegen "^2.1.0" esprima "^4.0.1" delayed-stream@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619" integrity sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ== depd@2.0.0, depd@~2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/depd/-/depd-2.0.0.tgz#b696163cc757560d09cf22cc8fad1571b79e76df" integrity sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw== destroy@1.2.0, destroy@~1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/destroy/-/destroy-1.2.0.tgz#4803735509ad8be552934c67df614f94e66fa015" integrity sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg== detect-libc@^2.0.0, detect-libc@^2.0.2, detect-libc@^2.0.3: version "2.1.2" resolved "https://registry.yarnpkg.com/detect-libc/-/detect-libc-2.1.2.tgz#689c5dcdc1900ef5583a4cb9f6d7b473742074ad" integrity sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ== devtools-protocol@0.0.1203626: version "0.0.1203626" resolved "https://registry.yarnpkg.com/devtools-protocol/-/devtools-protocol-0.0.1203626.tgz#4366a4c81a7e0d4fd6924e9182c67f1e5941e820" integrity sha512-nEzHZteIUZfGCZtTiS1fRpC8UZmsfD1SiyPvaUNvS13dvKf666OAm8YTi0+Ca3n1nLEyu49Cy4+dPWpaHFJk9g== digest-fetch@^1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/digest-fetch/-/digest-fetch-1.3.0.tgz#898e69264d00012a23cf26e8a3e40320143fc661" integrity sha512-CGJuv6iKNM7QyZlM2T3sPAdZWd/p9zQiRNS9G+9COUCwzWFTs0Xp8NF5iePx7wtvhDykReiRRrSeNb4oMmB8lA== dependencies: base-64 "^0.1.0" md5 "^2.3.0" dingbat-to-unicode@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/dingbat-to-unicode/-/dingbat-to-unicode-1.0.1.tgz#5091dd673241453e6b5865e26e5a4452cdef5c83" integrity sha512-98l0sW87ZT58pU4i61wa2OHwxbiYSbuxsCBozaVnYX2iCnr3bLM3fIes1/ej7h1YdOKuKt/MLs706TVnALA65w== dom-serializer@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/dom-serializer/-/dom-serializer-2.0.0.tgz#e41b802e1eedf9f6cae183ce5e622d789d7d8e53" integrity sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg== dependencies: domelementtype "^2.3.0" domhandler "^5.0.2" entities "^4.2.0" domelementtype@^2.3.0: version "2.3.0" resolved "https://registry.yarnpkg.com/domelementtype/-/domelementtype-2.3.0.tgz#5c45e8e869952626331d7aab326d01daf65d589d" integrity sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw== domhandler@^5.0.2, domhandler@^5.0.3: version "5.0.3" resolved "https://registry.yarnpkg.com/domhandler/-/domhandler-5.0.3.tgz#cc385f7f751f1d1fc650c21374804254538c7d31" integrity sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w== dependencies: domelementtype "^2.3.0" domutils@^3.0.1: version "3.2.2" resolved "https://registry.yarnpkg.com/domutils/-/domutils-3.2.2.tgz#edbfe2b668b0c1d97c24baf0f1062b132221bc78" integrity sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw== dependencies: dom-serializer "^2.0.0" domelementtype "^2.3.0" domhandler "^5.0.3" dotenv@^16.0.3: version "16.6.1" resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.6.1.tgz#773f0e69527a8315c7285d5ee73c4459d20a8020" integrity sha512-uBq4egWHTcTt33a72vpSG0z3HnPuIl6NqYcTrKEg2azoEyl2hpW0zqlxysq2pK9HlDIHyHyakeYaYnSAwd8bow== duck@^0.1.12: version "0.1.12" resolved "https://registry.yarnpkg.com/duck/-/duck-0.1.12.tgz#de7adf758421230b6d7aee799ce42670586b9efa" integrity sha512-wkctla1O6VfP89gQ+J/yDesM0S7B7XLXjKGzXxMDVFg7uEn706niAtyYovKbyq1oT9YwDcly721/iUWoc8MVRg== dependencies: underscore "^1.13.1" dunder-proto@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/dunder-proto/-/dunder-proto-1.0.1.tgz#d7ae667e1dc83482f8b70fd0f6eefc50da30f58a" integrity sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A== dependencies: call-bind-apply-helpers "^1.0.1" es-errors "^1.3.0" gopd "^1.2.0" ee-first@1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/ee-first/-/ee-first-1.1.1.tgz#590c61156b0ae2f4f0255732a158b266bc56b21d" integrity sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow== emoji-regex@^8.0.0: version "8.0.0" resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-8.0.0.tgz#e818fd69ce5ccfcb404594f842963bf53164cc37" integrity sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A== enabled@2.0.x: version "2.0.0" resolved "https://registry.yarnpkg.com/enabled/-/enabled-2.0.0.tgz#f9dd92ec2d6f4bbc0d5d1e64e21d61cd4665e7c2" integrity sha512-AKrN98kuwOzMIdAizXGI86UFBoo26CL21UM763y1h/GMSJ4/OHU9k2YlsmBpyScFo/wbLzWQJBMCW4+IO3/+OQ== encodeurl@~1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.2.tgz#ad3ff4c86ec2d029322f5a02c3a9a606c95b3f59" integrity sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w== encodeurl@~2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-2.0.0.tgz#7b8ea898077d7e409d3ac45474ea38eaf0857a58" integrity sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg== encoding-japanese@2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/encoding-japanese/-/encoding-japanese-2.2.0.tgz#0ef2d2351250547f432a2dd155453555c16deb59" integrity sha512-EuJWwlHPZ1LbADuKTClvHtwbaFn4rOD+dRAbWysqEOXRc2Uui0hJInNJrsdH0c+OhJA4nrCBdSkW4DD5YxAo6A== end-of-stream@^1.0.0, end-of-stream@^1.1.0, end-of-stream@^1.4.1: version "1.4.5" resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.5.tgz#7344d711dea40e0b74abc2ed49778743ccedb08c" integrity sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg== dependencies: once "^1.4.0" entities@^4.2.0, entities@^4.4.0: version "4.5.0" resolved "https://registry.yarnpkg.com/entities/-/entities-4.5.0.tgz#5d268ea5e7113ec74c4d033b79ea5a35a488fb48" integrity sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw== "epub2@git+https://github.com/Mintplex-Labs/epub2-static.git#main": version "3.0.2" resolved "git+https://github.com/Mintplex-Labs/epub2-static.git#eb0a45cd41ac1a5b4c97766e9935d97583104d35" dependencies: adm-zip "^0.5.10" array-hyper-unique "^2.1.4" bluebird "^3.7.2" crlf-normalize "^1.0.19" tslib "^2.6.2" xml2js "^0.6.2" error-ex@^1.3.1: version "1.3.4" resolved "https://registry.yarnpkg.com/error-ex/-/error-ex-1.3.4.tgz#b3a8d8bb6f92eecc1629e3e27d3c8607a8a32414" integrity sha512-sqQamAnR14VgCr1A618A3sGrygcpK+HEbenA/HiEAkkUwcZIIB/tgWqHFxWgOyDh4nB4JCRimh79dR5Ywc9MDQ== dependencies: is-arrayish "^0.2.1" es-define-property@^1.0.0, es-define-property@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/es-define-property/-/es-define-property-1.0.1.tgz#983eb2f9a6724e9303f61addf011c72e09e0b0fa" integrity sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g== es-errors@^1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/es-errors/-/es-errors-1.3.0.tgz#05f75a25dab98e4fb1dcd5e1472c0546d5057c8f" integrity sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw== es-object-atoms@^1.0.0, es-object-atoms@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/es-object-atoms/-/es-object-atoms-1.1.1.tgz#1c4f2c4837327597ce69d2ca190a7fdd172338c1" integrity sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA== dependencies: es-errors "^1.3.0" es-set-tostringtag@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz#f31dbbe0c183b00a6d26eb6325c810c0fd18bd4d" integrity sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA== dependencies: es-errors "^1.3.0" get-intrinsic "^1.2.6" has-tostringtag "^1.0.2" hasown "^2.0.2" escalade@^3.1.1: version "3.2.0" resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.2.0.tgz#011a3f69856ba189dffa7dc8fcce99d2a87903e5" integrity sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA== escape-html@~1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/escape-html/-/escape-html-1.0.3.tgz#0258eae4d3d0c0974de1c169188ef0051d1d1988" integrity sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow== escape-string-regexp@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz#14ba83a5d373e3d311e5afca29cf5bfad965bf34" integrity sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA== escodegen@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/escodegen/-/escodegen-2.1.0.tgz#ba93bbb7a43986d29d6041f99f5262da773e2e17" integrity sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w== dependencies: esprima "^4.0.1" estraverse "^5.2.0" esutils "^2.0.2" optionalDependencies: source-map "~0.6.1" eslint-config-prettier@^9.0.0: version "9.1.2" resolved "https://registry.yarnpkg.com/eslint-config-prettier/-/eslint-config-prettier-9.1.2.tgz#90deb4fa0259592df774b600dbd1d2249a78ce91" integrity sha512-iI1f+D2ViGn+uvv5HuHVUamg8ll4tN+JRHGc6IJi4TP9Kl976C57fzPXgseXNs8v0iA8aSJpHsTWjDb9QJamGQ== eslint-plugin-prettier@^5.0.0: version "5.5.5" resolved "https://registry.yarnpkg.com/eslint-plugin-prettier/-/eslint-plugin-prettier-5.5.5.tgz#9eae11593faa108859c26f9a9c367d619a0769c0" integrity sha512-hscXkbqUZ2sPithAuLm5MXL+Wph+U7wHngPBv9OMWwlP8iaflyxpjTYZkmdgB4/vPIhemRlBEoLrH7UC1n7aUw== dependencies: prettier-linter-helpers "^1.0.1" synckit "^0.11.12" eslint-plugin-unused-imports@^4.0.0: version "4.4.1" resolved "https://registry.yarnpkg.com/eslint-plugin-unused-imports/-/eslint-plugin-unused-imports-4.4.1.tgz#a831f0a2937d7631eba30cb87091ab7d3a5da0e1" integrity sha512-oZGYUz1X3sRMGUB+0cZyK2VcvRX5lm/vB56PgNNcU+7ficUCKm66oZWKUubXWnOuPjQ8PvmXtCViXBMONPe7tQ== eslint-scope@^8.4.0: version "8.4.0" resolved "https://registry.yarnpkg.com/eslint-scope/-/eslint-scope-8.4.0.tgz#88e646a207fad61436ffa39eb505147200655c82" integrity sha512-sNXOfKCn74rt8RICKMvJS7XKV/Xk9kA7DyJr8mJik3S7Cwgy3qlkkmyS2uQB3jiJg6VNdZd/pDBJu0nvG2NlTg== dependencies: esrecurse "^4.3.0" estraverse "^5.2.0" eslint-visitor-keys@^3.4.3: version "3.4.3" resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz#0cd72fe8550e3c2eae156a96a4dddcd1c8ac5800" integrity sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag== eslint-visitor-keys@^4.2.1: version "4.2.1" resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-4.2.1.tgz#4cfea60fe7dd0ad8e816e1ed026c1d5251b512c1" integrity sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ== eslint@^9.0.0: version "9.39.3" resolved "https://registry.yarnpkg.com/eslint/-/eslint-9.39.3.tgz#08d63df1533d7743c0907b32a79a7e134e63ee2f" integrity sha512-VmQ+sifHUbI/IcSopBCF/HO3YiHQx/AVd3UVyYL6weuwW+HvON9VYn5l6Zl1WZzPWXPNZrSQpxwkkZ/VuvJZzg== dependencies: "@eslint-community/eslint-utils" "^4.8.0" "@eslint-community/regexpp" "^4.12.1" "@eslint/config-array" "^0.21.1" "@eslint/config-helpers" "^0.4.2" "@eslint/core" "^0.17.0" "@eslint/eslintrc" "^3.3.1" "@eslint/js" "9.39.3" "@eslint/plugin-kit" "^0.4.1" "@humanfs/node" "^0.16.6" "@humanwhocodes/module-importer" "^1.0.1" "@humanwhocodes/retry" "^0.4.2" "@types/estree" "^1.0.6" ajv "^6.12.4" chalk "^4.0.0" cross-spawn "^7.0.6" debug "^4.3.2" escape-string-regexp "^4.0.0" eslint-scope "^8.4.0" eslint-visitor-keys "^4.2.1" espree "^10.4.0" esquery "^1.5.0" esutils "^2.0.2" fast-deep-equal "^3.1.3" file-entry-cache "^8.0.0" find-up "^5.0.0" glob-parent "^6.0.2" ignore "^5.2.0" imurmurhash "^0.1.4" is-glob "^4.0.0" json-stable-stringify-without-jsonify "^1.0.1" lodash.merge "^4.6.2" minimatch "^3.1.2" natural-compare "^1.4.0" optionator "^0.9.3" espree@^10.0.1, espree@^10.4.0: version "10.4.0" resolved "https://registry.yarnpkg.com/espree/-/espree-10.4.0.tgz#d54f4949d4629005a1fa168d937c3ff1f7e2a837" integrity sha512-j6PAQ2uUr79PZhBjP5C5fhl8e39FmRnOjsD5lGnWrFU8i2G776tBK7+nP8KuQUTTyAZUwfQqXAgrVH5MbH9CYQ== dependencies: acorn "^8.15.0" acorn-jsx "^5.3.2" eslint-visitor-keys "^4.2.1" esprima@^4.0.1: version "4.0.1" resolved "https://registry.yarnpkg.com/esprima/-/esprima-4.0.1.tgz#13b04cdb3e6c5d19df91ab6987a8695619b0aa71" integrity sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A== esquery@^1.5.0: version "1.7.0" resolved "https://registry.yarnpkg.com/esquery/-/esquery-1.7.0.tgz#08d048f261f0ddedb5bae95f46809463d9c9496d" integrity sha512-Ap6G0WQwcU/LHsvLwON1fAQX9Zp0A2Y6Y/cJBl9r/JbW90Zyg4/zbG6zzKa2OTALELarYHmKu0GhpM5EO+7T0g== dependencies: estraverse "^5.1.0" esrecurse@^4.3.0: version "4.3.0" resolved "https://registry.yarnpkg.com/esrecurse/-/esrecurse-4.3.0.tgz#7ad7964d679abb28bee72cec63758b1c5d2c9921" integrity sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag== dependencies: estraverse "^5.2.0" estraverse@^5.1.0, estraverse@^5.2.0: version "5.3.0" resolved "https://registry.yarnpkg.com/estraverse/-/estraverse-5.3.0.tgz#2eea5290702f26ab8fe5370370ff86c965d21123" integrity sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA== esutils@^2.0.2: version "2.0.3" resolved "https://registry.yarnpkg.com/esutils/-/esutils-2.0.3.tgz#74d2eb4de0b8da1293711910d50775b9b710ef64" integrity sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g== etag@~1.8.1: version "1.8.1" resolved "https://registry.yarnpkg.com/etag/-/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887" integrity sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg== event-target-shim@^5.0.0: version "5.0.1" resolved "https://registry.yarnpkg.com/event-target-shim/-/event-target-shim-5.0.1.tgz#5d4d3ebdf9583d63a5333ce2deb7480ab2b05789" integrity sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ== eventemitter3@^4.0.4: version "4.0.7" resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-4.0.7.tgz#2de9b68f6528d5644ef5c59526a1b4a07306169f" integrity sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw== events-universal@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/events-universal/-/events-universal-1.0.1.tgz#b56a84fd611b6610e0a2d0f09f80fdf931e2dfe6" integrity sha512-LUd5euvbMLpwOF8m6ivPCbhQeSiYVNb8Vs0fQ8QjXo0JTkEHpz8pxdQf0gStltaPpw0Cca8b39KxvK9cfKRiAw== dependencies: bare-events "^2.7.0" events@^3.3.0: version "3.3.0" resolved "https://registry.yarnpkg.com/events/-/events-3.3.0.tgz#31a95ad0a924e2d2c419a813aeb2c4e878ea7400" integrity sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q== execa@^5.1.1: version "5.1.1" resolved "https://registry.yarnpkg.com/execa/-/execa-5.1.1.tgz#f80ad9cbf4298f7bd1d4c9555c21e93741c411dd" integrity sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg== dependencies: cross-spawn "^7.0.3" get-stream "^6.0.0" human-signals "^2.1.0" is-stream "^2.0.0" merge-stream "^2.0.0" npm-run-path "^4.0.1" onetime "^5.1.2" signal-exit "^3.0.3" strip-final-newline "^2.0.0" expand-template@^2.0.3: version "2.0.3" resolved "https://registry.yarnpkg.com/expand-template/-/expand-template-2.0.3.tgz#6e14b3fcee0f3a6340ecb57d2e8918692052a47c" integrity sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg== expr-eval@^2.0.2: version "2.0.2" resolved "https://registry.yarnpkg.com/expr-eval/-/expr-eval-2.0.2.tgz#fa6f044a7b0c93fde830954eb9c5b0f7fbc7e201" integrity sha512-4EMSHGOPSwAfBiibw3ndnP0AvjDWLsMvGOvWEZ2F96IGk0bIVdjQisOHxReSkE13mHcfbuCiXw+G4y0zv6N8Eg== express@^4.21.2: version "4.22.1" resolved "https://registry.yarnpkg.com/express/-/express-4.22.1.tgz#1de23a09745a4fffdb39247b344bb5eaff382069" integrity sha512-F2X8g9P1X7uCPZMA3MVf9wcTqlyNp7IhH5qPCI0izhaOIYXaW9L535tGA3qmjRzpH+bZczqq7hVKxTR4NWnu+g== dependencies: accepts "~1.3.8" array-flatten "1.1.1" body-parser "~1.20.3" content-disposition "~0.5.4" content-type "~1.0.4" cookie "~0.7.1" cookie-signature "~1.0.6" debug "2.6.9" depd "2.0.0" encodeurl "~2.0.0" escape-html "~1.0.3" etag "~1.8.1" finalhandler "~1.3.1" fresh "~0.5.2" http-errors "~2.0.0" merge-descriptors "1.0.3" methods "~1.1.2" on-finished "~2.4.1" parseurl "~1.3.3" path-to-regexp "~0.1.12" proxy-addr "~2.0.7" qs "~6.14.0" range-parser "~1.2.1" safe-buffer "5.2.1" send "~0.19.0" serve-static "~1.16.2" setprototypeof "1.2.0" statuses "~2.0.1" type-is "~1.6.18" utils-merge "1.0.1" vary "~1.1.2" extract-zip@2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/extract-zip/-/extract-zip-2.0.1.tgz#663dca56fe46df890d5f131ef4a06d22bb8ba13a" integrity sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg== dependencies: debug "^4.1.1" get-stream "^5.1.0" yauzl "^2.10.0" optionalDependencies: "@types/yauzl" "^2.9.1" fast-deep-equal@^3.1.1, fast-deep-equal@^3.1.3: version "3.1.3" resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz#3a7d56b559d6cbc3eb512325244e619a65c6c525" integrity sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q== fast-diff@^1.1.2: version "1.3.0" resolved "https://registry.yarnpkg.com/fast-diff/-/fast-diff-1.3.0.tgz#ece407fa550a64d638536cd727e129c61616e0f0" integrity sha512-VxPP4NqbUjj6MaAOafWeUn2cXWLcCtljklUtZf0Ind4XQ+QPtmA0b18zZy0jIQx+ExRVCR/ZQpBmik5lXshNsw== fast-fifo@^1.2.0, fast-fifo@^1.3.2: version "1.3.2" resolved "https://registry.yarnpkg.com/fast-fifo/-/fast-fifo-1.3.2.tgz#286e31de96eb96d38a97899815740ba2a4f3640c" integrity sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ== fast-json-stable-stringify@^2.0.0: version "2.1.0" resolved "https://registry.yarnpkg.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz#874bf69c6f404c2b5d99c481341399fd55892633" integrity sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw== fast-levenshtein@^2.0.6: version "2.0.6" resolved "https://registry.yarnpkg.com/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz#3d8a5c66883a16a30ca8643e851f19baa7797917" integrity sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw== fd-slicer@~1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/fd-slicer/-/fd-slicer-1.1.0.tgz#25c7c89cb1f9077f8891bbe61d8f390eae256f1e" integrity sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g== dependencies: pend "~1.2.0" fecha@^4.2.0: version "4.2.3" resolved "https://registry.yarnpkg.com/fecha/-/fecha-4.2.3.tgz#4d9ccdbc61e8629b259fdca67e65891448d569fd" integrity sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw== file-entry-cache@^8.0.0: version "8.0.0" resolved "https://registry.yarnpkg.com/file-entry-cache/-/file-entry-cache-8.0.0.tgz#7787bddcf1131bffb92636c69457bbc0edd6d81f" integrity sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ== dependencies: flat-cache "^4.0.0" file-type@^16.5.4: version "16.5.4" resolved "https://registry.yarnpkg.com/file-type/-/file-type-16.5.4.tgz#474fb4f704bee427681f98dd390058a172a6c2fd" integrity sha512-/yFHK0aGjFEgDJjEKP0pWCplsPFPhwyfwevf/pVxiN0tmE4L9LmwWxWukdJSHdoCli4VgQLehjJtwQBnqmsKcw== dependencies: readable-web-to-node-stream "^3.0.0" strtok3 "^6.2.4" token-types "^4.1.1" file-type@^3.8.0: version "3.9.0" resolved "https://registry.yarnpkg.com/file-type/-/file-type-3.9.0.tgz#257a078384d1db8087bc449d107d52a52672b9e9" integrity sha512-RLoqTXE8/vPmMuTI88DAzhMYC99I8BWv7zYP4A1puo5HIjEJ5EX48ighy4ZyKMG9EDXxBgW6e++cn7d1xuFghA== file-type@^5.2.0: version "5.2.0" resolved "https://registry.yarnpkg.com/file-type/-/file-type-5.2.0.tgz#2ddbea7c73ffe36368dfae49dc338c058c2b8ad6" integrity sha512-Iq1nJ6D2+yIO4c8HHg4fyVb8mAJieo1Oloy1mLLaB2PvezNedhBVm+QU7g0qM42aiMbRXTxKKwGD17rjKNJYVQ== file-type@^6.1.0: version "6.2.0" resolved "https://registry.yarnpkg.com/file-type/-/file-type-6.2.0.tgz#e50cd75d356ffed4e306dc4f5bcf52a79903a919" integrity sha512-YPcTBDV+2Tm0VqjybVd32MHdlEGAtuxS3VAYsumFokDSMG+ROT5wawGlnHDoz7bfMcMDt9hxuXvXwoKUx2fkOg== fill-range@^7.1.1: version "7.1.1" resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.1.1.tgz#44265d3cac07e3ea7dc247516380643754a05292" integrity sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg== dependencies: to-regex-range "^5.0.1" finalhandler@~1.3.1: version "1.3.2" resolved "https://registry.yarnpkg.com/finalhandler/-/finalhandler-1.3.2.tgz#1ebc2228fc7673aac4a472c310cc05b77d852b88" integrity sha512-aA4RyPcd3badbdABGDuTXCMTtOneUCAYH/gxoYRTZlIJdF0YPWuGqiAsIrhNnnqdXGswYk6dGujem4w80UJFhg== dependencies: debug "2.6.9" encodeurl "~2.0.0" escape-html "~1.0.3" on-finished "~2.4.1" parseurl "~1.3.3" statuses "~2.0.2" unpipe "~1.0.0" find-up@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/find-up/-/find-up-5.0.0.tgz#4c92819ecb7083561e4f4a240a86be5198f536fc" integrity sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng== dependencies: locate-path "^6.0.0" path-exists "^4.0.0" fix-path@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/fix-path/-/fix-path-4.0.0.tgz#bc1d14f038edb734ac46944a45454106952ca429" integrity sha512-g31GX207Tt+psI53ZSaB1egprYbEN0ZYl90aKcO22A2LmCNnFsSq3b5YpoKp3E/QEiWByTXGJOkFQG4S07Bc1A== dependencies: shell-path "^3.0.0" flat-cache@^4.0.0: version "4.0.1" resolved "https://registry.yarnpkg.com/flat-cache/-/flat-cache-4.0.1.tgz#0ece39fcb14ee012f4b0410bd33dd9c1f011127c" integrity sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw== dependencies: flatted "^3.2.9" keyv "^4.5.4" flat@^5.0.2: version "5.0.2" resolved "https://registry.yarnpkg.com/flat/-/flat-5.0.2.tgz#8ca6fe332069ffa9d324c327198c598259ceb241" integrity sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ== flatbuffers@^1.12.0: version "1.12.0" resolved "https://registry.yarnpkg.com/flatbuffers/-/flatbuffers-1.12.0.tgz#72e87d1726cb1b216e839ef02658aa87dcef68aa" integrity sha512-c7CZADjRcl6j0PlvFy0ZqXQ67qSEZfrVPynmnL+2zPc+NtMvrF8Y0QceMo7QqnSPc7+uWjUIAbvCQ5WIKlMVdQ== flatted@^3.2.9: version "3.3.4" resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.3.4.tgz#0986e681008f0f13f58e18656c47967682db5ff6" integrity sha512-3+mMldrTAPdta5kjX2G2J7iX4zxtnwpdA8Tr2ZSjkyPSanvbZAcy6flmtnXbEybHrDcU9641lxrMfFuUxVz9vA== fn.name@1.x.x: version "1.1.0" resolved "https://registry.yarnpkg.com/fn.name/-/fn.name-1.1.0.tgz#26cad8017967aea8731bc42961d04a3d5988accc" integrity sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw== for-each@^0.3.5: version "0.3.5" resolved "https://registry.yarnpkg.com/for-each/-/for-each-0.3.5.tgz#d650688027826920feeb0af747ee7b9421a41d47" integrity sha512-dKx12eRCVIzqCxFGplyFKJMPvLEWgmNtUrpTiJIR5u97zEhRG8ySrtboPHZXx7daLxQVrl643cTzbab2tkQjxg== dependencies: is-callable "^1.2.7" foreground-child@^3.1.0: version "3.3.1" resolved "https://registry.yarnpkg.com/foreground-child/-/foreground-child-3.3.1.tgz#32e8e9ed1b68a3497befb9ac2b6adf92a638576f" integrity sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw== dependencies: cross-spawn "^7.0.6" signal-exit "^4.0.1" form-data-encoder@1.7.2: version "1.7.2" resolved "https://registry.yarnpkg.com/form-data-encoder/-/form-data-encoder-1.7.2.tgz#1f1ae3dccf58ed4690b86d87e4f57c654fbab040" integrity sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A== form-data@^4.0.4: version "4.0.5" resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.5.tgz#b49e48858045ff4cbf6b03e1805cebcad3679053" integrity sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w== dependencies: asynckit "^0.4.0" combined-stream "^1.0.8" es-set-tostringtag "^2.1.0" hasown "^2.0.2" mime-types "^2.1.12" formdata-node@^4.3.2: version "4.4.1" resolved "https://registry.yarnpkg.com/formdata-node/-/formdata-node-4.4.1.tgz#23f6a5cb9cb55315912cbec4ff7b0f59bbd191e2" integrity sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ== dependencies: node-domexception "1.0.0" web-streams-polyfill "4.0.0-beta.3" forwarded@0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/forwarded/-/forwarded-0.2.0.tgz#2269936428aad4c15c7ebe9779a84bf0b2a81811" integrity sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow== fresh@0.5.2, fresh@~0.5.2: version "0.5.2" resolved "https://registry.yarnpkg.com/fresh/-/fresh-0.5.2.tgz#3d8cadd90d976569fa835ab1f8e4b23a105605a7" integrity sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q== fs-constants@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/fs-constants/-/fs-constants-1.0.0.tgz#6be0de9be998ce16af8afc24497b9ee9b7ccd9ad" integrity sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow== fsevents@~2.3.2: version "2.3.3" resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.3.tgz#cac6407785d03675a2a5e1a5305c697b347d90d6" integrity sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw== function-bind@^1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.2.tgz#2c02d864d97f3ea6c8830c464cbd11ab6eab7a1c" integrity sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA== get-caller-file@^2.0.5: version "2.0.5" resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-2.0.5.tgz#4f94412a82db32f36e3b0b9741f8a97feb031f7e" integrity sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg== get-intrinsic@^1.2.4, get-intrinsic@^1.2.5, get-intrinsic@^1.2.6, get-intrinsic@^1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.3.0.tgz#743f0e3b6964a93a5491ed1bffaae054d7f98d01" integrity sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ== dependencies: call-bind-apply-helpers "^1.0.2" es-define-property "^1.0.1" es-errors "^1.3.0" es-object-atoms "^1.1.1" function-bind "^1.1.2" get-proto "^1.0.1" gopd "^1.2.0" has-symbols "^1.1.0" hasown "^2.0.2" math-intrinsics "^1.1.0" get-proto@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/get-proto/-/get-proto-1.0.1.tgz#150b3f2743869ef3e851ec0c49d15b1d14d00ee1" integrity sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g== dependencies: dunder-proto "^1.0.1" es-object-atoms "^1.0.0" get-stream@^2.2.0: version "2.3.1" resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-2.3.1.tgz#5f38f93f346009666ee0150a054167f91bdd95de" integrity sha512-AUGhbbemXxrZJRD5cDvKtQxLuYaIbNtDTK8YqupCI393Q2KSTreEsLUN3ZxAWFGiKTzL6nKuzfcIvieflUX9qA== dependencies: object-assign "^4.0.1" pinkie-promise "^2.0.0" get-stream@^5.1.0: version "5.2.0" resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-5.2.0.tgz#4966a1795ee5ace65e706c4b7beb71257d6e22d3" integrity sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA== dependencies: pump "^3.0.0" get-stream@^6.0.0: version "6.0.1" resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-6.0.1.tgz#a262d8eef67aced57c2852ad6167526a43cbf7b7" integrity sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg== get-uri@^6.0.1: version "6.0.5" resolved "https://registry.yarnpkg.com/get-uri/-/get-uri-6.0.5.tgz#714892aa4a871db671abc5395e5e9447bc306a16" integrity sha512-b1O07XYq8eRuVzBNgJLstU6FYc1tS6wnMtF1I1D9lE8LxZSOGZ7LhxN54yPP6mGw5f2CkXY2BQUL9Fx41qvcIg== dependencies: basic-ftp "^5.0.2" data-uri-to-buffer "^6.0.2" debug "^4.3.4" github-from-package@0.0.0: version "0.0.0" resolved "https://registry.yarnpkg.com/github-from-package/-/github-from-package-0.0.0.tgz#97fb5d96bfde8973313f20e8288ef9a167fa64ce" integrity sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw== glob-parent@^6.0.2: version "6.0.2" resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-6.0.2.tgz#6d237d99083950c79290f24c7642a3de9a28f9e3" integrity sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A== dependencies: is-glob "^4.0.3" glob-parent@~5.1.2: version "5.1.2" resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-5.1.2.tgz#869832c58034fe68a4093c17dc15e8340d8401c4" integrity sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow== dependencies: is-glob "^4.0.1" glob@^10.3.7: version "10.5.0" resolved "https://registry.yarnpkg.com/glob/-/glob-10.5.0.tgz#8ec0355919cd3338c28428a23d4f24ecc5fe738c" integrity sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg== dependencies: foreground-child "^3.1.0" jackspeak "^3.1.2" minimatch "^9.0.4" minipass "^7.1.2" package-json-from-dist "^1.0.0" path-scurry "^1.11.1" globals@^14.0.0: version "14.0.0" resolved "https://registry.yarnpkg.com/globals/-/globals-14.0.0.tgz#898d7413c29babcf6bafe56fcadded858ada724e" integrity sha512-oahGvuMGQlPw/ivIYBjVSrWAfWLBeku5tpPE2fOPLi+WHffIWbuh2tCjhyQhTBPMf5E9jDEH4FOmTYgYwbKwtQ== globals@^17.4.0: version "17.4.0" resolved "https://registry.yarnpkg.com/globals/-/globals-17.4.0.tgz#33d7d297ed1536b388a0e2f4bcd0ff19c8ff91b5" integrity sha512-hjrNztw/VajQwOLsMNT1cbJiH2muO3OROCHnbehc8eY5JyD2gqz4AcMHPqgaOR59DjgUjYAYLeH699g/eWi2jw== gopd@^1.0.1, gopd@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/gopd/-/gopd-1.2.0.tgz#89f56b8217bdbc8802bd299df6d7f1081d7e51a1" integrity sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg== graceful-fs@^4.1.10: version "4.2.11" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.11.tgz#4183e4e8bf08bb6e05bbb2f7d2e0c8f712ca40e3" integrity sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ== guid-typescript@^1.0.9: version "1.0.9" resolved "https://registry.yarnpkg.com/guid-typescript/-/guid-typescript-1.0.9.tgz#e35f77003535b0297ea08548f5ace6adb1480ddc" integrity sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ== has-flag@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-3.0.0.tgz#b5d454dc2199ae225699f3467e5a07f3b955bafd" integrity sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw== has-flag@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-4.0.0.tgz#944771fd9c81c81265c4d6941860da06bb59479b" integrity sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ== has-property-descriptors@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz#963ed7d071dc7bf5f084c5bfbe0d1b6222586854" integrity sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg== dependencies: es-define-property "^1.0.0" has-symbols@^1.0.3, has-symbols@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/has-symbols/-/has-symbols-1.1.0.tgz#fc9c6a783a084951d0b971fe1018de813707a338" integrity sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ== has-tostringtag@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/has-tostringtag/-/has-tostringtag-1.0.2.tgz#2cdc42d40bef2e5b4eeab7c01a73c54ce7ab5abc" integrity sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw== dependencies: has-symbols "^1.0.3" hasown@^2.0.2: version "2.0.2" resolved "https://registry.yarnpkg.com/hasown/-/hasown-2.0.2.tgz#003eaf91be7adc372e84ec59dc37252cedb80003" integrity sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ== dependencies: function-bind "^1.1.2" he@1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/he/-/he-1.2.0.tgz#84ae65fa7eafb165fddb61566ae14baf05664f0f" integrity sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw== html-to-text@9.0.5, html-to-text@^9.0.5: version "9.0.5" resolved "https://registry.yarnpkg.com/html-to-text/-/html-to-text-9.0.5.tgz#6149a0f618ae7a0db8085dca9bbf96d32bb8368d" integrity sha512-qY60FjREgVZL03vJU6IfMV4GDjGBIoOyvuFdpBDIX9yTlDw0TjxVBQp+P8NvpdIXNJvfWBTNul7fsAQJq2FNpg== dependencies: "@selderee/plugin-htmlparser2" "^0.11.0" deepmerge "^4.3.1" dom-serializer "^2.0.0" htmlparser2 "^8.0.2" selderee "^0.11.0" htmlparser2@^8.0.2: version "8.0.2" resolved "https://registry.yarnpkg.com/htmlparser2/-/htmlparser2-8.0.2.tgz#f002151705b383e62433b5cf466f5b716edaec21" integrity sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA== dependencies: domelementtype "^2.3.0" domhandler "^5.0.3" domutils "^3.0.1" entities "^4.4.0" http-errors@2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-2.0.0.tgz#b7774a1486ef73cf7667ac9ae0858c012c57b9d3" integrity sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ== dependencies: depd "2.0.0" inherits "2.0.4" setprototypeof "1.2.0" statuses "2.0.1" toidentifier "1.0.1" http-errors@~2.0.0, http-errors@~2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-2.0.1.tgz#36d2f65bc909c8790018dd36fb4d93da6caae06b" integrity sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ== dependencies: depd "~2.0.0" inherits "~2.0.4" setprototypeof "~1.2.0" statuses "~2.0.2" toidentifier "~1.0.1" http-proxy-agent@^7.0.0: version "7.0.2" resolved "https://registry.yarnpkg.com/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz#9a8b1f246866c028509486585f62b8f2c18c270e" integrity sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig== dependencies: agent-base "^7.1.0" debug "^4.3.4" https-proxy-agent@^7.0.2, https-proxy-agent@^7.0.6: version "7.0.6" resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz#da8dfeac7da130b05c2ba4b59c9b6cd66611a6b9" integrity sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw== dependencies: agent-base "^7.1.2" debug "4" human-signals@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-2.1.0.tgz#dc91fcba42e4d06e4abaed33b3e7a3c02f514ea0" integrity sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw== humanize-duration@^3.25.1: version "3.33.1" resolved "https://registry.yarnpkg.com/humanize-duration/-/humanize-duration-3.33.1.tgz#e4df2ce6660f24a6a3bf4a7b3bc63edb5be7826f" integrity sha512-hwzSCymnRdFx9YdRkQQ0OYequXiVAV6ZGQA2uzocwB0F4309Ke6pO8dg0P8LHhRQJyVjGteRTAA/zNfEcpXn8A== humanize-ms@^1.2.1: version "1.2.1" resolved "https://registry.yarnpkg.com/humanize-ms/-/humanize-ms-1.2.1.tgz#c46e3159a293f6b896da29316d8b6fe8bb79bbed" integrity sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ== dependencies: ms "^2.0.0" iconv-lite@0.6.3, iconv-lite@^0.6.3: version "0.6.3" resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.6.3.tgz#a52f80bf38da1952eb5c681790719871a1a72501" integrity sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw== dependencies: safer-buffer ">= 2.1.2 < 3.0.0" iconv-lite@0.7.0: version "0.7.0" resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.7.0.tgz#c50cd80e6746ca8115eb98743afa81aa0e147a3e" integrity sha512-cf6L2Ds3h57VVmkZe+Pn+5APsT7FpqJtEhhieDCvrE2MK5Qk9MyffgQyuxQTm6BChfeZNtcOLHp9IcWRVcIcBQ== dependencies: safer-buffer ">= 2.1.2 < 3.0.0" iconv-lite@~0.4.24: version "0.4.24" resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.24.tgz#2022b4b25fbddc21d2f524974a474aafe733908b" integrity sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA== dependencies: safer-buffer ">= 2.1.2 < 3" idb-keyval@^6.2.0: version "6.2.2" resolved "https://registry.yarnpkg.com/idb-keyval/-/idb-keyval-6.2.2.tgz#b0171b5f73944854a3291a5cdba8e12768c4854a" integrity sha512-yjD9nARJ/jb1g+CvD0tlhUHOrJ9Sy0P8T9MF3YaLlHnSRpwPfpTX0XIvpmw3gAJUmEu3FiICLBDPXVwyEvrleg== ieee754@^1.1.13, ieee754@^1.2.1: version "1.2.1" resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352" integrity sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA== ignore-by-default@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/ignore-by-default/-/ignore-by-default-1.0.1.tgz#48ca6d72f6c6a3af00a9ad4ae6876be3889e2b09" integrity sha512-Ius2VYcGNk7T90CppJqcIkS5ooHUZyIQK+ClZfMfMNFEF9VSE73Fq+906u/CWu92x4gzZMWOwfFYckPObzdEbA== ignore@^5.2.0, ignore@^5.3.0: version "5.3.2" resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.3.2.tgz#3cd40e729f3643fd87cb04e50bf0eb722bc596f5" integrity sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g== immediate@~3.0.5: version "3.0.6" resolved "https://registry.yarnpkg.com/immediate/-/immediate-3.0.6.tgz#9db1dbd0faf8de6fbe0f5dd5e56bb606280de69b" integrity sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ== import-fresh@^3.2.1, import-fresh@^3.3.0: version "3.3.1" resolved "https://registry.yarnpkg.com/import-fresh/-/import-fresh-3.3.1.tgz#9cecb56503c0ada1f2741dbbd6546e4b13b57ccf" integrity sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ== dependencies: parent-module "^1.0.0" resolve-from "^4.0.0" imurmurhash@^0.1.4: version "0.1.4" resolved "https://registry.yarnpkg.com/imurmurhash/-/imurmurhash-0.1.4.tgz#9218b9b2b928a238b13dc4fb6b6d576f231453ea" integrity sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA== inherits@2.0.4, inherits@^2.0.3, inherits@^2.0.4, inherits@~2.0.3, inherits@~2.0.4: version "2.0.4" resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== ini@~1.3.0: version "1.3.8" resolved "https://registry.yarnpkg.com/ini/-/ini-1.3.8.tgz#a29da425b48806f34767a4efce397269af28432c" integrity sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew== ip-address@^10.0.1: version "10.1.0" resolved "https://registry.yarnpkg.com/ip-address/-/ip-address-10.1.0.tgz#d8dcffb34d0e02eb241427444a6e23f5b0595aa4" integrity sha512-XXADHxXmvT9+CRxhXg56LJovE+bmWnEWB78LB83VZTprKTmaC5QfruXocxzTZ2Kl0DNwKuBdlIhjL8LeY8Sf8Q== ipaddr.js@1.9.1: version "1.9.1" resolved "https://registry.yarnpkg.com/ipaddr.js/-/ipaddr.js-1.9.1.tgz#bff38543eeb8984825079ff3a2a8e6cbd46781b3" integrity sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g== is-any-array@^2.0.0: version "2.0.1" resolved "https://registry.yarnpkg.com/is-any-array/-/is-any-array-2.0.1.tgz#9233242a9c098220290aa2ec28f82ca7fa79899e" integrity sha512-UtilS7hLRu++wb/WBAw9bNuP1Eg04Ivn1vERJck8zJthEvXCBEBpGR/33u/xLKWEQf95803oalHrVDptcAvFdQ== is-arrayish@^0.2.1: version "0.2.1" resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.2.1.tgz#77c99840527aa8ecb1a8ba697b80645a7a926a9d" integrity sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg== is-arrayish@^0.3.1: version "0.3.4" resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.3.4.tgz#1ee5553818511915685d33bb13d31bf854e5059d" integrity sha512-m6UrgzFVUYawGBh1dUsWR5M2Clqic9RVXC/9f8ceNlv2IcO9j9J/z8UoCLPqtsPBFNzEpfR3xftohbfqDx8EQA== is-binary-path@~2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/is-binary-path/-/is-binary-path-2.1.0.tgz#ea1f7f3b80f064236e83470f86c09c254fb45b09" integrity sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw== dependencies: binary-extensions "^2.0.0" is-buffer@~1.1.6: version "1.1.6" resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be" integrity sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w== is-callable@^1.2.7: version "1.2.7" resolved "https://registry.yarnpkg.com/is-callable/-/is-callable-1.2.7.tgz#3bc2a85ea742d9e36205dcacdd72ca1fdc51b055" integrity sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA== is-extglob@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/is-extglob/-/is-extglob-2.1.1.tgz#a88c02535791f02ed37c76a1b9ea9773c833f8c2" integrity sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ== is-fullwidth-code-point@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz#f116f8064fe90b3f7844a38997c0b75051269f1d" integrity sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg== is-glob@^4.0.0, is-glob@^4.0.1, is-glob@^4.0.3, is-glob@~4.0.1: version "4.0.3" resolved "https://registry.yarnpkg.com/is-glob/-/is-glob-4.0.3.tgz#64f61e42cbbb2eec2071a9dac0b28ba1e65d5084" integrity sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg== dependencies: is-extglob "^2.1.1" is-natural-number@^4.0.1: version "4.0.1" resolved "https://registry.yarnpkg.com/is-natural-number/-/is-natural-number-4.0.1.tgz#ab9d76e1db4ced51e35de0c72ebecf09f734cde8" integrity sha512-Y4LTamMe0DDQIIAlaer9eKebAlDSV6huy+TWhJVPlzZh2o4tRP5SQWFlLn5N0To4mDD22/qdOq+veo1cSISLgQ== is-number@^7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/is-number/-/is-number-7.0.0.tgz#7535345b896734d5f80c4d06c50955527a14f12b" integrity sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng== is-stream@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-1.1.0.tgz#12d4a3dd4e68e0b79ceb8dbc84173ae80d91ca44" integrity sha512-uQPm8kcs47jx38atAcWTVxyltQYoPT68y9aWYdV6yWXSyW8mzSat0TL6CiWdZeCdF3KrAvpVtnHbTv4RN+rqdQ== is-stream@^2.0.0: version "2.0.1" resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-2.0.1.tgz#fac1e3d53b97ad5a9d0ae9cef2389f5810a5c077" integrity sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg== is-typed-array@^1.1.14: version "1.1.15" resolved "https://registry.yarnpkg.com/is-typed-array/-/is-typed-array-1.1.15.tgz#4bfb4a45b61cee83a5a46fba778e4e8d59c0ce0b" integrity sha512-p3EcsicXjit7SaskXHs1hA91QxgTw46Fv6EFKKGS5DRFLD8yKnohjF3hxoju94b/OcMZoQukzpPpBE9uLVKzgQ== dependencies: which-typed-array "^1.1.16" is-url@^1.2.4: version "1.2.4" resolved "https://registry.yarnpkg.com/is-url/-/is-url-1.2.4.tgz#04a4df46d28c4cff3d73d01ff06abeb318a1aa52" integrity sha512-ITvGim8FhRiYe4IQ5uHSkj7pVaPDrCTkNd3yq3cV7iZAcJdHTUMPMEHcqSOy9xZ9qFenQCvi+2wjH9a1nXqHww== isarray@^2.0.5: version "2.0.5" resolved "https://registry.yarnpkg.com/isarray/-/isarray-2.0.5.tgz#8af1e4c1221244cc62459faf38940d4e644a5723" integrity sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw== isarray@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" integrity sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ== isexe@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10" integrity sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw== jackspeak@^3.1.2: version "3.4.3" resolved "https://registry.yarnpkg.com/jackspeak/-/jackspeak-3.4.3.tgz#8833a9d89ab4acde6188942bd1c53b6390ed5a8a" integrity sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw== dependencies: "@isaacs/cliui" "^8.0.2" optionalDependencies: "@pkgjs/parseargs" "^0.11.0" jintr@^1.1.0: version "1.2.0" resolved "https://registry.yarnpkg.com/jintr/-/jintr-1.2.0.tgz#f000275ee86f32a7d3d9eb9b71ffdb3c93600f50" integrity sha512-OYPVJPlDih+G7LXjnLR5Brsz8ClxkXYn0ZmWyJVIUmFI6lySQu/dzpGRK/ujBLAYCvuJu/VqXzf84C1edowbLg== dependencies: acorn "^8.8.0" js-tiktoken@^1.0.12, js-tiktoken@^1.0.7, js-tiktoken@^1.0.8: version "1.0.21" resolved "https://registry.yarnpkg.com/js-tiktoken/-/js-tiktoken-1.0.21.tgz#368a9957591a30a62997dd0c4cf30866f00f8221" integrity sha512-biOj/6M5qdgx5TKjDnFT1ymSpM5tbd3ylwDtrQvFQSu0Z7bBYko2dF+W/aUkXUPuk6IVpRxk/3Q2sHOzGlS36g== dependencies: base64-js "^1.5.1" js-tokens@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-4.0.0.tgz#19203fb59991df98e3a287050d4647cdeaf32499" integrity sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ== js-yaml@^4.1.0, js-yaml@^4.1.1: version "4.1.1" resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-4.1.1.tgz#854c292467705b699476e1a2decc0c8a3458806b" integrity sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA== dependencies: argparse "^2.0.1" json-buffer@3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/json-buffer/-/json-buffer-3.0.1.tgz#9338802a30d3b6605fbe0613e094008ca8c05a13" integrity sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ== json-parse-even-better-errors@^2.3.0: version "2.3.1" resolved "https://registry.yarnpkg.com/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz#7c47805a94319928e05777405dc12e1f7a4ee02d" integrity sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w== json-schema-traverse@^0.4.1: version "0.4.1" resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz#69f6a87d9513ab8bb8fe63bdb0979c448e684660" integrity sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg== json-stable-stringify-without-jsonify@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz#9db7b59496ad3f3cfef30a75142d2d930ad72651" integrity sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw== jsonpointer@^5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/jsonpointer/-/jsonpointer-5.0.1.tgz#2110e0af0900fd37467b5907ecd13a7884a1b559" integrity sha512-p/nXbhSEcu3pZRdkW1OfJhpsVtW1gd4Wa1fnQc9YLiTfAjn0312eMKimbdIQzuZl9aa9xUGaRlP9T/CJE/ditQ== jszip@^3.7.1: version "3.10.1" resolved "https://registry.yarnpkg.com/jszip/-/jszip-3.10.1.tgz#34aee70eb18ea1faec2f589208a157d1feb091c2" integrity sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g== dependencies: lie "~3.3.0" pako "~1.0.2" readable-stream "~2.3.6" setimmediate "^1.0.5" keyv@^4.5.4: version "4.5.4" resolved "https://registry.yarnpkg.com/keyv/-/keyv-4.5.4.tgz#a879a99e29452f942439f2a405e3af8b31d4de93" integrity sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw== dependencies: json-buffer "3.0.1" kuler@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/kuler/-/kuler-2.0.0.tgz#e2c570a3800388fb44407e851531c1d670b061b3" integrity sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A== langchain@0.1.36: version "0.1.36" resolved "https://registry.yarnpkg.com/langchain/-/langchain-0.1.36.tgz#2f201f681d83fb265716e28e7dfcfe61fbeef2c2" integrity sha512-NTbnCL/jKWIeEI//Nm1oG8nhW3vkYWvEMr1MPotmTThTfeKfO87eV/OAzAyh6Ruy6GFs/qofRgQZGIe6XvXTNQ== dependencies: "@anthropic-ai/sdk" "^0.9.1" "@langchain/community" "~0.0.47" "@langchain/core" "~0.1.60" "@langchain/openai" "~0.0.28" "@langchain/textsplitters" "~0.0.0" binary-extensions "^2.2.0" js-tiktoken "^1.0.7" js-yaml "^4.1.0" jsonpointer "^5.0.1" langchainhub "~0.0.8" langsmith "~0.1.7" ml-distance "^4.0.0" openapi-types "^12.1.3" p-retry "4" uuid "^9.0.0" yaml "^2.2.1" zod "^3.22.4" zod-to-json-schema "^3.22.3" langchain@~0.2.3: version "0.2.20" resolved "https://registry.yarnpkg.com/langchain/-/langchain-0.2.20.tgz#05bf348d684ece96cba73e0b8afe6b9778164857" integrity sha512-tbels6Rr524iMM3VOQ4aTGnEOOjAA1BQuBR8u/8gJ2yT48lMtIQRAN32Y4KVjKK+hEWxHHlmLBrtgLpTphFjNA== dependencies: "@langchain/core" ">=0.2.21 <0.3.0" "@langchain/openai" ">=0.1.0 <0.3.0" "@langchain/textsplitters" "~0.0.0" binary-extensions "^2.2.0" js-tiktoken "^1.0.12" js-yaml "^4.1.0" jsonpointer "^5.0.1" langsmith "^0.1.56-rc.1" openapi-types "^12.1.3" p-retry "4" uuid "^10.0.0" yaml "^2.2.1" zod "^3.22.4" zod-to-json-schema "^3.22.3" langchainhub@~0.0.8: version "0.0.11" resolved "https://registry.yarnpkg.com/langchainhub/-/langchainhub-0.0.11.tgz#2ce22def9c84699dcbd4fd4b78270d34bd2a9ae9" integrity sha512-WnKI4g9kU2bHQP136orXr2bcRdgz9iiTBpTN0jWt9IlScUKnJBoD0aa2HOzHURQKeQDnt2JwqVmQ6Depf5uDLQ== langsmith@^0.1.56-rc.1, langsmith@~0.1.1, langsmith@~0.1.30, langsmith@~0.1.7: version "0.1.68" resolved "https://registry.yarnpkg.com/langsmith/-/langsmith-0.1.68.tgz#848332e822fe5e6734a07f1c36b6530cc1798afb" integrity sha512-otmiysWtVAqzMx3CJ4PrtUBhWRG5Co8Z4o7hSZENPjlit9/j3/vm3TSvbaxpDYakZxtMjhkcJTqrdYFipISEiQ== dependencies: "@types/uuid" "^10.0.0" commander "^10.0.1" p-queue "^6.6.2" p-retry "4" semver "^7.6.3" uuid "^10.0.0" leac@^0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/leac/-/leac-0.6.0.tgz#dcf136e382e666bd2475f44a1096061b70dc0912" integrity sha512-y+SqErxb8h7nE/fiEX07jsbuhrpO9lL8eca7/Y1nuWV2moNlXhyd59iDGcRf6moVyDMbmTNzL40SUyrFU/yDpg== levn@^0.4.1: version "0.4.1" resolved "https://registry.yarnpkg.com/levn/-/levn-0.4.1.tgz#ae4562c007473b932a6200d403268dd2fffc6ade" integrity sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ== dependencies: prelude-ls "^1.2.1" type-check "~0.4.0" libbase64@1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/libbase64/-/libbase64-1.3.0.tgz#053314755a05d2e5f08bbfc48d0290e9322f4406" integrity sha512-GgOXd0Eo6phYgh0DJtjQ2tO8dc0IVINtZJeARPeiIJqge+HdsWSuaDTe8ztQ7j/cONByDZ3zeB325AHiv5O0dg== libmime@5.3.7: version "5.3.7" resolved "https://registry.yarnpkg.com/libmime/-/libmime-5.3.7.tgz#3835b6443d982d5cd1ac32ee241adbbc11b34406" integrity sha512-FlDb3Wtha8P01kTL3P9M+ZDNDWPKPmKHWaU/cG/lg5pfuAwdflVpZE+wm9m7pKmC5ww6s+zTxBKS1p6yl3KpSw== dependencies: encoding-japanese "2.2.0" iconv-lite "0.6.3" libbase64 "1.3.0" libqp "2.1.1" libqp@2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/libqp/-/libqp-2.1.1.tgz#f1be767a58f966f500597997cab72cfc1e17abfa" integrity sha512-0Wd+GPz1O134cP62YU2GTOPNA7Qgl09XwCqM5zpBv87ERCXdfDtyKXvV7c9U22yWJh44QZqBocFnXN11K96qow== lie@~3.3.0: version "3.3.0" resolved "https://registry.yarnpkg.com/lie/-/lie-3.3.0.tgz#dcf82dee545f46074daf200c7c1c5a08e0f40f6a" integrity sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ== dependencies: immediate "~3.0.5" lines-and-columns@^1.1.6: version "1.2.4" resolved "https://registry.yarnpkg.com/lines-and-columns/-/lines-and-columns-1.2.4.tgz#eca284f75d2965079309dc0ad9255abb2ebc1632" integrity sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg== linkify-it@5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/linkify-it/-/linkify-it-5.0.0.tgz#9ef238bfa6dc70bd8e7f9572b52d369af569b421" integrity sha512-5aHCbzQRADcdP+ATqnDuhhJ/MRIqDkZX5pyjFHRRysS8vZ5AbqGEoFIb6pYHPZ+L/OC2Lc+xT8uHVVR5CAK/wQ== dependencies: uc.micro "^2.0.0" locate-path@^6.0.0: version "6.0.0" resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-6.0.0.tgz#55321eb309febbc59c4801d931a72452a681d286" integrity sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw== dependencies: p-locate "^5.0.0" lodash.merge@^4.6.2: version "4.6.2" resolved "https://registry.yarnpkg.com/lodash.merge/-/lodash.merge-4.6.2.tgz#558aa53b43b661e1925a0afdfa36a9a1085fe57a" integrity sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ== lodash@^4.17.21: version "4.17.21" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== logform@^2.7.0: version "2.7.0" resolved "https://registry.yarnpkg.com/logform/-/logform-2.7.0.tgz#cfca97528ef290f2e125a08396805002b2d060d1" integrity sha512-TFYA4jnP7PVbmlBIfhlSe+WKxs9dklXMTEGcBCIvLhE/Tn3H6Gk1norupVW7m5Cnd4bLcr08AytbyV/xj7f/kQ== dependencies: "@colors/colors" "1.6.0" "@types/triple-beam" "^1.3.2" fecha "^4.2.0" ms "^2.1.1" safe-stable-stringify "^2.3.1" triple-beam "^1.3.0" long@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/long/-/long-4.0.0.tgz#9a7b71cfb7d361a194ea555241c92f7468d5bf28" integrity sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA== lop@^0.4.2: version "0.4.2" resolved "https://registry.yarnpkg.com/lop/-/lop-0.4.2.tgz#c9c2f958a39b9da1c2f36ca9ad66891a9fe84640" integrity sha512-RefILVDQ4DKoRZsJ4Pj22TxE3omDO47yFpkIBoDKzkqPRISs5U1cnAdg/5583YPkWPaLIYHOKRMQSvjFsO26cw== dependencies: duck "^0.1.12" option "~0.2.1" underscore "^1.13.1" lru-cache@^10.2.0: version "10.4.3" resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-10.4.3.tgz#410fc8a17b70e598013df257c2446b7f3383f119" integrity sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ== lru-cache@^7.14.1: version "7.18.3" resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-7.18.3.tgz#f793896e0fd0e954a59dfdd82f0773808df6aa89" integrity sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA== mailparser@^3.0.1: version "3.9.0" resolved "https://registry.yarnpkg.com/mailparser/-/mailparser-3.9.0.tgz#3b71732a26aa7c3390cd39540713eed7daf0aa2c" integrity sha512-jpaNLhDjwy0w2f8sySOSRiWREjPqssSc0C2czV98btCXCRX3EyNloQ2IWirmMDj1Ies8Fkm0l96bZBZpDG7qkg== dependencies: "@zone-eu/mailsplit" "5.4.7" encoding-japanese "2.2.0" he "1.2.0" html-to-text "9.0.5" iconv-lite "0.7.0" libmime "5.3.7" linkify-it "5.0.0" nodemailer "7.0.10" punycode.js "2.3.1" tlds "1.261.0" make-dir@^1.0.0: version "1.3.0" resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-1.3.0.tgz#79c1033b80515bd6d24ec9933e860ca75ee27f0c" integrity sha512-2w31R7SJtieJJnQtGc7RVL2StM2vGYVfqUOvUDxH6bC6aJTxPxTF0GnIgCyu7tjockiUWAYQRbxa7vKn34s5sQ== dependencies: pify "^3.0.0" mammoth@^1.6.0: version "1.11.0" resolved "https://registry.yarnpkg.com/mammoth/-/mammoth-1.11.0.tgz#f6c68624eaffcf56728a792fcccd3495d688bac5" integrity sha512-BcEqqY/BOwIcI1iR5tqyVlqc3KIaMRa4egSoK83YAVrBf6+yqdAAbtUcFDCWX8Zef8/fgNZ6rl4VUv+vVX8ddQ== dependencies: "@xmldom/xmldom" "^0.8.6" argparse "~1.0.3" base64-js "^1.5.1" bluebird "~3.4.0" dingbat-to-unicode "^1.0.1" jszip "^3.7.1" lop "^0.4.2" path-is-absolute "^1.0.0" underscore "^1.13.1" xmlbuilder "^10.0.0" math-intrinsics@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/math-intrinsics/-/math-intrinsics-1.1.0.tgz#a0dd74be81e2aa5c2f27e65ce283605ee4e2b7f9" integrity sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g== mbox-parser@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/mbox-parser/-/mbox-parser-1.0.1.tgz#cba6bb372a1d3c27aff9da356b4519aeee458e1f" integrity sha512-9PNE026G/SodrYYxZiDZDJ5YQ0piN8QCqO662MupIGwgsvBtJHFPxYc/uUsQyhUysl4kMG0NLYvfELNkM5+HzQ== dependencies: "@types/mailparser" "^3.0.0" humanize-duration "^3.25.1" mailparser "^3.0.1" md5@^2.3.0: version "2.3.0" resolved "https://registry.yarnpkg.com/md5/-/md5-2.3.0.tgz#c3da9a6aae3a30b46b7b0c349b87b110dc3bda4f" integrity sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g== dependencies: charenc "0.0.2" crypt "0.0.2" is-buffer "~1.1.6" media-typer@0.3.0: version "0.3.0" resolved "https://registry.yarnpkg.com/media-typer/-/media-typer-0.3.0.tgz#8710d7af0aa626f8fffa1ce00168545263255748" integrity sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ== merge-descriptors@1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/merge-descriptors/-/merge-descriptors-1.0.3.tgz#d80319a65f3c7935351e5cfdac8f9318504dbed5" integrity sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ== merge-stream@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/merge-stream/-/merge-stream-2.0.0.tgz#52823629a14dd00c9770fb6ad47dc6310f2c1f60" integrity sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w== methods@~1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/methods/-/methods-1.1.2.tgz#5529a4d67654134edcc5266656835b0f851afcee" integrity sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w== mime-db@1.52.0: version "1.52.0" resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.52.0.tgz#bbabcdc02859f4987301c856e3387ce5ec43bf70" integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg== mime-types@^2.1.12, mime-types@~2.1.24, mime-types@~2.1.34: version "2.1.35" resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.35.tgz#381a871b62a734450660ae3deee44813f70d959a" integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw== dependencies: mime-db "1.52.0" mime@1.6.0: version "1.6.0" resolved "https://registry.yarnpkg.com/mime/-/mime-1.6.0.tgz#32cd9e5c64553bd58d19a568af452acff04981b1" integrity sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg== mime@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/mime/-/mime-3.0.0.tgz#b374550dca3a0c18443b0c950a6a58f1931cf7a7" integrity sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A== mimic-fn@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-2.1.0.tgz#7ed2c2ccccaf84d3ffcb7a69b57711fc2083401b" integrity sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg== mimic-response@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-3.1.0.tgz#2d1d59af9c1b129815accc2c46a022a5ce1fa3c9" integrity sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ== minimatch@^3.1.2: version "3.1.2" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.1.2.tgz#19cd194bfd3e428f049a70817c038d89ab4be35b" integrity sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw== dependencies: brace-expansion "^1.1.7" minimatch@^3.1.3: version "3.1.5" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.1.5.tgz#580c88f8d5445f2bd6aa8f3cadefa0de79fbd69e" integrity sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w== dependencies: brace-expansion "^1.1.7" minimatch@^9.0.4: version "9.0.5" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-9.0.5.tgz#d74f9dd6b57d83d8e98cfb82133b03978bc929e5" integrity sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow== dependencies: brace-expansion "^2.0.1" minimist@^1.2.0, minimist@^1.2.3: version "1.2.8" resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.8.tgz#c1a464e7693302e082a075cee0c057741ac4772c" integrity sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA== "minipass@^5.0.0 || ^6.0.2 || ^7.0.0", minipass@^7.1.2: version "7.1.2" resolved "https://registry.yarnpkg.com/minipass/-/minipass-7.1.2.tgz#93a9626ce5e5e66bd4db86849e7515e92340a707" integrity sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw== mitt@3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/mitt/-/mitt-3.0.1.tgz#ea36cf0cc30403601ae074c8f77b7092cdab36d1" integrity sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw== mkdirp-classic@^0.5.2, mkdirp-classic@^0.5.3: version "0.5.3" resolved "https://registry.yarnpkg.com/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz#fa10c9115cc6d8865be221ba47ee9bed78601113" integrity sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A== ml-array-mean@^1.1.6: version "1.1.6" resolved "https://registry.yarnpkg.com/ml-array-mean/-/ml-array-mean-1.1.6.tgz#d951a700dc8e3a17b3e0a583c2c64abd0c619c56" integrity sha512-MIdf7Zc8HznwIisyiJGRH9tRigg3Yf4FldW8DxKxpCCv/g5CafTw0RRu51nojVEOXuCQC7DRVVu5c7XXO/5joQ== dependencies: ml-array-sum "^1.1.6" ml-array-sum@^1.1.6: version "1.1.6" resolved "https://registry.yarnpkg.com/ml-array-sum/-/ml-array-sum-1.1.6.tgz#d1d89c20793cd29c37b09d40e85681aa4515a955" integrity sha512-29mAh2GwH7ZmiRnup4UyibQZB9+ZLyMShvt4cH4eTK+cL2oEMIZFnSyB3SS8MlsTh6q/w/yh48KmqLxmovN4Dw== dependencies: is-any-array "^2.0.0" ml-distance-euclidean@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/ml-distance-euclidean/-/ml-distance-euclidean-2.0.0.tgz#3a668d236649d1b8fec96380b9435c6f42c9a817" integrity sha512-yC9/2o8QF0A3m/0IXqCTXCzz2pNEzvmcE/9HFKOZGnTjatvBbsn4lWYJkxENkA4Ug2fnYl7PXQxnPi21sgMy/Q== ml-distance@^4.0.0: version "4.0.1" resolved "https://registry.yarnpkg.com/ml-distance/-/ml-distance-4.0.1.tgz#4741d17a1735888c5388823762271dfe604bd019" integrity sha512-feZ5ziXs01zhyFUUUeZV5hwc0f5JW0Sh0ckU1koZe/wdVkJdGxcP06KNQuF0WBTj8FttQUzcvQcpcrOp/XrlEw== dependencies: ml-array-mean "^1.1.6" ml-distance-euclidean "^2.0.0" ml-tree-similarity "^1.0.0" ml-tree-similarity@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/ml-tree-similarity/-/ml-tree-similarity-1.0.0.tgz#24705a107e32829e24d945e87219e892159c53f0" integrity sha512-XJUyYqjSuUQkNQHMscr6tcjldsOoAekxADTplt40QKfwW6nd++1wHWV9AArl0Zvw/TIHgNaZZNvr8QGvE8wLRg== dependencies: binary-search "^1.3.5" num-sort "^2.0.0" moment@^2.29.4: version "2.30.1" resolved "https://registry.yarnpkg.com/moment/-/moment-2.30.1.tgz#f8c91c07b7a786e30c59926df530b4eac96974ae" integrity sha512-uEmtNhbDOrWPFS+hdjFCBfy9f2YoyzRpwcl+DqpC6taX21FzsTLQVbMV/W7PzNSX6x/bhC1zA3c2UQ5NzH6how== ms@2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" integrity sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A== ms@2.1.2: version "2.1.2" resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009" integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== ms@2.1.3, ms@^2.0.0, ms@^2.1.1, ms@^2.1.3: version "2.1.3" resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== mustache@^4.2.0: version "4.2.0" resolved "https://registry.yarnpkg.com/mustache/-/mustache-4.2.0.tgz#e5892324d60a12ec9c2a73359edca52972bf6f64" integrity sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ== napi-build-utils@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/napi-build-utils/-/napi-build-utils-2.0.0.tgz#13c22c0187fcfccce1461844136372a47ddc027e" integrity sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA== natural-compare@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7" integrity sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw== negotiator@0.6.3: version "0.6.3" resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.3.tgz#58e323a72fedc0d6f9cd4d31fe49f51479590ccd" integrity sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg== netmask@^2.0.2: version "2.0.2" resolved "https://registry.yarnpkg.com/netmask/-/netmask-2.0.2.tgz#8b01a07644065d536383835823bc52004ebac5e7" integrity sha512-dBpDMdxv9Irdq66304OLfEmQ9tbNRFnFTuZiLo+bD+r332bBmMJ8GBLXklIXXgxd3+v9+KUnZaUR5PJMa75Gsg== node-abi@^3.3.0: version "3.85.0" resolved "https://registry.yarnpkg.com/node-abi/-/node-abi-3.85.0.tgz#b115d575e52b2495ef08372b058e13d202875a7d" integrity sha512-zsFhmbkAzwhTft6nd3VxcG0cvJsT70rL+BIGHWVq5fi6MwGrHwzqKaxXE+Hl2GmnGItnDKPPkO5/LQqjVkIdFg== dependencies: semver "^7.3.5" node-addon-api@^6.1.0: version "6.1.0" resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-6.1.0.tgz#ac8470034e58e67d0c6f1204a18ae6995d9c0d76" integrity sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA== node-domexception@1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/node-domexception/-/node-domexception-1.0.0.tgz#6888db46a1f71c0b76b3f7555016b63fe64766e5" integrity sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ== node-ensure@^0.0.0: version "0.0.0" resolved "https://registry.yarnpkg.com/node-ensure/-/node-ensure-0.0.0.tgz#ecae764150de99861ec5c810fd5d096b183932a7" integrity sha512-DRI60hzo2oKN1ma0ckc6nQWlHU69RH6xN0sjQTjMpChPfTYvKZdcQFfdYK2RWbJcKyUizSIy/l8OTGxMAM1QDw== node-fetch@^2.6.12, node-fetch@^2.6.7, node-fetch@^2.6.9: version "2.7.0" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d" integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A== dependencies: whatwg-url "^5.0.0" node-html-parser@^6.1.13: version "6.1.13" resolved "https://registry.yarnpkg.com/node-html-parser/-/node-html-parser-6.1.13.tgz#a1df799b83df5c6743fcd92740ba14682083b7e4" integrity sha512-qIsTMOY4C/dAa5Q5vsobRpOOvPfC4pB61UVW2uSwZNUp0QU/jCekTal1vMmbO0DgdHeLUJpv/ARmDqErVxA3Sg== dependencies: css-select "^5.1.0" he "1.2.0" node-xlsx@^0.24.0: version "0.24.0" resolved "https://registry.yarnpkg.com/node-xlsx/-/node-xlsx-0.24.0.tgz#a6a365acb18ad37c66c2b254b6ebe0c22dc9dc6f" integrity sha512-1olwK48XK9nXZsyH/FCltvGrQYvXXZuxVitxXXv2GIuRm51aBi1+5KwR4rWM4KeO61sFU+00913WLZTD+AcXEg== dependencies: xlsx "https://cdn.sheetjs.com/xlsx-0.20.2/xlsx-0.20.2.tgz" nodemailer@7.0.10: version "7.0.10" resolved "https://registry.yarnpkg.com/nodemailer/-/nodemailer-7.0.10.tgz#540062dbbe574220b42e79d2d949956d3eac5a46" integrity sha512-Us/Se1WtT0ylXgNFfyFSx4LElllVLJXQjWi2Xz17xWw7amDKO2MLtFnVp1WACy7GkVGs+oBlRopVNUzlrGSw1w== nodemon@^2.0.22: version "2.0.22" resolved "https://registry.yarnpkg.com/nodemon/-/nodemon-2.0.22.tgz#182c45c3a78da486f673d6c1702e00728daf5258" integrity sha512-B8YqaKMmyuCO7BowF1Z1/mkPqLk6cs/l63Ojtd6otKjMx47Dq1utxfRxcavH1I7VSaL8n5BUaoutadnsX3AAVQ== dependencies: chokidar "^3.5.2" debug "^3.2.7" ignore-by-default "^1.0.1" minimatch "^3.1.2" pstree.remy "^1.1.8" semver "^5.7.1" simple-update-notifier "^1.0.7" supports-color "^5.5.0" touch "^3.1.0" undefsafe "^2.0.5" normalize-path@^3.0.0, normalize-path@~3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/normalize-path/-/normalize-path-3.0.0.tgz#0dcd69ff23a1c9b11fd0978316644a0388216a65" integrity sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA== npm-run-path@^4.0.1: version "4.0.1" resolved "https://registry.yarnpkg.com/npm-run-path/-/npm-run-path-4.0.1.tgz#b7ecd1e5ed53da8e37a55e1c2269e0b97ed748ea" integrity sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw== dependencies: path-key "^3.0.0" nth-check@^2.0.1: version "2.1.1" resolved "https://registry.yarnpkg.com/nth-check/-/nth-check-2.1.1.tgz#c9eab428effce36cd6b92c924bdb000ef1f1ed1d" integrity sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w== dependencies: boolbase "^1.0.0" num-sort@^2.0.0: version "2.1.0" resolved "https://registry.yarnpkg.com/num-sort/-/num-sort-2.1.0.tgz#1cbb37aed071329fdf41151258bc011898577a9b" integrity sha512-1MQz1Ed8z2yckoBeSfkQHHO9K1yDRxxtotKSJ9yvcTUUxSvfvzEq5GwBrjjHEpMlq/k5gvXdmJ1SbYxWtpNoVg== object-assign@^4, object-assign@^4.0.1: version "4.1.1" resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" integrity sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg== object-inspect@^1.13.3: version "1.13.4" resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.13.4.tgz#8375265e21bc20d0fa582c22e1b13485d6e00213" integrity sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew== officeparser@^4.0.5: version "4.2.0" resolved "https://registry.yarnpkg.com/officeparser/-/officeparser-4.2.0.tgz#6d0baf411bd21e7e31a4b1f206af7034a4506ca2" integrity sha512-LXSfaET8ZOBNjmSev4K1N6AiKTaY7m9NkddeCaMUdEe5D/HUuv2byB8VoPIaiLldtKun0I92tbhO+VGDUr/aXQ== dependencies: "@xmldom/xmldom" "^0.8.10" decompress "^4.2.1" file-type "^16.5.4" node-ensure "^0.0.0" rimraf "^5.0.10" on-finished@2.4.1, on-finished@~2.4.1: version "2.4.1" resolved "https://registry.yarnpkg.com/on-finished/-/on-finished-2.4.1.tgz#58c8c44116e54845ad57f14ab10b03533184ac3f" integrity sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg== dependencies: ee-first "1.1.1" once@^1.3.1, once@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1" integrity sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w== dependencies: wrappy "1" one-time@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/one-time/-/one-time-1.0.0.tgz#e06bc174aed214ed58edede573b433bbf827cb45" integrity sha512-5DXOiRKwuSEcQ/l0kGCF6Q3jcADFv5tSmRaJck/OqkVFcOzutB134KRSfF0xDrL39MNnqxbHBbUUcjZIhTgb2g== dependencies: fn.name "1.x.x" onetime@^5.1.2: version "5.1.2" resolved "https://registry.yarnpkg.com/onetime/-/onetime-5.1.2.tgz#d0e96ebb56b07476df1dd9c4806e5237985ca45e" integrity sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg== dependencies: mimic-fn "^2.1.0" onnx-proto@^4.0.4: version "4.0.4" resolved "https://registry.yarnpkg.com/onnx-proto/-/onnx-proto-4.0.4.tgz#2431a25bee25148e915906dda0687aafe3b9e044" integrity sha512-aldMOB3HRoo6q/phyB6QRQxSt895HNNw82BNyZ2CMh4bjeKv7g/c+VpAFtJuEMVfYLMbRx61hbuqnKceLeDcDA== dependencies: protobufjs "^6.8.8" onnxruntime-common@~1.14.0: version "1.14.0" resolved "https://registry.yarnpkg.com/onnxruntime-common/-/onnxruntime-common-1.14.0.tgz#2bb5dac5261269779aa5fb6536ca379657de8bf6" integrity sha512-3LJpegM2iMNRX2wUmtYfeX/ytfOzNwAWKSq1HbRrKc9+uqG/FsEA0bbKZl1btQeZaXhC26l44NWpNUeXPII7Ew== onnxruntime-node@1.14.0: version "1.14.0" resolved "https://registry.yarnpkg.com/onnxruntime-node/-/onnxruntime-node-1.14.0.tgz#c4ae6c355cfae7d83abaf36dd39a905c4a010217" integrity sha512-5ba7TWomIV/9b6NH/1x/8QEeowsb+jBEvFzU6z0T4mNsFwdPqXeFUM7uxC6QeSRkEbWu3qEB0VMjrvzN/0S9+w== dependencies: onnxruntime-common "~1.14.0" onnxruntime-web@1.14.0: version "1.14.0" resolved "https://registry.yarnpkg.com/onnxruntime-web/-/onnxruntime-web-1.14.0.tgz#c8cee538781b1d4c1c6b043934f4a3e6ddf1466e" integrity sha512-Kcqf43UMfW8mCydVGcX9OMXI2VN17c0p6XvR7IPSZzBf/6lteBzXHvcEVWDPmCKuGombl997HgLqj91F11DzXw== dependencies: flatbuffers "^1.12.0" guid-typescript "^1.0.9" long "^4.0.0" onnx-proto "^4.0.4" onnxruntime-common "~1.14.0" platform "^1.3.6" openai@4.95.1: version "4.95.1" resolved "https://registry.yarnpkg.com/openai/-/openai-4.95.1.tgz#7157697c2b150a546b13eb860180c4a6058051da" integrity sha512-IqJy+ymeW+k/Wq+2YVN3693OQMMcODRtHEYOlz263MdUwnN/Dwdl9c2EXSxLLtGEHkSHAfvzpDMHI5MaWJKXjQ== dependencies: "@types/node" "^18.11.18" "@types/node-fetch" "^2.6.4" abort-controller "^3.0.0" agentkeepalive "^4.2.1" form-data-encoder "1.7.2" formdata-node "^4.3.2" node-fetch "^2.6.7" openai@^4.41.1, openai@^4.57.3: version "4.104.0" resolved "https://registry.yarnpkg.com/openai/-/openai-4.104.0.tgz#c489765dc051b95019845dab64b0e5207cae4d30" integrity sha512-p99EFNsA/yX6UhVO93f5kJsDRLAg+CTA2RBqdHK4RtK8u5IJw32Hyb2dTGKbnnFmnuoBv5r7Z2CURI9sGZpSuA== dependencies: "@types/node" "^18.11.18" "@types/node-fetch" "^2.6.4" abort-controller "^3.0.0" agentkeepalive "^4.2.1" form-data-encoder "1.7.2" formdata-node "^4.3.2" node-fetch "^2.6.7" openapi-types@^12.1.3: version "12.1.3" resolved "https://registry.yarnpkg.com/openapi-types/-/openapi-types-12.1.3.tgz#471995eb26c4b97b7bd356aacf7b91b73e777dd3" integrity sha512-N4YtSYJqghVu4iek2ZUvcN/0aqH1kRDuNqzcycDxhOUpg7GdvLa2F3DgS6yBNhInhv2r/6I0Flkn7CqL8+nIcw== opencollective-postinstall@^2.0.3: version "2.0.3" resolved "https://registry.yarnpkg.com/opencollective-postinstall/-/opencollective-postinstall-2.0.3.tgz#7a0fff978f6dbfa4d006238fbac98ed4198c3259" integrity sha512-8AV/sCtuzUeTo8gQK5qDZzARrulB3egtLzFgteqB2tcT4Mw7B8Kt7JcDHmltjz6FOAHsvTevk70gZEbhM4ZS9Q== option@~0.2.1: version "0.2.4" resolved "https://registry.yarnpkg.com/option/-/option-0.2.4.tgz#fd475cdf98dcabb3cb397a3ba5284feb45edbfe4" integrity sha512-pkEqbDyl8ou5cpq+VsnQbe/WlEy5qS7xPzMS1U55OCG9KPvwFD46zDbxQIj3egJSFc3D+XhYOPUzz49zQAVy7A== optionator@^0.9.3: version "0.9.4" resolved "https://registry.yarnpkg.com/optionator/-/optionator-0.9.4.tgz#7ea1c1a5d91d764fb282139c88fe11e182a3a734" integrity sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g== dependencies: deep-is "^0.1.3" fast-levenshtein "^2.0.6" levn "^0.4.1" prelude-ls "^1.2.1" type-check "^0.4.0" word-wrap "^1.2.5" p-finally@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/p-finally/-/p-finally-1.0.0.tgz#3fbcfb15b899a44123b34b6dcc18b724336a2cae" integrity sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow== p-limit@^3.0.2: version "3.1.0" resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-3.1.0.tgz#e1daccbe78d0d1388ca18c64fea38e3e57e3706b" integrity sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ== dependencies: yocto-queue "^0.1.0" p-locate@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-5.0.0.tgz#83c8315c6785005e3bd021839411c9e110e6d834" integrity sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw== dependencies: p-limit "^3.0.2" p-queue@^6.6.2: version "6.6.2" resolved "https://registry.yarnpkg.com/p-queue/-/p-queue-6.6.2.tgz#2068a9dcf8e67dd0ec3e7a2bcb76810faa85e426" integrity sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ== dependencies: eventemitter3 "^4.0.4" p-timeout "^3.2.0" p-retry@4: version "4.6.2" resolved "https://registry.yarnpkg.com/p-retry/-/p-retry-4.6.2.tgz#9baae7184057edd4e17231cee04264106e092a16" integrity sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ== dependencies: "@types/retry" "0.12.0" retry "^0.13.1" p-timeout@^3.2.0: version "3.2.0" resolved "https://registry.yarnpkg.com/p-timeout/-/p-timeout-3.2.0.tgz#c7e17abc971d2a7962ef83626b35d635acf23dfe" integrity sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg== dependencies: p-finally "^1.0.0" pac-proxy-agent@^7.0.1: version "7.2.0" resolved "https://registry.yarnpkg.com/pac-proxy-agent/-/pac-proxy-agent-7.2.0.tgz#9cfaf33ff25da36f6147a20844230ec92c06e5df" integrity sha512-TEB8ESquiLMc0lV8vcd5Ql/JAKAoyzHFXaStwjkzpOpC5Yv+pIzLfHvjTSdf3vpa2bMiUQrg9i6276yn8666aA== dependencies: "@tootallnate/quickjs-emscripten" "^0.23.0" agent-base "^7.1.2" debug "^4.3.4" get-uri "^6.0.1" http-proxy-agent "^7.0.0" https-proxy-agent "^7.0.6" pac-resolver "^7.0.1" socks-proxy-agent "^8.0.5" pac-resolver@^7.0.1: version "7.0.1" resolved "https://registry.yarnpkg.com/pac-resolver/-/pac-resolver-7.0.1.tgz#54675558ea368b64d210fd9c92a640b5f3b8abb6" integrity sha512-5NPgf87AT2STgwa2ntRMr45jTKrYBGkVU36yT0ig/n/GMAa3oPqhZfIQ2kMEimReg0+t9kZViDVZ83qfVUlckg== dependencies: degenerator "^5.0.0" netmask "^2.0.2" package-json-from-dist@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz#4f1471a010827a86f94cfd9b0727e36d267de505" integrity sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw== pako@~1.0.2: version "1.0.11" resolved "https://registry.yarnpkg.com/pako/-/pako-1.0.11.tgz#6c9599d340d54dfd3946380252a35705a6b992bf" integrity sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw== parent-module@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/parent-module/-/parent-module-1.0.1.tgz#691d2709e78c79fae3a156622452d00762caaaa2" integrity sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g== dependencies: callsites "^3.0.0" parse-json@^5.2.0: version "5.2.0" resolved "https://registry.yarnpkg.com/parse-json/-/parse-json-5.2.0.tgz#c76fc66dee54231c962b22bcc8a72cf2f99753cd" integrity sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg== dependencies: "@babel/code-frame" "^7.0.0" error-ex "^1.3.1" json-parse-even-better-errors "^2.3.0" lines-and-columns "^1.1.6" parseley@^0.12.0: version "0.12.1" resolved "https://registry.yarnpkg.com/parseley/-/parseley-0.12.1.tgz#4afd561d50215ebe259e3e7a853e62f600683aef" integrity sha512-e6qHKe3a9HWr0oMRVDTRhKce+bRO8VGQR3NyVwcjwrbhMmFCX9KszEV35+rn4AdilFAq9VPxP/Fe1wC9Qjd2lw== dependencies: leac "^0.6.0" peberminta "^0.9.0" parseurl@~1.3.3: version "1.3.3" resolved "https://registry.yarnpkg.com/parseurl/-/parseurl-1.3.3.tgz#9da19e7bee8d12dff0513ed5b76957793bc2e8d4" integrity sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ== path-exists@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-4.0.0.tgz#513bdbe2d3b95d7762e8c1137efa195c6c61b5b3" integrity sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w== path-is-absolute@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f" integrity sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg== path-key@^3.0.0, path-key@^3.1.0: version "3.1.1" resolved "https://registry.yarnpkg.com/path-key/-/path-key-3.1.1.tgz#581f6ade658cbba65a0d3380de7753295054f375" integrity sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q== path-scurry@^1.11.1: version "1.11.1" resolved "https://registry.yarnpkg.com/path-scurry/-/path-scurry-1.11.1.tgz#7960a668888594a0720b12a911d1a742ab9f11d2" integrity sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA== dependencies: lru-cache "^10.2.0" minipass "^5.0.0 || ^6.0.2 || ^7.0.0" path-to-regexp@~0.1.12: version "0.1.12" resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-0.1.12.tgz#d5e1a12e478a976d432ef3c58d534b9923164bb7" integrity sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ== path-type@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/path-type/-/path-type-4.0.0.tgz#84ed01c0a7ba380afe09d90a8c180dcd9d03043b" integrity sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw== pdf-parse@^1.1.1: version "1.1.4" resolved "https://registry.yarnpkg.com/pdf-parse/-/pdf-parse-1.1.4.tgz#97bca6f46758130dafb1fdd9df905efd07581f4a" integrity sha512-XRIRcLgk6ZnUbsHsYXExMw+krrPE81hJ6FQPLdBNhhBefqIQKXu/WeTgNBGSwPrfU0v+UCEwn7AoAUOsVKHFvQ== dependencies: node-ensure "^0.0.0" peberminta@^0.9.0: version "0.9.0" resolved "https://registry.yarnpkg.com/peberminta/-/peberminta-0.9.0.tgz#8ec9bc0eb84b7d368126e71ce9033501dca2a352" integrity sha512-XIxfHpEuSJbITd1H3EeQwpcZbTLHc+VVr8ANI9t5sit565tsI4/xK3KWTUFE2e6QiangUkh3B0jihzmGnNrRsQ== peek-readable@^4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/peek-readable/-/peek-readable-4.1.0.tgz#4ece1111bf5c2ad8867c314c81356847e8a62e72" integrity sha512-ZI3LnwUv5nOGbQzD9c2iDG6toheuXSZP5esSHBjopsXH4dg19soufvpUGA3uohi5anFtGb2lhAVdHzH6R/Evvg== pend@~1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/pend/-/pend-1.2.0.tgz#7a57eb550a6783f9115331fcf4663d5c8e007a50" integrity sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg== picocolors@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.1.1.tgz#3d321af3eab939b083c8f929a1d12cda81c26b6b" integrity sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA== picomatch@^2.0.4, picomatch@^2.2.1: version "2.3.1" resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.1.tgz#3ba3833733646d9d3e4995946c1365a67fb07a42" integrity sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA== pify@^2.3.0: version "2.3.0" resolved "https://registry.yarnpkg.com/pify/-/pify-2.3.0.tgz#ed141a6ac043a849ea588498e7dca8b15330e90c" integrity sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog== pify@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/pify/-/pify-3.0.0.tgz#e5a4acd2c101fdf3d9a4d07f0dbc4db49dd28176" integrity sha512-C3FsVNH1udSEX48gGX1xfvwTWfsYWj5U+8/uK15BGzIGrKoUpghX8hWZwa/OFnakBiiVNmBvemTJR5mcy7iPcg== pinkie-promise@^2.0.0: version "2.0.1" resolved "https://registry.yarnpkg.com/pinkie-promise/-/pinkie-promise-2.0.1.tgz#2135d6dfa7a358c069ac9b178776288228450ffa" integrity sha512-0Gni6D4UcLTbv9c57DfxDGdr41XfgUjqWZu492f0cIGr16zDU06BWP/RAEvOuo7CQ0CNjHaLlM59YJJFm3NWlw== dependencies: pinkie "^2.0.0" pinkie@^2.0.0: version "2.0.4" resolved "https://registry.yarnpkg.com/pinkie/-/pinkie-2.0.4.tgz#72556b80cfa0d48a974e80e77248e80ed4f7f870" integrity sha512-MnUuEycAemtSaeFSjXKW/aroV7akBbY+Sv+RkyqFjgAe73F+MR0TBWKBRDkmfWq/HiFmdavfZ1G7h4SPZXaCSg== platform@^1.3.6: version "1.3.6" resolved "https://registry.yarnpkg.com/platform/-/platform-1.3.6.tgz#48b4ce983164b209c2d45a107adb31f473a6e7a7" integrity sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg== possible-typed-array-names@^1.0.0: version "1.1.0" resolved "https://registry.yarnpkg.com/possible-typed-array-names/-/possible-typed-array-names-1.1.0.tgz#93e3582bc0e5426586d9d07b79ee40fc841de4ae" integrity sha512-/+5VFTchJDoVj3bhoqi6UeymcD00DAwb1nJwamzPvHEszJ4FpF6SNNbUbOS8yI56qHzdV8eK0qEfOSiodkTdxg== prebuild-install@^7.1.1: version "7.1.3" resolved "https://registry.yarnpkg.com/prebuild-install/-/prebuild-install-7.1.3.tgz#d630abad2b147443f20a212917beae68b8092eec" integrity sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug== dependencies: detect-libc "^2.0.0" expand-template "^2.0.3" github-from-package "0.0.0" minimist "^1.2.3" mkdirp-classic "^0.5.3" napi-build-utils "^2.0.0" node-abi "^3.3.0" pump "^3.0.0" rc "^1.2.7" simple-get "^4.0.0" tar-fs "^2.0.0" tunnel-agent "^0.6.0" prelude-ls@^1.2.1: version "1.2.1" resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.2.1.tgz#debc6489d7a6e6b0e7611888cec880337d316396" integrity sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g== prettier-linter-helpers@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/prettier-linter-helpers/-/prettier-linter-helpers-1.0.1.tgz#6a31f88a4bad6c7adda253de12ba4edaea80ebcd" integrity sha512-SxToR7P8Y2lWmv/kTzVLC1t/GDI2WGjMwNhLLE9qtH8Q13C+aEmuRlzDst4Up4s0Wc8sF2M+J57iB3cMLqftfg== dependencies: fast-diff "^1.1.2" prettier@^2.4.1: version "2.8.8" resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.8.8.tgz#e8c5d7e98a4305ffe3de2e1fc4aca1a71c28b1da" integrity sha512-tdN8qQGvNjw4CHbY+XXk0JgCXn9QiF21a55rBe5LJAU+kDyC4WQn4+awm2Xfk2lQMk5fKup9XgzTZtGkjBdP9Q== process-nextick-args@~2.0.0: version "2.0.1" resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2" integrity sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag== process@^0.11.10: version "0.11.10" resolved "https://registry.yarnpkg.com/process/-/process-0.11.10.tgz#7332300e840161bda3e69a1d1d91a7d4bc16f182" integrity sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A== progress@2.0.3: version "2.0.3" resolved "https://registry.yarnpkg.com/progress/-/progress-2.0.3.tgz#7e8cf8d8f5b8f239c1bc68beb4eb78567d572ef8" integrity sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA== protobufjs@^6.8.8: version "6.11.4" resolved "https://registry.yarnpkg.com/protobufjs/-/protobufjs-6.11.4.tgz#29a412c38bf70d89e537b6d02d904a6f448173aa" integrity sha512-5kQWPaJHi1WoCpjTGszzQ32PG2F4+wRY6BmAT4Vfw56Q2FZ4YZzK20xUYQH4YkfehY1e6QSICrJquM6xXZNcrw== dependencies: "@protobufjs/aspromise" "^1.1.2" "@protobufjs/base64" "^1.1.2" "@protobufjs/codegen" "^2.0.4" "@protobufjs/eventemitter" "^1.1.0" "@protobufjs/fetch" "^1.1.0" "@protobufjs/float" "^1.0.2" "@protobufjs/inquire" "^1.1.0" "@protobufjs/path" "^1.1.2" "@protobufjs/pool" "^1.1.0" "@protobufjs/utf8" "^1.1.0" "@types/long" "^4.0.1" "@types/node" ">=13.7.0" long "^4.0.0" proxy-addr@~2.0.7: version "2.0.7" resolved "https://registry.yarnpkg.com/proxy-addr/-/proxy-addr-2.0.7.tgz#f19fe69ceab311eeb94b42e70e8c2070f9ba1025" integrity sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg== dependencies: forwarded "0.2.0" ipaddr.js "1.9.1" proxy-agent@6.3.1: version "6.3.1" resolved "https://registry.yarnpkg.com/proxy-agent/-/proxy-agent-6.3.1.tgz#40e7b230552cf44fd23ffaf7c59024b692612687" integrity sha512-Rb5RVBy1iyqOtNl15Cw/llpeLH8bsb37gM1FUfKQ+Wck6xHlbAhWGUFiTRHtkjqGTA5pSHz6+0hrPW/oECihPQ== dependencies: agent-base "^7.0.2" debug "^4.3.4" http-proxy-agent "^7.0.0" https-proxy-agent "^7.0.2" lru-cache "^7.14.1" pac-proxy-agent "^7.0.1" proxy-from-env "^1.1.0" socks-proxy-agent "^8.0.2" proxy-from-env@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/proxy-from-env/-/proxy-from-env-1.1.0.tgz#e102f16ca355424865755d2c9e8ea4f24d58c3e2" integrity sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg== pstree.remy@^1.1.8: version "1.1.8" resolved "https://registry.yarnpkg.com/pstree.remy/-/pstree.remy-1.1.8.tgz#c242224f4a67c21f686839bbdb4ac282b8373d3a" integrity sha512-77DZwxQmxKnu3aR542U+X8FypNzbfJ+C5XQDk3uWjWxn6151aIMGthWYRXTqT1E5oJvg+ljaa2OJi+VfvCOQ8w== pump@^3.0.0: version "3.0.3" resolved "https://registry.yarnpkg.com/pump/-/pump-3.0.3.tgz#151d979f1a29668dc0025ec589a455b53282268d" integrity sha512-todwxLMY7/heScKmntwQG8CXVkWUOdYxIvY2s0VWAAMh/nd8SoYiRaKjlr7+iCs984f2P8zvrfWcDDYVb73NfA== dependencies: end-of-stream "^1.1.0" once "^1.3.1" punycode.js@2.3.1: version "2.3.1" resolved "https://registry.yarnpkg.com/punycode.js/-/punycode.js-2.3.1.tgz#6b53e56ad75588234e79f4affa90972c7dd8cdb7" integrity sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA== punycode@^2.1.0: version "2.3.1" resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.3.1.tgz#027422e2faec0b25e1549c3e1bd8309b9133b6e5" integrity sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg== puppeteer-core@21.5.2: version "21.5.2" resolved "https://registry.yarnpkg.com/puppeteer-core/-/puppeteer-core-21.5.2.tgz#6d3de4efb2ae65f1ee072043787b75594e88035f" integrity sha512-v4T0cWnujSKs+iEfmb8ccd7u4/x8oblEyKqplqKnJ582Kw8PewYAWvkH4qUWhitN3O2q9RF7dzkvjyK5HbzjLA== dependencies: "@puppeteer/browsers" "1.8.0" chromium-bidi "0.4.33" cross-fetch "4.0.0" debug "4.3.4" devtools-protocol "0.0.1203626" ws "8.14.2" puppeteer@~21.5.2: version "21.5.2" resolved "https://registry.yarnpkg.com/puppeteer/-/puppeteer-21.5.2.tgz#0a4a72175c0fd0944d6486f4734807e1671d527b" integrity sha512-BaAGJOq8Fl6/cck6obmwaNLksuY0Bg/lIahCLhJPGXBFUD2mCffypa4A592MaWnDcye7eaHmSK9yot0pxctY8A== dependencies: "@puppeteer/browsers" "1.8.0" cosmiconfig "8.3.6" puppeteer-core "21.5.2" qs@~6.14.0: version "6.14.0" resolved "https://registry.yarnpkg.com/qs/-/qs-6.14.0.tgz#c63fa40680d2c5c941412a0e899c89af60c0a930" integrity sha512-YWWTjgABSKcvs/nWBi9PycY/JiPJqOD4JA6o9Sej2AtvSGarXxKC3OQSk4pAarbdQlKAh5D4FCQkJNkW+GAn3w== dependencies: side-channel "^1.1.0" range-parser@~1.2.1: version "1.2.1" resolved "https://registry.yarnpkg.com/range-parser/-/range-parser-1.2.1.tgz#3cf37023d199e1c24d1a55b84800c2f3e6468031" integrity sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg== raw-body@~2.5.3: version "2.5.3" resolved "https://registry.yarnpkg.com/raw-body/-/raw-body-2.5.3.tgz#11c6650ee770a7de1b494f197927de0c923822e2" integrity sha512-s4VSOf6yN0rvbRZGxs8Om5CWj6seneMwK3oDb4lWDH0UPhWcxwOWw5+qk24bxq87szX1ydrwylIOp2uG1ojUpA== dependencies: bytes "~3.1.2" http-errors "~2.0.1" iconv-lite "~0.4.24" unpipe "~1.0.0" rc@^1.2.7: version "1.2.8" resolved "https://registry.yarnpkg.com/rc/-/rc-1.2.8.tgz#cd924bf5200a075b83c188cd6b9e211b7fc0d3ed" integrity sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw== dependencies: deep-extend "^0.6.0" ini "~1.3.0" minimist "^1.2.0" strip-json-comments "~2.0.1" readable-stream@^2.3.0, readable-stream@^2.3.5, readable-stream@~2.3.6: version "2.3.8" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.8.tgz#91125e8042bba1b9887f49345f6277027ce8be9b" integrity sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA== dependencies: core-util-is "~1.0.0" inherits "~2.0.3" isarray "~1.0.0" process-nextick-args "~2.0.0" safe-buffer "~5.1.1" string_decoder "~1.1.1" util-deprecate "~1.0.1" readable-stream@^3.1.1, readable-stream@^3.4.0, readable-stream@^3.6.2: version "3.6.2" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.2.tgz#56a9b36ea965c00c5a93ef31eb111a0f11056967" integrity sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA== dependencies: inherits "^2.0.3" string_decoder "^1.1.1" util-deprecate "^1.0.1" readable-stream@^4.7.0: version "4.7.0" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-4.7.0.tgz#cedbd8a1146c13dfff8dab14068028d58c15ac91" integrity sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg== dependencies: abort-controller "^3.0.0" buffer "^6.0.3" events "^3.3.0" process "^0.11.10" string_decoder "^1.3.0" readable-web-to-node-stream@^3.0.0: version "3.0.4" resolved "https://registry.yarnpkg.com/readable-web-to-node-stream/-/readable-web-to-node-stream-3.0.4.tgz#392ba37707af5bf62d725c36c1b5d6ef4119eefc" integrity sha512-9nX56alTf5bwXQ3ZDipHJhusu9NTQJ/CVPtb/XHAJCXihZeitfJvIRS4GqQ/mfIoOE3IelHMrpayVrosdHBuLw== dependencies: readable-stream "^4.7.0" readdirp@~3.6.0: version "3.6.0" resolved "https://registry.yarnpkg.com/readdirp/-/readdirp-3.6.0.tgz#74a370bd857116e245b29cc97340cd431a02a6c7" integrity sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA== dependencies: picomatch "^2.2.1" regenerator-runtime@^0.13.3: version "0.13.11" resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.13.11.tgz#f6dca3e7ceec20590d07ada785636a90cdca17f9" integrity sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg== require-directory@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42" integrity sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q== resolve-from@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-4.0.0.tgz#4abcd852ad32dd7baabfe9b40e00a36db5f392e6" integrity sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g== retry@^0.13.1: version "0.13.1" resolved "https://registry.yarnpkg.com/retry/-/retry-0.13.1.tgz#185b1587acf67919d63b357349e03537b2484658" integrity sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg== rimraf@^5.0.10: version "5.0.10" resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-5.0.10.tgz#23b9843d3dc92db71f96e1a2ce92e39fd2a8221c" integrity sha512-l0OE8wL34P4nJH/H2ffoaniAokM2qSmrtXHmlpvYr5AVVX8msAyW0l8NVJFDxlSK4u3Uh/f41cQheDVdnYijwQ== dependencies: glob "^10.3.7" safe-buffer@5.2.1, safe-buffer@^5.0.1, safe-buffer@^5.1.1, safe-buffer@^5.2.1, safe-buffer@~5.2.0: version "5.2.1" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6" integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== safe-buffer@~5.1.0, safe-buffer@~5.1.1: version "5.1.2" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g== safe-stable-stringify@^2.3.1: version "2.5.0" resolved "https://registry.yarnpkg.com/safe-stable-stringify/-/safe-stable-stringify-2.5.0.tgz#4ca2f8e385f2831c432a719b108a3bf7af42a1dd" integrity sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA== "safer-buffer@>= 2.1.2 < 3", "safer-buffer@>= 2.1.2 < 3.0.0": version "2.1.2" resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== sax@>=0.6.0: version "1.4.3" resolved "https://registry.yarnpkg.com/sax/-/sax-1.4.3.tgz#fcebae3b756cdc8428321805f4b70f16ec0ab5db" integrity sha512-yqYn1JhPczigF94DMS+shiDMjDowYO6y9+wB/4WgO0Y19jWYk0lQ4tuG5KI7kj4FTp1wxPj5IFfcrz/s1c3jjQ== seek-bzip@^1.0.5: version "1.0.6" resolved "https://registry.yarnpkg.com/seek-bzip/-/seek-bzip-1.0.6.tgz#35c4171f55a680916b52a07859ecf3b5857f21c4" integrity sha512-e1QtP3YL5tWww8uKaOCQ18UxIT2laNBXHjV/S2WYCiK4udiv8lkG89KRIoCjUagnAmCBurjF4zEVX2ByBbnCjQ== dependencies: commander "^2.8.1" selderee@^0.11.0: version "0.11.0" resolved "https://registry.yarnpkg.com/selderee/-/selderee-0.11.0.tgz#6af0c7983e073ad3e35787ffe20cefd9daf0ec8a" integrity sha512-5TF+l7p4+OsnP8BCCvSyZiSPc4x4//p5uPwK8TCnVPJYRmU2aYKMpOXvw8zM5a5JvuuCGN1jmsMwuU2W02ukfA== dependencies: parseley "^0.12.0" semver@^5.7.1: version "5.7.2" resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.2.tgz#48d55db737c3287cd4835e17fa13feace1c41ef8" integrity sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g== semver@^7.3.5, semver@^7.5.4, semver@^7.6.3: version "7.7.3" resolved "https://registry.yarnpkg.com/semver/-/semver-7.7.3.tgz#4b5f4143d007633a8dc671cd0a6ef9147b8bb946" integrity sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q== semver@~7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/semver/-/semver-7.0.0.tgz#5f3ca35761e47e05b206c6daff2cf814f0316b8e" integrity sha512-+GB6zVA9LWh6zovYQLALHwv5rb2PHGlJi3lfiqIHxR0uuwCgefcOJc59v9fv1w8GbStwxuuqqAjI9NMAOOgq1A== send@0.19.0: version "0.19.0" resolved "https://registry.yarnpkg.com/send/-/send-0.19.0.tgz#bbc5a388c8ea6c048967049dbeac0e4a3f09d7f8" integrity sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw== dependencies: debug "2.6.9" depd "2.0.0" destroy "1.2.0" encodeurl "~1.0.2" escape-html "~1.0.3" etag "~1.8.1" fresh "0.5.2" http-errors "2.0.0" mime "1.6.0" ms "2.1.3" on-finished "2.4.1" range-parser "~1.2.1" statuses "2.0.1" send@~0.19.0: version "0.19.1" resolved "https://registry.yarnpkg.com/send/-/send-0.19.1.tgz#1c2563b2ee4fe510b806b21ec46f355005a369f9" integrity sha512-p4rRk4f23ynFEfcD9LA0xRYngj+IyGiEYyqqOak8kaN0TvNmuxC2dcVeBn62GpCeR2CpWqyHCNScTP91QbAVFg== dependencies: debug "2.6.9" depd "2.0.0" destroy "1.2.0" encodeurl "~2.0.0" escape-html "~1.0.3" etag "~1.8.1" fresh "0.5.2" http-errors "2.0.0" mime "1.6.0" ms "2.1.3" on-finished "2.4.1" range-parser "~1.2.1" statuses "2.0.1" serve-static@~1.16.2: version "1.16.2" resolved "https://registry.yarnpkg.com/serve-static/-/serve-static-1.16.2.tgz#b6a5343da47f6bdd2673848bf45754941e803296" integrity sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw== dependencies: encodeurl "~2.0.0" escape-html "~1.0.3" parseurl "~1.3.3" send "0.19.0" set-function-length@^1.2.2: version "1.2.2" resolved "https://registry.yarnpkg.com/set-function-length/-/set-function-length-1.2.2.tgz#aac72314198eaed975cf77b2c3b6b880695e5449" integrity sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg== dependencies: define-data-property "^1.1.4" es-errors "^1.3.0" function-bind "^1.1.2" get-intrinsic "^1.2.4" gopd "^1.0.1" has-property-descriptors "^1.0.2" setimmediate@^1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/setimmediate/-/setimmediate-1.0.5.tgz#290cbb232e306942d7d7ea9b83732ab7856f8285" integrity sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA== setprototypeof@1.2.0, setprototypeof@~1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.2.0.tgz#66c9a24a73f9fc28cbe66b09fed3d33dcaf1b424" integrity sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw== sharp@^0.32.0: version "0.32.6" resolved "https://registry.yarnpkg.com/sharp/-/sharp-0.32.6.tgz#6ad30c0b7cd910df65d5f355f774aa4fce45732a" integrity sha512-KyLTWwgcR9Oe4d9HwCwNM2l7+J0dUQwn/yf7S0EnTtb0eVS4RxO0eUSvxPtzT4F3SY+C4K6fqdv/DO27sJ/v/w== dependencies: color "^4.2.3" detect-libc "^2.0.2" node-addon-api "^6.1.0" prebuild-install "^7.1.1" semver "^7.5.4" simple-get "^4.0.1" tar-fs "^3.0.4" tunnel-agent "^0.6.0" sharp@^0.33.5: version "0.33.5" resolved "https://registry.yarnpkg.com/sharp/-/sharp-0.33.5.tgz#13e0e4130cc309d6a9497596715240b2ec0c594e" integrity sha512-haPVm1EkS9pgvHrQ/F3Xy+hgcuMV0Wm9vfIBSiwZ05k+xgb0PkBQpGsAA/oWdDobNaZTH5ppvHtzCFbnSEwHVw== dependencies: color "^4.2.3" detect-libc "^2.0.3" semver "^7.6.3" optionalDependencies: "@img/sharp-darwin-arm64" "0.33.5" "@img/sharp-darwin-x64" "0.33.5" "@img/sharp-libvips-darwin-arm64" "1.0.4" "@img/sharp-libvips-darwin-x64" "1.0.4" "@img/sharp-libvips-linux-arm" "1.0.5" "@img/sharp-libvips-linux-arm64" "1.0.4" "@img/sharp-libvips-linux-s390x" "1.0.4" "@img/sharp-libvips-linux-x64" "1.0.4" "@img/sharp-libvips-linuxmusl-arm64" "1.0.4" "@img/sharp-libvips-linuxmusl-x64" "1.0.4" "@img/sharp-linux-arm" "0.33.5" "@img/sharp-linux-arm64" "0.33.5" "@img/sharp-linux-s390x" "0.33.5" "@img/sharp-linux-x64" "0.33.5" "@img/sharp-linuxmusl-arm64" "0.33.5" "@img/sharp-linuxmusl-x64" "0.33.5" "@img/sharp-wasm32" "0.33.5" "@img/sharp-win32-ia32" "0.33.5" "@img/sharp-win32-x64" "0.33.5" shebang-command@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/shebang-command/-/shebang-command-2.0.0.tgz#ccd0af4f8835fbdc265b82461aaf0c36663f34ea" integrity sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA== dependencies: shebang-regex "^3.0.0" shebang-regex@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/shebang-regex/-/shebang-regex-3.0.0.tgz#ae16f1644d873ecad843b0307b143362d4c42172" integrity sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A== shell-env@^4.0.1: version "4.0.1" resolved "https://registry.yarnpkg.com/shell-env/-/shell-env-4.0.1.tgz#883302d9426095d398a39b102a851adb306b8cb8" integrity sha512-w3oeZ9qg/P6Lu6qqwavvMnB/bwfsz67gPB3WXmLd/n6zuh7TWQZtGa3iMEdmua0kj8rivkwl+vUjgLWlqZOMPw== dependencies: default-shell "^2.0.0" execa "^5.1.1" strip-ansi "^7.0.1" shell-path@^3.0.0: version "3.1.0" resolved "https://registry.yarnpkg.com/shell-path/-/shell-path-3.1.0.tgz#950671fe15de70fb4d984b886d55e8a2f10bfe33" integrity sha512-s/9q9PEtcRmDTz69+cJ3yYBAe9yGrL7e46gm2bU4pQ9N48ecPK9QrGFnLwYgb4smOHskx4PL7wCNMktW2AoD+g== dependencies: shell-env "^4.0.1" side-channel-list@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/side-channel-list/-/side-channel-list-1.0.0.tgz#10cb5984263115d3b7a0e336591e290a830af8ad" integrity sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA== dependencies: es-errors "^1.3.0" object-inspect "^1.13.3" side-channel-map@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/side-channel-map/-/side-channel-map-1.0.1.tgz#d6bb6b37902c6fef5174e5f533fab4c732a26f42" integrity sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA== dependencies: call-bound "^1.0.2" es-errors "^1.3.0" get-intrinsic "^1.2.5" object-inspect "^1.13.3" side-channel-weakmap@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz#11dda19d5368e40ce9ec2bdc1fb0ecbc0790ecea" integrity sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A== dependencies: call-bound "^1.0.2" es-errors "^1.3.0" get-intrinsic "^1.2.5" object-inspect "^1.13.3" side-channel-map "^1.0.1" side-channel@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/side-channel/-/side-channel-1.1.0.tgz#c3fcff9c4da932784873335ec9765fa94ff66bc9" integrity sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw== dependencies: es-errors "^1.3.0" object-inspect "^1.13.3" side-channel-list "^1.0.0" side-channel-map "^1.0.1" side-channel-weakmap "^1.0.2" signal-exit@^3.0.3: version "3.0.7" resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.7.tgz#a9a1767f8af84155114eaabd73f99273c8f59ad9" integrity sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ== signal-exit@^4.0.1: version "4.1.0" resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-4.1.0.tgz#952188c1cbd546070e2dd20d0f41c0ae0530cb04" integrity sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw== simple-concat@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/simple-concat/-/simple-concat-1.0.1.tgz#f46976082ba35c2263f1c8ab5edfe26c41c9552f" integrity sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q== simple-get@^4.0.0, simple-get@^4.0.1: version "4.0.1" resolved "https://registry.yarnpkg.com/simple-get/-/simple-get-4.0.1.tgz#4a39db549287c979d352112fa03fd99fd6bc3543" integrity sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA== dependencies: decompress-response "^6.0.0" once "^1.3.1" simple-concat "^1.0.0" simple-swizzle@^0.2.2: version "0.2.4" resolved "https://registry.yarnpkg.com/simple-swizzle/-/simple-swizzle-0.2.4.tgz#a8d11a45a11600d6a1ecdff6363329e3648c3667" integrity sha512-nAu1WFPQSMNr2Zn9PGSZK9AGn4t/y97lEm+MXTtUDwfP0ksAIX4nO+6ruD9Jwut4C49SB1Ws+fbXsm/yScWOHw== dependencies: is-arrayish "^0.3.1" simple-update-notifier@^1.0.7: version "1.1.0" resolved "https://registry.yarnpkg.com/simple-update-notifier/-/simple-update-notifier-1.1.0.tgz#67694c121de354af592b347cdba798463ed49c82" integrity sha512-VpsrsJSUcJEseSbMHkrsrAVSdvVS5I96Qo1QAQ4FxQ9wXFcB+pjj7FB7/us9+GcgfW4ziHtYMc1J0PLczb55mg== dependencies: semver "~7.0.0" slugify@^1.6.6: version "1.6.6" resolved "https://registry.yarnpkg.com/slugify/-/slugify-1.6.6.tgz#2d4ac0eacb47add6af9e04d3be79319cbcc7924b" integrity sha512-h+z7HKHYXj6wJU+AnS/+IH8Uh9fdcX1Lrhg1/VMdf9PwoBQXFcXiAdsy2tSK0P6gKwJLXp02r90ahUCqHk9rrw== smart-buffer@^4.2.0: version "4.2.0" resolved "https://registry.yarnpkg.com/smart-buffer/-/smart-buffer-4.2.0.tgz#6e1d71fa4f18c05f7d0ff216dd16a481d0e8d9ae" integrity sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg== socks-proxy-agent@^8.0.2, socks-proxy-agent@^8.0.5: version "8.0.5" resolved "https://registry.yarnpkg.com/socks-proxy-agent/-/socks-proxy-agent-8.0.5.tgz#b9cdb4e7e998509d7659d689ce7697ac21645bee" integrity sha512-HehCEsotFqbPW9sJ8WVYB6UbmIMv7kUUORIF2Nncq4VQvBfNBLibW9YZR5dlYCSUhwcD628pRllm7n+E+YTzJw== dependencies: agent-base "^7.1.2" debug "^4.3.4" socks "^2.8.3" socks@^2.8.3: version "2.8.7" resolved "https://registry.yarnpkg.com/socks/-/socks-2.8.7.tgz#e2fb1d9a603add75050a2067db8c381a0b5669ea" integrity sha512-HLpt+uLy/pxB+bum/9DzAgiKS8CX1EvbWxI4zlmgGCExImLdiad2iCwXT5Z4c9c3Eq8rP2318mPW2c+QbtjK8A== dependencies: ip-address "^10.0.1" smart-buffer "^4.2.0" source-map@~0.6.1: version "0.6.1" resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263" integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g== sprintf-js@~1.0.2: version "1.0.3" resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" integrity sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g== stack-trace@0.0.x: version "0.0.10" resolved "https://registry.yarnpkg.com/stack-trace/-/stack-trace-0.0.10.tgz#547c70b347e8d32b4e108ea1a2a159e5fdde19c0" integrity sha512-KGzahc7puUKkzyMt+IqAep+TVNbKP+k2Lmwhub39m1AsTSkaDutx56aDCo+HLDzf/D26BIHTJWNiTG1KAJiQCg== statuses@2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/statuses/-/statuses-2.0.1.tgz#55cb000ccf1d48728bd23c685a063998cf1a1b63" integrity sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ== statuses@~2.0.1, statuses@~2.0.2: version "2.0.2" resolved "https://registry.yarnpkg.com/statuses/-/statuses-2.0.2.tgz#8f75eecef765b5e1cfcdc080da59409ed424e382" integrity sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw== streamx@^2.15.0, streamx@^2.21.0: version "2.23.0" resolved "https://registry.yarnpkg.com/streamx/-/streamx-2.23.0.tgz#7d0f3d00d4a6c5de5728aecd6422b4008d66fd0b" integrity sha512-kn+e44esVfn2Fa/O0CPFcex27fjIL6MkVae0Mm6q+E6f0hWv578YCERbv+4m02cjxvDsPKLnmxral/rR6lBMAg== dependencies: events-universal "^1.0.0" fast-fifo "^1.3.2" text-decoder "^1.1.0" "string-width-cjs@npm:string-width@^4.2.0": version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== dependencies: emoji-regex "^8.0.0" is-fullwidth-code-point "^3.0.0" strip-ansi "^6.0.1" string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3, string-width@^5.1.2: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== dependencies: emoji-regex "^8.0.0" is-fullwidth-code-point "^3.0.0" strip-ansi "^6.0.1" string_decoder@^1.1.1, string_decoder@^1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.3.0.tgz#42f114594a46cf1a8e30b0a84f56c78c3edac21e" integrity sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA== dependencies: safe-buffer "~5.2.0" string_decoder@~1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.1.1.tgz#9cf1611ba62685d7030ae9e4ba34149c3af03fc8" integrity sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg== dependencies: safe-buffer "~5.1.0" "strip-ansi-cjs@npm:strip-ansi@^6.0.1": version "6.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== dependencies: ansi-regex "^5.0.1" strip-ansi@^6.0.0, strip-ansi@^6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== dependencies: ansi-regex "^5.0.1" strip-ansi@^7.0.1, strip-ansi@^7.1.2: version "7.1.2" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.1.2.tgz#132875abde678c7ea8d691533f2e7e22bb744dba" integrity sha512-gmBGslpoQJtgnMAvOVqGZpEz9dyoKTCzy2nfz/n8aIFhN/jCE/rCmcxabB6jOOHV+0WNnylOxaxBQPSvcWklhA== dependencies: ansi-regex "^6.0.1" strip-dirs@^2.0.0: version "2.1.0" resolved "https://registry.yarnpkg.com/strip-dirs/-/strip-dirs-2.1.0.tgz#4987736264fc344cf20f6c34aca9d13d1d4ed6c5" integrity sha512-JOCxOeKLm2CAS73y/U4ZeZPTkE+gNVCzKt7Eox84Iej1LT/2pTWYpZKJuxwQpvX1LiZb1xokNR7RLfuBAa7T3g== dependencies: is-natural-number "^4.0.1" strip-final-newline@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/strip-final-newline/-/strip-final-newline-2.0.0.tgz#89b852fb2fcbe936f6f4b3187afb0a12c1ab58ad" integrity sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA== strip-json-comments@^3.1.1: version "3.1.1" resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-3.1.1.tgz#31f1281b3832630434831c310c01cccda8cbe006" integrity sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig== strip-json-comments@~2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-2.0.1.tgz#3c531942e908c2697c0ec344858c286c7ca0a60a" integrity sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ== strtok3@^6.2.4: version "6.3.0" resolved "https://registry.yarnpkg.com/strtok3/-/strtok3-6.3.0.tgz#358b80ffe6d5d5620e19a073aa78ce947a90f9a0" integrity sha512-fZtbhtvI9I48xDSywd/somNqgUHl2L2cstmXCCif0itOf96jeW18MBSyrLuNicYQVkvpOxkZtkzujiTJ9LW5Jw== dependencies: "@tokenizer/token" "^0.3.0" peek-readable "^4.1.0" supports-color@^5.5.0: version "5.5.0" resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-5.5.0.tgz#e2e69a44ac8772f78a1ec0b35b689df6530efc8f" integrity sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow== dependencies: has-flag "^3.0.0" supports-color@^7.1.0: version "7.2.0" resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-7.2.0.tgz#1b7dcdcb32b8138801b3e478ba6a51caa89648da" integrity sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw== dependencies: has-flag "^4.0.0" synckit@^0.11.12: version "0.11.12" resolved "https://registry.yarnpkg.com/synckit/-/synckit-0.11.12.tgz#abe74124264fbc00a48011b0d98bdc1cffb64a7b" integrity sha512-Bh7QjT8/SuKUIfObSXNHNSK6WHo6J1tHCqJsuaFDP7gP0fkzSfTxI8y85JrppZ0h8l0maIgc2tfuZQ6/t3GtnQ== dependencies: "@pkgr/core" "^0.2.9" tar-fs@3.0.4: version "3.0.4" resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-3.0.4.tgz#a21dc60a2d5d9f55e0089ccd78124f1d3771dbbf" integrity sha512-5AFQU8b9qLfZCX9zp2duONhPmZv0hGYiBPJsyUdqMjzq/mqVpy/rEUSeHk1+YitmxugaptgBh5oDGU3VsAJq4w== dependencies: mkdirp-classic "^0.5.2" pump "^3.0.0" tar-stream "^3.1.5" tar-fs@^2.0.0: version "2.1.4" resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-2.1.4.tgz#800824dbf4ef06ded9afea4acafe71c67c76b930" integrity sha512-mDAjwmZdh7LTT6pNleZ05Yt65HC3E+NiQzl672vQG38jIrehtJk/J3mNwIg+vShQPcLF/LV7CMnDW6vjj6sfYQ== dependencies: chownr "^1.1.1" mkdirp-classic "^0.5.2" pump "^3.0.0" tar-stream "^2.1.4" tar-fs@^3.0.4: version "3.1.1" resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-3.1.1.tgz#4f164e59fb60f103d472360731e8c6bb4a7fe9ef" integrity sha512-LZA0oaPOc2fVo82Txf3gw+AkEd38szODlptMYejQUhndHMLQ9M059uXR+AfS7DNo0NpINvSqDsvyaCrBVkptWg== dependencies: pump "^3.0.0" tar-stream "^3.1.5" optionalDependencies: bare-fs "^4.0.1" bare-path "^3.0.0" tar-stream@^1.5.2: version "1.6.2" resolved "https://registry.yarnpkg.com/tar-stream/-/tar-stream-1.6.2.tgz#8ea55dab37972253d9a9af90fdcd559ae435c555" integrity sha512-rzS0heiNf8Xn7/mpdSVVSMAWAoy9bfb1WOTYC78Z0UQKeKa/CWS8FOq0lKGNa8DWKAn9gxjCvMLYc5PGXYlK2A== dependencies: bl "^1.0.0" buffer-alloc "^1.2.0" end-of-stream "^1.0.0" fs-constants "^1.0.0" readable-stream "^2.3.0" to-buffer "^1.1.1" xtend "^4.0.0" tar-stream@^2.1.4: version "2.2.0" resolved "https://registry.yarnpkg.com/tar-stream/-/tar-stream-2.2.0.tgz#acad84c284136b060dc3faa64474aa9aebd77287" integrity sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ== dependencies: bl "^4.0.3" end-of-stream "^1.4.1" fs-constants "^1.0.0" inherits "^2.0.3" readable-stream "^3.1.1" tar-stream@^3.1.5: version "3.1.7" resolved "https://registry.yarnpkg.com/tar-stream/-/tar-stream-3.1.7.tgz#24b3fb5eabada19fe7338ed6d26e5f7c482e792b" integrity sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ== dependencies: b4a "^1.6.4" fast-fifo "^1.2.0" streamx "^2.15.0" tesseract.js-core@^6.0.0: version "6.0.0" resolved "https://registry.yarnpkg.com/tesseract.js-core/-/tesseract.js-core-6.0.0.tgz#6f25da94f70f8e8f02aff47a43be61d49e6f67c3" integrity sha512-1Qncm/9oKM7xgrQXZXNB+NRh19qiXGhxlrR8EwFbK5SaUbPZnS5OMtP/ghtqfd23hsr1ZvZbZjeuAGcMxd/ooA== tesseract.js@^6.0.0: version "6.0.1" resolved "https://registry.yarnpkg.com/tesseract.js/-/tesseract.js-6.0.1.tgz#5b2ff39aae92d59cef79589a43a0f3ab963801cc" integrity sha512-/sPvMvrCtgxnNRCjbTYbr7BRu0yfWDsMZQ2a/T5aN/L1t8wUQN6tTWv6p6FwzpoEBA0jrN2UD2SX4QQFRdoDbA== dependencies: bmp-js "^0.1.0" idb-keyval "^6.2.0" is-url "^1.2.4" node-fetch "^2.6.9" opencollective-postinstall "^2.0.3" regenerator-runtime "^0.13.3" tesseract.js-core "^6.0.0" wasm-feature-detect "^1.2.11" zlibjs "^0.3.1" text-decoder@^1.1.0: version "1.2.3" resolved "https://registry.yarnpkg.com/text-decoder/-/text-decoder-1.2.3.tgz#b19da364d981b2326d5f43099c310cc80d770c65" integrity sha512-3/o9z3X0X0fTupwsYvR03pJ/DjWuqqrfwBgTQzdWDiQSm9KitAyz/9WqsT2JQW7KV2m+bC2ol/zqpW37NHxLaA== dependencies: b4a "^1.6.4" text-hex@1.0.x: version "1.0.0" resolved "https://registry.yarnpkg.com/text-hex/-/text-hex-1.0.0.tgz#69dc9c1b17446ee79a92bf5b884bb4b9127506f5" integrity sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg== through@^2.3.8: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" integrity sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg== tlds@1.261.0: version "1.261.0" resolved "https://registry.yarnpkg.com/tlds/-/tlds-1.261.0.tgz#055e412e92f01f84a9c8ac0504d3472d68b3c4c9" integrity sha512-QXqwfEl9ddlGBaRFXIvNKK6OhipSiLXuRuLJX5DErz0o0Q0rYxulWLdFryTkV5PkdZct5iMInwYEGe/eR++1AA== to-buffer@^1.1.1: version "1.2.2" resolved "https://registry.yarnpkg.com/to-buffer/-/to-buffer-1.2.2.tgz#ffe59ef7522ada0a2d1cb5dfe03bb8abc3cdc133" integrity sha512-db0E3UJjcFhpDhAF4tLo03oli3pwl3dbnzXOUIlRKrp+ldk/VUxzpWYZENsw2SZiuBjHAk7DfB0VU7NKdpb6sw== dependencies: isarray "^2.0.5" safe-buffer "^5.2.1" typed-array-buffer "^1.0.3" to-regex-range@^5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/to-regex-range/-/to-regex-range-5.0.1.tgz#1648c44aae7c8d988a326018ed72f5b4dd0392e4" integrity sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ== dependencies: is-number "^7.0.0" toidentifier@1.0.1, toidentifier@~1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/toidentifier/-/toidentifier-1.0.1.tgz#3be34321a88a820ed1bd80dfaa33e479fbb8dd35" integrity sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA== token-types@^4.1.1: version "4.2.1" resolved "https://registry.yarnpkg.com/token-types/-/token-types-4.2.1.tgz#0f897f03665846982806e138977dbe72d44df753" integrity sha512-6udB24Q737UD/SDsKAHI9FCRP7Bqc9D/MQUV02ORQg5iskjtLJlZJNdN4kKtcdtwCeWIwIHDGaUsTsCCAa8sFQ== dependencies: "@tokenizer/token" "^0.3.0" ieee754 "^1.2.1" touch@^3.1.0: version "3.1.1" resolved "https://registry.yarnpkg.com/touch/-/touch-3.1.1.tgz#097a23d7b161476435e5c1344a95c0f75b4a5694" integrity sha512-r0eojU4bI8MnHr8c5bNo7lJDdI2qXlWWJk6a9EAFG7vbhTjElYhBVS3/miuE0uOuoLdb8Mc/rVfsmm6eo5o9GA== tr46@~0.0.3: version "0.0.3" resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a" integrity sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw== triple-beam@^1.3.0: version "1.4.1" resolved "https://registry.yarnpkg.com/triple-beam/-/triple-beam-1.4.1.tgz#6fde70271dc6e5d73ca0c3b24e2d92afb7441984" integrity sha512-aZbgViZrg1QNcG+LULa7nhZpJTZSLm/mXnHXnbAbjmN5aSa0y7V+wvv6+4WaBtpISJzThKy+PIPxc1Nq1EJ9mg== ts-type@>=2: version "3.0.1" resolved "https://registry.yarnpkg.com/ts-type/-/ts-type-3.0.1.tgz#b52e7623065e0beb43c77c426347d85cf81dff84" integrity sha512-cleRydCkBGBFQ4KAvLH0ARIkciduS745prkGVVxPGvcRGhMMoSJUB7gNR1ByKhFTEYrYRg2CsMRGYnqp+6op+g== dependencies: "@types/node" "*" tslib ">=2" typedarray-dts "^1.0.0" tslib@>=2, tslib@^2.0.1, tslib@^2.4.0, tslib@^2.5.0, tslib@^2.6.2: version "2.8.1" resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.8.1.tgz#612efe4ed235d567e8aba5f2a5fab70280ade83f" integrity sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w== tunnel-agent@^0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.6.0.tgz#27a5dea06b36b04a0a9966774b290868f0fc40fd" integrity sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w== dependencies: safe-buffer "^5.0.1" type-check@^0.4.0, type-check@~0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/type-check/-/type-check-0.4.0.tgz#07b8203bfa7056c0657050e3ccd2c37730bab8f1" integrity sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew== dependencies: prelude-ls "^1.2.1" type-detect@^4.0.0: version "4.1.0" resolved "https://registry.yarnpkg.com/type-detect/-/type-detect-4.1.0.tgz#deb2453e8f08dcae7ae98c626b13dddb0155906c" integrity sha512-Acylog8/luQ8L7il+geoSxhEkazvkslg7PSNKOX59mbB9cOveP5aq9h74Y7YU8yDpJwetzQQrfIwtf4Wp4LKcw== type-is@~1.6.18: version "1.6.18" resolved "https://registry.yarnpkg.com/type-is/-/type-is-1.6.18.tgz#4e552cd05df09467dcbc4ef739de89f2cf37c131" integrity sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g== dependencies: media-typer "0.3.0" mime-types "~2.1.24" typed-array-buffer@^1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/typed-array-buffer/-/typed-array-buffer-1.0.3.tgz#a72395450a4869ec033fd549371b47af3a2ee536" integrity sha512-nAYYwfY3qnzX30IkA6AQZjVbtK6duGontcQm1WSG1MD94YLqK0515GNApXkoxKOWMusVssAHWLh9SeaoefYFGw== dependencies: call-bound "^1.0.3" es-errors "^1.3.0" is-typed-array "^1.1.14" typedarray-dts@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/typedarray-dts/-/typedarray-dts-1.0.0.tgz#9dec9811386dbfba964c295c2606cf9a6b982d06" integrity sha512-Ka0DBegjuV9IPYFT1h0Qqk5U4pccebNIJCGl8C5uU7xtOs+jpJvKGAY4fHGK25hTmXZOEUl9Cnsg5cS6K/b5DA== uc.micro@^2.0.0: version "2.1.0" resolved "https://registry.yarnpkg.com/uc.micro/-/uc.micro-2.1.0.tgz#f8d3f7d0ec4c3dea35a7e3c8efa4cb8b45c9e7ee" integrity sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A== unbzip2-stream@1.4.3, unbzip2-stream@^1.0.9: version "1.4.3" resolved "https://registry.yarnpkg.com/unbzip2-stream/-/unbzip2-stream-1.4.3.tgz#b0da04c4371311df771cdc215e87f2130991ace7" integrity sha512-mlExGW4w71ebDJviH16lQLtZS32VKqsSfk80GCfUlwT/4/hNRFsoscrF/c++9xinkMzECL1uL9DDwXqFWkruPg== dependencies: buffer "^5.2.1" through "^2.3.8" undefsafe@^2.0.5: version "2.0.5" resolved "https://registry.yarnpkg.com/undefsafe/-/undefsafe-2.0.5.tgz#38733b9327bdcd226db889fb723a6efd162e6e2c" integrity sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA== underscore@^1.13.1: version "1.13.7" resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.13.7.tgz#970e33963af9a7dda228f17ebe8399e5fbe63a10" integrity sha512-GMXzWtsc57XAtguZgaQViUOzs0KTkk8ojr3/xAxXLITqf/3EMwxC0inyETfDFjH/Krbhuep0HNbbjI9i/q3F3g== undici-types@~5.26.4: version "5.26.5" resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-5.26.5.tgz#bcd539893d00b56e964fd2657a4866b221a65617" integrity sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA== undici-types@~7.16.0: version "7.16.0" resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-7.16.0.tgz#ffccdff36aea4884cbfce9a750a0580224f58a46" integrity sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw== undici@^5.19.1: version "5.29.0" resolved "https://registry.yarnpkg.com/undici/-/undici-5.29.0.tgz#419595449ae3f2cdcba3580a2e8903399bd1f5a3" integrity sha512-raqeBD6NQK4SkWhQzeYKd1KmIG6dllBOTt55Rmkt4HtI9mwdWtJljnrXjAFUBLTSN67HWrOIZ3EPF4kjUw80Bg== dependencies: "@fastify/busboy" "^2.0.0" unpipe@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec" integrity sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ== uri-js@^4.2.2: version "4.4.1" resolved "https://registry.yarnpkg.com/uri-js/-/uri-js-4.4.1.tgz#9b1a52595225859e55f669d928f88c6c57f2a77e" integrity sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg== dependencies: punycode "^2.1.0" url-pattern@^1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/url-pattern/-/url-pattern-1.0.3.tgz#0409292471b24f23c50d65a47931793d2b5acfc1" integrity sha512-uQcEj/2puA4aq1R3A2+VNVBgaWYR24FdWjl7VNW83rnWftlhyzOZ/tBjezRiC2UkIzuxC8Top3IekN3vUf1WxA== urlpattern-polyfill@9.0.0: version "9.0.0" resolved "https://registry.yarnpkg.com/urlpattern-polyfill/-/urlpattern-polyfill-9.0.0.tgz#bc7e386bb12fd7898b58d1509df21d3c29ab3460" integrity sha512-WHN8KDQblxd32odxeIgo83rdVDE2bvdkb86it7bMhYZwWKJz0+O0RK/eZiHYnM+zgt/U7hAHOlCQGfjjvSkw2g== util-deprecate@^1.0.1, util-deprecate@~1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" integrity sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw== utils-merge@1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.1.tgz#9f95710f50a267947b2ccc124741c1028427e713" integrity sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA== uuid@^10.0.0: version "10.0.0" resolved "https://registry.yarnpkg.com/uuid/-/uuid-10.0.0.tgz#5a95aa454e6e002725c79055fd42aaba30ca6294" integrity sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ== uuid@^9.0.0: version "9.0.1" resolved "https://registry.yarnpkg.com/uuid/-/uuid-9.0.1.tgz#e188d4c8853cc722220392c424cd637f32293f30" integrity sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA== vary@^1, vary@~1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/vary/-/vary-1.1.2.tgz#2299f02c6ded30d4a5961b0b9f74524a18f634fc" integrity sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg== wasm-feature-detect@^1.2.11: version "1.8.0" resolved "https://registry.yarnpkg.com/wasm-feature-detect/-/wasm-feature-detect-1.8.0.tgz#4e9f55b0a64d801f372fbb0324ed11ad3abd0c78" integrity sha512-zksaLKM2fVlnB5jQQDqKXXwYHLQUVH9es+5TOOHwGOVJOCeRBCiPjwSg+3tN2AdTCzjgli4jijCH290kXb/zWQ== wavefile@^11.0.0: version "11.0.0" resolved "https://registry.yarnpkg.com/wavefile/-/wavefile-11.0.0.tgz#9302165874327ff63a704d00b154c753eaa1b8e7" integrity sha512-/OBiAALgWU24IG7sC84cDO/KfFuvajWc5Uec0oV2zrpOOZZDgGdOwHwgEzOrwh8jkubBk7PtZfQBIcI1OaE5Ng== web-streams-polyfill@4.0.0-beta.3: version "4.0.0-beta.3" resolved "https://registry.yarnpkg.com/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz#2898486b74f5156095e473efe989dcf185047a38" integrity sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug== web-streams-polyfill@^3.2.1: version "3.3.3" resolved "https://registry.yarnpkg.com/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz#2073b91a2fdb1fbfbd401e7de0ac9f8214cecb4b" integrity sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw== webidl-conversions@^3.0.0: version "3.0.1" resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871" integrity sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ== whatwg-url@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d" integrity sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw== dependencies: tr46 "~0.0.3" webidl-conversions "^3.0.0" which-typed-array@^1.1.16: version "1.1.19" resolved "https://registry.yarnpkg.com/which-typed-array/-/which-typed-array-1.1.19.tgz#df03842e870b6b88e117524a4b364b6fc689f956" integrity sha512-rEvr90Bck4WZt9HHFC4DJMsjvu7x+r6bImz0/BrbWb7A2djJ8hnZMrWnHo9F8ssv0OMErasDhftrfROTyqSDrw== dependencies: available-typed-arrays "^1.0.7" call-bind "^1.0.8" call-bound "^1.0.4" for-each "^0.3.5" get-proto "^1.0.1" gopd "^1.2.0" has-tostringtag "^1.0.2" which@^2.0.1: version "2.0.2" resolved "https://registry.yarnpkg.com/which/-/which-2.0.2.tgz#7c6a8dd0a636a0327e10b59c9286eee93f3f51b1" integrity sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA== dependencies: isexe "^2.0.0" winston-transport@^4.9.0: version "4.9.0" resolved "https://registry.yarnpkg.com/winston-transport/-/winston-transport-4.9.0.tgz#3bba345de10297654ea6f33519424560003b3bf9" integrity sha512-8drMJ4rkgaPo1Me4zD/3WLfI/zPdA9o2IipKODunnGDcuqbHwjsbB79ylv04LCGGzU0xQ6vTznOMpQGaLhhm6A== dependencies: logform "^2.7.0" readable-stream "^3.6.2" triple-beam "^1.3.0" winston@^3.13.0: version "3.18.3" resolved "https://registry.yarnpkg.com/winston/-/winston-3.18.3.tgz#93ac10808c8e1081d723bc8811cd2f445ddfdcd1" integrity sha512-NoBZauFNNWENgsnC9YpgyYwOVrl2m58PpQ8lNHjV3kosGs7KJ7Npk9pCUE+WJlawVSe8mykWDKWFSVfs3QO9ww== dependencies: "@colors/colors" "^1.6.0" "@dabh/diagnostics" "^2.0.8" async "^3.2.3" is-stream "^2.0.0" logform "^2.7.0" one-time "^1.0.0" readable-stream "^3.4.0" safe-stable-stringify "^2.3.1" stack-trace "0.0.x" triple-beam "^1.3.0" winston-transport "^4.9.0" word-wrap@^1.2.5: version "1.2.5" resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.5.tgz#d2c45c6dd4fbce621a66f136cbe328afd0410b34" integrity sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA== "wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== dependencies: ansi-styles "^4.0.0" string-width "^4.1.0" strip-ansi "^6.0.0" wrap-ansi@^7.0.0, wrap-ansi@^8.1.0: version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== dependencies: ansi-styles "^4.0.0" string-width "^4.1.0" strip-ansi "^6.0.0" wrappy@1: version "1.0.2" resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" integrity sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ== ws@8.14.2: version "8.14.2" resolved "https://registry.yarnpkg.com/ws/-/ws-8.14.2.tgz#6c249a806eb2db7a20d26d51e7709eab7b2e6c7f" integrity sha512-wEBG1ftX4jcglPxgFCMJmZ2PLtSbJ2Peg6TmpJFTbe9GZYOQCDPdMYu/Tm0/bGZkw8paZnJY45J4K2PZrLYq8g== "xlsx@https://cdn.sheetjs.com/xlsx-0.20.2/xlsx-0.20.2.tgz": version "0.20.2" resolved "https://cdn.sheetjs.com/xlsx-0.20.2/xlsx-0.20.2.tgz#0f64eeed3f1a46e64724620c3553f2dbd3cd2d7d" xml2js@^0.6.2: version "0.6.2" resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.6.2.tgz#dd0b630083aa09c161e25a4d0901e2b2a929b499" integrity sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA== dependencies: sax ">=0.6.0" xmlbuilder "~11.0.0" xmlbuilder@^10.0.0: version "10.1.1" resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-10.1.1.tgz#8cae6688cc9b38d850b7c8d3c0a4161dcaf475b0" integrity sha512-OyzrcFLL/nb6fMGHbiRDuPup9ljBycsdCypwuyg5AAHvyWzGfChJpCXMG88AGTIMFhGZ9RccFN1e6lhg3hkwKg== xmlbuilder@~11.0.0: version "11.0.1" resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3" integrity sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA== xtend@^4.0.0: version "4.0.2" resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54" integrity sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ== y18n@^5.0.5: version "5.0.8" resolved "https://registry.yarnpkg.com/y18n/-/y18n-5.0.8.tgz#7f4934d0f7ca8c56f95314939ddcd2dd91ce1d55" integrity sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA== yaml@^2.2.1: version "2.8.1" resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.8.1.tgz#1870aa02b631f7e8328b93f8bc574fac5d6c4d79" integrity sha512-lcYcMxX2PO9XMGvAJkJ3OsNMw+/7FKes7/hgerGUYWIoWu5j/+YQqcZr5JnPZWzOsEBgMbSbiSTn/dv/69Mkpw== yargs-parser@^21.1.1: version "21.1.1" resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-21.1.1.tgz#9096bceebf990d21bb31fa9516e0ede294a77d35" integrity sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw== yargs@17.7.2: version "17.7.2" resolved "https://registry.yarnpkg.com/yargs/-/yargs-17.7.2.tgz#991df39aca675a192b816e1e0363f9d75d2aa269" integrity sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w== dependencies: cliui "^8.0.1" escalade "^3.1.1" get-caller-file "^2.0.5" require-directory "^2.1.1" string-width "^4.2.3" y18n "^5.0.5" yargs-parser "^21.1.1" yauzl@^2.10.0, yauzl@^2.4.2: version "2.10.0" resolved "https://registry.yarnpkg.com/yauzl/-/yauzl-2.10.0.tgz#c7eb17c93e112cb1086fa6d8e51fb0667b79a5f9" integrity sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g== dependencies: buffer-crc32 "~0.2.3" fd-slicer "~1.1.0" yocto-queue@^0.1.0: version "0.1.0" resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-0.1.0.tgz#0294eb3dee05028d31ee1a5fa2c556a6aaf10a1b" integrity sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q== youtube-transcript-plus@^1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/youtube-transcript-plus/-/youtube-transcript-plus-1.1.2.tgz#f86851852a056088c11f4f6523ab0f8dba7d9711" integrity sha512-bLlqkA6gVVUorZpcc+THuECXyAwOpnHqW2lOav9g6gGovxAP3FCD8s9GBFVjmSl3cWWwwPPXtG/zY1nD+GvQ7A== youtubei.js@^9.1.0: version "9.4.0" resolved "https://registry.yarnpkg.com/youtubei.js/-/youtubei.js-9.4.0.tgz#ccccaf4a295b96e3e17134a66730bbc82461594b" integrity sha512-8plCOZD2WabqWSEgZU3RjzigIIeR7sF028EERJENYrC9xO/6awpLMZfeoE1gNrNEbKcA+bzbMvonqlvBdxGdKg== dependencies: jintr "^1.1.0" tslib "^2.5.0" undici "^5.19.1" zlibjs@^0.3.1: version "0.3.1" resolved "https://registry.yarnpkg.com/zlibjs/-/zlibjs-0.3.1.tgz#50197edb28a1c42ca659cc8b4e6a9ddd6d444554" integrity sha512-+J9RrgTKOmlxFSDHo0pI1xM6BLVUv+o0ZT9ANtCxGkjIVCCUdx9alUF8Gm+dGLKbkkkidWIHFDZHDMpfITt4+w== zod-to-json-schema@^3.22.3, zod-to-json-schema@^3.22.5: version "3.25.0" resolved "https://registry.yarnpkg.com/zod-to-json-schema/-/zod-to-json-schema-3.25.0.tgz#df504c957c4fb0feff467c74d03e6aab0b013e1c" integrity sha512-HvWtU2UG41LALjajJrML6uQejQhNJx+JBO9IflpSja4R03iNWfKXrj6W2h7ljuLyc1nKS+9yDyL/9tD1U/yBnQ== zod@^3.22.3, zod@^3.22.4: version "3.25.76" resolved "https://registry.yarnpkg.com/zod/-/zod-3.25.76.tgz#26841c3f6fd22a6a2760e7ccb719179768471e34" integrity sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ== ================================================ FILE: docker/.env.example ================================================ SERVER_PORT=3001 STORAGE_DIR="/app/server/storage" UID='1000' GID='1000' # SIG_KEY='passphrase' # Please generate random string at least 32 chars long. # SIG_SALT='salt' # Please generate random string at least 32 chars long. # JWT_SECRET="my-random-string-for-seeding" # Only needed if AUTH_TOKEN is set. Please generate random string at least 12 chars long. # JWT_EXPIRY="30d" # (optional) https://docs.anythingllm.com/configuration#custom-ttl-for-sessions ########################################### ######## LLM API SElECTION ################ ########################################### # LLM_PROVIDER='openai' # OPEN_AI_KEY= # OPEN_MODEL_PREF='gpt-4o' # LLM_PROVIDER='gemini' # GEMINI_API_KEY= # GEMINI_LLM_MODEL_PREF='gemini-2.0-flash-lite' # LLM_PROVIDER='azure' # AZURE_OPENAI_ENDPOINT= # AZURE_OPENAI_KEY= # AZURE_OPENAI_MODEL_PREF='my-gpt35-deployment' # This is the "deployment" on Azure you want to use. Not the base model. # EMBEDDING_MODEL_PREF='embedder-model' # This is the "deployment" on Azure you want to use for embeddings. Not the base model. Valid base model is text-embedding-ada-002 # LLM_PROVIDER='anthropic' # ANTHROPIC_API_KEY=sk-ant-xxxx # ANTHROPIC_MODEL_PREF='claude-2' # ANTHROPIC_CACHE_CONTROL="5m" # Enable prompt caching (5m=5min cache, 1h=1hour cache). Reduces costs and improves speed by caching system prompts. # LLM_PROVIDER='lmstudio' # LMSTUDIO_BASE_PATH='http://your-server:1234/v1' # LMSTUDIO_MODEL_PREF='Loaded from Chat UI' # this is a bug in LMStudio 0.2.17 # LMSTUDIO_MODEL_TOKEN_LIMIT=4096 # LMSTUDIO_AUTH_TOKEN='your-lmstudio-auth-token-here' # LLM_PROVIDER='localai' # LOCAL_AI_BASE_PATH='http://host.docker.internal:8080/v1' # LOCAL_AI_MODEL_PREF='luna-ai-llama2' # LOCAL_AI_MODEL_TOKEN_LIMIT=4096 # LOCAL_AI_API_KEY="sk-123abc" # LLM_PROVIDER='ollama' # OLLAMA_BASE_PATH='http://host.docker.internal:11434' # OLLAMA_MODEL_PREF='llama2' # OLLAMA_MODEL_TOKEN_LIMIT=4096 # OLLAMA_AUTH_TOKEN='your-ollama-auth-token-here (optional, only for ollama running behind auth - Bearer token)' # OLLAMA_RESPONSE_TIMEOUT=7200000 (optional, max timeout in milliseconds for ollama response to conclude. Default is 5min before aborting) # LLM_PROVIDER='togetherai' # TOGETHER_AI_API_KEY='my-together-ai-key' # TOGETHER_AI_MODEL_PREF='mistralai/Mixtral-8x7B-Instruct-v0.1' # LLM_PROVIDER='mistral' # MISTRAL_API_KEY='example-mistral-ai-api-key' # MISTRAL_MODEL_PREF='mistral-tiny' # LLM_PROVIDER='perplexity' # PERPLEXITY_API_KEY='my-perplexity-key' # PERPLEXITY_MODEL_PREF='codellama-34b-instruct' # LLM_PROVIDER='openrouter' # OPENROUTER_API_KEY='my-openrouter-key' # OPENROUTER_MODEL_PREF='openrouter/auto' # LLM_PROVIDER='huggingface' # HUGGING_FACE_LLM_ENDPOINT=https://uuid-here.us-east-1.aws.endpoints.huggingface.cloud # HUGGING_FACE_LLM_API_KEY=hf_xxxxxx # HUGGING_FACE_LLM_TOKEN_LIMIT=8000 # LLM_PROVIDER='groq' # GROQ_API_KEY=gsk_abcxyz # GROQ_MODEL_PREF=llama3-8b-8192 # LLM_PROVIDER='koboldcpp' # KOBOLD_CPP_BASE_PATH='http://127.0.0.1:5000/v1' # KOBOLD_CPP_MODEL_PREF='koboldcpp/codellama-7b-instruct.Q4_K_S' # KOBOLD_CPP_MODEL_TOKEN_LIMIT=4096 # LLM_PROVIDER='textgenwebui' # TEXT_GEN_WEB_UI_BASE_PATH='http://127.0.0.1:5000/v1' # TEXT_GEN_WEB_UI_TOKEN_LIMIT=4096 # TEXT_GEN_WEB_UI_API_KEY='sk-123abc' # LLM_PROVIDER='generic-openai' # GENERIC_OPEN_AI_BASE_PATH='http://proxy.url.openai.com/v1' # GENERIC_OPEN_AI_MODEL_PREF='gpt-3.5-turbo' # GENERIC_OPEN_AI_MODEL_TOKEN_LIMIT=4096 # GENERIC_OPEN_AI_API_KEY=sk-123abc # GENERIC_OPEN_AI_CUSTOM_HEADERS="X-Custom-Auth:my-secret-key,X-Custom-Header:my-value" (useful if using a proxy that requires authentication or other headers) # LLM_PROVIDER='litellm' # LITE_LLM_MODEL_PREF='gpt-3.5-turbo' # LITE_LLM_MODEL_TOKEN_LIMIT=4096 # LITE_LLM_BASE_PATH='http://127.0.0.1:4000' # LITE_LLM_API_KEY='sk-123abc' # LLM_PROVIDER='novita' # NOVITA_LLM_API_KEY='your-novita-api-key-here' check on https://novita.ai/settings/key-management # NOVITA_LLM_MODEL_PREF='deepseek/deepseek-r1' # LLM_PROVIDER='cometapi' # COMETAPI_LLM_API_KEY='your-cometapi-api-key-here' # Get one at https://api.cometapi.com/console/token # COMETAPI_LLM_MODEL_PREF='gpt-5-mini' # COMETAPI_LLM_TIMEOUT_MS=500 # Optional; stream idle timeout in ms (min 500ms) # LLM_PROVIDER='cohere' # COHERE_API_KEY= # COHERE_MODEL_PREF='command-r' # LLM_PROVIDER='bedrock' # AWS_BEDROCK_LLM_ACCESS_KEY_ID= # AWS_BEDROCK_LLM_ACCESS_KEY= # AWS_BEDROCK_LLM_REGION=us-west-2 # AWS_BEDROCK_LLM_MODEL_PREFERENCE=meta.llama3-1-8b-instruct-v1:0 # AWS_BEDROCK_LLM_MODEL_TOKEN_LIMIT=8191 # AWS_BEDROCK_LLM_CONNECTION_METHOD=iam # AWS_BEDROCK_LLM_MAX_OUTPUT_TOKENS=4096 # AWS_BEDROCK_LLM_SESSION_TOKEN= # Only required if CONNECTION_METHOD is 'sessionToken' # or even use Short and Long Term API keys # AWS_BEDROCK_LLM_CONNECTION_METHOD="apiKey" # AWS_BEDROCK_LLM_API_KEY= # LLM_PROVIDER='fireworksai' # FIREWORKS_AI_LLM_API_KEY='my-fireworks-ai-key' # FIREWORKS_AI_LLM_MODEL_PREF='accounts/fireworks/models/llama-v3p1-8b-instruct' # LLM_PROVIDER='apipie' # APIPIE_LLM_API_KEY='sk-123abc' # APIPIE_LLM_MODEL_PREF='openrouter/llama-3.1-8b-instruct' # LLM_PROVIDER='xai' # XAI_LLM_API_KEY='xai-your-api-key-here' # XAI_LLM_MODEL_PREF='grok-beta' # LLM_PROVIDER='zai' # ZAI_API_KEY="your-zai-api-key-here" # ZAI_MODEL_PREF="glm-4.5" # LLM_PROVIDER='nvidia-nim' # NVIDIA_NIM_LLM_BASE_PATH='http://127.0.0.1:8000' # NVIDIA_NIM_LLM_MODEL_PREF='meta/llama-3.2-3b-instruct' # LLM_PROVIDER='deepseek' # DEEPSEEK_API_KEY='your-deepseek-api-key-here' # DEEPSEEK_MODEL_PREF='deepseek-chat' # LLM_PROVIDER='ppio' # PPIO_API_KEY='your-ppio-api-key-here' # PPIO_MODEL_PREF=deepseek/deepseek-v3/community # LLM_PROVIDER='moonshotai' # MOONSHOT_AI_API_KEY='your-moonshot-api-key-here' # MOONSHOT_AI_MODEL_PREF='moonshot-v1-32k' # LLM_PROVIDER='foundry' # FOUNDRY_BASE_PATH='http://127.0.0.1:55776' # FOUNDRY_MODEL_PREF='phi-3.5-mini' # FOUNDRY_MODEL_TOKEN_LIMIT=4096 # LLM_PROVIDER='giteeai' # GITEE_AI_API_KEY= # GITEE_AI_MODEL_PREF= # GITEE_AI_MODEL_TOKEN_LIMIT= # LLM_PROVIDER='docker-model-runner' # DOCKER_MODEL_RUNNER_BASE_PATH='http://127.0.0.1:12434' # DOCKER_MODEL_RUNNER_LLM_MODEL_PREF='phi-3.5-mini' # DOCKER_MODEL_RUNNER_LLM_MODEL_TOKEN_LIMIT=4096 # LLM_PROVIDER='privatemode' # PRIVATEMODE_LLM_BASE_PATH='http://127.0.0.1:8080' # PRIVATEMODE_LLM_MODEL_PREF='gemma-3-27b' # LLM_PROVIDER='sambanova' # SAMBANOVA_LLM_API_KEY='xxx-xxx-xxx' # SAMBANOVA_LLM_MODEL_PREF='gpt-oss-120b' ########################################### ######## Embedding API SElECTION ########## ########################################### # This will be the assumed default embedding seleciton and model # EMBEDDING_ENGINE='native' # EMBEDDING_MODEL_PREF='Xenova/all-MiniLM-L6-v2' # Only used if you are using an LLM that does not natively support embedding (openai or Azure) # EMBEDDING_ENGINE='openai' # OPEN_AI_KEY=sk-xxxx # EMBEDDING_MODEL_PREF='text-embedding-ada-002' # EMBEDDING_ENGINE='azure' # AZURE_OPENAI_ENDPOINT= # AZURE_OPENAI_KEY= # EMBEDDING_MODEL_PREF='my-embedder-model' # This is the "deployment" on Azure you want to use for embeddings. Not the base model. Valid base model is text-embedding-ada-002 # EMBEDDING_ENGINE='localai' # EMBEDDING_BASE_PATH='http://localhost:8080/v1' # EMBEDDING_MODEL_PREF='text-embedding-ada-002' # EMBEDDING_MODEL_MAX_CHUNK_LENGTH=1000 # The max chunk size in chars a string to embed can be # EMBEDDING_ENGINE='ollama' # EMBEDDING_BASE_PATH='http://host.docker.internal:11434' # EMBEDDING_MODEL_PREF='nomic-embed-text:latest' # EMBEDDING_MODEL_MAX_CHUNK_LENGTH=8192 # EMBEDDING_ENGINE='lmstudio' # EMBEDDING_BASE_PATH='https://host.docker.internal:1234/v1' # EMBEDDING_MODEL_PREF='nomic-ai/nomic-embed-text-v1.5-GGUF/nomic-embed-text-v1.5.Q4_0.gguf' # EMBEDDING_MODEL_MAX_CHUNK_LENGTH=8192 # EMBEDDING_ENGINE='cohere' # COHERE_API_KEY= # EMBEDDING_MODEL_PREF='embed-english-v3.0' # EMBEDDING_ENGINE='voyageai' # VOYAGEAI_API_KEY= # EMBEDDING_MODEL_PREF='voyage-large-2-instruct' # EMBEDDING_ENGINE='litellm' # EMBEDDING_MODEL_PREF='text-embedding-ada-002' # EMBEDDING_MODEL_MAX_CHUNK_LENGTH=8192 # LITE_LLM_BASE_PATH='http://127.0.0.1:4000' # LITE_LLM_API_KEY='sk-123abc' # EMBEDDING_ENGINE='generic-openai' # EMBEDDING_MODEL_PREF='text-embedding-ada-002' # EMBEDDING_MODEL_MAX_CHUNK_LENGTH=8192 # EMBEDDING_BASE_PATH='http://127.0.0.1:4000' # GENERIC_OPEN_AI_EMBEDDING_API_KEY='sk-123abc' # GENERIC_OPEN_AI_EMBEDDING_MAX_CONCURRENT_CHUNKS=500 # GENERIC_OPEN_AI_EMBEDDING_API_DELAY_MS=1000 # EMBEDDING_ENGINE='gemini' # GEMINI_EMBEDDING_API_KEY= # EMBEDDING_MODEL_PREF='text-embedding-004' # EMBEDDING_ENGINE='openrouter' # EMBEDDING_MODEL_PREF='baai/bge-m3' # OPENROUTER_API_KEY='' # EMBEDDING_ENGINE='lemonade' # EMBEDDING_BASE_PATH='http://127.0.0.1:8000' # EMBEDDING_MODEL_PREF='Qwen3-embedder' # EMBEDDING_MODEL_MAX_CHUNK_LENGTH=8192 ########################################### ######## Vector Database Selection ######## ########################################### # Enable all below if you are using vector database: LanceDB. # VECTOR_DB="lancedb" # Enable all below if you are using vector database: Weaviate. # VECTOR_DB="pgvector" # PGVECTOR_CONNECTION_STRING="postgresql://dbuser:dbuserpass@localhost:5432/yourdb" # PGVECTOR_TABLE_NAME="anythingllm_vectors" # optional, but can be defined # Enable all below if you are using vector database: Chroma. # VECTOR_DB="chroma" # CHROMA_ENDPOINT='http://host.docker.internal:8000' # CHROMA_API_HEADER="X-Api-Key" # CHROMA_API_KEY="sk-123abc" # Enable all below if you are using vector database: Chroma Cloud. # VECTOR_DB="chromacloud" # CHROMACLOUD_API_KEY="ck-your-api-key" # CHROMACLOUD_TENANT= # CHROMACLOUD_DATABASE= # Enable all below if you are using vector database: Pinecone. # VECTOR_DB="pinecone" # PINECONE_API_KEY= # PINECONE_INDEX= # Enable all below if you are using vector database: Weaviate. # VECTOR_DB="weaviate" # WEAVIATE_ENDPOINT="http://localhost:8080" # WEAVIATE_API_KEY= # Enable all below if you are using vector database: Qdrant. # VECTOR_DB="qdrant" # QDRANT_ENDPOINT="http://localhost:6333" # QDRANT_API_KEY= # Enable all below if you are using vector database: Milvus. # VECTOR_DB="milvus" # MILVUS_ADDRESS="http://localhost:19530" # MILVUS_USERNAME= # MILVUS_PASSWORD= # Enable all below if you are using vector database: Zilliz Cloud. # VECTOR_DB="zilliz" # ZILLIZ_ENDPOINT="https://sample.api.gcp-us-west1.zillizcloud.com" # ZILLIZ_API_TOKEN=api-token-here # Enable all below if you are using vector database: Astra DB. # VECTOR_DB="astra" # ASTRA_DB_APPLICATION_TOKEN= # ASTRA_DB_ENDPOINT= ########################################### ######## Audio Model Selection ############ ########################################### # (default) use built-in whisper-small model. # WHISPER_PROVIDER="local" # use openai hosted whisper model. # WHISPER_PROVIDER="openai" # OPEN_AI_KEY=sk-xxxxxxxx ########################################### ######## TTS/STT Model Selection ########## ########################################### # TTS_PROVIDER="native" # TTS_PROVIDER="openai" # TTS_OPEN_AI_KEY=sk-example # TTS_OPEN_AI_VOICE_MODEL=nova # TTS_PROVIDER="generic-openai" # TTS_OPEN_AI_COMPATIBLE_KEY=sk-example # TTS_OPEN_AI_COMPATIBLE_MODEL=tts-1 # TTS_OPEN_AI_COMPATIBLE_VOICE_MODEL=nova # TTS_OPEN_AI_COMPATIBLE_ENDPOINT="https://api.openai.com/v1" # TTS_PROVIDER="elevenlabs" # TTS_ELEVEN_LABS_KEY= # TTS_ELEVEN_LABS_VOICE_MODEL=21m00Tcm4TlvDq8ikWAM # Rachel # CLOUD DEPLOYMENT VARIRABLES ONLY # AUTH_TOKEN="hunter2" # This is the password to your application if remote hosting. # DISABLE_TELEMETRY="false" ########################################### ######## PASSWORD COMPLEXITY ############## ########################################### # Enforce a password schema for your organization users. # Documentation on how to use https://github.com/kamronbatman/joi-password-complexity # Default is only 8 char minimum # PASSWORDMINCHAR=8 # PASSWORDMAXCHAR=250 # PASSWORDLOWERCASE=1 # PASSWORDUPPERCASE=1 # PASSWORDNUMERIC=1 # PASSWORDSYMBOL=1 # PASSWORDREQUIREMENTS=4 ########################################### ######## ENABLE HTTPS SERVER ############## ########################################### # By enabling this and providing the path/filename for the key and cert, # the server will use HTTPS instead of HTTP. #ENABLE_HTTPS="true" #HTTPS_CERT_PATH="sslcert/cert.pem" #HTTPS_KEY_PATH="sslcert/key.pem" ########################################### ######## AGENT SERVICE KEYS ############### ########################################### #------ SEARCH ENGINES ------- #============================= #------ Google Search -------- https://programmablesearchengine.google.com/controlpanel/create # AGENT_GSE_KEY= # AGENT_GSE_CTX= #------ SearchApi.io ----------- https://www.searchapi.io/ # AGENT_SEARCHAPI_API_KEY= # AGENT_SEARCHAPI_ENGINE=google #------ SerpApi ----------- https://serpapi.com/ # AGENT_SERPAPI_API_KEY= # AGENT_SERPAPI_ENGINE=google #------ Serper.dev ----------- https://serper.dev/ # AGENT_SERPER_DEV_KEY= #------ Bing Search ----------- https://portal.azure.com/ # AGENT_BING_SEARCH_API_KEY= #------ Serply.io ----------- https://serply.io/ # AGENT_SERPLY_API_KEY= #------ SearXNG ----------- https://github.com/searxng/searxng # AGENT_SEARXNG_API_URL= #------ Tavily ----------- https://www.tavily.com/ # AGENT_TAVILY_API_KEY= #------ Exa Search ----------- https://www.exa.ai/ # AGENT_EXA_API_KEY= #------ Perplexity Search ----------- [https://console.perplexity.ai](https://console.perplexity.ai) # AGENT_PERPLEXITY_API_KEY= ########################################### ######## Other Configurations ############ ########################################### # Disable viewing chat history from the UI and frontend APIs. # See https://docs.anythingllm.com/configuration#disable-view-chat-history for more information. # DISABLE_VIEW_CHAT_HISTORY=1 # Enable simple SSO passthrough to pre-authenticate users from a third party service. # See https://docs.anythingllm.com/configuration#simple-sso-passthrough for more information. # SIMPLE_SSO_ENABLED=1 # SIMPLE_SSO_NO_LOGIN=1 # SIMPLE_SSO_NO_LOGIN_REDIRECT=https://your-custom-login-url.com (optional) # Allow scraping of any IP address in collector - must be string "true" to be enabled # See https://docs.anythingllm.com/configuration#local-ip-address-scraping for more information. # COLLECTOR_ALLOW_ANY_IP="true" # Specify the target languages for when using OCR to parse images and PDFs. # This is a comma separated list of language codes as a string. Unsupported languages will be ignored. # Default is English. See https://tesseract-ocr.github.io/tessdoc/Data-Files-in-different-versions.html for a list of valid language codes. # TARGET_OCR_LANG=eng,deu,ita,spa,fra,por,rus,nld,tur,hun,pol,ita,spa,fra,por,rus,nld,tur,hun,pol # Runtime flags for built-in pupeeteer Chromium instance # This is only required on Linux machines running AnythingLLM via Docker # and do not want to use the --cap-add=SYS_ADMIN docker argument # ANYTHINGLLM_CHROMIUM_ARGS="--no-sandbox,--disable-setuid-sandbox" # Disable Swagger API documentation endpoint. # Set to "true" to disable the /api/docs endpoint (recommended for production deployments). # DISABLE_SWAGGER_DOCS="true" # Disable MCP cooldown timer for agent calls # this can lead to infinite recursive calls of the same function # for some model/provider combinations # MCP_NO_COOLDOWN="true # Allow native tool calling for specific providers. # This can VASTLY improve performance and speed of agent calls. # Check code for supported providers who can be enabled here via this flag # PROVIDER_SUPPORTS_NATIVE_TOOL_CALLING="generic-openai,bedrock,localai,groq,litellm,openrouter" # (optional) Maximum number of tools an agent can chain for a single response. # This prevents some lower-end models from infinite recursive tool calls. # AGENT_MAX_TOOL_CALLS=10 # Enable agent tool reranking to reduce token usage by selecting only the most relevant tools # for each query. Uses the native embedding reranker to score tools against the user's prompt. # Set to "true" to enable. This can reduce token costs by 80% when you have # many tools/MCP servers enabled. # AGENT_SKILL_RERANKER_ENABLED="true" # AGENT_SKILL_RERANKER_TOP_N=15 # (optional) Number of top tools to keep after reranking (default: 15) ================================================ FILE: docker/Dockerfile ================================================ # Setup base image FROM ubuntu:noble-20251013 AS base # Build arguments ARG ARG_UID=1000 ARG ARG_GID=1000 FROM base AS build-arm64 RUN echo "Preparing build of AnythingLLM image for arm64 architecture" SHELL ["/bin/bash", "-o", "pipefail", "-c"] # Install system dependencies # hadolint ignore=DL3008,DL3013 RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \ unzip curl gnupg libgfortran5 libgbm1 tzdata netcat-openbsd \ libasound2t64 libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 \ libgcc1 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libx11-6 libx11-xcb1 libxcb1 \ libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 \ libxss1 libxtst6 ca-certificates fonts-liberation libappindicator3-1 libnss3 lsb-release \ xdg-utils git build-essential ffmpeg && \ mkdir -p /etc/apt/keyrings && \ curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \ echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_18.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list && \ apt-get update && \ # Install node and yarn apt-get install -yq --no-install-recommends nodejs && \ curl -LO https://github.com/yarnpkg/yarn/releases/download/v1.22.19/yarn_1.22.19_all.deb \ && dpkg -i yarn_1.22.19_all.deb \ && rm yarn_1.22.19_all.deb && \ # Install uvx (pinned to 0.6.10) for MCP support curl -LsSf https://astral.sh/uv/0.6.10/install.sh | sh && \ mv /root/.local/bin/uv /usr/local/bin/uv && \ mv /root/.local/bin/uvx /usr/local/bin/uvx && \ echo "Installed uvx! $(uv --version)" && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* # Create a group and user with specific UID and GID # First, remove any existing user/group with the target UID/GID to avoid conflicts RUN (getent passwd "$ARG_UID" && userdel -f "$(getent passwd "$ARG_UID" | cut -d: -f1)") || true && \ (getent group "$ARG_GID" && groupdel "$(getent group "$ARG_GID" | cut -d: -f1)") || true && \ groupadd -g "$ARG_GID" anythingllm && \ useradd -l -u "$ARG_UID" -m -d /app -s /bin/bash -g anythingllm anythingllm && \ mkdir -p /app/frontend/ /app/server/ /app/collector/ && chown -R anythingllm:anythingllm /app # Copy docker helper scripts COPY ./docker/docker-entrypoint.sh /usr/local/bin/ COPY ./docker/docker-healthcheck.sh /usr/local/bin/ COPY --chown=anythingllm:anythingllm ./docker/.env.example /app/server/.env # Ensure the scripts are executable RUN chmod +x /usr/local/bin/docker-entrypoint.sh && \ chmod +x /usr/local/bin/docker-healthcheck.sh USER anythingllm WORKDIR /app # Puppeteer does not ship with an ARM86 compatible build for Chromium # so web-scraping would be broken in arm docker containers unless we patch it # by manually installing a compatible chromedriver. RUN echo "Need to patch Puppeteer x Chromium support for ARM86 - installing dep!" && \ curl -fSL https://webassets.anythingllm.com/chromium-1088-linux-arm64.zip -o chrome-linux.zip && \ unzip chrome-linux.zip && \ rm -rf chrome-linux.zip ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true ENV CHROME_PATH=/app/chrome-linux/chrome ENV PUPPETEER_EXECUTABLE_PATH=/app/chrome-linux/chrome RUN echo "Done running arm64 specific installation steps" ############################################# # amd64-specific stage FROM base AS build-amd64 RUN echo "Preparing build of AnythingLLM image for non-ARM architecture" SHELL ["/bin/bash", "-o", "pipefail", "-c"] # Install system dependencies # hadolint ignore=DL3008,DL3013 RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \ curl gnupg libgfortran5 libgbm1 tzdata netcat-openbsd \ libasound2t64 libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 \ libgcc1 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libx11-6 libx11-xcb1 libxcb1 \ libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 \ libxss1 libxtst6 ca-certificates fonts-liberation libappindicator3-1 libnss3 lsb-release \ xdg-utils git build-essential ffmpeg && \ mkdir -p /etc/apt/keyrings && \ curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \ echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_18.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list && \ apt-get update && \ # Install node and yarn apt-get install -yq --no-install-recommends nodejs && \ curl -LO https://github.com/yarnpkg/yarn/releases/download/v1.22.19/yarn_1.22.19_all.deb \ && dpkg -i yarn_1.22.19_all.deb \ && rm yarn_1.22.19_all.deb && \ # Install uvx (pinned to 0.6.10) for MCP support curl -LsSf https://astral.sh/uv/0.6.10/install.sh | sh && \ mv /root/.local/bin/uv /usr/local/bin/uv && \ mv /root/.local/bin/uvx /usr/local/bin/uvx && \ echo "Installed uvx! $(uv --version)" && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* # Create a group and user with specific UID and GID # First, remove any existing user/group with the target UID/GID to avoid conflicts RUN (getent passwd "$ARG_UID" && userdel -f "$(getent passwd "$ARG_UID" | cut -d: -f1)") || true && \ (getent group "$ARG_GID" && groupdel "$(getent group "$ARG_GID" | cut -d: -f1)") || true && \ groupadd -g "$ARG_GID" anythingllm && \ useradd -l -u "$ARG_UID" -m -d /app -s /bin/bash -g anythingllm anythingllm && \ mkdir -p /app/frontend/ /app/server/ /app/collector/ && chown -R anythingllm:anythingllm /app # Copy docker helper scripts COPY ./docker/docker-entrypoint.sh /usr/local/bin/ COPY ./docker/docker-healthcheck.sh /usr/local/bin/ COPY --chown=anythingllm:anythingllm ./docker/.env.example /app/server/.env # Ensure the scripts are executable RUN chmod +x /usr/local/bin/docker-entrypoint.sh && \ chmod +x /usr/local/bin/docker-healthcheck.sh ############################################# # COMMON BUILD FLOW FOR ALL ARCHS ############################################# # hadolint ignore=DL3006 FROM build-${TARGETARCH} AS build RUN echo "Running common build flow of AnythingLLM image for all architectures" USER anythingllm WORKDIR /app # Install & Build frontend layer # Use BUILDPLATFORM to run on the native host architecture (not emulated). # This avoids esbuild crashing under QEMU when cross-compiling. # The output (static HTML/CSS/JS) is platform-independent. FROM --platform=$BUILDPLATFORM node:18-slim AS frontend-build WORKDIR /app/frontend COPY ./frontend/package.json ./frontend/yarn.lock ./ RUN yarn install --network-timeout 100000 && yarn cache clean COPY ./frontend/ ./ RUN yarn build WORKDIR /app # Install server layer # Also pull and build collector deps (chromium issues prevent bad bindings) FROM build AS backend-build COPY --chown=anythingllm:anythingllm ./server /app/server/ WORKDIR /app/server RUN yarn install --production --network-timeout 100000 && yarn cache clean WORKDIR /app # Install collector dependencies COPY --chown=anythingllm:anythingllm ./collector/ ./collector/ WORKDIR /app/collector ENV PUPPETEER_DOWNLOAD_BASE_URL=https://storage.googleapis.com/chrome-for-testing-public RUN yarn install --production --network-timeout 100000 && yarn cache clean WORKDIR /app USER anythingllm # Since we are building from backend-build we just need to move built frontend into server/public FROM backend-build AS production-build WORKDIR /app COPY --chown=anythingllm:anythingllm --from=frontend-build /app/frontend/dist /app/server/public # Setup the environment ENV NODE_ENV=production ENV ANYTHING_LLM_RUNTIME=docker ENV DEPLOYMENT_VERSION=1.11.2 # Setup the healthcheck HEALTHCHECK --interval=1m --timeout=10s --start-period=1m \ CMD /bin/bash /usr/local/bin/docker-healthcheck.sh || exit 1 # Run the server # CMD ["sh", "-c", "tail -f /dev/null"] # For development: keep container open ENTRYPOINT ["/bin/bash", "/usr/local/bin/docker-entrypoint.sh"] ================================================ FILE: docker/HOW_TO_USE_DOCKER.md ================================================ # How to use Dockerized Anything LLM Use the Dockerized version of AnythingLLM for a much faster and complete startup of AnythingLLM. ### Minimum Requirements > [!TIP] > Running AnythingLLM on AWS/GCP/Azure? > You should aim for at least 2GB of RAM. Disk storage is proportional to however much data > you will be storing (documents, vectors, models, etc). Minimum 10GB recommended. - `docker` installed on your machine - `yarn` and `node` on your machine - access to an LLM running locally or remotely \*AnythingLLM by default uses a built-in vector database powered by [LanceDB](https://github.com/lancedb/lancedb) \*AnythingLLM by default embeds text on instance privately [Learn More](../server/storage/models/README.md) ## Recommend way to run dockerized AnythingLLM! > [!IMPORTANT] > If you are running another service on localhost like Chroma, LocalAi, or LMStudio > you will need to use http://host.docker.internal:xxxx to access the service from within > the docker container using AnythingLLM as `localhost:xxxx` will not resolve for the host system. > > **Requires** Docker v18.03+ on Win/Mac and 20.10+ on Linux/Ubuntu for host.docker.internal to resolve! > > _Linux_: add `--add-host=host.docker.internal:host-gateway` to docker run command for this to resolve. > > eg: Chroma host URL running on localhost:8000 on host machine needs to be http://host.docker.internal:8000 > when used in AnythingLLM. > [!TIP] > It is best to mount the containers storage volume to a folder on your host machine > so that you can pull in future updates without deleting your existing data! Pull in the latest image from docker. Supports both `amd64` and `arm64` CPU architectures. ```shell docker pull mintplexlabs/anythingllm ```
Mount the storage locally and run AnythingLLM in Docker
Linux/MacOs ```shell export STORAGE_LOCATION=$HOME/anythingllm && \ mkdir -p $STORAGE_LOCATION && \ touch "$STORAGE_LOCATION/.env" && \ docker run -d -p 3001:3001 \ --cap-add SYS_ADMIN \ -v ${STORAGE_LOCATION}:/app/server/storage \ -v ${STORAGE_LOCATION}/.env:/app/server/.env \ -e STORAGE_DIR="/app/server/storage" \ mintplexlabs/anythingllm ```
Windows ```powershell # Run this in powershell terminal $env:STORAGE_LOCATION="$HOME\Documents\anythingllm"; ` If(!(Test-Path $env:STORAGE_LOCATION)) {New-Item $env:STORAGE_LOCATION -ItemType Directory}; ` If(!(Test-Path "$env:STORAGE_LOCATION\.env")) {New-Item "$env:STORAGE_LOCATION\.env" -ItemType File}; ` docker run -d -p 3001:3001 ` --cap-add SYS_ADMIN ` -v "$env:STORAGE_LOCATION`:/app/server/storage" ` -v "$env:STORAGE_LOCATION\.env:/app/server/.env" ` -e STORAGE_DIR="/app/server/storage" ` mintplexlabs/anythingllm; ```
Docker Compose ```yaml version: '3.8' services: anythingllm: image: mintplexlabs/anythingllm container_name: anythingllm ports: - "3001:3001" cap_add: - SYS_ADMIN environment: # Adjust for your environment - STORAGE_DIR=/app/server/storage - JWT_SECRET="make this a large list of random numbers and letters 20+" - LLM_PROVIDER=ollama - OLLAMA_BASE_PATH=http://127.0.0.1:11434 - OLLAMA_MODEL_PREF=llama2 - OLLAMA_MODEL_TOKEN_LIMIT=4096 - EMBEDDING_ENGINE=ollama - EMBEDDING_BASE_PATH=http://127.0.0.1:11434 - EMBEDDING_MODEL_PREF=nomic-embed-text:latest - EMBEDDING_MODEL_MAX_CHUNK_LENGTH=8192 - VECTOR_DB=lancedb - WHISPER_PROVIDER=local - TTS_PROVIDER=native - PASSWORDMINCHAR=8 # Add any other keys here for services or settings # you can find in the docker/.env.example file volumes: - anythingllm_storage:/app/server/storage restart: always volumes: anythingllm_storage: driver: local driver_opts: type: none o: bind device: /path/on/local/disk ```
Go to `http://localhost:3001` and you are now using AnythingLLM! All your data and progress will persist between container rebuilds or pulls from Docker Hub. ## How to use the user interface - To access the full application, visit `http://localhost:3001` in your browser. ## About UID and GID in the ENV - The UID and GID are set to 1000 by default. This is the default user in the Docker container and on most host operating systems. If there is a mismatch between your host user UID and GID and what is set in the `.env` file, you may experience permission issues. ## Build locally from source _not recommended for casual use_ - `git clone` this repo and `cd anything-llm` to get to the root directory. - `touch server/storage/anythingllm.db` to create empty SQLite DB file. - `cd docker/` - `cp .env.example .env` **you must do this before building** - `docker-compose up -d --build` to build the image - this will take a few moments. Your docker host will show the image as online once the build process is completed. This will build the app to `http://localhost:3001`. ## Integrations and one-click setups The integrations below are templates or tooling built by the community to make running the docker experience of AnythingLLM easier. ### Use the Midori AI Subsystem to Manage AnythingLLM Follow the setup found on [Midori AI Subsystem Site](https://io.midori-ai.xyz/subsystem/manager/) for your host OS After setting that up install the AnythingLLM docker backend to the Midori AI Subsystem. Once that is done, you are all set! ## Common questions and fixes ### Cannot connect to service running on localhost! If you are in docker and cannot connect to a service running on your host machine running on a local interface or loopback: - `localhost` - `127.0.0.1` - `0.0.0.0` > [!IMPORTANT] > On linux `http://host.docker.internal:xxxx` does not work. > Use `http://172.17.0.1:xxxx` instead to emulate this functionality. Then in docker you need to replace that localhost part with `host.docker.internal`. For example, if running Ollama on the host machine, bound to http://127.0.0.1:11434 you should put `http://host.docker.internal:11434` into the connection URL in AnythingLLM. ### API is not working, cannot login, LLM is "offline"? You are likely running the docker container on a remote machine like EC2 or some other instance where the reachable URL is not `http://localhost:3001` and instead is something like `http://193.xx.xx.xx:3001` - in this case all you need to do is add the following to your `frontend/.env.production` before running `docker-compose up -d --build` ``` # frontend/.env.production GENERATE_SOURCEMAP=false VITE_API_BASE="http://:3001/api" ``` For example, if the docker instance is available on `192.186.1.222` your `VITE_API_BASE` would look like `VITE_API_BASE="http://192.186.1.222:3001/api"` in `frontend/.env.production`. ### Having issues with Ollama? If you are getting errors like `llama:streaming - could not stream chat. Error: connect ECONNREFUSED 172.17.0.1:11434` then visit the README below. [Fix common issues with Ollama](../server/utils/AiProviders/ollama/README.md) ### Still not working? [Ask for help on Discord](https://discord.gg/6UyHPeGZAC) ================================================ FILE: docker/docker-compose.yml ================================================ name: anythingllm networks: anything-llm: driver: bridge services: anything-llm: container_name: anythingllm build: context: ../. dockerfile: ./docker/Dockerfile args: ARG_UID: ${UID:-1000} ARG_GID: ${GID:-1000} cap_add: - SYS_ADMIN volumes: - "./.env:/app/server/.env" - "../server/storage:/app/server/storage" - "../collector/hotdir/:/app/collector/hotdir" - "../collector/outputs/:/app/collector/outputs" user: "${UID:-1000}:${GID:-1000}" ports: - "3001:3001" env_file: - .env networks: - anything-llm extra_hosts: - "host.docker.internal:host-gateway" ================================================ FILE: docker/docker-entrypoint.sh ================================================ #!/bin/bash # Check if STORAGE_DIR is set if [ -z "$STORAGE_DIR" ]; then echo "================================================================" echo "⚠️ ⚠️ ⚠️ WARNING: STORAGE_DIR environment variable is not set! ⚠️ ⚠️ ⚠️" echo "" echo "Not setting this will result in data loss on container restart since" echo "the application will not have a persistent storage location." echo "It can also result in weird errors in various parts of the application." echo "" echo "Please run the container with the official docker command at" echo "https://docs.anythingllm.com/installation-docker/quickstart" echo "" echo "⚠️ ⚠️ ⚠️ WARNING: STORAGE_DIR environment variable is not set! ⚠️ ⚠️ ⚠️" echo "================================================================" fi { cd /app/server/ && # Disable Prisma CLI telemetry (https://www.prisma.io/docs/orm/tools/prisma-cli#how-to-opt-out-of-data-collection) export CHECKPOINT_DISABLE=1 && npx prisma generate --schema=./prisma/schema.prisma && npx prisma migrate deploy --schema=./prisma/schema.prisma && node /app/server/index.js } & { node /app/collector/index.js; } & wait -n exit $? ================================================ FILE: docker/docker-healthcheck.sh ================================================ #!/bin/bash # Send a request to the specified URL response=$(curl --write-out '%{http_code}' --silent --output /dev/null http://localhost:3001/api/ping) # If the HTTP response code is 200 (OK), the server is up if [ "$response" -eq 200 ]; then echo "Server is up" exit 0 else echo "Server is down" exit 1 fi ================================================ FILE: docker/vex/CVE-2019-10790.vex.json ================================================ { "@context": "https://openvex.dev/ns/v0.2.0", "@id": "https://openvex.dev/docs/public/vex-6750d79bb005487e11d10f81d0b3ac92c47e6e259292c6b2d02558f9f4bca52d", "author": "tim@mintplexlabs.com", "timestamp": "2024-07-22T13:49:12.883675-07:00", "version": 1, "statements": [ { "vulnerability": { "name": "CVE-2019-10790" }, "timestamp": "2024-07-22T13:49:12.883678-07:00", "products": [ { "@id": "pkg:npm/taffydb@2.6.2" } ], "status": "not_affected", "justification": "vulnerable_code_not_in_execute_path" } ] } ================================================ FILE: docker/vex/CVE-2024-29415.vex.json ================================================ { "@context": "https://openvex.dev/ns/v0.2.0", "@id": "https://openvex.dev/docs/public/vex-939548c125c5bfebd3fd91e64c1c53bffacbde06b3611b4474ea90fa58045004", "author": "tim@mintplexlabs.com", "timestamp": "2024-07-19T16:08:47.147169-07:00", "version": 1, "statements": [ { "vulnerability": { "name": "CVE-2024-29415" }, "timestamp": "2024-07-19T16:08:47.147172-07:00", "products": [ { "@id": "pkg:npm/ip@2.0.0" } ], "status": "not_affected", "justification": "vulnerable_code_not_present" } ] } ================================================ FILE: docker/vex/CVE-2024-37890.vex.json ================================================ { "@context": "https://openvex.dev/ns/v0.2.0", "@id": "https://openvex.dev/docs/public/vex-939548c125c5bfebd3fd91e64c1c53bffacbde06b3611b4474ea90fa58045004", "author": "tim@mintplexlabs.com", "timestamp": "2024-07-19T16:08:47.147169-07:00", "version": 1, "statements": [ { "vulnerability": { "name": "CVE-2024-37890" }, "timestamp": "2024-07-19T16:08:47.147172-07:00", "products": [ { "@id": "pkg:npm/ws@8.14.2" } ], "status": "not_affected", "justification": "vulnerable_code_not_in_execute_path" } ] } ================================================ FILE: docker/vex/CVE-2024-4068.vex.json ================================================ { "@context": "https://openvex.dev/ns/v0.2.0", "@id": "https://openvex.dev/docs/public/vex-939548c125c5bfebd3fd91e64c1c53bffacbde06b3611b4474ea90fa58045004", "author": "tim@mintplexlabs.com", "timestamp": "2024-07-19T16:08:47.147169-07:00", "version": 1, "statements": [ { "vulnerability": { "name": "CVE-2024-4068" }, "timestamp": "2024-07-19T16:08:47.147172-07:00", "products": [ { "@id": "pkg:npm/braces@3.0.2" } ], "status": "not_affected", "justification": "vulnerable_code_not_present" } ] } ================================================ FILE: eslint.config.js ================================================ import globals from "./server/node_modules/globals/index.js" import eslintRecommended from "./server/node_modules/@eslint/js/src/index.js" import eslintConfigPrettier from "./server/node_modules/eslint-config-prettier/index.js" import prettier from "./server/node_modules/eslint-plugin-prettier/eslint-plugin-prettier.js" import react from "./server/node_modules/eslint-plugin-react/index.js" import reactRefresh from "./server/node_modules/eslint-plugin-react-refresh/index.js" import reactHooks from "./server/node_modules/eslint-plugin-react-hooks/index.js" import ftFlow from "./server/node_modules/eslint-plugin-ft-flow/dist/index.js" import hermesParser from "./server/node_modules/hermes-eslint/dist/index.js" const reactRecommended = react.configs.recommended const jsxRuntime = react.configs["jsx-runtime"] export default [ eslintRecommended.configs.recommended, eslintConfigPrettier, { ignores: ["**/*.test.js"], languageOptions: { parser: hermesParser, parserOptions: { ecmaFeatures: { jsx: true } }, ecmaVersion: 2020, sourceType: "module", globals: { ...globals.browser, ...globals.es2020, ...globals.node } }, linterOptions: { reportUnusedDisableDirectives: true }, settings: { react: { version: "18.2" } }, plugins: { ftFlow, react, "jsx-runtime": jsxRuntime, "react-hooks": reactHooks, prettier }, rules: { ...reactRecommended.rules, ...reactHooks.configs.recommended.rules, ...ftFlow.recommended, "no-unused-vars": "warn", "no-undef": "warn", "no-empty": "warn", "no-extra-boolean-cast": "warn", "no-prototype-builtins": "off", "prettier/prettier": "warn" } }, { files: ["frontend/src/**/*.js"], plugins: { ftFlow, prettier }, rules: { "prettier/prettier": "warn" } }, { files: [ "server/endpoints/**/*.js", "server/models/**/*.js", "server/swagger/**/*.js", "server/utils/**/*.js", "server/index.js" ], rules: { "no-undef": "warn" } }, { files: ["frontend/src/**/*.jsx"], plugins: { ftFlow, react, "jsx-runtime": jsxRuntime, "react-hooks": reactHooks, "react-refresh": reactRefresh, prettier }, rules: { ...jsxRuntime.rules, "react/prop-types": "off", // FIXME "react-refresh/only-export-components": "warn" } } ] ================================================ FILE: extras/scripts/verifyPackageVersions.mjs ================================================ import serverPackageJson from '../../server/package.json' assert { type: 'json' }; import collectorPackageJson from '../../collector/package.json' assert { type: 'json' }; const { dependencies: serverDependencies } = serverPackageJson; const { dependencies: collectorDependencies } = collectorPackageJson; const serverDependenciesKeys = Object.keys(serverDependencies); const collectorDependenciesKeys = Object.keys(collectorDependencies); const commonDependencies = Array.from(new Set([ ...serverDependenciesKeys.filter((key) => collectorDependenciesKeys.includes(key)), ...collectorDependenciesKeys.filter((key) => serverDependenciesKeys.includes(key)), ])); const ignores = [ "@langchain/community" // We are slowly removing this dependency from the app - its use is not critical ] console.log(`${commonDependencies.length} common dependencies found`, commonDependencies); console.log(`Verifying (serverVersion == collectorVersion) for each common dependency`); const failed = []; commonDependencies.forEach((dependency) => { console.log(`Verifying ${dependency}: ${serverDependencies[dependency]} == ${collectorDependencies[dependency]}`); if (serverDependencies[dependency] !== collectorDependencies[dependency]) { if (ignores.includes(dependency)) console.log(`${dependency} is in ignore list.`); else failed.push({ dependency, serverVersion: serverDependencies[dependency], collectorVersion: collectorDependencies[dependency] }); } }); if (failed.length > 0) { console.log(`❌ ${failed.length} dependencies failed to verify`, JSON.stringify(failed, null, 2)); throw new Error(`${failed.length} dependencies failed to verify!`); } console.log(`👍 All dependencies match between server and collector!`); ================================================ FILE: extras/support/announcements/2025-04-08.json ================================================ [ { "thumbnail_url": "https://cdn.anythingllm.com/support/announcements/assets/mcp.jpg", "title": "MCP Support", "short_description": "Import and leverage MCP tools using AnythingLLM.", "goto": "https://docs.anythingllm.com/mcp-compatibility/overview", "author": "AnythingLLM", "date": "April 8, 2025" }, { "thumbnail_url": "https://blogs.nvidia.com/wp-content/uploads/2025/03/nv-raig-032525-nv-blog-1280x680-1-scaled.jpg", "title": "NVIDIA NIM Support", "short_description": "Unlock the power of NVIDIA NIM on Windows with RTX GPU in our latest update via the NVIDIA NIM LLM provider.", "goto": "https://blogs.nvidia.com/blog/rtx-ai-garage-nim-blueprints-g-assist", "author": "NVIDIA", "date": "March 25, 2025" }, { "thumbnail_url": null, "title": "Community Hub Updates", "short_description": "We refreshed the Community Hub with a new look and feel. Check it out!", "goto": "https://hub.anythingllm.com", "author": "AnythingLLM", "date": "March 12, 2025" } ] ================================================ FILE: extras/support/announcements/2025-07-08.json ================================================ [ { "thumbnail_url": "https://cdn.anythingllm.com/support/announcements/assets/private-browsing.png", "title": "Private Web Scraping", "short_description": "Scrape private, gated, or authenticated websites using the browser tool.", "goto": "https://docs.anythingllm.com/features/browser-tool", "author": "AnythingLLM", "date": "July 8, 2025" }, { "thumbnail_url": "https://cdn.anythingllm.com/support/announcements/assets/mobile.png", "title": "AnythingLLM Mobile Beta", "short_description": "AnythingLLM on-device, offline, and private. Syncs with AnythingLLM.", "goto": "https://docs.google.com/forms/d/e/1FAIpQLSdrMRCUXVDWKrtNTcHzVcHQk_SRoiT8X8tiawTFtAhjMq_L6Q/viewform", "author": "AnythingLLM", "date": "July 3, 2025" }, { "title": "Community Hub updates", "short_description": "You can now easily push and pull Agent Flows, System Prompts, and more to the AnythingLLM Community Hub.", "goto": "https://hub.anythingllm.com", "author": "AnythingLLM", "date": "June 25, 2025" } ] ================================================ FILE: extras/support/announcements/2026-01-12.json ================================================ [ { "thumbnail_url": "https://cdn.anythingllm.com/support/announcements/assets/meeting-assistant.png", "title": "Meeting Assistant", "short_description": "Transcribe meetings and generate meeting notes entirely on device.", "goto": "https://docs.anythingllm.com/meeting-assistant/introduction", "author": "AnythingLLM", "date": "January 21, 2026" }, { "thumbnail_url": "https://cdn.anythingllm.com/support/announcements/assets/mobile.png", "title": "AnythingLLM Mobile", "short_description": "AnythingLLM Mobile is now available on the Google Play Store.", "goto": "https://play.google.com/store/apps/details?id=com.anythingllm", "author": "AnythingLLM", "date": "January 5, 2026" }, { "title": "50K Stars on Github", "short_description": "AnythingLLM broke 50K stars on Github!", "goto": "https://github.com/mintplex-labs/anything-llm", "author": "AnythingLLM", "date": "October 21, 2025" } ] ================================================ FILE: extras/support/announcements/list.txt ================================================ 2026-01-12.json 2025-07-08.json 2025-04-08.json ================================================ FILE: extras/translator/.env.example ================================================ DOCKER_MODEL_RUNNER_BASE_PATH='http://127.0.0.1:12434/engines/llama.cpp/v1' DOCKER_MODEL_RUNNER_LLM_MODEL_TOKEN_LIMIT=128000 ================================================ FILE: extras/translator/README.md ================================================ # AnythingLLM Auto-translater The AnythingLLM Auto-translator is a way for us to translate our locales but at no cost or overhead. However these are expected to be run manually and while they may not be 100% accurate improvement in models have made this work trivial without the need for an api. ## Getting started - Install Ollama [w/Docker Model Runner](https://ollama.com) - `ollama pull translategemma:4b` - `cd extras/translator && cp .env.example .env` ## Run the script All translations are based on english dictionary. So the English dictionary must have an entry for it to be supported. `cd extras/translator` **Target a specific language** `node index.mjs ` eg: `node index.mjs es` for Spanish. **Do all languages** `node index.mjs --all` _this is NOT recommended_ ## Gotchas - Sometimes translations go on for a while until the token window is exceeded - you can see this by the massive lines extending beyond the page. - Some languages operate different with single words or special symbols like "@" and will go off the rails. If the English text is one word and the translated text is 100 words, you dont need to be linguist to know that it is probably wrong. - You should always review every line for discrepencies or removal of `{{}}` inputs or brand name hallunications eg: `AnyLLM` instead of `AnythingLLM` ================================================ FILE: extras/translator/index.mjs ================================================ import fs from 'fs'; import {resources} from '../../frontend/src/locales/resources.js'; import "../../server/node_modules/dotenv/lib/main.js"; function getNestedValue(obj, path) { const keys = path.split('.'); let result = obj; for (const key of keys) { if (result == null) return undefined; result = result[key]; } return result; } function setNestedValue(obj, path, value) { const keys = path.split('.'); let result = obj; for (let i = 0; i < keys.length - 1; i++) { const key = keys[i]; if (result[key] == null) result[key] = {}; result = result[key]; } result[keys[keys.length - 1]] = value; } /** * Extract {{variableName}} placeholders from text and replace with tokens. * Returns the modified text and a map to restore the originals. * @param {string} text * @returns {{ text: string, placeholders: string[] }} */ function extractPlaceholders(text) { const placeholders = []; const modifiedText = text.replace(/\{\{([^}]+)\}\}/g, (match) => { const index = placeholders.length; placeholders.push(match); return `__PLACEHOLDER_${index}__`; }); return { text: modifiedText, placeholders }; } /** * Restore original {{variableName}} placeholders from tokens. * @param {string} text * @param {string[]} placeholders * @returns {string} */ function restorePlaceholders(text, placeholders) { return text.replace(/__PLACEHOLDER_(\d+)__/g, (_, index) => { return placeholders[parseInt(index, 10)] || `__PLACEHOLDER_${index}__`; }); } /** * Extract Trans component tags like , , , , etc. * These are used by react-i18next Trans component for rich text formatting. * @param {string} text * @returns {{ text: string, tags: string[] }} */ function extractTransTags(text) { const tags = []; // Match opening tags and closing tags // Also matches self-closing tags const modifiedText = text.replace(/<\/?([a-zA-Z][a-zA-Z0-9]*)\s*\/?>/g, (match) => { const index = tags.length; tags.push(match); return `__TAG_${index}__`; }); return { text: modifiedText, tags }; } /** * Restore original Trans component tags from tokens. * @param {string} text * @param {string[]} tags * @returns {string} */ function restoreTransTags(text, tags) { return text.replace(/__TAG_(\d+)__/g, (_, index) => { return tags[parseInt(index, 10)] || `__TAG_${index}__`; }); } /** * Validate that all placeholders from source exist in translated text. * @param {string} sourceText * @param {string} translatedText * @returns {{ valid: boolean, missing: string[] }} */ function validatePlaceholders(sourceText, translatedText) { const sourceMatches = sourceText.match(/\{\{([^}]+)\}\}/g) || []; const translatedMatches = translatedText.match(/\{\{([^}]+)\}\}/g) || []; const missing = sourceMatches.filter(p => !translatedMatches.includes(p)); return { valid: missing.length === 0, missing }; } /** * Validate that all Trans component tags from source exist in translated text. * @param {string} sourceText * @param {string} translatedText * @returns {{ valid: boolean, missing: string[] }} */ function validateTransTags(sourceText, translatedText) { const sourceMatches = sourceText.match(/<\/?([a-zA-Z][a-zA-Z0-9]*)\s*\/?>/g) || []; const translatedMatches = translatedText.match(/<\/?([a-zA-Z][a-zA-Z0-9]*)\s*\/?>/g) || []; const missing = sourceMatches.filter(t => !translatedMatches.includes(t)); return { valid: missing.length === 0, missing }; } class Translator { static modelTag = 'translategemma:4b' constructor() { this.localeObj = new Intl.DisplayNames(Object.keys(resources), { type: 'language' }); } getLanguageName(localeCode) { try { return this.localeObj.of(localeCode); } catch (error) { console.error("Error getting language name:", error); return null; } } #log(text, ...args) { console.log(`\x1b[32m[Translator]\x1b[0m ${text}`, ...args); } static slog(text, ...args) { console.log(`\x1b[32m[Translator]\x1b[0m ${text}`, ...args); } buildPrompt(text, sourceLangCode, targetLangCode, { hasPlaceholders = false, hasTags = false } = {}) { const sourceLanguage = this.getLanguageName(sourceLangCode); const targetLanguage = this.getLanguageName(targetLangCode); let specialInstructions = ''; if (hasPlaceholders || hasTags) { const items = []; if (hasPlaceholders) items.push('__PLACEHOLDER_0__, __PLACEHOLDER_1__'); if (hasTags) items.push('__TAG_0__, __TAG_1__'); specialInstructions = `\nIMPORTANT: The text contains tokens like ${items.join(', ')}, etc. You MUST keep these tokens exactly as they are in the translation - do not translate, modify, or remove them.`; } return `You are a professional ${sourceLanguage} (${sourceLangCode.toLowerCase()}) to ${targetLanguage} (${targetLangCode.toLowerCase()}) translator. Your goal is to accurately convey the meaning and nuances of the original ${sourceLanguage} text while adhering to ${targetLanguage} grammar, vocabulary, and cultural sensitivities.${specialInstructions} Produce only the ${targetLanguage} translation, without any additional explanations or commentary. Please translate the following ${sourceLanguage} text into ${targetLanguage}: ${text}` } /** * Clean the output text from the model * Output text: `在助手回复中呈现 HTML 响应。这可以显著提高回复的质量，但也可能带来潜在的安全风险。<|im_end|>` * We want to remove the <|im_end|> or im_start tags * @param {*} text * @returns */ cleanOutputText(text) { return text.replace(/<\|im_end\|>|<\|im_start\|>/g, '').trim(); } async translate(text, sourceLangCode, targetLangCode) { // Extract placeholders like {{variableName}} and replace with tokens const { text: textWithPlaceholders, placeholders } = extractPlaceholders(text); const hasPlaceholders = placeholders.length > 0; // Extract Trans component tags like , , etc. const { text: textWithTokens, tags } = extractTransTags(textWithPlaceholders); const hasTags = tags.length > 0; const prompt = this.buildPrompt(textWithTokens, sourceLangCode, targetLangCode, { hasPlaceholders, hasTags }); const response = await fetch(`http://127.0.0.1:11434/api/chat`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ model: Translator.modelTag, messages: [{ role: 'user', content: prompt }], temperature: 0.1, stream: false, }), }); if(!response.ok) throw new Error(`Failed to translate: ${response.statusText}`); const data = await response.json(); let translatedText = this.cleanOutputText(data.message.content); // Restore Trans component tags first (order matters since tags may contain placeholders) if (hasTags) { translatedText = restoreTransTags(translatedText, tags); // Validate all tags were preserved const tagValidation = validateTransTags(text, translatedText); if (!tagValidation.valid) { console.warn(`Warning: Missing Trans tags in translation: ${tagValidation.missing.join(', ')}`); for (let i = 0; i < tags.length; i++) { if (!translatedText.includes(tags[i])) { console.warn(` Tag ${tags[i]} was lost in translation`); } } } } // Restore original placeholders if (hasPlaceholders) { translatedText = restorePlaceholders(translatedText, placeholders); // Validate all placeholders were preserved const validation = validatePlaceholders(text, translatedText); if (!validation.valid) { console.warn(`Warning: Missing placeholders in translation: ${validation.missing.join(', ')}`); // Attempt to fix by checking if tokens remain untranslated for (let i = 0; i < placeholders.length; i++) { if (!translatedText.includes(placeholders[i])) { console.warn(` Placeholder ${placeholders[i]} was lost in translation`); } } } } return translatedText; } writeTranslations(langCode, translations) { let langFilename = langCode.toLowerCase(); // Special cases if(langCode === 'pt') langFilename = 'pt_BR'; if(langCode === 'zh-tw') langFilename = 'zh_TW'; if(langCode === 'vi') langFilename = 'vn'; fs.writeFileSync( `../../frontend/src/locales/${langFilename}/common.js`, `// Anything with "null" requires a translation. Contribute to translation via a PR! const TRANSLATIONS = ${JSON.stringify(translations, null, 2)} export default TRANSLATIONS;` ); console.log(`Updated ${langCode} translations file`); } } // Deep traverse the english translations and get all the path to any all keys const translator = new Translator(); const englishTranslations = resources.en.common; const allKeys = []; function traverseTranslations(translations, parentKey = '') { for(const [key, value] of Object.entries(translations)) { const fullKey = !parentKey ? key : `${parentKey}.${key}`; if(typeof value === 'object' && value !== null) { traverseTranslations(value, fullKey); } else { allKeys.push(fullKey); } } } traverseTranslations(englishTranslations); delete resources.en; async function translateAllLanguages() { for(const [langCode, { common }] of Object.entries(resources)) { console.log(`Translating ${translator.getLanguageName(langCode)}(${langCode}) to all languages`); await translateSingleLanguage(langCode); } } async function translateSingleLanguage(langCode) { let totalTranslations = 0; for(const key of allKeys) { const sourceText = getNestedValue(englishTranslations, key); if(!sourceText) continue; // If the source text is @agent, set the translation to @agent - this has no // direct translation and must be handled manually if(sourceText === '@agent') { setNestedValue(resources[langCode].common, key, '@agent'); continue; } if(sourceText === '/reset') { setNestedValue(resources[langCode].common, key, '/reset'); continue; } const value = getNestedValue(resources[langCode].common, key); if(!!value) continue; // If the translation is already present, skip it console.log(`Translation not found for ${translator.getLanguageName(langCode)}(${langCode})`, { key, sourceText, }); const outputText = await translator.translate(sourceText, 'en', langCode); if(!outputText) { console.log('No output text - skipping'); continue; } console.log(`Output text: ${outputText}`); setNestedValue(resources[langCode].common, key, outputText); console.log(`--------------------------------`); totalTranslations++; } if(totalTranslations === 0) return console.log('No translations performed!'); console.log(`--------------------------------`); console.log(`Translated ${totalTranslations} translations for ${langCode}`); translator.writeTranslations(langCode, resources[langCode].common); console.log(`--------------------------------`); } let langArg = process.argv[2]; if(langArg) { if(langArg.toLowerCase() === '--all') await translateAllLanguages(); else { if(!Object.keys(resources).includes(langArg)) throw new Error(`Language ${langArg} not found in resources`); await translateSingleLanguage(langArg); } } else { throw new Error('Please provide a language code as an argument or --all to translate all languages'); } ================================================ FILE: frontend/.env.example ================================================ VITE_API_BASE='http://localhost:3001/api' # Use this URL when developing locally # VITE_API_BASE="https://$CODESPACE_NAME-3001.$GITHUB_CODESPACES_PORT_FORWARDING_DOMAIN/api" # for GitHub Codespaces # VITE_API_BASE='/api' # Use this URL deploying on non-localhost address OR in docker. ================================================ FILE: frontend/.gitignore ================================================ # Logs logs *.log npm-debug.log* yarn-debug.log* yarn-error.log* pnpm-debug.log* lerna-debug.log* node_modules dist dist-ssr *.local # Editor directories and files .vscode/* !.vscode/extensions.json .idea .DS_Store *.suo *.ntvs* *.njsproj *.sln *.sw? bundleinspector.html .env.production flow-typed ================================================ FILE: frontend/.nvmrc ================================================ v18.18.0 ================================================ FILE: frontend/eslint.config.js ================================================ import js from "@eslint/js" import globals from "globals" import pluginReact from "eslint-plugin-react" import pluginReactHooks from "eslint-plugin-react-hooks" import pluginPrettier from "eslint-plugin-prettier" import configPrettier from "eslint-config-prettier" import unusedImports from "eslint-plugin-unused-imports" export default [ { ignores: ["**/*.min.js", "src/media/**/*"] }, // Base JS recommended rules js.configs.recommended, // Your React/JSX files { files: ["src/**/*.{js,jsx,mjs,cjs}"], languageOptions: { ecmaVersion: "latest", sourceType: "module", parserOptions: { ecmaFeatures: { jsx: true } }, globals: globals.browser }, plugins: { react: pluginReact, "react-hooks": pluginReactHooks, "unused-imports": unusedImports, prettier: pluginPrettier }, settings: { react: { version: "detect" } }, rules: { // React recommended rules (inline, since we're not "extending" in flat config) ...pluginReact.configs.flat.recommended.rules, // If you want hooks rules, add these (recommended) ...pluginReactHooks.configs.recommended.rules, // Prettier: disable conflicting stylistic rules + optionally enforce formatting ...configPrettier.rules, "prettier/prettier": "error", // Your overrides "react/react-in-jsx-scope": "off", "react-hooks/exhaustive-deps": "off", "react/prop-types": "off", "react-hooks/set-state-in-effect": "off", "react/jsx-no-target-blank": "error", "react/no-unescaped-entities": "off", "react/display-name": "off", "react-hooks/immutability": "off", "react-hooks/preserve-manual-memoization": "off", "no-extra-boolean-cast": "off", "no-prototype-builtins": "off", "no-empty": "off", "no-useless-escape": "off", "no-undef": "error", "no-unsafe-optional-chaining": "off", "no-constant-binary-expression": "off", // Unused cleanup "no-unused-vars": "off", "unused-imports/no-unused-imports": "error", "unused-imports/no-unused-vars": [ "warn", { vars: "all", varsIgnorePattern: "^_", args: "after-used", argsIgnorePattern: "^_" } ] } } ] ================================================ FILE: frontend/index.html ================================================ AnythingLLM | Your personal LLM trained on anything
================================================ FILE: frontend/jsconfig.json ================================================ { "compilerOptions": { "module": "commonjs", "target": "esnext", "jsx": "react", "paths": { "@/*": [ "./src/*" ] } } } ================================================ FILE: frontend/package.json ================================================ { "name": "anything-llm-frontend", "private": false, "license": "MIT", "type": "module", "scripts": { "start": "vite --open", "dev": "cross-env NODE_ENV=development vite --debug --host=0.0.0.0", "build": "vite build && node scripts/postbuild.js", "lint:check": "eslint src", "lint": "eslint --fix src", "preview": "vite preview" }, "dependencies": { "@lobehub/icons": "^4.0.3", "@microsoft/fetch-event-source": "^2.0.1", "@mintplex-labs/piper-tts-web": "^1.0.4", "@phosphor-icons/react": "^2.1.7", "@tremor/react": "^3.15.1", "dompurify": "^3.0.8", "file-saver": "^2.0.5", "he": "^1.2.0", "highlight.js": "^11.9.0", "i18next": "^23.11.3", "i18next-browser-languagedetector": "^7.2.1", "js-levenshtein": "^1.1.6", "katex": "^0.6.0", "lodash.debounce": "^4.0.8", "markdown-it": "^13.0.1", "moment": "^2.30.1", "onnxruntime-web": "^1.18.0", "pluralize": "^8.0.0", "qrcode.react": "^4.2.0", "react": "^18.2.0", "react-beautiful-dnd": "13.1.1", "react-confetti-explosion": "^2.1.2", "react-device-detect": "^2.2.2", "react-dom": "^18.2.0", "react-dropzone": "^14.2.3", "react-error-boundary": "^6.0.0", "react-highlight-words": "^0.21.0", "react-i18next": "^14.1.1", "react-loading-skeleton": "^3.1.0", "react-router-dom": "^6.3.0", "react-speech-recognition": "^3.10.0", "react-tag-input-component": "^2.0.2", "react-toastify": "^9.1.3", "react-tooltip": "^5.25.2", "recharts": "^2.12.5", "recharts-to-png": "^2.3.1", "text-case": "^1.0.9", "truncate": "^3.0.0", "uuid": "^9.0.0" }, "devDependencies": { "@esbuild-plugins/node-globals-polyfill": "^0.1.1", "@eslint/js": "^9.39.2", "@types/react": "^18.2.23", "@types/react-dom": "^18.2.8", "@types/react-router-dom": "^5.3.3", "@vitejs/plugin-react": "^4.0.0-beta.0", "autoprefixer": "^10.4.14", "buffer": "^6.0.3", "cross-env": "^7.0.3", "eslint": "^9.39.2", "eslint-config-prettier": "^9.0.0", "eslint-plugin-ft-flow": "^3.0.0", "eslint-plugin-prettier": "^5.0.0", "eslint-plugin-react": "^7.37.5", "eslint-plugin-react-hooks": "^7.0.1", "eslint-plugin-react-refresh": "^0.4.3", "eslint-plugin-unused-imports": "^4.3.0", "flow-bin": "^0.217.0", "flow-remove-types": "^2.217.1", "globals": "^16.5.0", "hermes-eslint": "^0.15.0", "postcss": "^8.4.23", "prettier": "^3.0.3", "rollup-plugin-visualizer": "^5.9.0", "tailwindcss": "^3.3.1", "vite": "^4.3.0" } } ================================================ FILE: frontend/postcss.config.js ================================================ import tailwind from 'tailwindcss' import autoprefixer from 'autoprefixer' import tailwindConfig from './tailwind.config.js' export default { plugins: [tailwind(tailwindConfig), autoprefixer], } ================================================ FILE: frontend/public/manifest.json ================================================ { "name": "AnythingLLM", "short_name": "AnythingLLM", "display": "standalone", "orientation": "portrait", "start_url": "/", "icons": [ { "src": "/favicon.png", "sizes": "any" } ] } ================================================ FILE: frontend/public/robots.txt ================================================ User-agent: * Disallow: / ================================================ FILE: frontend/public/service-workers/push-notifications.js ================================================ function parseEventData(event) { try { return event.data.json(); } catch (e) { console.error('Failed to parse event data - is payload valid? .text():\n', event.data.text()); return null } } self.addEventListener('push', function (event) { const payload = parseEventData(event); if (!payload) return; // options: https://developer.mozilla.org/en-US/docs/Web/API/ServiceWorkerRegistration/showNotification#options self.registration.showNotification(payload.title || 'AnythingLLM', { ...payload, icon: '/favicon.png', }); }); self.addEventListener('notificationclick', function (event) { event.notification.close(); const { onClickUrl = null } = event.notification.data || {}; if (!onClickUrl) return; event.waitUntil(clients.openWindow(onClickUrl)); }); ================================================ FILE: frontend/scripts/postbuild.js ================================================ import { renameSync } from 'fs'; import { fileURLToPath } from 'url'; import path from 'path'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); console.log(`Running frontend post build script...`) renameSync(path.resolve(__dirname, '../dist/index.html'), path.resolve(__dirname, '../dist/_index.html')); console.log(`index.html renamed to _index.html so SSR of the index page can be assumed.`); ================================================ FILE: frontend/src/App.jsx ================================================ import React, { Suspense } from "react"; import { Outlet, useLocation } from "react-router-dom"; import { I18nextProvider } from "react-i18next"; import { AuthProvider } from "@/AuthContext"; import { ToastContainer } from "react-toastify"; import "react-toastify/dist/ReactToastify.css"; import i18n from "./i18n"; import { PfpProvider } from "./PfpContext"; import { LogoProvider } from "./LogoContext"; import { FullScreenLoader } from "./components/Preloader"; import { ThemeProvider } from "./ThemeContext"; import { PWAModeProvider } from "./PWAContext"; import KeyboardShortcutsHelp from "@/components/KeyboardShortcutsHelp"; import { ErrorBoundary } from "react-error-boundary"; import ErrorBoundaryFallback from "./components/ErrorBoundaryFallback"; export default function App() { const location = useLocation(); return ( }> ); } ================================================ FILE: frontend/src/AuthContext.jsx ================================================ import React, { useState, createContext, useEffect } from "react"; import { AUTH_TIMESTAMP, AUTH_TOKEN, AUTH_USER, USER_PROMPT_INPUT_MAP, } from "@/utils/constants"; import System from "./models/system"; import { useNavigate } from "react-router-dom"; import { safeJsonParse } from "@/utils/request"; export const AuthContext = createContext(null); export function AuthProvider(props) { const localUser = localStorage.getItem(AUTH_USER); const localAuthToken = localStorage.getItem(AUTH_TOKEN); const [store, setStore] = useState({ user: localUser ? safeJsonParse(localUser, null) : null, authToken: localAuthToken ? localAuthToken : null, }); const navigate = useNavigate(); /* NOTE: * 1. There's no reason for these helper functions to be stateful. They could * just be regular funcs or methods on a basic object. * 2. These actions are not being invoked anywhere in the * codebase, dead code. */ const [actions] = useState({ updateUser: (user, authToken = "") => { localStorage.setItem(AUTH_USER, JSON.stringify(user)); localStorage.setItem(AUTH_TOKEN, authToken); setStore({ user, authToken }); }, unsetUser: () => { localStorage.removeItem(AUTH_USER); localStorage.removeItem(AUTH_TOKEN); localStorage.removeItem(AUTH_TIMESTAMP); localStorage.removeItem(USER_PROMPT_INPUT_MAP); setStore({ user: null, authToken: null }); }, }); /* * On initial mount and whenever the token changes, fetch a new user object * If the user is suspended, (success === false and data === null) logout the user and redirect to the login page * If success is true and data is not null, update the user object in the store (multi-user mode only) * If success is true and data is null, do nothing (single-user mode only) with or without password protection */ useEffect(() => { async function refreshUser() { const { success, user: refreshedUser } = await System.refreshUser(); if (success && refreshedUser === null) return; if (!success) { localStorage.removeItem(AUTH_USER); localStorage.removeItem(AUTH_TOKEN); localStorage.removeItem(AUTH_TIMESTAMP); localStorage.removeItem(USER_PROMPT_INPUT_MAP); setStore({ user: null, authToken: null }); navigate("/login"); return; } localStorage.setItem(AUTH_USER, JSON.stringify(refreshedUser)); setStore((prev) => ({ ...prev, user: refreshedUser, })); } if (store.authToken) refreshUser(); }, [store.authToken]); return ( {props.children} ); } ================================================ FILE: frontend/src/LogoContext.jsx ================================================ import { createContext, useEffect, useState } from "react"; import AnythingLLM from "./media/logo/anything-llm.png"; import AnythingLLMDark from "./media/logo/anything-llm-dark.png"; import DefaultLoginLogoLight from "./media/illustrations/login-logo.svg"; import DefaultLoginLogoDark from "./media/illustrations/login-logo-light.svg"; import System from "./models/system"; export const REFETCH_LOGO_EVENT = "refetch-logo"; function isLightMode() { return document.documentElement.getAttribute("data-theme") === "light"; } export const LogoContext = createContext(); export function LogoProvider({ children }) { const [logo, setLogo] = useState(""); const [loginLogo, setLoginLogo] = useState(""); const [isCustomLogo, setIsCustomLogo] = useState(false); async function fetchInstanceLogo() { const DefaultLoginLogo = isLightMode() ? DefaultLoginLogoDark : DefaultLoginLogoLight; try { const { isCustomLogo, logoURL } = await System.fetchLogo(); if (logoURL) { setLogo(logoURL); setLoginLogo(isCustomLogo ? logoURL : DefaultLoginLogo); setIsCustomLogo(isCustomLogo); } else { isLightMode() ? setLogo(AnythingLLMDark) : setLogo(AnythingLLM); setLoginLogo(DefaultLoginLogo); setIsCustomLogo(false); } } catch (err) { isLightMode() ? setLogo(AnythingLLMDark) : setLogo(AnythingLLM); setLoginLogo(DefaultLoginLogo); setIsCustomLogo(false); console.error("Failed to fetch logo:", err); } } useEffect(() => { fetchInstanceLogo(); window.addEventListener(REFETCH_LOGO_EVENT, fetchInstanceLogo); return () => { window.removeEventListener(REFETCH_LOGO_EVENT, fetchInstanceLogo); }; }, []); return ( {children} ); } ================================================ FILE: frontend/src/PWAContext.jsx ================================================ import React, { createContext, useContext, useEffect, useMemo, useState, } from "react"; /** * Detects if the application is running as a standalone PWA * @returns {boolean} True if running as standalone PWA */ function isStandalonePWA() { if (typeof window === "undefined") return false; const matchesStandaloneDisplayMode = typeof window.matchMedia === "function" ? window.matchMedia("(display-mode: standalone)")?.matches : false; const isIOSStandalone = window.navigator?.standalone === true; // iOS Safari const androidReferrer = typeof document !== "undefined" && document?.referrer ? document.referrer.includes("android-app://") : false; return Boolean( matchesStandaloneDisplayMode || isIOSStandalone || androidReferrer ); } const PWAModeContext = createContext({ isPWA: false }); export function PWAModeProvider({ children }) { const [isPWA, setIsPWA] = useState(() => isStandalonePWA()); useEffect(() => { if (typeof window === "undefined") return undefined; const mediaQuery = typeof window.matchMedia === "function" ? window.matchMedia("(display-mode: standalone)") : null; const updateStatus = () => setIsPWA(isStandalonePWA()); updateStatus(); if (mediaQuery?.addEventListener) { mediaQuery.addEventListener("change", updateStatus); } else if (mediaQuery?.addListener) { mediaQuery.addListener(updateStatus); } window.addEventListener("appinstalled", updateStatus); window.addEventListener("visibilitychange", updateStatus); return () => { if (mediaQuery?.removeEventListener) { mediaQuery.removeEventListener("change", updateStatus); } else if (mediaQuery?.removeListener) { mediaQuery.removeListener(updateStatus); } window.removeEventListener("appinstalled", updateStatus); window.removeEventListener("visibilitychange", updateStatus); }; }, []); useEffect(() => { if (typeof document === "undefined") return undefined; document.body.classList.toggle("pwa", isPWA); document.documentElement?.setAttribute( "data-pwa", isPWA ? "true" : "false" ); return () => { document.body.classList.remove("pwa"); document.documentElement?.removeAttribute("data-pwa"); }; }, [isPWA]); const value = useMemo(() => ({ isPWA }), [isPWA]); return ( {children} ); } export function usePWAMode() { return useContext(PWAModeContext); } ================================================ FILE: frontend/src/PfpContext.jsx ================================================ import React, { createContext, useState, useEffect } from "react"; import useUser from "./hooks/useUser"; import System from "./models/system"; export const PfpContext = createContext(); export function PfpProvider({ children }) { const [pfp, setPfp] = useState(null); const { user } = useUser(); useEffect(() => { async function fetchPfp() { if (!user?.id) return; try { const pfpUrl = await System.fetchPfp(user.id); setPfp(pfpUrl); } catch (err) { setPfp(null); console.error("Failed to fetch pfp:", err); } } fetchPfp(); }, [user?.id]); return ( {children} ); } ================================================ FILE: frontend/src/ThemeContext.jsx ================================================ import React, { createContext, useContext } from "react"; import { useTheme } from "./hooks/useTheme"; const ThemeContext = createContext(); export function ThemeProvider({ children }) { const themeValue = useTheme(); return ( {children} ); } export function useThemeContext() { return useContext(ThemeContext); } ================================================ FILE: frontend/src/components/CanViewChatHistory/index.jsx ================================================ import { useEffect, useState } from "react"; import { FullScreenLoader } from "@/components/Preloader"; import System from "@/models/system"; import paths from "@/utils/paths"; /** * Protects the view from system set ups who cannot view chat history. * If the user cannot view chat history, they are redirected to the home page. * @param {React.ReactNode} children */ export function CanViewChatHistory({ children }) { const { loading, viewable } = useCanViewChatHistory(); if (loading) return ; if (!viewable) { window.location.href = paths.home(); return ; } return <>{children}; } /** * Provides the `viewable` state to the children. * @returns {React.ReactNode} */ export function CanViewChatHistoryProvider({ children }) { const { loading, viewable } = useCanViewChatHistory(); if (loading) return null; return <>{children({ viewable })}; } /** * Hook that fetches the can view chat history state from local storage or the system settings. * @returns {Promise<{viewable: boolean, error: string | null}>} */ export function useCanViewChatHistory() { const [loading, setLoading] = useState(true); const [viewable, setViewable] = useState(false); useEffect(() => { async function fetchViewable() { const { viewable } = await System.fetchCanViewChatHistory(); setViewable(viewable); setLoading(false); } fetchViewable(); }, []); return { loading, viewable }; } ================================================ FILE: frontend/src/components/ChangeWarning/index.jsx ================================================ import { Warning, X } from "@phosphor-icons/react"; export default function ChangeWarningModal({ warningText = "", onClose, onConfirm, }) { return (

WARNING - This action is irreversible

{warningText.split("\\n").map((line, index) => ( {line}
))}

Are you sure you want to proceed?

); } ================================================ FILE: frontend/src/components/ChatBubble/index.jsx ================================================ import React from "react"; import UserIcon from "../UserIcon"; import { userFromStorage } from "@/utils/request"; import renderMarkdown from "@/utils/chat/markdown"; import DOMPurify from "@/utils/chat/purify"; export default function ChatBubble({ message, type }) { const isUser = type === "user"; return (
); } ================================================ FILE: frontend/src/components/CommunityHub/PublishEntityModal/AgentFlows/index.jsx ================================================ import { useState, useRef } from "react"; import { useTranslation } from "react-i18next"; import CommunityHub from "@/models/communityHub"; import showToast from "@/utils/toast"; import paths from "@/utils/paths"; import { X, CaretRight } from "@phosphor-icons/react"; import { BLOCK_INFO } from "@/pages/Admin/AgentBuilder/BlockList"; import { Link } from "react-router-dom"; export default function AgentFlows({ entity }) { const { t } = useTranslation(); const formRef = useRef(null); const [isSubmitting, setIsSubmitting] = useState(false); const [tags, setTags] = useState([]); const [tagInput, setTagInput] = useState(""); const [isSuccess, setIsSuccess] = useState(false); const [itemId, setItemId] = useState(null); const [expandedStep, setExpandedStep] = useState(null); const handleSubmit = async (e) => { e.preventDefault(); e.stopPropagation(); setIsSubmitting(true); try { const form = new FormData(formRef.current); const data = { name: form.get("name"), description: form.get("description"), tags: tags, visibility: "private", flow: JSON.stringify({ name: form.get("name"), description: form.get("description"), steps: entity.steps, tags: tags, visibility: "private", }), }; const { success, error, itemId } = await CommunityHub.createAgentFlow(data); if (!success) throw new Error(error); setItemId(itemId); setIsSuccess(true); } catch (error) { console.error("Failed to publish agent flow:", error); showToast(`Failed to publish agent flow: ${error.message}`, "error", { clear: true, }); } finally { setIsSubmitting(false); } }; const handleKeyDown = (e) => { if (e.key === "Enter" || e.key === ",") { e.preventDefault(); const value = tagInput.trim(); if (value.length > 20) return; if (value && !tags.includes(value)) { setTags((prevTags) => [...prevTags, value].slice(0, 5)); // Limit to 5 tags setTagInput(""); } } }; const removeTag = (tagToRemove) => { setTags(tags.filter((tag) => tag !== tagToRemove)); }; if (isSuccess) { return (

{t("community_hub.publish.agent_flow.success_title")}

{t("community_hub.publish.agent_flow.success_description")}

{t("community_hub.publish.agent_flow.success_thank_you")}

{t("community_hub.publish.agent_flow.view_on_hub")}
); } return ( <>

{t("community_hub.publish.agent_flow.modal_title")}

{t("community_hub.publish.agent_flow.name_description")}
{t("community_hub.publish.agent_flow.description_description")}
); } ================================================ FILE: frontend/src/components/WorkspaceChat/ChatContainer/PromptInput/ToolsMenu/Tabs/SlashCommands/SlashPresets/EditPresetModal.jsx ================================================ import { useState, useEffect } from "react"; import { X } from "@phosphor-icons/react"; import ModalWrapper from "@/components/ModalWrapper"; import { CMD_REGEX } from "./constants"; export default function EditPresetModal({ isOpen, onClose, onSave, onDelete, preset, }) { const [command, setCommand] = useState(""); const [deleting, setDeleting] = useState(false); useEffect(() => { if (preset && isOpen) { setCommand(preset.command?.slice(1) || ""); } }, [preset, isOpen]); const handleSubmit = (e) => { e.preventDefault(); const form = new FormData(e.target); const sanitizedCommand = command.replace(CMD_REGEX, ""); onSave({ id: preset.id, command: `/${sanitizedCommand}`, prompt: form.get("prompt"), description: form.get("description"), }); }; const handleCommandChange = (e) => { const value = e.target.value.replace(CMD_REGEX, ""); setCommand(value); }; const handleDelete = async () => { if (!window.confirm("Are you sure you want to delete this preset?")) return; setDeleting(true); await onDelete(preset.id); setDeleting(false); onClose(); }; return (

Edit Preset

/
); } ================================================ FILE: frontend/src/components/WorkspaceChat/ChatContainer/PromptInput/ToolsMenu/Tabs/SlashCommands/SlashPresets/constants.js ================================================ export const CMD_REGEX = /[^a-zA-Z0-9_-]/g; ================================================ FILE: frontend/src/components/WorkspaceChat/ChatContainer/PromptInput/ToolsMenu/Tabs/SlashCommands/index.jsx ================================================ import { useState, useEffect, useMemo, useCallback } from "react"; import { Plus } from "@phosphor-icons/react"; import { useTranslation } from "react-i18next"; import System from "@/models/system"; import { useModal } from "@/hooks/useModal"; import AddPresetModal from "./SlashPresets/AddPresetModal"; import EditPresetModal from "./SlashPresets/EditPresetModal"; import PublishEntityModal from "@/components/CommunityHub/PublishEntityModal"; import showToast from "@/utils/toast"; import { useIsAgentSessionActive } from "@/utils/chat/agent"; import { PROMPT_INPUT_EVENT } from "@/components/WorkspaceChat/ChatContainer/PromptInput"; import useToolsMenuItems from "../../useToolsMenuItems"; import SlashCommandRow from "./SlashCommandRow"; export default function SlashCommandsTab({ sendCommand, setShowing, promptRef, highlightedIndex = -1, registerItemCount, }) { const { t } = useTranslation(); const isActiveAgentSession = useIsAgentSessionActive(); const { isOpen: isAddModalOpen, openModal: openAddModal, closeModal: closeAddModal, } = useModal(); const { isOpen: isEditModalOpen, openModal: openEditModal, closeModal: closeEditModal, } = useModal(); const { isOpen: isPublishModalOpen, openModal: openPublishModal, closeModal: closePublishModal, } = useModal(); const [presets, setPresets] = useState([]); const [selectedPreset, setSelectedPreset] = useState(null); const [presetToPublish, setPresetToPublish] = useState(null); useEffect(() => { fetchPresets(); }, []); const fetchPresets = async () => { const presets = await System.getSlashCommandPresets(); setPresets(presets); }; // Build the list of selectable items for keyboard navigation and rendering // Command names must stay as static English strings since the backend // matches against exact "/reset" and "/exit" commands. const items = useMemo(() => { const builtIn = isActiveAgentSession ? { command: "/exit", description: t("chat_window.preset_exit_description"), autoSubmit: true, } : { command: "/reset", description: t("chat_window.preset_reset_description"), autoSubmit: true, }; return [ builtIn, ...presets.map((preset) => ({ command: preset.command, description: preset.description, autoSubmit: false, preset, })), ]; }, [isActiveAgentSession, presets]); const handleUseCommand = useCallback( (command, autoSubmit = false) => { setShowing(false); // Auto-submit commands (/reset, /exit) fire immediately if (autoSubmit) { sendCommand({ text: command, autoSubmit: true }); promptRef?.current?.focus(); return; } // Insert the command at the cursor, replacing a trailing "/" if present const textarea = promptRef?.current; if (!textarea) return; const cursor = textarea.selectionStart; const value = textarea.value; const charBefore = cursor > 0 ? value[cursor - 1] : ""; const insertStart = charBefore === "/" ? cursor - 1 : cursor; const newValue = value.slice(0, insertStart) + command + value.slice(cursor); window.dispatchEvent( new CustomEvent(PROMPT_INPUT_EVENT, { detail: { messageContent: newValue }, }) ); textarea.focus(); const newCursor = insertStart + command.length; setTimeout(() => textarea.setSelectionRange(newCursor, newCursor), 0); }, [sendCommand, setShowing, promptRef] ); useToolsMenuItems({ items, highlightedIndex, onSelect: (item) => { const text = item.preset ? `${item.command} ` : item.command; handleUseCommand(text, item.autoSubmit); }, registerItemCount, }); const handleSavePreset = async (preset) => { const { error } = await System.createSlashCommandPreset(preset); if (error) { showToast(error, "error"); return false; } fetchPresets(); closeAddModal(); return true; }; const handleEditPreset = (preset) => { setSelectedPreset(preset); openEditModal(); }; const handleUpdatePreset = async (updatedPreset) => { const { error } = await System.updateSlashCommandPreset( updatedPreset.id, updatedPreset ); if (error) { showToast(error, "error"); return; } fetchPresets(); closeEditModal(); setSelectedPreset(null); }; const handleDeletePreset = async (presetId) => { await System.deleteSlashCommandPreset(presetId); fetchPresets(); closeEditModal(); setSelectedPreset(null); }; const handlePublishPreset = (preset) => { setPresetToPublish({ name: preset.command.slice(1), description: preset.description, command: preset.command, prompt: preset.prompt, }); openPublishModal(); }; return ( <> {items.map((item, index) => ( handleUseCommand( item.preset ? `${item.command} ` : item.command, item.autoSubmit ) } onEdit={item.preset ? () => handleEditPreset(item.preset) : undefined} onPublish={ item.preset ? () => handlePublishPreset(item.preset) : undefined } showMenu={!!item.preset} highlighted={highlightedIndex === index} /> ))} {/* Add new */} {!isActiveAgentSession && (
{t("chat_window.add_new")}
)} {/* Modals */} {selectedPreset && ( { closeEditModal(); setSelectedPreset(null); }} onSave={handleUpdatePreset} onDelete={handleDeletePreset} preset={selectedPreset} /> )} ); } ================================================ FILE: frontend/src/components/WorkspaceChat/ChatContainer/PromptInput/ToolsMenu/index.jsx ================================================ import { useState, useEffect, useCallback, useRef, useMemo } from "react"; import { useTranslation } from "react-i18next"; import useUser from "@/hooks/useUser"; import AgentSkillsTab from "./Tabs/AgentSkills"; import SlashCommandsTab from "./Tabs/SlashCommands"; export const TOOLS_MENU_KEYBOARD_EVENT = "tools-menu-keyboard"; function getTabs(t, user) { const tabs = [ { key: "slash-commands", label: t("chat_window.slash_commands"), component: SlashCommandsTab, }, ]; // Only show agent skills tab for admins or when multiuser mode is off const canSeeAgentSkills = !user?.hasOwnProperty("role") || user.role === "admin"; if (canSeeAgentSkills) { tabs.push({ key: "agent-skills", label: t("chat_window.agent_skills"), component: AgentSkillsTab, }); } return tabs; } /** * @param {Workspace} props.workspace - the workspace object * @param {boolean} props.showing * @param {function} props.setShowing * @param {function} props.sendCommand * @param {object} props.promptRef * @param {boolean} [props.centered] - when true, popup opens below the input */ export default function ToolsMenu({ workspace, showing, setShowing, sendCommand, promptRef, centered = false, highlightedIndexRef, }) { const { t } = useTranslation(); const { user } = useUser(); const TABS = useMemo(() => getTabs(t, user), [t, user]); const [activeTab, setActiveTab] = useState(TABS[0].key); const [highlightedIndex, setHighlightedIndex] = useState(-1); const itemCountRef = useRef(0); // Always open to the slash commands useEffect(() => { if (showing) setActiveTab(TABS[0].key); }, [showing]); // Reset highlight when switching tabs or closing useEffect(() => { setHighlightedIndex(-1); }, [activeTab, showing]); // Keep the parent ref in sync so PromptInput can check it on Enter useEffect(() => { if (highlightedIndexRef) highlightedIndexRef.current = highlightedIndex; }, [highlightedIndex]); const registerItemCount = useCallback((count) => { itemCountRef.current = count; }, []); useEffect(() => { if (!showing) return; function handleKeyboard(e) { const { key } = e.detail; if (key === "ArrowLeft" || key === "ArrowRight") { const currentIdx = TABS.findIndex((tab) => tab.key === activeTab); const nextIdx = key === "ArrowLeft" ? (currentIdx - 1 + TABS.length) % TABS.length : (currentIdx + 1) % TABS.length; setActiveTab(TABS[nextIdx].key); return; } if (key === "ArrowUp" || key === "ArrowDown") { const count = itemCountRef.current; if (count === 0) return; setHighlightedIndex((prev) => { if (key === "ArrowDown") { return prev < count - 1 ? prev + 1 : 0; } return prev > 0 ? prev - 1 : count - 1; }); return; } // Enter is handled by the tab components via highlightedIndex } window.addEventListener(TOOLS_MENU_KEYBOARD_EVENT, handleKeyboard); return () => window.removeEventListener(TOOLS_MENU_KEYBOARD_EVENT, handleKeyboard); }, [showing, activeTab]); if (!showing) return null; const { component: ActiveTab } = TABS.find((tab) => tab.key === activeTab); return ( <>
e.preventDefault()} onClick={() => setShowing(false)} />
{ // Prevents prompt textarea from losing focus when clicking inside the menu. // Skip for portaled modals so their inputs can still receive focus. if (e.currentTarget.contains(e.target)) e.preventDefault(); }} className={`absolute left-2 right-2 md:left-14 md:right-auto md:w-[400px] z-50 bg-zinc-800 light:bg-white border border-zinc-700 light:border-slate-300 rounded-lg p-3 flex flex-col gap-2.5 shadow-lg overflow-hidden ${ centered ? "top-full mt-2 max-h-[min(360px,calc(100dvh-25rem))]" : "bottom-full mb-2 max-h-[min(360px,calc(100dvh-11rem))]" }`} >
{TABS.map((tab) => ( setActiveTab(tab.key)} > {tab.label} ))}
); } function TabButton({ active, onClick, children }) { return ( ); } ================================================ FILE: frontend/src/components/WorkspaceChat/ChatContainer/PromptInput/ToolsMenu/useToolsMenuItems.js ================================================ import { useEffect } from "react"; import { TOOLS_MENU_KEYBOARD_EVENT } from "./"; /** * Shared hook for ToolsMenu tabs that registers the item count * for Up/Down navigation and handles Enter to select the highlighted item. * @param {Array} items - the list of items rendered in the tab * @param {number} highlightedIndex - currently highlighted index from parent * @param {function} onSelect - called with the highlighted item on Enter * @param {function} registerItemCount - callback to register total item count with parent */ export default function useToolsMenuItems({ items, highlightedIndex, onSelect, registerItemCount, }) { useEffect(() => { registerItemCount?.(items.length); }, [items.length, registerItemCount]); useEffect(() => { if (highlightedIndex < 0 || highlightedIndex >= items.length) return; function handleEnter(e) { if (e.detail.key !== "Enter") return; onSelect(items[highlightedIndex]); } window.addEventListener(TOOLS_MENU_KEYBOARD_EVENT, handleEnter); return () => window.removeEventListener(TOOLS_MENU_KEYBOARD_EVENT, handleEnter); }, [highlightedIndex, items, onSelect]); } ================================================ FILE: frontend/src/components/WorkspaceChat/ChatContainer/PromptInput/index.jsx ================================================ import { useState, useRef, useEffect } from "react"; import debounce from "lodash.debounce"; import { ArrowUp, At } from "@phosphor-icons/react"; import StopGenerationButton from "./StopGenerationButton"; import SpeechToText from "./SpeechToText"; import { Tooltip } from "react-tooltip"; import AttachmentManager from "./Attachments"; import AttachItem from "./AttachItem"; import { ATTACHMENTS_PROCESSED_EVENT, ATTACHMENTS_PROCESSING_EVENT, PASTE_ATTACHMENT_EVENT, } from "../DnDWrapper"; import useTextSize from "@/hooks/useTextSize"; import { useTranslation } from "react-i18next"; import Appearance from "@/models/appearance"; import usePromptInputStorage from "@/hooks/usePromptInputStorage"; import ToolsMenu, { TOOLS_MENU_KEYBOARD_EVENT } from "./ToolsMenu"; import { useSearchParams } from "react-router-dom"; import { useIsAgentSessionActive } from "@/utils/chat/agent"; export const PROMPT_INPUT_ID = "primary-prompt-input"; export const PROMPT_INPUT_EVENT = "set_prompt_input"; const MAX_EDIT_STACK_SIZE = 100; /** * @param {Workspace} props.workspace - workspace object * @param {function} props.submit - form submit handler * @param {boolean} props.isStreaming - disables input while streaming response * @param {function} props.sendCommand - handler for slash commands and agent mentions * @param {Array} [props.attachments] - file attachments array * @param {boolean} [props.centered] - renders in centered layout mode (for home page) * @param {string} [props.workspaceSlug] - workspace slug for home page context * @param {string} [props.threadSlug] - thread slug for home page context */ export default function PromptInput({ workspace = {}, submit, isStreaming, sendCommand, attachments = [], centered = false, workspaceSlug = null, threadSlug = null, }) { const { t } = useTranslation(); const { showAgentCommand = true } = workspace ?? {}; const { isDisabled } = useIsDisabled(); const agentSessionActive = useIsAgentSessionActive(); const [promptInput, setPromptInput] = useState(""); const [showTools, setShowTools] = useState(false); const autoOpenedToolsRef = useRef(false); const toolsHighlightRef = useRef(-1); const formRef = useRef(null); const textareaRef = useRef(null); const [_, setFocused] = useState(false); const undoStack = useRef([]); const redoStack = useRef([]); const { textSizeClass } = useTextSize(); const [searchParams] = useSearchParams(); // Synchronizes prompt input value with localStorage, scoped to the current thread. usePromptInputStorage({ promptInput, setPromptInput, }); /* * @checklist-item * If the URL has the agent param, open the agent menu for the user * automatically when the component mounts. */ useEffect(() => { if (searchParams.get("action") === "set-agent-chat") { sendCommand({ text: "@agent " }); textareaRef.current?.focus(); } }, [textareaRef.current]); /** * To prevent too many re-renders we remotely listen for updates from the parent * via an event cycle. Otherwise, using message as a prop leads to a re-render every * change on the input. * @param {{detail: {messageContent: string, writeMode: 'replace' | 'append'}}} e */ function handlePromptUpdate(e) { const { messageContent, writeMode = "replace" } = e?.detail ?? {}; if (writeMode === "append") setPromptInput((prev) => prev + messageContent); else if (writeMode === "prepend") setPromptInput((prev) => messageContent + " " + prev); else setPromptInput(messageContent ?? ""); } useEffect(() => { if (!!window) window.addEventListener(PROMPT_INPUT_EVENT, handlePromptUpdate); return () => window?.removeEventListener(PROMPT_INPUT_EVENT, handlePromptUpdate); }, []); useEffect(() => { if (!isStreaming && textareaRef.current) textareaRef.current.focus(); resetTextAreaHeight(); }, [isStreaming]); /** * Save the current state before changes * @param {number} adjustment */ function saveCurrentState(adjustment = 0) { if (undoStack.current.length >= MAX_EDIT_STACK_SIZE) undoStack.current.shift(); undoStack.current.push({ value: promptInput, cursorPositionStart: textareaRef.current.selectionStart + adjustment, cursorPositionEnd: textareaRef.current.selectionEnd + adjustment, }); } const debouncedSaveState = debounce(saveCurrentState, 250); function handleSubmit(e) { // Ignore submits from portaled modals (slash command preset forms) if (e.target !== e.currentTarget) return; setFocused(false); setShowTools(false); submit(e); } function resetTextAreaHeight() { if (!textareaRef.current) return; textareaRef.current.style.height = "auto"; } /** * Capture enter key press to handle submission, redo, or undo * via keyboard shortcuts * @param {KeyboardEvent} event */ function captureEnterOrUndo(event) { // Forward keyboard events to the ToolsMenu when open if (showTools) { if ( ["ArrowUp", "ArrowDown", "ArrowLeft", "ArrowRight"].includes(event.key) ) { event.preventDefault(); window.dispatchEvent( new CustomEvent(TOOLS_MENU_KEYBOARD_EVENT, { detail: { key: event.key }, }) ); return; } // When an item is highlighted via arrow keys, Enter selects it. // Otherwise, Enter falls through to submit the form normally. if (event.key === "Enter" && toolsHighlightRef.current >= 0) { event.preventDefault(); window.dispatchEvent( new CustomEvent(TOOLS_MENU_KEYBOARD_EVENT, { detail: { key: "Enter" }, }) ); return; } if (event.key === "Escape") { event.preventDefault(); setShowTools(false); textareaRef.current?.focus(); return; } } // "/" toggles the Tools menu only when the input is empty if ( event.key === "/" && !event.ctrlKey && !event.metaKey && promptInput.trim() === "" ) { setShowTools((prev) => { autoOpenedToolsRef.current = !prev; return !prev; }); return; } // Is simple enter key press w/o shift key if (event.keyCode === 13 && !event.shiftKey) { event.preventDefault(); if (isStreaming || isDisabled) return; // Prevent submission if streaming or disabled setShowTools(false); return submit(event); } // Is undo with Ctrl+Z or Cmd+Z + Shift key = Redo if ( (event.ctrlKey || event.metaKey) && event.key === "z" && event.shiftKey ) { event.preventDefault(); if (redoStack.current.length === 0) return; const nextState = redoStack.current.pop(); if (!nextState) return; undoStack.current.push({ value: promptInput, cursorPositionStart: textareaRef.current.selectionStart, cursorPositionEnd: textareaRef.current.selectionEnd, }); setPromptInput(nextState.value); setTimeout(() => { textareaRef.current.setSelectionRange( nextState.cursorPositionStart, nextState.cursorPositionEnd ); }, 0); } // Undo with Ctrl+Z or Cmd+Z if ( (event.ctrlKey || event.metaKey) && event.key === "z" && !event.shiftKey ) { if (undoStack.current.length === 0) return; const lastState = undoStack.current.pop(); if (!lastState) return; redoStack.current.push({ value: promptInput, cursorPositionStart: textareaRef.current.selectionStart, cursorPositionEnd: textareaRef.current.selectionEnd, }); setPromptInput(lastState.value); setTimeout(() => { textareaRef.current.setSelectionRange( lastState.cursorPositionStart, lastState.cursorPositionEnd ); }, 0); } } function adjustTextArea(event) { const element = event.target; element.style.height = "auto"; element.style.height = `${element.scrollHeight}px`; } function handlePasteEvent(e) { e.preventDefault(); if (e.clipboardData.items.length === 0) return false; // paste any clipboard items that are images. for (const item of e.clipboardData.items) { if (item.type.startsWith("image/")) { const file = item.getAsFile(); window.dispatchEvent( new CustomEvent(PASTE_ATTACHMENT_EVENT, { detail: { files: [file] }, }) ); continue; } // handle files specifically that are not images as uploads if (item.kind === "file") { const file = item.getAsFile(); window.dispatchEvent( new CustomEvent(PASTE_ATTACHMENT_EVENT, { detail: { files: [file] }, }) ); continue; } } const pasteText = e.clipboardData.getData("text/plain"); if (pasteText) { const textarea = textareaRef.current; const start = textarea.selectionStart; const end = textarea.selectionEnd; const newPromptInput = promptInput.substring(0, start) + pasteText + promptInput.substring(end); setPromptInput(newPromptInput); // Set the cursor position after the pasted text // we need to use setTimeout to prevent the cursor from being set to the end of the text setTimeout(() => { textarea.selectionStart = textarea.selectionEnd = start + pasteText.length; adjustTextArea({ target: textarea }); }, 0); } return; } function handleChange(e) { debouncedSaveState(-1); adjustTextArea(e); const value = e.target.value; setPromptInput(value); // Auto-dismiss the tools menu when the "/" that opened it is modified if (autoOpenedToolsRef.current && showTools && value !== "/") { setShowTools(false); autoOpenedToolsRef.current = false; } } return (