Repository: etkecc/baibot Branch: main Commit: 20cb33bc66ed Files: 221 Total size: 877.6 KB Directory structure: gitextract_wac8c_3x/ ├── .dockerignore ├── .editorconfig ├── .github/ │ └── workflows/ │ ├── ci.yml │ └── publish.yml ├── .gitignore ├── .pre-commit-config.yaml ├── CHANGELOG.md ├── Cargo.toml ├── Dockerfile ├── Dockerfile.ci ├── LICENSE ├── README.md ├── docs/ │ ├── README.md │ ├── access.md │ ├── agents.md │ ├── configuration/ │ │ ├── README.md │ │ ├── authentication.md │ │ ├── handlers.md │ │ ├── image-generation.md │ │ ├── speech-to-text.md │ │ ├── text-generation.md │ │ └── text-to-speech.md │ ├── development.md │ ├── features.md │ ├── installation.md │ ├── providers.md │ ├── sample-provider-configs/ │ │ ├── anthropic.yml │ │ ├── groq.yml │ │ ├── localai.yml │ │ ├── mistral.yml │ │ ├── ollama.yml │ │ ├── openai-compatible.yml │ │ ├── openai.yml │ │ ├── openrouter.yml │ │ └── together-ai.yml │ └── usage.md ├── etc/ │ ├── app/ │ │ └── config.yml.dist │ ├── assets/ │ │ └── baibot.xcf │ └── services/ │ ├── continuwuity/ │ │ ├── compose.yml │ │ ├── config/ │ │ │ └── continuwuity.toml │ │ └── register-user.sh │ ├── element-web/ │ │ ├── compose.yml │ │ └── config.json.dist │ ├── env.dist │ ├── localai/ │ │ └── compose.yml │ ├── ollama/ │ │ └── compose.yml │ └── synapse/ │ ├── compose.yml │ └── config/ │ ├── homeserver.yaml │ ├── synapse.127.0.0.1.nip.io.log.config │ └── synapse.127.0.0.1.nip.io.signing.key ├── justfile ├── mise.toml ├── renovate.json ├── rust-toolchain.toml └── src/ ├── agent/ │ ├── definition.rs │ ├── identifier.rs │ ├── instantiation.rs │ ├── manager.rs │ ├── mod.rs │ ├── provider/ │ │ ├── anthropic/ │ │ │ ├── config.rs │ │ │ ├── controller.rs │ │ │ ├── mod.rs │ │ │ └── utils.rs │ │ ├── config.rs │ │ ├── controller.rs │ │ ├── entity/ │ │ │ ├── agent_provider.rs │ │ │ ├── image.rs │ │ │ ├── mod.rs │ │ │ ├── ping.rs │ │ │ ├── speech_to_text.rs │ │ │ ├── text_generation/ │ │ │ │ ├── mod.rs │ │ │ │ └── prompt_variables.rs │ │ │ └── text_to_speech.rs │ │ ├── groq/ │ │ │ └── mod.rs │ │ ├── localai/ │ │ │ └── mod.rs │ │ ├── mistral/ │ │ │ └── mod.rs │ │ ├── mod.rs │ │ ├── ollama/ │ │ │ └── mod.rs │ │ ├── openai/ │ │ │ ├── config.rs │ │ │ ├── controller.rs │ │ │ ├── mod.rs │ │ │ └── utils.rs │ │ ├── openai_compat/ │ │ │ ├── config.rs │ │ │ ├── controller.rs │ │ │ ├── mod.rs │ │ │ └── utils.rs │ │ ├── openrouter/ │ │ │ └── mod.rs │ │ └── togetherai/ │ │ └── mod.rs │ ├── purpose.rs │ └── utils.rs ├── bot/ │ ├── implementation.rs │ ├── load_config.rs │ ├── messaging.rs │ ├── mod.rs │ ├── reacting.rs │ └── rooms.rs ├── controller/ │ ├── access/ │ │ ├── determination/ │ │ │ ├── mod.rs │ │ │ └── tests.rs │ │ ├── dispatching.rs │ │ ├── help.rs │ │ ├── mod.rs │ │ ├── room_local_agent_managers.rs │ │ └── users.rs │ ├── agent/ │ │ ├── create/ │ │ │ ├── mod.rs │ │ │ └── tests.rs │ │ ├── delete/ │ │ │ └── mod.rs │ │ ├── details/ │ │ │ └── mod.rs │ │ ├── determination/ │ │ │ ├── mod.rs │ │ │ └── tests.rs │ │ ├── help/ │ │ │ └── mod.rs │ │ ├── list/ │ │ │ └── mod.rs │ │ └── mod.rs │ ├── cfg/ │ │ ├── common/ │ │ │ ├── generic_setting.rs │ │ │ └── mod.rs │ │ ├── controller_type.rs │ │ ├── determination/ │ │ │ ├── mod.rs │ │ │ ├── speech_to_text/ │ │ │ │ ├── mod.rs │ │ │ │ └── tests.rs │ │ │ ├── tests.rs │ │ │ ├── text_generation/ │ │ │ │ ├── mod.rs │ │ │ │ └── tests.rs │ │ │ └── text_to_speech/ │ │ │ ├── mod.rs │ │ │ └── tests.rs │ │ ├── dispatching/ │ │ │ ├── mod.rs │ │ │ ├── speech_to_text.rs │ │ │ ├── text_generation.rs │ │ │ └── text_to_speech.rs │ │ ├── global_config/ │ │ │ ├── generic_setting.rs │ │ │ ├── handler.rs │ │ │ └── mod.rs │ │ ├── help.rs │ │ ├── mod.rs │ │ ├── room_config/ │ │ │ ├── generic_setting.rs │ │ │ ├── handler.rs │ │ │ └── mod.rs │ │ └── status.rs │ ├── chat_completion/ │ │ └── mod.rs │ ├── controller_type.rs │ ├── determination/ │ │ ├── mod.rs │ │ └── tests.rs │ ├── dispatching.rs │ ├── help/ │ │ └── mod.rs │ ├── image/ │ │ ├── determination/ │ │ │ ├── mod.rs │ │ │ └── tests.rs │ │ ├── edit.rs │ │ ├── generation.rs │ │ ├── mod.rs │ │ └── prompt.rs │ ├── join/ │ │ └── mod.rs │ ├── mod.rs │ ├── provider/ │ │ └── mod.rs │ ├── reaction/ │ │ ├── mod.rs │ │ └── text_to_speech.rs │ ├── usage/ │ │ └── mod.rs │ └── utils/ │ ├── agent.rs │ ├── mod.rs │ └── text_to_speech.rs ├── conversation/ │ ├── llm/ │ │ ├── entity.rs │ │ ├── mod.rs │ │ ├── tests.rs │ │ ├── tokenization.rs │ │ └── utils.rs │ ├── matrix/ │ │ ├── entity.rs │ │ ├── mod.rs │ │ ├── room_display_name_fetcher.rs │ │ ├── room_event_fetcher.rs │ │ └── utils/ │ │ ├── mod.rs │ │ └── tests.rs │ ├── matrix_llm_bridge.rs │ └── mod.rs ├── entity/ │ ├── catch_up_marker/ │ │ ├── delayed_catch_up_marker_manager.rs │ │ ├── entity.rs │ │ └── mod.rs │ ├── cfg/ │ │ ├── config.rs │ │ ├── config_tests.rs │ │ ├── defaults.rs │ │ ├── env.rs │ │ └── mod.rs │ ├── globalconfig/ │ │ ├── entity.rs │ │ └── mod.rs │ ├── interaction_context.rs │ ├── message_context.rs │ ├── message_payload.rs │ ├── mod.rs │ ├── room_config_context.rs │ ├── roomconfig/ │ │ ├── defaults.rs │ │ ├── entity/ │ │ │ ├── handler.rs │ │ │ ├── mod.rs │ │ │ ├── speech_to_text.rs │ │ │ ├── text_generation.rs │ │ │ └── text_to_speech.rs │ │ └── mod.rs │ └── trigger_event_info.rs ├── lib.rs ├── main.rs ├── strings/ │ ├── access.rs │ ├── agent.rs │ ├── cfg.rs │ ├── error.rs │ ├── global_config.rs │ ├── help/ │ │ ├── access.rs │ │ ├── agent.rs │ │ ├── cfg.rs │ │ ├── mod.rs │ │ ├── provider.rs │ │ └── usage.rs │ ├── image_edit.rs │ ├── image_generation.rs │ ├── introduction.rs │ ├── mod.rs │ ├── provider.rs │ ├── room_config.rs │ ├── speech_to_text.rs │ ├── text_to_speech.rs │ └── usage.rs └── utils/ ├── base64.rs ├── mime.rs ├── mod.rs ├── status.rs ├── text.rs └── text_to_speech.rs ================================================ FILE CONTENTS ================================================ ================================================ FILE: .dockerignore ================================================ /target /var ================================================ FILE: .editorconfig ================================================ # This file is the top-most EditorConfig file root = true # All Files [*] charset = utf-8 end_of_line = lf indent_style = tab indent_size = 4 insert_final_newline = true trim_trailing_whitespace = true ######################### # File Extension Settings ######################### [*.{yml,yaml,yml.dist}] indent_style = space indent_size = 2 [*.rs] indent_style = space indent_size = 4 # Markdown Files # # Two spaces at the end of a line in Markdown mean "new line", # so trimming trailing whitespace for such files can cause breakage. [*.md] trim_trailing_whitespace = false indent_style = space indent_size = 2 ================================================ FILE: .github/workflows/ci.yml ================================================ name: CI on: workflow_dispatch: pull_request: branches: [ "main" ] push: branches: - "**" tags: [ "v*" ] permissions: contents: read pull-requests: read concurrency: group: ci-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true jobs: test-and-clippy: name: Unit testing and linting runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 - uses: dtolnay/rust-toolchain@1.93.0 - name: Install SQLite3 run: sudo apt-get update && sudo apt-get install -y libsqlite3-dev - run: cargo test --all-features - run: cargo clippy ================================================ FILE: .github/workflows/publish.yml ================================================ name: Publish on: workflow_run: workflows: [ "CI" ] types: [ "completed" ] permissions: contents: read concurrency: group: publish-${{ github.event.workflow_run.id || github.ref }} cancel-in-progress: false jobs: docker-clean-metadata: if: | github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'push' && ( github.event.workflow_run.head_branch == 'main' || startsWith(github.event.workflow_run.head_branch || '', 'v') ) runs-on: ubuntu-latest outputs: json: ${{ steps.meta.outputs.json }} steps: - name: Checkout uses: actions/checkout@v6 with: ref: ${{ github.event.workflow_run.head_sha }} fetch-depth: 0 - name: Extract metadata (tags, labels) for Docker id: meta uses: docker/metadata-action@v6 with: images: | ghcr.io/${{ github.repository }} tags: | type=raw,value=latest,enable=${{ github.event.workflow_run.head_branch == 'main' }} type=semver,pattern={{raw}},value=${{ github.event.workflow_run.head_branch }},enable=${{ startsWith(github.event.workflow_run.head_branch || '', 'v') }} docker-build: if: | github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'push' && ( github.event.workflow_run.head_branch == 'main' || startsWith(github.event.workflow_run.head_branch || '', 'v') ) permissions: contents: read packages: write attestations: write id-token: write strategy: matrix: include: - os: self-hosted arch: amd64 - os: ubuntu-24.04-arm arch: arm64 runs-on: ${{ matrix.os }} steps: - name: Checkout uses: actions/checkout@v6 with: ref: ${{ github.event.workflow_run.head_sha }} fetch-depth: 0 - name: Log in to the GitHub Container registry uses: docker/login-action@v4 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Extract metadata (tags, labels) for Docker id: meta uses: docker/metadata-action@v6 with: tags: | type=raw,value=latest,enable=${{ github.event.workflow_run.head_branch == 'main' }} type=semver,pattern={{raw}},value=${{ github.event.workflow_run.head_branch }},enable=${{ startsWith(github.event.workflow_run.head_branch || '', 'v') }} flavor: | latest=auto suffix=-${{ matrix.arch }},onlatest=true images: | ghcr.io/${{ github.repository }} - name: Build and push Docker images uses: docker/build-push-action@v7 with: push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} docker-manifest: if: | github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'push' && ( github.event.workflow_run.head_branch == 'main' || startsWith(github.event.workflow_run.head_branch || '', 'v') ) permissions: contents: read packages: write needs: - docker-build - docker-clean-metadata runs-on: ubuntu-latest strategy: matrix: image: ${{ fromJson(needs.docker-clean-metadata.outputs.json).tags }} steps: - name: Log in to the GitHub Container registry uses: docker/login-action@v4 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Create and push manifest run: | docker buildx imagetools create -t ${{ matrix.image }} ${{ matrix.image }}-amd64 ${{ matrix.image }}-arm64 ================================================ FILE: .gitignore ================================================ /target /var ================================================ FILE: .pre-commit-config.yaml ================================================ repos: # Fast built-in hooks (Rust-native, no dependencies) - repo: builtin hooks: - id: trailing-whitespace - id: end-of-file-fixer - id: check-yaml - id: check-merge-conflict - id: check-added-large-files args: ['--maxkb=1024'] # Local hooks that run project-specific tools - repo: local hooks: - id: cargo-fmt-check name: Cargo Format Check entry: cargo fmt --all -- --check language: system files: '\.rs$' pass_filenames: false - id: cargo-clippy name: Cargo Clippy entry: cargo clippy -- -D warnings language: system files: '\.rs$' pass_filenames: false priority: 100 - id: test-unit name: Unit Tests entry: just test language: system files: '\.rs$' pass_filenames: false priority: 100 ================================================ FILE: CHANGELOG.md ================================================ # (2026-05-21) Version 1.19.2 - (**Internal Improvement**) Update [async-openai](https://crates.io/crates/async-openai) to 0.40.0. - (**Internal Improvement**) Dependency updates. # (2026-05-09) Version 1.19.1 - (**Internal Improvement**) Update [async-openai](https://crates.io/crates/async-openai) to 0.38.0. - (**Internal Improvement**) Dependency updates. # (2026-05-09) Version 1.19.0 - (**Internal Improvement**) Update [matrix-sdk](https://crates.io/crates/matrix-sdk) from 0.16 to 0.17 and [mxlink](https://crates.io/crates/mxlink) to 1.14.0. matrix-sdk 0.17 dropped its `native-tls` feature and now uses [rustls](https://github.com/rustls/rustls) exclusively as its TLS backend. - (**Internal Improvement**) Bump the pinned Rust toolchain from 1.93.0 to 1.95.0 (in `rust-toolchain.toml` and the Docker build images). - (**Internal Improvement**) Dependency updates. # (2026-04-11) Version 1.18.0 - (**Bugfix**) Fix the bot not sending a welcome message when joining a room on homeservers (like [Continuwuity](https://continuwuity.org/)) that place the join membership event in the sync response's `state` block rather than the `timeline` block, via [mxlink](https://crates.io/crates/mxlink) 1.13.1 - (**Improvement**) Update [tiktoken-rs](https://crates.io/crates/tiktoken-rs) to 0.11, adding tokenization support for newer GPT models (gpt-5.x, codex, etc.) and fixing context sizes for o1-mini/chatgpt-4o/gpt-4.5 - (**Internal Improvement**) Dependency updates # (2026-03-25) Version 1.17.0 - (**Feature**) Add `text-generation sender-context-mode` for attaching sender metadata to conversation messages. See the [💬 Text Generation](./docs/configuration/text-generation.md#-sender-context-mode) documentation for details. Thanks to [kschwank](https://github.com/kschwank) for the contribution in [#104](https://github.com/etkecc/baibot/pull/104)! # (2026-03-24) Version 1.16.1 - (**Bugfix**) Fix compatibility with [async-openai](https://crates.io/crates/async-openai) 0.34.0 by populating the new `phase` field required for OpenAI Responses API message inputs. baibot does not currently distinguish between assistant `commentary` and `final_answer` turns, so using `None` preserves the previous behavior while remaining compatible with the updated crate. - (**Internal Improvement**) Dependency updates. # (2026-03-20) Version 1.16.0 - (**Feature**) Add support for file attachments (`m.file` Matrix messages) in conversations. Files like PDFs, text documents, spreadsheets, code files, etc. are now downloaded and forwarded to the LLM alongside the conversation context, similar to how images (`m.image`) are already handled. See the [💬 Text Generation](./docs/features.md#-text-generation) documentation for details and known limitations. - (**Improvement**) Use the [mime_guess](https://crates.io/crates/mime_guess) crate for MIME type detection from file extensions, replacing a hand-maintained mapping. This covers hundreds of file extensions out of the box. # (2026-03-07) Version 1.15.0 - (**Feature**) Add support for authentication via access tokens (for [Matrix Authentication Service](https://github.com/element-hq/matrix-authentication-service)/OIDC-enabled homeservers) as an alternative to password authentication. See [🔐 Authentication](./docs/configuration/authentication.md) for setup details. Thanks to [Taylor Southwick](https://github.com/twsouthwick) for the contribution in [#83](https://github.com/etkecc/baibot/pull/83)! - (**Internal Improvement**) Pin the Rust toolchain to `1.93.0` in both CI and local development to avoid `matrix-sdk` build failures on newer stable toolchains. - (**Internal Improvement**) Documentation updates. - (**Internal Improvement**) Dependency updates. # (2026-02-18) Version 1.14.3 - (**Internal Improvement**) Add [Renovate](https://docs.renovatebot.com/) configuration for automated dependency updates - (**Internal Improvement**) Dependency updates # (2026-02-18) Version 1.14.2 - (**Internal Improvement**) Dependency updates - (**Internal Improvement**) Reorganize the development environment to support [Continuwuity](https://continuwuity.org/) as a homeserver choice (in addition to [Synapse](https://github.com/element-hq/synapse)). Continuwuity is now the default for its lighter footprint (no external database required). See [development docs](./docs/development.md) for details. # (2026-02-10) Version 1.14.1 - (**Security**) Dependency updates to fix security vulnerabilities ([time](https://crates.io/crates/time) stack exhaustion DoS, [bytes](https://crates.io/crates/bytes) integer overflow), via [mxlink](https://crates.io/crates/mxlink) 1.12.0 - (**Internal Improvement**) Switch from deprecated [serde_yaml](https://crates.io/crates/serde_yaml) to its maintained fork [serde_yaml_ng](https://crates.io/crates/serde_yaml_ng) - (**Internal Improvement**) Add [prek](https://github.com/nicholasgasior/prek) pre-commit hooks via [mise](https://mise.jdx.dev/) for automated code quality checks (formatting, clippy, tests) - (**Internal Improvement**) Fix clippy warnings and formatting issues # (2026-02-04) Version 1.14.0 - (**Feature**) The `openai` provider now uses OpenAI's [Responses API](https://platform.openai.com/docs/api-reference/responses) (instead of the older Chat Completions API), adding support for [🛠️ built-in tools](./docs/features.md#️-built-in-tools-openai-only) (`web_search` and `code_interpreter`). These tools are **disabled by default** and can be enabled via the `text_generation.tools` configuration (see the [sample configuration](https://github.com/etkecc/baibot/blob/c70387b0c38d8d0f30bba2179a2a21a3710dbeaf/docs/sample-provider-configs/openai.yml#L12-L15)). To enable tools on an existing agent, you need to [update the agent](./docs/agents.md#updating-agents) to re-create it with the `text_generation.tools` section added and enable the tools you need. Thanks to [Layla Manley](https://github.com/yeslayla) for the contribution in [#62](https://github.com/etkecc/baibot/pull/62)! - (**Bugfix**) Fix sticker generation for newer GPT image models (`gpt-image-1`, `gpt-image-1-mini`, `gpt-image-1.5`) which don't support the previously hardcoded `256x256` size (minimum is `1024x1024`) - (**Internal Improvement**) Dependency updates # (2026-01-23) Version 1.13.0 - (**Improvement**) Extend auto-switching to support cheaper models (`gpt-image-1-mini`) for `gpt-image-1` and `gpt-image-1.5` when generating stickers ([e0b4a40](https://github.com/etkecc/baibot/commit/e0b4a40)) - (**Internal Improvement**) Upgrade Rust compiler (1.92.0 -> 1.93.0) ([691aeeb](https://github.com/etkecc/baibot/commit/691aeeb)) - (**Internal Improvement**) Dependency updates # (2025-12-21) Version 1.12.0 - (**Improvement**) Upgrade [async-openai](https://crates.io/crates/async-openai) (0.31.1 -> 0.32.2) and add support for OpenAI's `gpt-image-1.5` model ([08c689a](https://github.com/etkecc/baibot/commit/08c689a), [f7bf3d7](https://github.com/etkecc/baibot/commit/f7bf3d7)) - (**Internal Improvement**) Dependency updates # (2025-12-15) Version 1.11.0 - (**Feature**) Add support for custom avatars via file path and for keeping the already-set avatar (for those who wish to manage it by themselves via other means). See the [sample config](./etc/app/config.yml.dist) for details. ([062fbbb](https://github.com/etkecc/baibot/commit/062fbbb8ef9ad600db483a431c5c782402191023)) - (**Internal Improvement**) Dependency updates ([99bde53](https://github.com/etkecc/baibot/commit/99bde53ef648a5a9086a96778fde4a9dbc1ede58)) - (**Internal Improvement**) Documentation updates ([b3fd8e5](https://github.com/etkecc/baibot/commit/b3fd8e548f83fe46398ced4760d7e2bb7588c24d)) - (**Internal Improvement**) Upgrade Rust compiler (1.91.1 -> 1.92.0) ([22906aa](https://github.com/etkecc/baibot/commit/22906aa2d3cae51815fad2560a545eaa69c247b6)) # (2025-12-06) Version 1.10.0 - (**Internal Improvement**) Dependency updates. This version is based on [mxlink](https://crates.io/crates/mxlink)@1.11.0 (which is based on the newly released [matrix-sdk](https://crates.io/crates/matrix-sdk)@[0.16.0](https://github.com/matrix-org/matrix-rust-sdk/releases/tag/matrix-sdk-0.16.0). # (2025-11-30) Version 1.9.0 - (**Internal Improvement**) Upgrade [async-openai](https://crates.io/crates/async-openai) from our own etkecc fork (0.28.1-patched) to the official upstream version 0.31.1. This upgrade required some code adaptations to the new module structure, etc. While tested, regressions are possible. # (2025-11-28) Version 1.8.3 - (**Improvement**) Add support for the `BAIBOT_PERSISTENCE_SESSION_ENCRYPTION_KEY` environment variable for configuring `persistence.session_encryption_key` - (**Improvement**) Add support for the `BAIBOT_USER_ENCRYPTION_RECOVERY_RESET_ALLOWED` environment variable for configuring `user.encryption.recovery_reset_allowed` - (**Internal Improvement**) Dependency updates. # (2025-11-20) Version 1.8.2 - (**Internal Improvement**) Dependency and compiler updates (Rust 1.89.0 -> 1.91.1). # (2025-09-12) Version 1.8.1 - (**Internal Improvement**) Dependency updates. # (2025-09-08) Version 1.8.0 - (**Internal Improvement**) Upgrade [mxlink](https://crates.io/crates/mxlink) (1.9.0 -> 1.10.0) and [matrix-sdk](https://crates.io/crates/matrix-sdk) (0.13.0 -> 0.14.0) - (**Internal Improvement**) Upgrade [Rust](https://www.rust-lang.org/) (1.88.0 -> 1.89.0) - (**Internal Improvement**) Upgrade Debian base for container images (12/bookworm -> 13/trixie) # (2025-07-11) Version 1.7.6 - (**Internal Improvement**) Dependency updates. This version is based on [mxlink](https://crates.io/crates/mxlink)@1.9.0 (which is based on the newly released [matrix-sdk](https://crates.io/crates/matrix-sdk)@[0.13.0](https://github.com/matrix-org/matrix-rust-sdk/releases/tag/matrix-sdk-0.13.0), which contains fixes for some security vulnerabilities) # (2025-06-10) Version 1.7.5 - (**Internal Improvement**) Dependency and compiler updates (Rust 1.86 -> 1.86). # (2025-06-10) Version 1.7.4 - (**Internal Improvement**) Dependency updates. # (2025-06-10) Version 1.7.3 - (**Internal Improvement**) Dependency updates. This version is based on [mxlink](https://crates.io/crates/mxlink)@1.8.0 (which is based on the newly released [matrix-sdk](https://crates.io/crates/matrix-sdk)@[0.12.0](https://github.com/matrix-org/matrix-rust-sdk/releases/tag/matrix-sdk-0.12.0), which contains fixes for important security vulnerabilities) # (2025-05-11) Version 1.7.2 - (**Bugfix**) Allow `image_generation.size` configuration value for OpenAI to be `null` to allow the model to choose the size automatically and default to that # (2025-05-11) Version 1.7.1 - (**Bugfix**) Fix lack of documentation for the new [image-editing](./docs/features.md#-image-editing) feature in the `!bai usage` command's output # (2025-05-10) Version 1.7.0 - (**Feature**) Add vision support to the OpenAI and Anthropic providers. You can now mix text and images in your conversations - fixes [issue #5](https://github.com/etkecc/baibot/issues/5) - (**Feature**) Add [image-editing](./docs/features.md#-image-editing) support to the OpenAI provider - (**Improvement**) Add compatibility with OpenAI's `gpt-image-1` model - fixes [issue #40](https://github.com/etkecc/baibot/issues/40) - (**Change**) Rework [image-creation](./docs/features.md#-image-creation) to avoid command conflicts with [image-editing](./docs/features.md#-image-editing). The image-creation command syntax is now `!bai image create ` (previously: `!bai image `). - (**Internal Improvement**) Dependency and compiler updates > [!WARNING] > Unlike other releases, this release is not published to [crates.io](https://crates.io), because it relies on multiple library forks (`async-openai` and `anthropic-rs`) sourced from Github. # (2025-04-12) Version 1.6.0 - (**Internal Improvement**) Dependency updates. This version is based on [mxlink](https://crates.io/crates/mxlink)@1.7.0 (which is based on the newly released [matrix-sdk](https://crates.io/crates/matrix-sdk)@[0.11.0](https://github.com/matrix-org/matrix-rust-sdk/releases/tag/matrix-sdk-0.11.0)) # (2025-03-31) Version 1.5.1 - (**Internal Improvement**) Dependency updates # (2025-02-27) Version 1.5.0 - (**Feature**) Add support for sending Speech-to-Text replies for [Transcribe-only mode](./docs/features.md#transcribe-only-mode) as regular text messages instead of notices and doing it so by default ([a1bd292752](https://github.com/etkecc/baibot/commit/a1bd292752bdd37a196788c73d00b5619e843a78)) - improvement for [issue #14](https://github.com/etkecc/baibot/issues/14). See [🦻 Speech-to-Text / 🪄 Message Type for non-threaded only-transcribed messages](./docs/configuration/speech-to-text.md#-message-type-for-non-threaded-only-transcribed-messages) for details. - (**Feature**) Add config setting controlling if a self-introduction message is posted after joining a room ([c051da2f4a](https://github.com/etkecc/baibot/commit/c051da2f4a161de0974ebb917f7a52d01f5a001f)) - fixes [issue #32](https://github.com/etkecc/baibot/issues/32). You may wish to add a `room.post_join_self_introduction_enabled` property to your configuration. See the [sample config](./etc/app/config.yml.dist) for details. If unspecified, it defaults to `true` anyway which preserves the old behavior. - (**Feature**) Add support for configuring `max_completion_tokens` for OpenAI ([47d8edea70](https://github.com/etkecc/baibot/commit/47d8edea705a44aa25a9bfaec4888c0f9ea8700e)) - (**Improvement**) Dependency updates. This version is based on [mxlink](https://crates.io/crates/mxlink)@1.6.1 (which is based on the newly released [matrix-sdk](https://crates.io/crates/matrix-sdk)@[0.10.0](https://github.com/matrix-org/matrix-rust-sdk/releases/tag/matrix-sdk-0.10.0)) - (**Improvement**) Populate image/audio attachment `body` with a filename, not with text to avoid incorrect rendering in Element Web, etc. ([ec1879d212](https://github.com/etkecc/baibot/commit/ec1879d212fa8d6e5f8590486e94c72abfcb75a5)) - (**Improvement**) Replace Anthropic library ([anthropic-rs](https://crates.io/crates/anthropic-rs) -> [anthropic](https://crates.io/crates/anthropic)) and switch default recommended model (`claude-3-5-sonnet-20240620` -> `claude-3-7-sonnet-20250219`) ([692d61b239](https://github.com/etkecc/baibot/commit/692d61b2398f073b81d32d4cbe8145ab3929e48c)) - fixes [issue #22](https://github.com/etkecc/baibot/issues/22) - (**Internal Improvement**) Switch to native building of `arm64` container images to decrease total build times from ~40 minutes to ~8 minutes ([6719538530b](https://github.com/etkecc/baibot/commit/6719538530bf76b3ff2d24077b2a7fa868276b79)) - (**Internal Improvement**) Various other internal changes, including upgrading [Rust from 1.82 to 1.85 and switching to Rust edition 2024](https://blog.rust-lang.org/2025/02/20/Rust-1.85.0.html) # (2024-12-12) Version 1.4.1 - (**Bugfix**) Fix detection for whether the bot is the last member in a room, to avoid incorrectly leaving multi-user rooms that have had at least one person `leave` ([3c47d40781](https://github.com/etkecc/baibot/commit/3c47d407819aa9c0121117a411858238724f06da)) # (2024-11-19) Version 1.4.0 - (**Improvement**) Dependency updates. This version is based on [mxlink](https://crates.io/crates/mxlink)@1.4.0 (which is based on the newly released [matrix-sdk](https://crates.io/crates/matrix-sdk)@[0.8.0](https://github.com/matrix-org/matrix-rust-sdk/releases/tag/matrix-sdk-0.8.0)). Once you run this version at least once and your matrix-sdk datastore gets upgraded to the new schema, **you will not be able to downgrade to older baibot versions** (based on the older matrix-sdk), unless you start with an empty datastore. - (**Bugfix**) Add missing typing notices sending functionality while generating images ([9d166e35ba](https://github.com/etkecc/baibot/commit/9d166e35ba6fc0daaf69318870e92436f3302056)) - (**Feature**) Support for [Matrix authenticated media](https://matrix.org/docs/spec-guides/authed-media-servers/), thanks to upgrading [mxlink](https://crates.io/crates/mxlink) / [matrix-sdk](https://crates.io/crates/matrix-sdk) - fixes [issue #12](https://github.com/etkecc/baibot/issues/12) # (2024-11-12) Version 1.3.2 Dependency updates. # (2024-10-03) Version 1.3.1 - (**Improvement**) Improves fallback user mentions support for old clients (like Element iOS) which use the bot's display name (not its full Matrix User ID). ([d9a045a5e4](https://github.com/etkecc/baibot/commit/d9a045a5e41d2b99694f92ec9e90f47529546d89)) # (2024-10-03) Version 1.3.0 **TLDR**: you can now use OpenAI's [o1](https://platform.openai.com/docs/models/o1) models, benefit from [prompt caching](https://platform.openai.com/docs/guides/prompt-caching) and mention the bot again from old clients lacking proper [user mentions support](https://spec.matrix.org/latest/client-server-api/#user-and-room-mentions) (like Element iOS). - (**Feature**) Introduces a new `baibot_conversation_start_time_utc` [prompt variable](./docs/configuration/text-generation.md#️-prompt-override) which is not a moving target (like the `baibot_now_utc` variable) and allows [prompt caching](https://platform.openai.com/docs/guides/prompt-caching) to work. All default/sample configs have been adjusted to make use of this new variable, but users need to adjust your existing dynamically-created agents to start using it. ([85e66406dc](https://github.com/etkecc/baibot/commit/85e66406dc6f430741c7819f420e2df4ae6e8d3b)) - (**Improvement**) Allows for the `max_response_tokens` configuration value for the [OpenAI provider](./docs/providers.md#openai) to be set to `null` to allow [o1](https://platform.openai.com/docs/models/o1) models (which do not support `max_response_tokens`) to be used. See the new o1 sample config [here](./docs/sample-provider-configs/openai-o1.yml). ([db9422740c](https://github.com/etkecc/baibot/commit/db9422740ceca32956d9628b6326b8be206344e2)) - (**Improvement**) Switches the sample configs for the [OpenAI provider](./docs/providers.md#openai) to point to the `gpt-4o` model, which since 2024-10-02 is the same as the `gpt-4o-2024-08-06` model. We previously explicitly pointed the bot to the `gpt-4o-2024-08-06` model, because it was much better (longer context window). Now that `gpt-4o` points to the same powerful model, we don't need to pin its version anymore. Existing users may wish to adjust their configuration to match. ([90fbad5b64](https://github.com/etkecc/baibot/commit/90fbad5b643cd06c23179f055a309ec6a7cba161)) - (**Bugfix**) Restores fallback user mentions support (via regular text, not via the [user mentions spec](https://spec.matrix.org/latest/client-server-api/#user-and-room-mentions)) to allow certain old clients (like Element iOS) to be able to mention the bot again. Support for this was intentionally removed recently (in [v1.2.0](#2024-10-01-version-120)), but it turned out to be too early to do this. ([b40226826f](https://github.com/etkecc/baibot/commit/b40226826fe914d0d5d265230ebc5bac8058b6f7)) # (2024-10-01) Version 1.2.0 - (**Feature**) Adds support for [on-demand involvement](./docs/features.md#on-demand-involvement) of the bot (via mention) in arbitrary threads and reply chains ([9908512968](https://github.com/etkecc/baibot/commit/990851296828168c2106eb3f4668833e9e5a7463)) - fixes [issue #15](https://github.com/etkecc/baibot/issues/15) - (**Improvement**) Simplifies [Transcribe-only mode](./docs/features.md#transcribe-only-mode) reply format (removing `> 🦻` prefixing) to allow easier forwarding, etc. ([e6aa956423](https://github.com/etkecc/baibot/commit/e6aa95642376ee7d87932d0e66dcfedf261b188b)) - fixes [issue #14](https://github.com/etkecc/baibot/issues/14) - (**Bugfix**) Fixes speech-to-text replies rendering incorrectly in certain clients, due to them confusing our old reply format with [fallback for rich replies](https://spec.matrix.org/v1.11/client-server-api/#fallbacks-for-rich-replies) ([e6aa956423](https://github.com/etkecc/baibot/commit/e6aa95642376ee7d87932d0e66dcfedf261b188b)) - fixes [issue #17](https://github.com/etkecc/baibot/issues/17) # (2024-09-22) Version 1.1.1 - (**Bugfix**) Fix thread messages being lost due to lack of pagination support ([d4ddd29660](https://github.com/etkecc/baibot/commit/d4ddd29660d9f51d248119dd6032e68ab29e7d35)) - fixes [issue #13](https://github.com/etkecc/baibot/issues/13) - (**Bugfix**) Fix Anthropic conversations getting stuck when being impatient and sending multiple consecutive messages ([8b12bdf2b3](https://github.com/etkecc/baibot/commit/8b12bdf2b3196abea0e8db33d7c50fff48341cb9)) - fixes [issue #13](https://github.com/etkecc/baibot/issues/13) # (2024-09-21) Version 1.1.0 - (**Feature**) Adds support for [prompt variables](./docs/configuration/text-generation.md#️-prompt-override) (date/time, bot name, model id) ([2a5a2d6a4d](https://github.com/etkecc/baibot/commit/2a5a2d6a4dbf5fd7cb504ac07d4187fdc32ae395)) - fixes [issue #10](https://github.com/etkecc/baibot/issues/10) - (**Improvement**) [Dockerfile](./Dockerfile) changes to produce ~20MB smaller container images ([354063abb7](https://github.com/etkecc/baibot/commit/354063abb79035069bd3b26c53214874e9cdd95d)) - (**Improvement**) [Dockerfile](./Dockerfile) changes to optimize local (debug) runs in a container ([c8c5e0e540](https://github.com/etkecc/baibot/commit/c8c5e0e540ab981e849452eb3ddb0378105e1fc6)) - (**Improvement**) CI changes to try and work around multi-arch image issues like [this one](https://github.com/etkecc/baibot/issues/2) ([5de7559ed6](https://github.com/etkecc/baibot/commit/5de7559ed685a41c22dfc12283681f02f4c2ee00)) # (2024-09-19) Version 1.0.6 Improvements to: - messages sent by the bot - better onboarding flow, especially when no agents have been created yet - documentation pages # (2024-09-14) Version 1.0.5 Further [improves](https://github.com/etkecc/baibot/commit/3b25b92a81a05ebaf1c6dbabf675fbfbe6c9f418) the typing notification logic, so that it tolerates edge cases better. # (2024-09-14) Version 1.0.4 [Improves](https://github.com/etkecc/baibot/commit/dd1dd78312e3db7f92b37fb3b4750fbe35de7115) the typing notification logic. # (2024-09-13) Version 1.0.3 Contains [fixes](https://github.com/etkecc/rust-mxlink/commit/f339fc85e69aa7f614394ad303d1614cd307319c) for [some](https://github.com/etkecc/baibot/issues/1) startup failures caused by partial initialization (errors during startup). # (2024-09-12) Version 1.0.0 Initial release. 🎉 ================================================ FILE: Cargo.toml ================================================ [package] name = "baibot" description = "A Matrix bot for using diffent capabilities (text-generation, text-to-speech, speech-to-text, image-generation, etc.) of AI / Large Language Models" authors = ["Slavi Pantaleev "] repository = "https://github.com/etkecc/baibot" license = "AGPL-3.0-or-later" readme = "README.md" keywords = ["matrix", "chat", "bot", "AI", "LLM"] include = ["/etc/assets/baibot-torso-768.png", "/src", "/README.md", "/CHANGELOG.md", "/LICENSE"] version = "1.19.2" edition = "2024" [lib] name = "baibot" path = "src/lib.rs" [dependencies] anthropic = { git = "https://github.com/etkecc/anthropic-rs.git", branch = "fix-content-block-image" } anyhow = "1.0.*" async-openai = { version = "0.40.0", features = ["audio", "chat-completion", "image", "responses"] } base64 = "0.22.*" chrono = { version = "0.4.*", default-features = false, features = ["std", "now"] } # We'd rather not depend on this, but we cannot use the ruma-events EventContent macro without it. matrix-sdk = { version = "0.17.0", default-features = false } mime_guess = "2.0.*" mxidwc = "1.0.*" mxlink = ">=1.14.0" etke_openai_api_rust = "0.1.*" quick_cache = "0.6.*" regex = "1.12.*" serde = { version = "1.0.*", features = ["derive"], default-features = false } serde_json = "1.0.*" serde_yaml_ng = "0.10.*" tempfile = "3.27.*" tiktoken-rs = { version = "0.11.*", default-features = false } tokio = { version = "1.52.*", features = ["rt", "rt-multi-thread", "macros"] } tracing = "0.1.*" tracing-subscriber = { version = "0.3.*", features = ["env-filter"] } url = "2.5.*" [profile.release] strip = true opt-level = "z" lto = "thin" ================================================ FILE: Dockerfile ================================================ ####################################### # # # Stage 1: building # # # ####################################### FROM docker.io/rust:1.95.0-slim-trixie AS build RUN apt-get update && apt-get install -y build-essential pkg-config libssl-dev libsqlite3-dev ENV CARGO_HOME=/cargo ENV CARGO_TARGET_DIR=/target WORKDIR /app COPY . /app ARG RELEASE_BUILD=true RUN --mount=type=cache,target=/cargo,sharing=locked \ --mount=type=cache,target=/target,sharing=locked \ if [ "$RELEASE_BUILD" = "true" ]; then \ cargo build --release; \ else \ cargo build; \ fi # Move it out of the mounted cache, so we can copy it in the next stage. RUN --mount=type=cache,target=/target,sharing=locked \ if [ "$RELEASE_BUILD" = "true" ]; then \ cp /target/release/baibot /baibot; \ else \ cp /target/debug/baibot /baibot; \ fi ####################################### # # # Stage 2: packaging # # # ####################################### FROM docker.io/debian:trixie-slim RUN apt-get update && apt-get install -y ca-certificates sqlite3 && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* WORKDIR /app COPY --from=build /baibot . ENTRYPOINT ["/bin/sh", "-c"] CMD ["/app/baibot"] ================================================ FILE: Dockerfile.ci ================================================ ####################################### # # # Stage 1: building # # # ####################################### FROM docker.io/rust:1.95.0-slim-trixie AS build RUN apt-get update && apt-get install -y build-essential pkg-config libssl-dev libsqlite3-dev WORKDIR /app COPY . /app RUN cargo build --release ####################################### # # # Stage 2: packaging # # # ####################################### FROM docker.io/debian:trixie-slim RUN apt-get update && apt-get install -y ca-certificates sqlite3 && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* WORKDIR /app COPY --from=build /app/target/release/baibot . ENTRYPOINT ["/bin/sh", "-c"] CMD ["/app/baibot"] ================================================ FILE: LICENSE ================================================ GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 Copyright (C) 2007 Free Software Foundation, Inc. Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. Preamble The GNU Affero General Public License is a free, copyleft license for software and other kinds of works, specifically designed to ensure cooperation with the community in the case of network server software. The licenses for most software and other practical works are designed to take away your freedom to share and change the works. By contrast, our General Public Licenses are intended to guarantee your freedom to share and change all versions of a program--to make sure it remains free software for all its users. When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for them if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs, and that you know you can do these things. Developers that use our General Public Licenses protect your rights with two steps: (1) assert copyright on the software, and (2) offer you this License which gives you legal permission to copy, distribute and/or modify the software. A secondary benefit of defending all users' freedom is that improvements made in alternate versions of the program, if they receive widespread use, become available for other developers to incorporate. Many developers of free software are heartened and encouraged by the resulting cooperation. However, in the case of software used on network servers, this result may fail to come about. The GNU General Public License permits making a modified version and letting the public access it on a server without ever releasing its source code to the public. The GNU Affero General Public License is designed specifically to ensure that, in such cases, the modified source code becomes available to the community. It requires the operator of a network server to provide the source code of the modified version running there to the users of that server. Therefore, public use of a modified version, on a publicly accessible server, gives the public access to the source code of the modified version. An older license, called the Affero General Public License and published by Affero, was designed to accomplish similar goals. This is a different license, not a version of the Affero GPL, but Affero has released a new version of the Affero GPL which permits relicensing under this license. The precise terms and conditions for copying, distribution and modification follow. TERMS AND CONDITIONS 0. Definitions. "This License" refers to version 3 of the GNU Affero General Public License. "Copyright" also means copyright-like laws that apply to other kinds of works, such as semiconductor masks. "The Program" refers to any copyrightable work licensed under this License. Each licensee is addressed as "you". "Licensees" and "recipients" may be individuals or organizations. To "modify" a work means to copy from or adapt all or part of the work in a fashion requiring copyright permission, other than the making of an exact copy. The resulting work is called a "modified version" of the earlier work or a work "based on" the earlier work. A "covered work" means either the unmodified Program or a work based on the Program. To "propagate" a work means to do anything with it that, without permission, would make you directly or secondarily liable for infringement under applicable copyright law, except executing it on a computer or modifying a private copy. Propagation includes copying, distribution (with or without modification), making available to the public, and in some countries other activities as well. To "convey" a work means any kind of propagation that enables other parties to make or receive copies. Mere interaction with a user through a computer network, with no transfer of a copy, is not conveying. An interactive user interface displays "Appropriate Legal Notices" to the extent that it includes a convenient and prominently visible feature that (1) displays an appropriate copyright notice, and (2) tells the user that there is no warranty for the work (except to the extent that warranties are provided), that licensees may convey the work under this License, and how to view a copy of this License. If the interface presents a list of user commands or options, such as a menu, a prominent item in the list meets this criterion. 1. Source Code. The "source code" for a work means the preferred form of the work for making modifications to it. "Object code" means any non-source form of a work. A "Standard Interface" means an interface that either is an official standard defined by a recognized standards body, or, in the case of interfaces specified for a particular programming language, one that is widely used among developers working in that language. The "System Libraries" of an executable work include anything, other than the work as a whole, that (a) is included in the normal form of packaging a Major Component, but which is not part of that Major Component, and (b) serves only to enable use of the work with that Major Component, or to implement a Standard Interface for which an implementation is available to the public in source code form. A "Major Component", in this context, means a major essential component (kernel, window system, and so on) of the specific operating system (if any) on which the executable work runs, or a compiler used to produce the work, or an object code interpreter used to run it. The "Corresponding Source" for a work in object code form means all the source code needed to generate, install, and (for an executable work) run the object code and to modify the work, including scripts to control those activities. However, it does not include the work's System Libraries, or general-purpose tools or generally available free programs which are used unmodified in performing those activities but which are not part of the work. For example, Corresponding Source includes interface definition files associated with source files for the work, and the source code for shared libraries and dynamically linked subprograms that the work is specifically designed to require, such as by intimate data communication or control flow between those subprograms and other parts of the work. The Corresponding Source need not include anything that users can regenerate automatically from other parts of the Corresponding Source. The Corresponding Source for a work in source code form is that same work. 2. Basic Permissions. All rights granted under this License are granted for the term of copyright on the Program, and are irrevocable provided the stated conditions are met. This License explicitly affirms your unlimited permission to run the unmodified Program. The output from running a covered work is covered by this License only if the output, given its content, constitutes a covered work. This License acknowledges your rights of fair use or other equivalent, as provided by copyright law. You may make, run and propagate covered works that you do not convey, without conditions so long as your license otherwise remains in force. You may convey covered works to others for the sole purpose of having them make modifications exclusively for you, or provide you with facilities for running those works, provided that you comply with the terms of this License in conveying all material for which you do not control copyright. Those thus making or running the covered works for you must do so exclusively on your behalf, under your direction and control, on terms that prohibit them from making any copies of your copyrighted material outside their relationship with you. Conveying under any other circumstances is permitted solely under the conditions stated below. Sublicensing is not allowed; section 10 makes it unnecessary. 3. Protecting Users' Legal Rights From Anti-Circumvention Law. No covered work shall be deemed part of an effective technological measure under any applicable law fulfilling obligations under article 11 of the WIPO copyright treaty adopted on 20 December 1996, or similar laws prohibiting or restricting circumvention of such measures. When you convey a covered work, you waive any legal power to forbid circumvention of technological measures to the extent such circumvention is effected by exercising rights under this License with respect to the covered work, and you disclaim any intention to limit operation or modification of the work as a means of enforcing, against the work's users, your or third parties' legal rights to forbid circumvention of technological measures. 4. Conveying Verbatim Copies. You may convey verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice; keep intact all notices stating that this License and any non-permissive terms added in accord with section 7 apply to the code; keep intact all notices of the absence of any warranty; and give all recipients a copy of this License along with the Program. You may charge any price or no price for each copy that you convey, and you may offer support or warranty protection for a fee. 5. Conveying Modified Source Versions. You may convey a work based on the Program, or the modifications to produce it from the Program, in the form of source code under the terms of section 4, provided that you also meet all of these conditions: a) The work must carry prominent notices stating that you modified it, and giving a relevant date. b) The work must carry prominent notices stating that it is released under this License and any conditions added under section 7. This requirement modifies the requirement in section 4 to "keep intact all notices". c) You must license the entire work, as a whole, under this License to anyone who comes into possession of a copy. This License will therefore apply, along with any applicable section 7 additional terms, to the whole of the work, and all its parts, regardless of how they are packaged. This License gives no permission to license the work in any other way, but it does not invalidate such permission if you have separately received it. d) If the work has interactive user interfaces, each must display Appropriate Legal Notices; however, if the Program has interactive interfaces that do not display Appropriate Legal Notices, your work need not make them do so. A compilation of a covered work with other separate and independent works, which are not by their nature extensions of the covered work, and which are not combined with it such as to form a larger program, in or on a volume of a storage or distribution medium, is called an "aggregate" if the compilation and its resulting copyright are not used to limit the access or legal rights of the compilation's users beyond what the individual works permit. Inclusion of a covered work in an aggregate does not cause this License to apply to the other parts of the aggregate. 6. Conveying Non-Source Forms. You may convey a covered work in object code form under the terms of sections 4 and 5, provided that you also convey the machine-readable Corresponding Source under the terms of this License, in one of these ways: a) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by the Corresponding Source fixed on a durable physical medium customarily used for software interchange. b) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by a written offer, valid for at least three years and valid for as long as you offer spare parts or customer support for that product model, to give anyone who possesses the object code either (1) a copy of the Corresponding Source for all the software in the product that is covered by this License, on a durable physical medium customarily used for software interchange, for a price no more than your reasonable cost of physically performing this conveying of source, or (2) access to copy the Corresponding Source from a network server at no charge. c) Convey individual copies of the object code with a copy of the written offer to provide the Corresponding Source. This alternative is allowed only occasionally and noncommercially, and only if you received the object code with such an offer, in accord with subsection 6b. d) Convey the object code by offering access from a designated place (gratis or for a charge), and offer equivalent access to the Corresponding Source in the same way through the same place at no further charge. You need not require recipients to copy the Corresponding Source along with the object code. If the place to copy the object code is a network server, the Corresponding Source may be on a different server (operated by you or a third party) that supports equivalent copying facilities, provided you maintain clear directions next to the object code saying where to find the Corresponding Source. Regardless of what server hosts the Corresponding Source, you remain obligated to ensure that it is available for as long as needed to satisfy these requirements. e) Convey the object code using peer-to-peer transmission, provided you inform other peers where the object code and Corresponding Source of the work are being offered to the general public at no charge under subsection 6d. A separable portion of the object code, whose source code is excluded from the Corresponding Source as a System Library, need not be included in conveying the object code work. A "User Product" is either (1) a "consumer product", which means any tangible personal property which is normally used for personal, family, or household purposes, or (2) anything designed or sold for incorporation into a dwelling. In determining whether a product is a consumer product, doubtful cases shall be resolved in favor of coverage. For a particular product received by a particular user, "normally used" refers to a typical or common use of that class of product, regardless of the status of the particular user or of the way in which the particular user actually uses, or expects or is expected to use, the product. A product is a consumer product regardless of whether the product has substantial commercial, industrial or non-consumer uses, unless such uses represent the only significant mode of use of the product. "Installation Information" for a User Product means any methods, procedures, authorization keys, or other information required to install and execute modified versions of a covered work in that User Product from a modified version of its Corresponding Source. The information must suffice to ensure that the continued functioning of the modified object code is in no case prevented or interfered with solely because modification has been made. If you convey an object code work under this section in, or with, or specifically for use in, a User Product, and the conveying occurs as part of a transaction in which the right of possession and use of the User Product is transferred to the recipient in perpetuity or for a fixed term (regardless of how the transaction is characterized), the Corresponding Source conveyed under this section must be accompanied by the Installation Information. But this requirement does not apply if neither you nor any third party retains the ability to install modified object code on the User Product (for example, the work has been installed in ROM). The requirement to provide Installation Information does not include a requirement to continue to provide support service, warranty, or updates for a work that has been modified or installed by the recipient, or for the User Product in which it has been modified or installed. Access to a network may be denied when the modification itself materially and adversely affects the operation of the network or violates the rules and protocols for communication across the network. Corresponding Source conveyed, and Installation Information provided, in accord with this section must be in a format that is publicly documented (and with an implementation available to the public in source code form), and must require no special password or key for unpacking, reading or copying. 7. Additional Terms. "Additional permissions" are terms that supplement the terms of this License by making exceptions from one or more of its conditions. Additional permissions that are applicable to the entire Program shall be treated as though they were included in this License, to the extent that they are valid under applicable law. If additional permissions apply only to part of the Program, that part may be used separately under those permissions, but the entire Program remains governed by this License without regard to the additional permissions. When you convey a copy of a covered work, you may at your option remove any additional permissions from that copy, or from any part of it. (Additional permissions may be written to require their own removal in certain cases when you modify the work.) You may place additional permissions on material, added by you to a covered work, for which you have or can give appropriate copyright permission. Notwithstanding any other provision of this License, for material you add to a covered work, you may (if authorized by the copyright holders of that material) supplement the terms of this License with terms: a) Disclaiming warranty or limiting liability differently from the terms of sections 15 and 16 of this License; or b) Requiring preservation of specified reasonable legal notices or author attributions in that material or in the Appropriate Legal Notices displayed by works containing it; or c) Prohibiting misrepresentation of the origin of that material, or requiring that modified versions of such material be marked in reasonable ways as different from the original version; or d) Limiting the use for publicity purposes of names of licensors or authors of the material; or e) Declining to grant rights under trademark law for use of some trade names, trademarks, or service marks; or f) Requiring indemnification of licensors and authors of that material by anyone who conveys the material (or modified versions of it) with contractual assumptions of liability to the recipient, for any liability that these contractual assumptions directly impose on those licensors and authors. All other non-permissive additional terms are considered "further restrictions" within the meaning of section 10. If the Program as you received it, or any part of it, contains a notice stating that it is governed by this License along with a term that is a further restriction, you may remove that term. If a license document contains a further restriction but permits relicensing or conveying under this License, you may add to a covered work material governed by the terms of that license document, provided that the further restriction does not survive such relicensing or conveying. If you add terms to a covered work in accord with this section, you must place, in the relevant source files, a statement of the additional terms that apply to those files, or a notice indicating where to find the applicable terms. Additional terms, permissive or non-permissive, may be stated in the form of a separately written license, or stated as exceptions; the above requirements apply either way. 8. Termination. You may not propagate or modify a covered work except as expressly provided under this License. Any attempt otherwise to propagate or modify it is void, and will automatically terminate your rights under this License (including any patent licenses granted under the third paragraph of section 11). However, if you cease all violation of this License, then your license from a particular copyright holder is reinstated (a) provisionally, unless and until the copyright holder explicitly and finally terminates your license, and (b) permanently, if the copyright holder fails to notify you of the violation by some reasonable means prior to 60 days after the cessation. Moreover, your license from a particular copyright holder is reinstated permanently if the copyright holder notifies you of the violation by some reasonable means, this is the first time you have received notice of violation of this License (for any work) from that copyright holder, and you cure the violation prior to 30 days after your receipt of the notice. Termination of your rights under this section does not terminate the licenses of parties who have received copies or rights from you under this License. If your rights have been terminated and not permanently reinstated, you do not qualify to receive new licenses for the same material under section 10. 9. Acceptance Not Required for Having Copies. You are not required to accept this License in order to receive or run a copy of the Program. Ancillary propagation of a covered work occurring solely as a consequence of using peer-to-peer transmission to receive a copy likewise does not require acceptance. However, nothing other than this License grants you permission to propagate or modify any covered work. These actions infringe copyright if you do not accept this License. Therefore, by modifying or propagating a covered work, you indicate your acceptance of this License to do so. 10. Automatic Licensing of Downstream Recipients. Each time you convey a covered work, the recipient automatically receives a license from the original licensors, to run, modify and propagate that work, subject to this License. You are not responsible for enforcing compliance by third parties with this License. An "entity transaction" is a transaction transferring control of an organization, or substantially all assets of one, or subdividing an organization, or merging organizations. If propagation of a covered work results from an entity transaction, each party to that transaction who receives a copy of the work also receives whatever licenses to the work the party's predecessor in interest had or could give under the previous paragraph, plus a right to possession of the Corresponding Source of the work from the predecessor in interest, if the predecessor has it or can get it with reasonable efforts. You may not impose any further restrictions on the exercise of the rights granted or affirmed under this License. For example, you may not impose a license fee, royalty, or other charge for exercise of rights granted under this License, and you may not initiate litigation (including a cross-claim or counterclaim in a lawsuit) alleging that any patent claim is infringed by making, using, selling, offering for sale, or importing the Program or any portion of it. 11. Patents. A "contributor" is a copyright holder who authorizes use under this License of the Program or a work on which the Program is based. The work thus licensed is called the contributor's "contributor version". A contributor's "essential patent claims" are all patent claims owned or controlled by the contributor, whether already acquired or hereafter acquired, that would be infringed by some manner, permitted by this License, of making, using, or selling its contributor version, but do not include claims that would be infringed only as a consequence of further modification of the contributor version. For purposes of this definition, "control" includes the right to grant patent sublicenses in a manner consistent with the requirements of this License. Each contributor grants you a non-exclusive, worldwide, royalty-free patent license under the contributor's essential patent claims, to make, use, sell, offer for sale, import and otherwise run, modify and propagate the contents of its contributor version. In the following three paragraphs, a "patent license" is any express agreement or commitment, however denominated, not to enforce a patent (such as an express permission to practice a patent or covenant not to sue for patent infringement). To "grant" such a patent license to a party means to make such an agreement or commitment not to enforce a patent against the party. If you convey a covered work, knowingly relying on a patent license, and the Corresponding Source of the work is not available for anyone to copy, free of charge and under the terms of this License, through a publicly available network server or other readily accessible means, then you must either (1) cause the Corresponding Source to be so available, or (2) arrange to deprive yourself of the benefit of the patent license for this particular work, or (3) arrange, in a manner consistent with the requirements of this License, to extend the patent license to downstream recipients. "Knowingly relying" means you have actual knowledge that, but for the patent license, your conveying the covered work in a country, or your recipient's use of the covered work in a country, would infringe one or more identifiable patents in that country that you have reason to believe are valid. If, pursuant to or in connection with a single transaction or arrangement, you convey, or propagate by procuring conveyance of, a covered work, and grant a patent license to some of the parties receiving the covered work authorizing them to use, propagate, modify or convey a specific copy of the covered work, then the patent license you grant is automatically extended to all recipients of the covered work and works based on it. A patent license is "discriminatory" if it does not include within the scope of its coverage, prohibits the exercise of, or is conditioned on the non-exercise of one or more of the rights that are specifically granted under this License. You may not convey a covered work if you are a party to an arrangement with a third party that is in the business of distributing software, under which you make payment to the third party based on the extent of your activity of conveying the work, and under which the third party grants, to any of the parties who would receive the covered work from you, a discriminatory patent license (a) in connection with copies of the covered work conveyed by you (or copies made from those copies), or (b) primarily for and in connection with specific products or compilations that contain the covered work, unless you entered into that arrangement, or that patent license was granted, prior to 28 March 2007. Nothing in this License shall be construed as excluding or limiting any implied license or other defenses to infringement that may otherwise be available to you under applicable patent law. 12. No Surrender of Others' Freedom. If conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot convey a covered work so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not convey it at all. For example, if you agree to terms that obligate you to collect a royalty for further conveying from those to whom you convey the Program, the only way you could satisfy both those terms and this License would be to refrain entirely from conveying the Program. 13. Remote Network Interaction; Use with the GNU General Public License. Notwithstanding any other provision of this License, if you modify the Program, your modified version must prominently offer all users interacting with it remotely through a computer network (if your version supports such interaction) an opportunity to receive the Corresponding Source of your version by providing access to the Corresponding Source from a network server at no charge, through some standard or customary means of facilitating copying of software. This Corresponding Source shall include the Corresponding Source for any work covered by version 3 of the GNU General Public License that is incorporated pursuant to the following paragraph. Notwithstanding any other provision of this License, you have permission to link or combine any covered work with a work licensed under version 3 of the GNU General Public License into a single combined work, and to convey the resulting work. The terms of this License will continue to apply to the part which is the covered work, but the work with which it is combined will remain governed by version 3 of the GNU General Public License. 14. Revised Versions of this License. The Free Software Foundation may publish revised and/or new versions of the GNU Affero General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. Each version is given a distinguishing version number. If the Program specifies that a certain numbered version of the GNU Affero General Public License "or any later version" applies to it, you have the option of following the terms and conditions either of that numbered version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of the GNU Affero General Public License, you may choose any version ever published by the Free Software Foundation. If the Program specifies that a proxy can decide which future versions of the GNU Affero General Public License can be used, that proxy's public statement of acceptance of a version permanently authorizes you to choose that version for the Program. Later license versions may give you additional or different permissions. However, no additional obligations are imposed on any author or copyright holder as a result of your choosing to follow a later version. 15. Disclaimer of Warranty. THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 16. Limitation of Liability. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. 17. Interpretation of Sections 15 and 16. If the disclaimer of warranty and limitation of liability provided above cannot be given local legal effect according to their terms, reviewing courts shall apply local law that most closely approximates an absolute waiver of all civil liability in connection with the Program, unless a warranty or assumption of liability accompanies a copy of the Program in return for a fee. END OF TERMS AND CONDITIONS How to Apply These Terms to Your New Programs If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms. To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively state the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found. Copyright (C) This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see . Also add information on how to contact you by electronic and paper mail. If your software can interact with users remotely through a computer network, you should also make sure that it provides a way for users to get its source. For example, if your program is a web application, its interface could display a "Source" link that leads users to an archive of the code. There are many ways you could offer source, and different solutions will be better for different programs; see section 13 for the specific requirements. You should also get your employer (if you work as a programmer) or school, if any, to sign a "copyright disclaimer" for the program, if necessary. For more information on this, and how to apply and follow the GNU AGPL, see . ================================================ FILE: README.md ================================================

baibot logo

baibot

🤖 baibot is an [AI](https://en.wikipedia.org/wiki/Artificial_intelligence) ([Large Language Model](https://en.wikipedia.org/wiki/Large_language_model)) bot for [Matrix](https://matrix.org/) built by [etke.cc](https://etke.cc/) (managed Matrix servers). The name is pronounced 'bye'-bot and is a play on [AI](https://en.wikipedia.org/wiki/Artificial_intelligence), referencing the fictional character [🇧🇬 Bai Ganyo](https://en.wikipedia.org/wiki/Bay_Ganyo). It's designed as a more private and [featureful](#-features) alternative to [matrix-chatgpt-bot](https://github.com/matrixgpt/matrix-chatgpt-bot). It's influenced by [chaz](https://github.com/arcuru/chaz), but does **not** use the [AIChat](https://github.com/sigoden/aichat) CLI tool and instead does everything in-process, without forking. ## 🌟 Features - 🎨 Encourages **[provider](./docs/providers.md) choice** ([Anthropic](./docs/providers.md#anthropic), [Groq](./docs/providers.md#groq), [LocalAI](./docs/providers.md#localai), [OpenAI](./docs/providers.md#openai) and [☁️ many more](./docs/providers.md#️-providers)) as well as **[mixing & matching models](./docs/features.md#-mixing--matching-models)**: - Supports **different use purposes** (depending on the [☁️ provider](./docs/providers.md) & model): - [💬 text-generation](./docs/features.md#-text-generation): communicating with you via text (though certain models may "see" images as well). The [OpenAI provider](./docs/providers.md#openai) also supports [🛠️ built-in tools](./docs/features.md#️-built-in-tools-openai-only) (web search, code interpreter) - [🦻 speech-to-text](./docs/features.md#-speech-to-text): turning your voice messages into text - [🗣️ text-to-speech](./docs/features.md#%EF%B8%8F-text-to-speech): turning bot or users text messages into voice messages - [🖌️ image-generation](./docs/features.md#image-generation): creating and editing images based on instructions - 🪄 Supports [seamless voice interaction](./docs/features.md#seamless-voice-interaction) (turning user voice messages into text, answering in text, then turning that text back into voice) - 🦻 Supports [transcribe-only mode](./docs/features.md#transcribe-only-mode) (turning user voice messages into text, without doing text-generation) - 🗣️ Supports [text-to-speech-only mode](./docs/features.md#text-to-speech-only-mode) (turning user text messages into voice, without doing text-generation) - 🔒 Supports [encryption](./docs/features.md#-encryption) for Matrix communication and Account-Data-stored configuration - ♻️ Supports [context-management](./docs/configuration/text-generation.md#️-context-management) handling on some models (automatically adjusting the message history length, etc.) - 🛠️ Allows **customizing much of the bot's [configuration](./docs/configuration/README.md)** at runtime (using commands sent via chat) - 👥 **Actively maintained** by the team at [etke.cc](https://etke.cc/) ## 🖼️ Screenshots ![Introduction and general usage](./docs/screenshots/introduction-and-general-usage.webp) You can find more screenshots on the [🌟 Features](./docs/features.md) and other [📚 Documentation](./docs/README.md) pages, as well as in the [docs/screenshots](./docs/screenshots) directory. ## 🚀 Getting Started 🗲 For a quick experiment, you can refer to the [🧑‍💻 development documentation](./docs/development.md) which contains information on how to build and run the bot (and its various dependency services) locally. For a real installation, see the [🚀 Installation](./docs/installation.md) documentation which contains information on [🐋 Running in a container](./docs/installation.md#-running-in-a-container) and [🖥️️️️️ Running a binary](./docs/installation.md#-running-a-binary). ## 📚 Documentation See the bot's [📚 documentation](./docs/README.md) for more information on how to use and configure the bot. ## 💻 Development See the bot's [🧑‍💻 development documentation](./docs/development.md) for more information on how to develop on the bot. ## 📜 Changes This bot evolves over time, sometimes with backward-incompatible changes. When updating the bot, refer to [the changelog](CHANGELOG.md) to catch up with what's new. ## 🆘 Support - Matrix room: [#baibot:etke.cc](https://matrix.to/#/#baibot:etke.cc) - GitHub issues: [etkecc/baibot/issues](https://github.com/etkecc/baibot/issues) - (for [etke.cc](https://etke.cc/) customers): etke.cc [support](https://etke.cc/contacts/) ================================================ FILE: docs/README.md ================================================ # Table of Contents - [🔒 Access](./access.md) - [🤖 Agents](./agents.md) - [🛠️ Configuration](./configuration/README.md) - [🌟 Features](./features.md) - [🤝 Handlers](./configuration/handlers.md) - [☁️ Providers](./providers.md) - [📖 Usage](./usage.md) - [🚀 Installation](./installation.md) - [💻 Development](./development.md) ================================================ FILE: docs/access.md ================================================ ## 🔒 Access This bot employs access control to decide who can use its services and manage its configuration. ### 👋 Joining rooms The bot automatically joins rooms only when invited by someone considered a bot [👥 user](#-users). ### 👥 Users The bot can be used by users that match some [dynamically](./configuration/README.md#dynamic-configuration) configured [Matrix user id](https://spec.matrix.org/v1.11/#users) patterns. Users: - ✅ can **invite the bot to rooms** - ✅ can **use all the bot's [features](./features.md)** ([💬 Text Generation](./features.md#-text-generation), [🦻 Speech-to-Text](./features.md#-speech-to-text), etc.) by sending room messages - ✅ can **mention the bot** in threads and reply chains to provoke it to respond to non-user messages (see [🌟 Features / 💬 Text Generation / On-demand involvement](./features.md#on-demand-involvement)) - ✅ can **change the bot's configuration in a room** (e.g. `!bai config room ...` commands) - ❌ cannot **change the bot's global configuration** (e.g. `!bai config global ...` commands) - ❌ cannot **create new [🤖 Agents](./agents.md)** (neither in rooms, nor globally). See [💼 Room-local agent managers](#-room-local-agent-managers) for controlling which users can create agents. The following commands are available: - **Show** the currently allowed users: `!bai access users` - **Set** the list of allowed users: `!bai access set-users SPACE_SEPARATED_PATTERNS` Example patterns: `@*:example.com @*:another.com @someone:company.org` ### 👮‍♂️ Administrators Administrators can **manage the bot's configuration and access control**. Administrators are [👥 Users](#-users) and [💼 Room-local agent managers](#-room-local-agent-managers) implicitly, so they inherit all their permissions. The bot can be administrated by users that match some [statically](./configuration/README.md#static-configuration) configured [Matrix user id](https://spec.matrix.org/v1.11/#users) patterns. Administrators cannot be changed without adjusting the bot's configuration on the server. ### 💼 Room-local agent managers Room-local agent managers are users privileged to **create their own [agents](./agents.md)** (see `!bai agent`) in rooms. > [!WARNING] > Letting regular users create agents which contact arbitrary network services **may be a security issue**. The following commands are available: - **Show** the currently allowed users: `!bai access room-local-agent-managers` - **Set** the list of allowed users: `!bai access set-room-local-agent-managers SPACE_SEPARATED_PATTERNS` Example patterns: `@*:example.com @*:another.com @someone:company.org` ================================================ FILE: docs/agents.md ================================================ ## 🤖 Agents An agent is an instantiation and configuration of some [☁️ provider](./providers.md). It can support different capabilities (text-generation, speech-to-text, etc.) depending on the provider used and on the configuration of the agent. Agents can be set as **[🤝 handlers](./configuration/handlers.md) for various purposes** (text-generation, speech-to-text, etc.) globally or in specific rooms. Send a `!bai config status` command to see the current configuration. Agents can be **defined [statically](./configuration/README.md#static-configuration)** (in the server configuration) **or dynamically** (via commands sent to the bot). When [creating agents](#creating-agents) dynamically, you can do it **per-room or globally**. Globally-defined agents can be used by any authorized bot user in any room, while room-local agents can only be used in the room where they were defined. Agent configuration (like all other configuration) is stored in the Matrix Account Data of the bot user and is **potentially encrypted** (if enabled in the configuration), so that your configuration data is safe even on untrusted homeservers. ### Listing agents To **list** all available agents: `!bai agent list` #### Creating agents See a [🖼️ Screenshot of the agent creation process](./screenshots/agent-creation.webp). To **create** a new agent, you need to specify the [provider](./providers.md) and an agent id of your choosing. - **Create** a new agent: - (Accessible in **this room only**) `!bai agent create-room-local PROVIDER_ID AGENT_ID` - (Accessible in **all rooms**) `!bai agent create-global PROVIDER AGENT_ID` - Example: `!bai agent create-room-local openai my-openai-agent` The `AGENT_ID` is a unique identifier for the agent. It can be any string which **doesn't contain spaces and `/`**. Depending on where the agent is defined (within a room, globally, or [statically](./configuration/README.md#static-configuration)), this id will get a prefix (e.g. `room-local/`, `global/` or `static/`). The combined id (prefix + agent id) makes the **full agent identifier** (refered to as `FULL_AGENT_IDENTIFIER` in commands below). When creating an agent, you will be given some sample [YAML](https://en.wikipedia.org/wiki/YAML) configuration which you can use to customize the agent's behavior. This configuration varies depending on the [☁️ provider](./providers.md) used and the capabilities of the agent. Based on the configuration keys you pass, certain features will be enabled or disabled. For example, if you skip the `image_generation` key for an [OpenAI](./providers.md#openai) agent, it won't be able to generate images (see [🖌️ Image Creation](./features.md#-image-creation), [🎨 Image Editing](./features.md#-image-editing), [🫵 Sticker Creation](./features.md#-sticker-creation)). After making your modifications to the sample YAML, you submit it back to the bot and the new agent will be created. **To make use of the agent**, you need to [🤝 configure it as a handler for a given purpose](./configuration/handlers.md). ### Showing agent details To **show** full details for a given agent: `!bai agent details FULL_AGENT_IDENTIFIER` This command requires a full agent identifier (e.g. `room-local/agent-id`). ### Deleting agents To **delete** an agent: `!bai agent delete FULL_AGENT_IDENTIFIER` This command requires a full agent identifier (e.g. `room-local/agent-id`). ### Updating agents To **update** a given agent's configuration: show the agent's [details](#showing-agent-details) (current configuration), then [delete](#deleting-agents) it and finally [re-create](#creating-agents) it. ================================================ FILE: docs/configuration/README.md ================================================ ## 🛠️ Configuration The bot's behavior is controlled by a combination of [static](#static-configuration) and [dynamic](#dynamic-configuration) configuration. ### Static configuration The bot can be configured using a [YAML](https://en.wikipedia.org/wiki/YAML) configuration file as well as [environment variables](https://en.wikipedia.org/wiki/Environment_variable). When running the bot locally (during [🧑‍💻 development](../development.md)), the bot's configuration is read from the `var/app/config.yml` file. This file is created from the template found in [etc/app/config.yml.dist](../../etc/app/config.yml.dist). Certain keys can be left unset, in which case [📝 hardcoded defaults](../../src/entity/cfg/defaults.rs) would be used. Some configuration keys found in the YAML configuration can be overridden by setting an environment variable (dots should be replaced with `_`). Example: - to override `command_prefix`, set an environment variable `BAIBOT_COMMAND_PREFIX` - to override `homeserver.server_name`, set an environment variable `BAIBOT_HOMESERVER_SERVER_NAME` You can see the list of supported environment variables in the [🦀 src/entity/cfg/env.rs](../../src/entity/cfg/env.rs) file. > [!WARNING] > The static configuration contains an `initial_global_config` key, which is used to populate the bot's global configuration (stored as [dynamic configuration](#dynamic-configuration)) the first time the bot starts. Modifying this subsequently will not have any effect. After initial global configuration creation, it's expected to be managed dynamically via chat commands. For Matrix-account authentication setup, see [🔐 Authentication](./authentication.md). ### Dynamic configuration Besides the bot's [static configuration](#static-configuration), **the bot can also be configured dynamically at runtime (via chat messages)**. This includes changes to [🔒 Access](../access.md), [🤖 Agents](../agents.md) and [🛠️ Room Settings](#room-settings). #### Room Settings Room Settings come from 3 different levels with priority in the following order (higher to lower): - 📍 per-room (`!bai config room ..` commands) - 🌐 globally (`!bai config global ..` commands) - 📝 as [hardcoded defaults](../../src/entity/cfg/defaults.rs) You can adjust the following settings per room and/or globally: - [💬 Text Generation](text-generation.md) - [🦻 Speech-to-Text](speech-to-text.md) - [🗣️ Text-to-Speech](text-to-speech.md) - [🖌️ Image Creation](image-generation.md) - [🤝 Handlers](handlers.md) Refer to the bot's help messages (as a response to a `!bai config` help command) for the most up-to-date information on what Room Settings can be configured. You can **get an overview of the configuration affecting the current room** (a mix of hardcoded defaults, agent defaults, global and room-level settings) by sending a `!bai config status` command to the room. ================================================ FILE: docs/configuration/authentication.md ================================================ ## 🔐 Authentication baibot supports 2 authentication modes for the Matrix account (`user.*` keys in config). Set **exactly one** mode. If both are set (or neither is set), startup validation fails. ### Password authentication - Config key: `user.password` - Environment variable: `BAIBOT_USER_PASSWORD` ### Access token authentication - Config keys: `user.access_token` + `user.device_id` - Environment variables: `BAIBOT_USER_ACCESS_TOKEN` + `BAIBOT_USER_DEVICE_ID` Access-token authentication is useful for OIDC-enabled homeservers (e.g. those using [Matrix Authentication Service](https://github.com/element-hq/matrix-authentication-service)). Example token-generation command: ```sh mas-cli manage issue-compatibility-token [device_id] ``` ================================================ FILE: docs/configuration/handlers.md ================================================ ## 🤝 Handlers ### Introduction You can use **different models in different rooms** (e.g. [OpenAI](../providers.md#openai) GPT-4o alongside [Llama](https://en.wikipedia.org/wiki/Llama_(language_model)) running on [Groq](../providers.md#groq), etc.) You can also use **different models within the same room** (e.g. [💬 text-generation](#-text-generation) handled by one [agent](./agents.md), [🦻 speech-to-text](#-speech-to-text) handled by another, [🗣️ text-to-speech](#️-text-to-speech) by a 3rd, etc.) The bot supports the following use-purposes: - [💬 text-generation](../features.md#-text-generation): communicating with you via text (though certain models may also process images and files) - [🦻 speech-to-text](../features.md#-speech-to-text): turning your voice messages into text - [🗣️ text-to-speech](../features.md#️-text-to-speech): turning bot or users text messages into voice messages - [🖌️ image-generation](../features.md#image-generation): generating images based on instructions In a given room, each different purpose can be served by a different [provider](../providers.md) and model. This combination of provider and model configuration is called an [🤖 agent](../agents.md). Each purpose can be served by a different **handler** agent. See a [🖼️ Screenshot of an example room configuration](./screenshots/config-status-handlers.webp). ### Configuring Handlers can be configured [dynamically](./README.md#dynamic-configuration): - either per-room (e.g. `!bai config room set-handler text-generation room-local/openai-gpt-4o`) - or globally (e.g. `!bai config global set-handler text-generation global/openai-gpt-4o`) The per-room configuration takes priority over the global configuration. There's also a `catch-all` purpose that can be used as a fallback handler for messages that don't match any other handler. 💡 It's a good idea to globally-configure a powerful agent as a catch-all handler, so that the bot can always handle messages of any kind. You can then override individual handlers per room or globally. ================================================ FILE: docs/configuration/image-generation.md ================================================ ## Image Generation The Image Creation and Image Editing features are not configurable at this moment. You may also wish to see: - [🌟 Features / Image Generation / 🖌️ Image Creation](../features.md#-image-creation) for a higher-level introduction to the Image Creation features - [🌟 Features / Image Generation / 🎨 Image Editing](../features.md#-image-editing) for a higher-level introduction to the Image Editing features - [📖 Usage / Image Generation / 🖌️ Creating Images](../usage.md#-creating-images) section for more details on how to use the bot for Image Creation in a room - [📖 Usage / Image Generation / 🎨 Editing images](../usage.md#-editing-images) section for more details on how to use the bot for Image Editing in a room ================================================ FILE: docs/configuration/speech-to-text.md ================================================ ## 🦻 Speech-to-Text Below are some configuration settings related to Speech-to-Text. You may also wish to see: - [🌟 Features / 🦻 Speech-to-Text](../features.md#-speech-to-text) for a higher-level introduction to the Speech-to-Text features - [📖 Usage / 🦻 Speech-to-Text](../usage.md#-speech-to-text) section for more details on how to use the bot for Speech-to-Text in a room ### 🪄 Flow Type Controls how voice messages sent by [👥 user](../access.md#-users) are handled. The following configuration values are recognized: - (default) `transcribe_and_generate_text`: the bot will turn [👥 user](../access.md#-users) voice messages into text and then generate text messages via [💬 Text Generation](../features.md#-text-generation). This is the default setting to allow for [Seamless voice interaction](../features.md#seamless-voice-interaction). - `ignore`: the bot will ignore all audio messages - `only_transcribe`: the bot will turn [👥 user](../access.md#-users) voice messages into text, but will **not** proceed with [💬 Text Generation](../features.md#-text-generation). Switching to this may be useful in some cases, as in [Transcribe-only mode](../features.md#transcribe-only-mode). Example: `!bai config room speech-to-text set-flow-type ignore` (this can also be set globally, see [🛠️ Room Settings](./README.md#room-settings)) ### 🪄 Message Type for non-threaded only-transcribed messages Controls how the transcribed text of voice messages is sent to the chat when Flow Type = `only_transcribe`. The following configuration values are recognized: - (default) `text`: the transcribed text is sent as a regular message. This is more convenient if you'd like to forward the transcribed message to other rooms. - `notice`: the transcribed text is sent as a notice message. This provides better compatibility with other bots in the room, as they are less likely to interact with messages of type notice. Example: `!bai config room speech-to-text set-msg-type-for-non-threaded-only-transcribed-messages notice` (this can also be set globally, see [🛠️ Room Settings](./README.md#room-settings)) ### 🔤 Language Lets you specify the language of the input voice messages, to avoid using auto-detection. Supplying the input language using a 2-letter code (e.g. `ja`) as per [ISO-639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) may improve accuracy & latency. ![Speech-to-Text Language setting usage example](../screenshots/speech-to-text-language.webp) In the above example screenshot, even without a language specified, the voice was understood correctly as [Bulgarian](https://en.wikipedia.org/wiki/Bulgarian_language), but was produced in latin, not [Cyrillic](https://en.wikipedia.org/wiki/Cyrillic_script), which is wrong. If different [👥 user](../access.md#-users) are using different languages, do not specify a language. 💡 Certain models (like [OpenAI](../providers.md#openai)'s Whisper) may perform auto-translation if you specify a language, but you're speaking another one. You may abuse this side-effect for performing voice-to-text translation, but be aware that not all models behave this way. Example (setting it to Japanese): `!bai config room speech-to-text set-language ja` (this can also be set globally, see [🛠️ Room Settings](./README.md#room-settings)) ================================================ FILE: docs/configuration/text-generation.md ================================================ ## 💬 Text Generation Below are some [🛠️ dynamic configuration settings](./README.md#dynamic-configuration) related to Text Generation. You may also wish to see: - [🌟 Features / 💬 Text Generation](../features.md#-text-generation) for a higher-level introduction to the Text Generation features - [📖 Usage / 💬 Text Generation](../usage.md#-text-generation) section for more details on how to use the bot for Text Generation in a room ### 🗟 Prefix Requirement Type In Direct Message rooms with the bot (1:1 rooms), it most usually makes sense for the bot to respond to **all** of your messages, as shown on this [🖼️ screenshot](../screenshots/text-generation.webp). In group rooms (with multiple users), it may be more appropriate for the bot to only respond to messages that are **prefixed** with the command prefix (e.g. `!bai`) or which are [mentioning](https://spec.matrix.org/latest/client-server-api/#user-and-room-mentions) the bot (e.g. `@baibot`), so that other chat exchange in the room will not trigger it. Such a setup is shown on the [🖼️ On-demand involvement in the room](../screenshots/text-generation-prefix-requirement.webp) screenshot. There are exceptions to these rules, and you can configure the bot to respond only to prefixed messages in a 1:1 room, or to respond to all messages even in a multi-user group room. To support such use-cases, the bot has a `text-generation prefix-requirement-type` setting, which can be set to: - (default) `no`: indicates that the bot would not require a prefix and would respond to all messages - `command_prefix`: indicates that the bot would require that messages be prefixed with the command prefix (e.g. `!bai`) and would ignore all messages that are not prefixed By default, the bot is **auto-configured (upon joining a new room)** to use the `no` setting in rooms that only include 2 users (you and the bot), and `command_prefix` in rooms with more than 2 users. To prevent surprises, the bot will **not** adjust this setting subsequently. You can manually adjust it via `!bai config room text-generation set-prefix-requirement-type VALUE`. Example: `!bai config room text-generation set-prefix-requirement-type command_prefix` (this can also be set globally, see [🛠️ Room Settings](./README.md#room-settings)) Regardless of this configuration, **the bot will also respond to messages by allowed [👥 Users](../access.md#-users) which directly [mention](https://spec.matrix.org/latest/client-server-api/#user-and-room-mentions) the bot** (e.g. `@baibot`), even if they are not prefixed. An example of this can be seen on these screenshots: - [🖼️ On-demand involvement in a thread](../screenshots/text-generation-on-demand-thread-involvement.webp) - [🖼️ On-demand involvement in a reply chain](../screenshots/text-generation-on-demand-reply-involvement.webp) ### 🪄 Auto Usage Text generation is enabled by default (the `text-generation auto-usage` setting being set to `always`), but can be set to: - (default) `always`: generate text for all messages (also see [🗟 Prefix Requirement Type](#-prefix-requirement-type)) - `never`: never generate text for messages - `only_for_voice`: only generate text when the original user message was a voice message, later transcribed via [🦻 Speech-to-Text](../features.md#-speech-to-text) - `only_for_text`: only generate text when original user message was a text message Example: `!bai config room text-generation set-auto-usage only_for_voice` (this can also be set globally, see [🛠️ Room Settings](./README.md#room-settings)) ### ♻️ Context Management The bot also supports ♻️ **context management**, which automatically adjusts the message history length, etc. This feature relies on [tokenization](https://en.wikipedia.org/wiki/Large_language_model#Tokenization) performed by the [tiktoken-rs](https://github.com/zurawiki/tiktoken-rs) library which is [poorly well-maintained](https://github.com/zurawiki/tiktoken-rs/issues/50) and only works well for [OpenAI](../providers.md#openai) models. This setting is **disabled by default**, but can be enabled via `!bai config room text-generation set-context-management-enabled true` (this can also be set globally, see [🛠️ Room Settings](./README.md#room-settings)). ### 👤 Sender Context Mode In multi-user rooms, it may be useful for the model to know which participant sent each message in the conversation context. To support this, the bot has a `text-generation sender-context-mode` setting, which can be set to: - (default) `disabled`: do not attach sender metadata to messages before sending them to the model - `matrix_user_id`: prefix text messages with the sender's Matrix user ID, for example: `[sender=@alice:example.com] Hello bot` - `matrix_user_id_and_timestamp`: prefix text messages with the sender's Matrix user ID and the message timestamp, for example: `[sender=@alice:example.com sent_at=2026-03-23T14:30:00Z] Hello bot` This sender metadata is attached to conversation messages before they are sent to the model provider. It applies to user and assistant text messages, but not to system prompts or non-text content. ⚠️ Enabling this sends Matrix user IDs, and optionally timestamps, to the model provider. Example: `!bai config room text-generation set-sender-context-mode matrix_user_id` (this can also be set globally, see [🛠️ Room Settings](./README.md#room-settings)) ### ⌨️ Prompt Override You can override the [system prompt](https://huggingface.co/docs/transformers/en/tasks/prompting) configured at the [🤖 agent](../agents.md) level. Example (multi-line is supported): ``` !bai config room text-generation set-prompt-override You're a UI/UX expert. Everything you say needs to consider design and usability. Where appropriate, you'll mention best practices and common pitfalls. ``` A prompt override can also be set globally, see [🛠️ Room Settings](./README.md#room-settings). Prompts may contain the following **placeholder variables** which will be replaced *every time* the bot is interacted with: | Placeholder | Description | Example | |---------------------------|-------------|---------| | `{{ baibot_name }}` | Name of the bot as configured in the `user.name` field in the [Static configuration](./README.md#static-configuration) | `Baibot` | | `{{ baibot_model_id }}` | Text-Generation model ID as configured in the [🤖 agent](../agents.md)'s configuration | `gpt-4o` | | `{{ baibot_now_utc }}` | Current date and time in UTC (⚠️ usage may break prompt caching - see below) | `2024-09-20 (Friday), 14:26:42 UTC` | | `{{ baibot_conversation_start_time_utc }}` | The date and time in UTC that the conversation started | `2024-09-20 (Friday), 14:26:42 UTC` | 💡 `{{ baibot_now_utc }}` changes as time goes on, which prevents [prompt caching](https://platform.openai.com/docs/guides/prompt-caching) from working. It's better to use `{{ baibot_conversation_start_time_utc }}` in prompts, as its value doesn't change yet still orients the bot to the current date/time. Here's a prompt that combines some of the above variables: > You are a brief, but helpful bot called {{ baibot_name }} powered by the {{ baibot_model_id }} model. The date/time of this conversation's start is: {{ baibot_conversation_start_time_utc }}." ### 🌡️ Temperature Override You can override the [temperature](https://blogs.novita.ai/what-are-large-language-model-settings-temperature-top-p-and-max-tokens/#what-is-llm-temperature) (randomness / creativity) parameter configured at the [🤖 agent](../agents.md) level. Example: `!bai config room text-generation set-temperature-override 3.5` (this can also be set globally, see [🛠️ Room Settings](./README.md#room-settings)) ================================================ FILE: docs/configuration/text-to-speech.md ================================================ ## 🗣️ Text-to-Speech Below are some configuration settings related to Text-to-Speech. You may also wish to see: - [🌟 Features / 🗣️ Text-to-Speech](../features.md#-text-generation) for a higher-level introduction to the Text-to-Speech features - [📖 Usage / 🗣️ Text-to-Speech](../usage.md#-text-generation) section for more details on how to use the bot for Text-to-Speech in a room ### 🪄 Bot Messages Flow Type Controls how automatic text-to-speech functions for **messages sent by the bot**. The following configuration values are recognized: - (default) `on_demand_for_voice`: the bot will turn its own text messages into audio (voice) messages only after an allowed [👥 user](../access.md#-users) **reacts** to a bot's message with 🗣️. To make it easier for users to react without having to hunt for this emoji, the bot will automatically add a 🗣️ reaction to its own messages which are in response to a user audio (voice) message. - `on_demand_always`: the bot will turn its own text messages into audio (voice) messages only after an allowed [👥 user](../access.md#-users) **reacts** to a bot's message with 🗣️. To make it easier for users to react without having to hunt for this emoji, the bot will automatically add a 🗣️ reaction to **all of its own messages**. - `only_for_voice`: the bot will turn its own text messages into audio (voice) messages only if the original user message was a voice message. This is to allow for [Seamless voice interaction](../features.md#seamless-voice-interaction), where you can speak to the bot and then hear its responses - `never`: the bot will never turn its own text messags into audio (voice) messages - `always`: the bot will turn all its text messages into audio (voice) messages. This also allows for [Seamless voice interaction](../features.md#seamless-voice-interaction). Example: `!bai config room text-to-speech set-bot-msgs-flow-type never` (this can also be set globally, see [🛠️ Room Settings](./README.md#room-settings)) ### 🪄 User Messages Flow Type Controls how automatic text-to-speech functions for **messages sent by [👥 users](../access.md#-users)**. **Only works when automatic text-generation is disabled** (see [💬 Text Generation / 🪄 Auto Usage](./text-generation.md#-auto-usage)). The following configuration values are recognized: - (default) `never`: the bot will never turn [👥 user](../access.md#-users) text messages into audio (voice) messages - `on_demand`: the bot will turn [👥 user](../access.md#-users) text messages into audio (voice) messages if the text message receives a 🗣️ reaction - `always`: the bot will turn all [👥 user](../access.md#-users) text messages into audio (voice) messages. This is to allow for [Text-to-Speech-only mode](../features.md/#text-to-speech-only-mode). Example: `!bai config room text-to-speech set-user-msgs-flow-type always` (this can also be set globally, see [🛠️ Room Settings](./README.md#room-settings)) ### 🗲 Speed override The speed override setting lets you speed up/down speech relative to the default speed configured at the [🤖 agent](../agents.md) level (usually `1.0`). Values typically range from `0.25` to `4.0`, but may vary depending on the selected model. Example: `!bai config room text-to-speech set-speed-override 1.5` (this can also be set globally, see [🛠️ Room Settings](./README.md#room-settings)) ### 👫 Voice override The voice override setting lets you change the voice being used by the text-to-speech model configured at the [🤖 agent](../agents.md) level (usually `onyx` when using [OpenAI](../providers.md#openai)). Possible values (e.g. `onyx`) depend on the model you're using. For example, for [OpenAI](../providers.md#openai)'s Whisper model, [these voices](https://platform.openai.com/docs/guides/text-to-speech/voice-options) are available. Example: `!bai config room text-to-speech set-voice-override nova` (this can also be set globally, see [🛠️ Room Settings](./README.md#room-settings)) ================================================ FILE: docs/development.md ================================================ ## 🧑‍💻 Development This documentation page contains information about **running the bot locally for development purposes**. This can also **helpful for quickly testing the bot in a containerized environment, with all dependency services included**. For running the bot against your Matrix server, see the [🚀 Installation](./installation.md) documentation. This bot is built in [🦀 Rust](https://www.rust-lang.org/) and uses the [mxlink](https://github.com/etkecc/rust-mxlink) library (built on top of [matrix-rust-sdk](https://github.com/matrix-org/matrix-rust-sdk)). For local development, we run all dependency services in [🐋 Docker](https://www.docker.com/) containers via [docker-compose](https://docs.docker.com/compose/). ### Prerequisites - [🐋 Docker](https://www.docker.com/) and [docker-compose](https://docs.docker.com/compose/) - [Just](https://github.com/casey/just) - (Optional) [🦀 Rust](https://www.rust-lang.org/) - for compiling and running outside of a container - (Optional) an API key for some Large Language Model [☁️ provider](./providers.md) (e.g. [OpenAI](./providers.md#openai)), though we recommend using [LocalAI](#localai) or [Ollama](#ollama) for local development ### Choosing a homeserver The development environment supports two homeserver implementations: - **[Continuwuity](https://continuwuity.org/)** (default) — lightweight, no external database required. Good for most development needs. - **[Synapse](https://github.com/element-hq/synapse)** — the reference implementation, bundled with Postgres. Use this if you need Synapse-specific behavior. To choose a homeserver (optional — defaults to Continuwuity if skipped): ```sh just homeserver-init continuwuity # or: just homeserver-init synapse ``` The choice is stored in `var/homeserver` and affects all subsequent commands. > **Note:** If you switch homeservers after initial setup, you will need to: > - Delete `var/app/local/` and/or `var/app/container/` (app config and data) > - Delete `var/services/element-web/` (to regenerate its config) > - Re-run the prepare and user registration steps ### Getting started guide Developing [locally](#running-locally) is possible, but requires a [Rust](https://www.rust-lang.org/) toolchain. If this dependency is problematic for you, consider [🐋 running in a container](#running-in-a-container). In any case, you will need [🐋 Docker](https://www.docker.com/) as [dependency services](../etc/services/) run there. #### Running locally 1. (Optional) Choose a homeserver: `just homeserver-init continuwuity` (or `synapse`). Default is `continuwuity`. 2. Start the homeserver and Element Web: `just services-start` 3. (Only the first time around) Prepare initial app configuration in `var/app/local/config.yml`: `just app-local-prepare` 4. (Only the first time around) [Prepare your configuration file](#prepare-your-configuration-file) 5. (Only the first time around) Prepare initial default Matrix user accounts (`admin` and `baibot`): `just users-prepare` 6. (Optional) Start additional services depending on which [agent provider you've chosen](#choosing-an-agent-provider): - for [LocalAI](#localai): - Start services: `just localai-start` - Wait a while for LocalAI to start up. It has a lot of models to download. Monitor progress using `just localai-tail-logs` - When ready, you'll be able to reach LocalAI's web interface at http://localai.127.0.0.1.nip.io:42027/ (not that you really need it) - for [Ollama](#ollama): - Start services: `just ollama-start` - (Only the first time around) Pull the model configured in `agents.static_definitions` in the configuration file: `just ollama-pull-model gemma2:2b` 7. Start the bot: `just run-locally` 8. Go to http://element.127.0.0.1.nip.io:42025/ and login with `admin` / `admin` 9. Create a new room and invite `@baibot:continuwuity.127.0.0.1.nip.io` (or `@baibot:synapse.127.0.0.1.nip.io` if using Synapse) 10. When done, stop the bot (`Ctrl` + `C`) 11. Stop the services: `just services-stop` 12. (Optional) Stop additional services: - for [LocalAI](#localai): `just localai-stop` - for [Ollama](#ollama): `just ollama-stop` #### Running in a container You can avoid having a [Rust](https://www.rust-lang.org/) toolchain installed locally and build/run this in a container. 1. (Optional) Choose a homeserver: `just homeserver-init continuwuity` (or `synapse`). Default is `continuwuity`. 2. Start the homeserver and Element Web: `just services-start` 3. (Only the first time around) Prepare initial app configuration in `var/app/container/config.yml`: `just app-container-prepare` 4. (Only the first time around) [Prepare your configuration file](#prepare-your-configuration-file) 5. (Only the first time around) Prepare initial default Matrix user accounts (`admin` and `baibot`): `just users-prepare` 6. (Optional) Start additional services depending on which [agent provider you've chosen](#choosing-an-agent-provider): - for [LocalAI](#localai): - Start services: `just localai-start` - Wait a while for LocalAI to start up. It has a lot of models to download. Monitor progress using `just localai-tail-logs` - When ready, you'll be able to reach LocalAI's web interface at http://localai.127.0.0.1.nip.io:42027/ (not that you really need it) - for [Ollama](#ollama): - Start services: `just ollama-start` - (Only the first time around) Pull the model configured in `agents.static_definitions` in the configuration file: `just ollama-pull-model gemma2:2b` 7. Start the bot: `just run-in-container` 8. Go to http://element.127.0.0.1.nip.io:42025/ and login with `admin` / `admin` 9. Create a new room and invite `@baibot:continuwuity.127.0.0.1.nip.io` (or `@baibot:synapse.127.0.0.1.nip.io` if using Synapse) 10. When done, stop the bot (`Ctrl` + `C`) 11. Stop the services: `just services-stop` 12. (Optional) Stop additional services: - for [LocalAI](#localai): `just localai-stop` - for [Ollama](#ollama): `just ollama-stop` #### Prepare your configuration file This is about editing your configuration. The initial configuration is created based on `etc/app/config.yml.dist` when you run `just app-local-prepare` or `just app-container-prepare`. Depending on whether you run locally or in a container, your configuration lives in a different file (`var/app/local/config.yml` and `var/app/container/config.yml`, respectively). Before starting the bot, you may wish to adjust this configuration. ##### Choosing an agent provider You can create [🤖 agents](./agents.md) either [statically](./configuration/README.md#static-configuration) or [dynamically](./configuration/README.md#dynamic-configuration) using any of the supported [☁️ providers](./providers.md). For getting started most quickly (and locally), we recommend using [LocalAI](#localai) or [Ollama](#ollama). These services are already configured to run as [local services via docker-compose](../etc/services/). **Ollama is most lightweight** (~2GB for the container image + ~1.6GB for the model), but supports only [💬 text-generation](./features.md#-text-generation). **LocalAI requires 4x more disk space** (~6GB for the container image + ~12GB for the models), but supports [💬 text-generation](./features.md#-text-generation), [🗣️ text-to-speech](./features.md#️-text-to-speech), [🦻 speech-to-text](./features.md#-speech-to-text) and [🖼️ image-generation](./features.md#️-image-creation). **OpenAI supports all of these capabilities** as well and does not require powerful hardware or lots of disk space. However, it requires signup and an API key. For local testing, **we recommend LocalAI**, because it runs fully locally and supports more features than Ollama. ###### LocalAI [LocalAI](./providers.md#localai) supports all [🌟 features](./features.md) of the bot. If you decided to go with [LocalAI](./providers.md#localai): - enable the `localai` entry in the `agents.static_definitions` list in the configuration file - adjust the `initial_global_config.handler.catch_all` setting in the configuration file (`null` -> `static/localai`) By default, we configure LocalAI to use the [All-In-One images](https://localai.io/basics/container/#all-in-one-images) running on the CPU. Performance is not great, but it should work reasonably well on good hardware. If you'd like to use GPU acceleration, you may adjust the `SERVICE_LOCALAI_IMAGE_NAME` variable in [var/services/env](../var/services/env) (this file is automatically prepared for you based on [etc/services/env.dist](../etc/services/env.dist)) to use [other available LocalAI All-In-One images](https://localai.io/basics/container/#available-aio-images). ###### Ollama [Ollama](./providers.md#ollama) only supports [💬 text-generation](./features.md#-text-generation). If you decided to go with [Ollama](./providers.md#ollama): - enable the `ollama` entry in the `agents.static_definitions` list in the configuration file - adjust the `initial_global_config.handler.catch_all` setting in the configuration file (`null` -> `static/ollama`) The [gemma2:2b](https://ollama.com/library/gemma2:2b) model was chosen as a default, because it's smallest/lightest and should run well under [Ollama](./providers.md#ollama) on most machines. ###### OpenAI [OpenAI](./providers.md#openai) supports all [🌟 features](./features.md) of the bot. If you decided to go with [OpenAI](./providers.md#openai): - enable the `openai` entry in the `agents.static_definitions` list in the configuration file - adjust the `initial_global_config.handler.catch_all` setting in the configuration file (`null` -> `static/openai`) ================================================ FILE: docs/features.md ================================================ ## 🌟 Features ### 🎨 Mixing & matching models You can use **different models in different rooms** (e.g. [OpenAI](./providers.md#openai) GPT-4o alongside [Llama](https://en.wikipedia.org/wiki/Llama_(language_model)) running on [Groq](./providers.md#groq), etc.) You can also use **different models within the same room** (e.g. [💬 text-generation](#-text-generation) handled by one [🤖 agent](./agents.md), [🦻 speech-to-text](#-speech-to-text) handled by another, [🗣️ text-to-speech](#️-text-to-speech) by a 3rd, etc.) The bot supports the following use-purposes: - [💬 text-generation](#-text-generation): communicating with you via text (though certain models may also process images and files) - [🦻 speech-to-text](#-speech-to-text): turning your voice messages into text - [🗣️ text-to-speech](#%EF%B8%8F-text-to-speech): turning bot or users text messages into voice messages - [🖌️ image-generation](#%EF%B8%8F-image-generation): generating images based on instructions In a given room, each different purpose can be served by a different [☁️ provider](./providers.md) and model. This combination of provider and model configuration is called an [🤖 agent](./agents.md). Each purpose can be served by a different **handler** agent. See a [🖼️ Screenshot of an example room configuration](./screenshots/config-status-handlers.webp). For more information about configuring handlers, see the [🤝 Handlers / Configuring](./configuration/handlers.md#configuring) documentation section. ### 💬 Text Generation Text Generation is the bot's ability to **respond to users' messages with text**. ![Screenshot of Text Generation - a user sends a message and the bot replies in a new conversation thread](./screenshots/text-generation.webp) Some models also support vision and document understanding, so you may be able to mix text, images, and files (PDFs, text documents, etc.) in the same conversation. Note that certain providers may not support all file types or may have issues with specific files (e.g. scanned/image-based PDFs). If a file is rejected by the provider, the conversation thread may become unusable — start a new thread to work around this. In multi-user (group) rooms, to avoid disturbing the normal conversation between people, the bot is auto-configured to only respond to messages starting with the command prefix (`!bai`) or direct mentions via the [💬 Text Generation / 🗟 Prefix Requirement Type](./configuration/text-generation.md#-prefix-requirement-type) setting. Normally, the bot only responds to allowed [👥 Users](./access.md#-users). In certain cases, it's useful for an allowed user to provoke the bot to respond even in foreign threads or reply chains. You can learn more about this feature in the [On-demand involvement](./features.md#on-demand-involvement) section below. If needed, the bot can also attach sender metadata to conversation messages before sending them to the model, which can help the model distinguish between participants in multi-user rooms. See [🛠️ Configuration / 💬 Text Generation / 👤 Sender Context Mode](./configuration/text-generation.md#-sender-context-mode). A few other features (like [🗣️ Text-to-Speech](#️-text-to-speech) and [🦻 Speech-to-Text](#-speech-to-text)) combine well with Text Generation, so you **don't necessarily need to communicate with the bot via text** (with [Seamless voice interaction](#seamless-voice-interaction), you can communicate only with voice). You may also wish to see: - [🛠️ Configuration / 💬 Text Generation](./configuration/text-generation.md) for configuration options related to Text Generation - [📖 Usage / 💬 Text Generation](./usage.md#-text-generation) section for more details on how to use the bot for Text Generation in a room #### 🛠️ Built-in Tools (OpenAI only) The [OpenAI provider](./providers.md#openai) supports built-in tools that extend the model's capabilities: - [🔍 Web Search](https://platform.openai.com/docs/guides/tools-web-search) (`web_search`): allows the model to search the web for up-to-date information. [🖼️ Screenshot](./screenshots/text-generation-tools-web-search.webp) - [💻 Code Interpreter](https://platform.openai.com/docs/guides/tools-code-interpreter) (`code_interpreter`): allows the model to write and execute Python code in a sandbox These tools are **disabled by default** and need to be explicitly enabled in the agent's `text_generation.tools` configuration. See the [OpenAI sample configuration](https://github.com/etkecc/baibot/blob/c70387b0c38d8d0f30bba2179a2a21a3710dbeaf/docs/sample-provider-configs/openai.yml#L12-L15) for reference. To enable tools on an existing dynamically-created agent, you need to [update the agent](./agents.md#updating-agents) to re-create it with the `text_generation.tools` section added and enable the tools you need 💡 **Note**: These tools run on OpenAI's infrastructure and may incur additional costs. Web search results include citations that are incorporated into the response. #### On-demand involvement In the following 2 cases, it's useful to involve the bot in conversations on-demand: 1. In multi-user rooms (with the [🗟 Prefix Requirement](./configuration/text-generation.md#-prefix-requirement-type) setting set to "required") 2. In rooms with foreign users (users that are not authorized bot [👥 users](./access.md#-users)) In these instances, an allowed [👥 user](./access.md#-users) can also provoke the bot to respond to **any** thread or reply chain by [mentioning](https://spec.matrix.org/latest/client-server-api/#user-and-room-mentions) the bot (e.g. `@baibot Hello!`). The following screenshots demonstrate this behavior: - [🖼️ On-demand involvement in the room](./screenshots/text-generation-prefix-requirement.webp) - [🖼️ On-demand involvement in a thread](./screenshots/text-generation-on-demand-thread-involvement.webp) (the Alice user in this example is not an allowed user, yet her messages are still considered as part of the conversation context) - [🖼️ On-demand involvement in a reply chain](./screenshots/text-generation-on-demand-reply-involvement.webp) (the Alice user in this example is not an allowed user, yet her messages are still considered as part of the conversation context) 💡 **NOTE**: Normally, the bot **only considers messages from allowed [👥 Users](./access.md#-users)** and ignores all other messages when responding. However, **when the bot is explicitly invoked (via mention)** in a thread or reply chain, **it will consider all messages** in the thread and reply chain (even those from foreign users) as part of the conversation context. ### 🗣️ Text-to-Speech Text-to-Speech is the bot's ability to **turn text messages into voice messages**. It can be performed **on the bot's own text messages** (responses to yours due to [💬 Text Generation](#-text-generation)) and/or **on your own text messages**. Text-to-Speech can be enabled to be done automatically or on-demand (only after reacting to a message with 🗣️), and is configurable for different message types ([🪄 Bot Messages Flow Type](./configuration/README.md#-bot-messages-flow-type) vs [🪄 User Messages Flow Type](./configuration/README.md#-user-messages-flow-type)). By default, the bot **doesn't** perform text-to-speech. It can be configured for [Seamless voice interaction](#seamless-voice-interaction), where you can **speak to the bot** (instead of typing) and then **hear its responses**. Another use-case is to have the bot operate in [Text-to-Speech-only mode](#text-to-speech-only-mode). You may also wish to see: - [🛠️ Configuration / 🗣️ Text-to-Speech](./configuration/text-to-speech.md) for configuration options related to Text-to-Speech - [📖 Usage / 🗣️ Text-to-Speech](./usage.md#-text-to-speech) section for more details on how to use the bot for Text-to-Speech in a room #### Text-to-Speech-only mode You may wish to have the bot **automatically turn your text messages into voice messages**, but **without** doing [💬 Text Generation](#-text-generation). ![Screenshot of Text-to-Speech-only mode - text messages are turned to audio and posted as a reply, without Text Generation happening](./screenshots/text-to-speech-only-mode.webp) This could be useful in a room with others, where you'd like to post text messages and have people in the room consume them more easily (by listening to audio). To allow for this use-case, you can: - disable [💬 Text Generation](#-text-generation) (via [💬 Text Generation / 🪄 Auto Usage](./configuration/text-generation.md#-auto-usage) setting): `!bai config room text-generation set-auto-usage never` - enable [🗣️ Text-to-Speech](#️-text-to-speech) for user messages (via [🗣️ Text-to-Speech / 🪄 User Messages Flow Type](./configuration/text-to-speech.md#-user-messages-flow-type)): `!bai config room text-to-speech set-user-msgs-flow-type always` (or `on_demand`) ### 🦻 Speech-to-Text Speech-to-Text is the bot's ability to **turn voice messages into text**. ![Default flow for Speech-to-Text and Text-Generation - your voice messages are transcribed to text and then answered via Text Generation](./screenshots/speech-to-text-default-flow.webp) The default flow is shown in the screenshot above: your voice messages are transcribed to text and [💬 Text Generation](#-text-generation) is performed. By default, the bot offers [🗣️ Text-to-Speech](#️-text-to-speech) for its answers via a 🗣️ emoji. You can click it to trigger text-to-speech on-demand. You may also configure the bot for [Seamless voice interaction](#seamless-voice-interaction) or [Transcribe-only mode](#transcribe-only-mode), etc. You may also wish to see: - [🛠️ Configuration / 🦻 Speech-to-Text](./configuration/speech-to-text.md) for configuration options related to Speech-to-Text - [📖 Usage / 🦻 Speech-to-Text](./usage.md#-speech-to-text) section for more details on how to use the bot for Speech-to-Text in a room #### Seamless voice interaction The bot can perform seamless voice interaction (🗣️-to-🗣️), allowing you to **speak to the bot** (instead of typing) and then **hear its responses**. ![Screenshot of the Seamless voice interaction mode - your voice messages are transcribed to text, then answered via Text Generation, and finally the answer is turned into a voice message](./screenshots/text-to-speech-seamless-voice-interaction.webp) The flow is like this: 1. 👤 You sending a voice message 2. 🤖 The bot: - (default) first turning your **voice message into text** ([🦻 Speech-to-Text](#-speech-to-text)) and posting it as a reply. This lets you you see what the bot heard. - (default) then **answering in text** ([💬 Text Generation](#-text-generation)). This lets you read/skim text, if you so prefer. - (can be enabled) finally **turning the answer's text into a voice message** ([🗣️ Text-to-Speech](#️-text-to-speech)) 3. 👤 You continuing the conversation via text or voice messages ⚠️ Certain clients (like [Element](https://element.io/)) only support sending voice messages as top-level room messages, not as thread replies. Until this client limitation is fixed, Element users can only send the 1st message as a voice message - subsequent replies in the same conversation thread will need to be sent as text messages. By default, the last part of the aforementioned flow is **not enabled**, because we assume **a saner default is to reply with text and merely *offer* text-to-speech to those who want it**. Offering is done by the bot reacting to its own message with 🗣️, and letting you click this emoji to trigger text-to-speech on-demand. To enable automatic text-to-speech for the bot's messages, set the [🗣️ Text-to-Speech / 🪄 Bot Messages Flow Type](./configuration/text-to-speech.md#-bot-messages-flow-type) setting to `only_for_voice` or `always` (e.g. `!bai config room text-to-speech set-bot-msgs-flow-type only_for_voice`). #### Transcribe-only mode If you'd like to have the bot **only turn voice messages into text** (without generating text messages or voice messages), you can configure the bot for that. ![Screenshot of Transcribe-only-mode for Speech-to-Text - your voice messages are transcribed to text, and the bot does not generate text messages or voice messages](./screenshots/speech-to-text-transcribe-only-mode.webp) To operate in this mode, you can: - disable [💬 Text Generation](#-text-generation) (via [💬 Text Generation / 🪄 Auto Usage](./configuration/text-generation.md#-auto-usage) setting): `!bai config room text-generation set-auto-usage never` - adjust the [🦻 Speech-to-Text / 🪄 Flow Type](./configuration/speech-to-text.md#-flow-type) setting to make the bot only transcribe (without doing [💬 Text Generation](#-text-generation)): `!bai config room speech-to-text set-flow-type only_transcribe` - optionally adjust [🦻 Speech-to-Text / 🪄 Message Type for non-threaded only-transcribed messages](./configuration/speech-to-text.md#-message-type-for-non-threaded-only-transcribed-messages), if you'd like to bot to send messages of type `notice` (for better compatibility with other bots in the room) instead of sending regular `text` messages (default) ### Image Generation #### 🖌️ Image Creation Image creation is the bot's ability to **create images** based on text prompts. See a [🖼️ Screenshot of the Image Creation feature](./screenshots/image-creation.webp). You may also wish to see: - [🛠️ Configuration / 🖌️ Image Generation](./configuration/image-generation.md) for configuration options related to Image Generation - [📖 Usage / Image Generation / 🖌️ Creating Images](./usage.md#-creating-images) section for more details on how to use the bot for Image Creation in a room - [🖌️ Image Editing](#️-image-editing) - another image generation feature - [🫵 Sticker Creation](#-sticker-creation) - a special case of Image Creation #### 🎨 Image Editing Image editing is the bot's ability to **edit images** based on a prompt and one or more existing images. See a [🖼️ Screenshot of the Image Editing feature (manipulating a single image)](./screenshots/image-editing-single-image.webp) and a [🖼️ Screenshot of the Image Editing feature (manipulating multiple images)](./screenshots/image-editing-multiple-images.webp). You may also wish to see: - [🛠️ Configuration / 🖌️ Image Generation](./configuration/image-generation.md) for configuration options related to Image Generation - [📖 Usage / Image Generation / 🎨 Editing images](./usage.md#-editing-images) section for more details on how to use the bot for Image Editing in a room - [🖌️ Image Creation](#️-image-creation) - another image generation feature #### 🫵 Sticker Creation Sticker generation is the bot's ability to **generate sticker** images based on text prompts. It's a special case of [🖌️ Image Creation](#️-image-creation). See a [🖼️ Screenshot of the Sticker Creation feature](./screenshots/sticker-generation.webp). See [📖 Usage / Image Generation / 🫵 Creating Stickers](./usage.md#-creating-stickers) for details. ### 🔒 Encryption #### Message exchange The bot works in both **unencrypted and encrypted Matrix rooms**. If configured, the bot can make use of **Matrix's Secure Storage (Recovery) feature**, so that it can restore its encryption keys even its local database gets lost. #### Configuration The bot also stores its [🛠️ configuration](./configuration/README.md) (both 📍 per-room and 🌐globally) in Matrix Account Data, which is **generally stored as plain-text in the server**. To overcome this Matrix limitation, the bot can **optionally encrypt the configuration data** before storing it in Account Data. This allows for the bot to be used securely even against untrusted servers, without leaking sensitive configuration data to them. ================================================ FILE: docs/installation.md ================================================ ## 🚀 Installation ☁️ The easiest way to use the bot is to **get a managed Matrix server from [etke.cc](https://etke.cc/)** and order baibot via the [order form](https://etke.cc/order/). Existing customers can request the inclusion of this additional service by [contacting support](https://etke.cc/contacts/). 💻 If you're managing your Matrix server with the help of the [matrix-docker-ansible-deploy](https://github.com/spantaleev/matrix-docker-ansible-deploy) Ansible playbook, you can easily **install the bot via the Ansible playbook**. See the playbook's [Setting up baibot](https://github.com/spantaleev/matrix-docker-ansible-deploy/blob/master/docs/configuring-playbook-bot-baibot.md) documentation page. 🐋 In other cases, we **recommend using our [prebuilt container images](https://github.com/etkecc/baibot/pkgs/container/baibot) and [running in a container](#-running-in-a-container)**. You can also [build a container image](#building-a-container-image) yourself. 🔨 If containers are not your thing, you can [build a binary](#-building-a-binary) yourself and [run it](#-running-a-binary). 🗲 For a quick experiment, you can refer to the [🧑‍💻 development documentation](./development.md) which contains information on how to build and run the bot (and its various dependency services) locally. ### 🐋 Building a container image We provide prebuilt container images for the `amd64` and `arm64` architectures, so **you don't necessarily need to build images yourself** and can jump to [Running in a container](#-running-in-a-container). If you nevertheless wish to build a container image yourself, you can do so by running: - (recommended) `just build-container-image-release` to build a release version of the container image - or `just build-container-image-debug` to build a debug version of the container image Debug images are faster to build but are larger in size. Release images are ~5x smaller in size, but are slower to build. Both of these commands will build and tag your container image as `localhost/baibot:latest`. ### 🐋 Running in a container We recommend using a **tagged-release** (e.g. `v1.0.0`, not `latest`) of our [prebuilt container images](https://github.com/etkecc/baibot/pkgs/container/baibot), but you can also [build a container image](#-building-a-container-image) yourself. You should: - [🛠️ prepare a configuration file](#-preparing-a-configuration-file) (e.g. `cp etc/app/config.yml.dist /path/to/config.yml` & edit it) - prepare a data directory (`mkdir /path/to/data`) The example below uses [🐋 Docker](https://www.docker.com/) to run the container, but other container runtimes like [Podman](https://podman.io/) should work as well. ```sh # Adjust the version tag to point to the latest available tagged version. # If building your own container image name, adjust to something like `localhost/baibot:latest`. CONTAINER_IMAGE_NAME=ghcr.io/etkecc/baibot:v1.0.0 /usr/bin/env docker run \ -it \ --rm \ --name=baibot \ --user=$(id -u):$(id -g) \ --cap-drop=ALL \ --read-only \ --env BAIBOT_PERSISTENCE_DATA_DIR_PATH=/data \ --mount type=bind,src=/path/to/config.yml,dst=/app/config.yml,ro \ --mount type=bind,src=/path/to/data,dst=/data \ --tmpfs=/tmp:rw,noexec,nosuid,size=1024m \ $CONTAINER_IMAGE_NAME ``` 💡 If you've defined the `persistence.data_dir_path` setting in the `config.yml` file, you can skip the `BAIBOT_PERSISTENCE_DATA_DIR_PATH` environment variable. ### 🔨 Building a binary To build a binary, you need a [🦀 Rust](https://www.rust-lang.org/) toolchain. Consult the [Dockerfile](../Dockerfile) file to learn what some of the build dependencies are (e.g. `libssl-dev`, `libsqlite3-dev`, etc., on Debian-based distros). You can build a binary from the current project's source code: - in `debug` mode via: `just build-debug`, yielding a binary in `target/debug/baibot` - (recommended) in `release` mode via: `just build-release`, yielding a binary in `target/release/baibot` 💡 Unless you're [🧑‍💻 developing](./development.md), you probably wish to build in release mode, as that provides a much smaller and more optimized binary. 📦 You can also install from the [baibot](https://crates.io/crates/baibot) crate published to [crates.io](https://crates.io) with the help of the [cargo](https://doc.rust-lang.org/cargo/) package manager by running: `cargo install baibot`. ### 🖥️ Running a binary Once you've [🔨 built a binary](#-building-a-binary) and [🛠️ prepared a configuration file](#-preparing-a-configuration-file), you can run it. Consult the [Dockerfile](../Dockerfile) file to learn what some of the runtime dependencies are (e.g. `ca-certificates`, `sqlite3`, etc., on Debian-based distros). You can run the binary like this: ```sh BAIBOT_CONFIG_FILE_PATH=/path/to/config.yml \ BAIBOT_PERSISTENCE_DATA_DIR_PATH=/path/to/data \ ./target/release/baibot ``` 💡 If you've defined the `persistence.data_dir_path` setting in the `config.yml` file, you can skip the `BAIBOT_PERSISTENCE_DATA_DIR_PATH` environment variable. 💡 If your `config.yml` file is in your working directory (which may be different than the directory the binary lives in), you can skip the `BAIBOT_CONFIG_FILE_PATH` environment variable. ### 🛠️ Preparing a configuration file For an introduction to the configuration file, see the [🛠️ Configuration](./configuration/README.md) page. Generally, you need to copy the configuration file template ([etc/app/config.yml.dist](../etc/app/config.yml.dist)) and make modifications as needed. ================================================ FILE: docs/providers.md ================================================ ## ☁️ Providers [🤖 Agents](./agents.md) are powered by a provider. The provider could be a **local service** or a **cloud service**. The list of supported providers is below. ### Table of contents - [How to choose a provider](#how-to-choose-a-provider) - [How to use a provider](#how-to-use-a-provider) - [Supported providers](#supported-providers) - [Anthropic](#anthropic) - [Groq](#groq) - [LocalAI](#localai) - [Mistral](#mistral) - [Ollama](#ollama) - [OpenAI](#openai) - [OpenAI Compatible](#openai-compatible) - [OpenRouter](#openrouter) - [Together AI](#together-ai) ### How to choose a provider If you're not sure which provider to start with, **we recommend [OpenAI](#openai)** as it's the most popular and has the **widest range of capabilities**: [💬 text-generation](./features.md#-text-generation) (incl. vision, incl. [🛠️ tools](./features.md#️-built-in-tools-openai-only)), [🖌️ image-generation](./features.md#️image-generation), [🦻 speech-to-text](./features.md#-speech-to-text), [🗣️ text-to-speech](./features.md#️-text-to-speech). You don't need to choose just one though. The bot supports [mixing & matching models](./features.md#-mixing--matching-models), so you can use multiple providers at the same time. ### How to use a provider 1. 📝 **Sign up for it** 2. 🔑 **Obtain an API key** 3. 🤖 **Create one or more agents** in a given room or globally. Next to each provider in the [list below](#supported-providers) you'll see **🗲 Quick start** commands, but you may also refer to the [agent creation guide](./agents.md#creating-agents). 4. 🤝 **Set the new agent as a handler** for a given use-purpose like text-generation, image-generation, etc. The agent creation wizard will tell you how, but you may also refer to the [🤝 Handlers](./configuration/handlers.md) guide. ### Supported providers ### Anthropic [Anthropic](https://www.anthropic.com/) is an American AI company founded by former OpenAI engineers and providing powerful language models. - 🆔 Identifier: `anthropic` - 🔗 Links: [🏠 Home page](https://www.anthropic.com/), [🌐 Wiki](https://en.wikipedia.org/wiki/Anthropic), [👤 Sign up](https://console.anthropic.com/), [📋 Models list](https://docs.anthropic.com/en/docs/about-claude/models) - 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation) (incl. vision, no tools) - 🗲 Quick start: - create a room-local agent: `!bai agent create-room-local anthropic my-anthropic-agent` - create a global agent: `!bai agent create-global anthropic my-anthropic-agent` 💡 When creating an agent, the bot will show you an up-to-date sample configuration for this provider which looks [like this](./sample-provider-configs/anthropic.yml). ### Groq [Groq](https://groq.com/) is an American company developing optimized Language Processing Units (LPU) and offering cloud service which runs various models (built by others) with very high performance. - 🆔 Identifier: `groq` - 🔗 Links: [🏠 Home page](https://groq.com/), [🌐 Wiki](https://en.wikipedia.org/wiki/Groq), [👤 Sign up](https://console.groq.com/login), [📋 Models list](https://console.groq.com/docs/models) - 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation) (no vision, no tools), [🦻 speech-to-text](./features.md#-speech-to-text) - 🗲 Quick start: - create a room-local agent: `!bai agent create-room-local groq my-groq-agent` - create a global agent: `!bai agent create-global groq my-groq-agent` 💡 When creating an agent, the bot will show you an up-to-date sample configuration for this provider which looks [like this](./sample-provider-configs/groq.yml). ### LocalAI [LocalAI](https://localai.io/) is the free, Open Source OpenAI alternative. LocalAI act as a drop-in replacement REST API that’s compatible with OpenAI API specifications for local inferencing. It allows you to run LLMs, generate images, audio (and not only) locally or on-prem with consumer grade hardware, supporting multiple model families and architectures. - 🆔 Identifier: `localai` - 🔗 Links: [🏠 Home page](https://localai.io/), [📋 Models list](https://localai.io/gallery.html) - 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation) (no vision, no tools), [🗣️ text-to-speech](./features.md#️-text-to-speech), [🦻 speech-to-text](./features.md#-speech-to-text) - 🗲 Quick start: - create a room-local agent: `!bai agent create-room-local localai my-localai-agent` - create a global agent: `!bai agent create-global localai my-localai-agent` 💡 When creating an agent, the bot will show you an up-to-date sample configuration for this provider which looks [like this](./sample-provider-configs/localai.yml). ### Mistral [Mistral AI](https://mistral.ai/) is a research lab based in Europe (France) which produces their own language models. - 🆔 Identifier: `mistral` - 🔗 Links: [🏠 Home page](https://mistral.ai/), [🌐 Wiki](https://en.wikipedia.org/wiki/Mistral_AI), [👤 Sign up](https://auth.mistral.ai/ui/registration), [📋 Models list](https://docs.mistral.ai/getting-started/models/) - 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation) (no vision, no tools) - 🗲 Quick start: - create a room-local agent: `!bai agent create-room-local mistral my-mistral-agent` - create a global agent: `!bai agent create-global mistral my-mistral-agent` 💡 When creating an agent, the bot will show you an up-to-date sample configuration for this provider which looks [like this](./sample-provider-configs/mistral.yml). ### Ollama [Ollama](https://ollama.com/) lets you run various models in a [self-hosted](https://github.com/ollama/ollama?tab=readme-ov-file#ollama) way. This is more advanced and requires powerful hardware for running some of the better models, but ensures your data stays with you. - 🆔 Identifier: `ollama` - 🔗 Links: [🏠 Home page](https://ollama.com/), [📋 Models list](https://ollama.com/library) - 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation) (no vision, no tools) - 🗲 Quick start: - create a room-local agent: `!bai agent create-room-local ollama my-ollama-agent` - create a global agent: `!bai agent create-global ollama my-ollama-agent` 💡 When creating an agent, the bot will show you an up-to-date sample configuration for this provider which looks [like this](./sample-provider-configs/ollama.yml). ### OpenAI [OpenAI](https://openai.com/) is an American AI company providing powerful language models. Use this provider either with the OpenAI API or with other OpenAI-compatible API services which **fully** adhere to the [OpenAI API spec](https://github.com/openai/openai-openapi/). For services which are not fully compatible with the OpenAI API, consider using the [OpenAI Compatible](#openai-compatible) provider. - 🆔 Identifier: `openai` - 🔗 Links: [🏠 Home page](https://openai.com/), [🌐 Wiki](https://en.wikipedia.org/wiki/OpenAI), [👤 Sign up](https://platform.openai.com/signup), [📋 Models list](https://platform.openai.com/docs/models) - 🌟 Capabilities: [🖌️ image-generation](./features.md#️-image-creation), [💬 text-generation](./features.md#-text-generation) (incl. vision, incl. [🛠️ tools](./features.md#️-built-in-tools-openai-only)), [🗣️ text-to-speech](./features.md#️-text-to-speech), [🦻 speech-to-text](./features.md#-speech-to-text) - 🗲 Quick start: - create a room-local agent: `!bai agent create-room-local openai my-openai-agent` - create a global agent: `!bai agent create-global openai my-openai-agent` 💡 When creating an agent, the bot will show you an up-to-date sample configuration for this provider which looks [like this](./sample-provider-configs/openai.yml). ### OpenAI Compatible This provider allows you to use OpenAI-compatible API services like [OpenRouter](https://openrouter.ai/), [Together AI](https://www.together.ai/), etc. Some of these popular services already have **shortcut** providers (leading to this one behind the scenes) - this make it easier to get started. This provider is just as featureful as the [OpenAI](#openai) provider, but is more compatible with services which do not fully adhere to the [OpenAI API spec](https://github.com/openai/openai-openapi/). - 🆔 Identifier: `openai-compatible` - 🌟 Capabilities: [🖌️ image-generation](./features.md#️-image-creation), [💬 text-generation](./features.md#-text-generation) (no vision, no tools), [🗣️ text-to-speech](./features.md#️-text-to-speech), [🦻 speech-to-text](./features.md#-speech-to-text) - 🗲 Quick start: - create a room-local agent: `!bai agent create-room-local openai-compatible my-openai-compatible-agent` - create a global agent: `!bai agent create-global openai-compatible my-openai-compatible-agent` 💡 When creating an agent, the bot will show you an up-to-date sample configuration for this provider which looks [like this](./sample-provider-configs/openai-compatible.yml). ### OpenRouter [OpenRouter](https://openrouter.ai/) is a unified interface for LLMs. The platform scouts for the lowest prices and best latencies/throughputs across dozens of providers, and lets you choose how to [prioritize](https://openrouter.ai/docs/provider-routing) them. - 🆔 Identifier: `openrouter` - 🔗 Links: [🏠 Home page](https://openrouter.ai/), [👤 Sign up](https://openrouter.ai/), [📋 Models list](https://openrouter.ai/models) - 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation) (no vision, no tools) - 🗲 Quick start: - create a room-local agent: `!bai agent create-room-local openrouter my-openrouter-agent` - create a global agent: `!bai agent create-global openrouter my-openrouter-agent` 💡 When creating an agent, the bot will show you an up-to-date sample configuration for this provider which looks [like this](./sample-provider-configs/openrouter.yml). ### Together AI [Together AI](https://www.together.ai/) makes it easy to run or [fine-tune](https://docs.together.ai/docs/fine-tuning-overview) leading open source models with only a few lines of code. - 🆔 Identifier: `together-ai` - 🔗 Links: [🏠 Home page](https://www.together.ai/), [👤 Sign up](https://api.together.ai/signup), [📋 Models list](https://api.together.xyz/models) - 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation) (no vision, no tools) - 🗲 Quick start: - create a room-local agent: `!bai agent create-room-local together-ai my-together-ai-agent` - create a global agent: `!bai agent create-global together-ai my-together-ai-agent` 💡 When creating an agent, the bot will show you an up-to-date sample configuration for this provider which looks [like this](./sample-provider-configs/together-ai.yml). ================================================ FILE: docs/sample-provider-configs/anthropic.yml ================================================ base_url: https://api.anthropic.com/v1 api_key: YOUR_API_KEY_HERE text_generation: model_id: claude-3-7-sonnet-20250219 prompt: "You are a brief, but helpful bot called {{ baibot_name }} powered by the {{ baibot_model_id }} model. The date/time of this conversation's start is: {{ baibot_conversation_start_time_utc }}." temperature: 1.0 max_response_tokens: 8192 max_context_tokens: 204800 ================================================ FILE: docs/sample-provider-configs/groq.yml ================================================ base_url: https://api.groq.com/openai/v1 api_key: YOUR_API_KEY_HERE text_generation: model_id: llama3-70b-8192 prompt: "You are a brief, but helpful bot called {{ baibot_name }} powered by the {{ baibot_model_id }} model. The date/time of this conversation's start is: {{ baibot_conversation_start_time_utc }}." temperature: 1.0 max_response_tokens: 4096 max_context_tokens: 131072 speech_to_text: model_id: whisper-large-v3 ================================================ FILE: docs/sample-provider-configs/localai.yml ================================================ base_url: http://my-localai-self-hosted-service:8080/v1 api_key: YOUR_API_KEY_HERE text_generation: model_id: gpt-4 prompt: "You are a brief, but helpful bot called {{ baibot_name }} powered by the {{ baibot_model_id }} model. The date/time of this conversation's start is: {{ baibot_conversation_start_time_utc }}." temperature: 1.0 max_response_tokens: 4096 max_context_tokens: 128000 speech_to_text: model_id: whisper-1 text_to_speech: model_id: tts-1 voice: onyx speed: 1.0 response_format: opus image_generation: model_id: stablediffusion style: vivid size: 1024x1024 quality: standard ================================================ FILE: docs/sample-provider-configs/mistral.yml ================================================ base_url: https://api.mistral.ai/v1 api_key: YOUR_API_KEY_HERE text_generation: model_id: mistral-large-latest prompt: "You are a brief, but helpful bot called {{ baibot_name }} powered by the {{ baibot_model_id }} model. The date/time of this conversation's start is: {{ baibot_conversation_start_time_utc }}." temperature: 1.0 max_response_tokens: 4096 max_context_tokens: 128000 ================================================ FILE: docs/sample-provider-configs/ollama.yml ================================================ base_url: http://my-ollama-self-hosted-service:11434/v1 api_key: YOUR_API_KEY_HERE text_generation: model_id: gemma2:2b prompt: "You are a brief, but helpful bot called {{ baibot_name }} powered by the {{ baibot_model_id }} model. The date/time of this conversation's start is: {{ baibot_conversation_start_time_utc }}." temperature: 1.0 max_response_tokens: 4096 max_context_tokens: 128000 ================================================ FILE: docs/sample-provider-configs/openai-compatible.yml ================================================ base_url: '' api_key: YOUR_API_KEY_HERE text_generation: model_id: some-model prompt: "You are a brief, but helpful bot called {{ baibot_name }} powered by the {{ baibot_model_id }} model. The date/time of this conversation's start is: {{ baibot_conversation_start_time_utc }}." temperature: 1.0 max_response_tokens: 4096 max_context_tokens: 128000 speech_to_text: model_id: whisper-1 ================================================ FILE: docs/sample-provider-configs/openai.yml ================================================ base_url: https://api.openai.com/v1 api_key: YOUR_API_KEY_HERE text_generation: model_id: gpt-5.4 prompt: "You are a brief, but helpful bot called {{ baibot_name }} powered by the {{ baibot_model_id }} model. The date/time of this conversation's start is: {{ baibot_conversation_start_time_utc }}." temperature: 1.0 # Reasoning models need to use `max_completion_tokens` instead of `max_response_tokens`. # If you're dealing with a non-reasoning model, specify `max_response_tokens` and unset `max_completion_tokens`. max_response_tokens: null max_completion_tokens: 128000 max_context_tokens: 400000 # Built-in tools tools: web_search: false code_interpreter: false speech_to_text: model_id: whisper-1 text_to_speech: model_id: tts-1-hd voice: onyx speed: 1.0 response_format: opus image_generation: model_id: gpt-image-1.5 style: null size: null quality: null ================================================ FILE: docs/sample-provider-configs/openrouter.yml ================================================ base_url: https://openrouter.ai/api/v1 api_key: YOUR_API_KEY_HERE text_generation: model_id: mattshumer/reflection-70b:free prompt: "You are a brief, but helpful bot called {{ baibot_name }} powered by the {{ baibot_model_id }} model. The date/time of this conversation's start is: {{ baibot_conversation_start_time_utc }}." temperature: 1.0 max_response_tokens: 2048 max_context_tokens: 8192 ================================================ FILE: docs/sample-provider-configs/together-ai.yml ================================================ base_url: https://api.together.xyz/v1 api_key: YOUR_API_KEY_HERE text_generation: model_id: meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo prompt: "You are a brief, but helpful bot called {{ baibot_name }} powered by the {{ baibot_model_id }} model. The date/time of this conversation's start is: {{ baibot_conversation_start_time_utc }}." temperature: 1.0 max_response_tokens: 2048 max_context_tokens: 8192 ================================================ FILE: docs/usage.md ================================================ ## 📖 Usage This document covers how to use the bot in a room. The [🌟 Features](./features.md) page also includes details about how each feature works and can be configured. ### 💬 Text Generation This is related to the [💬 Text Generation](./features.md#-text-generation) feature. If there's a text-generation handler agent configured, the bot **may** respond to messages sent in the room. Some models also support vision and document understanding, so you may be able to mix text, images, and files (PDFs, text documents, etc.) in the same conversation. See screenshots of: - 🖼️ [the default Text Generation flow](./screenshots/text-generation.webp) in 1:1 rooms - 🖼️ [the Text Generation flow in multi-user rooms](./screenshots/text-generation-prefix-requirement.webp) (where the [🗟 Prefix Requirement](./configuration/text-generation.md#-prefix-requirement-type) setting is auto-configured to "required") - the [on-demand involvement](./features.md#on-demand-involvement) feature Whether the bot responds depends on: - ([🔒 access](./access.md)) whether you're a whitelisted bot [👥 user](./access.md#-users) - [🛠️ configuration](./configuration/README.md) whether there's a configured `text-generation` handler agent (or a `catch-all` handler agent). See [Mixing & matching models](./features.md#-mixing--matching-models) - (🎨 agent capabilities) whether the configured `text-generation` (or `catch-all`) handler agent actually supports text-generation. The provider may lack support for this feature or it may be disabled in the [🤖 agents](./agents.md) configuration - (the [🗟 Prefix Requirement](./configuration/text-generation.md#-prefix-requirement-type) setting) whether a prefix (e.g. `!bai`) or user mention (e.g. `@baibot`) is required for messages sent to the room. For multi-user rooms, this setting defaults to "required". See [🌟 Features / 💬 Text Generation / On-demand involvement](./features.md#on-demand-involvement) for details. Room messages start a threaded conversation where you can continue back-and-forth communication with the bot. Using [on-demand involvement](./features.md#on-demand-involvement), you can can also mention the bot to provoke it to get involved in any conversation thread or reply chain. Unless you've enabled the [♻️ Context Management](./features.md#️-context-management) feature, all messages will be sent to the agent's API each time. If the context management feature is enabled, older messages may be dropped. ### 🗣️ Text-to-Speech This is related to the [🗣️ Text-to-Speech](./features.md#️-text-to-speech) feature. If there's a text-to-speech handler agent configured, the bot **may** convert text messages sent to the room to audio (voice). See: - a [🖼️ screenshot](./screenshots/text-to-speech-only-mode.webp) of the bot's [Text-to-Speech-only](./features.md#text-to-speech-only-mode) mode - a [🖼️ screenshot](./screenshots/text-to-speech-seamless-voice-interaction.webp) of the bot's [Seamless voice interaction](./features.md#seamless-voice-interaction) mode By default, the bot: - will offer tex-to-speech for its own messages which are a response to voice message from your, as part of the [Seamless voice interaction](./features.md#seamless-voice-interaction) feature. This can be adjusted via the [🗣️ Text-to-Speech / 🪄 Bot Messages Flow Type](./configuration/text-to-speech.md#-bot-messages-flow-type) setting. - does not turn your own text messages to audio (voice). If you'd like for the bot to operate in such a mode, use the [🗣️ Text-to-Speech / 🪄 User Messages Flow Type](./configuration/text-to-speech.md#-user-messages-flow-type) setting (see [Text-to-Speech-only mode](./features.md#text-to-speech-only-mode)). ### 🦻 Speech-to-Text This is related to the [🦻 Speech-to-Text](./features.md#-speech-to-text) feature. If there's a speech-to-text handler agent configured, the bot **may** transcribe voice messages sent to the room to text. See a [🖼️ Screenshot of the default flow for Speech-to-Text and Text-Generation](./screenshots/speech-to-text-default-flow.webp). The speech-to-text feature triggers automatically by default, but can be adjusted via the [🦻 Speech-to-Text / 🪄 Flow Type](./features.md#-speech-to-text-flow-type) setting. If all your messages are in the same language, you can improve accuracy & latency by configuring the language (see [🦻 Speech-to-Text / 🔤 Language](./configuration/speech-to-text.md#-language)). ### Image Generation This feature is not configurable at the moment. The configuration (size, quality, style) specified at the [🤖 agent](./agents.md) level will be used. Capabilities depend on the [☁️ provider](./providers.md) and model used. #### 🖌️ Creating images Simply send a command like `!bai image create A beautiful sunset over the ocean` and the bot will start a threaded conversation and post an image based on your prompt. See a [🖼️ Screenshot of the Image Creation feature](./screenshots/image-creation.webp). You can then respond in the same message thread with: - more messages, to add more criteria to your prompt. - a message saying `again`, to generate one more image with the current prompt. #### 🎨 Editing images Simply send a command like `!bai image edit Turn the following image into an anime-style drawing` and the bot will start a threaded conversation asking for more details. See a [🖼️ Screenshot of the Image Editing feature (manipulating a single image)](./screenshots/image-editing-single-image.webp) and a [🖼️ Screenshot of the Image Editing feature (manipulating multiple images)](./screenshots/image-editing-multiple-images.webp). You can then respond in the same message thread with: - more messages, to add more criteria to your prompt. - one or more images, to provide the images that the bot will operate on. - a message saying `go`, to start the image generation process. - a message saying `again`, to prompt the bot to generate one more image edit with the current prompt. #### 🫵 Creating stickers A variation of [creating images](#creating-images) is creating "sticker images". See a [🖼️ Screenshot of the Sticker Creation feature](./screenshots/sticker-generation.webp). To create a sticker, send a command like `!bai sticker A huge ramen bowl with lots of chashu and a mountain of beansprouts on top`. The difference from [creating images](#creating-images) is that the bot will: - generate a smaller-resolution image (currently hardcoded to `256x256`) - smaller/quicker, but still good enough for a sticker - potentially switch to a different (cheaper or otherwise more suitable) model, if available - post the image directly to the room (as a reply to your message), without starting a threaded conversation Some models (like [OpenAI](./providers.md#openai)'s [Dall-E-3](https://openai.com/index/dall-e-3/)) can only generate larger images (`1024x1024`, etc., for a higher charge), so we switching to a smaller/cheaper model (like [Dall-E-2](https://openai.com/index/dall-e-2/)) is a way to generate a sticker cheaply. ================================================ FILE: etc/app/config.yml.dist ================================================ homeserver: # The canonical homeserver domain name server_name: __HOMESERVER_SERVER_NAME__ url: __HOMESERVER_URL__ user: mxid_localpart: baibot # Authentication: set EITHER password OR access_token + device_id. # # Password-based login (traditional homeservers): password: baibot # Access token login (for Matrix Authentication Service/OIDC-enabled homeservers): # Generate a token via: mas-cli manage issue-compatibility-token [device_id] # access_token: null # device_id: null # The name the bot uses as a display name and when it refers to itself. # Leave empty to use the default (baibot). name: baibot # An optional path to an image file to be used as a custom avatar image. # - null or empty string: use the default avatar # - "keep": don't touch the avatar, keep whatever is already set # - any other value: path to a custom avatar image file avatar: null encryption: # An optional passphrase to use for backing up and recovering the bot's encryption keys. # You can use any string here. # # If set to null, the recovery module will not be used and losing your session/database (see persistence) # will mean you lose access to old messages in encrypted room. # # Changing this subsequently will also cause you to lose access to old messages in encrypted rooms. # If you really need to change this: # - Set `encryption_recovery_reset_allowed` to `true` and adjust the passphrase # - Remove your session file and database (see persistence) # - Restart the bot # - Then restore `encryption_recovery_reset_allowed` to `false` to prevent accidental resets in the future recovery_passphrase: long-and-secure-passphrase-here # An optional flag to reset the encryption recovery passphrase. recovery_reset_allowed: false # Command prefix. Leave empty to use the default (!bai). command_prefix: "!bai" room: # Whether the bot should send an introduction message after joining a room. post_join_self_introduction_enabled: true access: # Space-separated list of MXID patterns which specify who is an admin. admin_patterns: - "@admin:__HOMESERVER_SERVER_NAME__" persistence: # This is unset here, because we expect the configuration to come from an environment variable (BAIBOT_PERSISTENCE_DATA_DIR_PATH). # In your setup, you may wish to set this to a directory path. data_dir_path: null # An optional secret for encrypting the bot's session data (stored in data_dir_path). # This must be 32-bytes (64 characters when HEX-encoded). # Generate it with: `openssl rand -hex 32` # Leave null or empty to avoid using encryption. # Changing this subsequently requires that you also throw away all data stored in data_dir_path. session_encryption_key: 9701cd109ed56770687dd8410f7d7371a4390dd3feb8ed721f189a0756c40098 # An optional secret for encrypting bot configuration stored in Matrix's account data. # This must be 32-bytes (64 characters when HEX-encoded). # Generate it with: `openssl rand -hex 32` # Leave null or empty to avoid using encryption. # Changing this subsequently will make you lose your configuration. config_encryption_key: a9f1df98d288802ead20a8be2c701a627eabd31cf3d9e2aea28867ccd7a4ded7 agents: # A list of statically-defined agents. # # Below are a few common choices on popular providers, preconfigured for development purposes (see docs/development.md). # You may enable some of the ones you see below or define others. # You can also leave this list empty and only define agents dynamically (via chat). # # Uncomment one or more of these and potentially adjust their configuration (API key, etc). # Consider setting `initial_global_config.handler.*` to an agent that you enable here. static_definitions: # - id: openai # provider: openai # config: # base_url: https://api.openai.com/v1 # api_key: "" # text_generation: # model_id: gpt-5.4 # prompt: "You are a brief, but helpful bot called {{ baibot_name }} powered by the {{ baibot_model_id }} model. The date/time of this conversation's start is: {{ baibot_conversation_start_time_utc }}." # temperature: 1.0 # # Reasoning models need to use `max_completion_tokens` instead of `max_response_tokens`. # # If you're dealing with a non-reasoning model, specify `max_response_tokens` and unset `max_completion_tokens`. # max_response_tokens: null # max_completion_tokens: 128000 # max_context_tokens: 400000 # # Built-in tools # tools: # web_search: false # code_interpreter: false # speech_to_text: # model_id: whisper-1 # text_to_speech: # model_id: tts-1-hd # voice: onyx # speed: 1.0 # response_format: opus # image_generation: # model_id: gpt-image-1.5 # style: null # size: null # quality: null # # - id: localai # provider: localai # config: # base_url: http://127.0.0.1:42027/v1 # api_key: null # text_generation: # model_id: gpt-4 # prompt: "You are a brief, but helpful bot called {{ baibot_name }} powered by the {{ baibot_model_id }} model. The date/time of this conversation's start is: {{ baibot_conversation_start_time_utc }}." # temperature: 1.0 # max_response_tokens: 16384 # max_context_tokens: 128000 # speech_to_text: # model_id: whisper-1 # text_to_speech: # model_id: tts-1 # voice: onyx # speed: 1.0 # response_format: opus # image_generation: # model_id: stablediffusion # style: vivid # # Intentionally defaults to a small value to improve performance # size: 256x256 # quality: standard # # - id: ollama # provider: ollama # config: # base_url: "http://127.0.0.1:42026/v1" # api_key: null # text_generation: # model_id: "gemma2:2b" # prompt: "You are a brief, but helpful bot called {{ baibot_name }} powered by the {{ baibot_model_id }} model. The date/time of this conversation's start is: {{ baibot_conversation_start_time_utc }}." # temperature: 1.0 # max_response_tokens: 4096 # max_context_tokens: 128000 # Initial global configuration. This only affects the first run of the bot. # Configuration is later managed at runtime. initial_global_config: handler: catch_all: null text_generation: null text_to_speech: null speech_to_text: null image_generation: null # Space-separated list of MXID patterns which specify who can use the bot. # By default, we let anyone on the homeserver use the bot. user_patterns: - "@*:__HOMESERVER_SERVER_NAME__" # Controls logging. # # Sets all tracing targets (external crates) to warn, and our own logs to debug. # For even more verbose logging, one may also use trace. # # matrix_sdk_crypto may be chatty and could be added with an error level. # # Learn more here: https://stackoverflow.com/a/73735203 logging: warn,mxlink=debug,baibot=debug ================================================ FILE: etc/services/continuwuity/compose.yml ================================================ services: continuwuity: image: forgejo.ellis.link/continuwuation/continuwuity:v0.5.9 user: "${UID}:${GID}" restart: unless-stopped cap_drop: - ALL read_only: true environment: CONDUWUIT_CONFIG: /etc/continuwuity/continuwuity.toml CONDUWUIT_DATABASE_PATH: /var/lib/continuwuity ports: - "${SERVICE_CONTINUWUITY_BIND_PORT_CLIENT_API}:6167" volumes: - ../../etc/services/continuwuity/config:/etc/continuwuity:ro - ./continuwuity/data:/var/lib/continuwuity tmpfs: - /tmp:rw,noexec,nosuid,size=500m networks: default: name: ${NETWORK_NAME} external: true ================================================ FILE: etc/services/continuwuity/config/continuwuity.toml ================================================ [global] server_name = "continuwuity.127.0.0.1.nip.io" address = "0.0.0.0" port = 6167 database_path = "/var/lib/continuwuity" allow_registration = true yes_i_am_very_very_sure_i_want_an_open_registration_server_prone_to_abuse = true new_user_displayname_suffix = "" max_request_size = 20_000_000 allow_federation = false trusted_servers = ["matrix.org"] log = "info,state_res=warn,rocket=off,_=off,sled=off" ================================================ FILE: etc/services/continuwuity/register-user.sh ================================================ #!/bin/sh set -eu if [ $# -ne 3 ]; then echo "Usage: $0 " exit 1 fi ENV_FILE="$1" USERNAME="$2" PASSWORD="$3" SERVER="http://$(grep '^SERVICE_CONTINUWUITY_BIND_PORT_CLIENT_API=' "${ENV_FILE}" | cut -d= -f2)" REGISTER_URL="${SERVER}/_matrix/client/v3/register" echo "Registering user '${USERNAME}' on ${SERVER}..." SESSION_RESPONSE=$(curl -s -X POST "${REGISTER_URL}" \ -H 'Content-Type: application/json' \ -d "{\"username\": \"${USERNAME}\", \"password\": \"${PASSWORD}\"}") SESSION_ID=$(echo "${SESSION_RESPONSE}" | grep -o '"session":"[^"]*"' | head -1 | cut -d'"' -f4) if [ -z "${SESSION_ID}" ]; then echo "Error: Could not get session ID. Response: ${SESSION_RESPONSE}" exit 1 fi # Determine the required auth flow from the server response. # The first user requires m.login.registration_token (bootstrap token from logs). # Subsequent users use m.login.dummy (open registration). if echo "${SESSION_RESPONSE}" | grep -q 'm.login.registration_token'; then CONTAINER_ID=$(docker ps -q --filter name=baibot-continuwuity-continuwuity) REG_TOKEN=$(docker logs "${CONTAINER_ID}" 2>&1 | sed 's/\x1b\[[0-9;]*m//g' | grep 'using the registration token' | grep -oP 'registration token \K[A-Za-z0-9]+' | head -1) AUTH_BODY="{\"type\": \"m.login.registration_token\", \"token\": \"${REG_TOKEN}\", \"session\": \"${SESSION_ID}\"}" else AUTH_BODY="{\"type\": \"m.login.dummy\", \"session\": \"${SESSION_ID}\"}" fi RESULT=$(curl -s -X POST "${REGISTER_URL}" \ -H 'Content-Type: application/json' \ -d "{\"username\": \"${USERNAME}\", \"password\": \"${PASSWORD}\", \"auth\": ${AUTH_BODY}}") if echo "${RESULT}" | grep -q '"user_id"'; then echo "Successfully registered user: $(echo "${RESULT}" | grep -o '"user_id":"[^"]*"' | cut -d'"' -f4)" else echo "Registration failed. Response: ${RESULT}" exit 1 fi ================================================ FILE: etc/services/element-web/compose.yml ================================================ services: element-web: image: ghcr.io/element-hq/element-web:v1.12.18 user: "${UID}:${GID}" restart: unless-stopped environment: ELEMENT_WEB_PORT: 8080 ports: - "${SERVICE_ELEMENT_WEB_BIND_PORT_HTTP}:8080" volumes: - ./element-web/config.json:/app/config.json:ro tmpfs: - /var/cache/nginx:rw,mode=777 - /var/run:rw,mode=777 - /tmp/element-web-config:rw,mode=777 - /etc/nginx/conf.d:rw,mode=777 networks: default: name: ${NETWORK_NAME} external: true ================================================ FILE: etc/services/element-web/config.json.dist ================================================ { "default_hs_url": "__HOMESERVER_CLIENT_URL__", "default_is_url": "https://vector.im", "integrations_ui_url": "https://scalar.vector.im/", "integrations_rest_url": "https://scalar.vector.im/api", "bug_report_endpoint_url": "https://element.io/bugreports/submit", "enableLabs": true, "roomDirectory": { "servers": [ "matrix.org" ] } } ================================================ FILE: etc/services/env.dist ================================================ SERVICE_SYNAPSE_BIND_PORT_CLIENT_API=127.0.0.1:42020 SERVICE_SYNAPSE_BIND_PORT_FEDERATION_API=127.0.0.1:42028 SERVICE_ELEMENT_WEB_BIND_PORT_HTTP=127.0.0.1:42025 SERVICE_CONTINUWUITY_BIND_PORT_CLIENT_API=127.0.0.1:42030 SERVICE_OLLAMA_BIND_PORT_HTTP=127.0.0.1:42026 # See https://localai.io/basics/container/#all-in-one-images for the list of available images SERVICE_LOCALAI_IMAGE_NAME=docker.io/localai/localai:latest-aio-cpu SERVICE_LOCALAI_BIND_PORT_HTTP=127.0.0.1:42027 # Variables below are added later on, dynamically ================================================ FILE: etc/services/localai/compose.yml ================================================ services: localai: image: ${SERVICE_LOCALAI_IMAGE_NAME} restart: unless-stopped healthcheck: test: ["CMD", "curl", "-f", "http://localhost:8080/readyz"] interval: 1m timeout: 20m retries: 5 ports: - ${SERVICE_LOCALAI_BIND_PORT_HTTP}:8080 environment: - DEBUG=true volumes: - ./localai/models:/build/models:cached networks: default: name: ${NETWORK_NAME} external: true ================================================ FILE: etc/services/ollama/compose.yml ================================================ services: ollama: image: docker.io/ollama/ollama:0.24.0 restart: unless-stopped ports: - "${SERVICE_OLLAMA_BIND_PORT_HTTP}:11434" volumes: - ./ollama:/root/.ollama networks: default: name: ${NETWORK_NAME} external: true ================================================ FILE: etc/services/synapse/compose.yml ================================================ services: postgres: image: docker.io/postgres:18.4-alpine user: ${UID}:${GID} restart: unless-stopped environment: POSTGRES_USER: synapse POSTGRES_PASSWORD: synapse-password POSTGRES_DB: homeserver POSTGRES_INITDB_ARGS: --lc-collate C --lc-ctype C --encoding UTF8 PGDATA: /data volumes: - ./postgres:/data - /etc/passwd:/etc/passwd:ro synapse: image: ghcr.io/element-hq/synapse:v1.153.0 user: "${UID}:${GID}" restart: unless-stopped entrypoint: python command: "-m synapse.app.homeserver -c /config/homeserver.yaml" ports: - "${SERVICE_SYNAPSE_BIND_PORT_CLIENT_API}:8008" - "${SERVICE_SYNAPSE_BIND_PORT_FEDERATION_API}:8008" volumes: - ../../etc/services/synapse/config:/config:ro - ./synapse/media-store:/media-store networks: default: name: ${NETWORK_NAME} external: true ================================================ FILE: etc/services/synapse/config/homeserver.yaml ================================================ #jinja2: lstrip_blocks: "True" # Configuration file for Synapse. # # This is a YAML file: see [1] for a quick introduction. Note in particular # that *indentation is important*: all the elements of a list or dictionary # should have the same indentation. # # [1] https://docs.ansible.com/ansible/latest/reference_appendices/YAMLSyntax.html ## Modules ## # Server admins can expand Synapse's functionality with external modules. # # See https://matrix-org.github.io/synapse/latest/modules/index.html for more # documentation on how to configure or create custom modules for Synapse. # #modules: #- module: my_super_module.MySuperClass # config: # do_thing: true #- module: my_other_super_module.SomeClass # config: {} modules: [] ## Server ## # The domain name of the server, with optional explicit port. # This is used by remote servers to connect to this server, # e.g. matrix.org, localhost:8080, etc. # This is also the last part of your UserID. # server_name: "synapse.127.0.0.1.nip.io" # When running as a daemon, the file to store the pid in # pid_file: /homeserver.pid # The path to the web client which will be served at /_matrix/client/ # if 'webclient' is configured under the 'listeners' configuration. # #web_client_location: "/path/to/web/root" # The public-facing base URL that clients use to access this HS # (not including _matrix/...). This is the same URL a user would # enter into the 'custom HS URL' field on their client. If you # use synapse with a reverse proxy, this should be the URL to reach # synapse via the proxy. # #public_baseurl: https://example.com/ # Set the soft limit on the number of file descriptors synapse can use # Zero is used to indicate synapse should set the soft limit to the # hard limit. # #soft_file_limit: 0 # Set to false to disable presence tracking on this homeserver. # #use_presence: false # Whether to require authentication to retrieve profile data (avatars, # display names) of other users through the client API. Defaults to # 'false'. Note that profile data is also available via the federation # API, so this setting is of limited value if federation is enabled on # the server. # #require_auth_for_profile_requests: true # If set to 'false', requires authentication to access the server's public rooms # directory through the client API. Defaults to 'true'. # #allow_public_rooms_without_auth: false # If set to 'false', forbids any other homeserver to fetch the server's public # rooms directory via federation. Defaults to 'true'. # #allow_public_rooms_over_federation: false # The default room version for newly created rooms. # # Known room versions are listed here: # https://matrix.org/docs/spec/#complete-list-of-room-versions # # For example, for room version 1, default_room_version should be set # to "1". # #default_room_version: "4" # The GC threshold parameters to pass to `gc.set_threshold`, if defined # #gc_thresholds: [700, 10, 10] # Set the limit on the returned events in the timeline in the get # and sync operations. The default value is -1, means no upper limit. # #filter_timeline_limit: 5000 # Whether room invites to users on this server should be blocked # (except those sent by local server admins). The default is False. # #block_non_admin_invites: True # Room searching # # If disabled, new messages will not be indexed for searching and users # will receive errors when searching for messages. Defaults to enabled. # #enable_search: false # Restrict federation to the following whitelist of domains. # N.B. we recommend also firewalling your federation listener to limit # inbound federation traffic as early as possible, rather than relying # purely on this application-layer restriction. If not specified, the # default is to whitelist everything. # #federation_domain_whitelist: # - lon.example.com # - nyc.example.com # - syd.example.com # Prevent federation requests from being sent to the following # blacklist IP address CIDR ranges. If this option is not specified, or # specified with an empty list, no ip range blacklist will be enforced. # # (0.0.0.0 and :: are always blacklisted, whether or not they are explicitly # listed here, since they correspond to unroutable addresses.) # federation_ip_range_blacklist: - '127.0.0.0/8' - '10.0.0.0/8' - '172.16.0.0/12' - '192.168.0.0/16' - '100.64.0.0/10' - '169.254.0.0/16' - '::1/128' - 'fe80::/64' - 'fc00::/7' # List of ports that Synapse should listen on, their purpose and their # configuration. # # Options for each listener include: # # port: the TCP port to bind to # # bind_addresses: a list of local addresses to listen on. The default is # 'all local interfaces'. # # type: the type of listener. Normally 'http', but other valid options are: # 'manhole' (see docs/manhole.md), # 'metrics' (see docs/metrics-howto.rst), # 'replication' (see docs/workers.rst). # # tls: set to true to enable TLS for this listener. Will use the TLS # key/cert specified in tls_private_key_path / tls_certificate_path. # # x_forwarded: Only valid for an 'http' listener. Set to true to use the # X-Forwarded-For header as the client IP. Useful when Synapse is # behind a reverse-proxy. # # resources: Only valid for an 'http' listener. A list of resources to host # on this port. Options for each resource are: # # names: a list of names of HTTP resources. See below for a list of # valid resource names. # # compress: set to true to enable HTTP comression for this resource. # # additional_resources: Only valid for an 'http' listener. A map of # additional endpoints which should be loaded via dynamic modules. # # Valid resource names are: # # client: the client-server API (/_matrix/client), and the synapse admin # API (/_synapse/admin). Also implies 'media' and 'static'. # # consent: user consent forms (/_matrix/consent). See # docs/consent_tracking.md. # # federation: the server-server API (/_matrix/federation). Also implies # 'media', 'keys', 'openid' # # keys: the key discovery API (/_matrix/keys). # # media: the media API (/_matrix/media). # # metrics: the metrics interface. See docs/metrics-howto.rst. # # openid: OpenID authentication. # # replication: the HTTP replication API (/_synapse/replication). See # docs/workers.rst. # # static: static resources under synapse/static (/_matrix/static). (Mostly # useful for 'fallback authentication'.) # # webclient: A web client. Requires web_client_location to be set. # listeners: # TLS-enabled listener: for when matrix traffic is sent directly to synapse. # # Disabled by default. To enable it, uncomment the following. (Note that you # will also need to give Synapse a TLS key and certificate: see the TLS section # below.) # #- port: 8448 # type: http # tls: true # resources: # - names: [client, federation] # Unsecure HTTP listener: for when matrix traffic passes through a reverse proxy # that unwraps TLS. # # If you plan to use a reverse proxy, please see # https://github.com/matrix-org/synapse/blob/master/docs/reverse_proxy.rst. # - port: 8008 tls: false type: http x_forwarded: true resources: - names: [client, federation] compress: false # example additional_resources: # #additional_resources: # "/_matrix/my/custom/endpoint": # module: my_module.CustomRequestHandler # config: {} # Turn on the twisted ssh manhole service on localhost on the given # port. # #- port: 9000 # bind_addresses: ['::1', '127.0.0.1'] # type: manhole ## Homeserver blocking ## # How to reach the server admin, used in ResourceLimitError # #admin_contact: 'mailto:admin@server.com' # Global blocking # #hs_disabled: False #hs_disabled_message: 'Human readable reason for why the HS is blocked' #hs_disabled_limit_type: 'error code(str), to help clients decode reason' # Monthly Active User Blocking # # Used in cases where the admin or server owner wants to limit to the # number of monthly active users. # # 'limit_usage_by_mau' disables/enables monthly active user blocking. When # anabled and a limit is reached the server returns a 'ResourceLimitError' # with error type Codes.RESOURCE_LIMIT_EXCEEDED # # 'max_mau_value' is the hard limit of monthly active users above which # the server will start blocking user actions. # # 'mau_trial_days' is a means to add a grace period for active users. It # means that users must be active for this number of days before they # can be considered active and guards against the case where lots of users # sign up in a short space of time never to return after their initial # session. # #limit_usage_by_mau: False #max_mau_value: 50 #mau_trial_days: 2 # If enabled, the metrics for the number of monthly active users will # be populated, however no one will be limited. If limit_usage_by_mau # is true, this is implied to be true. # #mau_stats_only: False # Sometimes the server admin will want to ensure certain accounts are # never blocked by mau checking. These accounts are specified here. # #mau_limit_reserved_threepids: # - medium: 'email' # address: 'reserved_user@example.com' # Used by phonehome stats to group together related servers. #server_context: context # Whether to require a user to be in the room to add an alias to it. # Defaults to 'true'. # #require_membership_for_aliases: false # Whether to allow per-room membership profiles through the send of membership # events with profile information that differ from the target's global profile. # Defaults to 'true'. # #allow_per_room_profiles: false ## TLS ## # PEM-encoded X509 certificate for TLS. # This certificate, as of Synapse 1.0, will need to be a valid and verifiable # certificate, signed by a recognised Certificate Authority. # # See 'ACME support' below to enable auto-provisioning this certificate via # Let's Encrypt. # # If supplying your own, be sure to use a `.pem` file that includes the # full certificate chain including any intermediate certificates (for # instance, if using certbot, use `fullchain.pem` as your certificate, # not `cert.pem`). # #tls_certificate_path: "/data/synapse.127.0.0.1.nip.io.tls.crt" # PEM-encoded private key for TLS # #tls_private_key_path: "/data/synapse.127.0.0.1.nip.io.tls.key" # Whether to verify TLS server certificates for outbound federation requests. # # Defaults to `true`. To disable certificate verification, uncomment the # following line. # #federation_verify_certificates: false # The minimum TLS version that will be used for outbound federation requests. # # Defaults to `1`. Configurable to `1`, `1.1`, `1.2`, or `1.3`. Note # that setting this value higher than `1.2` will prevent federation to most # of the public Matrix network: only configure it to `1.3` if you have an # entirely private federation setup and you can ensure TLS 1.3 support. # #federation_client_minimum_tls_version: 1.2 # Skip federation certificate verification on the following whitelist # of domains. # # This setting should only be used in very specific cases, such as # federation over Tor hidden services and similar. For private networks # of homeservers, you likely want to use a private CA instead. # # Only effective if federation_verify_certicates is `true`. # #federation_certificate_verification_whitelist: # - lon.example.com # - *.domain.com # - *.onion # List of custom certificate authorities for federation traffic. # # This setting should only normally be used within a private network of # homeservers. # # Note that this list will replace those that are provided by your # operating environment. Certificates must be in PEM format. # #federation_custom_ca_list: # - myCA1.pem # - myCA2.pem # - myCA3.pem # ACME support: This will configure Synapse to request a valid TLS certificate # for your configured `server_name` via Let's Encrypt. # # Note that provisioning a certificate in this way requires port 80 to be # routed to Synapse so that it can complete the http-01 ACME challenge. # By default, if you enable ACME support, Synapse will attempt to listen on # port 80 for incoming http-01 challenges - however, this will likely fail # with 'Permission denied' or a similar error. # # There are a couple of potential solutions to this: # # * If you already have an Apache, Nginx, or similar listening on port 80, # you can configure Synapse to use an alternate port, and have your web # server forward the requests. For example, assuming you set 'port: 8009' # below, on Apache, you would write: # # ProxyPass /.well-known/acme-challenge http://localhost:8009/.well-known/acme-challenge # # * Alternatively, you can use something like `authbind` to give Synapse # permission to listen on port 80. # acme: # ACME support is disabled by default. Uncomment the following line # (and tls_certificate_path and tls_private_key_path above) to enable it. # #enabled: true # Endpoint to use to request certificates. If you only want to test, # use Let's Encrypt's staging url: # https://acme-staging.api.letsencrypt.org/directory # #url: https://acme-v01.api.letsencrypt.org/directory # Port number to listen on for the HTTP-01 challenge. Change this if # you are forwarding connections through Apache/Nginx/etc. # #port: 80 # Local addresses to listen on for incoming connections. # Again, you may want to change this if you are forwarding connections # through Apache/Nginx/etc. # #bind_addresses: ['::', '0.0.0.0'] # How many days remaining on a certificate before it is renewed. # #reprovision_threshold: 30 # The domain that the certificate should be for. Normally this # should be the same as your Matrix domain (i.e., 'server_name'), but, # by putting a file at 'https:///.well-known/matrix/server', # you can delegate incoming traffic to another server. If you do that, # you should give the target of the delegation here. # # For example: if your 'server_name' is 'example.com', but # 'https://example.com/.well-known/matrix/server' delegates to # 'matrix.example.com', you should put 'matrix.example.com' here. # # If not set, defaults to your 'server_name'. # #domain: matrix.example.com # file to use for the account key. This will be generated if it doesn't # exist. # # If unspecified, we will use CONFDIR/client.key. # account_key_file: /data/acme_account.key # List of allowed TLS fingerprints for this server to publish along # with the signing keys for this server. Other matrix servers that # make HTTPS requests to this server will check that the TLS # certificates returned by this server match one of the fingerprints. # # Synapse automatically adds the fingerprint of its own certificate # to the list. So if federation traffic is handled directly by synapse # then no modification to the list is required. # # If synapse is run behind a load balancer that handles the TLS then it # will be necessary to add the fingerprints of the certificates used by # the loadbalancers to this list if they are different to the one # synapse is using. # # Homeservers are permitted to cache the list of TLS fingerprints # returned in the key responses up to the "valid_until_ts" returned in # key. It may be necessary to publish the fingerprints of a new # certificate and wait until the "valid_until_ts" of the previous key # responses have passed before deploying it. # # You can calculate a fingerprint from a given TLS listener via: # openssl s_client -connect $host:$port < /dev/null 2> /dev/null | # openssl x509 -outform DER | openssl sha256 -binary | base64 | tr -d '=' # or by checking matrix.org/federationtester/api/report?server_name=$host # #tls_fingerprints: [{"sha256": ""}] ## Database ## database: # The database engine name name: "psycopg2" args: user: "synapse" password: "synapse-password" database: "homeserver" host: "postgres" cp_min: 5 cp_max: 10 # Number of events to cache in memory. # #event_cache_size: 10K ## Logging ## # A yaml python logging config file # log_config: "/config/synapse.127.0.0.1.nip.io.log.config" ## Ratelimiting ## # Ratelimiting settings for client actions (registration, login, messaging). # # Each ratelimiting configuration is made of two parameters: # - per_second: number of requests a client can send per second. # - burst_count: number of requests a client can send before being throttled. # # Synapse currently uses the following configurations: # - one for messages that ratelimits sending based on the account the client # is using # - one for registration that ratelimits registration requests based on the # client's IP address. # - one for login that ratelimits login requests based on the client's IP # address. # - one for login that ratelimits login requests based on the account the # client is attempting to log into. # - one for login that ratelimits login requests based on the account the # client is attempting to log into, based on the amount of failed login # attempts for this account. # # The defaults are as shown below. # #rc_message: # per_second: 0.2 # burst_count: 10 # #rc_registration: # per_second: 0.17 # burst_count: 3 # #rc_login: # address: # per_second: 0.17 # burst_count: 3 # account: # per_second: 0.17 # burst_count: 3 # failed_attempts: # per_second: 0.17 # burst_count: 3 rc_message: per_second: 100 burst_count: 1000 rc_registration: per_second: 100 burst_count: 1000 rc_login: address: per_second: 100 burst_count: 1000 account: per_second: 100 burst_count: 1000 failed_attempts: per_second: 100 burst_count: 1000 # Ratelimiting settings for incoming federation # # The rc_federation configuration is made up of the following settings: # - window_size: window size in milliseconds # - sleep_limit: number of federation requests from a single server in # a window before the server will delay processing the request. # - sleep_delay: duration in milliseconds to delay processing events # from remote servers by if they go over the sleep limit. # - reject_limit: maximum number of concurrent federation requests # allowed from a single server # - concurrent: number of federation requests to concurrently process # from a single server # # The defaults are as shown below. # #rc_federation: # window_size: 1000 # sleep_limit: 10 # sleep_delay: 500 # reject_limit: 50 # concurrent: 3 # Target outgoing federation transaction frequency for sending read-receipts, # per-room. # # If we end up trying to send out more read-receipts, they will get buffered up # into fewer transactions. # #federation_rr_transactions_per_room_per_second: 50 enable_authenticated_media: true # Directory where uploaded images and attachments are stored. # media_store_path: "/media-store" # Media storage providers allow media to be stored in different # locations. # #media_storage_providers: # - module: file_system # # Whether to write new local files. # store_local: false # # Whether to write new remote media # store_remote: false # # Whether to block upload requests waiting for write to this # # provider to complete # store_synchronous: false # config: # directory: /mnt/some/other/directory # Directory where in-progress uploads are stored. # uploads_path: "/tmp" # The largest allowed upload size in bytes # #max_upload_size: 10M # Maximum number of pixels that will be thumbnailed # #max_image_pixels: 32M # Whether to generate new thumbnails on the fly to precisely match # the resolution requested by the client. If true then whenever # a new resolution is requested by the client the server will # generate a new thumbnail. If false the server will pick a thumbnail # from a precalculated list. # #dynamic_thumbnails: false # List of thumbnails to precalculate when an image is uploaded. # #thumbnail_sizes: # - width: 32 # height: 32 # method: crop # - width: 96 # height: 96 # method: crop # - width: 320 # height: 240 # method: scale # - width: 640 # height: 480 # method: scale # - width: 800 # height: 600 # method: scale # Is the preview URL API enabled? # # 'false' by default: uncomment the following to enable it (and specify a # url_preview_ip_range_blacklist blacklist). # #url_preview_enabled: true # List of IP address CIDR ranges that the URL preview spider is denied # from accessing. There are no defaults: you must explicitly # specify a list for URL previewing to work. You should specify any # internal services in your network that you do not want synapse to try # to connect to, otherwise anyone in any Matrix room could cause your # synapse to issue arbitrary GET requests to your internal services, # causing serious security issues. # # (0.0.0.0 and :: are always blacklisted, whether or not they are explicitly # listed here, since they correspond to unroutable addresses.) # # This must be specified if url_preview_enabled is set. It is recommended that # you uncomment the following list as a starting point. # #url_preview_ip_range_blacklist: # - '127.0.0.0/8' # - '10.0.0.0/8' # - '172.16.0.0/12' # - '192.168.0.0/16' # - '100.64.0.0/10' # - '169.254.0.0/16' # - '::1/128' # - 'fe80::/64' # - 'fc00::/7' # List of IP address CIDR ranges that the URL preview spider is allowed # to access even if they are specified in url_preview_ip_range_blacklist. # This is useful for specifying exceptions to wide-ranging blacklisted # target IP ranges - e.g. for enabling URL previews for a specific private # website only visible in your network. # #url_preview_ip_range_whitelist: # - '192.168.1.1' # Optional list of URL matches that the URL preview spider is # denied from accessing. You should use url_preview_ip_range_blacklist # in preference to this, otherwise someone could define a public DNS # entry that points to a private IP address and circumvent the blacklist. # This is more useful if you know there is an entire shape of URL that # you know that will never want synapse to try to spider. # # Each list entry is a dictionary of url component attributes as returned # by urlparse.urlsplit as applied to the absolute form of the URL. See # https://docs.python.org/2/library/urlparse.html#urlparse.urlsplit # The values of the dictionary are treated as an filename match pattern # applied to that component of URLs, unless they start with a ^ in which # case they are treated as a regular expression match. If all the # specified component matches for a given list item succeed, the URL is # blacklisted. # #url_preview_url_blacklist: # # blacklist any URL with a username in its URI # - username: '*' # # # blacklist all *.google.com URLs # - netloc: 'google.com' # - netloc: '*.google.com' # # # blacklist all plain HTTP URLs # - scheme: 'http' # # # blacklist http(s)://www.acme.com/foo # - netloc: 'www.acme.com' # path: '/foo' # # # blacklist any URL with a literal IPv4 address # - netloc: '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$' # The largest allowed URL preview spidering size in bytes # #max_spider_size: 10M ## Captcha ## # See docs/CAPTCHA_SETUP for full details of configuring this. # This Home Server's ReCAPTCHA public key. # #recaptcha_public_key: "YOUR_PUBLIC_KEY" # This Home Server's ReCAPTCHA private key. # #recaptcha_private_key: "YOUR_PRIVATE_KEY" # Enables ReCaptcha checks when registering, preventing signup # unless a captcha is answered. Requires a valid ReCaptcha # public/private key. # #enable_registration_captcha: false # A secret key used to bypass the captcha test entirely. # #captcha_bypass_secret: "YOUR_SECRET_HERE" # The API endpoint to use for verifying m.login.recaptcha responses. # #recaptcha_siteverify_api: "https://www.recaptcha.net/recaptcha/api/siteverify" ## TURN ## # The public URIs of the TURN server to give to clients # #turn_uris: [] # The shared secret used to compute passwords for the TURN server # #turn_shared_secret: "YOUR_SHARED_SECRET" # The Username and password if the TURN server needs them and # does not use a token # #turn_username: "TURNSERVER_USERNAME" #turn_password: "TURNSERVER_PASSWORD" # How long generated TURN credentials last # #turn_user_lifetime: 1h # Whether guests should be allowed to use the TURN server. # This defaults to True, otherwise VoIP will be unreliable for guests. # However, it does introduce a slight security risk as it allows users to # connect to arbitrary endpoints without having first signed up for a # valid account (e.g. by passing a CAPTCHA). # #turn_allow_guests: True ## Registration ## # # Registration can be rate-limited using the parameters in the "Ratelimiting" # section of this file. # Enable registration for new users. # enable_registration: true # Optional account validity configuration. This allows for accounts to be denied # any request after a given period. # # ``enabled`` defines whether the account validity feature is enabled. Defaults # to False. # # ``period`` allows setting the period after which an account is valid # after its registration. When renewing the account, its validity period # will be extended by this amount of time. This parameter is required when using # the account validity feature. # # ``renew_at`` is the amount of time before an account's expiry date at which # Synapse will send an email to the account's email address with a renewal link. # This needs the ``email`` and ``public_baseurl`` configuration sections to be # filled. # # ``renew_email_subject`` is the subject of the email sent out with the renewal # link. ``%(app)s`` can be used as a placeholder for the ``app_name`` parameter # from the ``email`` section. # # Once this feature is enabled, Synapse will look for registered users without an # expiration date at startup and will add one to every account it found using the # current settings at that time. # This means that, if a validity period is set, and Synapse is restarted (it will # then derive an expiration date from the current validity period), and some time # after that the validity period changes and Synapse is restarted, the users' # expiration dates won't be updated unless their account is manually renewed. This # date will be randomly selected within a range [now + period - d ; now + period], # where d is equal to 10% of the validity period. # #account_validity: # enabled: True # period: 6w # renew_at: 1w # renew_email_subject: "Renew your %(app)s account" # Time that a user's session remains valid for, after they log in. # # Note that this is not currently compatible with guest logins. # # Note also that this is calculated at login time: changes are not applied # retrospectively to users who have already logged in. # # By default, this is infinite. # #session_lifetime: 24h # The user must provide all of the below types of 3PID when registering. # #registrations_require_3pid: # - email # - msisdn # Explicitly disable asking for MSISDNs from the registration # flow (overrides registrations_require_3pid if MSISDNs are set as required) # #disable_msisdn_registration: true # Mandate that users are only allowed to associate certain formats of # 3PIDs with accounts on this server. # #allowed_local_3pids: # - medium: email # pattern: '.*@matrix\.org' # - medium: email # pattern: '.*@vector\.im' # - medium: msisdn # pattern: '\+44' # Enable 3PIDs lookup requests to identity servers from this server. # #enable_3pid_lookup: true registration_requires_token: true # If set, allows registration of standard or admin accounts by anyone who # has the shared secret, even if registration is otherwise disabled. # registration_shared_secret: "y4aTYam;zxKZ#MnaHRrGDPs4&dS*3VEv_&Ck_;pe1=CrtM8*=7" # Set the number of bcrypt rounds used to generate password hash. # Larger numbers increase the work factor needed to generate the hash. # The default number is 12 (which equates to 2^12 rounds). # N.B. that increasing this will exponentially increase the time required # to register or login - e.g. 24 => 2^24 rounds which will take >20 mins. # #bcrypt_rounds: 12 # Allows users to register as guests without a password/email/etc, and # participate in rooms hosted on this server which have been made # accessible to anonymous users. # #allow_guest_access: false # The identity server which we suggest that clients should use when users log # in on this server. # # (By default, no suggestion is made, so it is left up to the client. # This setting is ignored unless public_baseurl is also set.) # #default_identity_server: https://matrix.org # The list of identity servers trusted to verify third party # identifiers by this server. # # Also defines the ID server which will be called when an account is # deactivated (one will be picked arbitrarily). # #trusted_third_party_id_servers: # - matrix.org # - vector.im # Users who register on this homeserver will automatically be joined # to these rooms # #auto_join_rooms: # - "#example:example.com" # Where auto_join_rooms are specified, setting this flag ensures that the # the rooms exist by creating them when the first user on the # homeserver registers. # Setting to false means that if the rooms are not manually created, # users cannot be auto-joined since they do not exist. # #autocreate_auto_join_rooms: true ## Metrics ### # Enable collection and rendering of performance metrics # #enable_metrics: False # Enable sentry integration # NOTE: While attempts are made to ensure that the logs don't contain # any sensitive information, this cannot be guaranteed. By enabling # this option the sentry server may therefore receive sensitive # information, and it in turn may then diseminate sensitive information # through insecure notification channels if so configured. # #sentry: # dsn: "..." # Whether or not to report anonymized homeserver usage statistics. report_stats: false ## API Configuration ## # A list of event types that will be included in the room_invite_state # #room_invite_state_types: # - "m.room.join_rules" # - "m.room.canonical_alias" # - "m.room.avatar" # - "m.room.encryption" # - "m.room.name" # A list of application service config files to use # #app_service_config_files: # - app_service_1.yaml # - app_service_2.yaml # Uncomment to enable tracking of application service IP addresses. Implicitly # enables MAU tracking for application service users. # #track_appservice_user_ips: True # a secret which is used to sign access tokens. If none is specified, # the registration_shared_secret is used, if one is given; otherwise, # a secret key is derived from the signing key. # macaroon_secret_key: "q_&1433pm~0X#wN@JF=f+aN=RknLk^U84+7J8fwGrWDQv2;.,F" # Used to enable access token expiration. # #expire_access_token: False # a secret which is used to calculate HMACs for form values, to stop # falsification of values. Must be specified for the User Consent # forms to work. # form_secret: "g2:tvyaCbugJrP#1w.+6Eta:xxIfvl*HIF:o#8+qTbU7tPlUhY" ## Signing Keys ## # Path to the signing key to sign messages with # signing_key_path: "/config/synapse.127.0.0.1.nip.io.signing.key" # The keys that the server used to sign messages with but won't use # to sign new messages. E.g. it has lost its private key # #old_signing_keys: # "ed25519:auto": # # Base64 encoded public key # key: "The public part of your old signing key." # # Millisecond POSIX timestamp when the key expired. # expired_ts: 123456789123 # How long key response published by this server is valid for. # Used to set the valid_until_ts in /key/v2 APIs. # Determines how quickly servers will query to check which keys # are still valid. # #key_refresh_interval: 1d # The trusted servers to download signing keys from. # # When we need to fetch a signing key, each server is tried in parallel. # # Normally, the connection to the key server is validated via TLS certificates. # Additional security can be provided by configuring a `verify key`, which # will make synapse check that the response is signed by that key. # # This setting supercedes an older setting named `perspectives`. The old format # is still supported for backwards-compatibility, but it is deprecated. # # Options for each entry in the list include: # # server_name: the name of the server. required. # # verify_keys: an optional map from key id to base64-encoded public key. # If specified, we will check that the response is signed by at least # one of the given keys. # # accept_keys_insecurely: a boolean. Normally, if `verify_keys` is unset, # and federation_verify_certificates is not `true`, synapse will refuse # to start, because this would allow anyone who can spoof DNS responses # to masquerade as the trusted key server. If you know what you are doing # and are sure that your network environment provides a secure connection # to the key server, you can set this to `true` to override this # behaviour. # # An example configuration might look like: # #trusted_key_servers: # - server_name: "my_trusted_server.example.com" # verify_keys: # "ed25519:auto": "abcdefghijklmnopqrstuvwxyzabcdefghijklmopqr" # - server_name: "my_other_trusted_server.example.com" # # The default configuration is: # #trusted_key_servers: # - server_name: "matrix.org" # Enable SAML2 for registration and login. Uses pysaml2. # # `sp_config` is the configuration for the pysaml2 Service Provider. # See pysaml2 docs for format of config. # # Default values will be used for the 'entityid' and 'service' settings, # so it is not normally necessary to specify them unless you need to # override them. # # Once SAML support is enabled, a metadata file will be exposed at # https://:/_matrix/saml2/metadata.xml, which you may be able to # use to configure your SAML IdP with. Alternatively, you can manually configure # the IdP to use an ACS location of # https://:/_matrix/saml2/authn_response. # #saml2_config: # sp_config: # # point this to the IdP's metadata. You can use either a local file or # # (preferably) a URL. # metadata: # #local: ["saml2/idp.xml"] # remote: # - url: https://our_idp/metadata.xml # # # By default, the user has to go to our login page first. If you'd like to # # allow IdP-initiated login, set 'allow_unsolicited: True' in a # # 'service.sp' section: # # # #service: # # sp: # # allow_unsolicited: True # # # The examples below are just used to generate our metadata xml, and you # # may well not need it, depending on your setup. Alternatively you # # may need a whole lot more detail - see the pysaml2 docs! # # description: ["My awesome SP", "en"] # name: ["Test SP", "en"] # # organization: # name: Example com # display_name: # - ["Example co", "en"] # url: "http://example.com" # # contact_person: # - given_name: Bob # sur_name: "the Sysadmin" # email_address": ["admin@example.com"] # contact_type": technical # # # Instead of putting the config inline as above, you can specify a # # separate pysaml2 configuration file: # # # config_path: "/data/sp_conf.py" # # # the lifetime of a SAML session. This defines how long a user has to # # complete the authentication process, if allow_unsolicited is unset. # # The default is 5 minutes. # # # # saml_session_lifetime: 5m # Enable CAS for registration and login. # #cas_config: # enabled: true # server_url: "https://cas-server.com" # service_url: "https://homeserver.domain.com:8448" # #required_attributes: # # name: value # The JWT needs to contain a globally unique "sub" (subject) claim. # #jwt_config: # enabled: true # secret: "a secret" # algorithm: "HS256" password_config: # Uncomment to disable password login # #enabled: false # Uncomment to disable authentication against the local password # database. This is ignored if `enabled` is false, and is only useful # if you have other password_providers. # #localdb_enabled: false # Uncomment and change to a secret random string for extra security. # DO NOT CHANGE THIS AFTER INITIAL SETUP! # #pepper: "EVEN_MORE_SECRET" # Enable sending emails for password resets, notification events or # account expiry notices # # If your SMTP server requires authentication, the optional smtp_user & # smtp_pass variables should be used # #email: # enable_notifs: false # smtp_host: "localhost" # smtp_port: 25 # SSL: 465, STARTTLS: 587 # smtp_user: "exampleusername" # smtp_pass: "examplepassword" # require_transport_security: False # notif_from: "Your Friendly %(app)s Home Server " # app_name: Matrix # # # Enable email notifications by default # # # notif_for_new_users: True # # # Defining a custom URL for Riot is only needed if email notifications # # should contain links to a self-hosted installation of Riot; when set # # the "app_name" setting is ignored # # # riot_base_url: "http://localhost/riot" # # # Enable sending password reset emails via the configured, trusted # # identity servers # # # # IMPORTANT! This will give a malicious or overtaken identity server # # the ability to reset passwords for your users! Make absolutely sure # # that you want to do this! It is strongly recommended that password # # reset emails be sent by the homeserver instead # # # # If this option is set to false and SMTP options have not been # # configured, resetting user passwords via email will be disabled # # # #trust_identity_server_for_password_resets: false # # # Configure the time that a validation email or text message code # # will expire after sending # # # # This is currently used for password resets # # # #validation_token_lifetime: 1h # # # Template directory. All template files should be stored within this # # directory. If not set, default templates from within the Synapse # # package will be used # # # # For the list of default templates, please see # # https://github.com/matrix-org/synapse/tree/master/synapse/res/templates # # # #template_dir: res/templates # # # Templates for email notifications # # # notif_template_html: notif_mail.html # notif_template_text: notif_mail.txt # # # Templates for account expiry notices # # # expiry_template_html: notice_expiry.html # expiry_template_text: notice_expiry.txt # # # Templates for password reset emails sent by the homeserver # # # #password_reset_template_html: password_reset.html # #password_reset_template_text: password_reset.txt # # # Templates for password reset success and failure pages that a user # # will see after attempting to reset their password # # # #password_reset_template_success_html: password_reset_success.html # #password_reset_template_failure_html: password_reset_failure.html #password_providers: # - module: "ldap_auth_provider.LdapAuthProvider" # config: # enabled: true # uri: "ldap://ldap.example.com:389" # start_tls: true # base: "ou=users,dc=example,dc=com" # attributes: # uid: "cn" # mail: "email" # name: "givenName" # #bind_dn: # #bind_password: # #filter: "(objectClass=posixAccount)" password_providers: [] # Clients requesting push notifications can either have the body of # the message sent in the notification poke along with other details # like the sender, or just the event ID and room ID (`event_id_only`). # If clients choose the former, this option controls whether the # notification request includes the content of the event (other details # like the sender are still included). For `event_id_only` push, it # has no effect. # # For modern android devices the notification content will still appear # because it is loaded by the app. iPhone, however will send a # notification saying only that a message arrived and who it came from. # #push: # include_content: true #spam_checker: # module: "my_custom_project.SuperSpamChecker" # config: # example_option: 'things' # User Directory configuration # # 'enabled' defines whether users can search the user directory. If # false then empty responses are returned to all queries. Defaults to # true. # # 'search_all_users' defines whether to search all users visible to your HS # when searching the user directory, rather than limiting to users visible # in public rooms. Defaults to false. If you set it True, you'll have to # rebuild the user_directory search indexes, see # https://github.com/matrix-org/synapse/blob/master/docs/user_directory.md # #user_directory: # enabled: true # search_all_users: false # User Consent configuration # # for detailed instructions, see # https://github.com/matrix-org/synapse/blob/master/docs/consent_tracking.md # # Parts of this section are required if enabling the 'consent' resource under # 'listeners', in particular 'template_dir' and 'version'. # # 'template_dir' gives the location of the templates for the HTML forms. # This directory should contain one subdirectory per language (eg, 'en', 'fr'), # and each language directory should contain the policy document (named as # '.html') and a success page (success.html). # # 'version' specifies the 'current' version of the policy document. It defines # the version to be served by the consent resource if there is no 'v' # parameter. # # 'server_notice_content', if enabled, will send a user a "Server Notice" # asking them to consent to the privacy policy. The 'server_notices' section # must also be configured for this to work. Notices will *not* be sent to # guest users unless 'send_server_notice_to_guests' is set to true. # # 'block_events_error', if set, will block any attempts to send events # until the user consents to the privacy policy. The value of the setting is # used as the text of the error. # # 'require_at_registration', if enabled, will add a step to the registration # process, similar to how captcha works. Users will be required to accept the # policy before their account is created. # # 'policy_name' is the display name of the policy users will see when registering # for an account. Has no effect unless `require_at_registration` is enabled. # Defaults to "Privacy Policy". # #user_consent: # template_dir: res/templates/privacy # version: 1.0 # server_notice_content: # msgtype: m.text # body: >- # To continue using this homeserver you must review and agree to the # terms and conditions at %(consent_uri)s # send_server_notice_to_guests: True # block_events_error: >- # To continue using this homeserver you must review and agree to the # terms and conditions at %(consent_uri)s # require_at_registration: False # policy_name: Privacy Policy # # Local statistics collection. Used in populating the room directory. # # 'bucket_size' controls how large each statistics timeslice is. It can # be defined in a human readable short form -- e.g. "1d", "1y". # # 'retention' controls how long historical statistics will be kept for. # It can be defined in a human readable short form -- e.g. "1d", "1y". # # #stats: # enabled: true # bucket_size: 1d # retention: 1y # Server Notices room configuration # # Uncomment this section to enable a room which can be used to send notices # from the server to users. It is a special room which cannot be left; notices # come from a special "notices" user id. # # If you uncomment this section, you *must* define the system_mxid_localpart # setting, which defines the id of the user which will be used to send the # notices. # # It's also possible to override the room name, the display name of the # "notices" user, and the avatar for the user. # #server_notices: # system_mxid_localpart: notices # system_mxid_display_name: "Server Notices" # system_mxid_avatar_url: "mxc://server.com/oumMVlgDnLYFaPVkExemNVVZ" # room_name: "Server Notices" # Uncomment to disable searching the public room list. When disabled # blocks searching local and remote room lists for local and remote # users by always returning an empty list for all queries. # #enable_room_list_search: false # The `alias_creation` option controls who's allowed to create aliases # on this server. # # The format of this option is a list of rules that contain globs that # match against user_id, room_id and the new alias (fully qualified with # server name). The action in the first rule that matches is taken, # which can currently either be "allow" or "deny". # # Missing user_id/room_id/alias fields default to "*". # # If no rules match the request is denied. An empty list means no one # can create aliases. # # Options for the rules include: # # user_id: Matches against the creator of the alias # alias: Matches against the alias being created # room_id: Matches against the room ID the alias is being pointed at # action: Whether to "allow" or "deny" the request if the rule matches # # The default is: # #alias_creation_rules: # - user_id: "*" # alias: "*" # room_id: "*" # action: allow # The `room_list_publication_rules` option controls who can publish and # which rooms can be published in the public room list. # # The format of this option is the same as that for # `alias_creation_rules`. # # If the room has one or more aliases associated with it, only one of # the aliases needs to match the alias rule. If there are no aliases # then only rules with `alias: *` match. # # If no rules match the request is denied. An empty list means no one # can publish rooms. # # Options for the rules include: # # user_id: Matches agaisnt the creator of the alias # room_id: Matches against the room ID being published # alias: Matches against any current local or canonical aliases # associated with the room # action: Whether to "allow" or "deny" the request if the rule matches # # The default is: # #room_list_publication_rules: # - user_id: "*" # alias: "*" # room_id: "*" # action: allow # Server admins can define a Python module that implements extra rules for # allowing or denying incoming events. In order to work, this module needs to # override the methods defined in synapse/events/third_party_rules.py. # # This feature is designed to be used in closed federations only, where each # participating server enforces the same rules. # #third_party_event_rules: # module: "my_custom_project.SuperRulesSet" # config: # example_option: 'things' ## Opentracing ## # These settings enable opentracing, which implements distributed tracing. # This allows you to observe the causal chains of events across servers # including requests, key lookups etc., across any server running # synapse or any other other services which supports opentracing # (specifically those implemented with Jaeger). # opentracing: # tracing is disabled by default. Uncomment the following line to enable it. # #enabled: true # The list of homeservers we wish to send and receive span contexts and span baggage. # See docs/opentracing.rst # This is a list of regexes which are matched against the server_name of the # homeserver. # # By defult, it is empty, so no servers are matched. # #homeserver_whitelist: # - ".*" ================================================ FILE: etc/services/synapse/config/synapse.127.0.0.1.nip.io.log.config ================================================ version: 1 formatters: precise: format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s' filters: context: (): synapse.util.logcontext.LoggingContextFilter request: "" handlers: console: class: logging.StreamHandler formatter: precise filters: [context] loggers: synapse: level: INFO shared_secret_authenticator: level: INFO rest_auth_provider: level: INFO synapse.storage.SQL: # beware: increasing this to DEBUG will make synapse log sensitive # information such as access tokens. level: INFO root: level: INFO handlers: [console] ================================================ FILE: etc/services/synapse/config/synapse.127.0.0.1.nip.io.signing.key ================================================ ed25519 a_FEMe JGs8Fk83GHIrVyhBYa/VRUFbU4+Fxtf8iOsJ7CMamcM ================================================ FILE: justfile ================================================ project_name := "baibot" container_image_name := "localhost/baibot" project_container_network := "baibot" admin_username := "admin" admin_password := "admin" bot_username := "baibot" bot_password := "baibot" homeserver := `cat var/homeserver 2>/dev/null || echo continuwuity` mise_data_dir := env("MISE_DATA_DIR", justfile_directory() / "var/mise") mise_trusted_config_paths := justfile_directory() / "mise.toml" # Show help by default default: @just --list --justfile {{ justfile() }} # Selects which homeserver implementation to use (continuwuity or synapse) homeserver-init value: #!/bin/sh mkdir -p {{ justfile_directory() }}/var echo {{ value }} > {{ justfile_directory() }}/var/homeserver echo "" echo "⚠️ If you had already prepared your app configuration (var/app/local/config.yml or var/app/container/config.yml)," echo " you will need to update it manually or delete it and re-run the prepare step." echo " You should also delete var/app/local/data and/or var/app/container/data," echo " as old application state is not compatible across homeserver implementations." echo "" echo "⚠️ If Element Web was already prepared, delete var/services/element-web/ to regenerate its config." # Builds and runs a development binary run-locally *extra_args: app-local-prepare RUST_BACKTRACE=1 \ BAIBOT_CONFIG_FILE_PATH={{ justfile_directory() }}/var/app/local/config.yml \ BAIBOT_PERSISTENCE_DATA_DIR_PATH={{ justfile_directory() }}/var/app/local/data \ cargo run -- {{ extra_args }} # Builds and runs the bot in a container run-in-container *extra_args: app-container-prepare build-container-image-debug /usr/bin/env docker run \ -it \ --rm \ --name={{ project_name }} \ --user=$(id -u):$(id -g) \ --cap-drop=ALL \ --read-only \ --network={{ project_container_network }} \ --env BAIBOT_PERSISTENCE_DATA_DIR_PATH=/data \ --mount type=bind,src={{ justfile_directory() }}/var/app/container/config.yml,dst=/app/config.yml,ro \ --mount type=bind,src={{ justfile_directory() }}/var/app/container/data,dst=/data \ {{ container_image_name }}:latest {{ extra_args }} # Runs tests test *extra_args: RUST_BACKTRACE=1 cargo test {{ extra_args }} # Formats the code fmt: RUST_BACKTRACE=1 cargo fmt --all # Builds a debug binary (target/debug/*) build-debug *extra_args: RUST_BACKTRACE=1 cargo build {{ extra_args }} # Builds an optimized release binary (target/release/*) build-release *extra_args: (build-debug "--release") # Builds a container image (debug mode) build-container-image-debug tag='latest': (_build-container-image "false" tag) # Builds a container image (release mode) build-container-image-release tag='latest': (_build-container-image "true" tag) _build-container-image release_build tag: /usr/bin/env docker build \ --build-arg RELEASE_BUILD={{ release_build }} \ -f {{ justfile_directory() }}/Dockerfile \ -t {{ container_image_name }}:{{ tag }} \ . # Runs a docker-compose command docker-compose services_type *extra_args: /usr/bin/docker compose \ --project-directory var/services \ --env-file var/services/env \ -f etc/services/{{ services_type }}/compose.yml \ -p {{ project_name }}-{{ services_type }} \ {{ extra_args }} # Runs a docker-compose command against the synapse services docker-compose-synapse *extra_args: just docker-compose synapse {{ extra_args }} # Runs a docker-compose command against the element-web services docker-compose-element-web *extra_args: just docker-compose element-web {{ extra_args }} # Runs a docker-compose command against the localai services docker-compose-localai *extra_args: just docker-compose localai {{ extra_args }} # Runs a docker-compose command against the ollama services docker-compose-ollama *extra_args: just docker-compose ollama {{ extra_args }} # Runs a docker-compose command against the continuwuity services docker-compose-continuwuity *extra_args: just docker-compose continuwuity {{ extra_args }} # Runs the homeserver and Element Web (in the background) services-start: services-prepare just -f {{ justfile_directory() }}/justfile {{ homeserver }}-start just -f {{ justfile_directory() }}/justfile element-web-start # Stops Element Web and the homeserver services-stop: just -f {{ justfile_directory() }}/justfile element-web-stop just -f {{ justfile_directory() }}/justfile {{ homeserver }}-stop # Tails the logs for the homeserver and Element Web services-tail-logs: just -f {{ justfile_directory() }}/justfile {{ homeserver }}-tail-logs # Prepares the homeserver and Element Web for running services-prepare: just -f {{ justfile_directory() }}/justfile {{ homeserver }}-prepare just -f {{ justfile_directory() }}/justfile element-web-prepare # Runs Synapse (in the background) synapse-start: synapse-prepare (docker-compose-synapse "up" "-d") # Stops Synapse synapse-stop: (docker-compose-synapse "down") # Tails the logs for Synapse synapse-tail-logs: (docker-compose-synapse "logs" "-f") # Prepares Synapse for running synapse-prepare: _prepare-var-services-env _prepare-var-services-postgres _prepare-var-services-synapse _prepare-container-network # Runs Element Web (in the background) element-web-start: element-web-prepare (docker-compose-element-web "up" "-d") # Stops Element Web element-web-stop: (docker-compose-element-web "down") # Tails the logs for Element Web element-web-tail-logs: (docker-compose-element-web "logs" "-f") # Prepares Element Web for running element-web-prepare: _prepare-var-services-env _prepare-var-services-element-web _prepare-container-network # Runs LocalAI (in the background) localai-start: localai-prepare (docker-compose-localai "up" "-d") # Stops LocalAI localai-stop: (docker-compose-localai "down") # Tails the logs for LocalAI localai-tail-logs: (docker-compose-localai "logs" "-f") # Prepares LocalAI for running localai-prepare: _prepare-var-services-env _prepare-var-services-localai _prepare-container-network # Runs Ollama (in the background) ollama-start: ollama-prepare (docker-compose-ollama "up" "-d") # Stops Ollama ollama-stop: (docker-compose-ollama "down") # Tails the logs for Ollama ollama-tail-logs: (docker-compose-ollama "logs" "-f") # Prepares Ollama for running ollama-prepare: _prepare-var-services-env _prepare-var-services-ollama _prepare-container-network # Runs Continuwuity (in the background) continuwuity-start: continuwuity-prepare (docker-compose-continuwuity "up" "-d") # Stops Continuwuity continuwuity-stop: (docker-compose-continuwuity "down") # Tails the logs for Continuwuity continuwuity-tail-logs: (docker-compose-continuwuity "logs" "-f") # Prepares Continuwuity for running continuwuity-prepare: _prepare-var-services-env _prepare-var-services-continuwuity _prepare-container-network # Registers a user on Continuwuity via the Matrix Client-Server API continuwuity-register-user username password: {{ justfile_directory() }}/etc/services/continuwuity/register-user.sh {{ justfile_directory() }}/var/services/env {{ username }} {{ password }} # Prepares the Continuwuity user accounts continuwuity-users-prepare: continuwuity-prepare just -f {{ justfile_directory() }}/justfile continuwuity-register-user "{{ admin_username }}" "{{ admin_password }}" just -f {{ justfile_directory() }}/justfile continuwuity-register-user "{{ bot_username }}" "{{ bot_password }}" # Pulls an Ollama model ollama-pull-model model_id: just -f {{ justfile_directory() }}/justfile docker-compose-ollama \ exec ollama \ ollama pull {{ model_id }} # Prepares the app for running locally app-local-prepare: _prepare-var-app-local-config_yml _prepare-var-app-local-data # Prepares the app for running in a container app-container-prepare: _prepare-var-app-container-config_yml _prepare-var-app-container-data # Prepares the user accounts users-prepare: just -f {{ justfile_directory() }}/justfile {{ homeserver }}-users-prepare # Prepares the Synapse user accounts synapse-users-prepare: synapse-prepare just -f {{ justfile_directory() }}/justfile synapse-register-admin-user "{{ admin_username }}" "{{ admin_password }}" just -f {{ justfile_directory() }}/justfile synapse-register-regular-user "{{ bot_username }}" "{{ bot_password }}" # Starts a Postgres CLI (psql) postgres-cli: synapse-prepare (docker-compose-synapse "exec" "postgres" "/bin/sh" "-c" "'PGUSER=synapse PGPASSWORD=synapse-password PGDATABASE=homeserver psql -h postgres'") # Creates an administrator user on Synapse synapse-register-admin-user username password: synapse-prepare just -f {{ justfile_directory() }}/justfile docker-compose-synapse \ exec synapse \ register_new_matrix_user \ --admin \ -u {{ username }} \ -p {{ password }} \ -c /config/homeserver.yaml \ http://localhost:8008 # Creates a regular user on Synapse synapse-register-regular-user username password: synapse-prepare just -f {{ justfile_directory() }}/justfile docker-compose-synapse \ exec synapse \ register_new_matrix_user \ --no-admin \ -u {{ username }} \ -p {{ password }} \ -c /config/homeserver.yaml \ http://localhost:8008 # Runs the clippy linter clippy *extra_args: cargo clippy {{ extra_args }} # Checks that the code compiles without building check: cargo check # Invokes mise with the project-local data directory mise *args: _ensure_mise_data_directory #!/bin/sh export MISE_DATA_DIR="{{ mise_data_dir }}" export MISE_TRUSTED_CONFIG_PATHS="{{ mise_trusted_config_paths }}" mise {{ args }} # Runs prek (pre-commit hooks manager) with the given arguments prek *args: _ensure_mise_tools_installed @just --justfile {{ justfile() }} mise exec -- prek {{ args }} # Runs pre-commit hooks on staged files prek-run-on-staged *args: _ensure_mise_tools_installed @just --justfile {{ justfile() }} mise exec -- prek run {{ args }} # Runs pre-commit hooks on all files prek-run-on-all *args: _ensure_mise_tools_installed @just --justfile {{ justfile() }} mise exec -- prek run --all-files {{ args }} # Installs the git pre-commit hook (runs prek automatically before each commit) prek-install-git-pre-commit-hook: _ensure_mise_tools_installed @just --justfile {{ justfile() }} mise exec -- prek install # Internal - ensures var/mise directory exists _ensure_mise_data_directory: #!/bin/sh if [ ! -d "{{ mise_data_dir }}" ]; then mkdir -p "{{ mise_data_dir }}" fi # Internal - ensures mise tools are installed _ensure_mise_tools_installed: _ensure_mise_data_directory @just --justfile {{ justfile() }} mise install --quiet _prepare-var-services-env: #!/bin/sh cd {{ justfile_directory() }}; if [ ! -f var/services/env ]; then mkdir -p var/services cp {{ justfile_directory() }}/etc/services/env.dist var/services/env echo 'UID='`id -u` >> var/services/env; echo 'GID='`id -g` >> var/services/env; echo 'NETWORK_NAME={{ project_container_network }}' >> var/services/env; fi _prepare-var-services-postgres: #!/bin/sh cd {{ justfile_directory() }}; if [ ! -f var/services/postgres ]; then mkdir -p var/services/postgres chown `id -u`:`id -g` var/services/postgres fi _prepare-var-services-synapse: #!/bin/sh cd {{ justfile_directory() }}; if [ ! -f var/services/synapse ]; then mkdir -p var/services/synapse/media-store fi _prepare-var-services-element-web: #!/bin/sh cd {{ justfile_directory() }}; if [ ! -f var/services/element-web/config.json ]; then mkdir -p var/services/element-web cp {{ justfile_directory() }}/etc/services/element-web/config.json.dist var/services/element-web/config.json homeserver="{{ homeserver }}" if [ "$homeserver" = "continuwuity" ]; then sed --in-place 's|__HOMESERVER_CLIENT_URL__|http://continuwuity.127.0.0.1.nip.io:42030|g' var/services/element-web/config.json elif [ "$homeserver" = "synapse" ]; then sed --in-place 's|__HOMESERVER_CLIENT_URL__|http://synapse.127.0.0.1.nip.io:42020|g' var/services/element-web/config.json fi fi _prepare-var-services-ollama: #!/bin/sh cd {{ justfile_directory() }}; if [ ! -f var/services/ollama ]; then mkdir -p var/services/ollama fi _prepare-var-services-continuwuity: #!/bin/sh cd {{ justfile_directory() }}; if [ ! -f var/services/continuwuity ]; then mkdir -p var/services/continuwuity/data fi _prepare-var-services-localai: #!/bin/sh cd {{ justfile_directory() }}; if [ ! -f var/services/localai ]; then mkdir -p var/services/localai fi _prepare-container-network: #!/bin/sh network_definition=$(/usr/bin/env docker network ls --filter='name={{ project_container_network }}' --format=json) if [ "$network_definition" = "" ]; then /usr/bin/docker network create {{ project_container_network }} fi _prepare-var-app-local-config_yml: #!/bin/sh cd {{ justfile_directory() }}; if [ ! -f var/app/local/config.yml ]; then mkdir -p var/app/local cp {{ justfile_directory() }}/etc/app/config.yml.dist var/app/local/config.yml homeserver="{{ homeserver }}" if [ "$homeserver" = "continuwuity" ]; then sed --in-place 's/__HOMESERVER_SERVER_NAME__/continuwuity.127.0.0.1.nip.io/g' var/app/local/config.yml sed --in-place 's|__HOMESERVER_URL__|http://continuwuity.127.0.0.1.nip.io:42030|g' var/app/local/config.yml elif [ "$homeserver" = "synapse" ]; then sed --in-place 's/__HOMESERVER_SERVER_NAME__/synapse.127.0.0.1.nip.io/g' var/app/local/config.yml sed --in-place 's|__HOMESERVER_URL__|http://synapse.127.0.0.1.nip.io:42020|g' var/app/local/config.yml fi fi _prepare-var-app-local-data: #!/bin/sh cd {{ justfile_directory() }}; if [ ! -f var/app/local/data ]; then mkdir -p var/app/local/data fi _prepare-var-app-container-config_yml: #!/bin/sh cd {{ justfile_directory() }}; if [ ! -f var/app/container/config.yml ]; then mkdir -p var/app/container cp {{ justfile_directory() }}/etc/app/config.yml.dist var/app/container/config.yml homeserver="{{ homeserver }}" if [ "$homeserver" = "continuwuity" ]; then sed --in-place 's/__HOMESERVER_SERVER_NAME__/continuwuity.127.0.0.1.nip.io/g' var/app/container/config.yml sed --in-place 's|__HOMESERVER_URL__|http://continuwuity.127.0.0.1.nip.io:42030|g' var/app/container/config.yml sed --in-place 's/continuwuity.127.0.0.1.nip.io:42030/continuwuity:6167/g' var/app/container/config.yml elif [ "$homeserver" = "synapse" ]; then sed --in-place 's/__HOMESERVER_SERVER_NAME__/synapse.127.0.0.1.nip.io/g' var/app/container/config.yml sed --in-place 's|__HOMESERVER_URL__|http://synapse.127.0.0.1.nip.io:42020|g' var/app/container/config.yml sed --in-place 's/synapse.127.0.0.1.nip.io:42020/synapse:8008/g' var/app/container/config.yml fi sed --in-place 's/127.0.0.1:42026/ollama:11434/g' var/app/container/config.yml sed --in-place 's/127.0.0.1:42027/localai:8080/g' var/app/container/config.yml fi _prepare-var-app-container-data: #!/bin/sh cd {{ justfile_directory() }}; if [ ! -f var/app/container/data ]; then mkdir -p var/app/container/data fi ================================================ FILE: mise.toml ================================================ [tools] prek = "0.4.1" [settings] # Disable automatic trust prompts - we trust this config yes = true ================================================ FILE: renovate.json ================================================ { "$schema": "https://docs.renovatebot.com/renovate-schema.json", "extends": [ "config:recommended" ], "labels": [ "dependencies" ] } ================================================ FILE: rust-toolchain.toml ================================================ [toolchain] channel = "1.95.0" components = ["rustfmt", "clippy"] profile = "default" ================================================ FILE: src/agent/definition.rs ================================================ use serde::de::Error as DeError; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use super::provider::AgentProvider; // Custom serialization for AgentProvider pub fn serialize_provider_to_string( value: &AgentProvider, serializer: S, ) -> Result where S: Serializer, { serializer.serialize_str(value.to_static_str()) } // Custom deserialization for AgentProvider pub fn deserialize_provider_from_string<'de, D>(deserializer: D) -> Result where D: Deserializer<'de>, { let s = String::deserialize(deserializer)?; AgentProvider::from_string(&s).map_err(DeError::custom) } #[derive(Debug, Clone, Deserialize, Serialize)] pub struct AgentDefinition { pub id: String, #[serde( serialize_with = "serialize_provider_to_string", deserialize_with = "deserialize_provider_from_string" )] pub provider: AgentProvider, pub config: serde_yaml_ng::Value, } impl AgentDefinition { pub fn new(id: String, provider: AgentProvider, config: serde_yaml_ng::Value) -> Self { Self { id, provider, config, } } } impl PartialEq for AgentDefinition { fn eq(&self, other: &Self) -> bool { self.id == other.id } } impl Eq for AgentDefinition {} ================================================ FILE: src/agent/identifier.rs ================================================ use std::fmt; #[derive(Debug, PartialEq, Eq, Clone)] pub enum PublicIdentifier { Static(String), DynamicGlobal(String), DynamicRoomLocal(String), } impl PublicIdentifier { pub fn from_str(s: &str) -> Option { if let Some(rest) = s.strip_prefix("static/") { return Some(PublicIdentifier::Static(rest.to_string())); } else if let Some(rest) = s.strip_prefix("global/") { return Some(PublicIdentifier::DynamicGlobal(rest.to_string())); } else if let Some(rest) = s.strip_prefix("room-local/") { return Some(PublicIdentifier::DynamicRoomLocal(rest.to_string())); } None } pub fn as_string(&self) -> String { match self { PublicIdentifier::Static(s) => format!("static/{}", s), PublicIdentifier::DynamicGlobal(s) => format!("global/{}", s), PublicIdentifier::DynamicRoomLocal(s) => format!("room-local/{}", s), } } pub fn prefixless(&self) -> String { match self { PublicIdentifier::Static(s) => s.to_owned(), PublicIdentifier::DynamicGlobal(s) => s.to_owned(), PublicIdentifier::DynamicRoomLocal(s) => s.to_owned(), } } pub fn validate(&self) -> Result<(), String> { let prefixless = self.prefixless(); if prefixless.is_empty() { return Err("The agent ID must not be empty.".to_owned()); } // We use a slash to separate the agent type from the agent ID. if prefixless.contains("/") { return Err("The agent ID must not contain the `/` character.".to_owned()); } // Spaces are used for separating command arguments, etc. if prefixless.contains(" ") { return Err("The agent ID must not contain spaces.".to_owned()); } Ok(()) } } impl fmt::Display for PublicIdentifier { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{}", self.as_string()) } } #[cfg(test)] mod tests { use super::*; #[test] fn test_public_identifier_from_str() { assert_eq!( PublicIdentifier::from_str("static/abc"), Some(PublicIdentifier::Static("abc".to_string())) ); assert_eq!( PublicIdentifier::from_str("global/abc"), Some(PublicIdentifier::DynamicGlobal("abc".to_string())) ); assert_eq!( PublicIdentifier::from_str("room-local/abc"), Some(PublicIdentifier::DynamicRoomLocal("abc".to_string())) ); assert_eq!(PublicIdentifier::from_str("abc"), None); } #[test] fn test_public_identifier_as_string() { assert_eq!( PublicIdentifier::Static("abc".to_string()).as_string(), "static/abc" ); assert_eq!( PublicIdentifier::DynamicGlobal("abc".to_string()).as_string(), "global/abc" ); assert_eq!( PublicIdentifier::DynamicRoomLocal("abc".to_string()).as_string(), "room-local/abc" ); } } ================================================ FILE: src/agent/instantiation.rs ================================================ use super::{ AgentDefinition, AgentProvider, PublicIdentifier, provider::{self, ControllerType}, }; // Dead-code is allowed. We do not use these enum struct payloads directly, // but these errors are being print-formatted (`{:?}`) in error messages, so we wish to keep them. #[derive(Debug)] #[allow(dead_code)] pub enum Error { // Contains the error message from the validation function ConfigFailsValidation(String), // Contains the agent ID ConfigForAgentIsNotAMapping(String), // Contains the error from the constructor function ConstructionFailed(anyhow::Error), // Contains the error from the YAML deserialization function Yaml(serde_yaml_ng::Error), } pub type Result = std::result::Result; #[derive(Debug, Clone)] pub struct AgentInstance { identifier: PublicIdentifier, definition: AgentDefinition, controller: ControllerType, } impl AgentInstance { pub fn new( identifier: PublicIdentifier, definition: AgentDefinition, controller: ControllerType, ) -> Self { Self { identifier, definition, controller, } } pub fn identifier(&self) -> &PublicIdentifier { &self.identifier } pub fn definition(&self) -> &AgentDefinition { &self.definition } pub fn controller(&self) -> &ControllerType { &self.controller } } pub(super) fn create( identifier: PublicIdentifier, definition: AgentDefinition, ) -> Result { let controller = create_controller_from_provider_and_json_value_config( &definition.id, &definition.provider, definition.config.clone(), )?; Ok(AgentInstance::new(identifier, definition, controller)) } pub fn create_from_provider_and_yaml_value_config( provider: &AgentProvider, identifier: &PublicIdentifier, config: serde_yaml_ng::Value, ) -> Result { let definition = AgentDefinition::new(identifier.prefixless(), provider.to_owned(), config); create(identifier.to_owned(), definition) } fn create_controller_from_provider_and_json_value_config( agent_id: &str, provider: &AgentProvider, config: serde_yaml_ng::Value, ) -> Result { match provider { AgentProvider::Anthropic => { provider::anthropic::create_controller_from_yaml_value_config(agent_id, config) } AgentProvider::Groq => { provider::openai_compat::create_controller_from_yaml_value_config(agent_id, config) } AgentProvider::Mistral => { provider::openai_compat::create_controller_from_yaml_value_config(agent_id, config) } AgentProvider::LocalAI => { provider::openai_compat::create_controller_from_yaml_value_config(agent_id, config) } AgentProvider::Ollama => { provider::openai_compat::create_controller_from_yaml_value_config(agent_id, config) } AgentProvider::OpenAI => { provider::openai::create_controller_from_yaml_value_config(agent_id, config) } AgentProvider::OpenAICompat => { provider::openai_compat::create_controller_from_yaml_value_config(agent_id, config) } AgentProvider::OpenRouter => { provider::openai_compat::create_controller_from_yaml_value_config(agent_id, config) } AgentProvider::TogetherAI => { provider::openai_compat::create_controller_from_yaml_value_config(agent_id, config) } } } pub fn default_config_for_provider(provider: &AgentProvider) -> serde_yaml_ng::Value { match provider { AgentProvider::Anthropic => { let config = super::provider::anthropic::default_config(); serde_yaml_ng::to_value(config).expect("Failed to serialize config") } AgentProvider::Groq => { let config = super::provider::groq::default_config(); serde_yaml_ng::to_value(config).expect("Failed to serialize config") } AgentProvider::LocalAI => { let config = super::provider::localai::default_config(); serde_yaml_ng::to_value(config).expect("Failed to serialize config") } AgentProvider::Mistral => { let config = super::provider::mistral::default_config(); serde_yaml_ng::to_value(config).expect("Failed to serialize config") } AgentProvider::Ollama => { let config = super::provider::ollama::default_config(); serde_yaml_ng::to_value(config).expect("Failed to serialize config") } AgentProvider::OpenAI => { let config = super::provider::openai::default_config(); serde_yaml_ng::to_value(config).expect("Failed to serialize config") } AgentProvider::OpenAICompat => { let config = super::provider::openai_compat::default_config(); serde_yaml_ng::to_value(config).expect("Failed to serialize config") } AgentProvider::OpenRouter => { let config = super::provider::openrouter::default_config(); serde_yaml_ng::to_value(config).expect("Failed to serialize config") } AgentProvider::TogetherAI => { let config = super::provider::togetherai::default_config(); serde_yaml_ng::to_value(config).expect("Failed to serialize config") } } } ================================================ FILE: src/agent/manager.rs ================================================ use super::AgentDefinition; use super::PublicIdentifier; use super::instantiation; use super::instantiation::AgentInstance; use crate::entity::RoomConfigContext; #[derive(Debug)] pub struct Manager { static_agents: Vec, } impl Manager { pub fn new(static_agent_definitions: Vec) -> anyhow::Result { let mut static_agents = Vec::with_capacity(static_agent_definitions.len()); for definition in static_agent_definitions { let identifier = PublicIdentifier::Static(definition.id.clone()); match instantiation::create(identifier.clone(), definition.to_owned()) { Ok(instance) => static_agents.push(instance), Err(e) => { return Err(anyhow::anyhow!( "Failed to create static agent {}: {:?}", identifier, e )); } } } Ok(Self { static_agents }) } pub fn available_room_agents_by_room_config_context( &self, room_config_context: &RoomConfigContext, ) -> Vec { let mut agents: Vec = vec![]; for agent in &self.static_agents { agents.push(agent.clone()); } for definition in &room_config_context.global_config.agents { let identifier = PublicIdentifier::DynamicGlobal(definition.id.clone()); match instantiation::create(identifier.clone(), definition.to_owned()) { Ok(instance) => agents.push(instance), Err(e) => { tracing::warn!("Failed to create {} agent: {:?}. Skipping.", identifier, e); } } } for definition in &room_config_context.room_config.agents { let identifier = PublicIdentifier::DynamicRoomLocal(definition.id.clone()); match instantiation::create(identifier.clone(), definition.to_owned()) { Ok(instance) => agents.push(instance), Err(e) => { tracing::warn!("Failed to create {} agent: {:?}. Skipping.", identifier, e); } } } agents } } ================================================ FILE: src/agent/mod.rs ================================================ mod definition; mod identifier; mod instantiation; mod manager; pub mod provider; mod purpose; pub mod utils; pub use identifier::PublicIdentifier; pub use manager::Manager; pub use definition::AgentDefinition; pub use instantiation::AgentInstance; pub use instantiation::Error as AgentInstantiationError; pub use instantiation::Result as AgentInstantiationResult; pub use instantiation::create_from_provider_and_yaml_value_config; pub use instantiation::default_config_for_provider; pub use provider::{AgentProvider, AgentProviderInfo, ControllerTrait}; pub use purpose::AgentPurpose; pub(super) fn default_prompt() -> &'static str { "You are a brief, but helpful bot called {{ baibot_name }} powered by the {{ baibot_model_id }} model. The date/time of this conversation's start is: {{ baibot_conversation_start_time_utc }}." } ================================================ FILE: src/agent/provider/anthropic/config.rs ================================================ use serde::{Deserialize, Serialize}; use crate::agent::{default_prompt, provider::ConfigTrait}; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Config { pub base_url: String, pub api_key: String, pub text_generation: Option, } impl Default for Config { fn default() -> Self { Self { base_url: "https://api.anthropic.com/v1".to_owned(), api_key: "YOUR_API_KEY_HERE".to_owned(), text_generation: Some(TextGenerationConfig::default()), } } } impl ConfigTrait for Config { fn validate(&self) -> Result<(), String> { if self.base_url.is_empty() { return Err("The base URL must not be empty.".to_owned()); } if !self.base_url.ends_with("/v1") { return Err("The base URL must end with '/v1'.".to_owned()); } if self.api_key.is_empty() { return Err("The API key must not be empty.".to_owned()); } Ok(()) } } #[derive(Debug, Clone, Serialize, Deserialize)] pub struct TextGenerationConfig { #[serde(default = "default_text_model_id")] pub model_id: String, #[serde(default)] pub prompt: Option, #[serde(default = "super::super::default_temperature")] pub temperature: f32, #[serde(default)] pub max_response_tokens: u32, #[serde(default)] pub max_context_tokens: u32, } impl Default for TextGenerationConfig { fn default() -> Self { Self { model_id: default_text_model_id(), prompt: Some(default_prompt().to_owned()), temperature: super::super::default_temperature(), max_response_tokens: 8192, max_context_tokens: 204_800, } } } fn default_text_model_id() -> String { "claude-3-7-sonnet-20250219".to_owned() } ================================================ FILE: src/agent/provider/anthropic/controller.rs ================================================ use std::fmt::Debug; use std::sync::Arc; use anthropic::client::{Client, ClientBuilder}; use anthropic::types::ContentBlock; use super::super::ControllerTrait; use crate::agent::AgentPurpose; use crate::agent::provider::entity::{ ImageEditResult, ImageGenerationResult, ImageSource, PingResult, TextGenerationParams, TextGenerationResult, TextToSpeechParams, TextToSpeechResult, }; use crate::agent::provider::{ ImageEditParams, ImageGenerationParams, SpeechToTextParams, SpeechToTextResult, }; use crate::conversation::llm::{ Author as LLMAuthor, Conversation as LLMConversation, Message as LLMMessage, MessageContent as LLMMessageContent, shorten_messages_list_to_context_size, }; use crate::strings; use super::config::Config; struct ControllerInner { client: Client, } #[derive(Clone)] pub struct Controller { config: Config, inner: Arc, } impl Debug for Controller { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("Controller") .field("config", &self.config) .finish() } } impl Controller { pub fn new(config: Config) -> anyhow::Result { // The previous library that we used expected a base URL that ends with "/v1" // (e.g. "https://api.anthropic.com/v1"), while the new one doesn't. // // To keep backward compatibility, we don't ask people to change their configuration // and rather adapt by removing the "/v1" from the base URL. if !config.base_url.ends_with("/v1") { return Err(anyhow::anyhow!("base_url must end with '/v1'")); } let base_url = &config.base_url[..config.base_url.len() - 3]; let client = ClientBuilder::default() .api_base(base_url.to_string()) .api_key(config.api_key.clone()) .build()?; Ok(Self { config, inner: Arc::new(ControllerInner { client }), }) } } impl ControllerTrait for Controller { async fn ping(&self) -> anyhow::Result { if !self.supports_purpose(AgentPurpose::TextGeneration) { return Ok(PingResult::Inconclusive); } let messages = vec![LLMMessage { author: LLMAuthor::User, sender_id: None, content: LLMMessageContent::Text("Hello!".to_string()), timestamp: chrono::Utc::now(), }]; let conversation = LLMConversation { messages }; self.generate_text(conversation, TextGenerationParams::default()) .await?; Ok(PingResult::Successful) } async fn generate_text( &self, conversation: LLMConversation, params: TextGenerationParams, ) -> anyhow::Result { let Some(text_generation_config) = &self.config.text_generation else { return Err(anyhow::anyhow!( strings::agent::no_configuration_for_purpose_so_cannot_be_used( &AgentPurpose::TextGeneration ), )); }; let prompt_text = params.prompt_variables.format( params .prompt_override .unwrap_or(self.text_generation_prompt().unwrap_or("".to_owned())) .trim(), ); let prompt_message = if prompt_text.is_empty() { None } else { Some(LLMMessage { author: LLMAuthor::Prompt, sender_id: None, content: LLMMessageContent::Text(prompt_text), timestamp: chrono::Utc::now(), }) }; // Avoid the situation where multiple user or assistant messages are sent consecutively, // to avoid errors like: // > API error: Error response: error Api error: invalid_request_error messages: roles must alternate between "user" and "assistant", but found multiple "user" roles in a row // as reported here: https://github.com/etkecc/baibot/issues/13 // // As https://docs.anthropic.com/en/api/messages says: // > Our models are trained to operate on alternating user and assistant conversational turns. let conversation = conversation.combine_consecutive_messages(); let mut conversation_messages = conversation.messages; if params.context_management_enabled { tracing::trace!("Shortening messages list to context size"); conversation_messages = shorten_messages_list_to_context_size( &text_generation_config.model_id, &prompt_message, conversation_messages, Some(text_generation_config.max_response_tokens), text_generation_config.max_context_tokens, ); tracing::trace!("Finished shortening messages list to context size"); }; let messages_count = conversation_messages.len(); let mut request = super::utils::create_anthropic_message_request(conversation_messages); let temperature = params .temperature_override .unwrap_or(text_generation_config.temperature); if let Some(prompt_message) = prompt_message && let LLMMessageContent::Text(text) = &prompt_message.content { request.system = text.clone(); } request.model = text_generation_config.model_id.clone(); request.temperature = Some(temperature as f64); request.max_tokens = text_generation_config.max_response_tokens as usize; if let Ok(request_as_json) = serde_json::to_string(&request) { tracing::trace!( model = format!("{:?}", request.model), ?messages_count, request = request_as_json, "Sending Anthropic create message API request" ); } let response = self.inner.client.messages(request).await?; tracing::trace!(?response, "Got response from Anthropic create message API"); // response.content usually contains a single element, but we support handling multiple to account for all possibilities let mut text_parts = vec![]; for content in response.content { match content { ContentBlock::Text { text } => { text_parts.push(text); } ContentBlock::Image { .. } => { text_parts.push("The model responded with an image".to_string()); } } } if text_parts.is_empty() { return Err(anyhow::anyhow!( "No text content in response from the Anthropic create message API" )); } Ok(TextGenerationResult { text: text_parts.join("\n\n"), }) } async fn speech_to_text( &self, _mime_type: &mxlink::mime::Mime, _media: Vec, _params: SpeechToTextParams, ) -> anyhow::Result { Err(anyhow::anyhow!("Speech-to-Text not supported")) } async fn generate_image( &self, _prompt: &str, _params: ImageGenerationParams, ) -> anyhow::Result { Err(anyhow::anyhow!("Image generation not supported")) } async fn create_image_edit( &self, _prompt: &str, _images: Vec, _params: ImageEditParams, ) -> anyhow::Result { Err(anyhow::anyhow!("Image editing is not supported")) } async fn text_to_speech( &self, _input: &str, _params: TextToSpeechParams, ) -> anyhow::Result { Err(anyhow::anyhow!("Speech generation not supported")) } fn supports_purpose(&self, purpose: AgentPurpose) -> bool { match purpose { AgentPurpose::TextGeneration => self.config.text_generation.is_some(), AgentPurpose::SpeechToText => false, AgentPurpose::TextToSpeech => false, AgentPurpose::ImageGeneration => false, AgentPurpose::CatchAll => true, } } fn text_generation_model_id(&self) -> Option { self.config .text_generation .as_ref() .map(|config| config.model_id.to_owned()) } fn text_generation_prompt(&self) -> Option { self.config .text_generation .as_ref() .and_then(|config| config.prompt.clone()) } fn text_generation_temperature(&self) -> Option { self.config .text_generation .as_ref() .map(|config| config.temperature) } fn text_to_speech_voice(&self) -> Option { None } fn text_to_speech_speed(&self) -> Option { None } } ================================================ FILE: src/agent/provider/anthropic/mod.rs ================================================ mod config; mod controller; mod utils; pub use config::Config; pub use controller::Controller; use super::super::AgentInstantiationError; use super::super::AgentInstantiationResult; use super::ConfigTrait; use super::controller::ControllerType; pub fn create_controller_from_yaml_value_config( agent_id: &str, config: serde_yaml_ng::Value, ) -> AgentInstantiationResult { let config = match &config { serde_yaml_ng::Value::Mapping(_) => { let config: Config = serde_yaml_ng::from_value(config).map_err(AgentInstantiationError::Yaml)?; config .validate() .map_err(AgentInstantiationError::ConfigFailsValidation)?; config } _ => { return Err(AgentInstantiationError::ConfigForAgentIsNotAMapping( agent_id.to_owned(), )); } }; let controller = Controller::new(config).map_err(AgentInstantiationError::ConstructionFailed)?; Ok(ControllerType::Anthropic(Box::new(controller))) } pub fn default_config() -> Config { Config::default() } ================================================ FILE: src/agent/provider/anthropic/utils.rs ================================================ use anthropic::types::{ ContentBlock, ImageSource, Message, MessagesRequest, MessagesRequestBuilder, Role, }; use crate::conversation::llm::{ Author as LLMAuthor, Message as LLMMessage, MessageContent as LLMMessageContent, }; pub(super) fn create_anthropic_message_request(llm_messages: Vec) -> MessagesRequest { let mut messages = vec![]; for message in llm_messages { let role = match message.author { LLMAuthor::User => Role::User, LLMAuthor::Assistant => Role::Assistant, LLMAuthor::Prompt => { continue; } }; let content = match &message.content { LLMMessageContent::Text(text) => vec![ContentBlock::Text { text: text.clone() }], LLMMessageContent::Image(image_details) => { vec![ContentBlock::Image { source: ImageSource::Base64 { media_type: image_details.mime.to_string(), data: crate::utils::base64::base64_encode(&image_details.data), }, }] } LLMMessageContent::File(file_details) => { tracing::warn!( "The Anthropic provider's library does not support file/document content. This file message ({}) will be skipped.", file_details.filename(), ); continue; } }; let message = Message { role, content }; messages.push(message); } MessagesRequestBuilder::default() .messages(messages) .stream(false) .build() .expect("Failed to build messages request") } ================================================ FILE: src/agent/provider/config.rs ================================================ pub trait ConfigTrait { fn validate(&self) -> Result<(), String>; } ================================================ FILE: src/agent/provider/controller.rs ================================================ use crate::{agent::AgentPurpose, conversation::llm::Conversation}; use super::{ ImageEditParams, ImageGenerationParams, SpeechToTextParams, SpeechToTextResult, entity::{ ImageEditResult, ImageGenerationResult, ImageSource, PingResult, TextGenerationParams, TextGenerationResult, TextToSpeechParams, TextToSpeechResult, }, }; pub trait ControllerTrait { fn supports_purpose(&self, purpose: AgentPurpose) -> bool; fn ping(&self) -> impl std::future::Future> + Send; fn text_generation_model_id(&self) -> Option; fn text_generation_prompt(&self) -> Option; fn text_generation_temperature(&self) -> Option; fn text_to_speech_voice(&self) -> Option; fn text_to_speech_speed(&self) -> Option; fn generate_text( &self, conversation: Conversation, params: TextGenerationParams, ) -> impl std::future::Future> + Send; fn speech_to_text( &self, mime_type: &mxlink::mime::Mime, media: Vec, params: SpeechToTextParams, ) -> impl std::future::Future> + Send; fn generate_image( &self, prompt: &str, params: ImageGenerationParams, ) -> impl std::future::Future> + Send; fn create_image_edit( &self, prompt: &str, images: Vec, params: ImageEditParams, ) -> impl std::future::Future> + Send; fn text_to_speech( &self, text: &str, params: TextToSpeechParams, ) -> impl std::future::Future> + Send; } #[derive(Debug, Clone)] pub enum ControllerType { OpenAI(Box), OpenAICompat(Box), Anthropic(Box), } impl ControllerTrait for ControllerType { fn supports_purpose(&self, purpose: AgentPurpose) -> bool { match &self { ControllerType::OpenAI(controller) => controller.supports_purpose(purpose), ControllerType::OpenAICompat(controller) => controller.supports_purpose(purpose), ControllerType::Anthropic(controller) => controller.supports_purpose(purpose), } } fn text_generation_model_id(&self) -> Option { match &self { ControllerType::OpenAI(controller) => controller.text_generation_model_id(), ControllerType::OpenAICompat(controller) => controller.text_generation_model_id(), ControllerType::Anthropic(controller) => controller.text_generation_model_id(), } } fn text_generation_prompt(&self) -> Option { match &self { ControllerType::OpenAI(controller) => controller.text_generation_prompt(), ControllerType::OpenAICompat(controller) => controller.text_generation_prompt(), ControllerType::Anthropic(controller) => controller.text_generation_prompt(), } } fn text_to_speech_voice(&self) -> Option { match &self { ControllerType::OpenAI(controller) => controller.text_to_speech_voice(), ControllerType::OpenAICompat(controller) => controller.text_to_speech_voice(), ControllerType::Anthropic(controller) => controller.text_to_speech_voice(), } } fn text_to_speech_speed(&self) -> Option { match &self { ControllerType::OpenAI(controller) => controller.text_to_speech_speed(), ControllerType::OpenAICompat(controller) => controller.text_to_speech_speed(), ControllerType::Anthropic(controller) => controller.text_to_speech_speed(), } } fn text_generation_temperature(&self) -> Option { match &self { ControllerType::OpenAI(controller) => controller.text_generation_temperature(), ControllerType::OpenAICompat(controller) => controller.text_generation_temperature(), ControllerType::Anthropic(controller) => controller.text_generation_temperature(), } } async fn ping(&self) -> anyhow::Result { match &self { ControllerType::OpenAI(controller) => controller.ping().await, ControllerType::OpenAICompat(controller) => controller.ping().await, ControllerType::Anthropic(controller) => controller.ping().await, } } async fn generate_text( &self, conversation: Conversation, params: TextGenerationParams, ) -> anyhow::Result { match &self { ControllerType::OpenAI(controller) => { controller.generate_text(conversation, params).await } ControllerType::OpenAICompat(controller) => { controller.generate_text(conversation, params).await } ControllerType::Anthropic(controller) => { controller.generate_text(conversation, params).await } } } async fn speech_to_text( &self, mime_type: &mxlink::mime::Mime, media: Vec, params: SpeechToTextParams, ) -> anyhow::Result { match &self { ControllerType::OpenAI(controller) => { controller.speech_to_text(mime_type, media, params).await } ControllerType::OpenAICompat(controller) => { controller.speech_to_text(mime_type, media, params).await } ControllerType::Anthropic(controller) => { controller.speech_to_text(mime_type, media, params).await } } } async fn generate_image( &self, prompt: &str, params: ImageGenerationParams, ) -> anyhow::Result { match &self { ControllerType::OpenAI(controller) => controller.generate_image(prompt, params).await, ControllerType::OpenAICompat(controller) => { controller.generate_image(prompt, params).await } ControllerType::Anthropic(controller) => { controller.generate_image(prompt, params).await } } } async fn create_image_edit( &self, prompt: &str, images: Vec, params: ImageEditParams, ) -> anyhow::Result { match &self { ControllerType::OpenAI(controller) => { controller.create_image_edit(prompt, images, params).await } ControllerType::OpenAICompat(controller) => { controller.create_image_edit(prompt, images, params).await } ControllerType::Anthropic(controller) => { controller.create_image_edit(prompt, images, params).await } } } async fn text_to_speech( &self, text: &str, params: TextToSpeechParams, ) -> anyhow::Result { match &self { ControllerType::OpenAI(controller) => controller.text_to_speech(text, params).await, ControllerType::OpenAICompat(controller) => { controller.text_to_speech(text, params).await } ControllerType::Anthropic(controller) => controller.text_to_speech(text, params).await, } } } ================================================ FILE: src/agent/provider/entity/agent_provider.rs ================================================ use crate::agent::AgentPurpose; #[derive(Debug, Clone)] pub enum AgentProvider { Anthropic, Groq, LocalAI, Mistral, Ollama, OpenAI, OpenAICompat, OpenRouter, TogetherAI, } impl AgentProvider { pub fn choices() -> Vec<&'static Self> { vec![ &Self::Anthropic, &Self::Groq, &Self::LocalAI, &Self::Mistral, &Self::Ollama, &Self::OpenAI, &Self::OpenAICompat, &Self::OpenRouter, &Self::TogetherAI, ] } pub fn to_static_str(&self) -> &'static str { match &self { Self::Anthropic => "anthropic", Self::Groq => "groq", Self::LocalAI => "localai", Self::Mistral => "mistral", Self::Ollama => "ollama", Self::OpenAI => "openai", Self::OpenAICompat => "openai-compatible", Self::OpenRouter => "openrouter", Self::TogetherAI => "together-ai", } } pub fn from_string(s: &str) -> Result { match s { "anthropic" => Ok(Self::Anthropic), "groq" => Ok(Self::Groq), "localai" => Ok(Self::LocalAI), "mistral" => Ok(Self::Mistral), "ollama" => Ok(Self::Ollama), "openai" => Ok(Self::OpenAI), "openai-compatible" => Ok(Self::OpenAICompat), "openrouter" => Ok(Self::OpenRouter), "together-ai" => Ok(Self::TogetherAI), _ => Err("Unexpected string value"), } } pub fn info(&self) -> AgentProviderInfo { match &self { Self::Anthropic => AgentProviderInfo { id: Self::Anthropic.to_static_str(), name: "Anthropic", description: "Anthropic is an American AI company founded by former OpenAI engineers and providing powerful language models.", homepage_url: Some("https://www.anthropic.com/"), wiki_url: Some("https://en.wikipedia.org/wiki/Anthropic"), sign_up_url: Some("https://console.anthropic.com/"), models_list_url: Some("https://docs.anthropic.com/en/docs/about-claude/models"), supported_purposes: vec![AgentPurpose::TextGeneration], text_generation_supports_vision: true, text_generation_supports_tools: false, }, Self::Groq => AgentProviderInfo { id: Self::Groq.to_static_str(), name: "Groq", description: "Groq is an American company developing optimized Language Processing Units (LPU) and offering cloud service which runs various models (built by others) with very high performance.", homepage_url: Some("https://groq.com/"), wiki_url: Some("https://en.wikipedia.org/wiki/Groq"), sign_up_url: Some("https://console.groq.com/login"), models_list_url: Some("https://console.groq.com/docs/models"), supported_purposes: vec![AgentPurpose::TextGeneration, AgentPurpose::SpeechToText], text_generation_supports_vision: false, text_generation_supports_tools: false, }, Self::LocalAI => AgentProviderInfo { id: Self::LocalAI.to_static_str(), name: "LocalAI", description: "LocalAI is the free, Open Source OpenAI alternative. LocalAI act as a drop-in replacement REST API that's compatible with OpenAI API specifications for local inferencing. It allows you to run LLMs, generate images, audio (and not only) locally or on-prem with consumer grade hardware, supporting multiple model families and architectures.", homepage_url: Some("https://localai.io/"), wiki_url: None, sign_up_url: None, models_list_url: Some("https://localai.io/gallery.html"), supported_purposes: vec![ AgentPurpose::TextGeneration, AgentPurpose::TextToSpeech, AgentPurpose::SpeechToText, ], text_generation_supports_vision: false, text_generation_supports_tools: false, }, Self::Mistral => AgentProviderInfo { id: Self::Mistral.to_static_str(), name: "Mistral", description: "Mistral AI is a research lab based in Europe (France) which produces their own language models.", homepage_url: Some("https://mistral.ai/"), wiki_url: Some("https://en.wikipedia.org/wiki/Mistral_AI"), sign_up_url: Some("https://auth.mistral.ai/ui/registration"), models_list_url: Some("https://docs.mistral.ai/getting-started/models/"), supported_purposes: vec![AgentPurpose::TextGeneration], text_generation_supports_vision: false, text_generation_supports_tools: false, }, Self::Ollama => AgentProviderInfo { id: Self::Ollama.to_static_str(), name: "Ollama", description: "Ollama lets you run various models in a [self-hosted](https://github.com/ollama/ollama?tab=readme-ov-file#ollama) way. This is more advanced and requires powerful hardware for running some of the better models, but ensures your data stays with you.", homepage_url: Some("https://ollama.com/"), wiki_url: None, sign_up_url: None, models_list_url: Some("https://ollama.com/library"), supported_purposes: vec![AgentPurpose::TextGeneration], text_generation_supports_vision: false, text_generation_supports_tools: false, }, Self::OpenAI => AgentProviderInfo { id: Self::OpenAI.to_static_str(), name: "OpenAI", description: "OpenAI is an American AI company providing powerful language models.\n\nUse this provider either with the OpenAI API or with other OpenAI-compatible API services which **fully** adhere to the [OpenAI API spec](https://github.com/openai/openai-openapi/).\nFor services which are not fully compatible with the OpenAI API, consider using the **OpenAI Compatible** provider.", homepage_url: Some("https://openai.com/"), wiki_url: Some("https://en.wikipedia.org/wiki/OpenAI"), sign_up_url: Some("https://platform.openai.com/signup"), models_list_url: Some("https://platform.openai.com/docs/models"), supported_purposes: vec![ AgentPurpose::ImageGeneration, AgentPurpose::TextGeneration, AgentPurpose::TextToSpeech, AgentPurpose::SpeechToText, ], text_generation_supports_vision: true, text_generation_supports_tools: true, }, Self::OpenAICompat => AgentProviderInfo { id: Self::OpenAICompat.to_static_str(), name: "OpenAI Compatible", description: "This provider allows you to use OpenAI-compatible API services like [OpenRouter](https://openrouter.ai/), [Together AI](https://www.together.ai/), etc.\n\nSome of these popular services already have **shortcut** providers (leading to this one behind the scenes) - this make it easier to get started.\n\nThis provider just as featureful as the **OpenAI** provider, but is more compatible with services which do not fully adhere to the [OpenAI API spec](https://github.com/openai/openai-openapi/).", homepage_url: None, wiki_url: None, sign_up_url: None, models_list_url: None, supported_purposes: vec![ AgentPurpose::ImageGeneration, AgentPurpose::TextGeneration, AgentPurpose::TextToSpeech, AgentPurpose::SpeechToText, ], text_generation_supports_vision: false, text_generation_supports_tools: false, }, Self::OpenRouter => AgentProviderInfo { id: Self::OpenRouter.to_static_str(), name: "OpenRouter", description: "OpenRouter is a unified interface for LLMs. The platform scouts for the lowest prices and best latencies/throughputs across dozens of providers, and lets you choose how to [prioritize](https://openrouter.ai/docs/provider-routing) them.", homepage_url: Some("https://openrouter.ai/"), wiki_url: None, sign_up_url: Some("https://openrouter.ai/"), models_list_url: Some("https://openrouter.ai/models"), supported_purposes: vec![AgentPurpose::TextGeneration], text_generation_supports_vision: false, text_generation_supports_tools: false, }, Self::TogetherAI => AgentProviderInfo { id: Self::TogetherAI.to_static_str(), name: "Together AI", description: "Together AI makes it easy to run or [fine-tune](https://docs.together.ai/docs/fine-tuning-overview) leading open source models with only a few lines of code.", homepage_url: Some("https://www.together.ai/"), wiki_url: None, sign_up_url: Some("https://api.together.ai/signup"), models_list_url: Some("https://api.together.xyz/models"), supported_purposes: vec![AgentPurpose::TextGeneration], text_generation_supports_vision: false, text_generation_supports_tools: false, }, } } } impl std::fmt::Display for AgentProvider { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{}", self.to_static_str()) } } pub struct AgentProviderInfo { pub id: &'static str, pub name: &'static str, pub description: &'static str, pub homepage_url: Option<&'static str>, pub wiki_url: Option<&'static str>, pub sign_up_url: Option<&'static str>, pub models_list_url: Option<&'static str>, pub supported_purposes: Vec, pub text_generation_supports_vision: bool, pub text_generation_supports_tools: bool, } ================================================ FILE: src/agent/provider/entity/image.rs ================================================ use mxlink::mime; #[derive(Default)] pub struct ImageGenerationParams { pub smallest_size_possible: bool, pub cheaper_model_switching_allowed: bool, pub cheaper_quality_switching_allowed: bool, } impl ImageGenerationParams { pub fn with_smallest_size_possible(mut self, value: bool) -> Self { self.smallest_size_possible = value; self } pub fn with_cheaper_model_switching_allowed(mut self, value: bool) -> Self { self.cheaper_model_switching_allowed = value; self } pub fn with_cheaper_quality_switching_allowed(mut self, value: bool) -> Self { self.cheaper_quality_switching_allowed = value; self } } pub struct ImageGenerationResult { pub bytes: Vec, pub mime_type: mime::Mime, pub revised_prompt: Option, } #[derive(Default)] pub struct ImageEditParams {} pub struct ImageEditResult { pub bytes: Vec, pub mime_type: mime::Mime, } pub struct ImageSource { pub filename: String, pub bytes: Vec, pub mime_type: mime::Mime, } impl ImageSource { pub fn new(filename: String, bytes: Vec, mime_type: mime::Mime) -> Self { Self { filename, bytes, mime_type, } } } impl From for async_openai::types::images::ImageInput { fn from(value: ImageSource) -> Self { async_openai::types::images::ImageInput::from_vec_u8(value.filename, value.bytes) } } ================================================ FILE: src/agent/provider/entity/mod.rs ================================================ mod agent_provider; mod image; mod ping; mod speech_to_text; mod text_generation; mod text_to_speech; pub use agent_provider::{AgentProvider, AgentProviderInfo}; pub use image::{ ImageEditParams, ImageEditResult, ImageGenerationParams, ImageGenerationResult, ImageSource, }; pub use ping::PingResult; pub use speech_to_text::{SpeechToTextParams, SpeechToTextResult}; pub use text_generation::{ TextGenerationParams, TextGenerationPromptVariables, TextGenerationResult, }; pub use text_to_speech::{TextToSpeechParams, TextToSpeechResult}; ================================================ FILE: src/agent/provider/entity/ping.rs ================================================ pub enum PingResult { Inconclusive, Successful, } ================================================ FILE: src/agent/provider/entity/speech_to_text.rs ================================================ #[derive(Default)] pub struct SpeechToTextParams { pub language_override: Option, } pub struct SpeechToTextResult { pub text: String, } ================================================ FILE: src/agent/provider/entity/text_generation/mod.rs ================================================ mod prompt_variables; pub use prompt_variables::TextGenerationPromptVariables; #[derive(Default)] pub struct TextGenerationParams { pub context_management_enabled: bool, pub prompt_override: Option, pub temperature_override: Option, pub prompt_variables: TextGenerationPromptVariables, } pub struct TextGenerationResult { pub text: String, } ================================================ FILE: src/agent/provider/entity/text_generation/prompt_variables.rs ================================================ use chrono::{DateTime, Utc}; use std::collections::HashMap; pub struct TextGenerationPromptVariables { map: HashMap, } impl Default for TextGenerationPromptVariables { fn default() -> Self { let now = Utc::now(); Self::new("unnamed", "unknown-model", now, Some(now)) } } impl TextGenerationPromptVariables { pub fn new( bot_name: &str, model_id: &str, now_time: DateTime, conversation_start_time: Option>, ) -> Self { let mut map = HashMap::new(); map.insert("baibot_name".to_string(), bot_name.to_string()); map.insert("baibot_model_id".to_string(), model_id.to_string()); map.insert("baibot_now_utc".to_string(), format_utc_time(now_time)); let baibot_conversation_start_time_utc = match conversation_start_time { Some(conversation_start_time) => format_utc_time(conversation_start_time), None => "unknown".to_string(), }; map.insert( "baibot_conversation_start_time_utc".to_string(), baibot_conversation_start_time_utc, ); Self { map } } pub fn format(&self, text: &str) -> String { let mut formatted_text = text.to_string(); for (key, value) in &self.map { let placeholder = format!("{{{{ {} }}}}", key); formatted_text = formatted_text.replace(&placeholder, value); } formatted_text } } fn format_utc_time(time: DateTime) -> String { time.format("%Y-%m-%d (%A), %H:%M:%S UTC").to_string() } #[cfg(test)] mod tests { use super::*; use chrono::{TimeZone, Timelike}; #[test] fn test_new() { // Intentionally injecting some sub-seconds to ensure formatting would ignore them. let now_utc = Utc .with_ymd_and_hms(2024, 9, 20, 18, 34, 15) .unwrap() .with_nanosecond(250000000) .unwrap(); let conversation_start_time_utc = Utc .with_ymd_and_hms(2024, 9, 19, 18, 34, 15) .unwrap() .with_nanosecond(250000000) .unwrap(); let variables = TextGenerationPromptVariables::new( "baibot", "gpt-4o", now_utc, Some(conversation_start_time_utc), ); assert_eq!( variables.map.get("baibot_name"), Some(&"baibot".to_string()) ); assert_eq!( variables.map.get("baibot_model_id"), Some(&"gpt-4o".to_string()) ); assert_eq!( variables.map.get("baibot_now_utc"), Some(&format_utc_time(now_utc)) ); assert_eq!( variables.map.get("baibot_conversation_start_time_utc"), Some(&format_utc_time(conversation_start_time_utc)) ); let prompt = "Hello, I'm {{ baibot_name }} using {{ baibot_model_id }}. The date/time now is {{ baibot_now_utc }} and this conversation started at {{ baibot_conversation_start_time_utc }}."; let expected = "Hello, I'm baibot using gpt-4o. The date/time now is 2024-09-20 (Friday), 18:34:15 UTC and this conversation started at 2024-09-19 (Thursday), 18:34:15 UTC."; assert_eq!(variables.format(prompt), expected); } } ================================================ FILE: src/agent/provider/entity/text_to_speech.rs ================================================ #[derive(Default)] pub struct TextToSpeechParams { pub speed_override: Option, pub voice_override: Option, } pub struct TextToSpeechResult { pub bytes: Vec, pub mime_type: mxlink::mime::Mime, } ================================================ FILE: src/agent/provider/groq/mod.rs ================================================ // Groq is based on openai_compat, because it's not fully compatible with async-openai. use super::openai_compat::Config; pub fn default_config() -> Config { let mut config = Config { base_url: "https://api.groq.com/openai/v1".to_owned(), text_to_speech: None, image_generation: None, ..Default::default() }; if let Some(ref mut config) = config.text_generation.as_mut() { config.model_id = "llama3-70b-8192".to_owned(); config.max_context_tokens = 131_072; config.max_response_tokens = Some(4096); } if let Some(ref mut config) = config.speech_to_text.as_mut() { config.model_id = "whisper-large-v3".to_owned(); } config } ================================================ FILE: src/agent/provider/localai/mod.rs ================================================ // At the time of testing, LocalAI can be powered by `openai`, but we use `openai_compat` for better reliability // in the event of future updates to `async-openai`. use super::openai_compat::Config; pub fn default_config() -> Config { let mut config = Config { base_url: "http://my-localai-self-hosted-service:8080/v1".to_owned(), ..Default::default() }; if let Some(ref mut config) = config.text_generation.as_mut() { config.model_id = "gpt-4".to_owned(); config.max_context_tokens = 128_000; config.max_response_tokens = Some(4096); } if let Some(ref mut config) = config.text_to_speech.as_mut() { config.model_id = "tts-1".to_owned(); } if let Some(ref mut config) = config.speech_to_text.as_mut() { config.model_id = "whisper-1".to_owned(); } if let Some(ref mut config) = config.image_generation.as_mut() { config.model_id = "stablediffusion".to_owned(); } config } ================================================ FILE: src/agent/provider/mistral/mod.rs ================================================ // Mistral is based on openai_compat, because it's not fully compatible with async-openai. use super::openai_compat::Config; pub fn default_config() -> Config { let mut config = Config { base_url: "https://api.mistral.ai/v1".to_owned(), speech_to_text: None, text_to_speech: None, image_generation: None, ..Default::default() }; if let Some(ref mut config) = config.text_generation.as_mut() { config.model_id = "mistral-large-latest".to_owned(); config.max_context_tokens = 128_000; } config } ================================================ FILE: src/agent/provider/mod.rs ================================================ pub mod anthropic; mod config; mod controller; mod entity; pub(super) mod groq; pub mod localai; pub(super) mod mistral; pub mod ollama; pub mod openai; pub mod openai_compat; pub(super) mod openrouter; pub(super) mod togetherai; fn default_temperature() -> f32 { 1.0 } pub use controller::{ControllerTrait, ControllerType}; pub use config::ConfigTrait; pub use entity::{ AgentProvider, AgentProviderInfo, ImageEditParams, ImageGenerationParams, ImageSource, PingResult, SpeechToTextParams, SpeechToTextResult, TextGenerationParams, TextGenerationPromptVariables, TextToSpeechParams, }; ================================================ FILE: src/agent/provider/ollama/mod.rs ================================================ // At the time of testing, Ollama can be powered by `openai`, but we use `openai_compat` for better reliability // in the event of future updates to `async-openai`. use super::openai_compat::Config; pub fn default_config() -> Config { let mut config = Config { base_url: "http://my-ollama-self-hosted-service:11434/v1".to_owned(), text_to_speech: None, image_generation: None, speech_to_text: None, ..Default::default() }; if let Some(ref mut config) = config.text_generation.as_mut() { config.model_id = "gemma2:2b".to_owned(); config.max_context_tokens = 128_000; config.max_response_tokens = Some(4096); } config } ================================================ FILE: src/agent/provider/openai/config.rs ================================================ use serde::{Deserialize, Serialize}; use super::OPENAI_IMAGE_MODEL_GPT_IMAGE_1_DOT_5; use crate::agent::{default_prompt, provider::ConfigTrait}; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Config { pub base_url: String, pub api_key: String, #[serde(skip_serializing_if = "Option::is_none")] pub text_generation: Option, #[serde(skip_serializing_if = "Option::is_none")] pub speech_to_text: Option, #[serde(skip_serializing_if = "Option::is_none")] pub text_to_speech: Option, #[serde(skip_serializing_if = "Option::is_none")] pub image_generation: Option, } impl Default for Config { fn default() -> Self { Self { base_url: "https://api.openai.com/v1".to_owned(), api_key: "YOUR_API_KEY_HERE".to_owned(), text_generation: Some(TextGenerationConfig::default()), speech_to_text: Some(SpeechToTextConfig::default()), text_to_speech: Some(TextToSpeechConfig::default()), image_generation: Some(ImageGenerationConfig::default()), } } } impl ConfigTrait for Config { fn validate(&self) -> Result<(), String> { if self.base_url.is_empty() { return Err("The base URL must not be empty.".to_owned()); } Ok(()) } } #[derive(Debug, Clone, Serialize, Deserialize)] pub struct TextGenerationConfig { #[serde(default = "default_text_model_id")] pub model_id: String, #[serde(default)] pub prompt: Option, #[serde(default = "super::super::default_temperature")] pub temperature: f32, #[serde(default)] pub max_response_tokens: Option, #[serde(default)] pub max_completion_tokens: Option, #[serde(default)] pub max_context_tokens: u32, #[serde(default)] pub tools: ToolsConfig, } impl Default for TextGenerationConfig { fn default() -> Self { Self { model_id: default_text_model_id(), prompt: Some(default_prompt().to_owned()), temperature: super::super::default_temperature(), max_response_tokens: None, max_completion_tokens: Some(128_000), max_context_tokens: 400_000, tools: ToolsConfig::default(), } } } fn default_text_model_id() -> String { "gpt-5.4".to_owned() } #[derive(Debug, Clone, Serialize, Deserialize, Default)] pub struct ToolsConfig { #[serde(default)] pub web_search: bool, #[serde(default)] pub code_interpreter: bool, } #[derive(Debug, Clone, Serialize, Deserialize)] pub struct SpeechToTextConfig { #[serde(default = "default_speech_to_text_model_id")] pub model_id: String, } impl Default for SpeechToTextConfig { fn default() -> Self { Self { model_id: default_speech_to_text_model_id(), } } } fn default_speech_to_text_model_id() -> String { "whisper-1".to_owned() } #[derive(Debug, Clone, Serialize, Deserialize)] pub struct TextToSpeechConfig { #[serde(default = "default_text_to_speech_model_id")] pub model_id: async_openai::types::audio::SpeechModel, #[serde(default = "default_text_to_speech_voice")] pub voice: async_openai::types::audio::Voice, #[serde(default = "default_text_to_speech_speed")] pub speed: f32, #[serde(default = "default_text_to_speech_response_format")] pub response_format: async_openai::types::audio::SpeechResponseFormat, } impl Default for TextToSpeechConfig { fn default() -> Self { Self { model_id: default_text_to_speech_model_id(), voice: default_text_to_speech_voice(), speed: default_text_to_speech_speed(), response_format: default_text_to_speech_response_format(), } } } fn default_text_to_speech_model_id() -> async_openai::types::audio::SpeechModel { async_openai::types::audio::SpeechModel::Tts1Hd } fn default_text_to_speech_voice() -> async_openai::types::audio::Voice { async_openai::types::audio::Voice::Onyx } fn default_text_to_speech_speed() -> f32 { 1.0 } fn default_text_to_speech_response_format() -> async_openai::types::audio::SpeechResponseFormat { // The API defaults to mp3, but we prefer Opus because it's smaller. // Our clients should all have support for it. async_openai::types::audio::SpeechResponseFormat::Opus } #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ImageGenerationConfig { pub model_id: String, #[serde(default = "default_image_style")] pub style: Option, #[serde(default = "default_image_size")] pub size: Option, #[serde(default = "default_image_quality")] pub quality: Option, } impl Default for ImageGenerationConfig { fn default() -> Self { Self { model_id: OPENAI_IMAGE_MODEL_GPT_IMAGE_1_DOT_5.to_owned(), style: default_image_style(), size: default_image_size(), quality: default_image_quality(), } } } impl ImageGenerationConfig { pub fn model_id_as_openai_image_model( &self, ) -> Result { match self.model_id.as_str() { "dall-e-2" => Ok(async_openai::types::images::ImageModel::DallE2), "dall-e-3" => Ok(async_openai::types::images::ImageModel::DallE3), "gpt-image-1" => Ok(async_openai::types::images::ImageModel::GptImage1), "gpt-image-1.5" => Ok(async_openai::types::images::ImageModel::GptImage1dot5), "gpt-image-1-mini" => Ok(async_openai::types::images::ImageModel::GptImage1Mini), other => Ok(async_openai::types::images::ImageModel::Other( other.to_owned(), )), } } } fn default_image_style() -> Option { None } fn default_image_size() -> Option { None } fn default_image_quality() -> Option { None } ================================================ FILE: src/agent/provider/openai/controller.rs ================================================ use std::ops::Deref; use async_openai::{ Client as OpenAIClient, config::OpenAIConfig, types::{ audio::{AudioInput, CreateSpeechRequestArgs, CreateTranscriptionRequestArgs}, images::{ CreateImageEditRequestArgs, CreateImageRequestArgs, Image, ImageInput, ImageModel, ImageResponseFormat, }, responses::{ CodeInterpreterContainerAuto, CodeInterpreterTool, CodeInterpreterToolContainer, CreateResponseArgs, OutputItem, OutputMessageContent, Tool, WebSearchTool, }, }, }; use super::super::ControllerTrait; use crate::{ agent::provider::{ ImageEditParams, ImageGenerationParams, SpeechToTextParams, SpeechToTextResult, entity::{TextGenerationParams, TextGenerationResult}, }, conversation::llm::{ Author as LLMAuthor, Conversation as LLMConversation, Message as LLMMessage, MessageContent as LLMMessageContent, shorten_messages_list_to_context_size, }, utils::base64::base64_decode, }; use crate::{ agent::{ AgentPurpose, provider::entity::{ ImageEditResult, ImageGenerationResult, ImageSource, PingResult, TextToSpeechParams, TextToSpeechResult, }, }, strings, }; use super::config::Config; #[derive(Debug, Clone)] pub struct Controller { config: Config, client: OpenAIClient, } impl Controller { pub fn new(config: Config) -> Self { let openai_config = OpenAIConfig::new() .with_api_base(config.base_url.clone()) .with_api_key(config.api_key.clone()); let client = OpenAIClient::with_config(openai_config); Self { config, client } } } impl ControllerTrait for Controller { async fn ping(&self) -> anyhow::Result { if !self.supports_purpose(AgentPurpose::TextGeneration) { return Ok(PingResult::Inconclusive); } let messages = vec![LLMMessage { author: LLMAuthor::User, sender_id: None, content: LLMMessageContent::Text("Hello!".to_string()), timestamp: chrono::Utc::now(), }]; let conversation = LLMConversation { messages }; self.generate_text(conversation, TextGenerationParams::default()) .await?; Ok(PingResult::Successful) } async fn generate_text( &self, conversation: LLMConversation, params: TextGenerationParams, ) -> anyhow::Result { let Some(text_generation_config) = &self.config.text_generation else { return Err(anyhow::anyhow!( strings::agent::no_configuration_for_purpose_so_cannot_be_used( &AgentPurpose::TextGeneration ), )); }; let prompt_text = params.prompt_variables.format( params .prompt_override .unwrap_or(self.text_generation_prompt().unwrap_or("".to_owned())) .trim(), ); let prompt_message = if prompt_text.is_empty() { None } else { Some(LLMMessage { author: LLMAuthor::Prompt, sender_id: None, content: LLMMessageContent::Text(prompt_text), timestamp: chrono::Utc::now(), }) }; let mut conversation_messages = conversation.messages; if params.context_management_enabled { tracing::trace!("Shortening messages list to context size"); conversation_messages = shorten_messages_list_to_context_size( &text_generation_config.model_id, &prompt_message, conversation_messages, text_generation_config.max_response_tokens, text_generation_config.max_context_tokens, ); tracing::trace!("Finished shortening messages list to context size"); }; if let Some(prompt_message) = prompt_message { conversation_messages.insert(0, prompt_message); } let input = super::utils::convert_llm_messages_to_openai_response_input(conversation_messages); let messages_count = match &input { async_openai::types::responses::InputParam::Items(items) => items.len(), _ => 1, }; let temperature = params .temperature_override .unwrap_or(text_generation_config.temperature); let mut request_builder = CreateResponseArgs::default(); request_builder .model(&text_generation_config.model_id) .temperature(temperature) .input(input); let mut tools = Vec::new(); if text_generation_config.tools.web_search { tools.push(Tool::WebSearch(WebSearchTool::default())); } if text_generation_config.tools.code_interpreter { tools.push(Tool::CodeInterpreter(CodeInterpreterTool { container: CodeInterpreterToolContainer::Auto( CodeInterpreterContainerAuto::default(), ), })); } if !tools.is_empty() { request_builder.tools(tools); } if let Some(max_response_tokens) = text_generation_config.max_response_tokens { request_builder.max_output_tokens(max_response_tokens); } else if let Some(max_completion_tokens) = text_generation_config.max_completion_tokens { request_builder.max_output_tokens(max_completion_tokens); } let request = request_builder.build()?; if let Ok(request_as_json) = serde_json::to_string(&request) { tracing::trace!( model = format!("{:?}", request.model), ?messages_count, request = request_as_json, "Sending OpenAI response API request" ); } let response = self.client.responses().create(request).await?; tracing::trace!(?response, "Got response from the OpenAI response API"); for item in response.output { if let OutputItem::Message(message) = item { for content in message.content { if let OutputMessageContent::OutputText(text_content) = content { return Ok(TextGenerationResult { text: text_content.text, }); } } } } Err(anyhow::anyhow!( "No response messages choices were returned from the OpenAI response API" )) } async fn speech_to_text( &self, mime_type: &mxlink::mime::Mime, media: Vec, params: SpeechToTextParams, ) -> anyhow::Result { let Some(speech_to_text_config) = &self.config.speech_to_text else { return Err(anyhow::anyhow!( strings::agent::no_configuration_for_purpose_so_cannot_be_used( &AgentPurpose::SpeechToText ), )); }; let filename = audio_mime_type_to_file_name(mime_type).unwrap_or("audio.ogg".to_string()); let language = params.language_override.unwrap_or("".to_string()); let request = CreateTranscriptionRequestArgs::default() .model(&speech_to_text_config.model_id) .file(AudioInput::from_vec_u8(filename, media)) .language(language.clone()) .build()?; tracing::trace!( model_id = speech_to_text_config.model_id, ?language, "Sending OpenAI speech-to-text API request" ); let response = self.client.audio().transcription().create(request).await?; tracing::trace!( ?response, "Got response from the OpenAI audio transcription API" ); Ok(SpeechToTextResult { text: response.text, }) } async fn generate_image( &self, prompt: &str, params: ImageGenerationParams, ) -> anyhow::Result { let Some(image_generation_config) = &self.config.image_generation else { return Err(anyhow::anyhow!( strings::agent::no_configuration_for_purpose_so_cannot_be_used( &AgentPurpose::ImageGeneration ), )); }; let original_model = image_generation_config .model_id_as_openai_image_model() .map_err(|err| anyhow::anyhow!(err))?; let model = if params.cheaper_model_switching_allowed { // Switch to a cheaper model match original_model { ImageModel::DallE2 => ImageModel::DallE2, ImageModel::DallE3 => ImageModel::DallE2, ImageModel::GptImage1 => ImageModel::GptImage1Mini, ImageModel::GptImage1dot5 => ImageModel::GptImage1Mini, ImageModel::GptImage1Mini => ImageModel::GptImage1Mini, ImageModel::Other(_) => ImageModel::DallE2, } } else { original_model }; let quality = if params.cheaper_quality_switching_allowed { // Switch to a cheaper quality match &image_generation_config.quality { Some(quality) => match quality { async_openai::types::images::ImageQuality::Standard => { Some(async_openai::types::images::ImageQuality::Standard) } async_openai::types::images::ImageQuality::HD => { Some(async_openai::types::images::ImageQuality::Standard) } // New quality levels - keep as-is or downgrade to Standard async_openai::types::images::ImageQuality::High => { Some(async_openai::types::images::ImageQuality::Standard) } async_openai::types::images::ImageQuality::Medium => { Some(async_openai::types::images::ImageQuality::Medium) } async_openai::types::images::ImageQuality::Low => { Some(async_openai::types::images::ImageQuality::Low) } async_openai::types::images::ImageQuality::Auto => { Some(async_openai::types::images::ImageQuality::Auto) } }, None => None, } } else { image_generation_config.quality.clone() }; let size = if params.smallest_size_possible { Some(get_sticker_size(&model)) } else { image_generation_config.size }; let response_format = match model.clone() { ImageModel::DallE2 => Some(ImageResponseFormat::B64Json), ImageModel::DallE3 => Some(ImageResponseFormat::B64Json), // gpt-image-1 only outputs base64 and we don't need to specify the response format. // In fact, specifying the response format results in an error. ImageModel::GptImage1 => None, ImageModel::GptImage1Mini => None, ImageModel::GptImage1dot5 => None, ImageModel::Other(_) => Some(ImageResponseFormat::B64Json), }; let mut request_builder = CreateImageRequestArgs::default(); request_builder.model(model).prompt(prompt.to_owned()); if let Some(response_format) = response_format { request_builder.response_format(response_format); } if let Some(style) = &image_generation_config.style { request_builder.style(style.clone()); } if let Some(quality) = quality { request_builder.quality(quality.clone()); } if let Some(size) = size { request_builder.size(size); } let request = request_builder.build()?; tracing::trace!( ?prompt, model = format!("{:?}", request.model), size = format!("{:?}", request.size), style = format!("{:?}", request.style), quality = format!("{:?}", request.quality), "Sending OpenAI image generation API request" ); let response = self.client.images().generate(request).await?; if let Some(image) = response.data.into_iter().next() { match image.deref() { Image::B64Json { b64_json, revised_prompt, } => { let bytes = base64_decode(b64_json.as_ref())?; return Ok(ImageGenerationResult { bytes, mime_type: mxlink::mime::IMAGE_PNG, revised_prompt: revised_prompt.clone(), }); } _ => { return Err(anyhow::anyhow!("Unexpected image type")); } } } Err(anyhow::anyhow!( "The OpenAI image generation API returned no images" )) } async fn create_image_edit( &self, prompt: &str, images: Vec, _params: ImageEditParams, ) -> anyhow::Result { let Some(image_generation_config) = &self.config.image_generation else { return Err(anyhow::anyhow!( strings::agent::no_configuration_for_purpose_so_cannot_be_used( &AgentPurpose::ImageGeneration ), )); }; if images.is_empty() { return Err(anyhow::anyhow!("No image sources provided")); } let mut image_inputs: Vec = Vec::new(); for image in images { image_inputs.push(image.into()); } let dalle2_size = match image_generation_config.size { Some(async_openai::types::images::ImageSize::S256x256) => { Some(async_openai::types::images::ImageSize::S256x256) } Some(async_openai::types::images::ImageSize::S512x512) => { Some(async_openai::types::images::ImageSize::S512x512) } Some(async_openai::types::images::ImageSize::S1024x1024) => { Some(async_openai::types::images::ImageSize::S1024x1024) } _ => None, }; let model = image_generation_config .model_id_as_openai_image_model() .map_err(|err| anyhow::anyhow!(err))?; let response_format = match model.clone() { ImageModel::DallE2 => Some(ImageResponseFormat::B64Json), ImageModel::DallE3 => Some(ImageResponseFormat::B64Json), // gpt-image-1 only outputs base64 and we don't need to specify the response format. // In fact, specifying the response format results in an error. ImageModel::GptImage1 => None, ImageModel::GptImage1Mini => None, ImageModel::GptImage1dot5 => None, ImageModel::Other(_) => Some(ImageResponseFormat::B64Json), }; let mut request_builder = CreateImageEditRequestArgs::default(); request_builder .image(image_inputs) .prompt(prompt.to_owned()) .model(model); if let Some(size) = dalle2_size { request_builder.size(size); } if let Some(response_format) = response_format { request_builder.response_format(response_format); } let request = request_builder .build() .map_err(|e| anyhow::anyhow!("Failed to build CreateImageEditRequest: {}", e))?; tracing::trace!( model = format!("{:?}", request.model), size = format!("{:?}", request.size), response_format = format!("{:?}", request.response_format), "Sending OpenAI image edit API request" ); let response = self.client.images().edit(request).await?; if let Some(image_data) = response.data.into_iter().next() { match image_data.deref() { Image::B64Json { b64_json, .. } => { let bytes = base64_decode(b64_json.as_ref())?; return Ok(ImageEditResult { bytes, mime_type: mxlink::mime::IMAGE_PNG, }); } Image::Url { url, .. } => { tracing::warn!(?url, "Received URL instead of B64Json for image edit"); return Err(anyhow::anyhow!( "Unexpected image type (URL) when B64Json was requested" )); } } } Err(anyhow::anyhow!( "The OpenAI image edit API returned no images" )) } async fn text_to_speech( &self, input: &str, params: TextToSpeechParams, ) -> anyhow::Result { let Some(text_to_speech_config) = &self.config.text_to_speech else { return Err(anyhow::anyhow!( strings::agent::no_configuration_for_purpose_so_cannot_be_used( &AgentPurpose::TextToSpeech ), )); }; let speed = params.speed_override.unwrap_or(text_to_speech_config.speed); let voice = if let Some(voice_string) = params.voice_override { // This is a hacky way to construct a Voice enum from the string we have. let voice: serde_json::Result = serde_json::from_str(&format!("\"{}\"", voice_string)); match voice { Ok(voice) => voice, Err(err) => { tracing::debug!(?voice_string, ?err, "Failed to parse voice"); return Err(anyhow::anyhow!( "The configured voice ({}) is not supported.", voice_string )); } } } else { text_to_speech_config.voice.clone() }; let response_format = text_to_speech_config.response_format; let mime_type = response_format_to_mime_type(&response_format).unwrap_or( "audio/mp3" .parse() .expect("Failed parsing default mime type"), ); let request = CreateSpeechRequestArgs::default() .model(text_to_speech_config.model_id.clone()) .voice(voice) .speed(speed) .response_format(response_format) .input(input) .build()?; tracing::trace!( model = format!("{:?}", request.model), voice = format!("{:?}", request.voice), speed = format!("{:?}", request.speed), "Sending OpenAI text-to-speech API request" ); let result = self.client.audio().speech().create(request).await?; Ok(TextToSpeechResult { bytes: result.bytes.into(), mime_type, }) } fn supports_purpose(&self, purpose: AgentPurpose) -> bool { match purpose { AgentPurpose::TextGeneration => self.config.text_generation.is_some(), AgentPurpose::SpeechToText => self.config.speech_to_text.is_some(), AgentPurpose::TextToSpeech => self.config.text_to_speech.is_some(), AgentPurpose::ImageGeneration => self.config.image_generation.is_some(), AgentPurpose::CatchAll => true, } } fn text_generation_model_id(&self) -> Option { self.config .text_generation .as_ref() .map(|config| config.model_id.to_owned()) } fn text_generation_prompt(&self) -> Option { self.config .text_generation .as_ref() .and_then(|config| config.prompt.clone()) } fn text_generation_temperature(&self) -> Option { self.config .text_generation .as_ref() .map(|config| config.temperature) } fn text_to_speech_voice(&self) -> Option { let Some(text_to_speech_config) = &self.config.text_to_speech else { return None; }; // A hacky way to turn this enum to a string let voice_as_string = serde_json::to_string(&text_to_speech_config.voice).ok()?; Some(voice_as_string.replace("\"", "")) } fn text_to_speech_speed(&self) -> Option { let Some(text_to_speech_config) = &self.config.text_to_speech else { return None; }; Some(text_to_speech_config.speed) } } fn response_format_to_mime_type( response_format: &async_openai::types::audio::SpeechResponseFormat, ) -> Option { let content_type = match response_format { async_openai::types::audio::SpeechResponseFormat::Mp3 => "audio/mp3".to_owned(), async_openai::types::audio::SpeechResponseFormat::Wav => "audio/wav".to_owned(), async_openai::types::audio::SpeechResponseFormat::Opus => "audio/ogg".to_owned(), async_openai::types::audio::SpeechResponseFormat::Aac => "audio/aac".to_owned(), async_openai::types::audio::SpeechResponseFormat::Flac => "audio/flac".to_owned(), async_openai::types::audio::SpeechResponseFormat::Pcm => "audio/L8".to_owned(), }; match content_type.parse() { Ok(content_type) => Some(content_type), Err(err) => { tracing::error!(?err, "Failed to parse content type"); None } } } fn audio_mime_type_to_file_name(mime_type: &mxlink::mime::Mime) -> Option { let mime_type_string = mime_type.to_string(); let file_extension = match mime_type_string.as_str() { "audio/flac" => "flac", "audio/x-m4a" | "audio/m4a" => "m4a", "audio/mp3" | "audio/mpeg" => "mp3", "audio/mp4" => "mp4", "application/ogg" | "audio/ogg" => "ogg", "audio/wav" | "audio/x-wav" => "wav", "audio/webm" => "webm", _ => return None, }; Some(format!("audio.{}", file_extension)) } /// Returns the smallest supported size for stickers based on what the image model supports. fn get_sticker_size(model: &ImageModel) -> async_openai::types::images::ImageSize { use async_openai::types::images::ImageSize; match model { ImageModel::DallE2 => ImageSize::S256x256, ImageModel::DallE3 => ImageSize::S1024x1024, ImageModel::GptImage1 => ImageSize::S1024x1024, ImageModel::GptImage1Mini => ImageSize::S1024x1024, ImageModel::GptImage1dot5 => ImageSize::S1024x1024, ImageModel::Other(_) => ImageSize::S1024x1024, } } ================================================ FILE: src/agent/provider/openai/mod.rs ================================================ mod config; mod controller; mod utils; pub use config::Config; pub use controller::Controller; // openai_compat needs these, so it can convert from its own config types to these pub(super) use config::ImageGenerationConfig; pub(super) use config::SpeechToTextConfig; pub(super) use config::TextGenerationConfig; pub(super) use config::TextToSpeechConfig; use super::super::AgentInstantiationError; use super::super::AgentInstantiationResult; use super::ConfigTrait; use super::controller::ControllerType; pub const OPENAI_IMAGE_MODEL_GPT_IMAGE_1_DOT_5: &str = "gpt-image-1.5"; pub fn create_controller_from_yaml_value_config( agent_id: &str, config: serde_yaml_ng::Value, ) -> AgentInstantiationResult { let config = match &config { serde_yaml_ng::Value::Mapping(_) => { let config: Config = serde_yaml_ng::from_value(config).map_err(AgentInstantiationError::Yaml)?; config .validate() .map_err(AgentInstantiationError::ConfigFailsValidation)?; config } _ => { return Err(AgentInstantiationError::ConfigForAgentIsNotAMapping( agent_id.to_owned(), )); } }; Ok(ControllerType::OpenAI(Box::new(Controller::new(config)))) } pub fn default_config() -> Config { Config::default() } ================================================ FILE: src/agent/provider/openai/utils.rs ================================================ use async_openai::types::responses::{ EasyInputContent, EasyInputMessage, ImageDetail, InputContent, InputFileArgs, InputImageContent, InputItem, InputParam, MessageType, Role, }; use crate::conversation::llm::{ Author as LLMAuthor, Message as LLMMessage, MessageContent as LLMMessageContent, }; use crate::utils::base64::base64_encode; pub fn convert_llm_messages_to_openai_response_input( conversation_messages: Vec, ) -> InputParam { let mut items = Vec::with_capacity(conversation_messages.len()); for message in conversation_messages { let role = match message.author { LLMAuthor::Prompt => Role::System, LLMAuthor::Assistant => Role::Assistant, LLMAuthor::User => Role::User, }; let content = match message.content { LLMMessageContent::Text(text) => EasyInputContent::Text(text), LLMMessageContent::Image(image_details) => { let image_url = format!( "data:{};base64,{}", image_details.mime, base64_encode(&image_details.data) ); EasyInputContent::ContentList(vec![InputContent::InputImage(InputImageContent { image_url: Some(image_url), detail: ImageDetail::Auto, file_id: None, })]) } LLMMessageContent::File(file_details) => { let file_data = format!( "data:{};base64,{}", file_details.mime, base64_encode(&file_details.data) ); let file_content = InputFileArgs::default() .file_data(file_data) .filename(file_details.filename()) .build() .expect("Failed to build InputFileContent"); EasyInputContent::ContentList(vec![InputContent::InputFile(file_content)]) } }; items.push(InputItem::EasyMessage(EasyInputMessage { r#type: MessageType::Message, role, content, phase: None, })); } InputParam::Items(items) } ================================================ FILE: src/agent/provider/openai_compat/config.rs ================================================ use serde::{Deserialize, Serialize}; use crate::agent::default_prompt; use crate::agent::provider::openai::{ ImageGenerationConfig as OpenAIImageGenerationConfig, SpeechToTextConfig as OpenAISpeechToTextConfig, TextGenerationConfig as OpenAITextGenerationConfig, TextToSpeechConfig as OpenAITextToSpeechConfig, }; use crate::agent::provider::ConfigTrait; use super::utils::convert_string_to_enum; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Config { pub base_url: String, pub api_key: Option, #[serde(skip_serializing_if = "Option::is_none")] pub text_generation: Option, #[serde(skip_serializing_if = "Option::is_none")] pub speech_to_text: Option, #[serde(skip_serializing_if = "Option::is_none")] pub text_to_speech: Option, #[serde(skip_serializing_if = "Option::is_none")] pub image_generation: Option, } impl Default for Config { fn default() -> Self { Self { base_url: "".to_owned(), api_key: Some("YOUR_API_KEY_HERE".to_owned()), text_generation: Some(TextGenerationConfig::default()), speech_to_text: Some(SpeechToTextConfig::default()), text_to_speech: Some(TextToSpeechConfig::default()), image_generation: Some(ImageGenerationConfig::default()), } } } impl ConfigTrait for Config { fn validate(&self) -> Result<(), String> { if self.base_url.is_empty() { return Err("The base URL must not be empty.".to_owned()); } Ok(()) } } #[derive(Debug, Clone, Serialize, Deserialize)] pub struct TextGenerationConfig { #[serde(default = "default_text_model_id")] pub model_id: String, #[serde(default)] pub prompt: Option, #[serde(default = "super::super::default_temperature")] pub temperature: f32, #[serde(default)] pub max_response_tokens: Option, #[serde(default)] pub max_context_tokens: u32, } impl Default for TextGenerationConfig { fn default() -> Self { Self { model_id: default_text_model_id(), prompt: Some(default_prompt().to_owned()), temperature: super::super::default_temperature(), max_response_tokens: Some(4096), max_context_tokens: 128_000, } } } impl TryInto for TextGenerationConfig { type Error = anyhow::Error; fn try_into(self) -> Result { Ok(OpenAITextGenerationConfig { model_id: self.model_id, prompt: self.prompt, temperature: self.temperature, max_response_tokens: self.max_response_tokens, max_completion_tokens: None, max_context_tokens: self.max_context_tokens, tools: Default::default(), }) } } fn default_text_model_id() -> String { "some-model".to_owned() } #[derive(Debug, Clone, Serialize, Deserialize)] pub struct SpeechToTextConfig { #[serde(default = "default_speech_to_text_model_id")] pub model_id: String, } impl Default for SpeechToTextConfig { fn default() -> Self { Self { model_id: default_speech_to_text_model_id(), } } } impl TryInto for SpeechToTextConfig { type Error = anyhow::Error; fn try_into(self) -> Result { Ok(OpenAISpeechToTextConfig { model_id: self.model_id, }) } } fn default_speech_to_text_model_id() -> String { "whisper-1".to_owned() } #[derive(Debug, Clone, Serialize, Deserialize)] pub struct TextToSpeechConfig { #[serde(default = "default_text_to_speech_model_id")] pub model_id: String, #[serde(default = "default_text_to_speech_voice")] pub voice: String, #[serde(default = "default_text_to_speech_speed")] pub speed: f32, #[serde(default = "default_text_to_speech_response_format")] pub response_format: String, } impl Default for TextToSpeechConfig { fn default() -> Self { Self { model_id: default_text_to_speech_model_id(), voice: default_text_to_speech_voice(), speed: default_text_to_speech_speed(), response_format: default_text_to_speech_response_format(), } } } impl TryInto for TextToSpeechConfig { type Error = String; fn try_into(self) -> Result { let model_id = convert_string_to_enum::(&self.model_id)?; let voice = convert_string_to_enum::(&self.voice)?; let response_format = convert_string_to_enum::< async_openai::types::audio::SpeechResponseFormat, >(&self.response_format)?; Ok(OpenAITextToSpeechConfig { model_id, voice, speed: self.speed, response_format, }) } } fn default_text_to_speech_model_id() -> String { "tts-1".to_owned() } fn default_text_to_speech_voice() -> String { "onyx".to_owned() } fn default_text_to_speech_speed() -> f32 { 1.0 } fn default_text_to_speech_response_format() -> String { "opus".to_owned() } #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ImageGenerationConfig { pub model_id: String, #[serde(default = "default_image_style")] pub style: Option, #[serde(default = "default_image_size")] pub size: Option, #[serde(default = "default_image_quality")] pub quality: Option, } impl Default for ImageGenerationConfig { fn default() -> Self { Self { model_id: "stablediffusion".to_owned(), style: default_image_style(), size: default_image_size(), quality: default_image_quality(), } } } impl TryInto for ImageGenerationConfig { type Error = String; fn try_into(self) -> Result { let size = if let Some(size) = &self.size { Some(convert_string_to_enum::< async_openai::types::images::ImageSize, >(size)?) } else { None }; let style = if let Some(style) = &self.style { Some(convert_string_to_enum::< async_openai::types::images::ImageStyle, >(style)?) } else { None }; let quality = if let Some(quality) = &self.quality { Some(convert_string_to_enum::< async_openai::types::images::ImageQuality, >(quality)?) } else { None }; Ok(OpenAIImageGenerationConfig { model_id: self.model_id, style, size, quality, }) } } fn default_image_style() -> Option { Some("vivid".to_owned()) } fn default_image_size() -> Option { Some("1024x1024".to_owned()) } fn default_image_quality() -> Option { Some("standard".to_owned()) } ================================================ FILE: src/agent/provider/openai_compat/controller.rs ================================================ use etke_openai_api_rust::audio::{AudioApi, AudioBody}; use etke_openai_api_rust::chat::{ChatApi, ChatBody}; use etke_openai_api_rust::images::{ImagesApi, ImagesBody}; use etke_openai_api_rust::{Auth, Message, OpenAI}; const SMALLEST_IMAGE_SIZE: &str = "256x256"; use super::super::ControllerTrait; use crate::utils::base64::base64_decode; use crate::{ agent::provider::{ ImageEditParams, ImageGenerationParams, ImageSource, SpeechToTextParams, SpeechToTextResult, entity::{TextGenerationParams, TextGenerationResult}, }, conversation::llm::{ Author as LLMAuthor, Conversation as LLMConversation, Message as LLMMessage, MessageContent as LLMMessageContent, shorten_messages_list_to_context_size, }, }; use crate::{ agent::{ AgentPurpose, provider::entity::{ ImageEditResult, ImageGenerationResult, PingResult, TextToSpeechParams, TextToSpeechResult, }, }, strings, }; use super::Config; #[derive(Debug, Clone)] pub struct Controller { config: Config, client: OpenAI, } impl Controller { pub fn new(config: Config) -> Self { let api_key = config.api_key.clone().unwrap_or("".to_owned()); let auth = Auth::new(&api_key); // The library we use chokes if there's no trailing slash let base_url = if config.base_url.ends_with("/") { config.base_url.clone() } else { format!("{}/", config.base_url) }; let client = OpenAI::new(auth, &base_url); Self { config, client } } } impl ControllerTrait for Controller { async fn ping(&self) -> anyhow::Result { if !self.supports_purpose(AgentPurpose::TextGeneration) { return Ok(PingResult::Inconclusive); } let messages = vec![LLMMessage { author: LLMAuthor::User, sender_id: None, content: LLMMessageContent::Text("Hello!".to_string()), timestamp: chrono::Utc::now(), }]; let conversation = LLMConversation { messages }; self.generate_text(conversation, TextGenerationParams::default()) .await?; Ok(PingResult::Successful) } async fn generate_text( &self, conversation: LLMConversation, params: TextGenerationParams, ) -> anyhow::Result { let Some(text_generation_config) = &self.config.text_generation else { return Err(anyhow::anyhow!( strings::agent::no_configuration_for_purpose_so_cannot_be_used( &AgentPurpose::TextGeneration ), )); }; let prompt_text = params.prompt_variables.format( params .prompt_override .unwrap_or(self.text_generation_prompt().unwrap_or("".to_owned())) .trim(), ); let prompt_message = if prompt_text.is_empty() { None } else { Some(LLMMessage { author: LLMAuthor::Prompt, sender_id: None, content: LLMMessageContent::Text(prompt_text), timestamp: chrono::Utc::now(), }) }; let mut conversation_messages = conversation.messages; if params.context_management_enabled { tracing::trace!("Shortening messages list to context size"); conversation_messages = shorten_messages_list_to_context_size( &text_generation_config.model_id, &prompt_message, conversation_messages, text_generation_config.max_response_tokens, text_generation_config.max_context_tokens, ); tracing::trace!("Finished shortening messages list to context size"); }; if let Some(prompt_message) = prompt_message { conversation_messages.insert(0, prompt_message); } let openai_conversation_messages: Vec = super::utils::convert_llm_messages_to_openai_messages(conversation_messages); let messages_count = openai_conversation_messages.len(); let temperature = params .temperature_override .unwrap_or(text_generation_config.temperature); let max_tokens = text_generation_config .max_response_tokens .map(|max_response_tokens| { max_response_tokens .try_into() .expect("Failed converting max_response_tokens from u32 to i32") }); let request = ChatBody { model: text_generation_config.model_id.clone(), max_tokens, temperature: Some(temperature), top_p: None, n: Some(1), stream: Some(false), stop: None, presence_penalty: None, frequency_penalty: None, logit_bias: None, user: None, messages: openai_conversation_messages, }; if let Ok(request_as_json) = serde_json::to_string(&request) { tracing::trace!( model = format!("{:?}", request.model), ?messages_count, request = request_as_json, "Sending OpenAI-compat chat completion API request" ); } // This library is not async-aware, so we need to use `spawn_blocking` to run the request on a separate thread. let client = self.client.clone(); let response = tokio::task::spawn_blocking(move || client.chat_completion_create(&request)).await?; let response = match response { Ok(response) => response, Err(err) => { return Err(anyhow::anyhow!( "Failed to get response from the OpenAI-compat chat completion API: {:?}", err )); } }; tracing::trace!( ?response, "Got response from the OpenAI-compat chat completion API" ); // We only request 1 result, so there should only be 1 choice. if let Some(choice) = response.choices.into_iter().next() { let Some(message) = choice.message else { return Err(anyhow::anyhow!( "No response message in choice was returned from the OpenAI-compat chat completion API" )); }; return Ok(TextGenerationResult { text: message.content, }); } Err(anyhow::anyhow!( "No response messages choices were returned from the OpenAI-compat chat completion API" )) } async fn speech_to_text( &self, _mime_type: &mxlink::mime::Mime, media: Vec, params: SpeechToTextParams, ) -> anyhow::Result { let Some(speech_to_text_config) = &self.config.speech_to_text else { return Err(anyhow::anyhow!( strings::agent::no_configuration_for_purpose_so_cannot_be_used( &AgentPurpose::SpeechToText ), )); }; // This library does not support passing the audio data as a byte slice, so we need to write it to a temporary file :/ // // This temporary file will get auto-deleted when the variable goes out of scope. let temp_file = tokio::task::spawn_blocking(move || { let mut temp_file = match tempfile::NamedTempFile::new() { Ok(file) => file, Err(e) => return Err(e), }; match std::io::Write::write_all(&mut temp_file, &media) { Ok(_) => (), Err(e) => return Err(e), } Ok(temp_file) }) .await??; let file_path = temp_file .path() .to_str() .ok_or_else(|| anyhow::anyhow!("Failed to get temporary file path"))?; let language = params.language_override.clone(); let request = AudioBody { file: std::fs::File::open(file_path)?, model: speech_to_text_config.model_id.to_owned(), prompt: None, response_format: None, temperature: None, language: language.clone(), }; tracing::trace!( model_id = speech_to_text_config.model_id, ?language, "Sending OpenAI-compat speech-to-text API request" ); // This library is not async-aware, so we need to use `spawn_blocking` to run the request on a separate thread. let client = self.client.clone(); let response = tokio::task::spawn_blocking(move || client.audio_transcription_create(request)).await?; let response = match response { Ok(response) => response, Err(err) => { return Err(anyhow::anyhow!( "Failed to get response from the OpenAI-compat audio transcription API: {:?}", err )); } }; tracing::trace!( ?response, "Got response from the OpenAI-compat audio transcription API" ); let Some(text) = response.text else { return Err(anyhow::anyhow!( "No response text was returned from the OpenAI-compat audio transcription API" )); }; Ok(SpeechToTextResult { text }) } async fn generate_image( &self, prompt: &str, params: ImageGenerationParams, ) -> anyhow::Result { let Some(image_generation_config) = &self.config.image_generation else { return Err(anyhow::anyhow!( strings::agent::no_configuration_for_purpose_so_cannot_be_used( &AgentPurpose::ImageGeneration ), )); }; // It seems like some OpenAI-compatible providers (e.g. LocalAI with StableDiffusion) skip some requirements // when they span multiple lines. let prompt = prompt.replace("\n", " "); let size: Option = if params.smallest_size_possible { Some(SMALLEST_IMAGE_SIZE.to_owned()) } else { image_generation_config.size.clone() }; let request = ImagesBody { model: Some(image_generation_config.model_id.to_owned()), prompt: prompt.to_owned(), n: Some(1), quality: image_generation_config.quality.clone(), size, style: image_generation_config.style.clone(), response_format: Some("b64_json".to_string()), user: None, }; tracing::trace!( ?prompt, model = format!("{:?}", request.model), size = format!("{:?}", request.size), style = format!("{:?}", request.style), quality = format!("{:?}", request.quality), "Sending OpenAI-compat image generation API request" ); // This library is not async-aware, so we need to use `spawn_blocking` to run the request on a separate thread. let client = self.client.clone(); let response = tokio::task::spawn_blocking(move || client.image_create(&request)).await?; let response = match response { Ok(response) => response, Err(err) => { return Err(anyhow::anyhow!( "Failed to get response from the OpenAI-compat image creation API: {:?}", err )); } }; let Some(data) = response.data else { return Err(anyhow::anyhow!( "The OpenAI-compat image generationAPI returned no image data" )); }; if let Some(image) = data.into_iter().next() { let Some(b64_json) = &image.b64_json else { return Err(anyhow::anyhow!( "The OpenAI-compat image generation API returned no b64_json image data" )); }; let bytes = base64_decode(b64_json)?; return Ok(ImageGenerationResult { bytes, mime_type: mxlink::mime::IMAGE_PNG, revised_prompt: image.revised_prompt, }); } Err(anyhow::anyhow!( "The OpenAI image generation API returned no images" )) } async fn create_image_edit( &self, _prompt: &str, _images: Vec, _params: ImageEditParams, ) -> anyhow::Result { Err(anyhow::anyhow!( "The OpenAI image edit API is not supported by the OpenAI-compat provider" )) } async fn text_to_speech( &self, input: &str, params: TextToSpeechParams, ) -> anyhow::Result { // openai_api_rust does not support text-to-speech, so our only bet is to do it via async-openai and hope it works. // At the time of testing (2024-09-09), providers like LocalAI can be used for text-to-speech via async-openai. // // So.. below we try to convert our Config struct to the Config struct from the openai module // and invoke the openai controller. // Quick check to make sure doing work below is worth it let Some(_text_to_speech_config) = &self.config.text_to_speech else { return Err(anyhow::anyhow!( strings::agent::no_configuration_for_purpose_so_cannot_be_used( &AgentPurpose::TextToSpeech ), )); }; tracing::debug!("Converting OpenAI-compact config to OpenAI config.."); let openai_config = super::utils::convert_config_to_openai_config_lossy(&self.config); let Some(_text_to_speech_config) = &openai_config.text_to_speech else { return Err(anyhow::anyhow!( strings::agent::no_configuration_for_purpose_after_conversion_so_cannot_be_used( &AgentPurpose::TextToSpeech ), )); }; let openai_controller = super::super::openai::Controller::new(openai_config); tracing::error!("Invoking text-to-speech via the OpenAI controller.."); openai_controller.text_to_speech(input, params).await } fn supports_purpose(&self, purpose: AgentPurpose) -> bool { match purpose { AgentPurpose::ImageGeneration => self.config.image_generation.is_some(), AgentPurpose::TextGeneration => self.config.text_generation.is_some(), AgentPurpose::SpeechToText => self.config.speech_to_text.is_some(), AgentPurpose::TextToSpeech => self.config.text_to_speech.is_some(), AgentPurpose::CatchAll => true, } } fn text_generation_model_id(&self) -> Option { self.config .text_generation .as_ref() .map(|config| config.model_id.to_owned()) } fn text_generation_prompt(&self) -> Option { self.config .text_generation .as_ref() .and_then(|config| config.prompt.clone()) } fn text_generation_temperature(&self) -> Option { self.config .text_generation .as_ref() .map(|config| config.temperature) } fn text_to_speech_voice(&self) -> Option { let Some(text_to_speech_config) = &self.config.text_to_speech else { return None; }; // A hacky way to turn this enum to a string let voice_as_string = serde_json::to_string(&text_to_speech_config.voice).ok()?; Some(voice_as_string.replace("\"", "")) } fn text_to_speech_speed(&self) -> Option { let Some(text_to_speech_config) = &self.config.text_to_speech else { return None; }; Some(text_to_speech_config.speed) } } ================================================ FILE: src/agent/provider/openai_compat/mod.rs ================================================ // The openai_compat provider aims to support a wider ranger of OpenAI-compatible providers. // // The `openai` provider is based on `async-openai`, which only aims to support the OpenAI API spec. See: // - https://github.com/64bit/async-openai/issues/266 // - https://github.com/64bit/async-openai/blob/05d5a1b4fa6476829dd1a34447b80279cf89d4f8/async-openai/README.md#contributing // // This module uses its own configuration, which avoids using strict types tied to OpenAI, // and thus allows for more flexibility. // // Communication with the OpenAI-compatible API is handled by the `openai_api_rust` crate. // Since this crate is not async-aware, we need to use tokio's `spawn_blocking` to invoke it. // // Certain features (e.g. text-to-speech) are not supported by `openai_api_rust` yet, so we may try to delegate them to the `openai` provider. mod config; mod controller; mod utils; pub use config::Config; pub use controller::Controller; use super::super::AgentInstantiationError; use super::super::AgentInstantiationResult; use super::ConfigTrait; use super::controller::ControllerType; pub fn create_controller_from_yaml_value_config( agent_id: &str, config: serde_yaml_ng::Value, ) -> AgentInstantiationResult { let config = match &config { serde_yaml_ng::Value::Mapping(_) => { let config: Config = serde_yaml_ng::from_value(config).map_err(AgentInstantiationError::Yaml)?; config .validate() .map_err(AgentInstantiationError::ConfigFailsValidation)?; config } _ => { return Err(AgentInstantiationError::ConfigForAgentIsNotAMapping( agent_id.to_owned(), )); } }; Ok(ControllerType::OpenAICompat(Box::new(Controller::new( config, )))) } pub fn default_config() -> Config { let mut config = Config::default(); if let Some(text_generation) = &mut config.text_generation { text_generation.model_id = "some-model".to_string(); text_generation.max_response_tokens = Some(4096); text_generation.max_context_tokens = 128_000; } // We don't support these, so let's remove them from the configuration. config.text_to_speech = None; config.image_generation = None; config.base_url = "".to_owned(); config } ================================================ FILE: src/agent/provider/openai_compat/utils.rs ================================================ use etke_openai_api_rust::{Message, Role}; use crate::agent::provider::openai::Config as OpenAIConfig; use crate::conversation::llm::{ Author as LLMAuthor, Message as LLMMessage, MessageContent as LLMMessageContent, }; pub fn convert_llm_messages_to_openai_messages( conversation_messages: Vec, ) -> Vec { let mut openai_conversation_messages: Vec = Vec::with_capacity(conversation_messages.len()); for message in conversation_messages { let openai_message = convert_llm_message_to_openai_message(message); if let Some(openai_message) = openai_message { openai_conversation_messages.push(openai_message); } } openai_conversation_messages } fn convert_llm_message_to_openai_message(llm_message: LLMMessage) -> Option { let role = match llm_message.author { LLMAuthor::Prompt => Role::System, LLMAuthor::Assistant => Role::Assistant, LLMAuthor::User => Role::User, }; match &llm_message.content { LLMMessageContent::Text(text) => Some(Message { role, content: text.clone(), }), LLMMessageContent::Image(_image_details) => { tracing::warn!( "The OpenAI-compat provider's library does not support image content. This image message will be skipped." ); None } LLMMessageContent::File(_file_details) => { tracing::warn!( "The OpenAI-compat provider's library does not support file content. This file message will be skipped." ); None } } } pub(super) fn convert_config_to_openai_config_lossy(config: &super::Config) -> OpenAIConfig { let text_generation = config .text_generation .as_ref() .and_then(|tg| tg.clone().try_into().ok()); let speech_to_text = config .speech_to_text .as_ref() .and_then(|stt| stt.clone().try_into().ok()); let text_to_speech = config .text_to_speech .as_ref() .and_then(|tts| tts.clone().try_into().ok()); let image_generation = config .image_generation .as_ref() .and_then(|ig| ig.clone().try_into().ok()); OpenAIConfig { api_key: config.api_key.clone().unwrap_or("".to_string()), text_generation, speech_to_text, text_to_speech, image_generation, base_url: config.base_url.clone(), } } pub(super) fn convert_string_to_enum(value: &str) -> Result where T: serde::de::DeserializeOwned, { // This is a hacky way to construct an enum from the string we have. let enum_result: serde_json::Result = serde_json::from_str(&format!("\"{}\"", value)); match enum_result { Ok(enum_result) => Ok(enum_result), Err(err) => { tracing::debug!(?err, "Failed to parse into enum"); Err(format!("The value ({}) is not supported.", value)) } } } ================================================ FILE: src/agent/provider/openrouter/mod.rs ================================================ use super::openai_compat::Config; pub fn default_config() -> Config { let mut config = Config { base_url: "https://openrouter.ai/api/v1".to_owned(), text_to_speech: None, image_generation: None, speech_to_text: None, ..Default::default() }; if let Some(ref mut config) = config.text_generation.as_mut() { config.model_id = "mattshumer/reflection-70b:free".to_owned(); config.max_context_tokens = 8192; config.max_response_tokens = Some(2048); } config } ================================================ FILE: src/agent/provider/togetherai/mod.rs ================================================ use super::openai_compat::Config; pub fn default_config() -> Config { let mut config = Config { base_url: "https://api.together.xyz/v1".to_owned(), text_to_speech: None, image_generation: None, speech_to_text: None, ..Default::default() }; if let Some(ref mut config) = config.text_generation.as_mut() { config.model_id = "meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo".to_owned(); config.max_context_tokens = 8192; config.max_response_tokens = Some(2048); } config } ================================================ FILE: src/agent/purpose.rs ================================================ #[derive(Debug, Clone, Copy, PartialEq)] pub enum AgentPurpose { CatchAll, ImageGeneration, TextGeneration, TextToSpeech, SpeechToText, } impl AgentPurpose { pub fn from_str(s: &str) -> Option { match s { "catch-all" => Some(Self::CatchAll), "image-generation" => Some(Self::ImageGeneration), "text-generation" => Some(Self::TextGeneration), "text-to-speech" => Some(Self::TextToSpeech), "speech-to-text" => Some(Self::SpeechToText), _ => None, } } pub fn as_str(&self) -> &'static str { match self { Self::CatchAll => "catch-all", Self::ImageGeneration => "image-generation", Self::TextGeneration => "text-generation", Self::TextToSpeech => "text-to-speech", Self::SpeechToText => "speech-to-text", } } pub fn choices() -> Vec<&'static Self> { vec![ &Self::TextGeneration, &Self::SpeechToText, &Self::TextToSpeech, &Self::ImageGeneration, &Self::CatchAll, ] } pub fn emoji(&self) -> &'static str { match self { Self::CatchAll => "❓", Self::TextGeneration => "💬", Self::SpeechToText => "🦻", Self::TextToSpeech => "🗣️", Self::ImageGeneration => "🖌️", } } pub fn heading(&self) -> &'static str { match self { Self::CatchAll => "Catch-All", Self::TextGeneration => "Text Generation", Self::SpeechToText => "Speech-to-Text", Self::TextToSpeech => "Text-to-Speech", Self::ImageGeneration => "Image Generation", } } } impl std::fmt::Display for AgentPurpose { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str(self.as_str()) } } ================================================ FILE: src/agent/utils.rs ================================================ use crate::{ agent::{ AgentInstance, AgentPurpose, ControllerTrait, Manager as AgentManager, PublicIdentifier, }, entity::RoomConfigContext, strings, }; #[derive(Debug)] pub struct AgentForPurposeDeterminationInfo { pub instance: AgentInstance, pub configuration_source: AgentForPurposeDeterminationInfoConfigurationSource, } #[derive(Debug)] pub enum AgentForPurposeDeterminationInfoConfigurationSource { Room, Global, } #[derive(Debug)] pub enum AgentForPurposeDeterminationError { Unknown(String), NoneConfigured, ConfiguredButMissing(PublicIdentifier), ConfiguredButLacksSupport(PublicIdentifier), } pub async fn get_effective_agent_for_purpose( agent_manager: &AgentManager, room_config_context: &RoomConfigContext, agent_purpose: AgentPurpose, ) -> Result { let (agent_identifier, configuration_source) = match get_effective_room_agent_identifier_for_purpose(room_config_context, agent_purpose) .await { Ok((agent_identifier, configuration_source)) => { (agent_identifier, configuration_source) } Err(err) => { return Err(AgentForPurposeDeterminationError::Unknown(err)); } }; let Some(agent_identifier) = agent_identifier else { return Err(AgentForPurposeDeterminationError::NoneConfigured); }; let agents = agent_manager.available_room_agents_by_room_config_context(room_config_context); let Some(agent_instance) = agents.iter().find(|a| *a.identifier() == agent_identifier) else { return Err(AgentForPurposeDeterminationError::ConfiguredButMissing( agent_identifier, )); }; let agent_instance = agent_instance.clone(); let supports_purpose = agent_instance.controller().supports_purpose(agent_purpose); if !supports_purpose { return Err(AgentForPurposeDeterminationError::ConfiguredButLacksSupport(agent_identifier)); } Ok(AgentForPurposeDeterminationInfo { instance: agent_instance, configuration_source, }) } async fn get_effective_room_agent_identifier_for_purpose( room_config_context: &RoomConfigContext, purpose: AgentPurpose, ) -> Result< ( Option, AgentForPurposeDeterminationInfoConfigurationSource, ), String, > { let (agent_id, configuration_source) = get_effective_room_agent_raw_id_for_purpose(room_config_context, purpose).await; let Some(agent_id) = agent_id else { return Ok((None, configuration_source)); }; let agent_identifier = match PublicIdentifier::from_str(agent_id.as_str()) { Some(agent_identifier) => agent_identifier, None => return Err(strings::agent::invalid_id_generic()), }; Ok((Some(agent_identifier), configuration_source)) } async fn get_effective_room_agent_raw_id_for_purpose( room_config_context: &RoomConfigContext, purpose: AgentPurpose, ) -> ( Option, AgentForPurposeDeterminationInfoConfigurationSource, ) { let agent_id = room_config_context .room_config .settings .handler .get_by_purpose_with_catch_all_fallback(purpose); if let Some(agent_id) = agent_id { return ( Some(agent_id), AgentForPurposeDeterminationInfoConfigurationSource::Room, ); } tracing::trace!( ?purpose, "No specific agent found for purpose in room, falling back to global.", ); ( get_global_agent_id_for_purpose(room_config_context, purpose).await, AgentForPurposeDeterminationInfoConfigurationSource::Global, ) } async fn get_global_agent_id_for_purpose( room_config_context: &RoomConfigContext, purpose: AgentPurpose, ) -> Option { room_config_context .global_config .fallback_room_settings .handler .get_by_purpose_with_catch_all_fallback(purpose) } ================================================ FILE: src/bot/implementation.rs ================================================ use std::fs; use std::sync::Arc; use std::{future::Future, pin::Pin}; use mxlink::matrix_sdk::Room; use mxlink::matrix_sdk::media::{MediaFormat, MediaRequestParameters}; use mxlink::matrix_sdk::ruma::api::client::profile::{AvatarUrl, DisplayName}; use mxlink::matrix_sdk::ruma::{ MilliSecondsSinceUnixEpoch, OwnedUserId, events::room::MediaSource, }; use mxlink::{ InitConfig, LoginConfig, LoginCredentials, LoginEncryption, MatrixLink, PersistenceConfig, TypingNoticeGuard, }; use mxlink::helpers::account_data_config::{ ConfigError, GlobalConfigManager as AccountDataGlobalConfigManager, RoomConfigManager as AccountDataRoomConfigManager, }; use mxlink::helpers::encryption::Manager as EncryptionManager; use mxlink::mime::Mime; use crate::agent::Manager as AgentManager; use crate::entity::catch_up_marker::{ CatchUpMarker, CatchUpMarkerManager, DelayedCatchUpMarkerManager, }; use crate::entity::cfg::{Avatar, Config, ConfigUserAuth}; use crate::entity::globalconfig::{GlobalConfig, GlobalConfigurationManager}; use crate::entity::roomconfig::{RoomConfig, RoomConfigurationManager}; use crate::agent::Manager; use crate::conversation::matrix::{RoomDisplayNameFetcher, RoomEventFetcher}; const ROOM_EVENT_FETCHER_LRU_CACHE_SIZE: usize = 1000; const ROOM_DISPLAY_NAME_FETCHER_LRU_CACHE_SIZE: usize = 1000; const ROOM_CONFIG_MANAGER_LRU_CACHE_SIZE: usize = 1000; const LOGO_BYTES: &[u8] = include_bytes!("../../etc/assets/baibot-torso-768.png"); const LOGO_MIME_TYPE: &str = "image/png"; /// Controls how often we persist the catch-up marker to Account Data. /// Consult the `DelayedCatchUpMarkerManager` documentation for more information. const DELAYED_CATCH_UP_MARKER_MANAGER_PERSIST_INTERVAL_DURATION: std::time::Duration = std::time::Duration::from_secs(10); /// Controls what federation delay we will tolerate. The timestamp that gets persisted /// will be based on the last seen event's `origin_server_ts` minus this duration. /// Consult the `DelayedCatchUpMarkerManager` documentation for more information. const DELAYED_CATCH_UP_MARKER_MANAGER_FEDERATION_DELAY_TOLERANCE_DURATION: std::time::Duration = std::time::Duration::from_secs(90); struct BotInner { config: Config, matrix_link: MatrixLink, delayed_catch_up_marker_manager: DelayedCatchUpMarkerManager, global_config_manager: tokio::sync::Mutex, room_config_manager: tokio::sync::Mutex, room_event_fetcher: Arc, room_display_name_fetcher: Arc, agent_manager: Manager, admin_pattern_regexes: Vec, } /// Bot represents a bot instance. /// /// All of the state is held in an `Arc` so the `Bot` can be cloned freely. #[derive(Clone)] pub struct Bot { inner: Arc, } impl Bot { pub async fn new(config: Config) -> anyhow::Result { // Take some potentially problematic configuration values out of the config early on. // If we'd be failing, we'd like it to happen early, before we log in, etc. let initial_global_config: GlobalConfig = config.initial_global_config.clone().try_into()?; let admin_pattern_regexes = config.access.admin_pattern_regexes()?; let persistence_config_encryption_key = config.persistence.config_encryption_key()?; let agent_manager = AgentManager::new(config.agents.static_definitions.clone())?; let encryption_manager = EncryptionManager::new(persistence_config_encryption_key); let matrix_link = create_matrix_link(&config).await?; let catch_up_marker_manager = create_catch_up_marker_manager(matrix_link.clone()); let delayed_catch_up_marker_manager = DelayedCatchUpMarkerManager::new( catch_up_marker_manager, DELAYED_CATCH_UP_MARKER_MANAGER_PERSIST_INTERVAL_DURATION, DELAYED_CATCH_UP_MARKER_MANAGER_FEDERATION_DELAY_TOLERANCE_DURATION, ); let global_config_manager = tokio::sync::Mutex::new(create_global_configuration_manager( matrix_link.clone(), encryption_manager.clone(), initial_global_config, )); let room_config_manager = tokio::sync::Mutex::new(create_room_configuration_manager( matrix_link.clone(), encryption_manager.clone(), )); let room_event_fetcher = RoomEventFetcher::new(Some(ROOM_EVENT_FETCHER_LRU_CACHE_SIZE)); let room_display_name_fetcher = RoomDisplayNameFetcher::new( matrix_link.clone(), Some(ROOM_DISPLAY_NAME_FETCHER_LRU_CACHE_SIZE), ); Ok(Self { inner: Arc::new(BotInner { config, matrix_link, delayed_catch_up_marker_manager, global_config_manager, room_config_manager, room_event_fetcher: Arc::new(room_event_fetcher), room_display_name_fetcher: Arc::new(room_display_name_fetcher), agent_manager, admin_pattern_regexes, }), }) } pub(crate) fn admin_patterns(&self) -> &Vec { &self.inner.config.access.admin_patterns } pub(crate) fn name(&self) -> &str { &self.inner.config.user.name } pub(crate) fn command_prefix(&self) -> &str { &self.inner.config.command_prefix } pub(crate) fn post_join_self_introduction_enabled(&self) -> bool { self.inner.config.room.post_join_self_introduction_enabled } pub(crate) fn homeserver_name(&self) -> &str { &self.inner.config.homeserver.server_name } pub(crate) fn global_config_manager(&self) -> &tokio::sync::Mutex { &self.inner.global_config_manager } pub(crate) fn room_config_manager(&self) -> &tokio::sync::Mutex { &self.inner.room_config_manager } pub(crate) fn room_event_fetcher(&self) -> Arc { self.inner.room_event_fetcher.clone() } pub(crate) fn room_display_name_fetcher(&self) -> Arc { self.inner.room_display_name_fetcher.clone() } pub(crate) fn agent_manager(&self) -> &Manager { &self.inner.agent_manager } pub(crate) fn matrix_link(&self) -> &MatrixLink { &self.inner.matrix_link } pub(crate) fn user_id(&self) -> &OwnedUserId { self.matrix_link().user_id() } pub(crate) async fn user_display_name_in_room(&self, room: &Room) -> Option { let bot_display_name = self .room_display_name_fetcher() .own_display_name_in_room(room) .await; match bot_display_name { Ok(value) => value, Err(err) => { tracing::warn!( ?err, "Failed to fetch bot display name. Proceeding without it" ); None } } } pub(crate) fn reacting(&self) -> super::reacting::Reacting { super::reacting::Reacting::new(self.clone()) } pub(crate) fn rooms(&self) -> super::rooms::Rooms { super::rooms::Rooms::new(self.clone()) } pub(crate) fn messaging(&self) -> super::messaging::Messaging { super::messaging::Messaging::new(self.clone()) } pub(crate) fn admin_pattern_regexes(&self) -> &Vec { &self.inner.admin_pattern_regexes } pub(crate) async fn global_config(&self) -> Result { let mut global_config_manager_guard = self.inner.global_config_manager.lock().await; global_config_manager_guard.get_or_create().await } pub(crate) async fn is_caught_up( &self, event_origin_server_ts: MilliSecondsSinceUnixEpoch, ) -> Result { self.inner .delayed_catch_up_marker_manager .is_caught_up(event_origin_server_ts.0.into()) .await } pub(crate) async fn catch_up(&self, event_origin_server_ts: MilliSecondsSinceUnixEpoch) { self.inner .delayed_catch_up_marker_manager .catch_up(event_origin_server_ts.0.into()) .await } pub(crate) async fn start_typing_notice(&self, room: &Room) -> TypingNoticeGuard { self.inner .matrix_link .rooms() .start_typing_notice(room) .await } pub async fn start(&self) -> anyhow::Result<()> { self.rooms().attach_event_handlers().await; self.messaging().attach_event_handlers().await; self.reacting().attach_event_handlers().await; self.inner.delayed_catch_up_marker_manager.start().await; self.prepare_profile().await?; self.inner .matrix_link .start() .await .map_err(|e| anyhow::anyhow!("Failed to sync: {:?}", e)) } async fn prepare_profile(&self) -> anyhow::Result<()> { use std::time::Duration; use tokio::time::sleep; let mut delay = Duration::from_secs(3); let max_delay = Duration::from_secs(30); loop { match self.do_prepare_profile().await { Ok(_) => return Ok(()), Err(err) => { tracing::warn!( ?err, ?delay, "Failed to prepare profile.. Will retry after delay..." ); sleep(delay).await; delay = std::cmp::min(delay * 2, max_delay); } } } } async fn do_prepare_profile(&self) -> anyhow::Result<()> { tracing::debug!("Preparing profile.."); let desired_display_name = self.inner.config.user.name.clone(); let account = self.inner.matrix_link.client().account(); let media = self.inner.matrix_link.client().media(); let profile = account .fetch_user_profile() .await .map_err(|e| anyhow::anyhow!("Failed fetching profile: {:?}", e))?; let current_display_name = profile.get_static::()?; let current_avatar_url = profile.get_static::()?; let should_update_display_name = match ¤t_display_name { Some(displayname) => displayname != &desired_display_name, None => true, }; if should_update_display_name { tracing::info!( ?current_display_name, ?desired_display_name, "Updating display name.." ); if let Err(err) = account.set_display_name(Some(&desired_display_name)).await { return Err(anyhow::anyhow!("Failed setting display name: {:?}", err)); } } let desired_avatar: Option<(Vec, Mime)> = match &self.inner.config.user.avatar { Avatar::Keep => { tracing::info!("Avatar configured to keep current, skipping avatar management"); None } Avatar::Default => { tracing::info!("Avatar configured to use default"); Some(( LOGO_BYTES.to_vec(), LOGO_MIME_TYPE .parse() .expect("Failed parsing mime type for logo"), )) } Avatar::Custom(avatar_path) => { tracing::info!(?avatar_path, "Avatar configured to use custom path"); let bytes = fs::read(avatar_path).map_err(|e| { anyhow::anyhow!("Failed reading avatar from {:?}: {:?}", avatar_path, e) })?; let mime = mime_guess::from_path(avatar_path).first_or_octet_stream(); tracing::debug!(?mime, bytes_len = bytes.len(), "Loaded custom avatar"); Some((bytes, mime)) } }; if let Some((desired_bytes, mime_type)) = desired_avatar { let should_update_avatar = match ¤t_avatar_url { Some(avatar_url) => { tracing::debug!(?avatar_url, "Fetching current avatar to compare"); let request = MediaRequestParameters { source: MediaSource::Plain(avatar_url.to_owned()), format: MediaFormat::File, }; let content = media .get_media_content(&request, true) .await .map_err(|e| anyhow::anyhow!("Failed fetching existing avatar: {:?}", e))?; let needs_update = content.as_slice() != desired_bytes; tracing::debug!( current_bytes_len = content.len(), desired_bytes_len = desired_bytes.len(), ?needs_update, "Compared current and desired avatar" ); needs_update } None => { tracing::debug!("No current avatar set, will upload"); true } }; if should_update_avatar { tracing::info!("Updating avatar.."); account .upload_avatar(&mime_type, desired_bytes) .await .map_err(|e| anyhow::anyhow!("Failed uploading avatar: {:?}", e))?; tracing::info!("Avatar updated successfully"); } else { tracing::debug!("Avatar already up to date, skipping upload"); } } Ok(()) } } async fn create_matrix_link(config: &Config) -> anyhow::Result { let session_file_path = config.persistence.session_file_path()?; let session_encryption_key = config.persistence.session_encryption_key()?; let db_dir_path: std::path::PathBuf = config.persistence.db_dir_path()?; let user_auth = config.user.auth_config(&config.homeserver.server_name)?; let login_creds = match user_auth { ConfigUserAuth::UserPassword { username, password } => { LoginCredentials::UserPassword(username, password) } ConfigUserAuth::AccessToken { user_id, device_id, access_token, } => LoginCredentials::AccessToken { user_id, device_id, access_token, }, }; let login_encryption = LoginEncryption::new( config.user.encryption.recovery_passphrase.clone(), config.user.encryption.recovery_reset_allowed, ); let login_config = LoginConfig::new( config.homeserver.url.to_owned(), login_creds, Some(login_encryption), config.user.name.to_owned(), ); let persistence_config = PersistenceConfig::new(session_file_path, session_encryption_key, db_dir_path); let init_config = InitConfig::new(login_config, persistence_config); mxlink::init(&init_config).await.map_err(|e| e.into()) } pub fn create_global_configuration_manager( matrix_link: MatrixLink, encryption_manager: EncryptionManager, initial_global_config: GlobalConfig, ) -> GlobalConfigurationManager { let initial_global_config_callback = move || { let initial_global_config = initial_global_config.clone(); let future = create_initial_global_config(initial_global_config); // Explicitly box the future to match the expected type Box::pin(future) as Pin + Send>> }; AccountDataGlobalConfigManager::new( matrix_link, encryption_manager, initial_global_config_callback, ) } async fn create_initial_global_config(initial_global_config: GlobalConfig) -> GlobalConfig { initial_global_config } pub fn create_room_configuration_manager( matrix_link: MatrixLink, encryption_manager: EncryptionManager, ) -> RoomConfigurationManager { let initial_room_config_callback = |room: Room| { let future = create_initial_room_config(room); // Explicitly box the future to match the expected type Box::pin(future) as Pin + Send>> }; AccountDataRoomConfigManager::new( matrix_link.user_id().clone(), encryption_manager, initial_room_config_callback, Some(ROOM_CONFIG_MANAGER_LRU_CACHE_SIZE), ) } async fn create_initial_room_config(room: Room) -> RoomConfig { RoomConfig::default().with_room(room).await } pub fn create_catch_up_marker_manager(matrix_link: MatrixLink) -> CatchUpMarkerManager { let initial_global_config_callback = || { let future = create_initial_catch_up_marker(); // Explicitly box the future to match the expected type Box::pin(future) as Pin + Send>> }; // Intentionally not using encryption, to make this resilient even if we lose our encryption key. // We're not worried about the catch-up marker being read or tampered with, as it's not sensitive data. let encryption_manager = EncryptionManager::new(None); let catch_up_marker_manager: CatchUpMarkerManager = AccountDataGlobalConfigManager::new( matrix_link.clone(), encryption_manager, initial_global_config_callback, ); catch_up_marker_manager } async fn create_initial_catch_up_marker() -> CatchUpMarker { CatchUpMarker::new(0) } ================================================ FILE: src/bot/load_config.rs ================================================ use std::env; use std::path::PathBuf; use anyhow::anyhow; use crate::agent::AgentPurpose; pub use crate::entity::cfg::{Avatar, Config, defaults as cfg_defaults, env as cfg_env}; pub fn load() -> anyhow::Result { let config_file_path = env::var(cfg_env::BAIBOT_CONFIG_FILE_PATH) .unwrap_or_else(|_| cfg_defaults::config_file_path().to_owned()); let config_file_path = PathBuf::from(config_file_path); if !config_file_path.exists() { return Err(anyhow!( "Config file ({}) not found. Adjust the {} environment variable to use another config file.", config_file_path.display(), cfg_env::BAIBOT_CONFIG_FILE_PATH, )); } let config_str = std::fs::read_to_string(config_file_path)?; let mut config: Config = serde_yaml_ng::from_str(&config_str)?; // Allow environment variables to override some configuration keys for (key, value) in env::vars() { match key.as_str() { cfg_env::BAIBOT_HOMESERVER_SERVER_NAME => config.homeserver.server_name = value, cfg_env::BAIBOT_HOMESERVER_URL => config.homeserver.url = value, cfg_env::BAIBOT_USER_MXID_LOCALPART => config.user.mxid_localpart = value, cfg_env::BAIBOT_USER_PASSWORD => { config.user.password = optional_non_empty(value); } cfg_env::BAIBOT_USER_ACCESS_TOKEN => { config.user.access_token = optional_non_empty(value); } cfg_env::BAIBOT_USER_DEVICE_ID => { config.user.device_id = optional_non_empty(value); } cfg_env::BAIBOT_USER_ENCRYPTION_RECOVERY_PASSPHRASE => { config.user.encryption.recovery_passphrase = Some(value); } cfg_env::BAIBOT_USER_ENCRYPTION_RECOVERY_RESET_ALLOWED => { config.user.encryption.recovery_reset_allowed = value.parse::()?; } cfg_env::BAIBOT_USER_NAME => config.user.name = value, cfg_env::BAIBOT_USER_AVATAR => { config.user.avatar = Avatar::from_string(value); } cfg_env::BAIBOT_COMMAND_PREFIX => config.command_prefix = value, cfg_env::BAIBOT_ROOM_POST_JOIN_SELF_INTRODUCTION_ENABLED => { config.room.post_join_self_introduction_enabled = value.parse::()?; } cfg_env::BAIBOT_LOGGING => { config.logging = value; } cfg_env::BAIBOT_ACCESS_ADMIN_PATTERNS => { config.access.admin_patterns = value .split(' ') .map(|s| s.trim().to_string()) .filter(|s| !s.is_empty()) .collect(); } cfg_env::BAIBOT_PERSISTENCE_DATA_DIR_PATH => { config.persistence.data_dir_path = Some(value); } cfg_env::BAIBOT_PERSISTENCE_SESSION_ENCRYPTION_KEY => { config.persistence.session_encryption_key = Some(value); } cfg_env::BAIBOT_PERSISTENCE_CONFIG_ENCRYPTION_KEY => { config.persistence.config_encryption_key = Some(value); } cfg_env::BAIBOT_INITIAL_GLOBAL_CONFIG_HANDLER_CATCH_ALL => { let value = if value.is_empty() { None } else { Some(value) }; config .initial_global_config .handler .set_by_purpose(AgentPurpose::CatchAll, value); } cfg_env::BAIBOT_INITIAL_GLOBAL_CONFIG_HANDLER_TEXT_GENERATION => { let value = if value.is_empty() { None } else { Some(value) }; config .initial_global_config .handler .set_by_purpose(AgentPurpose::TextGeneration, value); } cfg_env::BAIBOT_INITIAL_GLOBAL_CONFIG_HANDLER_TEXT_TO_SPEECH => { let value = if value.is_empty() { None } else { Some(value) }; config .initial_global_config .handler .set_by_purpose(AgentPurpose::TextToSpeech, value); } cfg_env::BAIBOT_INITIAL_GLOBAL_CONFIG_HANDLER_SPEECH_TO_TEXT => { let value = if value.is_empty() { None } else { Some(value) }; config .initial_global_config .handler .set_by_purpose(AgentPurpose::SpeechToText, value); } cfg_env::BAIBOT_INITIAL_GLOBAL_CONFIG_HANDLER_IMAGE_GENERATION => { let value = if value.is_empty() { None } else { Some(value) }; config .initial_global_config .handler .set_by_purpose(AgentPurpose::ImageGeneration, value); } cfg_env::BAIBOT_INITIAL_GLOBAL_CONFIG_USER_PATTERNS => { config.initial_global_config.user_patterns = Some( value .split(' ') .map(|s| s.trim().to_string()) .filter(|s| !s.is_empty()) .collect(), ); } _ => {} } } config.validate().map_err(|s| anyhow!(s))?; Ok(config) } fn optional_non_empty(value: String) -> Option { if value.is_empty() { None } else { Some(value) } } ================================================ FILE: src/bot/messaging.rs ================================================ use mxlink::matrix_sdk::{ Room, ruma::{ OwnedEventId, api::client::receipt::create_receipt::v3::ReceiptType, events::room::message::OriginalSyncRoomMessageEvent, }, }; use mxlink::{CallbackError, MessageResponseType}; use tracing::Instrument; use crate::{ conversation::matrix::determine_interaction_context_for_room_event, entity::{MessageContext, MessagePayload, RoomConfigContext, TriggerEventInfo}, }; #[derive(Clone)] pub struct Messaging { bot: super::Bot, } impl Messaging { pub fn new(bot: super::Bot) -> Self { Self { bot } } pub async fn send_text_markdown_no_fail( &self, room: &Room, message: String, response_type: MessageResponseType, ) -> Option { let result = self .bot .matrix_link() .messaging() .send_text_markdown(room, message, response_type) .await; match result { Ok(result) => Some(result), Err(err) => { tracing::error!( room_id = format!("{:?}", room.room_id()), ?err, "Failed to send text message to room", ); None } } } pub async fn send_notice_markdown_no_fail( &self, room: &Room, message: String, response_type: MessageResponseType, ) -> Option { let result = self .bot .matrix_link() .messaging() .send_notice_markdown(room, message, response_type) .await; match result { Ok(result) => Some(result), Err(err) => { tracing::error!( room_id = format!("{:?}", room.room_id()), ?err, "Failed to send notice message to room", ); None } } } pub async fn send_tooltip_markdown_no_fail( &self, room: &Room, message: &str, response_type: MessageResponseType, ) -> Option { self.send_notice_markdown_no_fail( room, crate::utils::status::create_tooltip_message_text(message), response_type, ) .await } pub async fn send_success_markdown_no_fail( &self, room: &Room, message: &str, response_type: MessageResponseType, ) -> Option { self.send_notice_markdown_no_fail( room, crate::utils::status::create_success_message_text(message), response_type, ) .await } pub async fn send_error_markdown_no_fail( &self, room: &Room, err: &str, response_type: MessageResponseType, ) -> Option { self.send_notice_markdown_no_fail( room, crate::utils::status::create_error_message_text(err), response_type, ) .await } pub async fn redact_event_no_fail( &self, room: &Room, target_event_id: OwnedEventId, reason: Option, ) -> Option { let result = self .bot .matrix_link() .messaging() .redact_event(room, target_event_id.clone(), reason) .await; match result { Ok(result) => Some(result), Err(err) => { tracing::error!( room_id = format!("{:?}", room.room_id()), ?target_event_id, ?err, "Failed to send redaction to room", ); None } } } pub(super) async fn attach_event_handlers(&self) { let matrix_link_messaging = self.bot.matrix_link().messaging(); let this = self.clone(); matrix_link_messaging.on_actionable_room_message(|event, room| async move { this.on_actionable_message(event, room).await }); } #[tracing::instrument(name = "bot_on_actionable_message", skip_all, fields(room_id = room.room_id().as_str(), event_id = event.event_id.as_str()))] async fn on_actionable_message( &self, event: OriginalSyncRoomMessageEvent, room: Room, ) -> Result<(), CallbackError> { if self .bot .is_caught_up(event.origin_server_ts) .await .map_err(|e| { CallbackError::Unknown( format!("Failed to determine catch-up state: {:?}", e).into(), ) })? { tracing::debug!( event_origin_server_ts = format!("{:?}", event.origin_server_ts), "Ignoring old message event", ); return Ok(()); } tracing::info!("Processing message"); let global_config = self .bot .global_config() .await .map_err(|err| CallbackError::Unknown(err.into()))?; tracing::trace!(?global_config, "Global config"); let room_config = self .bot .room_config_manager() .lock() .await .get_or_create_for_room(&room) .await .map_err(|err| CallbackError::Unknown(err.into()))?; tracing::trace!(?room_config, "Room config"); let trigger_event_sender_is_admin = mxidwc::match_user_id( event.sender.clone().as_str(), self.bot.admin_pattern_regexes(), ); let trigger_event_sender_is_allowed_user = match &global_config.access.user_patterns { Some(user_patterns) => { let allowed_user_regexes = mxidwc::parse_patterns_vector(user_patterns) .map_err(|err| CallbackError::Unknown(err.into()))?; mxidwc::match_user_id(event.sender.clone().as_str(), &allowed_user_regexes) } None => false, }; if !trigger_event_sender_is_admin && !trigger_event_sender_is_allowed_user { tracing::debug!("Ignoring message from non-admin/non-allowed user"); return Ok(()); } let payload: Result = event.content.msgtype.clone().try_into(); let payload = match payload { Ok(payload) => payload, Err(err) => { tracing::debug!( msg_type = event.content.msgtype(), ?err, "Ignoring message not supported by us", ); return Ok(()); } }; let bot_display_name = self.bot.user_display_name_in_room(&room).await; let interaction_context = determine_interaction_context_for_room_event( self.bot.user_id(), &bot_display_name, &room, &event, &payload, &self.bot.room_event_fetcher(), ) .await; let interaction_context = match interaction_context { Ok(value) => value, Err(err) => { tracing::error!(?err, "Failed to determine interaction context for event"); return Ok(()); } }; let Some(interaction_context) = interaction_context else { tracing::debug!( "Ignoring message with unknown interaction context (likely not a message for us)" ); return Ok(()); }; let room_config_context = RoomConfigContext::new(global_config.clone(), room_config.clone()); let trigger_event_info = TriggerEventInfo::new( event.event_id.clone(), event.sender.clone(), payload, trigger_event_sender_is_admin, ); let message_context = MessageContext::new( room.clone(), room_config_context, self.bot.admin_pattern_regexes().clone(), trigger_event_info, interaction_context.thread_info.clone(), ) .with_bot_display_name(bot_display_name); let controller_type = crate::controller::determine_controller( self.bot.command_prefix(), &interaction_context.trigger, &message_context, ); tracing::info!(?controller_type, "Determined controller"); let _ = room .send_single_receipt( ReceiptType::Read, interaction_context.thread_info.clone().into(), event.event_id.clone(), ) .await; let start_time = std::time::Instant::now(); let event_span = tracing::error_span!("message_controller", ?controller_type); crate::controller::dispatch_controller(&controller_type, &message_context, &self.bot) .instrument(event_span) .await; let duration = std::time::Instant::now().duration_since(start_time); tracing::debug!(?duration, "Controller finished"); self.bot.catch_up(event.origin_server_ts).await; return Ok(()); } } ================================================ FILE: src/bot/mod.rs ================================================ mod implementation; mod load_config; mod messaging; mod reacting; mod rooms; pub use implementation::Bot; pub use load_config::load as load_config; ================================================ FILE: src/bot/reacting.rs ================================================ use mxlink::matrix_sdk::{ Room, ruma::{ OwnedEventId, OwnedUserId, events::{ AnySyncMessageLikeEvent, AnySyncTimelineEvent, SyncMessageLikeEvent, room::message::Relation, }, }, }; use mxlink::CallbackError; use mxlink::ThreadInfo; use tracing::Instrument; use crate::entity::{MessageContext, MessagePayload, RoomConfigContext, TriggerEventInfo}; #[derive(Clone)] pub struct Reacting { bot: super::Bot, } impl Reacting { pub fn new(bot: super::Bot) -> Self { Self { bot } } pub async fn react_no_fail( &self, room: &Room, target_event_id: OwnedEventId, reaction_key: String, ) -> Option { let result = self .bot .matrix_link() .reacting() .react(room, target_event_id.clone(), reaction_key) .await; match result { Ok(result) => Some(result), Err(err) => { tracing::error!( "Failed to send reaction to {} in room {:?}: {:?}", target_event_id, room.room_id(), err ); None } } } pub(super) async fn attach_event_handlers(&self) { let matrix_link_reacting = self.bot.matrix_link().reacting(); let this = self.clone(); matrix_link_reacting.on_actionable_reaction( |event, room, reaction_event_content| async move { this.on_actionable_reaction(event, room, reaction_event_content) .await }, ); } #[tracing::instrument(name = "bot_on_actionable_reaction", skip_all, fields(room_id = room.room_id().as_str(), event_id = event.event_id().as_str()))] async fn on_actionable_reaction( &self, event: AnySyncTimelineEvent, room: Room, reaction_event_content: mxlink::matrix_sdk::ruma::events::reaction::ReactionEventContent, ) -> Result<(), CallbackError> { if self .bot .is_caught_up(event.origin_server_ts()) .await .map_err(|e| { CallbackError::Unknown( format!("Failed to determine catch-up state: {:?}", e).into(), ) })? { tracing::debug!( event_origin_server_ts = format!("{:?}", event.origin_server_ts()), "Ignoring old reaction event", ); return Ok(()); } tracing::info!("Handling reaction"); let global_config = self .bot .global_config() .await .map_err(|err| CallbackError::Unknown(err.into()))?; tracing::trace!(?global_config, "Global config"); let trigger_event_sender_is_admin = mxidwc::match_user_id(event.sender().as_str(), self.bot.admin_pattern_regexes()); let trigger_event_sender_is_allowed_user = match &global_config.access.user_patterns { Some(user_patterns) => { let allowed_user_regexes = mxidwc::parse_patterns_vector(user_patterns) .map_err(|err| CallbackError::Unknown(err.into()))?; mxidwc::match_user_id(event.sender().as_str(), &allowed_user_regexes) } None => false, }; if !trigger_event_sender_is_admin && !trigger_event_sender_is_allowed_user { tracing::debug!("Ignoring reaction from non-admin/non-allowed user"); return Ok(()); } let reacted_to_event_id = &reaction_event_content.relates_to.event_id; let reacted_to_event = self .bot .room_event_fetcher() .fetch_event_in_room(reacted_to_event_id, &room) .await; let reacted_to_event = match reacted_to_event { Ok(value) => value, Err(err) => { tracing::error!( ?reacted_to_event_id, ?err, "Failed to fetch reacted-to event", ); return Ok(()); } }; let reacted_to_event_any_timeline_event = match reacted_to_event.raw().deserialize() { Ok(value) => value, Err(err) => { tracing::error!( ?reacted_to_event_id, ?err, "Failed to deserialize reacted-to event event", ); return Ok(()); } }; let reacted_to_event_sender_id: OwnedUserId = reacted_to_event_any_timeline_event.sender().to_owned(); let AnySyncTimelineEvent::MessageLike(reacted_to_event_message_like) = reacted_to_event_any_timeline_event else { tracing::debug!( ?reacted_to_event_id, "Ignoring non-MessageLike reacted-to event", ); return Ok(()); }; let AnySyncMessageLikeEvent::RoomMessage(reacted_to_event_room_message) = reacted_to_event_message_like else { tracing::debug!( ?reacted_to_event_id, "Ignoring non-RoomMessage reacted-to event", ); return Ok(()); }; let SyncMessageLikeEvent::Original(reacted_to_event_room_message_original) = reacted_to_event_room_message else { tracing::debug!(?reacted_to_event_id, "Ignoring redacted reacted-to event",); return Ok(()); }; let reacted_to_event_payload: Result = reacted_to_event_room_message_original .content .msgtype .clone() .try_into(); let Ok(reacted_to_event_payload) = reacted_to_event_payload else { tracing::debug!( msg_type = reacted_to_event_room_message_original.content.msgtype(), "Ignoring reaction to message of unknown type", ); return Ok(()); }; let thread_root_event_id = match reacted_to_event_room_message_original.content.relates_to { Some(relation) => { if let Relation::Thread(thread_id) = relation { thread_id.event_id.clone() } else { reacted_to_event_id.clone() } } None => reacted_to_event_id.clone(), }; let thread_info = ThreadInfo::new(thread_root_event_id, reacted_to_event_id.clone()); let room_config = self .bot .room_config_manager() .lock() .await .get_or_create_for_room(&room) .await .map_err(|err| CallbackError::Unknown(err.into()))?; tracing::trace!(?room_config, "Room config"); let room_config_context = RoomConfigContext::new(global_config.clone(), room_config.clone()); let trigger_event_info = TriggerEventInfo::new( event.event_id().to_owned(), event.sender().to_owned(), MessagePayload::Reaction { key: reaction_event_content.relates_to.key, reacted_to_event_payload: Box::new(reacted_to_event_payload), reacted_to_event_id: reaction_event_content.relates_to.event_id.clone(), reacted_to_event_sender_id, }, trigger_event_sender_is_admin, ); let message_context = MessageContext::new( room, room_config_context, self.bot.admin_pattern_regexes().clone(), trigger_event_info, thread_info, ); tracing::info!("Handling reaction via reaction controller"); let event_span = tracing::error_span!("reaction_controller"); crate::controller::reaction::handle( &self.bot, self.bot.matrix_link().clone(), &message_context, ) .instrument(event_span) .await .map_err(|err| CallbackError::Unknown(err.into()))?; self.bot.catch_up(event.origin_server_ts()).await; Ok(()) } } ================================================ FILE: src/bot/rooms.rs ================================================ use mxlink::{ InvitationDecision, matrix_sdk::{ Room, ruma::events::{AnySyncTimelineEvent, room::member::StrippedRoomMemberEvent}, }, }; use mxlink::CallbackError; use tracing::Instrument; use crate::entity::RoomConfigContext; #[derive(Clone)] pub struct Rooms { bot: super::Bot, } impl Rooms { pub fn new(bot: super::Bot) -> Self { Self { bot } } pub(super) async fn attach_event_handlers(&self) { let matrix_link_rooms = self.bot.matrix_link().rooms(); let this = self.clone(); matrix_link_rooms.on_being_last_member(|event, room| async move { this.on_being_last_member(event, room).await }); let this = self.clone(); matrix_link_rooms .on_invitation(|event, room| async move { this.on_invitation(event, room).await }); let this = self.clone(); matrix_link_rooms.on_joined(|event, room| async move { this.on_joined(event, room).await }); } async fn on_invitation( &self, room_member: StrippedRoomMemberEvent, _room: Room, ) -> Result { tracing::debug!("Deciding on room invitation"); let global_config = self .bot .global_config() .await .map_err(|e| CallbackError::Unknown(e.into()))?; let sender_is_admin = mxidwc::match_user_id( room_member.sender.clone().as_str(), self.bot.admin_pattern_regexes(), ); let sender_is_allowed_user = match &global_config.access.user_patterns { Some(user_patterns) => { let allowed_user_regexes = mxidwc::parse_patterns_vector(user_patterns) .map_err(|e| CallbackError::Unknown(e.into()))?; mxidwc::match_user_id(room_member.sender.clone().as_str(), &allowed_user_regexes) } None => false, }; if !(sender_is_admin || sender_is_allowed_user) { return Ok(InvitationDecision::Reject); } Ok(InvitationDecision::Join) } #[tracing::instrument(name = "bot_on_joined", skip_all, fields(room_id = room.room_id().as_str(), event_id = event.event_id().as_str()))] async fn on_joined( &self, event: AnySyncTimelineEvent, room: Room, ) -> Result<(), CallbackError> { if self .bot .is_caught_up(event.origin_server_ts()) .await .map_err(|e| { CallbackError::Unknown( format!("Failed to determine catch-up state: {:?}", e).into(), ) })? { tracing::debug!( event_origin_server_ts = format!("{:?}", event.origin_server_ts()), "Ignoring old room join event", ); return Ok(()); } tracing::info!("Handling room join"); let global_config = self .bot .global_config() .await .map_err(|e| CallbackError::Unknown(e.into()))?; let room_config_manager = self.bot.room_config_manager().lock().await; // We force-create a new config when we join anew to ensure we: // - always start from a known clean state // - record the last join timestamp, so we can accurately service the room (ignoring past messages, etc.) let room_config = room_config_manager .create_new_for_room(&room) .await .map_err(|e| CallbackError::Unknown(e.into()))?; let room_config_context = RoomConfigContext::new(global_config, room_config); let event_span = tracing::error_span!("join_controller"); let result = crate::controller::join::handle(&self.bot, &room, &room_config_context) .instrument(event_span) .await .map_err(|e| CallbackError::Unknown(e.into())); self.bot.catch_up(event.origin_server_ts()).await; result } async fn on_being_last_member( &self, _event: AnySyncTimelineEvent, room: mxlink::matrix_sdk::Room, ) -> Result<(), CallbackError> { tracing::info!( "Leaving room {} because we are the last member", room.room_id() ); // We are last in this room. Let's just leave room.leave().await.map_err(|e| e.into()) } } ================================================ FILE: src/controller/access/determination/mod.rs ================================================ #[cfg(test)] mod tests; use super::super::ControllerType; #[derive(Debug, PartialEq)] pub enum AccessControllerType { Help, GetUsers, SetUsers(Option>), GetRoomLocalAgentManagers, SetRoomLocalAgentManagers(Option>), } pub fn determine_controller(text: &str) -> ControllerType { if text.starts_with("users") { return ControllerType::Access(AccessControllerType::GetUsers); } if let Some(patterns_string) = text.strip_prefix("set-users") { let patterns_string = patterns_string.trim().to_owned(); let patterns_option = if patterns_string.is_empty() { None } else { let patterns_vector = patterns_string .split(" ") .map(|s| s.to_string()) .collect::>(); Some(patterns_vector) }; return ControllerType::Access(AccessControllerType::SetUsers(patterns_option)); } if text.starts_with("room-local-agent-managers") { return ControllerType::Access(AccessControllerType::GetRoomLocalAgentManagers); } if let Some(patterns_string) = text.strip_prefix("set-room-local-agent-managers") { let patterns_string = patterns_string.trim().to_owned(); let patterns_option = if patterns_string.is_empty() { None } else { let patterns_vector = patterns_string .split(" ") .map(|s| s.to_string()) .collect::>(); Some(patterns_vector) }; return ControllerType::Access(AccessControllerType::SetRoomLocalAgentManagers( patterns_option, )); } ControllerType::Access(AccessControllerType::Help) } ================================================ FILE: src/controller/access/determination/tests.rs ================================================ #[test] fn determine_controller() { struct TestCase { name: &'static str, input: &'static str, expected: super::ControllerType, } let test_cases = vec![ TestCase { name: "Top-level is help", input: "", expected: super::ControllerType::Access(super::AccessControllerType::Help), }, TestCase { name: "Anything else goes to top-level", input: "whatever", expected: super::ControllerType::Access(super::AccessControllerType::Help), }, TestCase { name: "Users", input: "users", expected: super::ControllerType::Access(super::AccessControllerType::GetUsers), }, TestCase { name: "Set-users", input: "set-users @user:example.com @bot.*:example.org", expected: super::ControllerType::Access(super::AccessControllerType::SetUsers(Some( vec![ "@user:example.com".to_owned(), "@bot.*:example.org".to_owned(), ], ))), }, TestCase { name: "Room-local-agent-managers", input: "room-local-agent-managers", expected: super::ControllerType::Access( super::AccessControllerType::GetRoomLocalAgentManagers, ), }, TestCase { name: "Set-room-local-agent-managers", input: "set-room-local-agent-managers @user:example.com @bot.*:example.org", expected: super::ControllerType::Access( super::AccessControllerType::SetRoomLocalAgentManagers(Some(vec![ "@user:example.com".to_owned(), "@bot.*:example.org".to_owned(), ])), ), }, ]; for test_case in test_cases { let result = super::determine_controller(test_case.input); assert_eq!(result, test_case.expected, "Test case: {}", test_case.name); } } ================================================ FILE: src/controller/access/dispatching.rs ================================================ use mxlink::MessageResponseType; use crate::{Bot, entity::MessageContext, strings}; use super::AccessControllerType; pub async fn dispatch_controller( handler: &AccessControllerType, message_context: &MessageContext, bot: &Bot, ) -> anyhow::Result<()> { // Only the help command is available without access control, so that all users can get familiar with how the bot's access system works. match handler { AccessControllerType::Help => {} _ => { if !message_context.sender_can_manage_global_config() { bot.messaging() .send_error_markdown_no_fail( message_context.room(), strings::global_config::no_permissions_to_administrate(), MessageResponseType::Reply( message_context.thread_info().root_event_id.clone(), ), ) .await; return Ok(()); } } }; match handler { AccessControllerType::Help => super::help::handle(bot, message_context).await, AccessControllerType::GetUsers => super::users::handle_get(bot, message_context).await, AccessControllerType::SetUsers(patterns) => { super::users::handle_set(bot, message_context, patterns).await } AccessControllerType::GetRoomLocalAgentManagers => { super::room_local_agent_managers::handle_get(bot, message_context).await } AccessControllerType::SetRoomLocalAgentManagers(patterns) => { super::room_local_agent_managers::handle_set(bot, message_context, patterns).await } } } ================================================ FILE: src/controller/access/help.rs ================================================ use mxlink::MessageResponseType; use crate::{Bot, entity::MessageContext, strings}; pub async fn handle(bot: &Bot, message_context: &MessageContext) -> anyhow::Result<()> { let mut message = String::new(); message.push_str(&build_section_intro()); message.push_str("\n\n"); message.push_str(&build_section_joining_rooms()); message.push_str("\n\n"); message.push_str(&build_section_users( bot.command_prefix(), bot.homeserver_name(), message_context, )); message.push_str("\n\n"); message.push_str(&build_section_administrators(bot.admin_patterns())); message.push_str("\n\n"); message.push_str(&build_section_room_local_agent_managers( bot.command_prefix(), bot.homeserver_name(), message_context, )); bot.messaging() .send_text_markdown_no_fail( message_context.room(), message, MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()), ) .await; Ok(()) } fn build_section_intro() -> String { let mut message = String::new(); message.push_str(&format!("## {}", strings::help::access::heading())); message.push_str("\n\n"); message.push_str(&strings::help::access::intro()); message } fn build_section_joining_rooms() -> String { let mut message = String::new(); message.push_str(&format!( "### {}", strings::help::access::room_auto_join_heading() )); message.push_str("\n\n"); message.push_str(&strings::help::access::room_auto_join_intro()); message.push_str("\n\n"); message } fn build_section_users( command_prefix: &str, homeserver_name: &str, message_context: &MessageContext, ) -> String { let mut message = String::new(); message.push_str(&format!("### {}", strings::help::access::users_heading())); message.push_str("\n\n"); message.push_str(&strings::help::access::users_intro()); message.push('\n'); message.push_str(&strings::help::access::users_access()); message.push_str("\n\n"); if let Some(user_patterns) = &message_context.global_config().access.user_patterns { if user_patterns.is_empty() { message.push_str(&strings::access::users_no_patterns()); } else { message.push_str(&strings::access::users_now_match_patterns(user_patterns)); } } else { message.push_str(&strings::access::users_no_patterns()); } if message_context.sender_can_manage_global_config() { message.push_str("\n\n"); message.push_str(strings::the_following_commands_are_available()); message.push('\n'); message.push_str(&strings::help::access::users_command_get(command_prefix)); message.push('\n'); message.push_str(&strings::help::access::users_command_set(command_prefix)); message.push_str("\n\n"); message.push_str(&strings::help::access::example_user_patterns( homeserver_name, )); } message } fn build_section_administrators(admin_patterns: &[String]) -> String { let mut message = String::new(); message.push_str(&format!( "### {}", strings::help::access::administrators_heading() )); message.push_str("\n\n"); message.push_str(&strings::help::access::administrators_intro()); message.push_str("\n\n"); message.push_str(&strings::help::access::administrators_now_match_patterns( admin_patterns, )); message.push_str("\n\n"); message.push_str(&strings::help::access::administrators_outro()); message } fn build_section_room_local_agent_managers( command_prefix: &str, homeserver_name: &str, message_context: &MessageContext, ) -> String { let mut message = String::new(); message.push_str(&format!( "### {}", strings::help::access::room_local_agent_managers_heading() )); message.push_str("\n\n"); message.push_str(&strings::help::access::room_local_agent_managers_intro( command_prefix, )); message.push('\n'); message.push_str(&strings::help::access::room_local_agent_managers_security_warning()); message.push_str("\n\n"); if let Some(user_patterns) = &message_context .global_config() .access .room_local_agent_manager_patterns { if user_patterns.is_empty() { message.push_str(&strings::access::room_local_agent_managers_no_patterns()); } else { message.push_str( &strings::access::room_local_agent_managers_now_match_patterns(user_patterns), ); } } else { message.push_str(&strings::access::room_local_agent_managers_no_patterns()); } if message_context.sender_can_manage_global_config() { message.push_str("\n\n"); message.push_str(strings::the_following_commands_are_available()); message.push('\n'); message.push_str( &strings::help::access::room_local_agent_managers_command_get(command_prefix), ); message.push('\n'); message.push_str( &strings::help::access::room_local_agent_managers_command_set(command_prefix), ); message.push_str("\n\n"); message.push_str(&strings::help::access::example_user_patterns( homeserver_name, )); } message } ================================================ FILE: src/controller/access/mod.rs ================================================ mod determination; mod dispatching; pub mod help; mod room_local_agent_managers; mod users; pub use determination::{AccessControllerType, determine_controller}; pub use dispatching::dispatch_controller; ================================================ FILE: src/controller/access/room_local_agent_managers.rs ================================================ use mxlink::MessageResponseType; use crate::{Bot, entity::MessageContext, strings}; pub async fn handle_get(bot: &Bot, message_context: &MessageContext) -> anyhow::Result<()> { let message = match &message_context .global_config() .access .room_local_agent_manager_patterns { Some(patterns) => strings::access::room_local_agent_managers_now_match_patterns(patterns), None => strings::access::room_local_agent_managers_no_patterns(), }; bot.messaging() .send_text_markdown_no_fail( message_context.room(), message, MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()), ) .await; Ok(()) } pub async fn handle_set( bot: &Bot, message_context: &MessageContext, patterns: &Option>, ) -> anyhow::Result<()> { if let Some(patterns) = patterns && let Err(err) = mxidwc::parse_patterns_vector(patterns) { bot.messaging() .send_error_markdown_no_fail( message_context.room(), &strings::access::failed_to_parse_patterns(&err.to_string()), MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()), ) .await; return Ok(()); } let mut global_config_manager_guard = bot.global_config_manager().lock().await; let mut global_config = global_config_manager_guard.get_or_create().await?; global_config.access.room_local_agent_manager_patterns = patterns.clone(); global_config_manager_guard.persist(&global_config).await?; let message = match patterns { Some(patterns) => strings::access::room_local_agent_managers_now_match_patterns(patterns), None => strings::access::room_local_agent_managers_no_patterns(), }; bot.messaging() .send_success_markdown_no_fail( message_context.room(), &message, MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()), ) .await; Ok(()) } ================================================ FILE: src/controller/access/users.rs ================================================ use mxlink::MessageResponseType; use crate::{Bot, entity::MessageContext, strings}; pub async fn handle_get(bot: &Bot, message_context: &MessageContext) -> anyhow::Result<()> { let message = match &message_context.global_config().access.user_patterns { Some(patterns) => strings::access::users_now_match_patterns(patterns), None => strings::access::users_no_patterns(), }; bot.messaging() .send_text_markdown_no_fail( message_context.room(), message, MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()), ) .await; Ok(()) } pub async fn handle_set( bot: &Bot, message_context: &MessageContext, patterns: &Option>, ) -> anyhow::Result<()> { if let Some(patterns) = patterns && let Err(err) = mxidwc::parse_patterns_vector(patterns) { bot.messaging() .send_error_markdown_no_fail( message_context.room(), &strings::access::failed_to_parse_patterns(&err.to_string()), MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()), ) .await; return Ok(()); } let mut global_config_manager_guard = bot.global_config_manager().lock().await; let mut global_config = global_config_manager_guard.get_or_create().await?; global_config.access.user_patterns = patterns.clone(); global_config_manager_guard.persist(&global_config).await?; let message = match patterns { Some(patterns) => strings::access::users_now_match_patterns(patterns), None => strings::access::users_no_patterns(), }; bot.messaging() .send_success_markdown_no_fail( message_context.room(), &message, MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()), ) .await; Ok(()) } ================================================ FILE: src/controller/agent/create/mod.rs ================================================ #[cfg(test)] mod tests; use mxlink::MessageResponseType; use crate::agent::PublicIdentifier; use crate::agent::provider::{ControllerTrait, PingResult}; use crate::agent::{AgentDefinition, create_from_provider_and_yaml_value_config}; use crate::agent::{AgentInstance, AgentProvider}; use crate::controller::utils::get_text_body_or_complain; use crate::entity::globalconfig::GlobalConfigurationManager; use crate::entity::roomconfig::RoomConfigurationManager; use crate::strings; use crate::{Bot, entity::MessageContext}; struct ParsedAgentConfig { agent: AgentInstance, config: serde_yaml_ng::Value, } pub async fn handle_room_local( bot: &Bot, room_config_manager: &tokio::sync::Mutex, message_context: &MessageContext, provider: &str, agent_id_prefixless: &str, ) -> anyhow::Result<()> { if !message_context.sender_can_manage_room_local_agents()? { bot.messaging() .send_error_markdown_no_fail( message_context.room(), &strings::agent::not_allowed_to_manage_room_local_agents_in_room(), MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()), ) .await; return Ok(()); } let Ok(provider) = AgentProvider::from_string(provider) else { bot.messaging() .send_error_markdown_no_fail( message_context.room(), &strings::provider::invalid(provider), MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()), ) .await; return Ok(()); }; let agent_identifier = PublicIdentifier::DynamicRoomLocal(agent_id_prefixless.to_owned()); if let Err(err) = agent_identifier.validate() { bot.messaging() .send_error_markdown_no_fail( message_context.room(), &strings::agent::invalid_id_validation_error(err), MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()), ) .await; return Ok(()); } let agent_exists = bot .agent_manager() .available_room_agents_by_room_config_context(message_context.room_config_context()) .iter() .any(|agent| *agent.identifier() == agent_identifier); if agent_exists { bot.messaging() .send_error_markdown_no_fail( message_context.room(), &strings::agent::already_exists_see_help(agent_id_prefixless, bot.command_prefix()), MessageResponseType::InThread(message_context.thread_info().clone()), ) .await; return Ok(()); } if message_context.thread_info().is_thread_root_only() { return send_guide(bot, message_context, &agent_identifier, &provider).await; } let Some(text_message_content) = get_text_body_or_complain(bot, message_context).await else { return Ok(()); }; let parsed_config = parse_agent_config_from_message_or_complain( bot, message_context, &provider, &agent_identifier, text_message_content, ) .await; let Some(parsed_config) = parsed_config else { return Ok(()); }; if !try_to_ping_agent_or_complain(bot, message_context, &parsed_config.agent).await { return Ok(()); } let agent_definition = AgentDefinition::new( agent_identifier.prefixless(), provider, parsed_config.config.clone(), ); let mut room_config = message_context.room_config().clone(); room_config.agents.push(agent_definition.clone()); room_config_manager .lock() .await .persist(message_context.room(), &room_config) .await?; send_completion_wrap_up( bot, message_context, &agent_identifier, &parsed_config.agent, ) .await; Ok(()) } pub async fn handle_global( bot: &Bot, global_config_manager: &tokio::sync::Mutex, message_context: &MessageContext, provider: &str, agent_id_prefixless: &str, ) -> anyhow::Result<()> { if !message_context.sender_can_manage_global_config() { bot.messaging() .send_error_markdown_no_fail( message_context.room(), strings::global_config::no_permissions_to_administrate(), MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()), ) .await; return Ok(()); } let Ok(provider) = AgentProvider::from_string(provider) else { bot.messaging() .send_error_markdown_no_fail( message_context.room(), &strings::provider::invalid(provider), MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()), ) .await; return Ok(()); }; let agent_identifier = PublicIdentifier::DynamicGlobal(agent_id_prefixless.to_owned()); if let Err(err) = agent_identifier.validate() { bot.messaging() .send_error_markdown_no_fail( message_context.room(), &strings::agent::invalid_id_validation_error(err), MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()), ) .await; return Ok(()); } let agent_exists = bot .agent_manager() .available_room_agents_by_room_config_context(message_context.room_config_context()) .iter() .any(|agent| *agent.identifier() == agent_identifier); if agent_exists { bot.messaging() .send_error_markdown_no_fail( message_context.room(), &strings::agent::already_exists_see_help(agent_id_prefixless, bot.command_prefix()), MessageResponseType::InThread(message_context.thread_info().clone()), ) .await; return Ok(()); } if message_context.thread_info().is_thread_root_only() { return send_guide(bot, message_context, &agent_identifier, &provider).await; } let Some(text_message_content) = get_text_body_or_complain(bot, message_context).await else { return Ok(()); }; let parsed_config = parse_agent_config_from_message_or_complain( bot, message_context, &provider, &agent_identifier, text_message_content, ) .await; let Some(parsed_config) = parsed_config else { return Ok(()); }; if !try_to_ping_agent_or_complain(bot, message_context, &parsed_config.agent).await { return Ok(()); } let agent_definition = AgentDefinition::new( agent_identifier.prefixless(), provider, parsed_config.config.clone(), ); let mut global_config = message_context.global_config().clone(); global_config.agents.push(agent_definition.clone()); global_config_manager .lock() .await .persist(&global_config) .await?; send_completion_wrap_up( bot, message_context, &agent_identifier, &parsed_config.agent, ) .await; Ok(()) } async fn send_guide( bot: &Bot, message_context: &MessageContext, agent_identifier: &PublicIdentifier, provider: &AgentProvider, ) -> anyhow::Result<()> { let sample_config = crate::agent::default_config_for_provider(provider); let sample_config_pretty_yaml = serde_yaml_ng::to_string(&sample_config)?; bot.messaging() .send_text_markdown_no_fail( message_context.room(), strings::agent::creation_guide(agent_identifier, provider, &sample_config_pretty_yaml), MessageResponseType::InThread(message_context.thread_info().clone()), ) .await; Ok(()) } fn parse_from_message_to_yaml_value(text: &str) -> Result { let mut text = text.trim(); if text.starts_with("```") { // Try to strip ```yml and ```yaml first and fall back to the generic ``` later. text = text.trim_start_matches("```yml"); text = text.trim_start_matches("```yaml"); text = text.trim_start_matches("```"); text = text.trim_end_matches("```"); } let config: serde_yaml_ng::Value = serde_yaml_ng::from_str(text).map_err(|e| e.to_string())?; match config { serde_yaml_ng::Value::Mapping(_) => {} _ => { return Err("Not a valid YAML hashmap".to_owned()); } }; Ok(config) } async fn parse_agent_config_from_message_or_complain( bot: &Bot, message_context: &MessageContext, provider: &AgentProvider, agent_identifier: &PublicIdentifier, text: &str, ) -> Option { let config_yaml_value = parse_from_message_to_yaml_value(text); let config_yaml_value = match config_yaml_value { Ok(config_yaml_value) => config_yaml_value, Err(err) => { bot.messaging() .send_error_markdown_no_fail( message_context.room(), &strings::agent::configuration_not_a_valid_yaml_hashmap(err), MessageResponseType::InThread(message_context.thread_info().clone()), ) .await; return None; } }; let agent = create_from_provider_and_yaml_value_config( provider, agent_identifier, config_yaml_value.clone(), ); let agent = match agent { Ok(agent) => ParsedAgentConfig { agent, config: config_yaml_value, }, Err(err) => { bot.messaging() .send_error_markdown_no_fail( message_context.room(), &strings::provider::invalid_configuration_for_provider(provider, err), MessageResponseType::InThread(message_context.thread_info().clone()), ) .await; return None; } }; Some(agent) } async fn try_to_ping_agent_or_complain( bot: &Bot, message_context: &MessageContext, agent_instance: &AgentInstance, ) -> bool { bot.messaging() .send_notice_markdown_no_fail( message_context.room(), format!("⏳ {}", strings::agent::configuration_agent_will_ping()), MessageResponseType::InThread(message_context.thread_info().clone()), ) .await; match agent_instance.controller().ping().await { Ok(ping_result) => { let message = match ping_result { PingResult::Inconclusive => format!( "❓ {}", strings::agent::configuration_agent_ping_inconclusive() ), PingResult::Successful => { format!("✅ {}", strings::agent::configuration_agent_ping_ok()) } }; bot.messaging() .send_notice_markdown_no_fail( message_context.room(), message, MessageResponseType::InThread(message_context.thread_info().clone()), ) .await; true } Err(err) => { bot.messaging() .send_error_markdown_no_fail( message_context.room(), &strings::agent::configuration_does_not_result_in_a_working_agent(err), MessageResponseType::InThread(message_context.thread_info().clone()), ) .await; false } } } async fn send_completion_wrap_up( bot: &Bot, message_context: &MessageContext, agent_identifier: &PublicIdentifier, agent_instance: &AgentInstance, ) { bot.messaging() .send_success_markdown_no_fail( message_context.room(), &strings::agent::created(agent_identifier), MessageResponseType::InThread(message_context.thread_info().clone()), ) .await; bot.messaging() .send_tooltip_markdown_no_fail( message_context.room(), &strings::agent::post_creation_helpful_commands( agent_identifier, agent_instance, bot.command_prefix(), ), MessageResponseType::InThread(message_context.thread_info().clone()), ) .await; } ================================================ FILE: src/controller/agent/create/tests.rs ================================================ #[test] fn agent_config_parsing_works() { struct TestCase { input: String, expected: Option, } let provider = crate::agent::AgentProvider::OpenAI; let sample_config = crate::agent::default_config_for_provider(&provider); let sample_config_pretty_yaml = serde_yaml_ng::to_string(&sample_config).unwrap(); let test_cases = vec![ // Invalid input TestCase { input: r#"Hello"#.to_owned(), expected: None, }, // Plain text TestCase { input: sample_config_pretty_yaml.clone(), expected: Some(sample_config.clone()), }, // Generic code block TestCase { input: format!("```\n{}```", sample_config_pretty_yaml), expected: Some(sample_config.clone()), }, // YAML code block (yaml) TestCase { input: format!("```yaml\n{}```", sample_config_pretty_yaml), expected: Some(sample_config.clone()), }, // YAML code block (yml) TestCase { input: format!("```yml\n{}```", sample_config_pretty_yaml), expected: Some(sample_config.clone()), }, // JSON code block TestCase { input: format!("```json\n{}```", sample_config_pretty_yaml), expected: None, }, ]; for (i, test_case) in test_cases.iter().enumerate() { let result = super::parse_from_message_to_yaml_value(&test_case.input); match result { Ok(config) => { assert_eq!( config, test_case.expected.clone().unwrap(), "Test case {} failed", i ); } Err(_) => { assert_eq!(test_case.expected, None, "Test case {} failed", i); } } } } ================================================ FILE: src/controller/agent/delete/mod.rs ================================================ use mxlink::MessageResponseType; use crate::entity::{ MessageContext, globalconfig::GlobalConfigurationManager, roomconfig::RoomConfigurationManager, }; use crate::{Bot, agent::PublicIdentifier, strings}; pub async fn handle( bot: &Bot, room_config_manager: &tokio::sync::Mutex, global_config_manager: &tokio::sync::Mutex, message_context: &MessageContext, agent_identifier: &PublicIdentifier, ) -> anyhow::Result<()> { let agents = bot .agent_manager() .available_room_agents_by_room_config_context(message_context.room_config_context()); let agent = agents.iter().find(|a| a.identifier() == agent_identifier); let Some(_) = agent else { bot.messaging() .send_error_markdown_no_fail( message_context.room(), &strings::agent::agent_with_given_identifier_missing(agent_identifier), MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()), ) .await; return Ok(()); }; match &agent_identifier { PublicIdentifier::DynamicRoomLocal(_) => { if !message_context.sender_can_manage_room_local_agents()? { bot.messaging() .send_error_markdown_no_fail( message_context.room(), &strings::agent::not_allowed_to_manage_room_local_agents_in_room(), MessageResponseType::Reply( message_context.thread_info().root_event_id.clone(), ), ) .await; return Ok(()); } delete_room_local_agent(bot, room_config_manager, message_context, agent_identifier) .await } PublicIdentifier::DynamicGlobal(_) => { if !message_context.sender_can_manage_global_config() { bot.messaging() .send_error_markdown_no_fail( message_context.room(), strings::global_config::no_permissions_to_administrate(), MessageResponseType::Reply( message_context.thread_info().root_event_id.clone(), ), ) .await; return Ok(()); } delete_global_agent( bot, global_config_manager, message_context, agent_identifier, ) .await } PublicIdentifier::Static(_) => { bot.messaging() .send_error_markdown_no_fail( message_context.room(), &strings::agent::not_allowed_to_manage_static_agents(), MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()), ) .await; Ok(()) } } } async fn delete_room_local_agent( bot: &Bot, room_config_manager: &tokio::sync::Mutex, message_context: &MessageContext, agent_id: &PublicIdentifier, ) -> anyhow::Result<()> { let mut room_config = message_context.room_config().clone(); let mut was_deleted = false; let agent_id_prefixless = agent_id.prefixless(); let mut agents = Vec::new(); for agent_config in room_config.agents { if agent_config.id == agent_id_prefixless { was_deleted = true; } else { agents.push(agent_config.clone()); } } if !was_deleted { bot.messaging() .send_error_markdown_no_fail( message_context.room(), &strings::agent::agent_with_given_identifier_missing(agent_id), MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()), ) .await; return Ok(()); } room_config.agents = agents; let room_config_manager = room_config_manager.lock().await; // We may unset all handlers in the room config which refer to this agent. // We intentionally do not do this, because we do not support "agent edit" yet and ask people to do "agent delete" and "agent create" instead. // We'd rather not magically reconfigure the room on agent deletion and obstruct this use case. room_config_manager .persist(message_context.room(), &room_config) .await?; bot.messaging() .send_success_markdown_no_fail( message_context.room(), &strings::agent::removed_room_local(agent_id, bot.command_prefix()), MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()), ) .await; Ok(()) } async fn delete_global_agent( bot: &Bot, global_config_manager: &tokio::sync::Mutex, message_context: &MessageContext, agent_id: &PublicIdentifier, ) -> anyhow::Result<()> { let mut global_config = message_context.global_config().clone(); let mut was_deleted = false; let agent_id_prefixless = agent_id.prefixless(); let mut agents = Vec::new(); for agent_config in global_config.agents { if agent_config.id == agent_id_prefixless { was_deleted = true; } else { agents.push(agent_config.clone()); } } if !was_deleted { bot.messaging() .send_error_markdown_no_fail( message_context.room(), &strings::agent::agent_with_given_identifier_missing(agent_id), MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()), ) .await; return Ok(()); } global_config.agents = agents; global_config_manager .lock() .await .persist(&global_config) .await?; bot.messaging() .send_success_markdown_no_fail( message_context.room(), &strings::agent::removed_global(agent_id, bot.command_prefix()), MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()), ) .await; Ok(()) } ================================================ FILE: src/controller/agent/details/mod.rs ================================================ use mxlink::MessageResponseType; use crate::{Bot, agent::PublicIdentifier, entity::MessageContext, strings}; pub async fn handle( bot: &Bot, message_context: &MessageContext, agent_identifier: &PublicIdentifier, ) -> anyhow::Result<()> { let agents = bot .agent_manager() .available_room_agents_by_room_config_context(message_context.room_config_context()); let agent = agents.iter().find(|a| a.identifier() == agent_identifier); let agent = match agent { Some(agent) => agent, None => { bot.messaging() .send_error_markdown_no_fail( message_context.room(), &strings::agent::agent_with_given_identifier_missing(agent_identifier), MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()), ) .await; return Ok(()); } }; // Access checks match &agent_identifier { PublicIdentifier::DynamicRoomLocal(_) => { if !message_context.sender_can_manage_room_local_agents()? { bot.messaging() .send_error_markdown_no_fail( message_context.room(), &strings::agent::not_allowed_to_manage_room_local_agents_in_room(), MessageResponseType::Reply( message_context.thread_info().root_event_id.clone(), ), ) .await; return Ok(()); } } PublicIdentifier::DynamicGlobal(_) => { if !message_context.sender_can_manage_global_config() { bot.messaging() .send_error_markdown_no_fail( message_context.room(), strings::global_config::no_permissions_to_administrate(), MessageResponseType::Reply( message_context.thread_info().root_event_id.clone(), ), ) .await; return Ok(()); } } PublicIdentifier::Static(_) => {} }; let config_yaml_pretty = serde_yaml_ng::to_string(&agent.definition().config)?; bot.messaging() .send_text_markdown_no_fail( message_context.room(), format!( "Configuration for agent `{}` (powered by the `{}` provider):\n```yml\n{}\n```", agent_identifier, agent.definition().provider.to_static_str(), config_yaml_pretty.trim(), ) .to_owned(), MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()), ) .await; Ok(()) } ================================================ FILE: src/controller/agent/determination/mod.rs ================================================ #[cfg(test)] mod tests; use crate::{agent::PublicIdentifier, controller::ControllerType, strings}; #[derive(Debug, PartialEq)] pub enum AgentControllerType { List, Details(PublicIdentifier), CreateRoomLocal { provider: String, agent_id: String }, CreateGlobal { provider: String, agent_id: String }, Delete(PublicIdentifier), Help, } pub fn determine_controller(command_prefix: &str, text: &str) -> ControllerType { if text.starts_with("list") { return ControllerType::Agent(AgentControllerType::List); } if let Some(agent_id_string) = text.strip_prefix("details") { let agent_id_string = agent_id_string.trim(); if agent_id_string.is_empty() || agent_id_string.contains(" ") { return ControllerType::Error( strings::agent::incorrect_invocation_expects_agent_id_arg(command_prefix), ); } let Some(agent_identifier) = PublicIdentifier::from_str(agent_id_string) else { return ControllerType::Error(strings::agent::invalid_id_generic()); }; return ControllerType::Agent(AgentControllerType::Details(agent_identifier)); } if let Some(remaining_text) = text.strip_prefix("create-room-local") { // `remaining_text` should be something like: `PROVIDER ID` let remaining_text = remaining_text.trim(); let parts = remaining_text.split_once(' '); let Some((provider, agent_id_string)) = parts else { return ControllerType::Error(strings::agent::incorrect_creation_invocation( command_prefix, )); }; if agent_id_string.contains(" ") { return ControllerType::Error(strings::agent::incorrect_creation_invocation( command_prefix, )); } return ControllerType::Agent(AgentControllerType::CreateRoomLocal { provider: provider.to_owned(), agent_id: agent_id_string.trim().to_owned(), }); } if let Some(remaining_text) = text.strip_prefix("create-global") { // `remaining_text` should be something like: `PROVIDER ID` let remaining_text = remaining_text.trim(); let parts = remaining_text.split_once(' '); let Some((provider, agent_id_string)) = parts else { return ControllerType::Error(strings::agent::incorrect_creation_invocation( command_prefix, )); }; if agent_id_string.contains(" ") { return ControllerType::Error(strings::agent::incorrect_creation_invocation( command_prefix, )); } return ControllerType::Agent(AgentControllerType::CreateGlobal { provider: provider.to_owned(), agent_id: agent_id_string.trim().to_owned(), }); } if let Some(agent_id_string) = text.strip_prefix("delete") { let agent_id_string = agent_id_string.trim(); if agent_id_string.is_empty() || agent_id_string.contains(" ") { return ControllerType::Error( strings::agent::incorrect_invocation_expects_agent_id_arg(command_prefix), ); } let Some(agent_identifier) = PublicIdentifier::from_str(agent_id_string) else { return ControllerType::Error(strings::agent::invalid_id_generic()); }; return ControllerType::Agent(AgentControllerType::Delete(agent_identifier)); } ControllerType::Agent(AgentControllerType::Help) } ================================================ FILE: src/controller/agent/determination/tests.rs ================================================ #[test] fn determine_controller() { use crate::agent::PublicIdentifier; struct TestCase { name: &'static str, input: &'static str, expected: super::ControllerType, } let command_prefix = "!bai"; let test_cases = vec![ TestCase { name: "Top-level is help", input: "", expected: super::ControllerType::Agent(super::AgentControllerType::Help), }, TestCase { name: "Anything else goes to top-level", input: "whatever", expected: super::ControllerType::Agent(super::AgentControllerType::Help), }, TestCase { name: "List", input: "list", expected: super::ControllerType::Agent(super::AgentControllerType::List), }, TestCase { name: "details", input: "details static/agent-id", expected: super::ControllerType::Agent(super::AgentControllerType::Details( PublicIdentifier::Static("agent-id".to_owned()), )), }, TestCase { name: "details with invalid agent identifier", input: "details agent-id", expected: super::ControllerType::Error(crate::strings::agent::invalid_id_generic()), }, TestCase { name: "create-room-local no arguments", input: "create-room-local", expected: super::ControllerType::Error( crate::strings::agent::incorrect_creation_invocation(command_prefix), ), }, TestCase { name: "create-room-local only with provider", input: "create-room-local openai", expected: super::ControllerType::Error( crate::strings::agent::incorrect_creation_invocation(command_prefix), ), }, TestCase { name: "create-room-local correct", input: "create-room-local openai my-agent-id", expected: super::ControllerType::Agent(super::AgentControllerType::CreateRoomLocal { provider: "openai".to_owned(), agent_id: "my-agent-id".trim().to_owned(), }), }, TestCase { name: "create-global extra arguments", input: "create-global openai my-agent-id more arguments here", expected: super::ControllerType::Error( crate::strings::agent::incorrect_creation_invocation(command_prefix), ), }, TestCase { name: "create-global no arguments", input: "create-global", expected: super::ControllerType::Error( crate::strings::agent::incorrect_creation_invocation(command_prefix), ), }, TestCase { name: "create-global only with provider", input: "create-global openai", expected: super::ControllerType::Error( crate::strings::agent::incorrect_creation_invocation(command_prefix), ), }, TestCase { name: "create-global correct", input: "create-global openai my-agent-id", expected: super::ControllerType::Agent(super::AgentControllerType::CreateGlobal { provider: "openai".to_owned(), agent_id: "my-agent-id".trim().to_owned(), }), }, TestCase { name: "create-global extra arguments", input: "create-global openai my-agent-id more arguments here", expected: super::ControllerType::Error( crate::strings::agent::incorrect_creation_invocation(command_prefix), ), }, TestCase { name: "delete no arguments", input: "delete", expected: super::ControllerType::Error( crate::strings::agent::incorrect_invocation_expects_agent_id_arg(command_prefix), ), }, TestCase { name: "delete too many arguments", input: "delete agent-id extra arguments", expected: super::ControllerType::Error( crate::strings::agent::incorrect_invocation_expects_agent_id_arg(command_prefix), ), }, TestCase { name: "delete", input: "delete static/agent-id", expected: super::ControllerType::Agent(super::AgentControllerType::Delete( PublicIdentifier::Static("agent-id".to_owned()), )), }, TestCase { name: "delete with invalid agent identifier", input: "delete agent-id", expected: super::ControllerType::Error(crate::strings::agent::invalid_id_generic()), }, ]; for test_case in test_cases { let result = super::determine_controller(command_prefix, test_case.input); assert_eq!(result, test_case.expected, "Test case: {}", test_case.name); } } ================================================ FILE: src/controller/agent/help/mod.rs ================================================ use mxlink::MessageResponseType; use crate::{Bot, entity::MessageContext, strings}; pub async fn handle(bot: &Bot, message_context: &MessageContext) -> anyhow::Result<()> { // Anyone can access this help command, because certain subcommands ("list") // are also useful to regular users and it'd be great for them to learn about them. let mut message = String::new(); let can_manage_agents = message_context.sender_can_manage_room_local_agents()?; message.push_str(&format!("## {}", strings::help::agent::heading())); message.push_str("\n\n"); message.push_str(&strings::help::agent::intro(bot.command_prefix())); message.push('\n'); message.push_str(&strings::help::agent::intro_capabilities()); message.push_str("\n\n"); message.push_str(&strings::help::agent::intro_handler_relation( bot.command_prefix(), )); if can_manage_agents { message.push_str("\n\n"); message.push_str(strings::help::available_commands_intro()); message.push('\n'); message.push_str(&strings::help::agent::list_agents(bot.command_prefix())); message.push('\n'); message.push_str(strings::help::agent::create_agent_intro()); message.push('\n'); message.push_str(&strings::help::agent::create_agent_room_local( bot.command_prefix(), )); message.push('\n'); if message_context.sender_can_manage_global_config() { message.push_str(&strings::help::agent::create_agent_global( bot.command_prefix(), )); message.push('\n'); } message.push_str(&strings::help::agent::create_agent_example( bot.command_prefix(), )); message.push('\n'); message.push_str(&strings::help::agent::show_agent_details( bot.command_prefix(), )); message.push('\n'); message.push_str(&strings::help::agent::delete_agent(bot.command_prefix())); message.push_str("\n\n"); message.push_str(strings::help::agent::available_commands_outro_update_note()); } else { message.push_str("\n\n"); message.push_str(strings::help::agent::no_permission_to_create_agents()); } bot.messaging() .send_text_markdown_no_fail( message_context.room(), message, MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()), ) .await; Ok(()) } ================================================ FILE: src/controller/agent/list/mod.rs ================================================ use mxlink::MessageResponseType; use crate::agent::AgentPurpose; use crate::strings; use crate::{Bot, entity::MessageContext}; pub async fn handle(bot: &Bot, message_context: &MessageContext) -> anyhow::Result<()> { let agents = bot .agent_manager() .available_room_agents_by_room_config_context(message_context.room_config_context()); let mut message = String::new(); if agents.is_empty() { message.push_str(strings::agent::agent_list_empty().as_str()); } else { message.push_str(&strings::agent::non_empty_agent_list_block(&agents)); message.push_str("\n\n"); message.push_str(strings::agent::agent_list_legend_intro().as_str()); for purpose in AgentPurpose::choices() { message.push_str(&format!( "\n- {} `{}` ({})", purpose.emoji(), purpose.as_str(), strings::agent::purpose_howto(purpose), )); } } bot.messaging() .send_text_markdown_no_fail( message_context.room(), message, MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()), ) .await; Ok(()) } ================================================ FILE: src/controller/agent/mod.rs ================================================ use crate::{Bot, entity::MessageContext}; pub mod create; pub mod delete; pub mod details; pub mod determination; pub mod help; pub mod list; pub use determination::{AgentControllerType, determine_controller}; pub async fn dispatch_controller( handler: &AgentControllerType, message_context: &MessageContext, bot: &Bot, ) -> anyhow::Result<()> { match handler { AgentControllerType::CreateRoomLocal { provider, agent_id } => { create::handle_room_local( bot, bot.room_config_manager(), message_context, provider, agent_id, ) .await } AgentControllerType::CreateGlobal { provider, agent_id } => { create::handle_global( bot, bot.global_config_manager(), message_context, provider, agent_id, ) .await } AgentControllerType::List => list::handle(bot, message_context).await, AgentControllerType::Details(agent_identifier) => { details::handle(bot, message_context, agent_identifier).await } AgentControllerType::Delete(agent_identifier) => { delete::handle( bot, bot.room_config_manager(), bot.global_config_manager(), message_context, agent_identifier, ) .await } AgentControllerType::Help => help::handle(bot, message_context).await, } } ================================================ FILE: src/controller/cfg/common/generic_setting.rs ================================================ use mxlink::MessageResponseType; use crate::{Bot, entity::MessageContext, strings}; pub async fn handle_get( bot: &Bot, message_context: &MessageContext, value: &Option, ) -> anyhow::Result<()> where T: std::fmt::Display, { match value { Some(value) => { bot.messaging() .send_text_markdown_no_fail( message_context.room(), strings::cfg::value_currently_set_to(value), MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()), ) .await; } None => { bot.messaging() .send_text_markdown_no_fail( message_context.room(), strings::cfg::value_currently_unset(), MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()), ) .await; } } Ok(()) } ================================================ FILE: src/controller/cfg/common/mod.rs ================================================ pub(super) mod generic_setting; ================================================ FILE: src/controller/cfg/controller_type.rs ================================================ use crate::{ agent::{AgentPurpose, PublicIdentifier}, entity::roomconfig::{ SpeechToTextFlowType, SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages, TextGenerationAutoUsage, TextGenerationPrefixRequirementType, TextGenerationSenderContextMode, TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType, }, }; #[derive(Debug, PartialEq)] pub enum SettingsStorageSource { Room, Global, } #[derive(Debug, PartialEq)] pub enum ConfigControllerType { Help, Status, SettingsRelated(SettingsStorageSource, ConfigSettingRelatedControllerType), } #[derive(Debug, PartialEq)] pub enum ConfigSettingRelatedControllerType { GetHandler(AgentPurpose), SetHandler(AgentPurpose, Option), TextGeneration(ConfigTextGenerationSettingRelatedControllerType), SpeechToText(ConfigSpeechToTextSettingRelatedControllerType), TextToSpeech(ConfigTextToSpeechSettingRelatedControllerType), } #[derive(Debug, PartialEq)] pub enum ConfigTextGenerationSettingRelatedControllerType { GetContextManagementEnabled, SetContextManagementEnabled(Option), GetPrefixRequirementType, SetPrefixRequirementType(Option), GetAutoUsage, SetAutoUsage(Option), GetPromptOverride, SetPromptOverride(Option), GetTemperatureOverride, SetTemperatureOverride(Option), GetSenderContextMode, SetSenderContextMode(Option), } #[derive(Debug, PartialEq)] pub enum ConfigSpeechToTextSettingRelatedControllerType { GetFlowType, SetFlowType(Option), GetMsgTypeForNonThreadedOnlyTranscribedMessages, SetMsgTypeForNonThreadedOnlyTranscribedMessages( Option, ), GetLanguage, SetLanguage(Option), } #[derive(Debug, PartialEq)] pub enum ConfigTextToSpeechSettingRelatedControllerType { GetBotMessagesFlowType, SetBotMessagesFlowType(Option), GetUserMessagesFlowType, SetUserMessagesFlowType(Option), GetSpeedOverride, SetSpeedOverride(Option), GetVoiceOverride, SetVoiceOverride(Option), } ================================================ FILE: src/controller/cfg/determination/mod.rs ================================================ #[cfg(test)] mod tests; mod speech_to_text; mod text_generation; mod text_to_speech; use crate::{ agent::{AgentPurpose, PublicIdentifier}, controller::ControllerType, strings, }; use super::controller_type::{ ConfigControllerType, ConfigSettingRelatedControllerType, SettingsStorageSource, }; pub fn determine_controller(text: &str) -> ControllerType { if text.starts_with("status") { return ControllerType::Config(ConfigControllerType::Status); } // Someone pasted our instructions verbatim. if text.strip_prefix("CONFIG_TYPE").is_some() { return ControllerType::Error(strings::cfg::error_config_type_not_replaced()); } if let Some(remaining_text) = text.strip_prefix("room ") { return match do_determine_controller(remaining_text.trim()) { Ok(handler) => ControllerType::Config(ConfigControllerType::SettingsRelated( SettingsStorageSource::Room, handler, )), Err(controller_type) => controller_type, }; } if let Some(remaining_text) = text.strip_prefix("global ") { return match do_determine_controller(remaining_text.trim()) { Ok(handler) => ControllerType::Config(ConfigControllerType::SettingsRelated( SettingsStorageSource::Global, handler, )), Err(controller_type) => controller_type, }; } ControllerType::Config(ConfigControllerType::Help) } fn do_determine_controller( text: &str, ) -> Result { if let Some(purpose_str) = text.strip_prefix("handler") { let purpose_str = purpose_str.trim(); if purpose_str.is_empty() { return Err(ControllerType::Error( strings::cfg::configuration_invocation_incorrect_more_values_expected().to_owned(), )); } let Some(purpose) = AgentPurpose::from_str(purpose_str) else { return Err(ControllerType::Error( strings::agent::purpose_unrecognized(purpose_str).to_owned(), )); }; return Ok(ConfigSettingRelatedControllerType::GetHandler(purpose)); } if let Some(remaining_text) = text.strip_prefix("set-handler") { // Something like: // - `PURPOSE ID` // - `PURPOSE` let remaining_text = remaining_text.trim(); if remaining_text.is_empty() { return Err(ControllerType::Error( strings::cfg::configuration_invocation_incorrect_more_values_expected().to_owned(), )); } // This will be None if we're just dealing with `PURPOSE` and lack an `ID`. // In such cases, the whole thing is the purpose string. let parts = remaining_text.split_once(' '); let (purpose_str, agent_id_string_option) = if let Some(parts) = parts { (parts.0, Some(parts.1.to_owned())) } else { (remaining_text, None) }; let Some(purpose) = AgentPurpose::from_str(purpose_str) else { return Err(ControllerType::Error( strings::agent::purpose_unrecognized(purpose_str).to_owned(), )); }; let agent_identifier = match agent_id_string_option { Some(agent_id_string) => { let Some(agent_identifier) = PublicIdentifier::from_str(&agent_id_string) else { return Err(ControllerType::Error( strings::agent::invalid_id_generic().to_owned(), )); }; Some(agent_identifier) } None => None, }; return Ok(ConfigSettingRelatedControllerType::SetHandler( purpose, agent_identifier, )); } if let Some(remaining_text) = text.strip_prefix("text-generation") { return match text_generation::determine(remaining_text.trim()) { Ok(handler) => Ok(ConfigSettingRelatedControllerType::TextGeneration(handler)), Err(controller_type) => Err(controller_type), }; } if let Some(remaining_text) = text.strip_prefix("text-to-speech") { return match text_to_speech::determine(remaining_text.trim()) { Ok(handler) => Ok(ConfigSettingRelatedControllerType::TextToSpeech(handler)), Err(controller_type) => Err(controller_type), }; } if let Some(remaining_text) = text.strip_prefix("speech-to-text") { return match speech_to_text::determine(remaining_text.trim()) { Ok(handler) => Ok(ConfigSettingRelatedControllerType::SpeechToText(handler)), Err(controller_type) => Err(controller_type), }; } Err(ControllerType::Unknown) } ================================================ FILE: src/controller/cfg/determination/speech_to_text/mod.rs ================================================ #[cfg(test)] mod tests; use crate::{ controller::ControllerType, entity::roomconfig::{ SpeechToTextFlowType, SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages, }, strings, }; use super::super::controller_type::ConfigSpeechToTextSettingRelatedControllerType; pub(super) fn determine( text: &str, ) -> Result { // Flow Type if let Some(remaining_text) = text.strip_prefix("flow-type") { let remaining_text = remaining_text.trim(); if !remaining_text.is_empty() { return Err(ControllerType::Error( strings::cfg::configuration_getter_used_with_extra_text( "flow-type", remaining_text, ) .to_owned(), )); } return Ok(ConfigSpeechToTextSettingRelatedControllerType::GetFlowType); } if let Some(value_string) = text.strip_prefix("set-flow-type") { let value_string = value_string.trim().to_owned(); let value_choice = if value_string.is_empty() { None } else { let value_choice = SpeechToTextFlowType::from_str(&value_string.to_lowercase()); if value_choice.is_none() { return Err(ControllerType::Error( strings::cfg::configuration_value_unrecognized(&value_string).to_owned(), )); } value_choice }; return Ok(ConfigSpeechToTextSettingRelatedControllerType::SetFlowType( value_choice, )); } // msg_type_for_non_threaded_only_transcribed_messages if let Some(remaining_text) = text.strip_prefix("msg-type-for-non-threaded-only-transcribed-messages") { let remaining_text = remaining_text.trim(); if !remaining_text.is_empty() { return Err(ControllerType::Error( strings::cfg::configuration_getter_used_with_extra_text( "msg-type-for-non-threaded-only-transcribed-messages", remaining_text, ) .to_owned(), )); } return Ok(ConfigSpeechToTextSettingRelatedControllerType::GetMsgTypeForNonThreadedOnlyTranscribedMessages); } if let Some(value_string) = text.strip_prefix("set-msg-type-for-non-threaded-only-transcribed-messages") { let value_string = value_string.trim().to_owned(); let value_choice = if value_string.is_empty() { None } else { let value_choice = SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::from_str( &value_string.to_lowercase(), ); if value_choice.is_none() { return Err(ControllerType::Error( strings::cfg::configuration_value_unrecognized(&value_string).to_owned(), )); } value_choice }; return Ok(ConfigSpeechToTextSettingRelatedControllerType::SetMsgTypeForNonThreadedOnlyTranscribedMessages( value_choice, )); } // Language if let Some(remaining_text) = text.strip_prefix("language") { let remaining_text = remaining_text.trim(); if !remaining_text.is_empty() { return Err(ControllerType::Error( strings::cfg::configuration_getter_used_with_extra_text("language", remaining_text) .to_owned(), )); } return Ok(ConfigSpeechToTextSettingRelatedControllerType::GetLanguage); } if let Some(value_string) = text.strip_prefix("set-language") { let value_string = value_string.trim().to_owned(); let value_string = if value_string.is_empty() { None } else { if value_string.len() != 2 { return Err(ControllerType::Error( strings::speech_to_text::language_code_invalid(&value_string).to_owned(), )); } Some(value_string) }; return Ok(ConfigSpeechToTextSettingRelatedControllerType::SetLanguage( value_string, )); } Err(ControllerType::Unknown) } ================================================ FILE: src/controller/cfg/determination/speech_to_text/tests.rs ================================================ #[test] fn determine_controller_other() { use super::ConfigSpeechToTextSettingRelatedControllerType; use super::ControllerType; struct TestCase { name: &'static str, input: &'static str, expected: Result, } let test_cases = vec![TestCase { name: "Unknown", input: "whatever", expected: Err(ControllerType::Unknown), }]; for test_case in test_cases { let result = super::determine(test_case.input); assert_eq!(result, test_case.expected, "Test case: {}", test_case.name); } } #[test] fn determine_controller_flow_type() { use super::ConfigSpeechToTextSettingRelatedControllerType; use super::ControllerType; use crate::entity::roomconfig::SpeechToTextFlowType; struct TestCase { name: &'static str, input: &'static str, expected: Result, } let test_cases = vec![ TestCase { name: "flow-type getter ok", input: "flow-type", expected: Ok(ConfigSpeechToTextSettingRelatedControllerType::GetFlowType), }, TestCase { name: "flow-type getter extra args", input: "flow-type some values here", expected: Err(ControllerType::Error( crate::strings::cfg::configuration_getter_used_with_extra_text( "flow-type", "some values here", ), )), }, TestCase { name: "flow-type setter", input: "set-flow-type transcribe_and_generate_text", expected: Ok(ConfigSpeechToTextSettingRelatedControllerType::SetFlowType( Some(SpeechToTextFlowType::TranscribeAndGenerateText), )), }, TestCase { name: "flow-type setter", input: "set-flow-type unknown-Value", expected: Err(ControllerType::Error( crate::strings::cfg::configuration_value_unrecognized("unknown-Value"), )), }, TestCase { name: "flow-type unsetter", input: "set-flow-type", expected: Ok(ConfigSpeechToTextSettingRelatedControllerType::SetFlowType( None, )), }, ]; for test_case in test_cases { let result = super::determine(test_case.input); assert_eq!(result, test_case.expected, "Test case: {}", test_case.name); } } #[test] fn determine_controller_language() { use super::ConfigSpeechToTextSettingRelatedControllerType; use super::ControllerType; struct TestCase { name: &'static str, input: &'static str, expected: Result, } let test_cases = vec![ TestCase { name: "language getter ok", input: "language", expected: Ok(ConfigSpeechToTextSettingRelatedControllerType::GetLanguage), }, TestCase { name: "language getter extra args", input: "language some values here", expected: Err(ControllerType::Error( crate::strings::cfg::configuration_getter_used_with_extra_text( "language", "some values here", ), )), }, TestCase { name: "language setter 2-letter code (ja)", input: "set-language ja", expected: Ok(ConfigSpeechToTextSettingRelatedControllerType::SetLanguage( Some("ja".to_owned()), )), }, // OpenAI does not support 3-letter codes, so we won't be allowing it either TestCase { name: "language setter 3-letter code (jpn) fails", input: "set-language jpn", expected: Err(ControllerType::Error( crate::strings::speech_to_text::language_code_invalid("jpn"), )), }, TestCase { name: "language unsetter", input: "set-language", expected: Ok(ConfigSpeechToTextSettingRelatedControllerType::SetLanguage( None, )), }, ]; for test_case in test_cases { let result = super::determine(test_case.input); assert_eq!(result, test_case.expected, "Test case: {}", test_case.name); } } ================================================ FILE: src/controller/cfg/determination/tests.rs ================================================ #[test] fn determine_controller() { use super::super::controller_type; use crate::agent::{AgentPurpose, PublicIdentifier}; struct TestCase { name: &'static str, input: &'static str, expected: super::ControllerType, } let test_cases = vec![ TestCase { name: "Top-level is help", input: "", expected: super::ControllerType::Config(controller_type::ConfigControllerType::Help), }, TestCase { name: "unknown commands is help", input: "whatever", expected: super::ControllerType::Config(controller_type::ConfigControllerType::Help), }, TestCase { name: "Status", input: "status", expected: super::ControllerType::Config(controller_type::ConfigControllerType::Status), }, TestCase { name: "per-room handler getter - catch-all", input: "room handler catch-all", expected: super::ControllerType::Config(controller_type::ConfigControllerType::SettingsRelated( controller_type::SettingsStorageSource::Room, controller_type::ConfigSettingRelatedControllerType::GetHandler(AgentPurpose::CatchAll), )), }, TestCase { name: "per-room handler getter - text-generation", input: "room handler text-generation", expected: super::ControllerType::Config(controller_type::ConfigControllerType::SettingsRelated( controller_type::SettingsStorageSource::Room, controller_type::ConfigSettingRelatedControllerType::GetHandler(AgentPurpose::TextGeneration), )), }, TestCase { name: "per-room handler getter - invalid purpose", input: "room handler invalid-purpose-here", expected: super::ControllerType::Error( crate::strings::agent::purpose_unrecognized("invalid-purpose-here").to_owned() ), }, TestCase { name: "per-room handler getter - invalid purpose with spaces", input: "room handler invalid purpose here", expected: super::ControllerType::Error( crate::strings::agent::purpose_unrecognized("invalid purpose here").to_owned() ), }, TestCase { name: "per-room handler setter - too few values", input: "room set-handler", expected: super::ControllerType::Error( crate::strings::cfg::configuration_invocation_incorrect_more_values_expected().to_owned() ), }, TestCase { name: "per-room handler setter - catch-all", input: "room set-handler catch-all static/agent-id", expected: super::ControllerType::Config(controller_type::ConfigControllerType::SettingsRelated( controller_type::SettingsStorageSource::Room, controller_type::ConfigSettingRelatedControllerType::SetHandler(AgentPurpose::CatchAll, Some( PublicIdentifier::Static("agent-id".to_owned()) )), )), }, TestCase { name: "per-room handler setter - catch-all with bare agent id", input: "room set-handler catch-all agent-id", expected: super::ControllerType::Error( crate::strings::agent::invalid_id_generic().to_owned() ), }, TestCase { name: "per-room handler setter - catch-all unsetter", input: "room set-handler catch-all", expected: super::ControllerType::Config(controller_type::ConfigControllerType::SettingsRelated( controller_type::SettingsStorageSource::Room, controller_type::ConfigSettingRelatedControllerType::SetHandler(AgentPurpose::CatchAll, None), )), }, TestCase { name: "per-room handler setter - text-generation", input: "room set-handler text-generation room-local/agent-id", expected: super::ControllerType::Config(controller_type::ConfigControllerType::SettingsRelated( controller_type::SettingsStorageSource::Room, controller_type::ConfigSettingRelatedControllerType::SetHandler(AgentPurpose::TextGeneration, Some( PublicIdentifier::DynamicRoomLocal("agent-id".to_owned()) )), )), }, TestCase { name: "per-room handler setter - too many values", input: "room set-handler text-generation agent-id more values here", expected: super::ControllerType::Error( crate::strings::agent::invalid_id_generic().to_owned() ), }, // We have few global handler test cases. We've exercised the per-room handlers enough. // These share the same code path, so we don't need to test all the permutations again. TestCase { name: "global handler getter - catch-all", input: "global handler catch-all", expected: super::ControllerType::Config(controller_type::ConfigControllerType::SettingsRelated( controller_type::SettingsStorageSource::Global, controller_type::ConfigSettingRelatedControllerType::GetHandler(AgentPurpose::CatchAll), )), }, TestCase { name: "global handler setter - text-generation with global agent", input: "global set-handler text-generation global/agent-id", expected: super::ControllerType::Config(controller_type::ConfigControllerType::SettingsRelated( controller_type::SettingsStorageSource::Global, controller_type::ConfigSettingRelatedControllerType::SetHandler(AgentPurpose::TextGeneration, Some( PublicIdentifier::DynamicGlobal("agent-id".to_owned()) )), )), }, // This test case passes, even though the handler function will subsequently reject using room-local agents for global handlers. TestCase { name: "global handler setter - text-generation with room-local agent", input: "global set-handler text-generation room-local/agent-id", expected: super::ControllerType::Config(controller_type::ConfigControllerType::SettingsRelated( controller_type::SettingsStorageSource::Global, controller_type::ConfigSettingRelatedControllerType::SetHandler(AgentPurpose::TextGeneration, Some( PublicIdentifier::DynamicRoomLocal("agent-id".to_owned()) )), )), }, // We'll only test one handler per sub-category to ensure proper routing is done here. // Extensive tests for each sub-category are done in their respective modules. TestCase { name: "per-room text-generation/context-management-enabled getter", input: "room text-generation context-management-enabled", expected: super::ControllerType::Config(controller_type::ConfigControllerType::SettingsRelated( controller_type::SettingsStorageSource::Room, controller_type::ConfigSettingRelatedControllerType::TextGeneration( controller_type::ConfigTextGenerationSettingRelatedControllerType::GetContextManagementEnabled, ), )), }, TestCase { name: "global text-generation/context-management-enabled getter", input: "global text-generation context-management-enabled", expected: super::ControllerType::Config(controller_type::ConfigControllerType::SettingsRelated( controller_type::SettingsStorageSource::Global, controller_type::ConfigSettingRelatedControllerType::TextGeneration( controller_type::ConfigTextGenerationSettingRelatedControllerType::GetContextManagementEnabled, ), )), }, TestCase { name: "per-room text-generation/sender-context-mode getter", input: "room text-generation sender-context-mode", expected: super::ControllerType::Config(controller_type::ConfigControllerType::SettingsRelated( controller_type::SettingsStorageSource::Room, controller_type::ConfigSettingRelatedControllerType::TextGeneration( controller_type::ConfigTextGenerationSettingRelatedControllerType::GetSenderContextMode, ), )), }, TestCase { name: "global text-generation/sender-context-mode getter", input: "global text-generation sender-context-mode", expected: super::ControllerType::Config(controller_type::ConfigControllerType::SettingsRelated( controller_type::SettingsStorageSource::Global, controller_type::ConfigSettingRelatedControllerType::TextGeneration( controller_type::ConfigTextGenerationSettingRelatedControllerType::GetSenderContextMode, ), )), }, TestCase { name: "per-room text-to-speech/speed-override getter", input: "room text-to-speech speed-override", expected: super::ControllerType::Config(controller_type::ConfigControllerType::SettingsRelated( controller_type::SettingsStorageSource::Room, controller_type::ConfigSettingRelatedControllerType::TextToSpeech( controller_type::ConfigTextToSpeechSettingRelatedControllerType::GetSpeedOverride, ), )), }, TestCase { name: "global text-to-speech/speed-override getter", input: "global text-to-speech speed-override", expected: super::ControllerType::Config(controller_type::ConfigControllerType::SettingsRelated( controller_type::SettingsStorageSource::Global, controller_type::ConfigSettingRelatedControllerType::TextToSpeech( controller_type::ConfigTextToSpeechSettingRelatedControllerType::GetSpeedOverride, ), )), }, TestCase { name: "room speech-to-text/flow-type getter", input: "room speech-to-text flow-type", expected: super::ControllerType::Config(controller_type::ConfigControllerType::SettingsRelated( controller_type::SettingsStorageSource::Room, controller_type::ConfigSettingRelatedControllerType::SpeechToText( controller_type::ConfigSpeechToTextSettingRelatedControllerType::GetFlowType, ), )), }, TestCase { name: "global speech-to-text/flow-type getter", input: "global speech-to-text flow-type", expected: super::ControllerType::Config(controller_type::ConfigControllerType::SettingsRelated( controller_type::SettingsStorageSource::Global, controller_type::ConfigSettingRelatedControllerType::SpeechToText( controller_type::ConfigSpeechToTextSettingRelatedControllerType::GetFlowType, ), )), }, ]; for test_case in test_cases { let result = super::determine_controller(test_case.input); assert_eq!(result, test_case.expected, "Test case: {}", test_case.name); } } ================================================ FILE: src/controller/cfg/determination/text_generation/mod.rs ================================================ #[cfg(test)] mod tests; use crate::{ controller::ControllerType, entity::roomconfig::{ TextGenerationAutoUsage, TextGenerationPrefixRequirementType, TextGenerationSenderContextMode, }, strings, }; use super::super::controller_type::ConfigTextGenerationSettingRelatedControllerType; pub(super) fn determine( text: &str, ) -> Result { if let Some(remaining_text) = text.strip_prefix("context-management-enabled") { let remaining_text = remaining_text.trim(); if !remaining_text.is_empty() { return Err(ControllerType::Error( strings::cfg::configuration_getter_used_with_extra_text( "context-management-enabled", remaining_text, ) .to_owned(), )); } return Ok(ConfigTextGenerationSettingRelatedControllerType::GetContextManagementEnabled); } if let Some(value_string) = text.strip_prefix("set-context-management-enabled") { let value_string = value_string.trim().to_owned(); let value_opt = if value_string.is_empty() { None } else { let value_string_lowercase = value_string.to_lowercase(); Some(match value_string_lowercase.as_str() { "true" => true, "false" => false, _ => { return Err(ControllerType::Error( strings::cfg::configuration_value_unrecognized(&value_string).to_owned(), )); } }) }; return Ok( ConfigTextGenerationSettingRelatedControllerType::SetContextManagementEnabled( value_opt, ), ); } if let Some(remaining_text) = text.strip_prefix("prefix-requirement-type") { let remaining_text = remaining_text.trim(); if !remaining_text.is_empty() { return Err(ControllerType::Error( strings::cfg::configuration_getter_used_with_extra_text( "prefix-requirement-type", remaining_text, ) .to_owned(), )); } return Ok(ConfigTextGenerationSettingRelatedControllerType::GetPrefixRequirementType); } if let Some(value_string) = text.strip_prefix("set-prefix-requirement-type") { let value_string = value_string.trim().to_owned(); let value_choice = if value_string.is_empty() { None } else { let value_choice = TextGenerationPrefixRequirementType::from_str(&value_string.to_lowercase()); if value_choice.is_none() { return Err(ControllerType::Error( strings::cfg::configuration_value_unrecognized(&value_string).to_owned(), )); } value_choice }; return Ok( ConfigTextGenerationSettingRelatedControllerType::SetPrefixRequirementType( value_choice, ), ); } if let Some(remaining_text) = text.strip_prefix("auto-usage") { let remaining_text = remaining_text.trim(); if !remaining_text.is_empty() { return Err(ControllerType::Error( strings::cfg::configuration_getter_used_with_extra_text( "auto-usage", remaining_text, ) .to_owned(), )); } return Ok(ConfigTextGenerationSettingRelatedControllerType::GetAutoUsage); } if let Some(value_string) = text.strip_prefix("set-auto-usage") { let value_string = value_string.trim().to_owned(); let value_choice = if value_string.is_empty() { None } else { let value_choice = TextGenerationAutoUsage::from_str(&value_string.to_lowercase()); if value_choice.is_none() { return Err(ControllerType::Error( strings::cfg::configuration_value_unrecognized(&value_string).to_owned(), )); } value_choice }; return Ok(ConfigTextGenerationSettingRelatedControllerType::SetAutoUsage(value_choice)); } if let Some(remaining_text) = text.strip_prefix("prompt-override") { let remaining_text = remaining_text.trim(); if !remaining_text.is_empty() { return Err(ControllerType::Error( strings::cfg::configuration_getter_used_with_extra_text( "prompt-override", remaining_text, ) .to_owned(), )); } return Ok(ConfigTextGenerationSettingRelatedControllerType::GetPromptOverride); } if let Some(value_string) = text.strip_prefix("set-prompt-override") { let value_string = value_string.trim().to_owned(); if value_string.is_empty() { return Ok(ConfigTextGenerationSettingRelatedControllerType::SetPromptOverride(None)); } return Ok( ConfigTextGenerationSettingRelatedControllerType::SetPromptOverride(Some(value_string)), ); } if let Some(remaining_text) = text.strip_prefix("temperature-override") { let remaining_text = remaining_text.trim(); if !remaining_text.is_empty() { return Err(ControllerType::Error( strings::cfg::configuration_getter_used_with_extra_text( "temperature-override", remaining_text, ) .to_owned(), )); } return Ok(ConfigTextGenerationSettingRelatedControllerType::GetTemperatureOverride); } if let Some(value_string) = text.strip_prefix("set-temperature-override") { let value_string = value_string.trim().to_owned(); if value_string.is_empty() { return Ok( ConfigTextGenerationSettingRelatedControllerType::SetTemperatureOverride(None), ); } let value_f32 = value_string.parse::(); let Ok(value_f32) = value_f32 else { return Err(ControllerType::Error( strings::cfg::configuration_value_not_f32(&value_string).to_owned(), )); }; return Ok( ConfigTextGenerationSettingRelatedControllerType::SetTemperatureOverride(Some( value_f32, )), ); } if let Some(remaining_text) = text.strip_prefix("sender-context-mode") { let remaining_text = remaining_text.trim(); if !remaining_text.is_empty() { return Err(ControllerType::Error( strings::cfg::configuration_getter_used_with_extra_text( "sender-context-mode", remaining_text, ) .to_owned(), )); } return Ok(ConfigTextGenerationSettingRelatedControllerType::GetSenderContextMode); } if let Some(value_string) = text.strip_prefix("set-sender-context-mode") { let value_string = value_string.trim().to_owned(); let value_choice = if value_string.is_empty() { None } else { let value_choice = TextGenerationSenderContextMode::from_str(&value_string.to_lowercase()); if value_choice.is_none() { return Err(ControllerType::Error( strings::cfg::configuration_value_unrecognized(&value_string).to_owned(), )); } value_choice }; return Ok( ConfigTextGenerationSettingRelatedControllerType::SetSenderContextMode(value_choice), ); } Err(ControllerType::Unknown) } ================================================ FILE: src/controller/cfg/determination/text_generation/tests.rs ================================================ #[test] fn determine_controller_other() { use super::ConfigTextGenerationSettingRelatedControllerType; use super::ControllerType; struct TestCase { name: &'static str, input: &'static str, expected: Result, } let test_cases = vec![TestCase { name: "Unknown", input: "whatever", expected: Err(ControllerType::Unknown), }]; for test_case in test_cases { let result = super::determine(test_case.input); assert_eq!(result, test_case.expected, "Test case: {}", test_case.name); } } #[test] fn determine_controller_context_management() { use super::ConfigTextGenerationSettingRelatedControllerType; use super::ControllerType; struct TestCase { name: &'static str, input: &'static str, expected: Result, } let test_cases = vec![ TestCase { name: "context-management-enabled getter ok", input: "context-management-enabled", expected: Ok( ConfigTextGenerationSettingRelatedControllerType::GetContextManagementEnabled, ), }, TestCase { name: "context-management-enabled getter extra args", input: "context-management-enabled some values here", expected: Err(ControllerType::Error( crate::strings::cfg::configuration_getter_used_with_extra_text( "context-management-enabled", "some values here", ), )), }, TestCase { name: "context-management-enabled setter", input: "set-context-management-enabled true", expected: Ok( ConfigTextGenerationSettingRelatedControllerType::SetContextManagementEnabled( Some(true), ), ), }, TestCase { name: "context-management-enabled setter uppercase", input: "set-context-management-enabled TRUE", expected: Ok( ConfigTextGenerationSettingRelatedControllerType::SetContextManagementEnabled( Some(true), ), ), }, TestCase { name: "context-management-enabled setter non-bool", input: "set-context-management-enabled non-Bool-Value", expected: Err(ControllerType::Error( crate::strings::cfg::configuration_value_unrecognized("non-Bool-Value"), )), }, TestCase { name: "context-management-enabled unsetter", input: "set-context-management-enabled", expected: Ok( ConfigTextGenerationSettingRelatedControllerType::SetContextManagementEnabled(None), ), }, ]; for test_case in test_cases { let result = super::determine(test_case.input); assert_eq!(result, test_case.expected, "Test case: {}", test_case.name); } } #[test] fn determine_controller_sender_context() { use super::ConfigTextGenerationSettingRelatedControllerType; use super::ControllerType; use crate::entity::roomconfig::TextGenerationSenderContextMode; struct TestCase { name: &'static str, input: &'static str, expected: Result, } let test_cases = vec![ TestCase { name: "sender-context-mode getter ok", input: "sender-context-mode", expected: Ok(ConfigTextGenerationSettingRelatedControllerType::GetSenderContextMode), }, TestCase { name: "sender-context-mode getter extra args", input: "sender-context-mode some values here", expected: Err(ControllerType::Error( crate::strings::cfg::configuration_getter_used_with_extra_text( "sender-context-mode", "some values here", ), )), }, TestCase { name: "sender-context-mode setter matrix_user_id", input: "set-sender-context-mode matrix_user_id", expected: Ok( ConfigTextGenerationSettingRelatedControllerType::SetSenderContextMode(Some( TextGenerationSenderContextMode::MatrixUserId, )), ), }, TestCase { name: "sender-context-mode setter uppercase", input: "set-sender-context-mode MATRIX_USER_ID_AND_TIMESTAMP", expected: Ok( ConfigTextGenerationSettingRelatedControllerType::SetSenderContextMode(Some( TextGenerationSenderContextMode::MatrixUserIdAndTimestamp, )), ), }, TestCase { name: "sender-context-mode setter invalid", input: "set-sender-context-mode non-Enum-Value", expected: Err(ControllerType::Error( crate::strings::cfg::configuration_value_unrecognized("non-Enum-Value"), )), }, TestCase { name: "sender-context-mode unsetter", input: "set-sender-context-mode", expected: Ok( ConfigTextGenerationSettingRelatedControllerType::SetSenderContextMode(None), ), }, ]; for test_case in test_cases { let result = super::determine(test_case.input); assert_eq!(result, test_case.expected, "Test case: {}", test_case.name); } } #[test] fn determine_controller_prefix_requirement_type() { use super::ConfigTextGenerationSettingRelatedControllerType; use super::ControllerType; use crate::entity::roomconfig::TextGenerationPrefixRequirementType; struct TestCase { name: &'static str, input: &'static str, expected: Result, } let test_cases = vec![ TestCase { name: "prefix-requirement-type getter ok", input: "prefix-requirement-type", expected: Ok( ConfigTextGenerationSettingRelatedControllerType::GetPrefixRequirementType, ), }, TestCase { name: "prefix-requirement-type getter extra args", input: "prefix-requirement-type some values here", expected: Err(ControllerType::Error( crate::strings::cfg::configuration_getter_used_with_extra_text( "prefix-requirement-type", "some values here", ), )), }, TestCase { name: "prefix-requirement-type setter (command_prefix)", input: "set-prefix-requirement-type command_prefix", expected: Ok( ConfigTextGenerationSettingRelatedControllerType::SetPrefixRequirementType(Some( TextGenerationPrefixRequirementType::CommandPrefix, )), ), }, TestCase { name: "prefix-requirement-type setter (no)", input: "set-prefix-requirement-type no", expected: Ok( ConfigTextGenerationSettingRelatedControllerType::SetPrefixRequirementType(Some( TextGenerationPrefixRequirementType::No, )), ), }, TestCase { name: "prefix-requirement-type setter", input: "set-prefix-requirement-type unknown-Value", expected: Err(ControllerType::Error( crate::strings::cfg::configuration_value_unrecognized("unknown-Value"), )), }, TestCase { name: "prefix-requirement-type unsetter", input: "set-prefix-requirement-type", expected: Ok( ConfigTextGenerationSettingRelatedControllerType::SetPrefixRequirementType(None), ), }, ]; for test_case in test_cases { let result = super::determine(test_case.input); assert_eq!(result, test_case.expected, "Test case: {}", test_case.name); } } #[test] fn determine_controller_auto_usage() { use super::ConfigTextGenerationSettingRelatedControllerType; use super::ControllerType; use crate::entity::roomconfig::TextGenerationAutoUsage; struct TestCase { name: &'static str, input: &'static str, expected: Result, } let test_cases = vec![ TestCase { name: "auto-usage getter ok", input: "auto-usage", expected: Ok(ConfigTextGenerationSettingRelatedControllerType::GetAutoUsage), }, TestCase { name: "auto-usage getter extra args", input: "auto-usage some values here", expected: Err(ControllerType::Error( crate::strings::cfg::configuration_getter_used_with_extra_text( "auto-usage", "some values here", ), )), }, TestCase { name: "auto-usage setter", input: "set-auto-usage only_for_voice", expected: Ok( ConfigTextGenerationSettingRelatedControllerType::SetAutoUsage(Some( TextGenerationAutoUsage::OnlyForVoice, )), ), }, TestCase { name: "auto-usage setter", input: "set-auto-usage unknown-Value", expected: Err(ControllerType::Error( crate::strings::cfg::configuration_value_unrecognized("unknown-Value"), )), }, TestCase { name: "auto-usage unsetter", input: "set-auto-usage", expected: Ok(ConfigTextGenerationSettingRelatedControllerType::SetAutoUsage(None)), }, ]; for test_case in test_cases { let result = super::determine(test_case.input); assert_eq!(result, test_case.expected, "Test case: {}", test_case.name); } } #[test] fn determine_controller_prompt_override() { use super::ConfigTextGenerationSettingRelatedControllerType; use super::ControllerType; struct TestCase { name: &'static str, input: &'static str, expected: Result, } let test_cases = vec![ TestCase { name: "prompt-override getter ok", input: "prompt-override", expected: Ok(ConfigTextGenerationSettingRelatedControllerType::GetPromptOverride), }, TestCase { name: "prompt-override getter extra args", input: "prompt-override some values here", expected: Err(ControllerType::Error( crate::strings::cfg::configuration_getter_used_with_extra_text( "prompt-override", "some values here", ), )), }, TestCase { name: "prompt-override setter with multiple words", input: "set-prompt-override Hello! You are a bot", expected: Ok( ConfigTextGenerationSettingRelatedControllerType::SetPromptOverride(Some( "Hello! You are a bot".to_owned(), )), ), }, TestCase { name: "prompt-override setter with multi-line", input: "set-prompt-override Hello!\n\nYou are a bot", expected: Ok( ConfigTextGenerationSettingRelatedControllerType::SetPromptOverride(Some( "Hello!\n\nYou are a bot".to_owned(), )), ), }, TestCase { name: "prompt-override unsetter", input: "set-prompt-override", expected: Ok(ConfigTextGenerationSettingRelatedControllerType::SetPromptOverride(None)), }, ]; for test_case in test_cases { let result = super::determine(test_case.input); assert_eq!(result, test_case.expected, "Test case: {}", test_case.name); } } #[test] fn determine_controller_temperature_override() { use super::ConfigTextGenerationSettingRelatedControllerType; use super::ControllerType; struct TestCase { name: &'static str, input: &'static str, expected: Result, } let test_cases = vec![ TestCase { name: "temperature-override getter ok", input: "temperature-override", expected: Ok(ConfigTextGenerationSettingRelatedControllerType::GetTemperatureOverride), }, TestCase { name: "temperature-override getter extra args", input: "temperature-override some values here", expected: Err(ControllerType::Error( crate::strings::cfg::configuration_getter_used_with_extra_text( "temperature-override", "some values here", ), )), }, TestCase { name: "temperature-override setter", input: "set-temperature-override 0.5", expected: Ok( ConfigTextGenerationSettingRelatedControllerType::SetTemperatureOverride(Some(0.5)), ), }, TestCase { name: "temperature-override setter", input: "set-temperature-override unknown-Value", expected: Err(ControllerType::Error( crate::strings::cfg::configuration_value_not_f32("unknown-Value"), )), }, TestCase { name: "temperature-override unsetter", input: "set-temperature-override", expected: Ok( ConfigTextGenerationSettingRelatedControllerType::SetTemperatureOverride(None), ), }, ]; for test_case in test_cases { let result = super::determine(test_case.input); assert_eq!(result, test_case.expected, "Test case: {}", test_case.name); } } ================================================ FILE: src/controller/cfg/determination/text_to_speech/mod.rs ================================================ #[cfg(test)] mod tests; use crate::{ controller::ControllerType, entity::roomconfig::{TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType}, strings, }; use super::super::controller_type::ConfigTextToSpeechSettingRelatedControllerType; pub(super) fn determine( text: &str, ) -> Result { if let Some(remaining_text) = text.strip_prefix("bot-msgs-flow-type") { let remaining_text = remaining_text.trim(); if !remaining_text.is_empty() { return Err(ControllerType::Error( strings::cfg::configuration_getter_used_with_extra_text( "bot-msgs-flow-type", remaining_text, ) .to_owned(), )); } return Ok(ConfigTextToSpeechSettingRelatedControllerType::GetBotMessagesFlowType); } if let Some(value_string) = text.strip_prefix("set-bot-msgs-flow-type") { let value_string = value_string.trim().to_owned(); let value_choice = if value_string.is_empty() { None } else { let value_choice = TextToSpeechBotMessagesFlowType::from_str(&value_string.to_lowercase()); if value_choice.is_none() { return Err(ControllerType::Error( strings::cfg::configuration_value_unrecognized(&value_string).to_owned(), )); } value_choice }; return Ok( ConfigTextToSpeechSettingRelatedControllerType::SetBotMessagesFlowType(value_choice), ); } if let Some(remaining_text) = text.strip_prefix("user-msgs-flow-type") { let remaining_text = remaining_text.trim(); if !remaining_text.is_empty() { return Err(ControllerType::Error( strings::cfg::configuration_getter_used_with_extra_text( "user-msgs-flow-type", remaining_text, ) .to_owned(), )); } return Ok(ConfigTextToSpeechSettingRelatedControllerType::GetUserMessagesFlowType); } if let Some(value_string) = text.strip_prefix("set-user-msgs-flow-type") { let value_string = value_string.trim().to_owned(); let value_choice = if value_string.is_empty() { None } else { let value_choice = TextToSpeechUserMessagesFlowType::from_str(&value_string.to_lowercase()); if value_choice.is_none() { return Err(ControllerType::Error( strings::cfg::configuration_value_unrecognized(&value_string).to_owned(), )); } value_choice }; return Ok( ConfigTextToSpeechSettingRelatedControllerType::SetUserMessagesFlowType(value_choice), ); } if let Some(remaining_text) = text.strip_prefix("speed-override") { let remaining_text = remaining_text.trim(); if !remaining_text.is_empty() { return Err(ControllerType::Error( strings::cfg::configuration_getter_used_with_extra_text( "speed-override", remaining_text, ) .to_owned(), )); } return Ok(ConfigTextToSpeechSettingRelatedControllerType::GetSpeedOverride); } if let Some(value_string) = text.strip_prefix("set-speed-override") { let value_string = value_string.trim().to_owned(); if value_string.is_empty() { return Ok(ConfigTextToSpeechSettingRelatedControllerType::SetSpeedOverride(None)); } let value_f32 = value_string.parse::(); let Ok(value_f32) = value_f32 else { return Err(ControllerType::Error( strings::cfg::configuration_value_not_f32(&value_string).to_owned(), )); }; return Ok( ConfigTextToSpeechSettingRelatedControllerType::SetSpeedOverride(Some(value_f32)), ); } if let Some(remaining_text) = text.strip_prefix("voice-override") { let remaining_text = remaining_text.trim(); if !remaining_text.is_empty() { return Err(ControllerType::Error( strings::cfg::configuration_getter_used_with_extra_text( "voice-override", remaining_text, ) .to_owned(), )); } return Ok(ConfigTextToSpeechSettingRelatedControllerType::GetVoiceOverride); } if let Some(value_string) = text.strip_prefix("set-voice-override") { let value_string = value_string.trim().to_owned(); if value_string.is_empty() { return Ok(ConfigTextToSpeechSettingRelatedControllerType::SetVoiceOverride(None)); } return Ok( ConfigTextToSpeechSettingRelatedControllerType::SetVoiceOverride(Some(value_string)), ); } Err(ControllerType::Unknown) } ================================================ FILE: src/controller/cfg/determination/text_to_speech/tests.rs ================================================ #[test] fn determine_controller_other() { use super::ConfigTextToSpeechSettingRelatedControllerType; use super::ControllerType; struct TestCase { name: &'static str, input: &'static str, expected: Result, } let test_cases = vec![TestCase { name: "Unknown", input: "whatever", expected: Err(ControllerType::Unknown), }]; for test_case in test_cases { let result = super::determine(test_case.input); assert_eq!(result, test_case.expected, "Test case: {}", test_case.name); } } #[test] fn determine_controller_bot_msgs_flow_type() { use super::ConfigTextToSpeechSettingRelatedControllerType; use super::ControllerType; use crate::entity::roomconfig::TextToSpeechBotMessagesFlowType; struct TestCase { name: &'static str, input: &'static str, expected: Result, } let test_cases = vec![ TestCase { name: "bot-msgs-flow-type getter ok", input: "bot-msgs-flow-type", expected: Ok(ConfigTextToSpeechSettingRelatedControllerType::GetBotMessagesFlowType), }, TestCase { name: "bot-msgs-flow-type getter extra args", input: "bot-msgs-flow-type some values here", expected: Err(ControllerType::Error( crate::strings::cfg::configuration_getter_used_with_extra_text( "bot-msgs-flow-type", "some values here", ), )), }, TestCase { name: "bot-msgs-flow-type setter", input: "set-bot-msgs-flow-type only_for_voice", expected: Ok( ConfigTextToSpeechSettingRelatedControllerType::SetBotMessagesFlowType(Some( TextToSpeechBotMessagesFlowType::OnlyForVoice, )), ), }, TestCase { name: "bot-msgs-flow-type setter", input: "set-bot-msgs-flow-type unknown-Value", expected: Err(ControllerType::Error( crate::strings::cfg::configuration_value_unrecognized("unknown-Value"), )), }, TestCase { name: "bot-msgs-flow-type unsetter", input: "set-bot-msgs-flow-type", expected: Ok( ConfigTextToSpeechSettingRelatedControllerType::SetBotMessagesFlowType(None), ), }, ]; for test_case in test_cases { let result = super::determine(test_case.input); assert_eq!(result, test_case.expected, "Test case: {}", test_case.name); } } #[test] fn determine_controller_user_msgs_flow_type() { use super::ConfigTextToSpeechSettingRelatedControllerType; use super::ControllerType; use crate::entity::roomconfig::TextToSpeechUserMessagesFlowType; struct TestCase { name: &'static str, input: &'static str, expected: Result, } let test_cases = vec![ TestCase { name: "user-msgs-flow-type getter ok", input: "user-msgs-flow-type", expected: Ok(ConfigTextToSpeechSettingRelatedControllerType::GetUserMessagesFlowType), }, TestCase { name: "user-msgs-flow-type getter extra args", input: "user-msgs-flow-type some values here", expected: Err(ControllerType::Error( crate::strings::cfg::configuration_getter_used_with_extra_text( "user-msgs-flow-type", "some values here", ), )), }, TestCase { name: "user-msgs-flow-type setter", input: "set-user-msgs-flow-type on_demand", expected: Ok( ConfigTextToSpeechSettingRelatedControllerType::SetUserMessagesFlowType(Some( TextToSpeechUserMessagesFlowType::OnDemand, )), ), }, TestCase { name: "user-msgs-flow-type setter", input: "set-user-msgs-flow-type unknown-Value", expected: Err(ControllerType::Error( crate::strings::cfg::configuration_value_unrecognized("unknown-Value"), )), }, TestCase { name: "user-msgs-flow-type unsetter", input: "set-user-msgs-flow-type", expected: Ok( ConfigTextToSpeechSettingRelatedControllerType::SetUserMessagesFlowType(None), ), }, ]; for test_case in test_cases { let result = super::determine(test_case.input); assert_eq!(result, test_case.expected, "Test case: {}", test_case.name); } } #[test] fn determine_controller_speed_override() { use super::ConfigTextToSpeechSettingRelatedControllerType; use super::ControllerType; struct TestCase { name: &'static str, input: &'static str, expected: Result, } let test_cases = vec![ TestCase { name: "speed-override getter ok", input: "speed-override", expected: Ok(ConfigTextToSpeechSettingRelatedControllerType::GetSpeedOverride), }, TestCase { name: "speed-override getter extra args", input: "speed-override some values here", expected: Err(ControllerType::Error( crate::strings::cfg::configuration_getter_used_with_extra_text( "speed-override", "some values here", ), )), }, TestCase { name: "speed-override setter", input: "set-speed-override 0.5", expected: Ok( ConfigTextToSpeechSettingRelatedControllerType::SetSpeedOverride(Some(0.5)), ), }, TestCase { name: "speed-override setter", input: "set-speed-override unknown-Value", expected: Err(ControllerType::Error( crate::strings::cfg::configuration_value_not_f32("unknown-Value"), )), }, TestCase { name: "speed-override unsetter", input: "set-speed-override", expected: Ok(ConfigTextToSpeechSettingRelatedControllerType::SetSpeedOverride(None)), }, ]; for test_case in test_cases { let result = super::determine(test_case.input); assert_eq!(result, test_case.expected, "Test case: {}", test_case.name); } } #[test] fn determine_controller_voice_override() { use super::ConfigTextToSpeechSettingRelatedControllerType; use super::ControllerType; struct TestCase { name: &'static str, input: &'static str, expected: Result, } let test_cases = vec![ TestCase { name: "voice-override getter ok", input: "voice-override", expected: Ok(ConfigTextToSpeechSettingRelatedControllerType::GetVoiceOverride), }, TestCase { name: "voice-override getter extra args", input: "voice-override some values here", expected: Err(ControllerType::Error( crate::strings::cfg::configuration_getter_used_with_extra_text( "voice-override", "some values here", ), )), }, TestCase { name: "voice-override setter", input: "set-voice-override alex", expected: Ok( ConfigTextToSpeechSettingRelatedControllerType::SetVoiceOverride(Some( "alex".to_owned(), )), ), }, TestCase { name: "voice-override setter preserves case", input: "set-voice-override Alex", expected: Ok( ConfigTextToSpeechSettingRelatedControllerType::SetVoiceOverride(Some( "Alex".to_owned(), )), ), }, TestCase { name: "voice-override unsetter", input: "set-voice-override", expected: Ok(ConfigTextToSpeechSettingRelatedControllerType::SetVoiceOverride(None)), }, ]; for test_case in test_cases { let result = super::determine(test_case.input); assert_eq!(result, test_case.expected, "Test case: {}", test_case.name); } } ================================================ FILE: src/controller/cfg/dispatching/mod.rs ================================================ use crate::strings; use crate::{Bot, entity::MessageContext}; use mxlink::MessageResponseType; use super::controller_type::{ ConfigControllerType, ConfigSettingRelatedControllerType, SettingsStorageSource, }; mod speech_to_text; mod text_generation; mod text_to_speech; pub async fn dispatch_controller( handler: &ConfigControllerType, message_context: &MessageContext, bot: &Bot, ) -> anyhow::Result<()> { // Anyone can access Help and Status. // Settings-related access checks are done in dispatch_config_related_handler(). match handler { ConfigControllerType::Help => super::help::handle(bot, message_context).await, ConfigControllerType::Status => super::status::handle(bot, message_context).await, ConfigControllerType::SettingsRelated(config_type, config_related_handler) => { dispatch_config_related_handler( config_type, config_related_handler, message_context, bot, ) .await } } } async fn dispatch_config_related_handler( config_type: &SettingsStorageSource, handler: &ConfigSettingRelatedControllerType, message_context: &MessageContext, bot: &Bot, ) -> anyhow::Result<()> { if let SettingsStorageSource::Global = config_type && !message_context.sender_can_manage_global_config() { bot.messaging() .send_error_markdown_no_fail( message_context.room(), strings::global_config::no_permissions_to_administrate(), MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()), ) .await; return Ok(()); } let room_settings = match config_type { SettingsStorageSource::Room => &message_context.room_config().settings, SettingsStorageSource::Global => &message_context.global_config().fallback_room_settings, }; match handler { ConfigSettingRelatedControllerType::GetHandler(purpose) => match config_type { SettingsStorageSource::Room => { super::room_config::handler::handle_get(bot, message_context, *purpose).await } SettingsStorageSource::Global => { super::global_config::handler::handle_get(bot, message_context, *purpose).await } }, ConfigSettingRelatedControllerType::SetHandler(purpose, agent_identifier) => { match config_type { SettingsStorageSource::Room => { super::room_config::handler::handle_set( bot, bot.room_config_manager(), message_context, *purpose, agent_identifier, ) .await } SettingsStorageSource::Global => { super::global_config::handler::handle_set( bot, bot.global_config_manager(), message_context, *purpose, agent_identifier, ) .await } } } ConfigSettingRelatedControllerType::TextGeneration(controller_type) => { text_generation::dispatch( controller_type, message_context, bot, room_settings, config_type, ) .await } ConfigSettingRelatedControllerType::SpeechToText(controller_type) => { speech_to_text::dispatch( controller_type, message_context, bot, room_settings, config_type, ) .await } ConfigSettingRelatedControllerType::TextToSpeech(controller_type) => { text_to_speech::dispatch( controller_type, message_context, bot, room_settings, config_type, ) .await } } } ================================================ FILE: src/controller/cfg/dispatching/speech_to_text.rs ================================================ use crate::entity::roomconfig::{ RoomSettings, SpeechToTextFlowType, SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages, }; use crate::{Bot, entity::MessageContext}; use super::super::controller_type::{ ConfigSpeechToTextSettingRelatedControllerType, SettingsStorageSource, }; use super::super::common::generic_setting::handle_get as setting_get; use super::super::global_config::generic_setting::handle_set as global_setting_set; use super::super::room_config::generic_setting::handle_set as room_setting_set; pub(super) async fn dispatch( handler: &ConfigSpeechToTextSettingRelatedControllerType, message_context: &MessageContext, bot: &Bot, room_settings: &RoomSettings, config_type: &SettingsStorageSource, ) -> anyhow::Result<()> { match handler { ConfigSpeechToTextSettingRelatedControllerType::GetFlowType => { let value = &room_settings.speech_to_text.flow_type; setting_get::(bot, message_context, value).await } ConfigSpeechToTextSettingRelatedControllerType::SetFlowType(value) => { let value = value.to_owned(); let setter_callback = Box::new(move |room_settings: &mut RoomSettings| { room_settings.speech_to_text.flow_type = value; }); match config_type { SettingsStorageSource::Room => { room_setting_set::( bot, message_context, &value, setter_callback, ) .await } SettingsStorageSource::Global => { global_setting_set::( bot, message_context, &value, setter_callback, ) .await } } } ConfigSpeechToTextSettingRelatedControllerType::GetMsgTypeForNonThreadedOnlyTranscribedMessages => { let value = &room_settings.speech_to_text.msg_type_for_non_threaded_only_transcribed_messages; setting_get::(bot, message_context, value).await } ConfigSpeechToTextSettingRelatedControllerType::SetMsgTypeForNonThreadedOnlyTranscribedMessages(value) => { let value = value.to_owned(); let setter_callback = Box::new(move |room_settings: &mut RoomSettings| { room_settings.speech_to_text.msg_type_for_non_threaded_only_transcribed_messages = value; }); match config_type { SettingsStorageSource::Room => { room_setting_set::( bot, message_context, &value, setter_callback, ) .await } SettingsStorageSource::Global => { global_setting_set::( bot, message_context, &value, setter_callback, ) .await } } } ConfigSpeechToTextSettingRelatedControllerType::GetLanguage => { let value = &room_settings.speech_to_text.language; setting_get::(bot, message_context, value).await } ConfigSpeechToTextSettingRelatedControllerType::SetLanguage(value) => { let value = value.to_owned(); let value_setter = value.clone(); let setter_callback = Box::new(move |room_settings: &mut RoomSettings| { room_settings.speech_to_text.language = value_setter; }); match config_type { SettingsStorageSource::Room => { room_setting_set::(bot, message_context, &value, setter_callback).await } SettingsStorageSource::Global => { global_setting_set::(bot, message_context, &value, setter_callback) .await } } } } } ================================================ FILE: src/controller/cfg/dispatching/text_generation.rs ================================================ use crate::entity::roomconfig::{ RoomSettings, TextGenerationAutoUsage, TextGenerationPrefixRequirementType, TextGenerationSenderContextMode, }; use crate::{Bot, entity::MessageContext}; use super::super::controller_type::{ ConfigTextGenerationSettingRelatedControllerType, SettingsStorageSource, }; use super::super::common::generic_setting::handle_get as setting_get; use super::super::global_config::generic_setting::handle_set as global_setting_set; use super::super::room_config::generic_setting::handle_set as room_setting_set; pub(super) async fn dispatch( handler: &ConfigTextGenerationSettingRelatedControllerType, message_context: &MessageContext, bot: &Bot, room_settings: &RoomSettings, config_type: &SettingsStorageSource, ) -> anyhow::Result<()> { match handler { ConfigTextGenerationSettingRelatedControllerType::GetContextManagementEnabled => { let value = &room_settings.text_generation.context_management_enabled; setting_get::(bot, message_context, value).await } ConfigTextGenerationSettingRelatedControllerType::SetContextManagementEnabled(value) => { let value = value.to_owned(); let setter_callback = Box::new(move |room_settings: &mut RoomSettings| { room_settings.text_generation.context_management_enabled = value; }); match config_type { SettingsStorageSource::Room => { room_setting_set::(bot, message_context, &value, setter_callback).await } SettingsStorageSource::Global => { global_setting_set::(bot, message_context, &value, setter_callback).await } } } ConfigTextGenerationSettingRelatedControllerType::GetPrefixRequirementType => { let value = &room_settings.text_generation.prefix_requirement_type; setting_get::(bot, message_context, value).await } ConfigTextGenerationSettingRelatedControllerType::SetPrefixRequirementType(value) => { let value = value.to_owned(); let setter_callback = Box::new(move |room_settings: &mut RoomSettings| { room_settings.text_generation.prefix_requirement_type = value; }); match config_type { SettingsStorageSource::Room => { room_setting_set::( bot, message_context, &value, setter_callback, ) .await } SettingsStorageSource::Global => { global_setting_set::( bot, message_context, &value, setter_callback, ) .await } } } ConfigTextGenerationSettingRelatedControllerType::GetAutoUsage => { let value = &room_settings.text_generation.auto_usage; setting_get::(bot, message_context, value).await } ConfigTextGenerationSettingRelatedControllerType::SetAutoUsage(value) => { let value = value.to_owned(); let setter_callback = Box::new(move |room_settings: &mut RoomSettings| { room_settings.text_generation.auto_usage = value; }); match config_type { SettingsStorageSource::Room => { room_setting_set::( bot, message_context, &value, setter_callback, ) .await } SettingsStorageSource::Global => { global_setting_set::( bot, message_context, &value, setter_callback, ) .await } } } ConfigTextGenerationSettingRelatedControllerType::GetPromptOverride => { let value = &room_settings.text_generation.prompt_override; setting_get::(bot, message_context, value).await } ConfigTextGenerationSettingRelatedControllerType::SetPromptOverride(value) => { let value = value.to_owned(); let value_setter = value.clone(); let setter_callback = Box::new(move |room_settings: &mut RoomSettings| { room_settings.text_generation.prompt_override = value_setter; }); match config_type { SettingsStorageSource::Room => { room_setting_set::(bot, message_context, &value, setter_callback).await } SettingsStorageSource::Global => { global_setting_set::(bot, message_context, &value, setter_callback) .await } } } ConfigTextGenerationSettingRelatedControllerType::GetTemperatureOverride => { let value = &room_settings.text_generation.temperature_override; setting_get::(bot, message_context, value).await } ConfigTextGenerationSettingRelatedControllerType::SetTemperatureOverride(value) => { let value = value.to_owned(); let setter_callback = Box::new(move |room_settings: &mut RoomSettings| { room_settings.text_generation.temperature_override = value; }); match config_type { SettingsStorageSource::Room => { room_setting_set::(bot, message_context, &value, setter_callback).await } SettingsStorageSource::Global => { global_setting_set::(bot, message_context, &value, setter_callback).await } } } ConfigTextGenerationSettingRelatedControllerType::GetSenderContextMode => { let value = &room_settings.text_generation.sender_context_mode; setting_get::(bot, message_context, value).await } ConfigTextGenerationSettingRelatedControllerType::SetSenderContextMode(value) => { let value = value.to_owned(); let setter_callback = Box::new(move |room_settings: &mut RoomSettings| { room_settings.text_generation.sender_context_mode = value; }); match config_type { SettingsStorageSource::Room => { room_setting_set::( bot, message_context, &value, setter_callback, ) .await } SettingsStorageSource::Global => { global_setting_set::( bot, message_context, &value, setter_callback, ) .await } } } } } ================================================ FILE: src/controller/cfg/dispatching/text_to_speech.rs ================================================ use crate::entity::roomconfig::{ RoomSettings, TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType, }; use crate::{Bot, entity::MessageContext}; use super::super::controller_type::{ ConfigTextToSpeechSettingRelatedControllerType, SettingsStorageSource, }; use super::super::common::generic_setting::handle_get as setting_get; use super::super::global_config::generic_setting::handle_set as global_setting_set; use super::super::room_config::generic_setting::handle_set as room_setting_set; pub(super) async fn dispatch( handler: &ConfigTextToSpeechSettingRelatedControllerType, message_context: &MessageContext, bot: &Bot, room_settings: &RoomSettings, config_type: &SettingsStorageSource, ) -> anyhow::Result<()> { match handler { ConfigTextToSpeechSettingRelatedControllerType::GetBotMessagesFlowType => { let value = &room_settings.text_to_speech.bot_msgs_flow_type; setting_get::(bot, message_context, value).await } ConfigTextToSpeechSettingRelatedControllerType::SetBotMessagesFlowType(value) => { let value = value.to_owned(); let setter_callback = Box::new(move |room_settings: &mut RoomSettings| { room_settings.text_to_speech.bot_msgs_flow_type = value; }); match config_type { SettingsStorageSource::Room => { room_setting_set::( bot, message_context, &value, setter_callback, ) .await } SettingsStorageSource::Global => { global_setting_set::( bot, message_context, &value, setter_callback, ) .await } } } ConfigTextToSpeechSettingRelatedControllerType::GetUserMessagesFlowType => { let value = &room_settings.text_to_speech.user_msgs_flow_type; setting_get::(bot, message_context, value).await } ConfigTextToSpeechSettingRelatedControllerType::SetUserMessagesFlowType(value) => { let value = value.to_owned(); let setter_callback = Box::new(move |room_settings: &mut RoomSettings| { room_settings.text_to_speech.user_msgs_flow_type = value; }); match config_type { SettingsStorageSource::Room => { room_setting_set::( bot, message_context, &value, setter_callback, ) .await } SettingsStorageSource::Global => { global_setting_set::( bot, message_context, &value, setter_callback, ) .await } } } ConfigTextToSpeechSettingRelatedControllerType::GetSpeedOverride => { let value = &room_settings.text_to_speech.speed_override; setting_get::(bot, message_context, value).await } ConfigTextToSpeechSettingRelatedControllerType::SetSpeedOverride(value) => { let value = value.to_owned(); let setter_callback = Box::new(move |room_settings: &mut RoomSettings| { room_settings.text_to_speech.speed_override = value; }); match config_type { SettingsStorageSource::Room => { room_setting_set::(bot, message_context, &value, setter_callback).await } SettingsStorageSource::Global => { global_setting_set::(bot, message_context, &value, setter_callback).await } } } ConfigTextToSpeechSettingRelatedControllerType::GetVoiceOverride => { let value = &room_settings.text_to_speech.voice_override; setting_get::(bot, message_context, value).await } ConfigTextToSpeechSettingRelatedControllerType::SetVoiceOverride(value) => { let value = value.to_owned(); let value_setter = value.clone(); let setter_callback = Box::new(move |room_settings: &mut RoomSettings| { room_settings.text_to_speech.voice_override = value_setter; }); match config_type { SettingsStorageSource::Room => { room_setting_set::(bot, message_context, &value, setter_callback).await } SettingsStorageSource::Global => { global_setting_set::(bot, message_context, &value, setter_callback) .await } } } } } ================================================ FILE: src/controller/cfg/global_config/generic_setting.rs ================================================ use mxlink::MessageResponseType; use crate::entity::{MessageContext, roomconfig::RoomSettings}; use crate::{Bot, strings}; pub async fn handle_set( bot: &Bot, message_context: &MessageContext, value: &Option, setter_callback: Box, ) -> anyhow::Result<()> where T: std::fmt::Display, { let mut global_config = message_context.global_config().clone(); setter_callback(&mut global_config.fallback_room_settings); bot.global_config_manager() .lock() .await .persist(&global_config) .await?; let message = match value { Some(value) => strings::global_config::value_was_set_to(value), None => strings::global_config::value_was_unset(), }; bot.messaging() .send_success_markdown_no_fail( message_context.room(), &message, MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()), ) .await; Ok(()) } ================================================ FILE: src/controller/cfg/global_config/handler.rs ================================================ use mxlink::MessageResponseType; use crate::{ Bot, agent::{AgentPurpose, PublicIdentifier}, entity::{MessageContext, globalconfig::GlobalConfigurationManager}, strings, }; pub async fn handle_get( bot: &Bot, message_context: &MessageContext, purpose: AgentPurpose, ) -> anyhow::Result<()> { let agent_id = message_context .global_config() .fallback_room_settings .handler .get_by_purpose(purpose); let Some(agent_id) = agent_id else { bot.messaging() .send_text_markdown_no_fail( message_context.room(), strings::global_config::global_config_lacks_specific_agent_for_purpose(purpose), MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()), ) .await; return Ok(()); }; let agent_identifier = match PublicIdentifier::from_str(agent_id.as_str()) { Some(agent_identifier) => agent_identifier, None => { bot.messaging() .send_error_markdown_no_fail( message_context.room(), &strings::agent::invalid_id_generic(), MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()), ) .await; return Ok(()); } }; let agent_exists = bot .agent_manager() .available_room_agents_by_room_config_context(message_context.room_config_context()) .iter() .any(|agent| *agent.identifier() == agent_identifier); if agent_exists { bot.messaging() .send_text_markdown_no_fail( message_context.room(), strings::global_config::configured_to_use_agent_for_purpose( &agent_identifier, purpose, ), MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()), ) .await; } else { bot.messaging() .send_text_markdown_no_fail( message_context.room(), strings::global_config::configures_agent_for_purpose_but_does_not_exist( &agent_identifier, purpose, ), MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()), ) .await; return Ok(()); } Ok(()) } pub async fn handle_set( bot: &Bot, global_config_manager: &tokio::sync::Mutex, message_context: &MessageContext, purpose: AgentPurpose, agent_identifier: &Option, ) -> anyhow::Result<()> { if let Some(agent_identifier) = agent_identifier { let is_allowed = match &agent_identifier { PublicIdentifier::Static(_) => true, PublicIdentifier::DynamicGlobal(_) => true, PublicIdentifier::DynamicRoomLocal(_) => false, }; if !is_allowed { bot.messaging() .send_error_markdown_no_fail( message_context.room(), &strings::global_config::not_allowed_to_use_agent_in_global_config( agent_identifier, ), MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()), ) .await; return Ok(()); } let agent_exists = bot .agent_manager() .available_room_agents_by_room_config_context(message_context.room_config_context()) .iter() .any(|agent| agent.identifier() == agent_identifier); if !agent_exists { bot.messaging() .send_error_markdown_no_fail( message_context.room(), &strings::agent::agent_with_given_identifier_missing(agent_identifier), MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()), ) .await; return Ok(()); } } let agent_id = agent_identifier .as_ref() .map(|agent_identifier| agent_identifier.as_string()); let mut global_config = global_config_manager.lock().await.get_or_create().await?; global_config .fallback_room_settings .handler .set_by_purpose(purpose, agent_id); global_config_manager .lock() .await .persist(&global_config) .await?; let message = match agent_identifier { Some(agent_identifier) => { strings::global_config::reconfigured_to_use_agent_for_purpose(agent_identifier, purpose) } None => strings::global_config::reconfigured_to_not_specify_agent_for_purpose(purpose), }; bot.messaging() .send_success_markdown_no_fail( message_context.room(), &message, MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()), ) .await; Ok(()) } ================================================ FILE: src/controller/cfg/global_config/mod.rs ================================================ pub(super) mod generic_setting; pub(super) mod handler; ================================================ FILE: src/controller/cfg/help.rs ================================================ use mxlink::MessageResponseType; use crate::{ Bot, entity::{ MessageContext, roomconfig::{ SpeechToTextFlowType, SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages, TextGenerationAutoUsage, TextGenerationPrefixRequirementType, TextGenerationSenderContextMode, TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType, }, }, strings, }; pub async fn handle(bot: &Bot, message_context: &MessageContext) -> anyhow::Result<()> { let mut message = String::new(); message.push_str(&build_section_intro()); message.push_str("\n\n"); message.push_str("\n---\n"); message.push_str(&build_section_status(bot.command_prefix())); message.push_str("\n\n"); message.push_str("\n---\n"); message.push_str(&build_section_handlers(bot.command_prefix())); message.push_str("\n\n"); message.push_str("\n---\n"); message.push_str(&build_section_text_generation( bot.command_prefix(), bot.user_id().localpart(), )); message.push_str("\n\n"); message.push_str("\n---\n"); message.push_str(&build_section_text_to_speech(bot.command_prefix())); message.push_str("\n\n"); message.push_str("\n---\n"); message.push_str(&build_section_speech_to_text(bot.command_prefix())); message.push_str("\n\n"); message.push_str("\n---\n"); message.push_str(&build_section_image_generation()); bot.messaging() .send_text_markdown_no_fail( message_context.room(), message, MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()), ) .await; Ok(()) } fn build_section_intro() -> String { let mut message = String::new(); message.push_str(&format!("## {}", strings::help::cfg::heading())); message.push_str("\n\n"); message.push_str(&strings::help::cfg::intro_long()); message } fn build_section_status(command_prefix: &str) -> String { let mut message = String::new(); message.push_str(&format!("### {}", strings::help::cfg::status_heading())); message.push_str("\n\n"); message.push_str(&strings::help::cfg::status_intro(command_prefix)); message } fn build_section_handlers(command_prefix: &str) -> String { let mut message = String::new(); message.push_str(&format!("### {}", strings::help::cfg::handlers_heading())); message.push_str("\n\n"); message.push_str(&strings::help::cfg::handlers_intro_common()); message.push('\n'); message.push_str(&strings::help::cfg::handlers_intro_purposes()); message.push_str("\n\n"); message.push_str(strings::help::available_commands_intro()); message.push('\n'); message.push_str(&format!( "- {}", strings::help::cfg::handlers_show(command_prefix) )); message.push('\n'); message.push_str(&format!( "- {}", strings::help::cfg::handlers_set(command_prefix) )); message.push('\n'); message.push_str(&format!( "- {}", strings::help::cfg::handlers_unset(command_prefix) )); message } fn build_section_text_generation(command_prefix: &str, bot_username: &str) -> String { let mut message = String::new(); message.push_str(&format!( "### {}", strings::help::cfg::text_generation_heading() )); message.push_str("\n\n"); message.push_str(&strings::help::cfg::text_generation_common()); message.push_str("\n\n"); // Prefix requirement type message.push_str(&format!( "#### {}", strings::help::cfg::text_generation_prefix_requirement_type_heading() )); message.push_str("\n\n"); message.push_str(&strings::help::cfg::text_generation_prefix_requirement_type_intro()); message.push('\n'); message.push_str( &strings::help::cfg::the_following_configuration_values_are_recognized( TextGenerationPrefixRequirementType::choices(), ), ); message.push_str("\n\n"); message .push_str(&strings::help::cfg::text_generation_prefix_requirement_type_outro(bot_username)); message.push('\n'); message.push_str(&format!( "- {}", &strings::help::cfg::current_setting_show( command_prefix, "text-generation prefix-requirement-type" ) )); message.push('\n'); message.push_str(&format!( "- {}", &strings::help::cfg::current_setting_set( command_prefix, "text-generation set-prefix-requirement-type VALUE" ) )); message.push('\n'); message.push_str(&format!( "- {}", &strings::help::cfg::current_setting_unset( command_prefix, "text-generation set-prefix-requirement-type" ) )); message.push_str("\n\n"); // Auto Usage message.push_str(&format!( "#### {}", strings::help::cfg::text_generation_auto_usage_heading() )); message.push_str("\n\n"); message.push_str(&strings::help::cfg::text_generation_auto_usage_intro()); message.push('\n'); message.push_str( &strings::help::cfg::the_following_configuration_values_are_recognized( TextGenerationAutoUsage::choices(), ), ); message.push_str("\n\n"); message.push_str(&format!( "- {}", &strings::help::cfg::current_setting_show(command_prefix, "text-generation auto-usage") )); message.push('\n'); message.push_str(&format!( "- {}", &strings::help::cfg::current_setting_set( command_prefix, "text-generation set-auto-usage VALUE" ) )); message.push('\n'); message.push_str(&format!( "- {}", &strings::help::cfg::current_setting_unset( command_prefix, "text-generation set-auto-usage" ) )); message.push_str("\n\n"); // Context Management message.push_str(&format!( "#### {}", strings::help::cfg::text_generation_context_management_heading() )); message.push_str("\n\n"); message.push_str(&strings::help::cfg::text_generation_context_management_intro()); message.push('\n'); message.push_str( &strings::help::cfg::the_following_configuration_values_are_recognized(vec![true, false]), ); message.push_str("\n\n"); message.push_str(&format!( "- {}", &strings::help::cfg::current_setting_show( command_prefix, "text-generation context-management-enabled" ) )); message.push('\n'); message.push_str(&format!( "- {}", &strings::help::cfg::current_setting_set( command_prefix, "text-generation set-context-management-enabled VALUE" ) )); message.push('\n'); message.push_str(&format!( "- {}", &strings::help::cfg::current_setting_unset( command_prefix, "text-generation set-context-management-enabled" ) )); message.push_str("\n\n"); // Sender Context message.push_str(&format!( "#### {}", strings::help::cfg::text_generation_sender_context_heading() )); message.push_str("\n\n"); message.push_str(&strings::help::cfg::text_generation_sender_context_intro()); message.push('\n'); message.push_str( &strings::help::cfg::the_following_configuration_values_are_recognized( TextGenerationSenderContextMode::choices(), ), ); message.push_str("\n\n"); message.push_str(&format!( "- {}", &strings::help::cfg::current_setting_show( command_prefix, "text-generation sender-context-mode" ) )); message.push('\n'); message.push_str(&format!( "- {}", &strings::help::cfg::current_setting_set( command_prefix, "text-generation set-sender-context-mode VALUE" ) )); message.push('\n'); message.push_str(&format!( "- {}", &strings::help::cfg::current_setting_unset( command_prefix, "text-generation set-sender-context-mode" ) )); message.push_str("\n\n"); // Prompt override message.push_str(&format!( "#### {}", strings::help::cfg::text_generation_prompt_override_heading() )); message.push_str("\n\n"); message.push_str(&strings::help::cfg::text_generation_prompt_override_intro()); message.push_str("\n\n"); message.push_str(&format!( "- {}", &strings::help::cfg::current_setting_show( command_prefix, "text-generation prompt-override" ) )); message.push('\n'); message.push_str(&format!( "- {}", &strings::help::cfg::current_setting_set( command_prefix, "text-generation set-prompt-override VALUE" ) )); message.push('\n'); message.push_str(&format!( "- {}", &strings::help::cfg::current_setting_unset( command_prefix, "text-generation set-prompt-override" ) )); message.push_str("\n\n"); // Speed override message.push_str(&format!( "#### {}", strings::help::cfg::text_generation_temperature_override_heading() )); message.push_str("\n\n"); message.push_str(&strings::help::cfg::text_generation_temperature_override_intro()); message.push_str("\n\n"); message.push_str(&format!( "- {}", &strings::help::cfg::current_setting_show( command_prefix, "text-generation temperature-override" ) )); message.push('\n'); message.push_str(&format!( "- {}", &strings::help::cfg::current_setting_set( command_prefix, "text-generation set-temperature-override VALUE" ) )); message.push('\n'); message.push_str(&format!( "- {}", &strings::help::cfg::current_setting_unset( command_prefix, "text-generation set-temperature-override" ) )); message } fn build_section_speech_to_text(command_prefix: &str) -> String { let mut message = String::new(); message.push_str(&format!( "### {}", strings::help::cfg::speech_to_text_heading() )); message.push_str("\n\n"); message.push_str(&strings::help::cfg::speech_to_text_common()); message.push_str("\n\n"); // Flow Type message.push_str(&format!( "#### {}", strings::help::cfg::speech_to_text_flow_type_heading() )); message.push_str("\n\n"); message.push_str(strings::help::cfg::speech_to_text_flow_type_intro()); message.push('\n'); message.push_str( &strings::help::cfg::the_following_configuration_values_are_recognized( SpeechToTextFlowType::choices(), ), ); message.push_str("\n\n"); message.push_str(&format!( "- {}", &strings::help::cfg::current_setting_show(command_prefix, "speech-to-text flow-type") )); message.push('\n'); message.push_str(&format!( "- {}", &strings::help::cfg::current_setting_set( command_prefix, "speech-to-text set-flow-type VALUE" ) )); message.push('\n'); message.push_str(&format!( "- {}", &strings::help::cfg::current_setting_unset(command_prefix, "speech-to-text set-flow-type") )); message.push_str("\n\n"); // Msg Type For Non Threaded Only Transcribed Messages message.push_str(&format!( "#### {}", strings::help::cfg::speech_to_text_msg_type_for_non_threaded_only_transcribed_messages_heading() )); message.push_str("\n\n"); message.push_str(strings::help::cfg::speech_to_text_msg_type_for_non_threaded_only_transcribed_messages_intro()); message.push('\n'); message.push_str( &strings::help::cfg::the_following_configuration_values_are_recognized( SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::choices(), ), ); message.push_str("\n\n"); message.push_str(&format!( "- {}", &strings::help::cfg::current_setting_show( command_prefix, "speech-to-text msg-type-for-non-threaded-only-transcribed-messages" ) )); message.push('\n'); message.push_str(&format!( "- {}", &strings::help::cfg::current_setting_set( command_prefix, "speech-to-text set-msg-type-for-non-threaded-only-transcribed-messages VALUE" ) )); message.push('\n'); message.push_str(&format!( "- {}", &strings::help::cfg::current_setting_unset( command_prefix, "speech-to-text set-msg-type-for-non-threaded-only-transcribed-messages" ) )); message.push_str("\n\n"); // Language message.push_str(&format!( "#### {}", strings::help::cfg::speech_to_text_language_heading() )); message.push_str("\n\n"); message.push_str(strings::help::cfg::speech_to_text_language_intro()); message.push_str("\n\n"); message.push_str(&format!( "- {}", &strings::help::cfg::current_setting_show(command_prefix, "speech-to-text language") )); message.push('\n'); message.push_str(&format!( "- {}", &strings::help::cfg::current_setting_set( command_prefix, "speech-to-text set-language VALUE" ) )); message.push('\n'); message.push_str(&format!( "- {}", &strings::help::cfg::current_setting_unset(command_prefix, "speech-to-text set-language") )); message.push_str("\n\n"); message } fn build_section_text_to_speech(command_prefix: &str) -> String { let mut message = String::new(); message.push_str(&format!( "### {}", strings::help::cfg::text_to_speech_heading() )); message.push_str("\n\n"); message.push_str(strings::help::cfg::text_to_speech_common()); message.push_str("\n\n"); // Bot Messages Flow Type message.push_str(&format!( "#### {}", strings::help::cfg::text_to_speech_bot_msgs_flow_type_heading() )); message.push_str("\n\n"); message.push_str(&strings::help::cfg::text_to_speech_bot_msgs_flow_type_intro()); message.push('\n'); message.push_str( &strings::help::cfg::the_following_configuration_values_are_recognized( TextToSpeechBotMessagesFlowType::choices(), ), ); message.push_str("\n\n"); message.push_str(&format!( "- {}", &strings::help::cfg::current_setting_show( command_prefix, "text-to-speech bot-msgs-flow-type" ) )); message.push('\n'); message.push_str(&format!( "- {}", &strings::help::cfg::current_setting_set( command_prefix, "text-to-speech set-bot-msgs-flow-type VALUE" ) )); message.push('\n'); message.push_str(&format!( "- {}", &strings::help::cfg::current_setting_unset( command_prefix, "text-to-speech set-bot-msgs-flow-type" ) )); message.push_str("\n\n"); // User Messages Flow Type message.push_str(&format!( "#### {}", strings::help::cfg::text_to_speech_user_msgs_flow_type_heading() )); message.push_str("\n\n"); message.push_str(&strings::help::cfg::text_to_speech_user_msgs_flow_type_intro()); message.push('\n'); message.push_str( &strings::help::cfg::the_following_configuration_values_are_recognized( TextToSpeechUserMessagesFlowType::choices(), ), ); message.push_str("\n\n"); message.push_str(&format!( "- {}", &strings::help::cfg::current_setting_show( command_prefix, "text-to-speech user-msgs-flow-type" ) )); message.push('\n'); message.push_str(&format!( "- {}", &strings::help::cfg::current_setting_set( command_prefix, "text-to-speech set-user-msgs-flow-type VALUE" ) )); message.push('\n'); message.push_str(&format!( "- {}", &strings::help::cfg::current_setting_unset( command_prefix, "text-to-speech set-user-msgs-flow-type" ) )); message.push_str("\n\n"); // Speed override message.push_str(&format!( "#### {}", strings::help::cfg::text_to_speech_speed_override_heading() )); message.push_str("\n\n"); message.push_str(&strings::help::cfg::text_to_speech_speed_override_intro()); message.push_str("\n\n"); message.push_str(&format!( "- {}", &strings::help::cfg::current_setting_show(command_prefix, "text-to-speech speed-override") )); message.push('\n'); message.push_str(&format!( "- {}", &strings::help::cfg::current_setting_set( command_prefix, "text-to-speech set-speed-override VALUE" ) )); message.push('\n'); message.push_str(&format!( "- {}", &strings::help::cfg::current_setting_unset( command_prefix, "text-to-speech set-speed-override" ) )); message.push_str("\n\n"); // Voice override message.push_str(&format!( "#### {}", strings::help::cfg::text_to_speech_voice_override_heading() )); message.push_str("\n\n"); message.push_str(&strings::help::cfg::text_to_speech_voice_override_intro()); message.push_str("\n\n"); message.push_str(&format!( "- {}", &strings::help::cfg::current_setting_show(command_prefix, "text-to-speech voice-override") )); message.push('\n'); message.push_str(&format!( "- {}", &strings::help::cfg::current_setting_set( command_prefix, "text-to-speech set-voice-override VALUE" ) )); message.push('\n'); message.push_str(&format!( "- {}", &strings::help::cfg::current_setting_unset( command_prefix, "text-to-speech set-voice-override" ) )); message.push_str("\n\n"); message } fn build_section_image_generation() -> String { let mut message = String::new(); message.push_str(&format!( "### {}", strings::help::cfg::image_generation_heading() )); message.push_str("\n\n"); message.push_str(strings::help::cfg::image_generation_common()); message } ================================================ FILE: src/controller/cfg/mod.rs ================================================ mod common; mod controller_type; mod determination; mod dispatching; mod global_config; mod help; mod room_config; mod status; pub use controller_type::ConfigControllerType; pub use determination::determine_controller; pub use dispatching::dispatch_controller; ================================================ FILE: src/controller/cfg/room_config/generic_setting.rs ================================================ use mxlink::MessageResponseType; use crate::entity::{MessageContext, roomconfig::RoomSettings}; use crate::{Bot, strings}; pub async fn handle_set( bot: &Bot, message_context: &MessageContext, value: &Option, setter_callback: Box, ) -> anyhow::Result<()> where T: std::fmt::Display, { let mut room_config = message_context.room_config().clone(); setter_callback(&mut room_config.settings); bot.room_config_manager() .lock() .await .persist(message_context.room(), &room_config) .await?; let message = match value { Some(value) => strings::room_config::value_was_set_to(value), None => strings::room_config::value_was_unset(), }; bot.messaging() .send_success_markdown_no_fail( message_context.room(), &message, MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()), ) .await; Ok(()) } ================================================ FILE: src/controller/cfg/room_config/handler.rs ================================================ use mxlink::MessageResponseType; use crate::{ Bot, agent::{AgentPurpose, PublicIdentifier}, entity::MessageContext, strings, }; use crate::entity::roomconfig::RoomConfigurationManager; pub async fn handle_get( bot: &Bot, message_context: &MessageContext, purpose: AgentPurpose, ) -> anyhow::Result<()> { let agent_id = message_context .room_config() .settings .handler .get_by_purpose(purpose); let Some(agent_id) = agent_id else { bot.messaging() .send_error_markdown_no_fail( message_context.room(), &strings::room_config::room_not_configured_with_specific_agent_for_purpose(purpose), MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()), ) .await; return Ok(()); }; let Some(agent_identifier) = PublicIdentifier::from_str(agent_id.as_str()) else { bot.messaging() .send_error_markdown_no_fail( message_context.room(), &strings::agent::invalid_id_generic(), MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()), ) .await; return Ok(()); }; let agent_exists = bot .agent_manager() .available_room_agents_by_room_config_context(message_context.room_config_context()) .iter() .any(|agent| *agent.identifier() == agent_identifier); if agent_exists { bot.messaging() .send_text_markdown_no_fail( message_context.room(), strings::room_config::configured_to_use_agent_for_purpose( &agent_identifier, purpose, ), MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()), ) .await; } else { bot.messaging() .send_text_markdown_no_fail( message_context.room(), strings::room_config::configures_agent_for_purpose_but_does_not_exist( &agent_identifier, purpose, ), MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()), ) .await; return Ok(()); } Ok(()) } pub async fn handle_set( bot: &Bot, room_config_manager: &tokio::sync::Mutex, message_context: &MessageContext, purpose: AgentPurpose, agent_identifier: &Option, ) -> anyhow::Result<()> { if let Some(agent_identifier) = agent_identifier { let agent_exists = bot .agent_manager() .available_room_agents_by_room_config_context(message_context.room_config_context()) .iter() .any(|agent| agent.identifier() == agent_identifier); if !agent_exists { bot.messaging() .send_error_markdown_no_fail( message_context.room(), &strings::agent::agent_with_given_identifier_missing(agent_identifier), MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()), ) .await; return Ok(()); } } let mut new_room_config = message_context.room_config().clone(); let agent_id = agent_identifier .as_ref() .map(|agent_identifier| agent_identifier.as_string()); new_room_config .settings .handler .set_by_purpose(purpose, agent_id); room_config_manager .lock() .await .persist(message_context.room(), &new_room_config) .await?; let message = match agent_identifier { Some(agent_identifier) => { strings::room_config::reconfigured_to_use_agent_for_purpose(agent_identifier, purpose) } None => strings::room_config::reconfigured_to_not_specify_agent_for_purpose(purpose), }; bot.messaging() .send_success_markdown_no_fail( message_context.room(), &message, MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()), ) .await; Ok(()) } ================================================ FILE: src/controller/cfg/room_config/mod.rs ================================================ pub(super) mod generic_setting; pub(super) mod handler; ================================================ FILE: src/controller/cfg/status.rs ================================================ use mxlink::MessageResponseType; use crate::{ Bot, agent::{ AgentInstance, AgentPurpose, ControllerTrait, Manager as AgentManager, PublicIdentifier, utils::get_effective_agent_for_purpose, }, entity::{ MessageContext, RoomConfigContext, roomconfig::{RoomConfig, RoomSettingsHandler}, }, strings, }; pub async fn handle(bot: &Bot, message_context: &MessageContext) -> anyhow::Result<()> { let mut message = String::new(); let agent_manager = bot.agent_manager(); let agents = agent_manager .available_room_agents_by_room_config_context(message_context.room_config_context()); // Room handlers message.push_str(&generate_room_handlers_section( &message_context.room_config().settings.handler, &agents, bot.command_prefix(), )); message.push_str("\n\n"); // Global handlers message.push_str(&generate_global_config_handlers_section( &message_context .global_config() .fallback_room_settings .handler, &agents, bot.command_prefix(), )); message.push_str("\n\n"); // Agents message.push_str(&generate_room_agents_section( message_context.room_config(), &agents, bot.command_prefix(), )); message.push_str("\n\n"); // Text Generation message.push_str( &generate_text_generation_section(agent_manager, message_context.room_config_context()) .await, ); message.push_str("\n\n"); // Text-to-Speech message.push_str( &generate_text_to_speech_section(agent_manager, message_context.room_config_context()) .await, ); message.push_str("\n\n"); // Speech-to-Text message.push_str( &generate_speech_to_text_section(agent_manager, message_context.room_config_context()) .await, ); message.push_str("\n\n"); // Image Creation message.push_str( &generate_image_generation_section(agent_manager, message_context.room_config_context()) .await, ); message.push_str("\n\n"); bot.messaging() .send_text_markdown_no_fail( message_context.room(), message, MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()), ) .await; Ok(()) } fn generate_room_handlers_section( handler_config: &RoomSettingsHandler, agents: &[AgentInstance], command_prefix: &str, ) -> String { let mut message = String::new(); message.push_str( format!( "## {}\n", strings::cfg::status_room_config_handlers_heading() ) .as_str(), ); message.push_str(strings::cfg::status_room_config_handlers_intro()); message.push_str("\n\n"); for purpose in AgentPurpose::choices() { message.push_str(&generate_handler_line_for_purpose( purpose, handler_config, agents, false, )); message.push('\n'); } message.push_str("\n\n"); message.push_str(strings::cfg::status_room_config_handlers_outro(command_prefix).as_str()); message } fn generate_global_config_handlers_section( handler_config: &RoomSettingsHandler, agents: &[AgentInstance], command_prefix: &str, ) -> String { let mut message = String::new(); message.push_str( format!( "## {}\n", strings::cfg::status_global_config_handlers_heading() ) .as_str(), ); message.push_str(strings::cfg::status_global_config_handlers_intro()); message.push_str("\n\n"); for purpose in AgentPurpose::choices() { message.push_str(&generate_handler_line_for_purpose( purpose, handler_config, agents, true, )); message.push('\n'); } message.push_str("\n\n"); message.push_str(strings::cfg::status_global_config_handlers_outro(command_prefix).as_str()); message } fn generate_handler_line_for_purpose( purpose: &AgentPurpose, handler_config: &RoomSettingsHandler, agents: &[AgentInstance], is_for_global_config: bool, ) -> String { let agent_id = handler_config.get_by_purpose(*purpose); match agent_id { Some(agent_id) => { let agent = agents .iter() .find(|a| *a.identifier().as_string() == agent_id); strings::cfg::status_handler_line_agent_found(purpose, &agent_id, agent) } None => match purpose { AgentPurpose::CatchAll => { if is_for_global_config { return strings::cfg::status_handler_line_catch_all_agent_not_set_globally(); } strings::cfg::status_handler_line_catch_all_agent_not_set_in_room_default_to_global( ) } _ => { if is_for_global_config { return strings::cfg::status_handler_line_non_catch_all_agent_not_set_globally( purpose, ); } strings::cfg::status_handler_line_non_catch_all_agent_not_set_in_room_default_to_global( purpose, ) } }, } } fn generate_room_agents_section( room_config: &RoomConfig, agents: &Vec, command_prefix: &str, ) -> String { let mut message = String::new(); message.push_str(format!("## {}\n", strings::cfg::status_room_agents_heading()).as_str()); if room_config.agents.is_empty() { message.push_str(strings::cfg::status_room_agents_empty()); message.push_str("\n\n"); message.push_str(&strings::help::learn_more_send_a_command( command_prefix, "agent", )); return message; } message.push_str(strings::cfg::status_room_agents_intro()); message.push_str("\n\n"); for agent in agents { let PublicIdentifier::DynamicRoomLocal(_) = agent.identifier() else { continue; }; message.push_str(&format!( "- `{}` ({})\n", agent.identifier(), strings::agent::create_support_badges_text(agent.controller()), )); } message.push_str("\n\n"); message.push_str(strings::cfg::status_room_agents_outro(command_prefix).as_str()); message } async fn generate_text_generation_section( agent_manager: &AgentManager, room_config_context: &RoomConfigContext, ) -> String { let mut message = String::new(); message.push_str(format!("## {}\n", strings::cfg::status_text_generation_heading()).as_str()); let text_generation_agent_info = get_effective_agent_for_purpose( agent_manager, room_config_context, AgentPurpose::TextGeneration, ) .await; // Effective agent let text_generation_agent = match text_generation_agent_info { Ok(text_generation_agent_info) => { message.push_str(&strings::cfg::status_entry_effective_agent( text_generation_agent_info.instance.identifier(), text_generation_agent_info.configuration_source, )); Some(text_generation_agent_info.instance) } Err(err) => { tracing::error!(?err, "Failed to determine text-generation agent"); message.push_str(&strings::cfg::status_entry_effective_agent_error()); None } }; // Prefix requirement type let effective_prefix_requirement_type = room_config_context.text_generation_prefix_requirement_type(); let room_config_prefix_requirement_type = room_config_context .room_config .settings .text_generation .prefix_requirement_type; let global_config_prefix_requirement_type = room_config_context .global_config .fallback_room_settings .text_generation .prefix_requirement_type; let prefix_requirement_type_set_where = if room_config_prefix_requirement_type.is_some() { strings::cfg::status_badge_set_in_room_config() } else if global_config_prefix_requirement_type.is_some() { strings::cfg::status_badge_set_in_global_config() } else { strings::cfg::status_badge_using_hardcoded_default() }; message.push_str( &strings::cfg::status_text_generation_entry_prefix_requirement_type( effective_prefix_requirement_type, prefix_requirement_type_set_where, ), ); // Auto usage let effective_auto_usage = room_config_context.auto_text_generation_usage(); let room_config_auto_usage = room_config_context .room_config .settings .text_generation .auto_usage; let global_config_auto_usage = room_config_context .global_config .fallback_room_settings .text_generation .auto_usage; let auto_usage_set_where = if room_config_auto_usage.is_some() { strings::cfg::status_badge_set_in_room_config() } else if global_config_auto_usage.is_some() { strings::cfg::status_badge_set_in_global_config() } else { strings::cfg::status_badge_using_hardcoded_default() }; message.push_str(&strings::cfg::status_text_generation_entry_auto_usage( effective_auto_usage, auto_usage_set_where, )); // Context Management let effective_context_management = room_config_context.text_generation_context_management_enabled(); let room_config_context_management = room_config_context .room_config .settings .text_generation .context_management_enabled; let global_config_context_management = room_config_context .global_config .fallback_room_settings .text_generation .context_management_enabled; let context_management_set_where = if room_config_context_management.is_some() { strings::cfg::status_badge_set_in_room_config() } else if global_config_context_management.is_some() { strings::cfg::status_badge_set_in_global_config() } else { strings::cfg::status_badge_using_hardcoded_default() }; message.push_str( &strings::cfg::status_text_generation_entry_context_management( effective_context_management, context_management_set_where, ), ); // Sender Context let effective_sender_context = room_config_context.text_generation_sender_context_mode(); let room_config_sender_context = room_config_context .room_config .settings .text_generation .sender_context_mode; let global_config_sender_context = room_config_context .global_config .fallback_room_settings .text_generation .sender_context_mode; let sender_context_set_where = if room_config_sender_context.is_some() { strings::cfg::status_badge_set_in_room_config() } else if global_config_sender_context.is_some() { strings::cfg::status_badge_set_in_global_config() } else { strings::cfg::status_badge_using_hardcoded_default() }; message.push_str(&strings::cfg::status_text_generation_entry_sender_context( effective_sender_context, sender_context_set_where, )); // Prompt override let text_agent_prompt = if let Some(text_generation_agent) = &text_generation_agent { text_generation_agent.controller().text_generation_prompt() } else { None }; let room_config_prompt_override = room_config_context .room_config .settings .text_generation .prompt_override .clone(); let global_config_prompt_override = room_config_context .global_config .fallback_room_settings .text_generation .prompt_override .clone(); let (prompt, prompt_set_where) = if let Some(room_config_prompt_override) = room_config_prompt_override { ( room_config_prompt_override, strings::cfg::status_badge_set_in_room_config(), ) } else if let Some(global_config_prompt_override) = global_config_prompt_override { ( global_config_prompt_override, strings::cfg::status_badge_set_in_global_config(), ) } else { ( text_agent_prompt.unwrap_or("".to_owned()), strings::cfg::status_badge_set_in_agent_config(), ) }; message.push_str(&strings::cfg::status_text_generation_entry_prompt( &prompt, prompt_set_where, )); // Temperature let text_agent_temperature = if let Some(text_generation_agent) = &text_generation_agent { text_generation_agent .controller() .text_generation_temperature() } else { None }; let room_config_temperature_override = room_config_context .room_config .settings .text_generation .temperature_override; let global_config_temperature_override = room_config_context .global_config .fallback_room_settings .text_generation .temperature_override; let (effective_temperature, set_where) = if let Some(room_config_temperature_override) = room_config_temperature_override { ( Some(room_config_temperature_override), strings::cfg::status_badge_set_in_room_config(), ) } else if let Some(global_config_temperature_override) = global_config_temperature_override { ( Some(global_config_temperature_override), strings::cfg::status_badge_set_in_global_config(), ) } else { ( text_agent_temperature, strings::cfg::status_badge_set_in_agent_config(), ) }; message.push_str(&strings::cfg::status_text_generation_entry_temperature( effective_temperature, set_where, )); message } async fn generate_speech_to_text_section( agent_manager: &AgentManager, room_config_context: &RoomConfigContext, ) -> String { let mut message = String::new(); message.push_str(format!("## {}\n", strings::cfg::status_speech_to_text_heading()).as_str()); let speech_to_text_agent_info = get_effective_agent_for_purpose( agent_manager, room_config_context, AgentPurpose::SpeechToText, ) .await; // Effective agent match speech_to_text_agent_info { Ok(speech_to_text_agent_info) => { message.push_str(&strings::cfg::status_entry_effective_agent( speech_to_text_agent_info.instance.identifier(), speech_to_text_agent_info.configuration_source, )); } Err(err) => { tracing::error!(?err, "Failed to determine speech-to-text agent"); message.push_str(&strings::cfg::status_entry_effective_agent_error()); } }; // Flow type let effective_flow_type = room_config_context.speech_to_text_flow_type(); let room_config_flow_type = room_config_context .room_config .settings .speech_to_text .flow_type; let global_config_flow_type = room_config_context .global_config .fallback_room_settings .speech_to_text .flow_type; let flow_type_set_where = if room_config_flow_type.is_some() { strings::cfg::status_badge_set_in_room_config() } else if global_config_flow_type.is_some() { strings::cfg::status_badge_set_in_global_config() } else { strings::cfg::status_badge_using_hardcoded_default() }; message.push_str(&strings::cfg::status_speech_to_text_entry_flow_type( effective_flow_type, flow_type_set_where, )); // Msg Type For Non Threaded Only Transcribed Messages let effective_msg_type_for_non_threaded_only_transcribed_messages = room_config_context.speech_to_text_msg_type_for_non_threaded_only_transcribed_messages(); let room_config_msg_type_for_non_threaded_only_transcribed_messages = room_config_context .room_config .settings .speech_to_text .msg_type_for_non_threaded_only_transcribed_messages; let global_config_msg_type_for_non_threaded_only_transcribed_messages = room_config_context .global_config .fallback_room_settings .speech_to_text .msg_type_for_non_threaded_only_transcribed_messages; let msg_type_for_non_threaded_only_transcribed_messages_set_where = if room_config_msg_type_for_non_threaded_only_transcribed_messages.is_some() { strings::cfg::status_badge_set_in_room_config() } else if global_config_msg_type_for_non_threaded_only_transcribed_messages.is_some() { strings::cfg::status_badge_set_in_global_config() } else { strings::cfg::status_badge_using_hardcoded_default() }; message.push_str(&strings::cfg::status_speech_to_text_entry_msg_type_for_non_threaded_only_transcribed_messages( effective_msg_type_for_non_threaded_only_transcribed_messages, msg_type_for_non_threaded_only_transcribed_messages_set_where, )); // Language let effective_language = room_config_context.speech_to_text_language(); let room_config_language = &room_config_context .room_config .settings .speech_to_text .language; let global_config_language = &room_config_context .global_config .fallback_room_settings .speech_to_text .language; let language_set_where = if room_config_language.is_some() { strings::cfg::status_badge_set_in_room_config() } else if global_config_language.is_some() { strings::cfg::status_badge_set_in_global_config() } else { strings::cfg::status_badge_using_hardcoded_default() }; message.push_str(&strings::cfg::status_speech_to_text_entry_language( effective_language, language_set_where, )); message } async fn generate_text_to_speech_section( agent_manager: &AgentManager, room_config_context: &RoomConfigContext, ) -> String { let mut message = String::new(); message.push_str(format!("## {}\n", strings::cfg::status_text_to_speech_heading()).as_str()); let text_to_speech_agent_info = get_effective_agent_for_purpose( agent_manager, room_config_context, AgentPurpose::TextToSpeech, ) .await; // Effective agent let text_to_speech_agent = match text_to_speech_agent_info { Ok(text_to_speech_agent_info) => { message.push_str(&strings::cfg::status_entry_effective_agent( text_to_speech_agent_info.instance.identifier(), text_to_speech_agent_info.configuration_source, )); Some(text_to_speech_agent_info.instance) } Err(err) => { tracing::error!(?err, "Failed to determine text-to-speech agent"); message.push_str(&strings::cfg::status_entry_effective_agent_error()); None } }; // Bot messages flow type let effective_bot_messages_tts_flow_type = room_config_context.text_to_speech_bot_messages_flow_type(); let room_config_bot_messages_tts_flow_type = room_config_context .room_config .settings .text_to_speech .bot_msgs_flow_type; let global_config_bot_messages_tts_flow_type = room_config_context .global_config .fallback_room_settings .text_to_speech .bot_msgs_flow_type; let bot_messages_tts_flow_type_set_where = if room_config_bot_messages_tts_flow_type.is_some() { strings::cfg::status_badge_set_in_room_config() } else if global_config_bot_messages_tts_flow_type.is_some() { strings::cfg::status_badge_set_in_global_config() } else { strings::cfg::status_badge_using_hardcoded_default() }; message.push_str( &strings::cfg::status_text_to_speech_entry_bot_msgs_flow_type( effective_bot_messages_tts_flow_type, bot_messages_tts_flow_type_set_where, ), ); // User messages flow type let effective_user_messages_tts_flow_type = room_config_context.text_to_speech_user_messages_flow_type(); let room_config_user_messages_tts_flow_type = room_config_context .room_config .settings .text_to_speech .user_msgs_flow_type; let global_config_user_messages_tts_flow_type = room_config_context .global_config .fallback_room_settings .text_to_speech .user_msgs_flow_type; let user_messages_tts_flow_type_set_where = if room_config_user_messages_tts_flow_type.is_some() { strings::cfg::status_badge_set_in_room_config() } else if global_config_user_messages_tts_flow_type.is_some() { strings::cfg::status_badge_set_in_global_config() } else { strings::cfg::status_badge_using_hardcoded_default() }; message.push_str( &strings::cfg::status_text_to_speech_entry_user_msgs_flow_type( effective_user_messages_tts_flow_type, user_messages_tts_flow_type_set_where, ), ); // Speed let agent_speed = if let Some(text_to_speech_agent) = &text_to_speech_agent { text_to_speech_agent.controller().text_to_speech_speed() } else { None }; let room_config_speed_override = room_config_context .room_config .settings .text_to_speech .speed_override; let global_config_speed_override = room_config_context .global_config .fallback_room_settings .text_to_speech .speed_override; let (effective_speed, set_where) = if let Some(room_config_speed_override) = room_config_speed_override { ( Some(room_config_speed_override), strings::cfg::status_badge_set_in_room_config(), ) } else if let Some(global_config_speed_override) = global_config_speed_override { ( Some(global_config_speed_override), strings::cfg::status_badge_set_in_global_config(), ) } else if agent_speed.is_some() { ( agent_speed, strings::cfg::status_badge_set_in_agent_config(), ) } else { (None, strings::cfg::status_badge_using_hardcoded_default()) }; message.push_str(&strings::cfg::status_text_to_speech_entry_speed( effective_speed, set_where, )); // Voice let agent_voice = if let Some(text_to_speech_agent) = &text_to_speech_agent { text_to_speech_agent.controller().text_to_speech_voice() } else { None }; let room_config_voice_override = room_config_context .room_config .settings .text_to_speech .voice_override .clone(); let global_config_voice_override = room_config_context .global_config .fallback_room_settings .text_to_speech .voice_override .clone(); let (effective_voice, set_where) = if let Some(room_config_voice_override) = room_config_voice_override { ( Some(room_config_voice_override), strings::cfg::status_badge_set_in_room_config(), ) } else if let Some(global_config_voice_override) = global_config_voice_override { ( Some(global_config_voice_override), strings::cfg::status_badge_set_in_global_config(), ) } else if agent_voice.is_some() { ( agent_voice, strings::cfg::status_badge_set_in_agent_config(), ) } else { (None, strings::cfg::status_badge_using_hardcoded_default()) }; message.push_str(&strings::cfg::status_text_to_speech_entry_voice( effective_voice, set_where, )); message } async fn generate_image_generation_section( agent_manager: &AgentManager, room_config_context: &RoomConfigContext, ) -> String { let mut message = String::new(); message.push_str(format!("## {}\n", strings::cfg::status_image_generation_heading()).as_str()); let image_generation_agent_info = get_effective_agent_for_purpose( agent_manager, room_config_context, AgentPurpose::ImageGeneration, ) .await; // Effective agent let _image_generation_agent = match image_generation_agent_info { Ok(image_generation_agent_info) => { message.push_str(&strings::cfg::status_entry_effective_agent( image_generation_agent_info.instance.identifier(), image_generation_agent_info.configuration_source, )); Some(image_generation_agent_info.instance) } Err(err) => { tracing::error!(?err, "Failed to determine image generation agent"); message.push_str(&strings::cfg::status_entry_effective_agent_error()); None } }; message } ================================================ FILE: src/controller/chat_completion/mod.rs ================================================ use mxlink::matrix_sdk::ruma::OwnedEventId; use mxlink::matrix_sdk::ruma::events::room::message::AudioMessageEventContent; use mxlink::{MatrixLink, MessageResponseType}; use tracing::Instrument; use crate::agent::AgentInstance; use crate::agent::AgentPurpose; use crate::agent::ControllerTrait; use crate::agent::provider::{ SpeechToTextParams, TextGenerationParams, TextGenerationPromptVariables, }; use crate::controller::utils::agent::get_effective_agent_for_purpose_or_complain; use crate::conversation::matrix::MatrixMessageProcessingParams; use crate::entity::MessagePayload; use crate::entity::roomconfig::{ SpeechToTextFlowType, SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages, TextGenerationSenderContextMode, TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType, }; use crate::strings; use crate::utils::text_to_speech::create_transcribed_message_text; use crate::{ Bot, conversation::{ create_llm_conversation_for_matrix_reply_chain, create_llm_conversation_for_matrix_thread, llm::{Author, Conversation, MessageContent}, matrix::create_list_of_bot_user_prefixes_to_strip, }, entity::MessageContext, }; #[derive(Debug, PartialEq)] pub enum ChatCompletionControllerType { // Invoked via a command prefix (e.g. `!bai Hello!`) TextCommand, // Invoked via a mention (e.g. `@baibot Hello!`) TextMention, // Invoked via a direct message (e.g. `Hello!`) TextDirect, Audio, Image, File, ThreadMention, ReplyMention, } struct TextToSpeechEligiblePayload { text: String, event_id: OwnedEventId, } enum TextToSpeechParams { Perform(TextToSpeechEligiblePayload, MessageResponseType), Offer(TextToSpeechEligiblePayload, MessageResponseType), } pub async fn handle( bot: &Bot, matrix_link: MatrixLink, message_context: &MessageContext, controller_type: &ChatCompletionControllerType, ) -> anyhow::Result<()> { let mut original_message_is_audio = false; let mut _typing_notice_guard: Option = None; let speech_to_text_flow_type = message_context .room_config_context() .speech_to_text_flow_type(); let mut speech_to_text_created_event_id: Option = None; if let MessagePayload::Audio(audio_content) = &message_context.payload() { original_message_is_audio = true; let (response_type, msg_type) = match speech_to_text_flow_type { SpeechToTextFlowType::Ignore => { tracing::debug!("Intentionally ignoring audio message"); return Ok(()); } SpeechToTextFlowType::TranscribeAndGenerateText => { tracing::debug!("Will be transcribing and possibly generating text.."); ( MessageResponseType::InThread(message_context.thread_info().clone()), SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::Notice, ) } SpeechToTextFlowType::OnlyTranscribe => { tracing::debug!("Will only be transcribing audio to text.."); if message_context.thread_info().is_thread_root_only() { let msg_type = message_context .room_config_context() .speech_to_text_msg_type_for_non_threaded_only_transcribed_messages(); ( MessageResponseType::Reply( message_context.thread_info().root_event_id.clone(), ), msg_type, ) } else { ( MessageResponseType::InThread(message_context.thread_info().clone()), SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::Notice, ) } } }; if _typing_notice_guard.is_none() { _typing_notice_guard = Some(bot.start_typing_notice(message_context.room()).await); } let Some(speech_to_text_created_event_id_result) = handle_stage_speech_to_text( bot, message_context, audio_content, response_type, msg_type, ) .await else { return Ok(()); }; speech_to_text_created_event_id = Some(speech_to_text_created_event_id_result); if speech_to_text_flow_type == SpeechToTextFlowType::OnlyTranscribe { tracing::debug!( "Intentionally not continuing with text generation after transcription" ); return Ok(()); } // We've pushed a transcription to the room. // Let's proceed below where we potentially handle text-generation. } let text_to_speech_stage_params: Option; if message_context .room_config_context() .should_auto_text_generate(original_message_is_audio) { if _typing_notice_guard.is_none() { _typing_notice_guard = Some(bot.start_typing_notice(message_context.room()).await); } let speech_to_text_created_event_id_reaction_event_id = if let Some(speech_to_text_created_event_id) = speech_to_text_created_event_id { let reaction_event_response = bot .reacting() .react_no_fail( message_context.room(), speech_to_text_created_event_id.clone(), strings::PROGRESS_INDICATOR_EMOJI.to_owned(), ) .await; reaction_event_response .map(|reaction_event_response| reaction_event_response.event_id) } else { None }; let response_type = match controller_type { // When we're triggered via a reply mention, we reply to the message that triggered us. ChatCompletionControllerType::ReplyMention => { MessageResponseType::Reply(message_context.thread_info().last_event_id.clone()) } // In all other cases, we're dealing with a threaded conversation, so we reply in the thread. _ => MessageResponseType::InThread(message_context.thread_info().clone()), }; let text_to_speech_eligible_payload = handle_stage_text_generation( bot, matrix_link.clone(), message_context, controller_type, response_type.clone(), ) .await; if let Some(speech_to_text_created_event_id_reaction_event_id) = speech_to_text_created_event_id_reaction_event_id { bot.messaging() .redact_event_no_fail( message_context.room(), speech_to_text_created_event_id_reaction_event_id, Some("Done".to_owned()), ) .await; } // If no text was generated (due to some issue), there's no point in continuing. let Some(text_to_speech_eligible_payload) = text_to_speech_eligible_payload else { return Ok(()); }; text_to_speech_stage_params = match message_context .room_config_context() .text_to_speech_bot_messages_flow_type() { TextToSpeechBotMessagesFlowType::Never => None, TextToSpeechBotMessagesFlowType::OnDemandAlways => Some(TextToSpeechParams::Offer( text_to_speech_eligible_payload, response_type, )), TextToSpeechBotMessagesFlowType::OnDemandForVoice => { if original_message_is_audio { Some(TextToSpeechParams::Offer( text_to_speech_eligible_payload, response_type, )) } else { None } } TextToSpeechBotMessagesFlowType::OnlyForVoice => { if original_message_is_audio { Some(TextToSpeechParams::Perform( text_to_speech_eligible_payload, response_type, )) } else { None } } TextToSpeechBotMessagesFlowType::Always => Some(TextToSpeechParams::Perform( text_to_speech_eligible_payload, response_type, )), }; } else { tracing::debug!("Not generating text due to auto-usage configuration"); let response_type = MessageResponseType::Reply(message_context.event_id().clone()); // If we got text from the user, perhaps it's eligible for text-to-speech. let MessagePayload::Text(text_payload) = &message_context.payload() else { // Audio message, or a notice or something else. // We don't wish to proceed with potential TTS for non-text messages. return Ok(()); }; let text_to_speech_eligible_payload = TextToSpeechEligiblePayload { text: text_payload.body.clone(), event_id: message_context.event_id().clone(), }; text_to_speech_stage_params = match message_context .room_config_context() .text_to_speech_user_messages_flow_type() { TextToSpeechUserMessagesFlowType::Never => None, TextToSpeechUserMessagesFlowType::OnDemand => Some(TextToSpeechParams::Offer( text_to_speech_eligible_payload, response_type, )), TextToSpeechUserMessagesFlowType::Always => Some(TextToSpeechParams::Perform( text_to_speech_eligible_payload, response_type, )), }; } // We're potentially dealing with some text in text_to_speech_eligible_payload - either coming directly from the user or generated by an agent. match text_to_speech_stage_params { Some(TextToSpeechParams::Perform(text_to_speech_eligible_payload, response_type)) => { if _typing_notice_guard.is_none() { _typing_notice_guard = Some(bot.start_typing_notice(message_context.room()).await); } let _tts_result = generate_and_send_tts_for_message( bot, matrix_link.clone(), message_context, response_type, text_to_speech_eligible_payload.event_id, &text_to_speech_eligible_payload.text, ) .await; } Some(TextToSpeechParams::Offer(text_to_speech_eligible_payload, response_type)) => { send_tts_offer_for_message( bot, message_context, response_type, text_to_speech_eligible_payload.event_id, ) .await; } None => {} } Ok(()) } async fn handle_stage_speech_to_text( bot: &Bot, message_context: &MessageContext, audio_content: &AudioMessageEventContent, response_type: MessageResponseType, msg_type: SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages, ) -> Option { let agent = get_effective_agent_for_purpose_or_complain( bot, message_context, AgentPurpose::SpeechToText, response_type.clone(), true, ) .await?; tracing::debug!( agent_id = agent.identifier().as_string(), "Handling speech-to-text", ); let reaction_event_response = bot .reacting() .react_no_fail( message_context.room(), message_context.event_id().clone(), strings::PROGRESS_INDICATOR_EMOJI.to_owned(), ) .await; let speech_to_text_created_event_id = handle_stage_speech_to_text_actual_transcribing( bot, message_context, &agent, audio_content, response_type.clone(), msg_type, ) .await; if let Some(reaction_event_response) = reaction_event_response { let redaction_reason = if speech_to_text_created_event_id.is_ok() { strings::speech_to_text::redaction_reason_done() } else { strings::speech_to_text::redaction_reason_failed() }; bot.messaging() .redact_event_no_fail( message_context.room(), reaction_event_response.event_id, Some(redaction_reason.to_owned()), ) .await; } let speech_to_text_created_event_id = match speech_to_text_created_event_id { Ok(event_id) => event_id, Err(err) => { tracing::warn!( "Error in room {} while trying to transcribe via agent {}: {:?}", message_context.room_id(), agent.identifier(), err, ); bot.messaging() .send_error_markdown_no_fail( message_context.room(), &strings::agent::error_while_serving_purpose( agent.identifier(), &AgentPurpose::SpeechToText, &err, ), response_type, ) .await; return None; } }; Some(speech_to_text_created_event_id) } async fn handle_stage_text_generation( bot: &Bot, matrix_link: MatrixLink, message_context: &MessageContext, controller_type: &ChatCompletionControllerType, response_type: MessageResponseType, ) -> Option { let agent = get_effective_agent_for_purpose_or_complain( bot, message_context, AgentPurpose::TextGeneration, response_type.clone(), true, ) .await?; // We only strip text from the first message if we're invoked via a command prefix. // Otherwise, we do bot-user mentions stripping on all messages below. let first_message_prefixes_to_strip = match controller_type { ChatCompletionControllerType::TextCommand => vec![bot.command_prefix().to_owned()], _ => vec![], }; let bot_user_prefixes_to_strip = create_list_of_bot_user_prefixes_to_strip( bot.user_id(), message_context.bot_display_name(), ); let allowed_users = match controller_type { // Regular chat completion only operates on messages from allowed users. ChatCompletionControllerType::TextCommand | ChatCompletionControllerType::TextMention | ChatCompletionControllerType::TextDirect | ChatCompletionControllerType::Audio | ChatCompletionControllerType::Image | ChatCompletionControllerType::File => { Some(message_context.combined_admin_and_user_regexes()) } // When we're triggered via an explicit mention (thread or reply), we wish to operate against the mention's whole context // (the whole thread or the whole reply chain upward of the message that triggered us). // // This is to allow admins and users to trigger text-generation for other users' messages. // When we're dragged into a conversation by a known (to us) user, we'd like to process all messages in the conversation, // not just those from allowed users. ChatCompletionControllerType::ThreadMention | ChatCompletionControllerType::ReplyMention => None, }; let params = MatrixMessageProcessingParams::new(bot.user_id().to_owned(), allowed_users) .with_first_message_prefixes_to_strip(first_message_prefixes_to_strip) .with_bot_user_prefixes_to_strip(bot_user_prefixes_to_strip); let conversation = match controller_type { // When we're triggered via a reply mention, the context is the whole reply chain upward of the message that triggered us. ChatCompletionControllerType::ReplyMention => { create_llm_conversation_for_matrix_reply_chain( &matrix_link, &bot.room_event_fetcher().clone(), message_context.room(), message_context.thread_info().last_event_id.clone(), ¶ms, ) .await } // Everything else is happening in a thread, so the context is the whole thread. _ => { create_llm_conversation_for_matrix_thread( &matrix_link, message_context.room(), message_context.thread_info().root_event_id.clone(), ¶ms, ) .await } }; let conversation = match conversation { Ok(conversation) => conversation, Err(err) => { tracing::warn!(?err, "Error while trying to create conversation"); bot.messaging() .send_error_markdown_no_fail( message_context.room(), &strings::agent::error_while_serving_purpose( agent.identifier(), &AgentPurpose::TextGeneration, &err, ), response_type, ) .await; return None; } }; let conversation = inject_sender_context( conversation, message_context .room_config_context() .text_generation_sender_context_mode(), ); tracing::debug!( agent_id = agent.identifier().as_string(), provider = format!("{}", agent.definition().provider.clone()), "Invoking LLM for text generation with conversation.." ); let span = tracing::debug_span!( "text_generation", agent_id = agent.identifier().as_string(), provider = format!("{}", agent.definition().provider.clone()), ); let start_time = std::time::Instant::now(); let controller = agent.controller(); let prompt_variables = TextGenerationPromptVariables::new( bot.name(), &controller .text_generation_model_id() .unwrap_or("unknown-model".to_owned()), chrono::Utc::now(), conversation.start_time(), ); let params = TextGenerationParams { context_management_enabled: message_context .room_config_context() .text_generation_context_management_enabled(), prompt_override: message_context .room_config_context() .text_generation_prompt_override(), temperature_override: message_context .room_config_context() .text_generation_temperature_override(), prompt_variables, }; let result = controller .generate_text(conversation, params) .instrument(span) .await; let duration = std::time::Instant::now().duration_since(start_time); tracing::debug!( agent_id = agent.identifier().as_string(), provider = format!("{}", agent.definition().provider.clone()), ?duration, "Done with LLM text generation" ); let result = match result { Ok(result) => result, Err(err) => { tracing::warn!( "Error in room {} while trying to generate text via agent {}: {:?}", message_context.room_id(), agent.identifier(), err, ); bot.messaging() .send_error_markdown_no_fail( message_context.room(), &strings::agent::error_while_serving_purpose( agent.identifier(), &AgentPurpose::TextGeneration, &err, ), response_type, ) .await; return None; } }; let text = result.text.clone().trim().to_owned(); if text.is_empty() { tracing::warn!( agent_id = agent.identifier().as_string(), "Agent returned empty text", ); bot.messaging() .send_error_markdown_no_fail( message_context.room(), &strings::agent::empty_response_returned(agent.identifier()), response_type, ) .await; return None; } let send_message_response = bot .messaging() .send_text_markdown_no_fail(message_context.room(), text.clone(), response_type) .await?; Some(TextToSpeechEligiblePayload { text, event_id: send_message_response.event_id, }) } async fn handle_stage_speech_to_text_actual_transcribing( bot: &Bot, message_context: &MessageContext, agent: &AgentInstance, audio_content: &AudioMessageEventContent, response_type: MessageResponseType, msg_type: SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages, ) -> anyhow::Result { let src = &audio_content.source; let media_request = mxlink::matrix_sdk::media::MediaRequestParameters { source: src.to_owned(), format: mxlink::matrix_sdk::media::MediaFormat::File, }; let media = message_context .room() .client() .media() .get_media_content(&media_request, true) .await?; let span = tracing::debug_span!( "speech_to_text_generation", agent_id = agent.identifier().as_string() ); let mime_type = audio_content .info .as_ref() .and_then(|info| info.mimetype.clone()) .unwrap_or_else(|| "audio/ogg".to_string()) .parse::() .map_err(|err| anyhow::anyhow!("Invalid MIME type: {}", err))?; let params = SpeechToTextParams { language_override: message_context .room_config_context() .speech_to_text_language(), }; let speech_to_text_result = agent .controller() .speech_to_text(&mime_type, media, params) .instrument(span) .await?; // Only use the `> 🦻 Transcribed text` format if we're posting in a thread. // // If we're dealing with a regular reply (which would be the case in "Transcribe-only mode" = speech-to-text/flow-type=only_transcribe), // we don't want to use the `> 🦻 Transcribed text` format for 2 reasons: // // 1. This kind of blockquote-formatting can be confused by clients for a fallback-for-rich-replies // (see https://spec.matrix.org/v1.11/client-server-api/#fallbacks-for-rich-replies). // It makes certain clients render our messages incorrectly. // // 2. Transcribe-only mode is typically used for memos. Sticking to a plain-text format // allows people to copy-paste the text or forward it to another room more easily (without having to strip formatting, etc.) // // When sending a bare reply, we'd better annotate the message with a 🦻 reaction instead, // to make it clear to users that it's a transcription. let (transcribed_text, annotate_message_with_reaction) = if let MessageResponseType::InThread(_) = response_type { ( create_transcribed_message_text(&speech_to_text_result.text), false, ) } else { (speech_to_text_result.text, true) }; let result = match msg_type { SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::Text => { bot.messaging() .send_text_markdown_no_fail(message_context.room(), transcribed_text, response_type) .await } SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::Notice => { bot.messaging() .send_notice_markdown_no_fail( message_context.room(), transcribed_text, response_type, ) .await } }; let event_id = result .map(|result| result.event_id) .ok_or_else(|| anyhow::anyhow!("Failed to send transcribed text"))?; if annotate_message_with_reaction { bot.reacting() .react_no_fail( message_context.room(), event_id.clone(), AgentPurpose::SpeechToText.emoji().to_owned(), ) .await; } Ok(event_id) } async fn send_tts_offer_for_message( bot: &Bot, message_context: &MessageContext, response_type: MessageResponseType, event_id: OwnedEventId, ) { // Offers may be enabled, but there's no guarantee that whatever agent is configured can actually do TTS. // So.. do not complain if there's no agent available. Just silently ignore it. let speech_agent = get_effective_agent_for_purpose_or_complain( bot, message_context, AgentPurpose::TextToSpeech, response_type, false, ) .await; if speech_agent.is_some() { bot.reacting() .react_no_fail( message_context.room(), event_id, AgentPurpose::TextToSpeech.emoji().to_owned(), ) .await; } } async fn generate_and_send_tts_for_message( bot: &Bot, matrix_link: MatrixLink, message_context: &MessageContext, response_type: MessageResponseType, event_id: OwnedEventId, text: &str, ) -> bool { let speech_agent = get_effective_agent_for_purpose_or_complain( bot, message_context, AgentPurpose::TextToSpeech, response_type.clone(), true, ) .await; let Some(speech_agent) = speech_agent else { return false; }; crate::controller::utils::text_to_speech::generate_and_send_tts_for_message( bot, matrix_link, message_context, response_type, &speech_agent, &event_id, text, ) .await } fn inject_sender_context( conversation: Conversation, sender_context_mode: TextGenerationSenderContextMode, ) -> Conversation { if sender_context_mode == TextGenerationSenderContextMode::Disabled { return conversation; } let include_timestamp = sender_context_mode == TextGenerationSenderContextMode::MatrixUserIdAndTimestamp; let messages = conversation .messages .into_iter() .map(|mut message| { if message.author == Author::Prompt { return message; } let Some(sender_id) = &message.sender_id else { return message; }; if let MessageContent::Text(ref mut text) = message.content { *text = if include_timestamp { let timestamp = message.timestamp.format("%Y-%m-%dT%H:%M:%SZ"); format!("[sender={} sent_at={}] {}", sender_id, timestamp, text) } else { format!("[sender={}] {}", sender_id, text) }; } message }) .collect(); Conversation { messages } } #[cfg(test)] mod sender_context_tests { use super::inject_sender_context; use crate::conversation::llm::{Author, Conversation, ImageDetails, Message, MessageContent}; use crate::entity::roomconfig::TextGenerationSenderContextMode; use chrono::{TimeZone, Utc}; use mxlink::matrix_sdk::ruma::events::room::message::ImageMessageEventContent; use mxlink::matrix_sdk::ruma::{OwnedMxcUri, OwnedUserId}; use mxlink::mime; #[test] fn test_inject_sender_context_prefixes_text_messages() { let timestamp = Utc.with_ymd_and_hms(2026, 3, 23, 14, 30, 0).unwrap(); let user_id = OwnedUserId::try_from("@alice:example.com").unwrap(); let conversation = Conversation { messages: vec![Message { author: Author::User, sender_id: Some(user_id), timestamp, content: MessageContent::Text("Hello bot".to_string()), }], }; let result = inject_sender_context( conversation, TextGenerationSenderContextMode::MatrixUserIdAndTimestamp, ); assert_eq!(result.messages.len(), 1); assert_eq!( result.messages[0].content, MessageContent::Text( "[sender=@alice:example.com sent_at=2026-03-23T14:30:00Z] Hello bot".to_string() ) ); } #[test] fn test_inject_sender_context_can_prefix_without_timestamp() { let timestamp = Utc.with_ymd_and_hms(2026, 3, 23, 14, 30, 0).unwrap(); let user_id = OwnedUserId::try_from("@alice:example.com").unwrap(); let conversation = Conversation { messages: vec![Message { author: Author::User, sender_id: Some(user_id), timestamp, content: MessageContent::Text("Hello bot".to_string()), }], }; let result = inject_sender_context(conversation, TextGenerationSenderContextMode::MatrixUserId); assert_eq!(result.messages.len(), 1); assert_eq!( result.messages[0].content, MessageContent::Text("[sender=@alice:example.com] Hello bot".to_string()) ); } #[test] fn test_inject_sender_context_prefixes_assistant_messages() { let timestamp = Utc.with_ymd_and_hms(2026, 3, 23, 14, 30, 0).unwrap(); let user_id = OwnedUserId::try_from("@baibot:example.com").unwrap(); let conversation = Conversation { messages: vec![Message { author: Author::Assistant, sender_id: Some(user_id), timestamp, content: MessageContent::Text("Hello human".to_string()), }], }; let result = inject_sender_context( conversation, TextGenerationSenderContextMode::MatrixUserIdAndTimestamp, ); assert_eq!(result.messages.len(), 1); assert_eq!( result.messages[0].content, MessageContent::Text( "[sender=@baibot:example.com sent_at=2026-03-23T14:30:00Z] Hello human".to_string() ) ); } #[test] fn test_inject_sender_context_skips_prompt_messages() { let timestamp = Utc.with_ymd_and_hms(2026, 3, 23, 14, 30, 0).unwrap(); let conversation = Conversation { messages: vec![Message { author: Author::Prompt, sender_id: None, timestamp, content: MessageContent::Text("You are a bot".to_string()), }], }; let result = inject_sender_context(conversation, TextGenerationSenderContextMode::MatrixUserId); assert_eq!( result.messages[0].content, MessageContent::Text("You are a bot".to_string()) ); } #[test] fn test_inject_sender_context_skips_messages_without_sender_id() { let timestamp = Utc.with_ymd_and_hms(2026, 3, 23, 14, 30, 0).unwrap(); let conversation = Conversation { messages: vec![Message { author: Author::User, sender_id: None, timestamp, content: MessageContent::Text("Transcribed text".to_string()), }], }; let result = inject_sender_context( conversation, TextGenerationSenderContextMode::MatrixUserIdAndTimestamp, ); assert_eq!( result.messages[0].content, MessageContent::Text("Transcribed text".to_string()) ); } #[test] fn test_inject_sender_context_leaves_non_text_content_unchanged() { let timestamp = Utc.with_ymd_and_hms(2026, 3, 23, 14, 30, 0).unwrap(); let user_id = OwnedUserId::try_from("@alice:example.com").unwrap(); let image_event_content = ImageMessageEventContent::plain( "image.png".to_string(), OwnedMxcUri::from("mxc://example.com/1234567890"), ); let conversation = Conversation { messages: vec![Message { author: Author::User, sender_id: Some(user_id), timestamp, content: MessageContent::Image(ImageDetails::new( image_event_content.clone(), mime::IMAGE_PNG, vec![], )), }], }; let result = inject_sender_context( conversation, TextGenerationSenderContextMode::MatrixUserIdAndTimestamp, ); assert_eq!( result.messages[0].content, MessageContent::Image(ImageDetails::new( image_event_content, mime::IMAGE_PNG, vec![] )) ); } #[test] fn test_inject_sender_context_none_leaves_text_unchanged() { let timestamp = Utc.with_ymd_and_hms(2026, 3, 23, 14, 30, 0).unwrap(); let user_id = OwnedUserId::try_from("@alice:example.com").unwrap(); let conversation = Conversation { messages: vec![Message { author: Author::User, sender_id: Some(user_id), timestamp, content: MessageContent::Text("Hello bot".to_string()), }], }; let result = inject_sender_context(conversation, TextGenerationSenderContextMode::Disabled); assert_eq!( result.messages[0].content, MessageContent::Text("Hello bot".to_string()) ); } } ================================================ FILE: src/controller/controller_type.rs ================================================ #[derive(Debug, PartialEq)] pub enum ControllerType { // Denotes that the message is to be ignored. Ignore, Help, UsageHelp, Unknown, Error(String), ErrorInThread(String, mxlink::ThreadInfo), ProviderHelp, Access(super::access::AccessControllerType), Agent(super::agent::AgentControllerType), Config(super::cfg::ConfigControllerType), ChatCompletion(super::chat_completion::ChatCompletionControllerType), ImageGeneration(String), ImageEdit(String), StickerGeneration(String), } ================================================ FILE: src/controller/determination/mod.rs ================================================ #[cfg(test)] mod tests; use super::chat_completion::ChatCompletionControllerType; use crate::{ entity::{ InteractionTrigger, MessageContext, MessagePayload, roomconfig::TextGenerationPrefixRequirementType, }, strings, }; use super::ControllerType; pub fn determine_controller( command_prefix: &str, first_thread_message: &InteractionTrigger, message_context: &MessageContext, ) -> ControllerType { match &first_thread_message.payload { MessagePayload::SynthethicChatCompletionTriggerInThread => { ControllerType::ChatCompletion(ChatCompletionControllerType::ThreadMention) } MessagePayload::SynthethicChatCompletionTriggerForReply => { ControllerType::ChatCompletion(ChatCompletionControllerType::ReplyMention) } MessagePayload::Text(text_message_content) => { let prefix_requirement_type = message_context .room_config_context() .text_generation_prefix_requirement_type(); determine_text_controller( command_prefix, &text_message_content.body, prefix_requirement_type, first_thread_message.is_mentioning_bot, ) } MessagePayload::Image(_image_message_content) => { let prefix_requirement_type = message_context .room_config_context() .text_generation_prefix_requirement_type(); match prefix_requirement_type { TextGenerationPrefixRequirementType::CommandPrefix => ControllerType::Ignore, TextGenerationPrefixRequirementType::No => { ControllerType::ChatCompletion(ChatCompletionControllerType::Image) } } } MessagePayload::Encrypted(thread_info) => { if thread_info.is_thread_root_only() { ControllerType::Error(strings::error::message_is_encrypted().to_owned()) } else { ControllerType::ErrorInThread( strings::error::first_message_in_thread_is_encrypted().to_owned(), thread_info.clone(), ) } } MessagePayload::File(_file_message_content) => { let prefix_requirement_type = message_context .room_config_context() .text_generation_prefix_requirement_type(); match prefix_requirement_type { TextGenerationPrefixRequirementType::CommandPrefix => ControllerType::Ignore, TextGenerationPrefixRequirementType::No => { ControllerType::ChatCompletion(ChatCompletionControllerType::File) } } } MessagePayload::Audio(_) => { ControllerType::ChatCompletion(ChatCompletionControllerType::Audio) } MessagePayload::Reaction { .. } => { panic!("Handling reaction as first message in thread does not make sense") } } } fn determine_text_controller( command_prefix: &str, text: &str, room_text_generation_prefix_requirement_type: TextGenerationPrefixRequirementType, is_mentioning_bot: bool, ) -> ControllerType { let text = text.trim(); if text.starts_with(&format!("{command_prefix} help")) || text == command_prefix { return ControllerType::Help; } if let Some(remaining) = text.strip_prefix(&format!("{command_prefix} access")) { return super::access::determine_controller(remaining.trim()); } if let Some(remaining) = text.strip_prefix(&format!("{command_prefix} provider")) { return super::provider::determine_controller(remaining.trim()); } if let Some(remaining) = text.strip_prefix(&format!("{command_prefix} agent")) { return super::agent::determine_controller(command_prefix, remaining.trim()); } if let Some(remaining) = text.strip_prefix(&format!("{command_prefix} config")) { return super::cfg::determine_controller(remaining.trim()); } if let Some(prompt) = text.strip_prefix(&format!("{command_prefix} image")) { return super::image::determine_controller(prompt.trim()); } if let Some(prompt) = text.strip_prefix(&format!("{command_prefix} sticker")) { return ControllerType::StickerGeneration(prompt.trim().to_owned()); } if let Some(remaining) = text.strip_prefix(&format!("{command_prefix} usage")) { return super::usage::determine_controller(remaining.trim()); } // Regular text message that does not match any command. // If it mentions the bot, it's a chat completion. // Otherwise, it depends on the prefix requirement for text generation - it may be routed for chat completion or ignored. if is_mentioning_bot { return ControllerType::ChatCompletion(ChatCompletionControllerType::TextMention); } // Regardless of what the prefix requirement is, if we encounter a command prefix, we'll consider it a chat completion via command prefix invokation. // This is to correctly indicate to the chat completion controller that a command prefix was used, // so that it can be stripped from the beginning of the message. if text.starts_with(command_prefix) { return ControllerType::ChatCompletion(ChatCompletionControllerType::TextCommand); } // We're dealing with a regular message that does not start with a command prefix. match room_text_generation_prefix_requirement_type { TextGenerationPrefixRequirementType::CommandPrefix => { // A prefix is required, but we've already checked (above) that the message does not start with a command prefix. // It's to be ignored. ControllerType::Ignore } TextGenerationPrefixRequirementType::No => { ControllerType::ChatCompletion(ChatCompletionControllerType::TextDirect) } } } ================================================ FILE: src/controller/determination/tests.rs ================================================ #[test] fn determine_text_controller() { use super::super::chat_completion::ChatCompletionControllerType; use super::ControllerType; use crate::controller; let command_prefix = "!bai"; struct TestCase { name: &'static str, input: &'static str, is_mentioning_bot: bool, expected: ControllerType, // This value only matters for some of the tests. // We default to using the No variant for most tests where it's irrelevant. room_text_generation_prefix_requirement_type: super::TextGenerationPrefixRequirementType, } // We only have top-level test cases here. // Each submodule defines its own test cases. let test_cases = vec![ TestCase { name: "Help", input: "!bai help", is_mentioning_bot: false, room_text_generation_prefix_requirement_type: super::TextGenerationPrefixRequirementType::No, expected: ControllerType::Help, }, TestCase { name: "Prefix only leads to help", input: "!bai", is_mentioning_bot: false, room_text_generation_prefix_requirement_type: super::TextGenerationPrefixRequirementType::No, expected: ControllerType::Help, }, TestCase { name: "Prefix and unknown command leads to chat completion", input: "!bai something-else", is_mentioning_bot: false, room_text_generation_prefix_requirement_type: super::TextGenerationPrefixRequirementType::No, expected: ControllerType::ChatCompletion(ChatCompletionControllerType::TextCommand), }, TestCase { name: "Access top-level", input: "!bai access", is_mentioning_bot: false, room_text_generation_prefix_requirement_type: super::TextGenerationPrefixRequirementType::No, expected: ControllerType::Access(controller::access::AccessControllerType::Help), }, TestCase { name: "Provider", input: "!bai provider", is_mentioning_bot: false, room_text_generation_prefix_requirement_type: super::TextGenerationPrefixRequirementType::No, expected: ControllerType::ProviderHelp, }, TestCase { name: "Usage", input: "!bai usage", is_mentioning_bot: false, room_text_generation_prefix_requirement_type: super::TextGenerationPrefixRequirementType::No, expected: ControllerType::UsageHelp, }, TestCase { name: "Agent top-level", input: "!bai agent", is_mentioning_bot: false, room_text_generation_prefix_requirement_type: super::TextGenerationPrefixRequirementType::No, expected: ControllerType::Agent(controller::agent::AgentControllerType::Help), }, TestCase { name: "Config top-level", input: "!bai config", is_mentioning_bot: false, room_text_generation_prefix_requirement_type: super::TextGenerationPrefixRequirementType::No, expected: ControllerType::Config(controller::cfg::ConfigControllerType::Help), }, TestCase { name: "Generic image command causes usage help", input: "!bai image Draw a cat!", is_mentioning_bot: false, room_text_generation_prefix_requirement_type: super::TextGenerationPrefixRequirementType::No, expected: ControllerType::UsageHelp, }, TestCase { name: "Image generation", input: "!bai image create Draw a cat!", is_mentioning_bot: false, room_text_generation_prefix_requirement_type: super::TextGenerationPrefixRequirementType::No, expected: ControllerType::ImageGeneration("Draw a cat!".to_owned()), }, TestCase { name: "Sticker generation", input: "!bai sticker A surprised cat", is_mentioning_bot: false, room_text_generation_prefix_requirement_type: super::TextGenerationPrefixRequirementType::No, expected: ControllerType::StickerGeneration("A surprised cat".to_owned()), }, TestCase { name: "Regular text triggers completion when prefix not required", input: "Regular text goes here", is_mentioning_bot: false, room_text_generation_prefix_requirement_type: super::TextGenerationPrefixRequirementType::No, expected: ControllerType::ChatCompletion(ChatCompletionControllerType::TextDirect), }, TestCase { name: "Regular text is ignored when prefix is required", input: "Regular text goes here", is_mentioning_bot: false, room_text_generation_prefix_requirement_type: super::TextGenerationPrefixRequirementType::CommandPrefix, expected: ControllerType::Ignore, }, TestCase { name: "Command-prefixed text triggers completion when prefix is required", input: "!bai Regular text goes here", is_mentioning_bot: false, room_text_generation_prefix_requirement_type: super::TextGenerationPrefixRequirementType::CommandPrefix, expected: ControllerType::ChatCompletion(ChatCompletionControllerType::TextCommand), }, TestCase { name: "Command-prefixed text triggers completion even when prefix is not required", input: "!bai Regular text goes here", is_mentioning_bot: false, room_text_generation_prefix_requirement_type: super::TextGenerationPrefixRequirementType::No, expected: ControllerType::ChatCompletion(ChatCompletionControllerType::TextCommand), }, TestCase { name: "Regular message with bot mention triggers completion (no prefix requirement)", input: "Regular text goes here", is_mentioning_bot: true, room_text_generation_prefix_requirement_type: super::TextGenerationPrefixRequirementType::No, expected: ControllerType::ChatCompletion(ChatCompletionControllerType::TextMention), }, // This test case is the same as the one above, just with a different prefix requirement setting. // We expect the same result. TestCase { name: "Regular message with bot mention triggers completion (command prefix requirement)", input: "Regular text goes here", is_mentioning_bot: true, room_text_generation_prefix_requirement_type: super::TextGenerationPrefixRequirementType::CommandPrefix, expected: ControllerType::ChatCompletion(ChatCompletionControllerType::TextMention), }, ]; for test_case in test_cases { let result = super::determine_text_controller( command_prefix, test_case.input, test_case.room_text_generation_prefix_requirement_type, test_case.is_mentioning_bot, ); assert_eq!(result, test_case.expected, "Test case: {}", test_case.name); } } ================================================ FILE: src/controller/dispatching.rs ================================================ use mxlink::MessageResponseType; use crate::{Bot, entity::MessageContext, strings}; use super::ControllerType; pub async fn dispatch_controller( controller_type: &ControllerType, message_context: &MessageContext, bot: &Bot, ) { let result = match controller_type { ControllerType::Access(controller_type) => { super::access::dispatch_controller(controller_type, message_context, bot).await } ControllerType::Agent(controller_type) => { super::agent::dispatch_controller(controller_type, message_context, bot).await } ControllerType::Config(controller_type) => { super::cfg::dispatch_controller(controller_type, message_context, bot).await } ControllerType::Help => super::help::handle(bot, message_context).await, ControllerType::Unknown => { bot.messaging() .send_error_markdown_no_fail( message_context.room(), &strings::error::unknown_command_see_help(bot.command_prefix()), MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()), ) .await; Ok(()) } ControllerType::ProviderHelp => super::provider::handle_help(message_context, bot).await, ControllerType::UsageHelp => super::usage::handle_help(message_context, bot).await, ControllerType::ChatCompletion(controller_type) => { super::chat_completion::handle( bot, bot.matrix_link().clone(), message_context, controller_type, ) .await } ControllerType::Error(message) => { bot.messaging() .send_error_markdown_no_fail( message_context.room(), message, MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()), ) .await; Ok(()) } ControllerType::ErrorInThread(message, thread_info) => { bot.messaging() .send_error_markdown_no_fail( message_context.room(), message, MessageResponseType::InThread(thread_info.clone()), ) .await; Ok(()) } ControllerType::Ignore => { tracing::trace!("Ignoring text message"); Ok(()) } ControllerType::ImageGeneration(prompt) => { super::image::generation::handle_image( bot, bot.matrix_link().clone(), message_context, prompt, ) .await } ControllerType::ImageEdit(prompt) => { super::image::edit::handle(bot, bot.matrix_link().clone(), message_context, prompt) .await } ControllerType::StickerGeneration(prompt) => { super::image::generation::handle_sticker( bot, bot.matrix_link().clone(), message_context, prompt, ) .await } }; if let Err(e) = result { tracing::error!( "Error handling message {} from sender {} in room {}: {:?}", message_context.event_id(), message_context.sender_id(), message_context.room_id(), e, ); bot.messaging() .send_error_markdown_no_fail( message_context.room(), strings::error::error_while_processing_message(), MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()), ) .await; } } ================================================ FILE: src/controller/help/mod.rs ================================================ use mxlink::MessageResponseType; use crate::{Bot, entity::MessageContext, strings}; pub async fn handle(bot: &Bot, message_context: &MessageContext) -> anyhow::Result<()> { let sender_can_manage_global_config = message_context.sender_can_manage_global_config(); let sender_can_manage_room_local_agents = message_context.sender_can_manage_room_local_agents()?; let mut message = String::from(""); message.push_str(&format!("## {}\n\n", strings::help::heading_introduction())); message.push_str(&strings::introduction::create_short_introduction( bot.name(), )); message.push_str("\n\n"); // Agents message.push_str(&format!("## {}", strings::help::agent::heading())); message.push_str("\n\n"); message.push_str(&strings::help::agent::intro(bot.command_prefix())); message.push_str("\n\n"); message.push_str(&strings::help::agent::intro_handler_relation( bot.command_prefix(), )); message.push_str("\n\n"); message.push_str(&strings::help::learn_more_send_a_command( bot.command_prefix(), "agent", )); message.push_str("\n\n"); // Providers if sender_can_manage_room_local_agents || sender_can_manage_global_config { message.push_str(&format!("## {}", strings::help::provider::heading())); message.push_str("\n\n"); message.push_str(&strings::help::provider::intro()); message.push_str("\n\n"); message.push_str(&strings::help::learn_more_send_a_command( bot.command_prefix(), "provider", )); message.push_str("\n\n"); } // Access message.push_str(&format!("## {}", strings::help::access::heading())); message.push_str("\n\n"); message.push_str(&strings::help::access::intro()); message.push_str("\n\n"); message.push_str(&strings::help::learn_more_send_a_command( bot.command_prefix(), "access", )); message.push_str("\n\n"); // Configuration message.push_str(&format!("## {}", strings::help::cfg::heading())); message.push_str("\n\n"); message.push_str(strings::help::cfg::intro_short()); message.push_str("\n\n"); message.push_str(&strings::help::learn_more_send_a_command( bot.command_prefix(), "config", )); message.push_str("\n\n"); // Usage message.push_str(&format!("## {}", strings::help::usage::heading())); message.push_str("\n\n"); message.push_str(strings::help::usage::intro()); message.push_str("\n\n"); message.push_str(&strings::help::learn_more_send_a_command( bot.command_prefix(), "usage", )); bot.messaging() .send_text_markdown_no_fail( message_context.room(), message, MessageResponseType::Reply(message_context.thread_info().last_event_id.clone()), ) .await; Ok(()) } ================================================ FILE: src/controller/image/determination/mod.rs ================================================ use crate::controller::ControllerType; mod tests; pub fn determine_controller(text: &str) -> ControllerType { let text = text.trim(); if let Some(prompt) = text.strip_prefix("create") { return ControllerType::ImageGeneration(prompt.trim().to_owned()); } if let Some(prompt) = text.strip_prefix("edit") { return ControllerType::ImageEdit(prompt.trim().to_owned()); } ControllerType::UsageHelp } ================================================ FILE: src/controller/image/determination/tests.rs ================================================ #[test] fn determine_controller() { struct TestCase { name: &'static str, input: &'static str, expected: super::ControllerType, } let test_cases = vec![ TestCase { name: "Top-level is usage help", input: "", expected: super::ControllerType::UsageHelp, }, TestCase { name: "Top-level with some text is usage help", input: "Some text", expected: super::ControllerType::UsageHelp, }, TestCase { name: "Image generation triggered by create prefix", input: "create Some prompt", expected: super::ControllerType::ImageGeneration("Some prompt".to_owned()), }, TestCase { name: "Image edit triggered by edit prefix", input: "edit Turn this into an anime-style image", expected: super::ControllerType::ImageEdit( "Turn this into an anime-style image".to_owned(), ), }, ]; for test_case in test_cases { let result = super::determine_controller(test_case.input); assert_eq!(result, test_case.expected, "Test case: {}", test_case.name); } } ================================================ FILE: src/controller/image/edit.rs ================================================ use mxlink::{MatrixLink, MessageResponseType}; use tracing::Instrument; use crate::agent::AgentPurpose; use crate::agent::ControllerTrait; use crate::agent::provider::ImageEditParams; use crate::agent::provider::ImageSource; use crate::controller::utils::agent::get_effective_agent_for_purpose_or_complain; use crate::conversation::create_llm_conversation_for_matrix_thread; use crate::conversation::matrix::MatrixMessageProcessingParams; use crate::strings; use crate::utils::mime::get_file_extension; use crate::{Bot, entity::MessageContext}; pub async fn handle( bot: &Bot, matrix_link: MatrixLink, message_context: &MessageContext, original_prompt: &str, ) -> anyhow::Result<()> { let response_type = MessageResponseType::InThread(message_context.thread_info().clone()); let Some(agent) = get_effective_agent_for_purpose_or_complain( bot, message_context, AgentPurpose::ImageGeneration, response_type.clone(), true, ) .await else { return Ok(()); }; if message_context.thread_info().is_thread_root_only() { return send_guide(bot, message_context).await; } let _typing_notice_guard = bot.start_typing_notice(message_context.room()).await; let params = MatrixMessageProcessingParams::new( bot.user_id().to_owned(), Some(message_context.combined_admin_and_user_regexes()), ); let conversation = create_llm_conversation_for_matrix_thread( &matrix_link, message_context.room(), message_context.thread_info().root_event_id.clone(), ¶ms, ) .await?; let prompt = if conversation.messages.len() >= 2 { // Skip the first message, which contains the original prompt (which we already have) let other_messages = conversation.messages.iter().skip(1).cloned().collect(); super::prompt::build(original_prompt, other_messages) } else { original_prompt.to_owned() }; let got_go_signal = conversation.messages.iter().any(|message| { if let crate::conversation::llm::MessageContent::Text(text) = &message.content { text.to_lowercase() == "go" } else { false } }); let image_sources: Vec = conversation .messages .iter() .filter_map(|message| { if let crate::conversation::llm::MessageContent::Image(image_content) = &message.content { Some(image_content.clone().into()) } else { None } }) .collect(); if !got_go_signal || image_sources.is_empty() { // We don't send the guide again here to avoid being annoying. return Ok(()); } let span = tracing::debug_span!("image_edit", agent_id = agent.identifier().as_string()); let result = agent .controller() .create_image_edit(&prompt, image_sources, ImageEditParams::default()) .instrument(span) .await; let response = match result { Ok(response) => response, Err(err) => { tracing::warn!( "Error in room {} while trying to generate image edit via agent {}: {:?}", message_context.room_id(), agent.identifier(), err, ); bot.messaging() .send_error_markdown_no_fail( message_context.room(), &strings::agent::error_while_serving_purpose( agent.identifier(), &AgentPurpose::ImageGeneration, &err, ), response_type, ) .await; return Ok(()); } }; let attachment_body_text = format!( "generated-image-edit.{}", get_file_extension(&response.mime_type) ); let mut event_content = matrix_link .media() .upload_and_prepare_event_content( message_context.room(), &response.mime_type, response.bytes, &attachment_body_text, ) .await .map_err(|e| anyhow::anyhow!("Failed to upload and prepare event: {}", e))?; matrix_link .messaging() .send_event( message_context.room(), &mut event_content, response_type.clone(), ) .await?; Ok(()) } async fn send_guide(bot: &Bot, message_context: &MessageContext) -> anyhow::Result<()> { bot.messaging() .send_text_markdown_no_fail( message_context.room(), strings::image_edit::guide_how_to_proceed(), MessageResponseType::InThread(message_context.thread_info().clone()), ) .await; Ok(()) } ================================================ FILE: src/controller/image/generation.rs ================================================ use mxlink::{MatrixLink, MessageResponseType}; use tracing::Instrument; use crate::agent::AgentPurpose; use crate::agent::ControllerTrait; use crate::agent::provider::ImageGenerationParams; use crate::controller::utils::agent::get_effective_agent_for_purpose_or_complain; use crate::conversation::create_llm_conversation_for_matrix_thread; use crate::conversation::matrix::MatrixMessageProcessingParams; use crate::strings; use crate::utils::mime::get_file_extension; use crate::{Bot, entity::MessageContext}; pub async fn handle_image( bot: &Bot, matrix_link: MatrixLink, message_context: &MessageContext, original_prompt: &str, ) -> anyhow::Result<()> { let response_type = MessageResponseType::InThread(message_context.thread_info().clone()); let Some(agent) = get_effective_agent_for_purpose_or_complain( bot, message_context, AgentPurpose::ImageGeneration, response_type.clone(), true, ) .await else { return Ok(()); }; let _typing_notice_guard = bot.start_typing_notice(message_context.room()).await; let params = MatrixMessageProcessingParams::new( bot.user_id().to_owned(), Some(message_context.combined_admin_and_user_regexes()), ); let conversation = create_llm_conversation_for_matrix_thread( &matrix_link, message_context.room(), message_context.thread_info().root_event_id.clone(), ¶ms, ) .await?; let prompt = if conversation.messages.len() >= 2 { // Skip the first message, which contains the original prompt (which we already have) let other_messages = conversation.messages.iter().skip(1).cloned().collect(); super::prompt::build(original_prompt, other_messages) } else { original_prompt.to_owned() }; let span = tracing::debug_span!( "image_generation", agent_id = agent.identifier().as_string() ); let result = agent .controller() .generate_image(&prompt, ImageGenerationParams::default()) .instrument(span) .await; let response = match result { Ok(response) => response, Err(err) => { tracing::warn!( "Error in room {} while trying to generate image via agent {}: {:?}", message_context.room_id(), agent.identifier(), err, ); bot.messaging() .send_error_markdown_no_fail( message_context.room(), &strings::agent::error_while_serving_purpose( agent.identifier(), &AgentPurpose::ImageGeneration, &err, ), response_type, ) .await; return Ok(()); } }; let actual_prompt = response.revised_prompt.as_deref().unwrap_or(&prompt); if *actual_prompt.trim() != *prompt.trim() { bot.messaging() .send_notice_markdown_no_fail( message_context.room(), strings::image_generation::revised_prompt(actual_prompt), response_type.clone(), ) .await; } let attachment_body_text = format!( "generated-image.{}", get_file_extension(&response.mime_type) ); let mut event_content = matrix_link .media() .upload_and_prepare_event_content( message_context.room(), &response.mime_type, response.bytes, &attachment_body_text, ) .await .map_err(|e| anyhow::anyhow!("Failed to upload and prepare event: {}", e))?; matrix_link .messaging() .send_event( message_context.room(), &mut event_content, response_type.clone(), ) .await?; if conversation.messages.len() == 1 { // If this is the beginning of the thread, send helpful instructions bot.messaging() .send_notice_markdown_no_fail( message_context.room(), strings::image_generation::guide_how_to_proceed(), response_type.clone(), ) .await; } Ok(()) } pub async fn handle_sticker( bot: &Bot, matrix_link: MatrixLink, message_context: &MessageContext, original_prompt: &str, ) -> anyhow::Result<()> { // Stickers are always sent directly to the room - no threading. let response_type = MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()); let Some(agent) = get_effective_agent_for_purpose_or_complain( bot, message_context, AgentPurpose::ImageGeneration, response_type.clone(), true, ) .await else { return Ok(()); }; let _typing_notice_guard = bot.start_typing_notice(message_context.room()).await; let span = tracing::debug_span!( "sticker_generation", agent_id = agent.identifier().as_string() ); let params = ImageGenerationParams::default() .with_smallest_size_possible(true) .with_cheaper_model_switching_allowed(true) .with_cheaper_quality_switching_allowed(true); let result = agent .controller() .generate_image(original_prompt, params) .instrument(span) .await; let response = match result { Ok(response) => response, Err(err) => { tracing::warn!( "Error in room {} while trying to generate sticker via agent {}: {:?}", message_context.room_id(), agent.identifier(), err, ); bot.messaging() .send_error_markdown_no_fail( message_context.room(), &strings::agent::error_while_serving_purpose( agent.identifier(), &AgentPurpose::ImageGeneration, &err, ), response_type, ) .await; return Ok(()); } }; let attachment_body_text = format!( "generated-sticker.{}", get_file_extension(&response.mime_type) ); let mut event_content = matrix_link .media() .upload_and_prepare_event_content( message_context.room(), &response.mime_type, response.bytes, &attachment_body_text, ) .await .map_err(|e| anyhow::anyhow!("Failed to upload and prepare event: {}", e))?; matrix_link .messaging() .send_event(message_context.room(), &mut event_content, response_type) .await?; Ok(()) } ================================================ FILE: src/controller/image/mod.rs ================================================ mod determination; pub mod edit; pub mod generation; mod prompt; pub use determination::determine_controller; ================================================ FILE: src/controller/image/prompt.rs ================================================ use crate::conversation::llm::{Author, Message, MessageContent}; /// Builds a prompt from the original prompt and other messages in the conversation. /// /// Only messages authored by the user are considered. /// /// Messages that say "Again" or "Go" (regardless of casing) are ignored. They are considered special messages /// which trigger re-generation and "start" respectively, and do not need to be included in the prompt criteria. pub fn build(original_prompt: &str, other_messages: Vec) -> String { let mut prompt = original_prompt.to_owned(); // Make a new messages vector that only contains messages we care about let other_messages: Vec = other_messages .into_iter() .filter(|message| { if let Author::User = message.author { if let MessageContent::Text(text) = &message.content { text.to_lowercase() != "again" && text.to_lowercase() != "go" } else { false } } else { false } }) .collect(); if !other_messages.is_empty() { prompt.push_str("\nOther criteria:"); for message in other_messages { if let MessageContent::Text(text) = &message.content { prompt.push_str(format!("\n- {}", text.replace("\n", ". ").as_str()).as_str()); } } } prompt } #[cfg(test)] mod tests { use super::build; use super::{Author, Message, MessageContent}; struct TestCase { original_prompt: &'static str, messages: Vec, expected_prompt: &'static str, } #[test] fn test_build_prompt() { let timestamp = chrono::Utc::now(); let test_cases = vec![ // Simple case TestCase { original_prompt: "Generate a picture of a cat", messages: vec![], expected_prompt: "Generate a picture of a cat", }, // Only a single user message TestCase { original_prompt: "Generate a picture of a dog", messages: vec![Message { author: Author::User, sender_id: None, content: MessageContent::Text("Must be blue".to_owned()), timestamp, }], expected_prompt: "Generate a picture of a dog\nOther criteria:\n- Must be blue", }, // Multiple complex user messages dispersed with assistant messages TestCase { original_prompt: "Generate a picture of an elephant", messages: vec![ Message { author: Author::User, sender_id: None, content: MessageContent::Text("Must be blue".to_owned()), timestamp, }, Message { author: Author::Assistant, sender_id: None, content: MessageContent::Text("Whatever".to_owned()), timestamp, }, Message { author: Author::User, sender_id: None, content: MessageContent::Text( "Must be 3-legged.\nMust be flying.".to_owned(), ), timestamp, }, ], expected_prompt: "Generate a picture of an elephant\nOther criteria:\n- Must be blue\n- Must be 3-legged.. Must be flying.", }, // "Again" is ignored. TestCase { original_prompt: "Generate a picture of a grizzly bear", messages: vec![ Message { author: Author::User, sender_id: None, content: MessageContent::Text("Must be blue".to_owned()), timestamp, }, Message { author: Author::Assistant, sender_id: None, content: MessageContent::Text("Whatever".to_owned()), timestamp, }, Message { author: Author::User, sender_id: None, content: MessageContent::Text("Again".to_owned()), timestamp, }, Message { author: Author::User, sender_id: None, content: MessageContent::Text("again".to_owned()), timestamp, }, ], expected_prompt: "Generate a picture of a grizzly bear\nOther criteria:\n- Must be blue", }, ]; for test_case in test_cases { let actual_prompt = build(test_case.original_prompt, test_case.messages); assert_eq!(actual_prompt, test_case.expected_prompt); } } } ================================================ FILE: src/controller/join/mod.rs ================================================ use mxlink::MessageResponseType; use crate::entity::RoomConfigContext; use crate::{Bot, strings}; pub async fn handle( bot: &Bot, room: &mxlink::matrix_sdk::Room, room_config_context: &RoomConfigContext, ) -> anyhow::Result<()> { if !bot.post_join_self_introduction_enabled() { tracing::debug!( "Post-join self-introduction is disabled - not sending introduction message" ); return Ok(()); } let agent_manager = bot.agent_manager(); bot.messaging() .send_text_markdown_no_fail( room, strings::introduction::create_on_join_introduction( bot.name(), bot.command_prefix(), agent_manager, room_config_context, ) .await, MessageResponseType::InRoom, ) .await; Ok(()) } ================================================ FILE: src/controller/mod.rs ================================================ mod controller_type; pub mod access; pub mod agent; pub mod cfg; pub mod chat_completion; mod determination; mod dispatching; pub mod help; pub mod image; pub mod join; pub mod provider; pub mod reaction; pub mod usage; mod utils; pub use controller_type::ControllerType; pub use determination::determine_controller; pub use dispatching::dispatch_controller; ================================================ FILE: src/controller/provider/mod.rs ================================================ use mxlink::MessageResponseType; use crate::{Bot, agent::AgentProvider, entity::MessageContext, strings}; use super::ControllerType; pub fn determine_controller(_text: &str) -> ControllerType { ControllerType::ProviderHelp } pub async fn handle_help(message_context: &MessageContext, bot: &Bot) -> anyhow::Result<()> { let can_create_global_agents = message_context.sender_can_manage_global_config(); let can_create_room_local_agents = message_context.sender_can_manage_room_local_agents()?; let mut message = String::new(); message.push_str(&format!("## {}", strings::help::provider::heading())); message.push_str("\n\n"); message.push_str(&strings::help::provider::intro()); message.push_str("\n\n"); message.push_str(&strings::provider::providers_list_intro()); message.push_str("\n\n"); // How to choose message.push_str(&format!( "### {}", strings::provider::help_how_to_choose_heading() )); message.push_str("\n\n"); message.push_str(&strings::provider::help_how_to_choose_description( bot.command_prefix(), )); message.push_str("\n\n"); // How to use message.push_str(&format!( "### {}", strings::provider::help_how_to_use_heading() )); message.push_str("\n\n"); message.push_str(&strings::provider::help_how_to_use_description( bot.command_prefix(), )); message.push_str("\n\n"); for provider in AgentProvider::choices() { let provider_info = provider.info(); message.push_str(&format!( "### {}", strings::provider::help_provider_heading( provider_info.name, &provider_info.homepage_url.as_ref().map(|s| s.to_string()) ) )); message.push_str("\n\n"); message.push_str(&strings::provider::help_provider_details( provider.to_static_str(), &provider_info, )); // We always show a "Quick start" section (even to unprivileged users), // because we're talking about it in a previous message. message.push_str("- 🗲 Quick start:"); if can_create_room_local_agents { message.push_str(&format!( "\n\t- create a room-local agent: `{command_prefix} agent create-room-local {provider_id} my-{provider_id}-agent`", command_prefix = bot.command_prefix(), provider_id = provider.to_static_str(), )); } if can_create_global_agents { message.push_str(&format!( "\n\t- create a global agent: `{command_prefix} agent create-global {provider_id} my-{provider_id}-agent`", command_prefix = bot.command_prefix(), provider_id = provider.to_static_str(), )); } if !can_create_room_local_agents && !can_create_global_agents { message.push_str(" ask an administrator to create an agent for you (you lack permissions to do so yourself)"); } message.push_str("\n\n"); } bot.messaging() .send_text_markdown_no_fail( message_context.room(), message, MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()), ) .await; Ok(()) } ================================================ FILE: src/controller/reaction/mod.rs ================================================ use std::ops::Deref; use mxlink::MatrixLink; use crate::{ Bot, agent::AgentPurpose, entity::{MessageContext, MessagePayload}, }; mod text_to_speech; pub async fn handle( bot: &Bot, matrix_link: MatrixLink, message_context: &MessageContext, ) -> anyhow::Result<()> { match &message_context.payload() { MessagePayload::Reaction { key, reacted_to_event_payload, reacted_to_event_id, reacted_to_event_sender_id, } => { if key == AgentPurpose::TextToSpeech.emoji() { if let MessagePayload::Text(text_content) = reacted_to_event_payload.deref() { return text_to_speech::handle( bot, matrix_link, message_context, reacted_to_event_id, reacted_to_event_sender_id, text_content, ) .await; } tracing::debug!("Ignoring text-to-speech reaction to non-text message"); return Ok(()); } tracing::debug!("Ignoring unknown reaction"); Ok(()) } _ => Err(anyhow::anyhow!( "Reaction controller called with a non-reaction message" )), } } ================================================ FILE: src/controller/reaction/text_to_speech.rs ================================================ use mxlink::{MatrixLink, MessageResponseType}; use mxlink::matrix_sdk::ruma::{ OwnedEventId, OwnedUserId, events::room::message::TextMessageEventContent, }; use crate::entity::roomconfig::{ TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType, }; use crate::{ Bot, agent::AgentPurpose, controller::utils::agent::get_effective_agent_for_purpose_or_complain, entity::MessageContext, }; pub(super) async fn handle( bot: &Bot, matrix_link: MatrixLink, message_context: &MessageContext, reacted_to_event_id: &OwnedEventId, reacted_to_event_sender_id: &OwnedUserId, text_content: &TextMessageEventContent, ) -> anyhow::Result<()> { // If we're in a thread, we're likely dealing with a bot message, so we should start in the thread. // Otherwise, we're likely operating in "TTS user messages" mode, so we should reply to the reacted-to message and avoid threads. let response_type = if message_context.thread_info().is_thread_root_only() { MessageResponseType::Reply(reacted_to_event_id.clone()) } else { MessageResponseType::InThread(message_context.thread_info().clone()) }; if !is_allowed_to_tts_for_event( message_context, reacted_to_event_sender_id, matrix_link.user_id(), ) { tracing::debug!( "Ignoring request for on-demand text-to-speech (via reaction) due to room configuration" ); return Ok(()); } let speech_agent = get_effective_agent_for_purpose_or_complain( bot, message_context, AgentPurpose::TextToSpeech, response_type.clone(), true, ) .await; let Some(speech_agent) = speech_agent else { // We've already complained about this in get_effective_agent_or_complain return Ok(()); }; let _typing_notice_guard = bot.start_typing_notice(message_context.room()).await; crate::controller::utils::text_to_speech::generate_and_send_tts_for_message( bot, matrix_link, message_context, response_type, &speech_agent, reacted_to_event_id, &text_content.body, ) .await; Ok(()) } fn is_allowed_to_tts_for_event( message_context: &MessageContext, sender_id: &OwnedUserId, bot_user_id: &OwnedUserId, ) -> bool { // Whether we're allowed depends on who the original message sender is (the bot or some user). // // The user may be an allowed bot user or someone else. // Regardless, we've been invoked by an allowed user, so if the user wants TTS for a foreign message, we should allow it. if *sender_id == *bot_user_id { match message_context .room_config_context() .text_to_speech_bot_messages_flow_type() { TextToSpeechBotMessagesFlowType::Never => false, TextToSpeechBotMessagesFlowType::OnDemandAlways => true, TextToSpeechBotMessagesFlowType::OnDemandForVoice => true, TextToSpeechBotMessagesFlowType::OnlyForVoice => true, TextToSpeechBotMessagesFlowType::Always => true, } } else { match message_context .room_config_context() .text_to_speech_user_messages_flow_type() { TextToSpeechUserMessagesFlowType::Never => false, TextToSpeechUserMessagesFlowType::OnDemand => true, TextToSpeechUserMessagesFlowType::Always => true, } } } ================================================ FILE: src/controller/usage/mod.rs ================================================ use mxlink::MessageResponseType; use crate::{Bot, entity::MessageContext, strings}; use super::ControllerType; pub fn determine_controller(_text: &str) -> ControllerType { ControllerType::UsageHelp } pub async fn handle_help(message_context: &MessageContext, bot: &Bot) -> anyhow::Result<()> { bot.messaging() .send_text_markdown_no_fail( message_context.room(), strings::usage::intro(bot.command_prefix()), MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()), ) .await; Ok(()) } ================================================ FILE: src/controller/utils/agent.rs ================================================ use mxlink::MessageResponseType; use crate::{ Bot, agent::{ AgentInstance, AgentPurpose, utils::{AgentForPurposeDeterminationError, get_effective_agent_for_purpose}, }, entity::MessageContext, strings, }; pub async fn get_effective_agent_for_purpose_or_complain( bot: &Bot, message_context: &MessageContext, agent_purpose: AgentPurpose, response_type: MessageResponseType, complain_when_purpose_unsupported: bool, ) -> Option { let agent_info = get_effective_agent_for_purpose( bot.agent_manager(), message_context.room_config_context(), agent_purpose, ) .await; match agent_info { Ok(agent_info) => Some(agent_info.instance), Err(err) => { let error_message = match err { AgentForPurposeDeterminationError::Unknown(err_string) => Some(err_string), AgentForPurposeDeterminationError::NoneConfigured => None, AgentForPurposeDeterminationError::ConfiguredButMissing(agent_identifier) => Some( strings::room_config::configures_agent_for_purpose_but_does_not_exist( &agent_identifier, agent_purpose, ), ), AgentForPurposeDeterminationError::ConfiguredButLacksSupport(agent_identifier) => { if complain_when_purpose_unsupported { Some(strings::room_config::configures_agent_for_purpose_but_agent_does_not_support_it( &agent_identifier, agent_purpose, )) } else { None } } }; if let Some(error_message) = error_message { bot.messaging() .send_error_markdown_no_fail( message_context.room(), &error_message, response_type, ) .await; }; None } } } ================================================ FILE: src/controller/utils/mod.rs ================================================ use mxlink::MessageResponseType; use crate::{ Bot, entity::{MessageContext, MessagePayload}, }; pub mod agent; pub mod text_to_speech; pub async fn get_text_body_or_complain<'a>( bot: &Bot, message_context: &'a MessageContext, ) -> Option<&'a str> { match &message_context.payload() { MessagePayload::Text(text_message_content) => Some(&text_message_content.body), _ => { bot.messaging() .send_text_markdown_no_fail( message_context.room(), "This command only works with text messages.".to_owned(), MessageResponseType::InThread(message_context.thread_info().clone()), ) .await; None } } } ================================================ FILE: src/controller/utils/text_to_speech.rs ================================================ use mxlink::matrix_sdk::ruma::OwnedEventId; use mxlink::{MatrixLink, MessageResponseType}; use tracing::Instrument; use crate::utils::mime::get_file_extension; use crate::{ Bot, agent::{AgentInstance, AgentPurpose, ControllerTrait, provider::TextToSpeechParams}, entity::MessageContext, strings, }; pub async fn generate_and_send_tts_for_message( bot: &Bot, matrix_link: MatrixLink, message_context: &MessageContext, response_type: MessageResponseType, speech_agent: &AgentInstance, text_message_event_id: &OwnedEventId, text_content: &str, ) -> bool { let reaction_event_response = bot .reacting() .react_no_fail( message_context.room(), text_message_event_id.clone(), strings::PROGRESS_INDICATOR_EMOJI.to_owned(), ) .await; let result = do_generate_and_send_tts_for_message( bot, matrix_link, message_context, response_type, speech_agent, text_content, ) .await; if let Some(reaction_event_response) = reaction_event_response { let redaction_reason = if result { strings::text_to_speech::redaction_reason_done() } else { strings::text_to_speech::redaction_reason_failed() }; bot.messaging() .redact_event_no_fail( message_context.room(), reaction_event_response.event_id, Some(redaction_reason.to_owned()), ) .await; } result } async fn do_generate_and_send_tts_for_message( bot: &Bot, matrix_link: MatrixLink, message_context: &MessageContext, response_type: MessageResponseType, speech_agent: &AgentInstance, text_content: &str, ) -> bool { let params = TextToSpeechParams { speed_override: message_context .room_config_context() .text_to_speech_speed_override(), voice_override: message_context .room_config_context() .text_to_speech_voice_override(), }; let text_content = if let Some(text_content) = text_content.strip_prefix(bot.command_prefix()) { text_content.trim() } else { text_content }; let span = tracing::debug_span!( "text_to_speech_generation", agent_id = speech_agent.identifier().as_string() ); let text_to_speech_result = speech_agent .controller() .text_to_speech(text_content, params) .instrument(span) .await; let text_to_speech_result = match text_to_speech_result { Ok(text_to_speech_result) => text_to_speech_result, Err(err) => { tracing::warn!( "Error in room {} while trying to generate TTS via agent {}: {:?}", message_context.room_id(), speech_agent.identifier(), err, ); bot.messaging() .send_error_markdown_no_fail( message_context.room(), &strings::agent::error_while_serving_purpose( speech_agent.identifier(), &AgentPurpose::SpeechToText, &err, ), response_type, ) .await; return false; } }; let attachment_body_text = format!( "generated-speech.{}", get_file_extension(&text_to_speech_result.mime_type) ); let event_content = matrix_link .media() .upload_and_prepare_event_content( message_context.room(), &text_to_speech_result.mime_type, text_to_speech_result.bytes, &attachment_body_text, ) .await; let mut event_content = match event_content { Ok(event_content) => event_content, Err(err) => { tracing::error!( ?err, "Error in room {} while trying to upload TTS via agent {}", message_context.room_id(), speech_agent.identifier(), ); return false; } }; let result = matrix_link .messaging() .send_event( message_context.room(), &mut event_content, response_type.clone(), ) .await; let Err(err) = result else { return true; }; tracing::error!( ?err, "Error in room {} while trying to send TTS payload", message_context.room_id(), ); false } ================================================ FILE: src/conversation/llm/entity.rs ================================================ use chrono::{DateTime, Utc}; use mxlink::matrix_sdk::ruma::OwnedUserId; use mxlink::matrix_sdk::ruma::events::room::message::{ FileMessageEventContent, ImageMessageEventContent, }; use mxlink::mime::Mime; use crate::agent::provider::ImageSource; #[derive(Debug, Clone, PartialEq)] pub enum Author { Prompt, Assistant, User, } #[derive(Debug, Clone)] pub struct Message { pub author: Author, pub sender_id: Option, pub timestamp: DateTime, pub content: MessageContent, } #[derive(Debug, Clone)] pub struct ImageDetails { pub event_content: ImageMessageEventContent, pub mime: Mime, pub data: Vec, } impl ImageDetails { pub fn new(event_content: ImageMessageEventContent, mime: Mime, data: Vec) -> Self { Self { event_content, mime, data, } } pub fn filename(&self) -> String { self.event_content .filename .clone() .unwrap_or(self.event_content.body.clone()) } } impl From for ImageSource { fn from(value: ImageDetails) -> Self { ImageSource::new(value.filename(), value.data.clone(), value.mime.clone()) } } #[derive(Debug, Clone)] pub struct FileDetails { pub event_content: FileMessageEventContent, pub mime: Mime, pub data: Vec, } impl FileDetails { pub fn new(event_content: FileMessageEventContent, mime: Mime, data: Vec) -> Self { Self { event_content, mime, data, } } pub fn filename(&self) -> String { self.event_content .filename .clone() .unwrap_or(self.event_content.body.clone()) } } #[derive(Debug, Clone)] pub enum MessageContent { Text(String), Image(ImageDetails), File(FileDetails), } impl PartialEq for MessageContent { fn eq(&self, other: &Self) -> bool { match (self, other) { (MessageContent::Text(a), MessageContent::Text(b)) => a == b, (MessageContent::Image(a), MessageContent::Image(b)) => { // We can probably do better than this by inspecting `.event_conten1t.source`, but for now this is good enough. a.filename() == b.filename() } (MessageContent::File(a), MessageContent::File(b)) => a.filename() == b.filename(), _ => false, } } } #[derive(Debug)] pub struct Conversation { pub messages: Vec, } impl Conversation { /// Combine consecutive messages by the same author into a single message. /// /// Certain models (like Anthropic) cannot tolerate consecutive messages by the same author, /// so combining them helps avoid issues. /// /// When multiple text messages by the same author are merged, the resulting message keeps a /// `sender_id` only if all merged messages came from the same sender. Mixed-sender merges are /// possible for user turns in multi-user rooms, so `sender_id` is cleared in that case to /// avoid incorrectly attributing the whole merged turn to the first sender. /// See: https://github.com/etkecc/baibot/issues/13 pub fn combine_consecutive_messages(&self) -> Conversation { // We'll likely get fewer messages, but let's reserve the maximum we expect. let mut new_messages = Vec::with_capacity(self.messages.len()); let mut last_seen_text_from_author: Option = None; for message in &self.messages { let MessageContent::Text(message_text_content) = &message.content else { last_seen_text_from_author = None; new_messages.push(message.clone()); continue; }; let Some(last_seen_author_clone) = last_seen_text_from_author.clone() else { last_seen_text_from_author = Some(message.author.clone()); new_messages.push(message.clone()); continue; }; if message.author != last_seen_author_clone { last_seen_text_from_author = Some(message.author.clone()); new_messages.push(message.clone()); continue; } let last_message = new_messages.last_mut().unwrap(); if let MessageContent::Text(ref mut text) = last_message.content { text.push('\n'); text.push_str(message_text_content); } if last_message.sender_id != message.sender_id { last_message.sender_id = None; } } Conversation { messages: new_messages, } } pub fn start_time(&self) -> Option> { self.messages.first().map(|message| message.timestamp) } } #[cfg(test)] mod tests { use super::*; use chrono::{TimeZone, Utc}; use mxlink::matrix_sdk::ruma::{OwnedMxcUri, OwnedUserId}; use mxlink::mime; #[test] fn combine_consecutive_messages() { let timestamp_1 = Utc.with_ymd_and_hms(2024, 9, 20, 18, 34, 15).unwrap(); let timestamp_2 = Utc.with_ymd_and_hms(2024, 9, 21, 18, 34, 16).unwrap(); let timestamp_3 = Utc.with_ymd_and_hms(2024, 9, 22, 18, 34, 17).unwrap(); let timestamp_4 = Utc.with_ymd_and_hms(2024, 9, 23, 18, 34, 18).unwrap(); let image_event_content = ImageMessageEventContent::plain( "image.png".to_string(), OwnedMxcUri::from("mxc://example.com/1234567890"), ); let conversation = Conversation { messages: vec![ // User's turn Message { author: Author::User, sender_id: None, content: MessageContent::Text("Hello".to_string()), timestamp: timestamp_1, }, Message { author: Author::User, sender_id: None, content: MessageContent::Text("How are you?".to_string()), timestamp: timestamp_2, }, Message { author: Author::User, sender_id: None, content: MessageContent::Text("I'm OK, btw.".to_string()), timestamp: timestamp_3, }, Message { author: Author::User, sender_id: None, content: MessageContent::Image(ImageDetails::new( image_event_content.clone(), mime::IMAGE_PNG, vec![], )), timestamp: timestamp_4, }, Message { author: Author::User, sender_id: None, content: MessageContent::Text("Above is an image.".to_string()), timestamp: timestamp_4, }, Message { author: Author::User, sender_id: None, content: MessageContent::Text("Would you take a look at it?".to_string()), timestamp: timestamp_4, }, // Assistant's turn Message { author: Author::Assistant, sender_id: None, content: MessageContent::Text("Hi there!".to_string()), timestamp: timestamp_2, }, Message { author: Author::Assistant, sender_id: None, content: MessageContent::Text("I'm doing well, thank you.".to_string()), timestamp: timestamp_3, }, // User's turn Message { author: Author::User, sender_id: None, content: MessageContent::Text("That's great!".to_string()), timestamp: timestamp_3, }, ], }; let conversation = conversation.combine_consecutive_messages(); assert_eq!(conversation.messages.len(), 5); assert_eq!(conversation.messages[0].author, Author::User); assert_eq!( conversation.messages[0].content, MessageContent::Text("Hello\nHow are you?\nI'm OK, btw.".to_string()) ); assert_eq!(conversation.messages[0].timestamp, timestamp_1); assert_eq!(conversation.messages[1].author, Author::User); assert_eq!( conversation.messages[1].content, MessageContent::Image(ImageDetails::new( image_event_content.clone(), mime::IMAGE_PNG, vec![], )) ); assert_eq!(conversation.messages[2].author, Author::User); assert_eq!( conversation.messages[2].content, MessageContent::Text("Above is an image.\nWould you take a look at it?".to_string()) ); assert_eq!(conversation.messages[2].timestamp, timestamp_4); assert_eq!(conversation.messages[3].author, Author::Assistant); assert_eq!( conversation.messages[3].content, MessageContent::Text("Hi there!\nI'm doing well, thank you.".to_string()) ); assert_eq!(conversation.messages[3].timestamp, timestamp_2); assert_eq!(conversation.messages[4].author, Author::User); assert_eq!( conversation.messages[4].content, MessageContent::Text("That's great!".to_string()) ); assert_eq!(conversation.messages[4].timestamp, timestamp_3); } #[test] fn combine_consecutive_messages_clears_sender_id_for_mixed_sender_turns() { let timestamp_1 = Utc.with_ymd_and_hms(2024, 9, 20, 18, 34, 15).unwrap(); let timestamp_2 = Utc.with_ymd_and_hms(2024, 9, 20, 18, 34, 16).unwrap(); let sender_1 = OwnedUserId::try_from("@alice:example.com").unwrap(); let sender_2 = OwnedUserId::try_from("@bob:example.com").unwrap(); let conversation = Conversation { messages: vec![ Message { author: Author::User, sender_id: Some(sender_1), content: MessageContent::Text("Hello".to_string()), timestamp: timestamp_1, }, Message { author: Author::User, sender_id: Some(sender_2), content: MessageContent::Text("Hi there".to_string()), timestamp: timestamp_2, }, ], }; let conversation = conversation.combine_consecutive_messages(); assert_eq!(conversation.messages.len(), 1); assert_eq!(conversation.messages[0].sender_id, None); assert_eq!( conversation.messages[0].content, MessageContent::Text("Hello\nHi there".to_string()) ); assert_eq!(conversation.messages[0].timestamp, timestamp_1); } } ================================================ FILE: src/conversation/llm/mod.rs ================================================ mod entity; mod tokenization; mod utils; #[cfg(test)] mod tests; pub use entity::*; pub use tokenization::shorten_messages_list_to_context_size; pub use utils::*; ================================================ FILE: src/conversation/llm/tests.rs ================================================ use mxlink::matrix_sdk::ruma::OwnedUserId; use crate::utils::status::create_error_message_text; use crate::utils::text_to_speech::create_transcribed_message_text; use super::*; #[test] fn test_messages_by_the_bot_are_identified_correctly() { let bot_user_id = OwnedUserId::try_from("@bot:example.com").expect("Failed to parse bot user ID"); let matrix_message = super::super::matrix::MatrixMessage { sender_id: bot_user_id.to_owned(), content: super::super::matrix::MatrixMessageContent::Text("Hello!".to_owned()), mentioned_users: vec![], timestamp: chrono::Utc::now(), }; let llm_message = convert_matrix_message_to_llm_message(&matrix_message, &bot_user_id).unwrap(); assert_eq!(llm_message.author, Author::Assistant); assert_eq!(llm_message.sender_id, Some(bot_user_id.clone())); assert_eq!( llm_message.content, MessageContent::Text("Hello!".to_string()) ); } #[test] fn test_notice_messages_by_bot_with_speech_to_text_prefix_are_cleaned_up_and_considered_sent_by_user() { let bot_user_id = OwnedUserId::try_from("@bot:example.com").expect("Failed to parse bot user ID"); let source_message_text = "Hello!"; let message_text = create_transcribed_message_text(source_message_text); assert_ne!(source_message_text, message_text); let matrix_message = super::super::matrix::MatrixMessage { sender_id: bot_user_id.to_owned(), content: super::super::matrix::MatrixMessageContent::Notice(message_text), mentioned_users: vec![], timestamp: chrono::Utc::now(), }; let llm_message = convert_matrix_message_to_llm_message(&matrix_message, &bot_user_id).unwrap(); assert_eq!(llm_message.author, Author::User); assert_eq!(llm_message.sender_id, None); assert_eq!( llm_message.content, MessageContent::Text(source_message_text.to_string()) ); } #[test] fn test_notice_error_messages_by_bot_are_ignored() { let bot_user_id = OwnedUserId::try_from("@bot:example.com").expect("Failed to parse bot user ID"); let source_message_text = "Some error happened"; let message_text = create_error_message_text(source_message_text); assert_ne!(source_message_text, message_text); let matrix_message = super::super::matrix::MatrixMessage { sender_id: bot_user_id.to_owned(), content: super::super::matrix::MatrixMessageContent::Notice(message_text), mentioned_users: vec![], timestamp: chrono::Utc::now(), }; let llm_message = convert_matrix_message_to_llm_message(&matrix_message, &bot_user_id); assert!(llm_message.is_none()); } #[test] fn test_user_messages_preserve_sender_id() { let bot_user_id = OwnedUserId::try_from("@bot:example.com").expect("Failed to parse bot user ID"); let user_id = OwnedUserId::try_from("@alice:example.com").expect("Failed to parse user ID"); let matrix_message = super::super::matrix::MatrixMessage { sender_id: user_id.clone(), content: super::super::matrix::MatrixMessageContent::Text("Hello!".to_owned()), mentioned_users: vec![], timestamp: chrono::Utc::now(), }; let llm_message = convert_matrix_message_to_llm_message(&matrix_message, &bot_user_id).unwrap(); assert_eq!(llm_message.author, Author::User); assert_eq!(llm_message.sender_id, Some(user_id)); assert_eq!( llm_message.content, MessageContent::Text("Hello!".to_string()) ); } #[test] fn test_other_notice_messages_by_the_bot_are_ignored() { // Also see `test_notice_error_messages_by_bot_are_ignored()`. // That one passes accidentally, because we ignore all messages by the bot that are notices // (except for speech-to-text-created transcriptions - see `test_notice_messages_by_bot_with_speech_to_text_prefix_are_cleaned_up_and_considered_sent_by_user()`). // This test is to make sure that we don't accidentally start accepting other notice messages. let bot_user_id = OwnedUserId::try_from("@bot:example.com").expect("Failed to parse bot user ID"); let message_text = "Something something"; let matrix_message = super::super::matrix::MatrixMessage { sender_id: bot_user_id.to_owned(), content: super::super::matrix::MatrixMessageContent::Notice(message_text.to_owned()), mentioned_users: vec![], timestamp: chrono::Utc::now(), }; let llm_message = convert_matrix_message_to_llm_message(&matrix_message, &bot_user_id); assert!(llm_message.is_none()); } ================================================ FILE: src/conversation/llm/tokenization.rs ================================================ use tiktoken_rs::CoreBPE; use tiktoken_rs::bpe_for_tokenizer; use tiktoken_rs::tokenizer; use super::{Author, Message, MessageContent}; fn get_bpe_for_model(model: &str) -> &'static CoreBPE { let tokenizer = tokenizer::get_tokenizer(model) .or_else(|| tokenizer::get_tokenizer("gpt-4")) .unwrap(); bpe_for_tokenizer(tokenizer).unwrap() } pub fn shorten_messages_list_to_context_size( model: &str, prompt_message: &Option, mut messages: Vec, max_response_tokens: Option, max_context_tokens: u32, ) -> Vec { // Loading the tokenization data is an expensive process, so // se construct the BPE instance once and then use it for all messages. let bpe = get_bpe_for_model(model); // We want to retain the prompt in all cases, so we always count it first. // We also always reserve enough tokens for the maximum response we expect. let mut current_context_length: u32 = if let Some(prompt_message) = prompt_message { calculate_token_size_for_message(bpe, model, prompt_message) + max_response_tokens.unwrap_or(0) } else { 0 }; messages.reverse(); let mut messages_to_keep: Vec = Vec::new(); for message in messages { let tokens_for_message = calculate_token_size_for_message(bpe, model, &message); if current_context_length + tokens_for_message > max_context_tokens { break; } current_context_length += tokens_for_message; messages_to_keep.push(message); } messages_to_keep.reverse(); messages_to_keep } /// Calculate the token size of a message for a given model, with a preloaded CoreBPE object. /// Related to `calculate_token_size_for_model_message`. fn calculate_token_size_for_message(bpe: &CoreBPE, model: &str, message: &Message) -> u32 { let (tokens_per_message, tokens_per_name) = if model.starts_with("gpt-3.5") { ( 4, // every message follows {role/name}\n{content}\n -1, // if there's a name, the role is omitted ) } else { (3, 1) }; let role_length = match message.author { Author::Assistant => bpe.encode_with_special_tokens("assistant").len() as i32, Author::User => bpe.encode_with_special_tokens("user").len() as i32, Author::Prompt => bpe.encode_with_special_tokens("system").len() as i32, }; let text_length = match &message.content { MessageContent::Text(text) => bpe.encode_with_special_tokens(text).len() as i32, MessageContent::Image(..) => 0, MessageContent::File(..) => 0, }; (text_length + role_length + tokens_per_message + tokens_per_name) as u32 } pub mod test { #[test] fn message_size_counting_works() { let model = "gpt-4"; let bpe = super::get_bpe_for_model(model); let message = super::Message { author: super::Author::User, sender_id: None, content: super::MessageContent::Text("Hello there!".to_string()), timestamp: chrono::Utc::now(), }; let tokens = super::calculate_token_size_for_message(bpe, model, &message); assert_eq!(8, tokens); } #[test] fn shortening_works_with_english() { let model = "gpt-4"; let bpe = super::get_bpe_for_model(model); let max_response_tokens: Option = Some(5); let prompt = super::Message { author: super::Author::Prompt, sender_id: None, content: super::MessageContent::Text("You are a bot!".to_string()), timestamp: chrono::Utc::now(), }; let prompt_length = 10; assert_eq!( prompt_length, super::calculate_token_size_for_message(bpe, model, &prompt) ); let mut conversation_messages = Vec::new(); let first = super::Message { author: super::Author::User, sender_id: None, content: super::MessageContent::Text("Hello there!".to_string()), timestamp: chrono::Utc::now(), }; let first_length = 8; assert_eq!( first_length, super::calculate_token_size_for_message(bpe, model, &first) ); conversation_messages.push(first); let second = super::Message { author: super::Author::Assistant, sender_id: None, content: super::MessageContent::Text("Hello!".to_string()), timestamp: chrono::Utc::now(), }; let second_length = 7; assert_eq!( second_length, super::calculate_token_size_for_message(bpe, model, &second) ); conversation_messages.push(second); let third = super::Message { author: super::Author::User, sender_id: None, content: super::MessageContent::Text( "This is the 3rd message in this conversation. It shall be preserved.".to_owned(), ), timestamp: chrono::Utc::now(), }; let third_length = 21; assert_eq!( third_length, super::calculate_token_size_for_message(bpe, model, &third) ); conversation_messages.push(third.clone()); let forth = super::Message { author: super::Author::Assistant, sender_id: None, content: super::MessageContent::Text( "This is yet another message that shall be preserved.".to_owned(), ), timestamp: chrono::Utc::now(), }; let forth_length = 15; assert_eq!( forth_length, super::calculate_token_size_for_message(bpe, model, &forth) ); conversation_messages.push(forth.clone()); assert_eq!(4, conversation_messages.len()); let new_conversation_messages = super::shorten_messages_list_to_context_size( model, &Some(prompt), conversation_messages, max_response_tokens, prompt_length + max_response_tokens.unwrap_or(0) + forth_length + third_length, ); assert_eq!(2, new_conversation_messages.len()); assert_eq!( new_conversation_messages.first().unwrap().content, third.content ); assert_eq!( new_conversation_messages.last().unwrap().content, forth.content ); } #[test] fn shortening_works_with_japanese() { let model = "gpt-4"; let bpe = super::get_bpe_for_model(model); let max_response_tokens: Option = Some(5); let prompt = super::Message { author: super::Author::User, sender_id: None, content: super::MessageContent::Text("あなたはボットです。".to_string()), timestamp: chrono::Utc::now(), }; let prompt_length = 14; assert_eq!( prompt_length, super::calculate_token_size_for_message(bpe, model, &prompt) ); let mut conversation_messages = Vec::new(); let first = super::Message { author: super::Author::User, sender_id: None, content: super::MessageContent::Text("こんにちは!".to_string()), timestamp: chrono::Utc::now(), }; let first_length = 7; assert_eq!( first_length, super::calculate_token_size_for_message(bpe, model, &first) ); conversation_messages.push(first); let second = super::Message { author: super::Author::Assistant, sender_id: None, content: super::MessageContent::Text("こんにちは。今日は元気ですか。".to_string()), timestamp: chrono::Utc::now(), }; let second_length = 15; assert_eq!( second_length, super::calculate_token_size_for_message(bpe, model, &second) ); conversation_messages.push(second); let third = super::Message { author: super::Author::User, sender_id: None, content: super::MessageContent::Text( "これは第3のメッセージなので、保存されます。".to_string(), ), timestamp: chrono::Utc::now(), }; let third_length = 22; assert_eq!( third_length, super::calculate_token_size_for_message(bpe, model, &third) ); conversation_messages.push(third.clone()); let forth = super::Message { author: super::Author::Assistant, sender_id: None, content: super::MessageContent::Text( "これはもう一つの保存されますメッセージです。".to_string(), ), timestamp: chrono::Utc::now(), }; let forth_length = 21; assert_eq!( forth_length, super::calculate_token_size_for_message(bpe, model, &forth) ); conversation_messages.push(forth.clone()); assert_eq!(4, conversation_messages.len()); let new_conversation_messages = super::shorten_messages_list_to_context_size( model, &Some(prompt), conversation_messages, max_response_tokens, prompt_length + max_response_tokens.unwrap_or(0) + forth_length + third_length, ); assert_eq!(2, new_conversation_messages.len()); assert_eq!( new_conversation_messages.first().unwrap().content, third.content ); assert_eq!( new_conversation_messages.last().unwrap().content, forth.content ); } } ================================================ FILE: src/conversation/llm/utils.rs ================================================ use mxlink::matrix_sdk::ruma::OwnedUserId; use super::entity::{Author, FileDetails, ImageDetails, Message, MessageContent}; use crate::conversation::matrix::{MatrixMessage, MatrixMessageContent}; use crate::utils::text_to_speech as text_to_speech_utils; pub fn convert_matrix_message_to_llm_message( matrix_message: &MatrixMessage, bot_user_id: &OwnedUserId, ) -> Option { if matrix_message.sender_id == bot_user_id.as_str() { return convert_bot_message(matrix_message); } convert_user_message(matrix_message) } fn convert_bot_message(matrix_message: &MatrixMessage) -> Option { match &matrix_message.content { MatrixMessageContent::Text(text) => convert_bot_text_message( text, &matrix_message.timestamp, matrix_message.sender_id.clone(), ), MatrixMessageContent::Notice(text) => { convert_bot_notice_message(text, &matrix_message.timestamp) } MatrixMessageContent::Image(image_content, mime_type, media_bytes) => Some(Message { author: Author::Assistant, sender_id: Some(matrix_message.sender_id.clone()), content: MessageContent::Image(ImageDetails::new( image_content.clone(), mime_type.clone(), media_bytes.clone(), )), timestamp: matrix_message.timestamp.to_owned(), }), MatrixMessageContent::File(file_content, mime_type, media_bytes) => Some(Message { author: Author::Assistant, sender_id: Some(matrix_message.sender_id.clone()), content: MessageContent::File(FileDetails::new( file_content.clone(), mime_type.clone(), media_bytes.clone(), )), timestamp: matrix_message.timestamp.to_owned(), }), } } fn convert_bot_text_message( text: &str, timestamp: &chrono::DateTime, sender_id: OwnedUserId, ) -> Option { Some(Message { author: Author::Assistant, sender_id: Some(sender_id), content: MessageContent::Text(text.to_owned()), timestamp: timestamp.to_owned(), }) } fn convert_bot_notice_message( text: &str, timestamp: &chrono::DateTime, ) -> Option { // Notice messages sent by the bot are usually transcriptions of previous messages sent by the user. // Such transcriptions are prefixed with an emoji and blockquoted. // If we find a notice that doesn't match this pattern, we skip it. // // It should be noted that transcriptions are sometimes posted as regular notice (or even text) messages which do not include // the `> 🦻` formatting. This function will not handle these properly. if let Some(text) = text_to_speech_utils::parse_transcribed_message_text(text) { // This is a transcription message. We remove the prefix and consider it as a message sent by the user. // sender_id is None because the original speaker is unknown. return Some(Message { author: Author::User, sender_id: None, content: MessageContent::Text(text.to_owned()), timestamp: timestamp.to_owned(), }); } None } fn convert_user_message(matrix_message: &MatrixMessage) -> Option { match &matrix_message.content { MatrixMessageContent::Text(text) => Some(Message { author: Author::User, sender_id: Some(matrix_message.sender_id.clone()), content: MessageContent::Text(text.clone()), timestamp: matrix_message.timestamp.to_owned(), }), MatrixMessageContent::Notice(text) => Some(Message { author: Author::User, sender_id: Some(matrix_message.sender_id.clone()), content: MessageContent::Text(text.clone()), timestamp: matrix_message.timestamp.to_owned(), }), MatrixMessageContent::Image(image_content, mime_type, media_bytes) => Some(Message { author: Author::User, sender_id: Some(matrix_message.sender_id.clone()), content: MessageContent::Image(ImageDetails::new( image_content.clone(), mime_type.clone(), media_bytes.clone(), )), timestamp: matrix_message.timestamp.to_owned(), }), MatrixMessageContent::File(file_content, mime_type, media_bytes) => Some(Message { author: Author::User, sender_id: Some(matrix_message.sender_id.clone()), content: MessageContent::File(FileDetails::new( file_content.clone(), mime_type.clone(), media_bytes.clone(), )), timestamp: matrix_message.timestamp.to_owned(), }), } } ================================================ FILE: src/conversation/matrix/entity.rs ================================================ use chrono::{DateTime, Utc}; use regex::Regex; use mxlink::matrix_sdk::ruma::OwnedUserId; use mxlink::matrix_sdk::ruma::events::room::message::{ FileMessageEventContent, ImageMessageEventContent, }; use mxlink::mime::Mime; #[derive(Clone)] pub struct MatrixMessage { pub sender_id: OwnedUserId, pub content: MatrixMessageContent, pub mentioned_users: Vec, pub timestamp: DateTime, } #[derive(Clone)] pub enum MatrixMessageContent { Text(String), Notice(String), Image(ImageMessageEventContent, Mime, Vec), File(FileMessageEventContent, Mime, Vec), } #[derive(Clone)] pub struct MatrixMessageProcessingParams { pub(crate) bot_user_id: OwnedUserId, /// The prefixes that will be stripped when processing the messages in the context (thread or reply chain), /// which are found to be mentioning the bot user (`bot_user_id`). pub(crate) bot_user_prefixes_to_strip: Vec, /// The prefixes that will be stripped when processing the 1st message in the context (thread or reply chain). pub(crate) first_message_prefixes_to_strip: Vec, /// A list of users whose messages are allowed. /// If None, all messages are allowed. /// If Some, only messages from the allowed users (and the bot itself, `bot_user_id`) are allowed. pub(crate) allowed_users: Option>, } impl MatrixMessageProcessingParams { pub fn new(bot_user_id: OwnedUserId, allowed_users: Option>) -> Self { Self { bot_user_id, bot_user_prefixes_to_strip: vec![], first_message_prefixes_to_strip: vec![], allowed_users, } } pub fn with_bot_user_prefixes_to_strip(mut self, value: Vec) -> Self { self.bot_user_prefixes_to_strip = value; self } pub fn with_first_message_prefixes_to_strip(mut self, value: Vec) -> Self { self.first_message_prefixes_to_strip = value; self } } ================================================ FILE: src/conversation/matrix/mod.rs ================================================ mod entity; mod room_display_name_fetcher; mod room_event_fetcher; mod utils; pub(crate) use room_display_name_fetcher::RoomDisplayNameFetcher; pub(crate) use room_event_fetcher::RoomEventFetcher; pub(crate) use entity::{MatrixMessage, MatrixMessageContent, MatrixMessageProcessingParams}; pub(crate) use utils::*; ================================================ FILE: src/conversation/matrix/room_display_name_fetcher.rs ================================================ use mxlink::matrix_sdk::Room; use mxlink::matrix_sdk::ruma::OwnedRoomId; use mxlink::MatrixLink; use quick_cache::sync::Cache; pub struct RoomDisplayNameFetcher { matrix_link: MatrixLink, lru_cache: Option>>, } impl RoomDisplayNameFetcher { pub fn new(matrix_link: MatrixLink, lru_cache_size: Option) -> Self { let lru_cache = lru_cache_size.map(Cache::new); Self { matrix_link, lru_cache, } } #[tracing::instrument(skip_all, fields(room_id = room.room_id().as_str()))] pub async fn own_display_name_in_room( &self, room: &Room, ) -> mxlink::matrix_sdk::Result> { let Some(lru_cache) = &self.lru_cache else { return self.get_uncached_value(room).await; }; let guard = lru_cache.get_value_or_guard_async(room.room_id()).await; match guard { Ok(value) => { tracing::debug!("Returning existing cached display name.."); return Ok(value); } Err(guard) => { let value = self.get_uncached_value(room).await?; let _ = guard.insert(value.clone()); tracing::debug!("Returning now-cached display name"); return Ok(value); } } } async fn get_uncached_value(&self, room: &Room) -> mxlink::matrix_sdk::Result> { self.matrix_link .rooms() .own_display_name_in_room(room) .await } } ================================================ FILE: src/conversation/matrix/room_event_fetcher.rs ================================================ use mxlink::matrix_sdk::Room; use mxlink::matrix_sdk::deserialized_responses::TimelineEvent; use mxlink::matrix_sdk::ruma::OwnedEventId; use quick_cache::sync::Cache; pub struct RoomEventFetcher { lru_cache: Option>, } impl RoomEventFetcher { pub fn new(lru_cache_size: Option) -> Self { let lru_cache = lru_cache_size.map(Cache::new); Self { lru_cache } } #[tracing::instrument(skip(self), fields(room_id = room.room_id().as_str(), event_id = event_id.as_str()))] pub async fn fetch_event_in_room( &self, event_id: &OwnedEventId, room: &Room, ) -> mxlink::matrix_sdk::Result { let Some(lru_cache) = &self.lru_cache else { return room.event(event_id, None).await; }; let guard = lru_cache.get_value_or_guard_async(event_id).await; match guard { Ok(config) => { tracing::trace!("Returning existing cached event.."); return Ok(config); } Err(guard) => { let event = room.event(event_id, None).await?; let _ = guard.insert(event.clone()); tracing::trace!("Returning now-cached event"); return Ok(event); } } } } ================================================ FILE: src/conversation/matrix/utils/mod.rs ================================================ #[cfg(test)] mod tests; use std::sync::Arc; use mxlink::matrix_sdk::ruma::{OwnedEventId, OwnedUserId}; use mxlink::matrix_sdk::{ Room, deserialized_responses::TimelineEvent, ruma::events::{ AnyMessageLikeEventContent, AnySyncMessageLikeEvent, AnySyncTimelineEvent, SyncMessageLikeEvent, relation::Thread, room::message::{ MessageType, OriginalSyncRoomMessageEvent, Relation, RoomMessageEventContent, sanitize::remove_plain_reply_fallback, }, }, }; use mxlink::{MatrixLink, ThreadGetMessagesParams, ThreadInfo}; use tracing::Instrument; use super::{MatrixMessage, MatrixMessageContent, MatrixMessageProcessingParams, RoomEventFetcher}; use crate::entity::{InteractionContext, InteractionTrigger, MessagePayload}; use crate::utils::mime::get_mime_type_from_file_name; struct DetailedMessagePayload { is_mentioning_bot: bool, message_payload: MessagePayload, } pub async fn get_matrix_messages_in_thread( matrix_link: &MatrixLink, room: &Room, thread_id: OwnedEventId, ) -> Result, mxlink::matrix_sdk::Error> { let messages_native = matrix_link .threads() .get_messages(room, thread_id, ThreadGetMessagesParams::default()) .await?; let mut messages: Vec = Vec::new(); for matrix_native_message in messages_native { let message_result = convert_matrix_native_event_to_matrix_message(matrix_link, &matrix_native_message) .await?; if let Some(message) = message_result { messages.push(message); } } Ok(messages) } pub async fn get_matrix_messages_in_reply_chain( matrix_link: &MatrixLink, event_fetcher: &Arc, room: &Room, event_id: OwnedEventId, ) -> Result, mxlink::matrix_sdk::Error> { let messages_native = get_matrix_messages_in_reply_chain_native(event_fetcher, room, event_id).await?; let mut messages: Vec = Vec::new(); for matrix_native_message in messages_native { let message_result = convert_matrix_native_event_to_matrix_message(matrix_link, &matrix_native_message) .await?; if let Some(message) = message_result { messages.push(message); } } Ok(messages) } async fn get_matrix_messages_in_reply_chain_native( event_fetcher: &Arc, room: &Room, event_id: OwnedEventId, ) -> Result, mxlink::matrix_sdk::Error> { let mut next_event_id = Some(event_id.clone()); let mut messages: Vec = Vec::new(); let mut handled_event_ids: Vec = Vec::new(); while let Some(next_event_id_in_loop) = next_event_id { let event = event_fetcher .fetch_event_in_room(&next_event_id_in_loop, room) .await .unwrap(); if handled_event_ids.contains(&next_event_id_in_loop) { tracing::warn!( "Not following loop-causing event: {}", next_event_id_in_loop ); break; } handled_event_ids.push(next_event_id_in_loop.clone()); let event_deserialized = event.raw().deserialize()?; let AnySyncTimelineEvent::MessageLike(message_like_event) = event_deserialized else { tracing::warn!( "Not proceeding past non-MessageLike event: {:?}", event_deserialized ); break; }; next_event_id = match message_like_event.clone() { AnySyncMessageLikeEvent::RoomEncrypted(_) => None, AnySyncMessageLikeEvent::RoomMessage(room_message) => { if let SyncMessageLikeEvent::Original(room_message_original) = room_message { match room_message_original.content.relates_to { Some(Relation::Reply(reply)) => Some(reply.in_reply_to.event_id.clone()), _ => None, } } else { None } } _ => None, }; messages.push(message_like_event); } messages.reverse(); Ok(messages) } pub async fn process_matrix_messages( messages: &[MatrixMessage], params: &MatrixMessageProcessingParams, ) -> Vec { let mut messages_filtered: Vec = Vec::new(); for (i, message) in messages.iter().enumerate() { if !is_message_from_allowed_sender( message, ¶ms.bot_user_id, params.allowed_users.as_deref(), ) { continue; } let mut message = message.clone(); if i == 0 && !params.first_message_prefixes_to_strip.is_empty() && let MatrixMessageContent::Text(message_text) = &message.content { let mut message_text = message_text.clone(); for prefix in ¶ms.first_message_prefixes_to_strip { if let Some(message_text_stripped) = message_text.strip_prefix(prefix) { message_text = message_text_stripped.to_owned(); } } message.content = MatrixMessageContent::Text(message_text.trim().to_owned()); } // We only strip `bot_user_prefixes_to_strip`-defined prefixes from messages that mention the bot user. if !params.bot_user_prefixes_to_strip.is_empty() && message.mentioned_users.contains(¶ms.bot_user_id) && let MatrixMessageContent::Text(message_text) = &message.content { let mut message_text = message_text.clone(); for prefix in ¶ms.bot_user_prefixes_to_strip { if let Some(message_text_stripped) = message_text.strip_prefix(prefix) { message_text = message_text_stripped.to_owned(); } } message.content = MatrixMessageContent::Text(message_text.trim().to_owned()); } messages_filtered.push(message); } messages_filtered } /// Tells if the given message is from an allowed sender. /// /// If allowed_users is None, all messages are allowed. /// If allowed_users is Some, only messages from the allowed users (and the `bot_user_id`) are allowed. fn is_message_from_allowed_sender( matrix_message: &MatrixMessage, bot_user_id: &OwnedUserId, allowed_users: Option<&[regex::Regex]>, ) -> bool { if matrix_message.sender_id == *bot_user_id { return true; } if let Some(allowed_users) = allowed_users { if mxidwc::match_user_id(matrix_message.sender_id.as_str(), allowed_users) { return true; } } else { // No allowed users configured, so all messages are allowed return true; } false } pub async fn convert_matrix_native_event_to_matrix_message( matrix_link: &MatrixLink, matrix_native_event: &AnySyncMessageLikeEvent, ) -> Result, mxlink::matrix_sdk::Error> { let Some(content) = matrix_native_event.original_content() else { // Redacted message return Ok(None); }; let AnyMessageLikeEventContent::RoomMessage(room_message) = content else { // Some state event, etc. return Ok(None); }; let (text, is_notice) = match &room_message.msgtype { MessageType::Text(text_content) => (text_content.body.clone(), false), MessageType::Notice(notice_content) => (notice_content.body.clone(), true), MessageType::Image(image_content) => (image_content.body.clone(), false), MessageType::File(file_content) => (file_content.body.clone(), false), _ => return Ok(None), }; let is_reply = matches!(room_message.relates_to, Some(Relation::Reply { .. })); let text = if is_reply { // For regular replies, we need to strip the fallback-for-rich replies part. // See: https://spec.matrix.org/v1.11/client-server-api/#fallbacks-for-rich-replies remove_plain_reply_fallback(&text).to_owned() } else { text }; let timestamp = chrono::DateTime::::from( matrix_native_event .origin_server_ts() .to_system_time() .unwrap_or_else(std::time::SystemTime::now), ); let mentioned_users = room_message .mentions .map(|m| m.user_ids.iter().map(|u| u.to_owned()).collect()) .unwrap_or(vec![]); if let MessageType::Image(image_content) = &room_message.msgtype { let media_request = mxlink::matrix_sdk::media::MediaRequestParameters { source: image_content.source.to_owned(), format: mxlink::matrix_sdk::media::MediaFormat::File, }; let file_name = image_content .filename .clone() .unwrap_or(image_content.body.clone()); let mime_type = get_mime_type_from_file_name(&file_name); tracing::debug!("Determined mime type {} for file {}", mime_type, file_name); let span = tracing::debug_span!("get_media_content", file_name = %file_name, mime_type = %mime_type); let media_bytes = matrix_link .client() .media() .get_media_content(&media_request, true) .instrument(span) .await?; return Ok(Some(MatrixMessage { sender_id: matrix_native_event.sender().to_owned(), content: MatrixMessageContent::Image(image_content.clone(), mime_type, media_bytes), mentioned_users, timestamp, })); } if let MessageType::File(file_content) = &room_message.msgtype { let media_request = mxlink::matrix_sdk::media::MediaRequestParameters { source: file_content.source.to_owned(), format: mxlink::matrix_sdk::media::MediaFormat::File, }; let file_name = file_content .filename .clone() .unwrap_or(file_content.body.clone()); let mime_type = file_content .info .as_ref() .and_then(|info| info.mimetype.clone()) .and_then(|mimetype| mimetype.parse::().ok()) .unwrap_or_else(|| get_mime_type_from_file_name(&file_name)); tracing::debug!("Determined mime type {} for file {}", mime_type, file_name); if mime_type == mxlink::mime::APPLICATION_OCTET_STREAM { tracing::debug!( "Skipping file {} with unsupported MIME type {}. It will be represented as a text message.", file_name, mime_type, ); return Ok(Some(MatrixMessage { sender_id: matrix_native_event.sender().to_owned(), content: MatrixMessageContent::Text(format!( "[A file ({}) was attached but skipped because its content type ({}) is not supported. Let the user know.]", file_name, mime_type, )), mentioned_users, timestamp, })); } let span = tracing::debug_span!("get_media_content", file_name = %file_name, mime_type = %mime_type); let media_bytes = matrix_link .client() .media() .get_media_content(&media_request, true) .instrument(span) .await?; tracing::debug!( "Downloaded {} bytes for file {}", media_bytes.len(), file_name ); return Ok(Some(MatrixMessage { sender_id: matrix_native_event.sender().to_owned(), content: MatrixMessageContent::File(file_content.clone(), mime_type, media_bytes), mentioned_users, timestamp, })); } Ok(Some(MatrixMessage { sender_id: matrix_native_event.sender().to_owned(), content: if is_notice { MatrixMessageContent::Notice(text) } else { MatrixMessageContent::Text(text) }, mentioned_users, timestamp, })) } /// Determines the interaction context for an incoming (new) room event. /// /// This context is created based on the "newest message" (`current_event`), which is: /// - either a top-level message, which may or may not be mentioning the bot /// - this function will inspect the event and will likely start a new threaded conversation /// /// - or a thread reply /// - this function will inspect the thread root event and will return the interaction context /// - if the bot only reacts to prefixed messsages (or mentions), this function may ignore the given thread reply, unless it mentions the bot (which causes a synthetic "first message" to be produced) /// - if the thread root event is not found, is redacted, or is of some unsupported MessagePayload type, this function will return `None` /// /// - or an in-room (non-threaded) reply to a room message, which may or may not be mentioning the bot /// - replies that do not mention the bot cause this function to return `None` /// - other replies create a interaction context which points to a "first message" which is synthetic #[tracing::instrument(name = "determine_interaction_context_for_room_event", skip_all, fields(room_id = room.room_id().as_str(), event_id = current_event.event_id.as_str()))] pub async fn determine_interaction_context_for_room_event( bot_user_id: &OwnedUserId, bot_display_name: &Option, room: &Room, current_event: &OriginalSyncRoomMessageEvent, current_event_payload: &MessagePayload, event_fetcher: &Arc, ) -> anyhow::Result> { let current_event_is_mentioning_bot = is_event_mentioning_bot(¤t_event.content, bot_user_id, bot_display_name); let Some(relation) = ¤t_event.content.relates_to else { // This is a top-level message. We consider it the start of the thread. let thread_info = ThreadInfo::new( current_event.event_id.clone(), current_event.event_id.clone(), ); return Ok(Some(InteractionContext { thread_info, trigger: InteractionTrigger { is_mentioning_bot: current_event_is_mentioning_bot, payload: current_event_payload.clone(), }, })); }; match relation { Relation::Thread(thread) => { determine_interaction_context_for_room_event_related_to_thread( bot_user_id, bot_display_name, room, current_event, event_fetcher, current_event_is_mentioning_bot, thread, ) .await } Relation::Reply(reply) => { determine_interaction_context_for_room_event_related_to_reply( current_event, current_event_is_mentioning_bot, reply.in_reply_to.event_id.clone(), ) .await } // This is a replacement or something else. It's not something we support. _ => return Ok(None), } } async fn determine_interaction_context_for_room_event_related_to_thread( bot_user_id: &OwnedUserId, bot_display_name: &Option, room: &Room, current_event: &OriginalSyncRoomMessageEvent, event_fetcher: &Arc, current_event_is_mentioning_bot: bool, thread: &Thread, ) -> anyhow::Result> { let thread_info = ThreadInfo::new(thread.event_id.clone(), current_event.event_id.clone()); tracing::trace!( ?current_event_is_mentioning_bot, is_thread_root_only = thread_info.is_thread_root_only(), "Dealing with a thread reply", ); if current_event_is_mentioning_bot && !thread_info.is_thread_root_only() { // If the current event is a thread reply and is mentioning the bot, // it's probably someone trying to involve us in the threaded conversation. // See: https://github.com/etkecc/baibot/issues/15 // // In such cases, we don't care what the thread root event is like or what the current event is like, // we want text-generation to be triggered for this whole thread regardless. return Ok(Some(InteractionContext { thread_info, trigger: InteractionTrigger { is_mentioning_bot: true, payload: MessagePayload::SynthethicChatCompletionTriggerInThread, }, })); } let start_time = std::time::Instant::now(); let thread_start_timeline_event = event_fetcher .fetch_event_in_room(&thread.event_id, room) .await; let thread_start_timeline_event = match thread_start_timeline_event { Ok(value) => value, Err(err) => { return Err(anyhow::format_err!( "Failed to fetch thread start event {}: {:?}", thread.event_id, err )); } }; let duration = start_time.elapsed(); tracing::trace!( thread_id = thread.event_id.as_str(), duration = ?duration, "Fetched thread start event" ); let thread_start_detailed_message_payload = timeline_event_to_detailed_message_payload( &thread.event_id, thread_start_timeline_event, thread_info.clone(), bot_user_id, bot_display_name, )?; let Some(detailed_message_payload) = thread_start_detailed_message_payload else { return Ok(None); }; Ok(Some(InteractionContext { thread_info, trigger: InteractionTrigger { is_mentioning_bot: detailed_message_payload.is_mentioning_bot, payload: detailed_message_payload.message_payload, }, })) } async fn determine_interaction_context_for_room_event_related_to_reply( current_event: &OriginalSyncRoomMessageEvent, current_event_is_mentioning_bot: bool, reply_to_event_id: OwnedEventId, ) -> anyhow::Result> { tracing::trace!(?current_event_is_mentioning_bot, "Dealing with a reply"); if !current_event_is_mentioning_bot { // If the current event is not mentioning the bot, we don't care about it. tracing::trace!("Ignoring reply event which does not mention the bot"); return Ok(None); } let thread_info = ThreadInfo::new(reply_to_event_id.clone(), current_event.event_id.clone()); Ok(Some(InteractionContext { thread_info, trigger: InteractionTrigger { is_mentioning_bot: true, payload: MessagePayload::SynthethicChatCompletionTriggerForReply, }, })) } fn is_event_mentioning_bot( event_content: &RoomMessageEventContent, bot_user_id: &OwnedUserId, bot_display_name: &Option, ) -> bool { if let Some(mentions) = &event_content.mentions { mentions .user_ids .iter() .any(|user_id| user_id == bot_user_id) } else { // For compatibility with clients that do not support the new Mentions specification // (see https://spec.matrix.org/latest/client-server-api/#user-and-room-mentions), // we also do string matching here. // // As of 2024-10-03, at least Element iOS does not support the new Mentions specification // and is still quite widespread. // // We may consider dropping this string-matching behavior altogether in the future, // so improving this compatibility block is not a high priority. if event_content.body().contains(bot_user_id.as_str()) { return true; } if let Some(bot_display_name) = bot_display_name { return event_content.body().contains(bot_display_name); } false } } fn timeline_event_to_detailed_message_payload( timeline_event_id: &OwnedEventId, timeline_event: TimelineEvent, thread_info: ThreadInfo, bot_user_id: &OwnedUserId, bot_display_name: &Option, ) -> anyhow::Result> { let timeline_event_deserialized = match timeline_event.raw().deserialize() { Ok(value) => value, Err(err) => { return Err(anyhow::format_err!( "Failed to deserialize timeline event {}: {:?}", timeline_event_id, err )); } }; let AnySyncTimelineEvent::MessageLike(thread_start_message_like_event) = timeline_event_deserialized else { tracing::trace!( "Ignoring non-MessageLike timeline event: {:?}", timeline_event_deserialized ); return Ok(None); }; let (is_mentioning_bot, message_payload) = match thread_start_message_like_event { AnySyncMessageLikeEvent::RoomEncrypted(room_message) => { tracing::warn!( "Could not inspect event {} because it failed to decrypt: {:?}", timeline_event_id.clone(), room_message ); // There's no way to know and it doesn't matter anyway. let is_mentioning_bot = false; ( is_mentioning_bot, MessagePayload::Encrypted(thread_info.clone()), ) } AnySyncMessageLikeEvent::RoomMessage(room_message) => { if let SyncMessageLikeEvent::Original(room_message_original) = room_message { let room_message_payload: Result = room_message_original.content.msgtype.clone().try_into(); let Ok(room_message_payload) = room_message_payload else { tracing::debug!( msg_type = room_message_original.content.msgtype(), "Ignoring event message of unknown type", ); return Ok(None); }; let is_mentioning_bot = is_event_mentioning_bot( &room_message_original.content, bot_user_id, bot_display_name, ); (is_mentioning_bot, room_message_payload) } else { tracing::error!("Ignoring event message which appears to be redacted"); return Ok(None); } } other => { tracing::trace!("Ignoring unknown MessageLike event: {:?}", other); return Ok(None); } }; Ok(Some(DetailedMessagePayload { is_mentioning_bot, message_payload, })) } /// Creates a list of prefixes to strip from the beginning of message texts that mention the bot user. /// /// Different clients do mentions differently. /// The body text containing the mention usually contains one of: /// - the full user ID (includes a @ prefix by default) /// - the localpart (with a @ prefix) /// - the localpart (without a @ prefix) /// - the display name (with a @ prefix) /// - the display name (without a @ prefix) /// /// Some add a `: ` suffix after the mention. /// /// There's no guarantee that the mention is at the start even. /// It being there is most common and we try to strip it from there /// as best as we can. pub fn create_list_of_bot_user_prefixes_to_strip( bot_user_id: &OwnedUserId, bot_display_name: &Option, ) -> Vec { let bot_user_id_localpart = bot_user_id.localpart(); let mut prefixes_to_strip = vec![ bot_user_id.as_str().to_owned(), format!("@{}", bot_user_id_localpart), bot_user_id_localpart.to_owned(), ]; if let Some(bot_display_name) = bot_display_name { prefixes_to_strip.push(format!("@{}", bot_display_name)); prefixes_to_strip.push(bot_display_name.to_owned()); } prefixes_to_strip.push(":".to_owned()); prefixes_to_strip } ================================================ FILE: src/conversation/matrix/utils/tests.rs ================================================ use chrono::{TimeZone, Utc}; use mxlink::matrix_sdk::ruma::OwnedUserId; use crate::conversation::matrix::{ MatrixMessage, MatrixMessageContent, MatrixMessageProcessingParams, }; #[test] fn is_message_from_allowed_sender() { let bot_user_id = OwnedUserId::try_from("@bot:example.com").expect("Failed to parse bot user ID"); let allowed_user_id = OwnedUserId::try_from("@user.someone:example.com").unwrap(); let unallowed_user_id = OwnedUserId::try_from("@another:example.com").unwrap(); let timestamp = Utc.with_ymd_and_hms(2024, 9, 20, 18, 34, 15).unwrap(); let bot_message = MatrixMessage { sender_id: bot_user_id.to_owned(), content: MatrixMessageContent::Text("Hello!".to_owned()), mentioned_users: vec![], timestamp, }; let allowed_user_message = MatrixMessage { sender_id: allowed_user_id.to_owned(), content: MatrixMessageContent::Text("Hello!".to_owned()), mentioned_users: vec![], timestamp, }; let unallowed_user_message = MatrixMessage { sender_id: unallowed_user_id.to_owned(), content: MatrixMessageContent::Text("Hello!".to_owned()), mentioned_users: vec![], timestamp, }; let parsed_regex = match mxidwc::parse_pattern("@user.*:example.com") { Ok(value) => value, Err(err) => { panic!("Error parsing regex: {}", err); } }; let allowed_users = vec![parsed_regex]; assert!( super::is_message_from_allowed_sender(&bot_message, &bot_user_id, Some(&allowed_users)), "Bot message should be allowed" ); assert!( super::is_message_from_allowed_sender( &allowed_user_message, &bot_user_id, Some(&allowed_users) ), "Allowed user message should be allowed" ); assert!( !super::is_message_from_allowed_sender( &unallowed_user_message, &bot_user_id, Some(&allowed_users), ), "Unallowed user message should be ignored" ); assert!( super::is_message_from_allowed_sender(&unallowed_user_message, &bot_user_id, None,), "An empty list of allowed users lets everyone through" ); } #[tokio::test] async fn process_matrix_messages() { let bot_user_id = OwnedUserId::try_from("@bot:example.com").expect("Failed to parse bot user ID"); let allowed_user_id = OwnedUserId::try_from("@user.someone:example.com").unwrap(); let unallowed_user_id = OwnedUserId::try_from("@another:example.com").unwrap(); let timestamp = Utc.with_ymd_and_hms(2024, 9, 20, 18, 34, 15).unwrap(); let allowed_user_message = MatrixMessage { sender_id: allowed_user_id.to_owned(), content: MatrixMessageContent::Text("Hello from the user!".to_owned()), mentioned_users: vec![], timestamp, }; let allowed_user_message_with_prefix = MatrixMessage { sender_id: allowed_user_id.to_owned(), content: MatrixMessageContent::Text("!bai Hello from the user!".to_owned()), mentioned_users: vec![], timestamp, }; let allowed_user_message_with_prefix_no_space = MatrixMessage { sender_id: allowed_user_id.to_owned(), content: MatrixMessageContent::Text("!baiHello from the user!".to_owned()), mentioned_users: vec![], timestamp, }; let allowed_user_message_with_prefix_full_width_space = MatrixMessage { sender_id: allowed_user_id.to_owned(), content: MatrixMessageContent::Text("!bai　Hello from the user!".to_owned()), mentioned_users: vec![], timestamp, }; let bot_message = MatrixMessage { sender_id: bot_user_id.to_owned(), content: MatrixMessageContent::Text("Hello from the bot!".to_owned()), mentioned_users: vec![], timestamp, }; let allowed_user_message_with_bot_mention = MatrixMessage { sender_id: allowed_user_id.to_owned(), content: MatrixMessageContent::Text("@baibot: Hello from the user!".to_owned()), mentioned_users: vec![bot_user_id.to_owned()], timestamp, }; // The message text is the same as above - it mentions the bot, but the actually-mentioned user is another user. let allowed_user_message_with_another_user_mention = MatrixMessage { sender_id: allowed_user_id.to_owned(), content: allowed_user_message_with_bot_mention.content.clone(), mentioned_users: vec![allowed_user_id.to_owned()], timestamp, }; let unallowed_user_message = MatrixMessage { sender_id: unallowed_user_id.to_owned(), content: MatrixMessageContent::Text("Hello from an unallowed user!".to_owned()), mentioned_users: vec![], timestamp, }; let parsed_regex = match mxidwc::parse_pattern("@user.*:example.com") { Ok(value) => value, Err(err) => { panic!("Error parsing regex: {}", err); } }; let allowed_users = vec![parsed_regex]; let message_processing_params_basic = super::MatrixMessageProcessingParams::new( bot_user_id.to_owned(), Some(allowed_users.clone()), ); let message_processing_params_with_prefix_stripping = super::MatrixMessageProcessingParams::new( bot_user_id.to_owned(), Some(allowed_users.clone()), ) .with_first_message_prefixes_to_strip(vec!["!bai".to_owned()]); let message_processing_params_with_bot_user_prefix_stripping = super::MatrixMessageProcessingParams::new( bot_user_id.to_owned(), Some(allowed_users.clone()), ) .with_bot_user_prefixes_to_strip(vec!["@baibot: ".to_owned(), "@baibot".to_owned()]); struct TestCase { name: String, messages: Vec, message_processing_params: MatrixMessageProcessingParams, expected_message_texts: Vec, } let test_cases = vec![ TestCase { name: "Messages by unallowed users are ignored".to_owned(), messages: vec![ allowed_user_message.clone(), bot_message.clone(), unallowed_user_message.clone(), ], message_processing_params: message_processing_params_basic.clone(), expected_message_texts: vec![ "Hello from the user!".to_owned(), "Hello from the bot!".to_owned(), ], }, TestCase { name: "The first message with a prefix gets stripped if params configure it (regular space)".to_owned(), messages: vec![ allowed_user_message_with_prefix.clone(), bot_message.clone(), allowed_user_message_with_prefix.clone(), unallowed_user_message.clone(), ], message_processing_params: message_processing_params_with_prefix_stripping.clone(), expected_message_texts: vec![ "Hello from the user!".to_owned(), "Hello from the bot!".to_owned(), "!bai Hello from the user!".to_owned(), ], }, TestCase { name: "The first message with a prefix gets stripped if params configure it (no space)".to_owned(), messages: vec![ allowed_user_message_with_prefix_no_space.clone(), bot_message.clone(), allowed_user_message_with_prefix_no_space.clone(), unallowed_user_message.clone(), ], message_processing_params: message_processing_params_with_prefix_stripping.clone(), expected_message_texts: vec![ "Hello from the user!".to_owned(), "Hello from the bot!".to_owned(), "!baiHello from the user!".to_owned(), ], }, TestCase { name: "The first message with a prefix gets stripped if params configure it (full-width-space)".to_owned(), messages: vec![ allowed_user_message_with_prefix_full_width_space.clone(), bot_message.clone(), allowed_user_message_with_prefix_full_width_space.clone(), unallowed_user_message.clone(), ], message_processing_params: message_processing_params_with_prefix_stripping.clone(), expected_message_texts: vec![ "Hello from the user!".to_owned(), "Hello from the bot!".to_owned(), "!bai　Hello from the user!".to_owned(), ], }, TestCase { name: "The first message with a prefix remains untouched if params leave it alone" .to_owned(), messages: vec![ allowed_user_message_with_prefix.clone(), bot_message.clone(), allowed_user_message_with_prefix.clone(), unallowed_user_message.clone(), ], message_processing_params: message_processing_params_basic.clone(), expected_message_texts: vec![ "!bai Hello from the user!".to_owned(), "Hello from the bot!".to_owned(), "!bai Hello from the user!".to_owned(), ], }, TestCase { name: "Messages that mention the bot user get the bot user prefix stripped" .to_owned(), messages: vec![ allowed_user_message_with_bot_mention.clone(), allowed_user_message_with_another_user_mention.clone(), ], message_processing_params: message_processing_params_with_bot_user_prefix_stripping.clone(), expected_message_texts: vec![ "Hello from the user!".to_owned(), "@baibot: Hello from the user!".to_owned(), ], }, ]; for test_case in test_cases { let processed_messages = super::process_matrix_messages( &test_case.messages, &test_case.message_processing_params, ) .await; let processed_message_texts = processed_messages .iter() .map(|message| match &message.content { MatrixMessageContent::Text(text) => text.clone(), _ => "".to_owned(), }) .collect::>(); assert_eq!( processed_message_texts, test_case.expected_message_texts, "Test case {} failed", test_case.name, ); } } #[test] fn create_list_of_bot_user_prefixes_to_strip() { let bot_user_id = OwnedUserId::try_from("@baibot:example.com").expect("Failed to parse bot user ID"); // Test case 1: Bot user with no display name let bot_display_name = None; let prefixes = super::create_list_of_bot_user_prefixes_to_strip(&bot_user_id, &bot_display_name); assert_eq!( prefixes, vec![ "@baibot:example.com".to_string(), "@baibot".to_string(), "baibot".to_string(), ":".to_string() ] ); // Test case 2: Bot user with display name let bot_display_name = Some("Assistant".to_string()); let prefixes = super::create_list_of_bot_user_prefixes_to_strip(&bot_user_id, &bot_display_name); assert_eq!( prefixes, vec![ "@baibot:example.com".to_string(), "@baibot".to_string(), "baibot".to_string(), "@Assistant".to_string(), "Assistant".to_string(), ":".to_string() ] ); } ================================================ FILE: src/conversation/matrix_llm_bridge.rs ================================================ use std::sync::Arc; use mxlink::MatrixLink; use mxlink::matrix_sdk::ruma::OwnedEventId; use crate::conversation::matrix::MatrixMessage; use super::llm::{Conversation, Message, convert_matrix_message_to_llm_message}; use super::matrix::{ MatrixMessageProcessingParams, RoomEventFetcher, get_matrix_messages_in_reply_chain, get_matrix_messages_in_thread, process_matrix_messages, }; pub async fn create_llm_conversation_for_matrix_thread( matrix_link: &MatrixLink, room: &mxlink::matrix_sdk::Room, thread_id: OwnedEventId, params: &MatrixMessageProcessingParams, ) -> Result { let messages = get_matrix_messages_in_thread(matrix_link, room, thread_id).await?; let llm_messages = filter_messages_and_convert_to_llm_messages(messages, params).await; Ok(Conversation { messages: llm_messages, }) } pub async fn create_llm_conversation_for_matrix_reply_chain( matrix_link: &MatrixLink, event_fetcher: &Arc, room: &mxlink::matrix_sdk::Room, event_id: OwnedEventId, params: &MatrixMessageProcessingParams, ) -> Result { let messages = get_matrix_messages_in_reply_chain(matrix_link, event_fetcher, room, event_id).await?; let llm_messages = filter_messages_and_convert_to_llm_messages(messages, params).await; Ok(Conversation { messages: llm_messages, }) } async fn filter_messages_and_convert_to_llm_messages( messages: Vec, params: &MatrixMessageProcessingParams, ) -> Vec { let messages_filtered = process_matrix_messages(&messages, params).await; let mut llm_messages: Vec = Vec::new(); for matrix_message in messages_filtered { let Some(llm_message) = convert_matrix_message_to_llm_message(&matrix_message, ¶ms.bot_user_id) else { continue; }; llm_messages.push(llm_message); } llm_messages } ================================================ FILE: src/conversation/mod.rs ================================================ pub(crate) mod llm; pub(crate) mod matrix; mod matrix_llm_bridge; pub(crate) use matrix_llm_bridge::{ create_llm_conversation_for_matrix_reply_chain, create_llm_conversation_for_matrix_thread, }; ================================================ FILE: src/entity/catch_up_marker/delayed_catch_up_marker_manager.rs ================================================ use std::sync::Arc; use tokio::sync::Mutex; use tokio::time::Duration; use mxlink::helpers::account_data_config::ConfigError; use super::CatchUpMarkerManager; /// A service that records roughly until when we're caught up on processing events. /// Roughly, because we account for potential federation delay and we don't persist the marker too often. /// /// If the matrix-sdk's state-store is kept intact, we (usually) won't be given the same event twice. /// In such a happy path, we don't need to keep track of anything and there's no problem. /// /// If the state-store is lost (a very rare, but possible event), we can recover our encryption keys, etc., /// but the Matrix SDK would try to feed us the same events again. /// Responding to many old events again is annoying to users and can be a huge waste of resources. /// /// In order to handle state-store-loss better, we need to record until when we're caught up in storage that won't get lost (such as account data for the user). /// Because state-store-loss is a very rare event, we don't need to be very exact about the specific timestamp we're caught up to. /// In fact, being behind is necessary, to allow for federation delay (see `federation_delay_tolerance_duration`). pub struct DelayedCatchUpMarkerManager { catch_up_marker_manager: Arc>, /// `persist_interval_duration` affects how often we persist the catch-up marker to Account Data /// A too small value means there's needless overhead. /// The downside to a larger interval value (and a larger federation delay tolerance value) is that that a state-store loss will mean that /// we will reprocess some of the same events. /// Since this is a very rare event and the downside is not so bad, a large value is recommended. persist_interval_duration: Duration, /// `federation_delay_tolerance_duration` affects what federation delay we will tolerate. /// A larger delay than this may mean we ignore events that are actually new to us. /// This is necessary because the timestamp given to us (see `catch_up()`) is based on the "origin server" timestamp. /// If federation is slow, we may actually receive old events later on - they'd still be new to us, /// but we may ignore them if we've marked this "origin server timestamp" value as "caught up". federation_delay_tolerance_duration: Duration, /// Holds the timestamp to use for updating the catch-up marker's `caught_up_until_event_origin_server_ts_millis`. /// A value of `0` is used to indicate that no update is scheduled and the next iteration should skip updating the marker. next_catch_up_marker_event_origin_server_ts_millis: Arc>, } impl DelayedCatchUpMarkerManager { pub fn new( catch_up_marker_manager: CatchUpMarkerManager, persist_interval_duration: Duration, federation_delay_tolerance_duration: Duration, ) -> Self { let next_catch_up_marker_event_origin_server_ts_millis = Arc::new(tokio::sync::Mutex::new(0)); let catch_up_marker_manager = Arc::new(Mutex::new(catch_up_marker_manager)); Self { catch_up_marker_manager, persist_interval_duration, federation_delay_tolerance_duration, next_catch_up_marker_event_origin_server_ts_millis, } } #[tracing::instrument(name = "catch_up", skip(self))] pub async fn catch_up(&self, event_origin_server_ts_millis: i64) { tracing::trace!("Locking to catch-up.."); let mut next_catch_up_marker_event_origin_server_ts_millis_guard = self .next_catch_up_marker_event_origin_server_ts_millis .lock() .await; if *next_catch_up_marker_event_origin_server_ts_millis_guard > event_origin_server_ts_millis { tracing::trace!( ?next_catch_up_marker_event_origin_server_ts_millis_guard, "Already have a more recent timestamp scheduled", ); return; } *next_catch_up_marker_event_origin_server_ts_millis_guard = event_origin_server_ts_millis; tracing::info!("Configured catch-up timestamp for the next update"); } /// Tells if we're caught up until the given timestamp. /// /// This intentionally uses the latest (cached) data stored in catch_up_marker_manager (Account Data), not the `next_catch_up_marker_event_origin_server_ts_millis` value. /// `next_catch_up_marker_event_origin_server_ts_millis` is used for scheduling the next update only. /// The actual timestamp that will get persisted durign the update will actually be adjusted by `federation_delay_tolerance_duration`, /// so comparing against `next_catch_up_marker_event_origin_server_ts_millis` in its raw form would be incorrect. #[tracing::instrument(name = "is_caught_up", skip(self))] pub(crate) async fn is_caught_up( &self, event_origin_ts_millis: i64, ) -> Result { tracing::trace!("Locking to check if caught up.."); let mut manager = self.catch_up_marker_manager.lock().await; let marker = manager.get_or_create().await?; let is_caught_up = marker.caught_up_until_event_origin_server_ts_millis >= event_origin_ts_millis; tracing::debug!( ?is_caught_up, ?marker.caught_up_until_event_origin_server_ts_millis, "Determined caught-up status" ); Ok(is_caught_up) } pub async fn start(&self) { let inner = Arc::clone(&self.catch_up_marker_manager); let persist_interval_duration = self.persist_interval_duration; let federation_delay_tolerance = self.federation_delay_tolerance_duration; let next_catch_up_marker_event_origin_server_ts_millis = Arc::clone(&self.next_catch_up_marker_event_origin_server_ts_millis); tokio::spawn(async move { let mut interval = tokio::time::interval(persist_interval_duration); loop { interval.tick().await; tracing::trace!("Catch-up manager doing work.."); let mut next_catch_up_marker_event_origin_server_ts_millis_guard = next_catch_up_marker_event_origin_server_ts_millis .lock() .await; if *next_catch_up_marker_event_origin_server_ts_millis_guard == 0 { tracing::trace!("No scheduled updates to the catch-up marker"); continue; } let mut manager = inner.lock().await; let marker = manager.get_or_create().await; let mut marker = match marker { Ok(marker) => marker, Err(err) => { tracing::error!(?err, "Failed to get or create catch-up marker"); continue; } }; // To allow for some federation delay (specified in federation_delay_tolerance), // we adjust the value we'll actually persist with that delay duration. // For more information, see the documentation for `Self`. let caught_up_until_event_origin_server_ts_millis = *next_catch_up_marker_event_origin_server_ts_millis_guard - (federation_delay_tolerance.as_millis() as i64); marker.caught_up_until_event_origin_server_ts_millis = caught_up_until_event_origin_server_ts_millis; tracing::debug!( ?caught_up_until_event_origin_server_ts_millis, next_catch_up_marker_event_origin_server_ts_millis = format!( "{:?}", next_catch_up_marker_event_origin_server_ts_millis_guard ), "Updating catch-up marker..", ); let result = manager.persist(&marker).await; if let Err(err) = result { tracing::error!(?err, "Failed to persist catch-up marker"); } *next_catch_up_marker_event_origin_server_ts_millis_guard = 0; } }); } } ================================================ FILE: src/entity/catch_up_marker/entity.rs ================================================ use mxlink::matrix_sdk::ruma::events::macros::EventContent; use serde::{Deserialize, Serialize}; use mxlink::helpers::account_data_config::GlobalConfig; use mxlink::helpers::account_data_config::GlobalConfigCarrierContent; #[derive(Clone, Debug, Default, Deserialize, Serialize, EventContent)] #[ruma_event(type = "cc.etke.baibot.catch_up_marker", kind = GlobalAccountData)] pub struct CatchUpMarkerCarrierContent { pub payload: String, } impl GlobalConfigCarrierContent for CatchUpMarkerCarrierContent { fn payload(&self) -> &str { &self.payload } fn new(payload: String) -> Self { Self { payload } } } #[derive(Clone, Debug, Default, Deserialize, Serialize)] pub struct CatchUpMarker { pub caught_up_until_event_origin_server_ts_millis: i64, } impl CatchUpMarker { pub fn new(caught_up_until_event_origin_server_ts_millis: i64) -> Self { Self { caught_up_until_event_origin_server_ts_millis, } } } impl GlobalConfig for CatchUpMarker {} ================================================ FILE: src/entity/catch_up_marker/mod.rs ================================================ mod delayed_catch_up_marker_manager; mod entity; use mxlink::helpers::account_data_config::GlobalConfigManager as AccountDataGlobalConfigManager; pub use entity::{CatchUpMarker, CatchUpMarkerCarrierContent}; pub type CatchUpMarkerManager = AccountDataGlobalConfigManager; pub use delayed_catch_up_marker_manager::DelayedCatchUpMarkerManager; ================================================ FILE: src/entity/cfg/config.rs ================================================ use std::path::PathBuf; use mxlink::helpers::encryption::EncryptionKey; use mxlink::matrix_sdk::ruma::{OwnedDeviceId, OwnedUserId}; use serde::{Deserialize, Deserializer, Serialize}; use crate::{ agent::{AgentDefinition, AgentPurpose, PublicIdentifier}, entity::{globalconfig::GlobalConfig, roomconfig::RoomSettingsHandler}, }; #[derive(Debug, Deserialize)] pub struct Config { pub homeserver: ConfigHomeserver, pub user: ConfigUser, pub persistence: PersistenceConfig, #[serde(default = "super::defaults::command_prefix")] pub command_prefix: String, #[serde(default)] pub room: ConfigRoom, pub access: ConfigAccess, pub agents: ConfigAgents, // Contains the initial global configuration values. // Not all properties of the object make sense to be configured statically, // so not all of them will be reflected onto the actual global configuration. pub initial_global_config: ConfigInitialGlobalConfig, #[serde(default = "super::defaults::logging")] pub logging: String, } impl Config { pub fn validate(&self) -> anyhow::Result<()> { self.homeserver.validate()?; self.user.validate(&self.homeserver.server_name)?; self.persistence.validate()?; self.room.validate()?; self.access.validate()?; if self.command_prefix.is_empty() { return Err(anyhow::anyhow!( "The command_prefix ({}) configuration must be set", super::env::BAIBOT_COMMAND_PREFIX )); } self.agents.validate()?; self.initial_global_config.clone().validate()?; Ok(()) } } #[derive(Debug)] pub enum ConfigUserAuth { UserPassword { username: String, password: String, }, AccessToken { user_id: OwnedUserId, device_id: OwnedDeviceId, access_token: String, }, } #[derive(Debug, Serialize, Deserialize)] pub struct ConfigHomeserver { pub server_name: String, pub url: String, } impl ConfigHomeserver { pub fn validate(&self) -> anyhow::Result<()> { if self.server_name.is_empty() { return Err(anyhow::anyhow!( "The homeserver.server_name ({}) configuration must be set", super::env::BAIBOT_HOMESERVER_SERVER_NAME )); } if self.url.is_empty() { return Err(anyhow::anyhow!( "The homeserver.url ({}) configuration must be set", super::env::BAIBOT_HOMESERVER_URL )); } Ok(()) } } /// Configuration for the bot's avatar. /// /// - `Default`: Use the built-in default avatar (null, empty string, or missing in config) /// - `Keep`: Don't touch the avatar, keep whatever is already set ("keep" in config) /// - `Custom(String)`: Use a custom avatar from the specified file path #[derive(Debug, Clone, Default, PartialEq, Serialize)] pub enum Avatar { /// Use the built-in default avatar #[default] Default, /// Keep the current avatar, don't change it Keep, /// Use a custom avatar from the specified file path Custom(String), } impl<'de> Deserialize<'de> for Avatar { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, { let value: Option = Option::deserialize(deserializer)?; Ok(match value { None => Avatar::Default, Some(s) => Avatar::from_string(s), }) } } impl Avatar { pub fn from_string(value: String) -> Self { if value.is_empty() { Avatar::Default } else if value.eq_ignore_ascii_case("keep") { Avatar::Keep } else { Avatar::Custom(value) } } } #[derive(Debug, Serialize, Deserialize)] pub struct ConfigUser { pub mxid_localpart: String, #[serde(default)] pub password: Option, #[serde(default)] pub access_token: Option, #[serde(default)] pub device_id: Option, #[serde(default = "super::defaults::name")] pub name: String, #[serde(default)] pub encryption: ConfigUserEncryption, #[serde(default)] pub avatar: Avatar, } impl ConfigUser { pub fn validate(&self, homeserver_server_name: &str) -> anyhow::Result<()> { if self.mxid_localpart.is_empty() { return Err(anyhow::anyhow!( "The user.mxid_localpart ({}) configuration must be set", super::env::BAIBOT_USER_MXID_LOCALPART )); } self.auth_config(homeserver_server_name)?; if self.name.is_empty() { return Err(anyhow::anyhow!( "The name ({}) configuration must be set", super::env::BAIBOT_USER_NAME )); } self.encryption.validate()?; Ok(()) } pub fn auth_config(&self, homeserver_server_name: &str) -> anyhow::Result { let password = self.password.as_deref().filter(|value| !value.is_empty()); let access_token = self .access_token .as_deref() .filter(|value| !value.is_empty()); match (password, access_token) { (Some(_), Some(_)) => Err(anyhow::anyhow!( "Set exactly one authentication method: either user.password ({}) OR user.access_token ({}) + user.device_id ({})", super::env::BAIBOT_USER_PASSWORD, super::env::BAIBOT_USER_ACCESS_TOKEN, super::env::BAIBOT_USER_DEVICE_ID )), (None, None) => Err(anyhow::anyhow!( "Set one authentication method: either user.password ({}) OR user.access_token ({}) + user.device_id ({})", super::env::BAIBOT_USER_PASSWORD, super::env::BAIBOT_USER_ACCESS_TOKEN, super::env::BAIBOT_USER_DEVICE_ID )), (Some(password), None) => Ok(ConfigUserAuth::UserPassword { username: self.mxid_localpart.to_owned(), password: password.to_owned(), }), (None, Some(access_token)) => { let device_id = self .device_id .as_deref() .filter(|value| !value.is_empty()) .ok_or_else(|| { anyhow::anyhow!( "user.device_id ({}) must be set when using access token authentication", super::env::BAIBOT_USER_DEVICE_ID ) })?; let user_id = OwnedUserId::try_from(format!( "@{}:{}", self.mxid_localpart, homeserver_server_name )) .map_err(|e| anyhow::anyhow!("Invalid user ID: {e}"))?; Ok(ConfigUserAuth::AccessToken { user_id, device_id: OwnedDeviceId::from(device_id), access_token: access_token.to_owned(), }) } } } } #[derive(Debug, Default, Serialize, Deserialize)] pub struct ConfigUserEncryption { pub recovery_passphrase: Option, pub recovery_reset_allowed: bool, } impl ConfigUserEncryption { pub fn validate(&self) -> anyhow::Result<()> { if let Some(passphrase) = &self.recovery_passphrase && passphrase.is_empty() { return Err(anyhow::anyhow!( "The user.encryption.recovery_passphrase ({}) configuration must either be null or set to a non-empty passphrase", super::env::BAIBOT_USER_ENCRYPTION_RECOVERY_PASSPHRASE )); } Ok(()) } } #[derive(Debug, Serialize, Deserialize)] pub struct PersistenceConfig { #[serde(default = "super::defaults::persistence_data_dir_path")] pub data_dir_path: Option, #[serde(default = "super::defaults::persistence_session_file_name")] session_file_name: String, #[serde(default = "super::defaults::persistence_db_dir_name")] db_dir_name: String, pub session_encryption_key: Option, pub config_encryption_key: Option, } impl PersistenceConfig { pub fn validate(&self) -> anyhow::Result<()> { if let Some(data_dir_path) = &self.data_dir_path { let path = PathBuf::from(data_dir_path); if !path.exists() { return Err(anyhow::anyhow!( "The persistence.data_dir_path ({}) directory ({}) must exist", super::env::BAIBOT_PERSISTENCE_DATA_DIR_PATH, data_dir_path, )); } } self.config_encryption_key() .map_err(|e| anyhow::anyhow!(e))?; Ok(()) } pub fn session_file_path(&self) -> anyhow::Result { let Some(data_dir_path) = &self.data_dir_path else { return Err(anyhow::anyhow!( "The persistence.data_dir_path ({}) directory must be set", super::env::BAIBOT_PERSISTENCE_DATA_DIR_PATH )); }; let mut path = PathBuf::from(data_dir_path); path.push(&self.session_file_name); Ok(path) } pub fn db_dir_path(&self) -> anyhow::Result { let Some(data_dir_path) = &self.data_dir_path else { return Err(anyhow::anyhow!( "The persistence.data_dir_path ({}) directory must be set", super::env::BAIBOT_PERSISTENCE_DATA_DIR_PATH )); }; let mut path = PathBuf::from(data_dir_path); path.push(&self.db_dir_name); Ok(path) } pub fn session_encryption_key(&self) -> anyhow::Result> { self.parse_encryption_key(&self.session_encryption_key).map_err(|err| { anyhow::anyhow!( "Encryption key specified in persistence.session_encryption_key ({}) is not valid: {}", super::env::BAIBOT_PERSISTENCE_SESSION_ENCRYPTION_KEY, err ) }) } pub fn config_encryption_key(&self) -> anyhow::Result> { self.parse_encryption_key(&self.config_encryption_key).map_err(|err| { anyhow::anyhow!( "Encryption key specified in persistence.config_encryption_key ({}) is not valid: {}", super::env::BAIBOT_PERSISTENCE_CONFIG_ENCRYPTION_KEY, err ) }) } fn parse_encryption_key( &self, value: &Option, ) -> anyhow::Result, String> { let key = match value { Some(key) => { if key.is_empty() { None } else { Some(EncryptionKey::from_hex_str(key)?) } } None => None, }; Ok(key) } } #[derive(Debug, Serialize, Deserialize)] pub struct ConfigRoom { #[serde(default = "super::defaults::room_post_join_self_introduction_enabled")] pub post_join_self_introduction_enabled: bool, } impl ConfigRoom { pub fn validate(&self) -> anyhow::Result<()> { Ok(()) } } impl Default for ConfigRoom { fn default() -> Self { Self { post_join_self_introduction_enabled: super::defaults::room_post_join_self_introduction_enabled(), } } } #[derive(Debug, Serialize, Deserialize)] pub struct ConfigAccess { // Contains the admin whitelist patterns before parsing into regex. // Example: `["@*:example.com"]` pub admin_patterns: Vec, } impl ConfigAccess { // Returns the the mxidwc-parsed regexes for the admin whitelist. // Example: `["^@\.*:example\.com$"]` pub fn admin_pattern_regexes(&self) -> anyhow::Result> { mxidwc::parse_patterns_vector(&self.admin_patterns).map_err(|e| { anyhow::anyhow!( "Failed parsing access.admin_patterns ({}): {:?}", super::env::BAIBOT_ACCESS_ADMIN_PATTERNS, e ) }) } pub fn validate(&self) -> anyhow::Result<()> { if self.admin_patterns.is_empty() { return Err(anyhow::anyhow!( "The access.admin_patterns ({}) configuration must contain at least one pattern", super::env::BAIBOT_ACCESS_ADMIN_PATTERNS )); } self.admin_pattern_regexes()?; Ok(()) } } #[derive(Debug, Serialize, Deserialize)] pub struct ConfigAgents { pub static_definitions: Vec, } impl ConfigAgents { pub fn validate(&self) -> anyhow::Result<()> { Ok(()) } } #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ConfigInitialGlobalConfig { #[serde(default)] pub handler: RoomSettingsHandler, pub user_patterns: Option>, } impl ConfigInitialGlobalConfig { fn user_pattern_regexes(&self) -> anyhow::Result>> { match &self.user_patterns { Some(user_patterns) => { let user_patterns = mxidwc::parse_patterns_vector(user_patterns).map_err(|e| { anyhow::anyhow!( "Failed parsing initial_global_config.user_patterns ({}): {}", super::env::BAIBOT_INITIAL_GLOBAL_CONFIG_USER_PATTERNS, e ) })?; Ok(Some(user_patterns)) } None => Ok(None), } } pub fn validate(self) -> anyhow::Result<()> { self.user_pattern_regexes()?; for purpose in AgentPurpose::choices() { let agent_id = self.handler.get_by_purpose(*purpose); let Some(agent_id) = agent_id else { // None is OK continue; }; let config_key = format!( "initial_global_config.handler.{}", purpose.as_str().replace("-", "_") ); if agent_id.is_empty() { return Err(anyhow::anyhow!( "The {} configuration key must be pointing to a valid agent id or be set to null", config_key, )); } let agent_identifier = PublicIdentifier::from_str(&agent_id); let Some(agent_identifier) = agent_identifier else { return Err(anyhow::anyhow!( "The {} configuration key specifies an agent id (`{}`) that cannot be parsed. {}", config_key, agent_id, crate::strings::agent::invalid_id_generic() )); }; // We only allow statically-defined agents for now, although DynamicGlobal may make sense too. let PublicIdentifier::Static(_) = agent_identifier else { return Err(anyhow::anyhow!( "The {} configuration key specifies an agent id (`{}`) which does not refer to a static agent.", config_key, agent_id, )); }; } let _: GlobalConfig = self.try_into()?; Ok(()) } } impl TryInto for ConfigInitialGlobalConfig { type Error = anyhow::Error; fn try_into(self) -> anyhow::Result { let mut entity = GlobalConfig::default(); if let Some(user_patterns) = self.user_patterns { // We'd rather fail parsing this during startup than at runtime let _ = mxidwc::parse_patterns_vector(&user_patterns).map_err(|err| { anyhow::anyhow!( "Bad initial_global_config.user_patterns ({}): {}", super::env::BAIBOT_INITIAL_GLOBAL_CONFIG_USER_PATTERNS, err ) })?; entity.access.user_patterns = if user_patterns.is_empty() { None } else { Some(user_patterns) }; } for purpose in AgentPurpose::choices() { let agent_id = self.handler.get_by_purpose(*purpose); entity .fallback_room_settings .handler .set_by_purpose(*purpose, agent_id); } Ok(entity) } } #[cfg(test)] #[path = "config_tests.rs"] mod config_tests; ================================================ FILE: src/entity/cfg/config_tests.rs ================================================ use super::{Avatar, ConfigUser, ConfigUserAuth, ConfigUserEncryption}; use crate::entity::cfg::env; fn base_user() -> ConfigUser { ConfigUser { mxid_localpart: "baibot".to_owned(), password: None, access_token: None, device_id: None, name: "baibot".to_owned(), encryption: ConfigUserEncryption { recovery_passphrase: None, recovery_reset_allowed: false, }, avatar: Avatar::Default, } } #[test] fn auth_config_uses_password_mode() { let mut user = base_user(); user.password = Some("secret".to_owned()); let auth = user .auth_config("example.com") .expect("password auth should be valid"); match auth { ConfigUserAuth::UserPassword { username, password } => { assert_eq!(username, "baibot"); assert_eq!(password, "secret"); } ConfigUserAuth::AccessToken { .. } => { panic!("expected password auth mode"); } } } #[test] fn auth_config_uses_access_token_mode() { let mut user = base_user(); user.access_token = Some("token123".to_owned()); user.device_id = Some("DEVICE1".to_owned()); let auth = user .auth_config("example.com") .expect("access token auth should be valid"); match auth { ConfigUserAuth::AccessToken { user_id, device_id, access_token, } => { assert_eq!(user_id.as_str(), "@baibot:example.com"); assert_eq!(device_id.as_str(), "DEVICE1"); assert_eq!(access_token, "token123"); } ConfigUserAuth::UserPassword { .. } => { panic!("expected access token auth mode"); } } } #[test] fn auth_config_rejects_both_auth_methods() { let mut user = base_user(); user.password = Some("secret".to_owned()); user.access_token = Some("token123".to_owned()); user.device_id = Some("DEVICE1".to_owned()); let err = user .auth_config("example.com") .expect_err("both auth methods should be rejected"); assert!( err.to_string() .contains("exactly one authentication method") ); } #[test] fn auth_config_rejects_missing_auth() { let user = base_user(); let err = user .auth_config("example.com") .expect_err("missing auth should be rejected"); assert!(err.to_string().contains("Set one authentication method")); } #[test] fn auth_config_rejects_access_token_without_device_id() { let mut user = base_user(); user.access_token = Some("token123".to_owned()); let err = user .auth_config("example.com") .expect_err("access token mode without device_id should be rejected"); assert!(err.to_string().contains(env::BAIBOT_USER_DEVICE_ID)); } #[test] fn auth_config_treats_empty_strings_as_unset() { let mut user = base_user(); user.password = Some(String::new()); user.access_token = Some(String::new()); user.device_id = Some(String::new()); let err = user .auth_config("example.com") .expect_err("empty auth values should be treated as unset"); assert!(err.to_string().contains("Set one authentication method")); } ================================================ FILE: src/entity/cfg/defaults.rs ================================================ const CONFIG_FILE_PATH: &str = "config.yml"; const NAME: &str = "baibot"; const COMMAND_PREFIX: &str = "!bai"; const PERSISTENCE_SESSION_FILE_NAME: &str = "session.json"; const PERSISTENCE_DB_DIR_NAME: &str = "db"; pub(crate) fn name() -> String { NAME.to_owned() } pub(crate) fn config_file_path() -> String { CONFIG_FILE_PATH.to_owned() } pub(super) fn command_prefix() -> String { COMMAND_PREFIX.to_owned() } pub(super) fn room_post_join_self_introduction_enabled() -> bool { true } pub(super) fn persistence_data_dir_path() -> Option { None } pub(super) fn persistence_session_file_name() -> String { PERSISTENCE_SESSION_FILE_NAME.to_owned() } pub(super) fn persistence_db_dir_name() -> String { PERSISTENCE_DB_DIR_NAME.to_owned() } pub(super) fn logging() -> String { "warn,mxlink=debug,baibot=debug".to_owned() } ================================================ FILE: src/entity/cfg/env.rs ================================================ pub const BAIBOT_CONFIG_FILE_PATH: &str = "BAIBOT_CONFIG_FILE_PATH"; pub const BAIBOT_HOMESERVER_SERVER_NAME: &str = "BAIBOT_HOMESERVER_SERVER_NAME"; pub const BAIBOT_HOMESERVER_URL: &str = "BAIBOT_HOMESERVER_URL"; pub const BAIBOT_USER_MXID_LOCALPART: &str = "BAIBOT_USER_MXID_LOCALPART"; pub const BAIBOT_USER_PASSWORD: &str = "BAIBOT_USER_PASSWORD"; pub const BAIBOT_USER_ACCESS_TOKEN: &str = "BAIBOT_USER_ACCESS_TOKEN"; pub const BAIBOT_USER_DEVICE_ID: &str = "BAIBOT_USER_DEVICE_ID"; pub const BAIBOT_USER_NAME: &str = "BAIBOT_USER_NAME"; pub const BAIBOT_USER_AVATAR: &str = "BAIBOT_USER_AVATAR"; pub const BAIBOT_USER_ENCRYPTION_RECOVERY_PASSPHRASE: &str = "BAIBOT_USER_ENCRYPTION_RECOVERY_PASSPHRASE"; pub const BAIBOT_USER_ENCRYPTION_RECOVERY_RESET_ALLOWED: &str = "BAIBOT_USER_ENCRYPTION_RECOVERY_RESET_ALLOWED"; pub const BAIBOT_COMMAND_PREFIX: &str = "BAIBOT_COMMAND_PREFIX"; pub const BAIBOT_ROOM_POST_JOIN_SELF_INTRODUCTION_ENABLED: &str = "BAIBOT_ROOM_POST_JOIN_SELF_INTRODUCTION_ENABLED"; pub const BAIBOT_LOGGING: &str = "BAIBOT_LOGGING"; pub const BAIBOT_ACCESS_ADMIN_PATTERNS: &str = "BAIBOT_ACCESS_ADMIN_PATTERNS"; pub const BAIBOT_INITIAL_GLOBAL_CONFIG_HANDLER_CATCH_ALL: &str = "BAIBOT_INITIAL_GLOBAL_CONFIG_HANDLER_CATCH_ALL"; pub const BAIBOT_INITIAL_GLOBAL_CONFIG_HANDLER_TEXT_GENERATION: &str = "BAIBOT_INITIAL_GLOBAL_CONFIG_HANDLER_TEXT_GENERATION"; pub const BAIBOT_INITIAL_GLOBAL_CONFIG_HANDLER_TEXT_TO_SPEECH: &str = "BAIBOT_INITIAL_GLOBAL_CONFIG_HANDLER_TEXT_TO_SPEECH"; pub const BAIBOT_INITIAL_GLOBAL_CONFIG_HANDLER_SPEECH_TO_TEXT: &str = "BAIBOT_INITIAL_GLOBAL_CONFIG_HANDLER_SPEECH_TO_TEXT"; pub const BAIBOT_INITIAL_GLOBAL_CONFIG_HANDLER_IMAGE_GENERATION: &str = "BAIBOT_INITIAL_GLOBAL_CONFIG_HANDLER_IMAGE_GENERATION"; pub const BAIBOT_INITIAL_GLOBAL_CONFIG_USER_PATTERNS: &str = "BAIBOT_INITIAL_GLOBAL_CONFIG_USER_PATTERNS"; pub const BAIBOT_PERSISTENCE_DATA_DIR_PATH: &str = "BAIBOT_PERSISTENCE_DATA_DIR_PATH"; pub const BAIBOT_PERSISTENCE_SESSION_ENCRYPTION_KEY: &str = "BAIBOT_PERSISTENCE_SESSION_ENCRYPTION_KEY"; pub const BAIBOT_PERSISTENCE_CONFIG_ENCRYPTION_KEY: &str = "BAIBOT_PERSISTENCE_CONFIG_ENCRYPTION_KEY"; ================================================ FILE: src/entity/cfg/mod.rs ================================================ mod config; pub mod defaults; pub mod env; pub use config::{Avatar, Config, ConfigUserAuth}; ================================================ FILE: src/entity/globalconfig/entity.rs ================================================ use mxlink::matrix_sdk::ruma::events::macros::EventContent; use serde::{Deserialize, Serialize}; use mxlink::helpers::account_data_config::GlobalConfig as GlobalConfigTrait; use mxlink::helpers::account_data_config::GlobalConfigCarrierContent as GlobalConfigCarrierContentTrait; use crate::agent::AgentDefinition; use crate::entity::roomconfig::RoomSettings; #[derive(Clone, Debug, Default, Deserialize, Serialize, EventContent)] #[ruma_event(type = "cc.etke.baibot.global_config", kind = GlobalAccountData)] pub struct GlobalConfigCarrierContent { pub payload: String, } impl GlobalConfigCarrierContentTrait for GlobalConfigCarrierContent { fn payload(&self) -> &str { &self.payload } fn new(payload: String) -> Self { Self { payload } } } #[derive(Clone, Debug, Default, Deserialize, Serialize)] pub struct GlobalConfig { pub fallback_room_settings: RoomSettings, pub access: GlobalConfigAccess, pub agents: Vec, } impl GlobalConfig { pub fn new(user_patterns: Option>) -> Self { Self { fallback_room_settings: RoomSettings::default(), access: GlobalConfigAccess { user_patterns, room_local_agent_manager_patterns: None, }, agents: vec![], } } } impl GlobalConfigTrait for GlobalConfig {} #[derive(Clone, Debug, Default, Deserialize, Serialize)] pub struct GlobalConfigAccess { // Contains a list of patterns that will be used to specify the "allowed bot users". // These remain as patterns and are turned into regex and made use of on demand. // Example: `["@*:example.com"]` pub user_patterns: Option>, // Contains a list of patterns that will be used to specify "allowed room-local agent managers". // These remain as patterns and are turned into regex and made use of on demand. // Example: `["@*:example.com"]` pub room_local_agent_manager_patterns: Option>, } ================================================ FILE: src/entity/globalconfig/mod.rs ================================================ mod entity; use mxlink::helpers::account_data_config::GlobalConfigManager as AccountDataGlobalConfigManager; pub use entity::{GlobalConfig, GlobalConfigCarrierContent}; pub type GlobalConfigurationManager = AccountDataGlobalConfigManager; ================================================ FILE: src/entity/interaction_context.rs ================================================ use mxlink::ThreadInfo; use super::MessagePayload; pub struct InteractionContext { pub thread_info: ThreadInfo, pub trigger: InteractionTrigger, } pub struct InteractionTrigger { pub is_mentioning_bot: bool, pub payload: MessagePayload, } ================================================ FILE: src/entity/message_context.rs ================================================ use mxlink::matrix_sdk::Room; use mxlink::matrix_sdk::ruma::{OwnedEventId, OwnedUserId, RoomId}; use mxlink::ThreadInfo; use super::{ MessagePayload, RoomConfigContext, TriggerEventInfo, globalconfig::GlobalConfig, roomconfig::RoomConfig, }; #[derive(Debug)] pub struct MessageContext { room: Room, room_config_context: RoomConfigContext, admin_whitelist_regexes: Vec, trigger_event_info: TriggerEventInfo, thread_info: ThreadInfo, bot_display_name: Option, } impl MessageContext { pub fn new( room: Room, room_config_context: RoomConfigContext, admin_whitelist_regexes: Vec, trigger_event_info: TriggerEventInfo, thread_info: ThreadInfo, ) -> Self { Self { room, room_config_context, admin_whitelist_regexes, trigger_event_info, thread_info, bot_display_name: None, } } pub fn with_bot_display_name(mut self, value: Option) -> Self { self.bot_display_name = value; self } pub fn bot_display_name(&self) -> &Option { &self.bot_display_name } pub fn room(&self) -> &Room { &self.room } pub fn room_id(&self) -> &RoomId { self.room.room_id() } pub fn global_config(&self) -> &GlobalConfig { &self.room_config_context.global_config } pub fn room_config(&self) -> &RoomConfig { &self.room_config_context.room_config } pub fn room_config_context(&self) -> &RoomConfigContext { &self.room_config_context } pub fn event_id(&self) -> &OwnedEventId { &self.trigger_event_info.event_id } pub fn sender_id(&self) -> &OwnedUserId { &self.trigger_event_info.sender } pub fn payload(&self) -> &MessagePayload { &self.trigger_event_info.payload } pub fn thread_info(&self) -> &ThreadInfo { &self.thread_info } pub fn sender_can_manage_global_config(&self) -> bool { self.trigger_event_info.sender_is_admin } pub fn sender_can_manage_room_local_agents(&self) -> mxidwc::Result { Ok(self.sender_can_manage_global_config() || self.sender_is_allowed_room_local_agent_manager()?) } pub fn combined_admin_and_user_regexes(&self) -> Vec { let mut combined = self.admin_whitelist_regexes.clone(); if let Some(user_patterns) = &self.global_config().access.user_patterns { let user_regexes = mxidwc::parse_patterns_vector(user_patterns); match user_regexes { Ok(user_regexes) => { combined.extend(user_regexes); } Err(err) => { tracing::warn!( "Error parsing user patterns for room {}: {:?}", self.room.room_id(), err ); } } } combined } fn sender_is_allowed_room_local_agent_manager(&self) -> mxidwc::Result { self.room_config_context() .is_user_allowed_room_local_agent_manager(self.sender_id().clone()) } } ================================================ FILE: src/entity/message_payload.rs ================================================ use mxlink::matrix_sdk::ruma::events::room::message::{ AudioMessageEventContent, FileMessageEventContent, ImageMessageEventContent, MessageType, TextMessageEventContent, }; use mxlink::matrix_sdk::ruma::{OwnedEventId, OwnedUserId}; use mxlink::ThreadInfo; /// MessagePayload is like matrix-sdk's MessageType, but represents only message types that the bot deals with and payloads are massaged a bit. /// /// This also includes a few synthetic events. #[derive(Debug, Clone)] pub enum MessagePayload { /// A synthetic message payload that indicates that the bot should produce a reply inside a thread. /// This does not represent an actual message event, it's just a way to trigger a chat completion. /// /// When this is invoked, the ThreadInfo contains the full thread details (which represents our context). /// /// See: https://github.com/etkecc/baibot/issues/15 SynthethicChatCompletionTriggerInThread, /// A synthetic message payload that indicates that the bot should produce a reply to a specific message. /// This does not represent an actual message event, it's just a way to trigger a chat completion. /// /// When this is invoked, the ThreadInfo would refer to the reply-message that triggered us. /// We can follow the chain upward from it to get the full context. /// /// See: https://github.com/etkecc/baibot/issues/15 SynthethicChatCompletionTriggerForReply, Text(TextMessageEventContent), Audio(AudioMessageEventContent), Image(ImageMessageEventContent), File(FileMessageEventContent), Reaction { key: String, reacted_to_event_payload: Box, reacted_to_event_id: OwnedEventId, reacted_to_event_sender_id: OwnedUserId, }, /// Represents an encrypted message Encrypted(ThreadInfo), } impl TryInto for MessageType { type Error = String; fn try_into(self) -> Result { let payload = match self { MessageType::Text(text_content) => MessagePayload::Text(text_content), MessageType::Audio(audio_content) => { // We can consider inspecting `audio_content.voice.is_some()` and ignoring audio which is not a voice message. // // However, at the time of this writing (2024-09-10), certain popular clients (Element iOS) send voice messages // as regular audio messages, without voice annotation as per MSC3245. // For this reason, we handle all audio. MessagePayload::Audio(audio_content) } MessageType::Image(image_content) => MessagePayload::Image(image_content), MessageType::File(file_content) => MessagePayload::File(file_content), other => { return Err(format!("Unsupported message type: {:?}", other)); } }; Ok(payload) } } ================================================ FILE: src/entity/mod.rs ================================================ pub mod catch_up_marker; pub mod cfg; pub mod globalconfig; mod interaction_context; mod message_context; mod message_payload; mod room_config_context; pub mod roomconfig; mod trigger_event_info; pub use interaction_context::{InteractionContext, InteractionTrigger}; pub use message_context::MessageContext; pub use message_payload::MessagePayload; pub use room_config_context::RoomConfigContext; pub use trigger_event_info::TriggerEventInfo; ================================================ FILE: src/entity/room_config_context.rs ================================================ use mxlink::matrix_sdk::ruma::OwnedUserId; use super::globalconfig::GlobalConfig; use super::roomconfig::RoomConfig; use crate::entity::roomconfig::{ SpeechToTextFlowType, SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages, TextGenerationAutoUsage, TextGenerationPrefixRequirementType, TextGenerationSenderContextMode, TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType, defaults as roomconfig_defaults, }; #[derive(Debug)] pub struct RoomConfigContext { pub(crate) global_config: GlobalConfig, pub(crate) room_config: RoomConfig, } impl RoomConfigContext { pub fn new(global_config: GlobalConfig, room_config: RoomConfig) -> RoomConfigContext { Self { global_config, room_config, } } pub fn speech_to_text_flow_type(&self) -> SpeechToTextFlowType { self.room_config .settings .speech_to_text .flow_type .or({ self.global_config .fallback_room_settings .speech_to_text .flow_type }) .unwrap_or(roomconfig_defaults::SPEECH_TO_TEXT_FLOW_TYPE) } pub fn speech_to_text_msg_type_for_non_threaded_only_transcribed_messages( &self, ) -> SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages { self.room_config .settings .speech_to_text .msg_type_for_non_threaded_only_transcribed_messages .or({ self.global_config .fallback_room_settings .speech_to_text .msg_type_for_non_threaded_only_transcribed_messages }) .unwrap_or( roomconfig_defaults::SPEECH_TO_TEXT_ONLY_TRANSCRIBE_NON_THREADED_MESSAGE_TYPE, ) } pub fn speech_to_text_language(&self) -> Option { self.room_config .settings .speech_to_text .language .clone() .or({ self.global_config .fallback_room_settings .speech_to_text .language .clone() }) } pub fn auto_text_generation_usage(&self) -> TextGenerationAutoUsage { self.room_config .settings .text_generation .auto_usage .or({ self.global_config .fallback_room_settings .text_generation .auto_usage }) .unwrap_or(roomconfig_defaults::TEXT_GENERATION_AUTO_USAGE) } pub fn should_auto_text_generate(&self, original_message_is_audio: bool) -> bool { match self.auto_text_generation_usage() { TextGenerationAutoUsage::Never => false, TextGenerationAutoUsage::Always => true, TextGenerationAutoUsage::OnlyForVoice => original_message_is_audio, TextGenerationAutoUsage::OnlyForText => !original_message_is_audio, } } pub fn text_generation_prompt_override(&self) -> Option { self.room_config .settings .text_generation .prompt_override .clone() .or_else(|| { self.global_config .fallback_room_settings .text_generation .prompt_override .clone() }) } pub fn text_generation_temperature_override(&self) -> Option { self.room_config .settings .text_generation .temperature_override .or({ self.global_config .fallback_room_settings .text_generation .temperature_override }) } pub fn text_generation_context_management_enabled(&self) -> bool { self.room_config .settings .text_generation .context_management_enabled .or({ self.global_config .fallback_room_settings .text_generation .context_management_enabled }) .unwrap_or(false) } pub fn text_generation_sender_context_mode(&self) -> TextGenerationSenderContextMode { self.room_config .settings .text_generation .sender_context_mode .or({ self.global_config .fallback_room_settings .text_generation .sender_context_mode }) .unwrap_or(roomconfig_defaults::TEXT_GENERATION_SENDER_CONTEXT_MODE) } pub fn text_generation_prefix_requirement_type(&self) -> TextGenerationPrefixRequirementType { self.room_config .settings .text_generation .prefix_requirement_type .or({ self.global_config .fallback_room_settings .text_generation .prefix_requirement_type }) .unwrap_or(roomconfig_defaults::TEXT_GENERATION_PREFIX_REQUIREMENT_TYPE) } pub fn text_to_speech_bot_messages_flow_type(&self) -> TextToSpeechBotMessagesFlowType { self.room_config .settings .text_to_speech .bot_msgs_flow_type .or({ self.global_config .fallback_room_settings .text_to_speech .bot_msgs_flow_type }) .unwrap_or(roomconfig_defaults::TEXT_TO_SPEECH_BOT_MESSAGES_FLOW_TYPE) } pub fn text_to_speech_user_messages_flow_type(&self) -> TextToSpeechUserMessagesFlowType { self.room_config .settings .text_to_speech .user_msgs_flow_type .or({ self.global_config .fallback_room_settings .text_to_speech .user_msgs_flow_type }) .unwrap_or(roomconfig_defaults::TEXT_TO_SPEECH_USER_MESSAGES_FLOW_TYPE) } pub fn text_to_speech_speed_override(&self) -> Option { self.room_config.settings.text_to_speech.speed_override.or({ self.global_config .fallback_room_settings .text_to_speech .speed_override }) } pub fn text_to_speech_voice_override(&self) -> Option { self.room_config .settings .text_to_speech .voice_override .clone() .or_else(|| { self.global_config .fallback_room_settings .text_to_speech .voice_override .clone() }) } pub fn is_user_allowed_room_local_agent_manager( &self, user_id: OwnedUserId, ) -> mxidwc::Result { match &self.global_config.access.room_local_agent_manager_patterns { None => Ok(false), Some(patterns) => { let allowed_regexes = mxidwc::parse_patterns_vector(patterns)?; Ok(mxidwc::match_user_id(user_id.as_str(), &allowed_regexes)) } } } } ================================================ FILE: src/entity/roomconfig/defaults.rs ================================================ use super::{SpeechToTextFlowType, SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages}; use super::{ TextGenerationAutoUsage, TextGenerationPrefixRequirementType, TextGenerationSenderContextMode, }; use super::{TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType}; pub const TEXT_GENERATION_PREFIX_REQUIREMENT_TYPE: TextGenerationPrefixRequirementType = TextGenerationPrefixRequirementType::No; pub const TEXT_GENERATION_AUTO_USAGE: TextGenerationAutoUsage = TextGenerationAutoUsage::Always; pub const TEXT_GENERATION_SENDER_CONTEXT_MODE: TextGenerationSenderContextMode = TextGenerationSenderContextMode::Disabled; pub const TEXT_TO_SPEECH_BOT_MESSAGES_FLOW_TYPE: TextToSpeechBotMessagesFlowType = TextToSpeechBotMessagesFlowType::OnDemandForVoice; pub const TEXT_TO_SPEECH_USER_MESSAGES_FLOW_TYPE: TextToSpeechUserMessagesFlowType = TextToSpeechUserMessagesFlowType::OnDemand; pub const SPEECH_TO_TEXT_FLOW_TYPE: SpeechToTextFlowType = SpeechToTextFlowType::TranscribeAndGenerateText; // While notice messages may be less desirable with other bots in the room, // it's probably a better default for most people who enable "transcribe-only" mode. pub const SPEECH_TO_TEXT_ONLY_TRANSCRIBE_NON_THREADED_MESSAGE_TYPE: SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages = SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::Text; ================================================ FILE: src/entity/roomconfig/entity/handler.rs ================================================ use serde::{Deserialize, Serialize}; use crate::agent::AgentPurpose; #[derive(Clone, Debug, Default, Deserialize, Serialize)] pub struct RoomSettingsHandler { /// The agent used for any of the tasks which do not have a dedicated agent for them catch_all: Option, /// The agent used for text generation text_generation: Option, /// The agent used for transcribing audio (voice) to text speech_to_text: Option, /// The agent used for converting text to audio (voice) text_to_speech: Option, /// The agent used for generating images image_generation: Option, } impl RoomSettingsHandler { pub fn get_by_purpose(&self, purpose: AgentPurpose) -> Option { match purpose { AgentPurpose::CatchAll => self.catch_all.clone(), AgentPurpose::TextGeneration => self.text_generation.clone(), AgentPurpose::SpeechToText => self.speech_to_text.clone(), AgentPurpose::TextToSpeech => self.text_to_speech.clone(), AgentPurpose::ImageGeneration => self.image_generation.clone(), } } pub fn get_by_purpose_with_catch_all_fallback(&self, purpose: AgentPurpose) -> Option { match self.get_by_purpose(purpose) { Some(agent_id) => Some(agent_id), None => self.catch_all.clone(), } } pub fn set_by_purpose(&mut self, purpose: AgentPurpose, agent_id: Option) { match purpose { AgentPurpose::CatchAll => { self.catch_all = agent_id; } AgentPurpose::TextGeneration => { self.text_generation = agent_id; } AgentPurpose::SpeechToText => { self.speech_to_text = agent_id; } AgentPurpose::TextToSpeech => { self.text_to_speech = agent_id; } AgentPurpose::ImageGeneration => { self.image_generation = agent_id; } }; } } ================================================ FILE: src/entity/roomconfig/entity/mod.rs ================================================ use mxlink::helpers::account_data_config::RoomConfig as RoomConfigTrait; use mxlink::helpers::account_data_config::RoomConfigCarrierContent as RoomConfigCarrierContentTrait; use mxlink::matrix_sdk::ruma::events::macros::EventContent; use mxlink::matrix_sdk::{Room, RoomMemberships}; use serde::{Deserialize, Serialize}; use crate::agent::AgentDefinition; mod handler; mod speech_to_text; mod text_generation; mod text_to_speech; pub use handler::RoomSettingsHandler; pub use speech_to_text::{ SpeechToTextFlowType, SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages, }; pub use text_generation::{ TextGenerationAutoUsage, TextGenerationPrefixRequirementType, TextGenerationSenderContextMode, }; pub use text_to_speech::{TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType}; #[derive(Clone, Debug, Deserialize, Serialize, EventContent)] #[ruma_event(type = "cc.etke.baibot.room_config", kind = RoomAccountData)] pub struct RoomConfigCarrierContent { pub payload: String, } impl RoomConfigCarrierContentTrait for RoomConfigCarrierContent { fn payload(&self) -> &str { &self.payload } fn new(payload: String) -> Self { Self { payload } } } #[derive(Clone, Debug, Default, Deserialize, Serialize)] pub struct RoomConfig { pub settings: RoomSettings, pub agents: Vec, } impl RoomConfigTrait for RoomConfig {} impl RoomConfig { pub async fn with_room(mut self, room: Room) -> Self { tracing::trace!( "Determining room members count to decide on a suitable text-generation/prefix-requirement-type default" ); let members = room.members(RoomMemberships::ACTIVE).await; let prefix_requirement_type = match members { Ok(members) => { let members_count = members.len(); let prefix_requirement_type = if members.len() > 2 { text_generation::TextGenerationPrefixRequirementType::CommandPrefix } else { text_generation::TextGenerationPrefixRequirementType::No }; tracing::info!( ?members_count, ?prefix_requirement_type, "Determined text-generation/prefix-requirement-type based on room members count" ); prefix_requirement_type } Err(err) => { tracing::error!( ?err, "Failed to get members of room - will default text-generation/prefix-requirement-type to No" ); text_generation::TextGenerationPrefixRequirementType::No } }; self.settings.text_generation.prefix_requirement_type = Some(prefix_requirement_type); self } } #[derive(Clone, Debug, Default, Deserialize, Serialize)] pub struct RoomSettings { pub handler: handler::RoomSettingsHandler, #[serde(default)] pub text_generation: text_generation::RoomSettingsTextGeneration, #[serde(default)] pub speech_to_text: speech_to_text::RoomSettingsSpeechToText, #[serde(default)] pub text_to_speech: text_to_speech::RoomSettingsTextToSpeech, } ================================================ FILE: src/entity/roomconfig/entity/speech_to_text.rs ================================================ use serde::{Deserialize, Serialize}; #[derive(Clone, Debug, Default, Deserialize, Serialize)] pub struct RoomSettingsSpeechToText { pub flow_type: Option, /// Controls how the transcribed message is posted when dealing with: /// - messages that only get transcribed (and do not trigger text-generation). /// See `flow_type` and `SpeechToTextFlowType::OnlyTranscribe` for more details. /// - incoming voice messages that are not part of a thread. /// For messages that are part of a thread, we need to reply within the thread in a way (with a notice message) /// that won't confuse the bot later, so we have no choice but to use a notice message. /// /// Text-generation may happen either as a direct result of the incoming voice message or as part of a threaded conversation. /// Transcribed messages should not be attributed to the bot for the purposes of text-generation, /// so any time there's a chance of text-generation happening, we should use `SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::Notice` /// and optionally prefix the message with `> 🦻`. pub msg_type_for_non_threaded_only_transcribed_messages: Option, /// The language of the input audio. /// Supplying the input language in [ISO-639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) format will improve accuracy and latency. pub language: Option, } #[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq)] pub enum SpeechToTextFlowType { /// Voice messages are to be ignored. #[serde(rename = "ignore")] Ignore, /// Voice messages are to trigger text-generation. /// This may potentially trigger speech-to-text, but that's not what we care about here. #[serde(rename = "transcribe_and_generate_text")] TranscribeAndGenerateText, // Voices messages are to trigger transcription. #[serde(rename = "only_transcribe")] OnlyTranscribe, } impl SpeechToTextFlowType { pub fn choices() -> Vec { vec![ Self::Ignore, Self::TranscribeAndGenerateText, Self::OnlyTranscribe, ] } pub fn from_str(s: &str) -> Option { match s { "ignore" => Some(Self::Ignore), "transcribe_and_generate_text" => Some(Self::TranscribeAndGenerateText), "only_transcribe" => Some(Self::OnlyTranscribe), _ => None, } } } impl std::fmt::Display for SpeechToTextFlowType { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { SpeechToTextFlowType::Ignore => { write!(f, "ignore") } SpeechToTextFlowType::TranscribeAndGenerateText => { write!(f, "transcribe_and_generate_text") } SpeechToTextFlowType::OnlyTranscribe => write!(f, "only_transcribe"), } } } #[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq)] pub enum SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages { /// Send the transcribed message text as a regular message #[serde(rename = "text")] Text, /// Send the transcribed message text as a notice message #[serde(rename = "notice")] Notice, } impl SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages { pub fn choices() -> Vec { vec![Self::Text, Self::Notice] } pub fn from_str(s: &str) -> Option { match s { "text" => Some(Self::Text), "notice" => Some(Self::Notice), _ => None, } } } impl std::fmt::Display for SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::Text => write!(f, "text"), SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages::Notice => { write!(f, "notice") } } } } ================================================ FILE: src/entity/roomconfig/entity/text_generation.rs ================================================ use serde::{Deserialize, Serialize}; #[derive(Clone, Debug, Default, Deserialize, Serialize)] pub struct RoomSettingsTextGeneration { /// Controls whether initial text messages require a prefix to trigger text generation. /// This could have been a bool, using an enum allows us to add more options (e.g. CustomPrefix) in the future. /// Even if set to "required", prefixless-triggering could still happen via an initial voice message (see auto_usage). pub prefix_requirement_type: Option, /// Controls whether text generation is automatically triggered (depending on message type). pub auto_usage: Option, /// Controls whether conversation context management is enabled. /// When enabled, the bot will automatically tokenize messages and try to shorten the message context intelligently. pub context_management_enabled: Option, /// Controls how each message in the conversation context is annotated with sender metadata. pub sender_context_mode: Option, /// Allows customizing the system prompt that the agent would use pub prompt_override: Option, /// Allows customizing the temperature that the agent would use pub temperature_override: Option, } #[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq)] pub enum TextGenerationPrefixRequirementType { /// Text Generation is to be triggered for any text message #[serde(rename = "no")] No, /// Text Generation is to be triggered only for messages that are prefixed with the command prefix #[serde(rename = "command_prefix")] CommandPrefix, } impl TextGenerationPrefixRequirementType { pub fn choices() -> Vec { vec![Self::No, Self::CommandPrefix] } pub fn from_str(s: &str) -> Option { match s { "no" => Some(Self::No), "command_prefix" => Some(Self::CommandPrefix), _ => None, } } } impl std::fmt::Display for TextGenerationPrefixRequirementType { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { TextGenerationPrefixRequirementType::No => write!(f, "no"), TextGenerationPrefixRequirementType::CommandPrefix => { write!(f, "command_prefix") } } } } #[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq)] pub enum TextGenerationAutoUsage { /// Text Generation is to never be performed #[serde(rename = "never")] Never, /// Text Generation is to always be performed #[serde(rename = "always")] Always, /// Text Generation is to be performed when the original message was sent as audio (voice). /// The voice message would be transcribed to text (subject to other configuration) /// and text generation would be triggered. /// /// Also see `SpeechToTextFlowType`. #[serde(rename = "only_for_voice")] OnlyForVoice, /// Text Generation is to be performed when the original message was sent as text #[serde(rename = "only_for_text")] OnlyForText, } impl TextGenerationAutoUsage { pub fn choices() -> Vec { vec![ Self::Never, Self::Always, Self::OnlyForVoice, Self::OnlyForText, ] } pub fn from_str(s: &str) -> Option { match s { "never" => Some(Self::Never), "always" => Some(Self::Always), "only_for_voice" => Some(Self::OnlyForVoice), "only_for_text" => Some(Self::OnlyForText), _ => None, } } } impl std::fmt::Display for TextGenerationAutoUsage { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { TextGenerationAutoUsage::Never => write!(f, "never"), TextGenerationAutoUsage::Always => write!(f, "always"), TextGenerationAutoUsage::OnlyForVoice => write!(f, "only_for_voice"), TextGenerationAutoUsage::OnlyForText => write!(f, "only_for_text"), } } } #[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq)] pub enum TextGenerationSenderContextMode { #[serde(rename = "disabled")] Disabled, #[serde(rename = "matrix_user_id")] MatrixUserId, #[serde(rename = "matrix_user_id_and_timestamp")] MatrixUserIdAndTimestamp, } impl TextGenerationSenderContextMode { pub fn choices() -> Vec { vec![ Self::Disabled, Self::MatrixUserId, Self::MatrixUserIdAndTimestamp, ] } pub fn from_str(s: &str) -> Option { match s { "disabled" => Some(Self::Disabled), "matrix_user_id" => Some(Self::MatrixUserId), "matrix_user_id_and_timestamp" => Some(Self::MatrixUserIdAndTimestamp), _ => None, } } } impl std::fmt::Display for TextGenerationSenderContextMode { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { TextGenerationSenderContextMode::Disabled => write!(f, "disabled"), TextGenerationSenderContextMode::MatrixUserId => write!(f, "matrix_user_id"), TextGenerationSenderContextMode::MatrixUserIdAndTimestamp => { write!(f, "matrix_user_id_and_timestamp") } } } } ================================================ FILE: src/entity/roomconfig/entity/text_to_speech.rs ================================================ use serde::{Deserialize, Serialize}; #[derive(Clone, Debug, Default, Deserialize, Serialize)] pub struct RoomSettingsTextToSpeech { pub bot_msgs_flow_type: Option, pub user_msgs_flow_type: Option, pub speed_override: Option, pub voice_override: Option, } #[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq)] pub enum TextToSpeechBotMessagesFlowType { /// Never do text-to-speech for bot messages automatically and don't offer it #[serde(rename = "never")] Never, /// Never do text-to-speech for bot messages automatically, but offer it via an emoji reaction for all messages #[serde(rename = "on_demand_always")] OnDemandAlways, /// Never do text-to-speech for bot messages automatically, but offer it via an emoji reaction if the user message that prompted the bot message was audio (voice) #[serde(rename = "on_demand_for_voice")] OnDemandForVoice, /// Convert all bot text messages to audio (voice) automatically if the user message that prompted the bot message was audio (voice) #[serde(rename = "only_for_voice")] OnlyForVoice, /// Convert all bot text messages to audio (voice) automatically #[serde(rename = "always")] Always, } impl TextToSpeechBotMessagesFlowType { pub fn choices() -> Vec { vec![ Self::Never, Self::OnDemandAlways, Self::OnDemandForVoice, Self::Always, Self::OnlyForVoice, ] } pub fn from_str(s: &str) -> Option { match s { "never" => Some(Self::Never), "on_demand_always" => Some(Self::OnDemandAlways), "on_demand_for_voice" => Some(Self::OnDemandForVoice), "only_for_voice" => Some(Self::OnlyForVoice), "always" => Some(Self::Always), _ => None, } } } impl std::fmt::Display for TextToSpeechBotMessagesFlowType { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { TextToSpeechBotMessagesFlowType::Never => write!(f, "never"), TextToSpeechBotMessagesFlowType::OnDemandAlways => write!(f, "on_demand_always"), TextToSpeechBotMessagesFlowType::OnDemandForVoice => write!(f, "on_demand_for_voice"), TextToSpeechBotMessagesFlowType::Always => write!(f, "always"), TextToSpeechBotMessagesFlowType::OnlyForVoice => write!(f, "only_for_voice"), } } } #[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq)] pub enum TextToSpeechUserMessagesFlowType { /// Never do text-to-speech for user messages automatically and don't offer it #[serde(rename = "never")] Never, /// Never do text-to-speech for user messages automatically, but offer it via an emoji reaction #[serde(rename = "on_demand")] OnDemand, /// Convert all user text messages to audio (voice) automatically #[serde(rename = "always")] Always, } impl TextToSpeechUserMessagesFlowType { pub fn choices() -> Vec { vec![Self::Never, Self::OnDemand, Self::Always] } pub fn from_str(s: &str) -> Option { match s { "never" => Some(Self::Never), "on_demand" => Some(Self::OnDemand), "always" => Some(Self::Always), _ => None, } } } impl std::fmt::Display for TextToSpeechUserMessagesFlowType { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { TextToSpeechUserMessagesFlowType::Never => write!(f, "never"), TextToSpeechUserMessagesFlowType::OnDemand => write!(f, "on_demand"), TextToSpeechUserMessagesFlowType::Always => write!(f, "always"), } } } ================================================ FILE: src/entity/roomconfig/mod.rs ================================================ pub mod defaults; mod entity; use mxlink::helpers::account_data_config::RoomConfigManager as AccountDataRoomConfigManager; pub use entity::{RoomConfig, RoomConfigCarrierContent, RoomSettings, RoomSettingsHandler}; pub use entity::{ SpeechToTextFlowType, SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages, TextGenerationAutoUsage, TextGenerationPrefixRequirementType, TextGenerationSenderContextMode, TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType, }; pub type RoomConfigurationManager = AccountDataRoomConfigManager; ================================================ FILE: src/entity/trigger_event_info.rs ================================================ use mxlink::matrix_sdk::ruma::{OwnedEventId, OwnedUserId}; use super::MessagePayload; #[derive(Debug)] pub struct TriggerEventInfo { pub event_id: OwnedEventId, pub sender: OwnedUserId, pub payload: MessagePayload, pub sender_is_admin: bool, } impl TriggerEventInfo { pub fn new( event_id: OwnedEventId, sender: OwnedUserId, payload: MessagePayload, sender_is_admin: bool, ) -> Self { Self { event_id, sender, payload, sender_is_admin, } } } ================================================ FILE: src/lib.rs ================================================ // rustc 1.94+ trips a query-depth overflow when computing async layouts in // the matrix-sdk timeline future graph. matrix-rust-sdk PR #6489 raises the // limit, but `recursion_limit` is per-crate and applies to the crate currently // being compiled — so the consumer has to repeat it. #![recursion_limit = "256"] mod agent; mod bot; mod controller; mod conversation; mod entity; mod strings; mod utils; pub use bot::{Bot, load_config}; pub use entity::cfg::Config; ================================================ FILE: src/main.rs ================================================ use tracing_subscriber::EnvFilter; use tracing_subscriber::fmt::format::FmtSpan; use baibot::{Bot, Config, load_config}; #[tokio::main] async fn main() -> anyhow::Result<()> { match load_config() { Ok(config) => run_with_config(config).await, Err(err) => Err(anyhow::anyhow!("Failed loading configuration: {}", err)), } } async fn run_with_config(config: Config) -> anyhow::Result<()> { let subscriber = tracing_subscriber::fmt() .with_file(true) .with_line_number(true) .with_thread_ids(true) .with_target(true) .with_span_events(FmtSpan::NONE) .with_env_filter(EnvFilter::new(config.logging.clone())) .finish(); tracing::subscriber::set_global_default(subscriber).expect("Failed setting global subscriber"); let bot = Bot::new(config).await?; bot.start().await?; Ok(()) } ================================================ FILE: src/strings/access.rs ================================================ pub fn users_no_patterns() -> String { "No user patterns are configured, so the bot can only be used by administrators.".to_owned() } pub fn users_now_match_patterns(patterns: &[String]) -> String { format!( "The bot can be used by users with a [Matrix user id](https://spec.matrix.org/v1.11/#users) matching the following patterns: `{}`", patterns.join(" "), ) } pub fn room_local_agent_managers_no_patterns() -> String { "No room-local agent manager patterns are configured, so new agents can only be created by administrators.".to_owned() } pub fn room_local_agent_managers_now_match_patterns(patterns: &[String]) -> String { format!( "The bot allows users with a [Matrix user id](https://spec.matrix.org/v1.11/#users) matching the following patterns to manage agents: `{}`", patterns.join(" "), ) } pub fn failed_to_parse_patterns(err: &str) -> String { format!("Failed to parse patterns: {}", err) } ================================================ FILE: src/strings/agent.rs ================================================ use crate::{ agent::{AgentInstance, AgentProvider, AgentPurpose, ControllerTrait, PublicIdentifier}, utils::text::block_quote, }; pub fn invalid_id_generic() -> String { "The provided agent ID is not valid.\n\nIt must have a prefix (like `static/`, `global/` or `room-local/`) followed by a unique identifier which does not contain `/` or spaces.".to_string() } pub fn invalid_id_validation_error(validation_error: String) -> String { format!("The provided agent ID is not valid. {}", validation_error) } pub fn agent_with_given_identifier_missing(agent_identifier: &PublicIdentifier) -> String { format!("There is no agent with an ID of `{}`.", agent_identifier) } pub fn already_exists_see_help(agent_id: &str, command_prefix: &str) -> String { format!( "An agent with the ID `{}` already exists. Send a `{} help` command to the room (**not in this thread**) for more information.", agent_id, command_prefix ) } pub fn incorrect_creation_invocation(command_prefix: &str) -> String { format!( "Incorrect command invocation. This command expects a provider ID and an agent ID. See `{command_prefix} agent` for help." ) } pub fn incorrect_invocation_expects_agent_id_arg(command_prefix: &str) -> String { format!( "Incorrect command invocation. This command expects an agent ID. See `{command_prefix} agent` for help." ) } pub fn not_allowed_to_manage_room_local_agents_in_room() -> String { "You are not allowed to manage room-local agents in this room.".to_string() } pub fn not_allowed_to_manage_static_agents() -> String { "Statically defined agents cannot be managed via commands to the bot. Consider editing the bot configuration.".to_string() } pub fn configuration_does_not_result_in_a_working_agent(err: anyhow::Error) -> String { format!( "The provided configuration does not result in a working agent. The following error was encountered when trying to talk to the agent API:\n```\n{}\n```", err, ) } pub fn configuration_agent_will_ping() -> &'static str { "Checking this agent's API. Please wait.." } pub fn configuration_agent_ping_inconclusive() -> String { "Basic check results are inconclusive - this agent may or may not work.".to_string() } pub fn configuration_agent_ping_ok() -> String { "Basic checks succeeded.".to_string() } pub fn created(agent_identifier: &PublicIdentifier) -> String { format!("Agent `{}` created.", agent_identifier) } pub fn post_creation_helpful_commands( agent_identifier: &PublicIdentifier, agent_instance: &AgentInstance, command_prefix: &str, ) -> String { let mut message = String::new(); message.push_str(&format!( "To make use of the new agent, set it as a handler for a given purpose ({}, {}, etc.) either globally or in this room.", AgentPurpose::TextGeneration, AgentPurpose::SpeechToText, )); message.push_str("\n\n"); let supported_purposes: Vec<&AgentPurpose> = AgentPurpose::choices() .into_iter() .filter(|&p| { if *p == AgentPurpose::CatchAll { true } else { agent_instance.controller().supports_purpose(*p) } }) .collect(); if !supported_purposes.is_empty() { message.push_str( "Choose and send to the room (**not in this thread**) one or a few of these commands:", ); message.push('\n'); let is_room_local = matches!(agent_identifier, PublicIdentifier::DynamicRoomLocal(_)); for purpose in supported_purposes { message.push_str(&format!( "\n- {}", &set_as_purpose_handler_in_room(agent_identifier, purpose, command_prefix,) )); if !is_room_local { message.push_str(&format!( "\n- {}", &set_as_purpose_handler_globally(agent_identifier, purpose, command_prefix,) )); } } } else { message.push_str( "This agent does not support any handler purposes and cannot really be made use of.", ); } message.push_str("\n\n"); message.push_str(&format!( "For more information about configuring handlers, see `{command_prefix} config`\n", )); message } fn set_as_purpose_handler_in_room( agent_identifier: &PublicIdentifier, purpose: &AgentPurpose, command_prefix: &str, ) -> String { let purpose_emoji = purpose.emoji(); format!( "{purpose_emoji} Set as **{purpose}** handler in **this room**: `{command_prefix} config room set-handler {purpose} {agent_identifier}`", ) } fn set_as_purpose_handler_globally( agent_identifier: &PublicIdentifier, purpose: &AgentPurpose, command_prefix: &str, ) -> String { let purpose_emoji = purpose.emoji(); format!( "{purpose_emoji} Set as fallback **{purpose}** handler **globally**: `{command_prefix} config global set-handler {purpose} {agent_identifier}`", ) } pub fn configuration_not_a_valid_yaml_hashmap(err: String) -> String { format!( "The provided configuration is not a valid YAML hashmap:\n```\n{}\n```", err ) } pub fn creation_guide( agent_identifier: &PublicIdentifier, provider: &AgentProvider, pretty_yaml: &str, ) -> String { let mut message = String::from(""); message.push_str(creation_welcome(agent_identifier, provider).as_str()); message.push('\n'); message.push_str(creation_example_config(pretty_yaml).as_str()); message.push_str("\n\n"); message.push_str(creation_howto().as_str()); message.push_str("\n\n"); message.push_str(creation_raw_or_codeblock_ok().as_str()); message } fn creation_welcome(agent_identifier: &PublicIdentifier, provider: &AgentProvider) -> String { format!( "You're defining a new agent (`{}`) powered by the `{}` provider.\n\nSend [YAML](https://en.wikipedia.org/wiki/YAML) configuration that describes it.", agent_identifier, provider.to_static_str(), ) } fn creation_example_config(pretty_yaml: &str) -> String { format!("Below is an example:\n```yml\n{}\n```", pretty_yaml.trim()) } fn creation_howto() -> String { format!( "{}\n\n{}", "Copy, modify (with your own values) and send back the configuration to this message thread.", "You may omit certain configuration keys (or set them to `null`) - this signals to the bot that certain capabilities are not supported by your agent.", ) } fn creation_raw_or_codeblock_ok() -> String { "You can send the configuration as-is in a plain-text message or optionally wrap it in a [Markdown codeblock](https://www.markdownguide.org/extended-syntax/#fenced-code-blocks).".to_string() } pub fn removed_room_local(agent_identifier: &PublicIdentifier, command_prefix: &str) -> String { let mut message = String::new(); message.push_str(&removed(agent_identifier)); message.push_str("\n\n"); message.push_str("This room may still have it configured as a handler. If so, handlers will fail with a friendly error message."); message.push_str("\n\n"); message.push_str(&format!( "Use the `{command_prefix} config status` command to see the handlers for this room.", )); message } pub fn removed_global(agent_identifier: &PublicIdentifier, command_prefix: &str) -> String { let mut message = String::new(); message.push_str(&removed(agent_identifier)); message.push_str("\n\n"); message.push_str("Neither per-room, nor global handlers were adjusted. Some rooms may still try to use this now-removed agent for a given purpose. If so, handlers for them will fail with a friendly error message."); message.push_str("\n\n"); message.push_str(&format!( "Use the `{command_prefix} config status` command to see the handlers for this room, as well as those configured as a global fallback." )); message.push('\n'); message.push_str("This does not cover everything, but it's a start."); message } fn removed(agent_identifier: &PublicIdentifier) -> String { format!("Agent `{}` removed.", agent_identifier) } pub fn purpose_unrecognized(purpose: &str) -> String { format!("The `{}` purpose is unrecognized.", purpose) } pub fn purpose_howto(purpose: &AgentPurpose) -> &'static str { match purpose { AgentPurpose::CatchAll => "used as a fallback, when no specific handler is configured", AgentPurpose::TextGeneration => "communicating with you via text", AgentPurpose::SpeechToText => "turning your voice messages into text", AgentPurpose::TextToSpeech => "turning bot or users text messages into voice messages", AgentPurpose::ImageGeneration => "generating images based on instructions", } } pub fn agent_list_empty() -> String { "No agents are available.".to_string() } pub fn non_empty_agent_list_block(agents: &Vec) -> String { let mut message = String::new(); message.push_str(&agent_list_intro()); message.push('\n'); for agent in agents { let provider_info = agent.definition().provider.info(); let provider_display = match provider_info.homepage_url { Some(url) => format!("[{}]({})", provider_info.name, url), None => provider_info.name.to_string(), }; message.push_str(&format!( "- `{}` ({}), powered by {}\n", agent.identifier(), create_support_badges_text(agent.controller()), provider_display, )); } message } fn agent_list_intro() -> String { "The following agents are available:".to_string() } pub fn agent_list_legend_intro() -> String { "Legend:".to_string() } pub fn error_while_serving_purpose( agent_identifier: &PublicIdentifier, purpose: &AgentPurpose, err: impl std::fmt::Display, ) -> String { format!( "There was a problem performing {} via the `{}` agent:\n\n{}", purpose, agent_identifier, block_quote(&err.to_string()) ) } pub fn empty_response_returned(agent_identifier: &PublicIdentifier) -> String { format!("The `{agent_identifier}` agent returned an empty response.") } pub fn no_configuration_for_purpose_so_cannot_be_used(purpose: &AgentPurpose) -> String { format!( "This agent does not contain configuration for {} {}, so it cannot be used for that.", purpose.emoji(), purpose ) } pub fn no_configuration_for_purpose_after_conversion_so_cannot_be_used( purpose: &AgentPurpose, ) -> String { format!( "This agent's configuration was converted to the OpenAI format, but conversion failed. There is no configuration for {} {}, so it cannot be used for that.", purpose.emoji(), purpose ) } pub fn create_support_badges_text(controller: &impl ControllerTrait) -> String { let mut support_badges = vec![]; for purpose in AgentPurpose::choices() { if *purpose == AgentPurpose::CatchAll { // This is not a real purpose that users care about here continue; } if controller.supports_purpose(*purpose) { support_badges.push(purpose.emoji()); } } if support_badges.is_empty() { return "❌".to_owned(); } support_badges.join(" ") } ================================================ FILE: src/strings/cfg.rs ================================================ use crate::{ agent::{ AgentInstance, AgentPurpose, PublicIdentifier, utils::AgentForPurposeDeterminationInfoConfigurationSource, }, entity::roomconfig::{ SpeechToTextFlowType, SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages, TextGenerationAutoUsage, TextGenerationPrefixRequirementType, TextToSpeechBotMessagesFlowType, TextToSpeechUserMessagesFlowType, }, utils::text::block_quote, }; pub fn error_config_type_not_replaced() -> String { "The `CONFIG_TYPE` placeholder in the command was not replaced.\n\nIt should either be set to `room` (for room-specific configuration) or `global` (for global configuration).".to_owned() } pub fn create_display_text_for_value(value: impl std::fmt::Display) -> String { let value = value.to_string(); if value.to_string().contains("\n") { format!("\n\n{}\n", block_quote(&value)) } else { format!(" `{}`", value) } } pub fn value_currently_set_to(value: impl std::fmt::Display) -> String { format!( "This configuration value is currently set to:{}", create_display_text_for_value(value) ) } pub fn value_currently_unset() -> String { "This configuration value is currently unset.".to_owned() } pub fn configuration_invocation_incorrect_more_values_expected() -> String { "The invocation for this command is incorrect. More values are expected in your command." .to_string() } pub fn configuration_getter_used_with_extra_text( getter_name: &str, remaining_text: &str, ) -> String { format!( "You're invoking a getter command (`{getter_name}`), but passing additional text (`{remaining_text}`) as if you're invoking a setter.\n\nPerhaps you meant to invoke `set-{getter_name}`?" ) } pub fn configuration_value_unrecognized(value: &str) -> String { format!("The value `{}` is not a recognized choice.", value) } pub fn configuration_value_not_f32(value: &str) -> String { format!( "The value `{}` could not be converted to a [floating point number](https://en.wikipedia.org/wiki/Floating-point_arithmetic).", value ) } pub fn status_room_config_handlers_heading() -> &'static str { "📍 Room-specific handlers" } pub fn status_room_config_handlers_intro() -> &'static str { "This **room's configuration** specifies the following handlers (**taking priority** over global handlers):" } pub fn status_room_config_handlers_outro(command_prefix: &str) -> String { let mut message = String::new(); message.push_str(&format!( "Use `{command_prefix} config room set-handler` commands (see how via `{command_prefix} config`) to configure the handlers for this room.", )); message.push_str("\n\n"); message.push_str( "If a particular handler is unset, the catch-all handler would be used. If the catch-all handler is also unset, the global configuration would be used.", ); message } pub fn status_global_config_handlers_heading() -> &'static str { "🌐 Global handlers" } pub fn status_global_config_handlers_intro() -> &'static str { "The **global configuration** specifies the following handlers:" } pub fn status_global_config_handlers_outro(command_prefix: &str) -> String { let mut message = String::new(); message.push_str(&format!( "Use `{command_prefix} config global set-handler` commands (see how via `{command_prefix} config`) to configure the default handlers globally.", )); message.push_str("\n\n"); message.push_str("If a particular handler is unset, the catch-all agent would be used."); message } fn status_agent_not_found() -> &'static str { "not found" } pub fn status_handler_line_agent_found( purpose: &AgentPurpose, agent_id: &str, agent: Option<&AgentInstance>, ) -> String { let agent_status = match agent { Some(agent) => super::agent::create_support_badges_text(agent.controller()), None => status_agent_not_found().to_string(), }; format!( "- {} {}: `{}` ({})", purpose.emoji(), purpose, agent_id, agent_status, ) .to_owned() } pub fn status_handler_line_catch_all_agent_not_set_globally() -> String { format!( "- {} {}: *not set*", AgentPurpose::CatchAll.emoji(), AgentPurpose::CatchAll, ) .to_owned() } pub fn status_handler_line_catch_all_agent_not_set_in_room_default_to_global() -> String { format!( "- {} {}: *not set, defaulting to global config*", AgentPurpose::CatchAll.emoji(), AgentPurpose::CatchAll, ) .to_owned() } pub fn status_handler_line_non_catch_all_agent_not_set_globally(purpose: &AgentPurpose) -> String { format!( "- {} {}: *not set, defaulting to {}*", purpose.emoji(), purpose, AgentPurpose::CatchAll, ) .to_owned() } pub fn status_handler_line_non_catch_all_agent_not_set_in_room_default_to_global( purpose: &AgentPurpose, ) -> String { format!( "- {} {}: *not set, defaulting to {} or global config*", purpose.emoji(), purpose, AgentPurpose::CatchAll, ) .to_owned() } pub fn status_room_agents_heading() -> &'static str { "🤖 Room-specific agents" } pub fn status_room_agents_intro() -> &'static str { "The following agents have been defined in this room:" } pub fn status_room_agents_empty() -> &'static str { "No agents have been defined in this room." } pub fn status_room_agents_outro(command_prefix: &str) -> String { let mut message = String::new(); message.push_str( format!("Use `{command_prefix} agent create-room-local` commands (see how via `{command_prefix} help`) to define agents in this room.").as_str() ); message.push_str("\n\n"); message.push_str(format!( "You may also use any of the globally defined agents. See `{command_prefix} agent list` to see the full list of agents." ).as_str()); message } pub fn status_text_generation_heading() -> String { format!( "{} {}", AgentPurpose::TextGeneration.emoji(), AgentPurpose::TextGeneration.heading() ) } pub fn status_speech_to_text_heading() -> String { format!( "{} {}", AgentPurpose::SpeechToText.emoji(), AgentPurpose::SpeechToText.heading() ) } pub fn status_text_to_speech_heading() -> String { format!( "{} {}", AgentPurpose::TextToSpeech.emoji(), AgentPurpose::TextToSpeech.heading() ) } pub fn status_image_generation_heading() -> String { format!( "{} {}", AgentPurpose::ImageGeneration.emoji(), AgentPurpose::ImageGeneration.heading() ) } pub fn status_text_generation_entry_prefix_requirement_type( value: TextGenerationPrefixRequirementType, set_where: &str, ) -> String { format!("- 🗟 Prefix Requirement Type: `{}` ({})\n", value, set_where) } pub fn status_text_generation_entry_auto_usage( value: TextGenerationAutoUsage, set_where: &str, ) -> String { format!("- 🪄 Auto usage: `{}` ({})\n", value, set_where) } pub fn status_text_generation_entry_context_management(value: bool, set_where: &str) -> String { format!("- ♻️ Context management: `{}` ({})\n", value, set_where) } pub fn status_text_generation_entry_sender_context( value: impl std::fmt::Display, set_where: &str, ) -> String { format!("- 👤 Sender context mode: `{}` ({})\n", value, set_where) } pub fn status_text_generation_entry_prompt(value: &str, set_where: &str) -> String { let value = value.trim(); if value.is_empty() { format!("- ⌨️ Prompt: not using a prompt ({})\n", set_where) } else { format!("- ⌨️ Prompt ({}):\n\n{}\n\n", set_where, block_quote(value)) } } pub fn status_text_generation_entry_temperature(value: Option, set_where: &str) -> String { let formatted = match value { Some(value) => format!("`{:.1}` ({})", value, set_where), None => "not set".to_string(), }; format!("- 🌡️ Temperature: {}\n", formatted) } pub fn status_speech_to_text_entry_flow_type( value: SpeechToTextFlowType, set_where: &str, ) -> String { format!("- 🪄 Flow type: `{}` ({})\n", value, set_where) } pub fn status_speech_to_text_entry_msg_type_for_non_threaded_only_transcribed_messages( value: SpeechToTextMessageTypeForNonThreadedOnlyTranscribedMessages, set_where: &str, ) -> String { format!( "- 🪄 Message type for non-threaded only-transcribed messages: `{}` ({})\n", value, set_where ) } pub fn status_speech_to_text_entry_language(value: Option, set_where: &str) -> String { let formatted = match value { Some(value) => format!("`{}` ({})", value, set_where), None => "not set, using auto-detection".to_string(), }; format!("- 🔤 Language: {}\n", formatted) } pub fn status_text_to_speech_entry_bot_msgs_flow_type( value: TextToSpeechBotMessagesFlowType, set_where: &str, ) -> String { format!( "- 🪄 Flow type for bot messages: `{}` ({})\n", value, set_where ) } pub fn status_text_to_speech_entry_user_msgs_flow_type( value: TextToSpeechUserMessagesFlowType, set_where: &str, ) -> String { format!( "- 🪄 Flow type for user messages: `{}` ({})\n", value, set_where ) } pub fn status_text_to_speech_entry_speed(value: Option, set_where: &str) -> String { let formatted = match value { Some(value) => format!("`{:.1}` ({})", value, set_where), None => "not set".to_string(), }; format!("- ⚡ Speed: {}\n", formatted) } pub fn status_text_to_speech_entry_voice(value: Option, set_where: &str) -> String { let formatted = match value { Some(value) => format!("`{}` ({})", value, set_where), None => "not set".to_string(), }; format!("- 👫 Voice: {}\n", formatted) } pub fn status_entry_effective_agent_error() -> String { "- 🤖 Effective handler agent: error determining agent\n".to_string() } pub fn status_entry_effective_agent( value: &PublicIdentifier, source: AgentForPurposeDeterminationInfoConfigurationSource, ) -> String { let set_where = match source { AgentForPurposeDeterminationInfoConfigurationSource::Room => { status_badge_set_in_room_config() } AgentForPurposeDeterminationInfoConfigurationSource::Global => { status_badge_set_in_global_config() } }; format!( "- 🤖 Effective handler agent: `{}` ({})\n", value, set_where ) } pub fn status_badge_set_in_room_config() -> &'static str { "**📍 set in room**" } pub fn status_badge_set_in_global_config() -> &'static str { "**🌐 set globally**" } pub fn status_badge_using_hardcoded_default() -> &'static str { "📝 using hardcoded default" } pub fn status_badge_set_in_agent_config() -> &'static str { "🤖 set at the agent level" } ================================================ FILE: src/strings/error.rs ================================================ pub fn unknown_command_see_help(command_prefix: &str) -> String { format!("Unknown command. See help (`{command_prefix} help`).") } pub fn error_while_processing_message() -> &'static str { "An error occurred while processing your message. Please try again." } pub fn message_is_encrypted() -> &'static str { "This message is encrypted and I cannot decrypt it right now, so I cannot properly serve you." } pub fn first_message_in_thread_is_encrypted() -> &'static str { "The first message in this chat thread is encrypted and I cannot decrypt it right now, so I cannot properly serve you." } ================================================ FILE: src/strings/global_config.rs ================================================ use crate::agent::{AgentPurpose, PublicIdentifier}; pub fn no_permissions_to_administrate() -> &'static str { "You do not have permission to administrate the global config." } pub fn not_allowed_to_use_agent_in_global_config(agent_identifier: &PublicIdentifier) -> String { format!( "The agent `{}` is not allowed to be used in the global configuration.", agent_identifier ) } pub fn global_config_lacks_specific_agent_for_purpose(purpose: AgentPurpose) -> String { format!( "The global configuration does not specify any agent for the `{}` purpose.", purpose ) } pub fn configured_to_use_agent_for_purpose( agent_identifier: &PublicIdentifier, purpose: AgentPurpose, ) -> String { format!( "The global configuration specifies that the `{}` agent is to be used for the `{}` purpose.", agent_identifier, purpose ) } pub fn configures_agent_for_purpose_but_does_not_exist( agent_identifier: &PublicIdentifier, purpose: AgentPurpose, ) -> String { format!( "The global configuration specifies that the `{}` agent is to be used for the `{}` purpose, but such an agent does not exist.", agent_identifier, purpose ) } pub fn reconfigured_to_use_agent_for_purpose( agent_identifier: &PublicIdentifier, purpose: AgentPurpose, ) -> String { format!( "The global configuration has been adjusted to use the `{}` agent for the `{}` purpose.", agent_identifier, purpose ) } pub fn reconfigured_to_not_specify_agent_for_purpose(purpose: AgentPurpose) -> String { format!( "The global configuration has been adjusted to not specify any agent for the `{}` purpose.", purpose ) } pub fn value_was_set_to(value: impl std::fmt::Display) -> String { format!( "This global configuration value was set to:{}", super::cfg::create_display_text_for_value(value) ) } pub fn value_was_unset() -> String { "This global configuration value has been unset.".to_owned() } ================================================ FILE: src/strings/help/access.rs ================================================ pub fn heading() -> String { "🔒 Access".to_owned() } pub fn intro() -> String { "This bot employs access control to decide who can use its services and manage its configuration.".to_string() } pub fn room_auto_join_heading() -> String { "👋 Joining rooms".to_owned() } pub fn room_auto_join_intro() -> String { "The bot automatically joins rooms when invited by someone considered a bot user (see below)." .to_string() } pub fn users_heading() -> String { "👥 Users".to_owned() } pub fn users_intro() -> String { "The bot will ignore messages (and room invitations) from unallowed users.".to_string() } pub fn users_access() -> String { "Users can **use all the bot's features** (text-generation, speech-to-text, etc.), but cannot manage the bot's configuration.".to_string() } pub fn users_command_get(command_prefix: &str) -> String { format!("- **Show** the currently allowed users: `{command_prefix} access users`") } pub fn users_command_set(command_prefix: &str) -> String { format!( "- **Set** the list of allowed users: `{command_prefix} access set-users SPACE_SEPARATED_PATTERNS`" ) } pub fn example_user_patterns(own_server_name: &str) -> String { format!("Example patterns: `@*:{own_server_name} @*:another.com @someone:company.org`") } pub fn administrators_heading() -> String { "👮‍♂️ Administrators".to_owned() } pub fn administrators_intro() -> String { "Administrators can **manage the bot's configuration and access control**.".to_string() } pub fn administrators_now_match_patterns(patterns: &[String]) -> String { format!( "The bot can be administrated by users with a [Matrix user id](https://spec.matrix.org/v1.11/#users) matching the following patterns: `{}`", patterns.join(" "), ) } pub fn administrators_outro() -> String { "Administrators cannot be changed without adjusting the bot's configuration on the server." .to_string() } pub fn room_local_agent_managers_heading() -> String { "💼 Room-local agent managers".to_owned() } pub fn room_local_agent_managers_intro(command_prefix: &str) -> String { format!( "Room-local agent managers are users privileged to **create their own agents** (see `{command_prefix} agent`) in rooms." ) } pub fn room_local_agent_managers_security_warning() -> String { "Letting regular users create agents which contact arbitrary network services **may be a security issue**.".to_string() } pub fn room_local_agent_managers_command_get(command_prefix: &str) -> String { format!( "- **Show** the currently allowed users: `{command_prefix} access room-local-agent-managers`" ) } pub fn room_local_agent_managers_command_set(command_prefix: &str) -> String { format!( "- **Set** the list of allowed users: `{command_prefix} access set-room-local-agent-managers SPACE_SEPARATED_PATTERNS`" ) } ================================================ FILE: src/strings/help/agent.rs ================================================ pub fn heading() -> String { "🤖 Agents".to_owned() } pub fn intro(command_prefix: &str) -> String { format!( "An agent is an instantiation and configuration of some **☁️ provider** (see `{command_prefix} provider`)." ) } pub fn intro_handler_relation(command_prefix: &str) -> String { format!( "Agents can be set as **handlers for various purposes** (text-generation, speech-to-text, etc.) globally or in specific rooms. Send a `{command_prefix} config status` command to see the current configuration." ) } pub fn intro_capabilities() -> String { "It can support different capabilities (text-generation, speech-to-text, etc.) depending on the provider used and on the configuration of the agent.".to_string() } pub fn no_permission_to_create_agents() -> &'static str { "⚠️ You are neither a bot administrator, nor a room-local agent manager, so **you cannot create new agents by yourself**." } pub fn list_agents(command_prefix: &str) -> String { format!("- **List** all available agents: `{command_prefix} agent list`") } pub fn show_agent_details(command_prefix: &str) -> String { format!( "- **Show** full details for a given agent: `{command_prefix} agent details FULL_AGENT_IDENTIFIER`" ) } pub fn create_agent_intro() -> &'static str { "- **Create** a new agent:" } pub fn create_agent_room_local(command_prefix: &str) -> String { format!( "\t- (Accessible in **this room only**) `{command_prefix} agent create-room-local PROVIDER_ID AGENT_ID`" ) } pub fn create_agent_global(command_prefix: &str) -> String { format!( "\t- (Accessible in **all rooms**) `{command_prefix} agent create-global PROVIDER_ID AGENT_ID`" ) } pub fn create_agent_example(command_prefix: &str) -> String { format!("\t- Example: `{command_prefix} agent create-room-local openai my-openai-agent`") } pub fn delete_agent(command_prefix: &str) -> String { format!("- **Delete** an agent: `{command_prefix} agent delete FULL_AGENT_IDENTIFIER`") } pub fn available_commands_outro_update_note() -> &'static str { "To **update** a given agent's configuration: show the agent's **details** (current configuration), then **delete** it and finally **re-create** it." } ================================================ FILE: src/strings/help/cfg.rs ================================================ use crate::agent::AgentPurpose; pub fn heading() -> String { "🛠️ Configuration".to_owned() } pub fn intro_short() -> &'static str { "Various settings for this bot can be configured **📍 per-room** and **🌐 globally**." } pub fn intro_long() -> String { format!( "{}\n\n{}\n{}\n\n{}", intro_short(), "Room-specific configuration values override the global configuration.", "When no configuration values are set, the bot uses hardcoded defaults.", "In commands below, **replace the `CONFIG_TYPE` value** with either `room` (for room-specific configuration) or `global` (for global configuration)." ) } pub fn status_heading() -> String { "📃 Status".to_owned() } pub fn status_intro(command_prefix: &str) -> String { format!( "To **show a summary** of the configuration affecting the current room: `{command_prefix} config status`" ) } pub fn handlers_heading() -> String { "🤖 Handler Agents".to_owned() } pub fn handlers_intro_common() -> String { format!( "{}\n\n{}", "Different messages (text, audio, requests for image generation, etc.) in the room are handled differently and can potentially be served by different agents.", "When no specific agent is configured for a given purpose, the catch-all agent would be used.", ) } pub fn handlers_intro_purposes() -> String { let mut message = String::new(); message.push_str("The following purposes are available:"); message.push('\n'); for purpose in AgentPurpose::choices() { message.push_str(&format!( "\n- {} {}: {}", purpose.emoji(), purpose.as_str(), super::super::agent::purpose_howto(purpose), )); } message } pub fn handlers_show(command_prefix: &str) -> String { format!( "**Show** the currently configured agent for the given purpose: `{command_prefix} config CONFIG_TYPE handler PURPOSE`" ) } pub fn handlers_set(command_prefix: &str) -> String { format!( "**Set** the agent to be used for the given purpose: `{command_prefix} config CONFIG_TYPE set-handler PURPOSE AGENT_ID`" ) } pub fn handlers_unset(command_prefix: &str) -> String { format!( "**Unset** the agent to be used for the given purpose: `{command_prefix} config CONFIG_TYPE set-handler PURPOSE`" ) } pub fn text_generation_heading() -> String { format!( "{} {}", AgentPurpose::TextGeneration.emoji(), AgentPurpose::TextGeneration.heading() ) } pub fn text_generation_common() -> String { let text_generation_description = "Text Generation is the bot's ability to generate text based on the input it receives."; let input_types = format!( "This input may be received directly as text, or as audio (a voice message) transcribed to text by the bot itself (see {} {}).", AgentPurpose::SpeechToText.emoji(), AgentPurpose::SpeechToText.heading() ); format!("{}\n{}", text_generation_description, input_types) } pub fn text_generation_prefix_requirement_type_heading() -> &'static str { "🗟 Prefix Requirement Type" } pub fn text_generation_prefix_requirement_type_intro() -> String { "Controls whether all messages trigger text generation or just those prefixed in a certain way." .to_owned() } pub fn text_generation_prefix_requirement_type_outro(bot_username: &str) -> String { format!( "Regardless of the setting, the bot will always respond to **direct mentions** (e.g. `@{bot_username}`)." ) } pub fn text_generation_auto_usage_heading() -> &'static str { "🪄 Auto usage" } pub fn text_generation_auto_usage_intro() -> String { "Controls how automatic text-generation functions.".to_owned() } pub fn text_generation_context_management_heading() -> &'static str { "♻️ Context Management" } pub fn text_generation_context_management_intro() -> String { format!( "{}\n{}", "Controls the bot's ability to **intelligently drop old messages from the conversation context** when it gets too large.", "This feature relies on [tokenization](https://en.wikipedia.org/wiki/Large_language_model#Tokenization) performed by the [tiktoken-rs](https://github.com/zurawiki/tiktoken-rs) library which is [poorly well-maintained](https://github.com/zurawiki/tiktoken-rs/issues/50) and only works well for [OpenAI](./providers.md#openai) models.", ) } pub fn text_generation_sender_context_heading() -> &'static str { "👤 Sender Context Mode" } pub fn text_generation_sender_context_intro() -> String { format!( "{}\n{}", "Controls whether the bot attaches sender information to conversation messages before sending them to the model.", "`disabled` leaves messages unchanged, `matrix_user_id` adds `[sender=@alice:example.com]`, and `matrix_user_id_and_timestamp` adds `[sender=@alice:example.com sent_at=2026-03-23T14:30:00Z]`. Enabling this sends Matrix user IDs, and optionally timestamps, to the model provider.", ) } pub fn text_generation_prompt_override_heading() -> &'static str { "⌨️ Prompt Override" } pub fn text_generation_prompt_override_intro() -> String { "Lets you override the [system prompt](https://huggingface.co/docs/transformers/en/tasks/prompting) parameter configured at the agent level.".to_string() } pub fn text_generation_temperature_override_heading() -> &'static str { "🌡️ Temperature Override" } pub fn text_generation_temperature_override_intro() -> String { "Lets you override the [temperature](https://blogs.novita.ai/what-are-large-language-model-settings-temperature-top-p-and-max-tokens/#what-is-llm-temperature) (randomness / creativity) parameter configured at the agent level.".to_string() } pub fn current_setting_show(command_prefix: &str, setting_path_parts: &str) -> String { format!( "**Show** the current setting: `{command_prefix} config CONFIG_TYPE {setting_path_parts}`" ) } pub fn current_setting_set(command_prefix: &str, setting_path_parts: &str) -> String { format!("**Set**: `{command_prefix} config CONFIG_TYPE {setting_path_parts}`") } pub fn current_setting_unset(command_prefix: &str, setting_path_parts: &str) -> String { format!("**Unset**: `{command_prefix} config CONFIG_TYPE {setting_path_parts}`") } pub fn the_following_configuration_values_are_recognized( values: Vec, ) -> String { let values_with_backticks = values .iter() .map(|v| format!("`{}`", v)) .collect::>(); format!( "The following configuration values are recognized: {}", values_with_backticks.join(", ") ) } pub fn speech_to_text_heading() -> String { format!( "{} {}", AgentPurpose::SpeechToText.emoji(), AgentPurpose::SpeechToText.heading() ) } pub fn speech_to_text_common() -> String { let intro = "Speech-to-Text is the bot's ability to **turn audio (voice) messages into text**."; let text_gen = format!( "The generated text can be used for {} {}, or not (transcription only).", AgentPurpose::TextGeneration.emoji(), AgentPurpose::TextGeneration.heading() ); let text_to_speech = format!( "The bot may also turn the generated text response back into a voice message (see {} {}).", AgentPurpose::TextToSpeech.emoji(), AgentPurpose::TextToSpeech.heading() ); format!("{}\n{}\n{}", intro, text_gen, text_to_speech) } pub fn speech_to_text_flow_type_heading() -> &'static str { "🪄 Flow Type" } pub fn speech_to_text_flow_type_intro() -> &'static str { "Controls how voice messages are handled." } pub fn speech_to_text_msg_type_for_non_threaded_only_transcribed_messages_heading() -> &'static str { "🪄 Message Type for non-threaded only-transcribed messages" } pub fn speech_to_text_msg_type_for_non_threaded_only_transcribed_messages_intro() -> &'static str { "Controls how the transcribed text of voice messages is sent to the chat when Flow Type = `only_transcribe`." } pub fn speech_to_text_language_heading() -> &'static str { "🔤 Language" } pub fn speech_to_text_language_intro() -> &'static str { "Lets you specify the language of the input voice messages, to avoid using auto-detection.\nSupplying the input language using a 2-letter code (e.g. `ja`) as per [ISO-639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) may improve accuracy & latency.\n\nIf different users are using different languages, do not specify a language." } pub fn text_to_speech_heading() -> String { format!( "{} {}", AgentPurpose::TextToSpeech.emoji(), AgentPurpose::TextToSpeech.heading() ) } pub fn text_to_speech_common() -> &'static str { "Text-to-Speech is the bot's ability to **turn text messages into voice messages**." } pub fn text_to_speech_bot_msgs_flow_type_heading() -> &'static str { "🪄 Bot Messages Flow Type" } pub fn text_to_speech_bot_msgs_flow_type_intro() -> String { "Controls how automatic text-to-speech functions for **messages sent by the bot**.".to_owned() } pub fn text_to_speech_user_msgs_flow_type_heading() -> &'static str { "🪄 User Messages Flow Type" } pub fn text_to_speech_user_msgs_flow_type_intro() -> String { "Controls how automatic text-to-speech functions for **messages sent by users**.\n**Only works when automatic text-generation is disabled** (see Text Generation / Auto usage).".to_owned() } pub fn text_to_speech_speed_override_heading() -> &'static str { "🗲 Speed override" } pub fn text_to_speech_speed_override_intro() -> String { format!( "{}\n{}", "Lets you speed up/down speech relative to the default speed (`1.0` when unset).", "Values typically range from `0.25` to `4.0`, but may vary depending on the selected model.", ) } pub fn text_to_speech_voice_override_heading() -> &'static str { "👫 Voice override" } pub fn text_to_speech_voice_override_intro() -> String { format!( "{}\n\n{}", "Lets you change the default voice configured in the agent configuration.", "Possible values (e.g. `onyx`) depend on the model you're using. For example, for OpenAI's Whisper model, [these voices](https://platform.openai.com/docs/guides/text-to-speech/voice-options) are available.", ) } pub fn image_generation_heading() -> String { format!( "{} {}", AgentPurpose::ImageGeneration.emoji(), AgentPurpose::ImageGeneration.heading() ) } pub fn image_generation_common() -> &'static str { "Image-generation is the bot's ability to **generate images** based on text prompts.\n\nThis feature is not configurable at the moment." } ================================================ FILE: src/strings/help/mod.rs ================================================ pub mod access; pub mod agent; pub mod cfg; pub mod provider; pub mod usage; pub fn heading_introduction() -> String { "👋 Introduction".to_owned() } pub fn available_commands_intro() -> &'static str { "You can run the following commands:" } pub fn learn_more_send_a_command(command_prefix: &str, command_parts: &str) -> String { format!("To learn more, send a `{command_prefix} {command_parts}` command.") } ================================================ FILE: src/strings/help/provider.rs ================================================ pub fn heading() -> String { "☁️ Providers".to_owned() } pub fn intro() -> String { "Agents are powered by a provider. The provider could be powered by a local service or a cloud service.".to_string() } ================================================ FILE: src/strings/help/usage.rs ================================================ pub fn heading() -> &'static str { "📖 Usage" } pub fn intro() -> &'static str { "The bot can perform various tasks, such as 💬 Text Generation, 🗣️ Text-to-Speech, 🦻 Speech-to-Text, 🖌️ Image Generation, and more." } ================================================ FILE: src/strings/image_edit.rs ================================================ pub fn guide_how_to_proceed() -> String { let mut message = String::new(); message.push_str("💡 Respond in this thread (in any order) with:\n"); message.push_str("- one or more images: to use the given images for creating an edit\n"); message.push_str("- more messages: to expand on your original prompt\n"); message.push_str("- a message saying `go`: to generate an edit with the current prompt\n"); message.push_str( "- a message saying `again`: to generate one more image edit with the current prompt\n", ); message } ================================================ FILE: src/strings/image_generation.rs ================================================ pub fn revised_prompt(prompt: &str) -> String { format!("💭 Revised prompt to: {}", prompt) } pub fn guide_how_to_proceed() -> String { let mut message = String::new(); message.push_str("💡 Respond in this thread with:\n"); message.push_str("- more messages: to expand on your original prompt\n"); message.push_str( "- a message saying `again`: to generate one more image with the current prompt\n", ); message } ================================================ FILE: src/strings/introduction.rs ================================================ use crate::agent::utils::AgentForPurposeDeterminationError; use crate::agent::utils::get_effective_agent_for_purpose; use crate::agent::{AgentPurpose, Manager as AgentManager}; use crate::entity::RoomConfigContext; use crate::entity::roomconfig::TextGenerationPrefixRequirementType; fn hello() -> &'static str { "Hello! 👋" } pub fn its_me(name: &str) -> String { let mut message = format!( "I'm {name} - a bot exposing the power of [AI](https://en.wikipedia.org/wiki/Artificial_intelligence) ([Large Language Models](https://en.wikipedia.org/wiki/Large_language_model)) to you. 🤖" ); if name == crate::entity::cfg::defaults::name() { message.push('\n'); message.push_str("My name is pronounced 'bye' and is a play on [AI](https://en.wikipedia.org/wiki/Artificial_intelligence), referencing the fictional character [🇧🇬 Bai Ganyo](https://en.wikipedia.org/wiki/Bay_Ganyo)."); } message } fn purposes_intro() -> &'static str { "I can typically be used for the following purposes:" } pub async fn create_on_join_introduction( name: &str, command_prefix: &str, agent_manager: &AgentManager, room_config_context: &RoomConfigContext, ) -> String { let mut message = String::new(); message.push_str(hello()); message.push_str("\n\n"); message.push_str(&create_short_introduction(name)); message.push_str("\n\n"); let mut got_text_generation_agent = false; message.push_str(purposes_intro()); for purpose in AgentPurpose::choices() { if *purpose == AgentPurpose::CatchAll { continue; } let mut purpose_intro_line = format!( "\n- {} {}: {}", purpose.emoji(), purpose.as_str(), super::agent::purpose_howto(purpose), ); let agent_info = get_effective_agent_for_purpose(agent_manager, room_config_context, *purpose).await; let current_status_text = match agent_info { Ok(agent_info) => { let agent_instance = agent_info.instance; let provider_info = agent_instance.definition().provider.info(); if *purpose == AgentPurpose::TextGeneration { got_text_generation_agent = true; } let provider_display = match provider_info.homepage_url { Some(url) => format!("[{}]({})", provider_info.name, url), None => provider_info.name.to_owned(), }; format!( "✅ enabled via the `{}` agent, powered by the {} provider", agent_instance.identifier(), provider_display, ) } Err(err) => match err { AgentForPurposeDeterminationError::Unknown(err) => { crate::utils::status::create_error_message_text(&err).to_owned() } AgentForPurposeDeterminationError::NoneConfigured => { "❌ no agent configured".to_string() } AgentForPurposeDeterminationError::ConfiguredButMissing(agent_identifier) => { format!("❌ configured via `{agent_identifier}`, but the agent is missing") } AgentForPurposeDeterminationError::ConfiguredButLacksSupport(agent_identifier) => { format!("❌ configured via `{agent_identifier}`, but support is missing") } }, }; purpose_intro_line.push_str(&format!(" ({})", current_status_text)); message.push_str(&purpose_intro_line); } message.push_str("\n\n"); if got_text_generation_agent { message.push_str(&make_use_of_me_simply_send_a_message( command_prefix, room_config_context.text_generation_prefix_requirement_type(), )); } else { message.push_str(&make_use_of_me_agent_creation( command_prefix, room_config_context.text_generation_prefix_requirement_type(), )); } message } pub fn create_short_introduction(name: &str) -> String { its_me(name) } fn make_use_of_me_simply_send_a_message( command_prefix: &str, prefix_requirement_type: TextGenerationPrefixRequirementType, ) -> String { let message = r#"**To make use of me**: 1. 👋 %send_a_message% 2. 📖 %learn_more% "#; message .replace("%command_prefix%", command_prefix) .replace( "%send_a_message%", &send_a_text_message(command_prefix, prefix_requirement_type), ) .replace( "%learn_more%", &learn_more_from_usage_or_help(command_prefix), ) } fn make_use_of_me_agent_creation( command_prefix: &str, prefix_requirement_type: TextGenerationPrefixRequirementType, ) -> String { let message = r#"**To make use of me**: 1. ☁️ **Choose an agent provider** (e.g. OpenAI, Mistral, etc). Send a `%command_prefix% provider` command to see the list. 2. 🤖 %create_one_or_more_agents% 3. 🤝 %set_new_agent_as_handler% 4. 👋 %send_a_message% 5. 📖 %learn_more% "#; message .replace("%command_prefix%", command_prefix) .replace( "%send_a_message%", &send_a_text_message(command_prefix, prefix_requirement_type), ) .replace( "%learn_more%", &learn_more_from_usage_or_help(command_prefix), ) .replace( "%create_one_or_more_agents%", &create_one_or_more_agents(command_prefix), ) .replace( "%set_new_agent_as_handler%", &set_new_agent_as_handler(command_prefix), ) } fn send_a_text_message( command_prefix: &str, prefix_requirement_type: TextGenerationPrefixRequirementType, ) -> String { match prefix_requirement_type { TextGenerationPrefixRequirementType::No => { "**Send a text message** in this room (e.g. `Hello!`) and see me reply.".to_owned() } TextGenerationPrefixRequirementType::CommandPrefix => { format!( "In this room, I'm configured to require the command prefix (`{command_prefix}`) for text messages. **Send a prefixed text message** (e.g. `{command_prefix} Hello!`) and see me reply." ) } } } fn learn_more_from_usage_or_help(command_prefix: &str) -> String { format!( "**Learn more** by sending a `{command_prefix} usage` or `{command_prefix} help` command." ) } pub fn create_one_or_more_agents(command_prefix: &str) -> String { format!( "**Create one or more agents** in this room or globally. The provider help message will show you **🗲 Quick start** commands, but you may also send a `{command_prefix} agent` command to see the guide." ) } pub fn set_new_agent_as_handler(command_prefix: &str) -> String { format!( "**Set the new agent as a handler** for a given use-purpose like text-generation, image-generation, etc. The agent-creation wizard will tell you how, but you may also send a `{command_prefix} config` command to see the guide (in the 🤖 *Handler Agents* section)." ) } ================================================ FILE: src/strings/mod.rs ================================================ pub mod access; pub mod agent; pub mod cfg; pub mod error; pub mod global_config; pub mod help; pub mod image_edit; pub mod image_generation; pub mod introduction; pub mod provider; pub mod room_config; pub mod speech_to_text; pub mod text_to_speech; pub mod usage; pub const PROGRESS_INDICATOR_EMOJI: &str = "⏳"; pub fn the_following_commands_are_available() -> &'static str { "The following commands are available:" } ================================================ FILE: src/strings/provider.rs ================================================ use crate::agent::AgentInstantiationError; use crate::agent::AgentProvider; use crate::agent::AgentProviderInfo; use crate::agent::AgentPurpose; pub fn invalid(provider: &str) -> String { let choices_string = AgentProvider::choices() .iter() .map(|choice| format!("`{}`", choice.to_static_str(),)) .collect::>() .join(", "); format!( "`{}` is not a valid provider choice. Valid choices are: {}", provider, choices_string ) } pub fn invalid_configuration_for_provider( provider: &AgentProvider, err: AgentInstantiationError, ) -> String { format!( "The provided configuration is not valid for the `{}` provider:\n```\n{:?}\n```", provider, err ) } pub fn providers_list_intro() -> String { "The list of supported providers is below.".to_owned() } pub fn help_how_to_choose_heading() -> String { "How to choose a provider".to_string() } pub fn help_how_to_choose_description(command_prefix: &str) -> String { let str = r#" If you're not sure which provider to start with, **we recommend OpenAI** as it's the most popular and has the **widest range of capabilities**. You don't need to choose just one though. The bot supports **mixing & matching models** (by setting different handlers for different types of messages - see `%command_prefix% config`), so you can use multiple providers at the same time. "#; str.replace("%command_prefix%", command_prefix) .trim() .to_owned() } pub fn help_how_to_use_heading() -> String { "How to use a provider".to_string() } pub fn help_how_to_use_description(command_prefix: &str) -> String { let str = r#" 1. 📝 **Sign up for it** 2. 🔑 **Obtain an API key** 3. 🤖 %create_one_or_more_agents% 4. 🤝 %set_new_agent_as_handler% "#; str.replace("%command_prefix%", command_prefix) .replace( "%create_one_or_more_agents%", &super::introduction::create_one_or_more_agents(command_prefix), ) .replace( "%set_new_agent_as_handler%", &super::introduction::set_new_agent_as_handler(command_prefix), ) .trim() .to_owned() } pub fn help_provider_heading(provider_name: &str, homepage_url: &Option) -> String { match homepage_url { Some(url) => format!("[{}]({})", provider_name, url), None => provider_name.to_owned(), } } pub fn help_provider_details(id: &str, info: &AgentProviderInfo) -> String { let mut message = String::new(); message.push_str(info.description.trim()); message.push_str("\n\n"); message.push_str(&format!("- 🆔 Identifier: `{}`\n", id)); let mut links = Vec::new(); if let Some(url) = info.homepage_url { links.push(format!("[🏠 Home page]({})", url)); } if let Some(url) = info.wiki_url { links.push(format!("[🌐 Wiki]({})", url)); } if let Some(url) = info.sign_up_url { links.push(format!("[👤 Sign up]({})", url)); } if let Some(url) = info.models_list_url { links.push(format!("[📋 Models list]({})", url)); } if !links.is_empty() { message.push_str(&format!("- 🔗 Links: {}\n", links.join(", "))); } let mut capabilities = vec![]; for purpose in info.supported_purposes.iter() { let mut purpose_line = format!("{} {}", purpose.emoji(), purpose.as_str()); if let AgentPurpose::TextGeneration = purpose { let mut extras = vec![]; if info.text_generation_supports_vision { extras.push("incl. vision"); } else { extras.push("no vision"); } if info.text_generation_supports_tools { extras.push("incl. tools"); } else { extras.push("no tools"); } purpose_line = format!("{} ({})", purpose_line, extras.join(", ")); } capabilities.push(purpose_line); } message.push_str(&format!("- 🌟 Capabilities: {}\n", capabilities.join(", "))); message } ================================================ FILE: src/strings/room_config.rs ================================================ use crate::agent::{AgentPurpose, PublicIdentifier}; pub fn room_not_configured_with_specific_agent_for_purpose(purpose: AgentPurpose) -> String { format!( "This room is not configured to use any specific agent for the `{}` purpose.", purpose ) } pub fn configured_to_use_agent_for_purpose( agent_identifier: &PublicIdentifier, purpose: AgentPurpose, ) -> String { format!( "This room is configured to use the `{agent_identifier}` agent for the `{purpose}` purpose.", ) } pub fn configures_agent_for_purpose_but_does_not_exist( agent_identifier: &PublicIdentifier, purpose: AgentPurpose, ) -> String { format!( "This room is configured to use the `{agent_identifier}` agent for the `{purpose}` purpose, but such an agent does not exist.", ) } pub fn configures_agent_for_purpose_but_agent_does_not_support_it( agent_identifier: &PublicIdentifier, purpose: AgentPurpose, ) -> String { format!( "This room is configured to use the `{}` agent for {} (either directly, or through a {} fallback), but this agent does not support being used for {}.", agent_identifier, purpose, AgentPurpose::CatchAll, purpose, ) } pub fn reconfigured_to_use_agent_for_purpose( agent_identifier: &PublicIdentifier, purpose: AgentPurpose, ) -> String { format!( "This room has been reconfigured to use the `{}` agent for the `{}` purpose.", agent_identifier, purpose ) } pub fn reconfigured_to_not_specify_agent_for_purpose(purpose: AgentPurpose) -> String { format!( "This room has been reconfigured to not specify any agent for the `{}` purpose.", purpose ) } pub fn value_was_set_to(value: impl std::fmt::Display) -> String { format!( "This room-specific configuration value was set to:{}", super::cfg::create_display_text_for_value(value) ) } pub fn value_was_unset() -> String { "This room-specific configuration value has been unset.".to_owned() } ================================================ FILE: src/strings/speech_to_text.rs ================================================ pub fn redaction_reason_done() -> &'static str { "Done transcribing" } pub fn redaction_reason_failed() -> &'static str { "Failed while transcribing" } pub fn language_code_invalid(value: &str) -> String { format!( "The value `{}` is not a valid 2-letter language code as per [ISO 639-1](https://en.wikipedia.org/wiki/List_of_ISO_639_language_codes).", value ) } ================================================ FILE: src/strings/text_to_speech.rs ================================================ pub fn redaction_reason_done() -> &'static str { "Done with speech-to-text" } pub fn redaction_reason_failed() -> &'static str { "Failed while doing speech-to-text" } ================================================ FILE: src/strings/usage.rs ================================================ pub fn intro(command_prefix: &str) -> String { let message = r#" ## 📖 Usage To get an **overview of the current configuration affecting this room**, send a `%command_prefix% config status` command. To **adjust settings**, see `%command_prefix% config`. ### 💬 Text Generation If there's a text-generation handler agent configured (see `%command_prefix% config status`), the bot **may** respond to messages sent in the room. Whether the bot responds depends on the **💬 Text Generation / 🗟 Prefix Requirement** setting (see `%command_prefix% config status`). Sometimes, a prefix (e.g. `%command_prefix%`) is required in front of messages sent to the room for the bot to respond. For multi-user rooms, this setting defaults to "required". Room messages start a threaded conversation where you can continue back-and-forth communication with the bot. ### 🗣️ Text-to-Speech If there's a text-to-speech handler agent configured (see `%command_prefix% config status`), the bot **may** convert text messages sent to the room to audio (voice). By default, the bot will offer text-to-speech for its own messages which are a response to voice messages coming from you. Simply click the 🗣️ reaction on the bot's message and the bot will convert the text message to audio. This is configurable via the **🗣️ Text-to-Speech / 🪄 Bot Messages Flow Type** setting. The bot may be configured to also turn your own text messages to audio (voice) via the **🗣️ Text-to-Speech / 🪄 User Messages Flow Type** setting. ### 🦻 Speech-to-Text If there's a speech-to-text handler agent configured (see `%command_prefix% config status`), the bot **may** transcribe voice messages sent to the room to text. By default, the bot will also perform 💬 Text Generation on the text. This is configurable via the **🦻 Speech-to-Text / 🪄 Flow Type** setting. If all your messages are in the same language, you can improve accuracy & latency by configuring the language via the **🦻 Speech-to-Text / 🔤 Language** setting. ### Image Generation #### 🖌️ Creating images Simply send a command like `%command_prefix% image create A beautiful sunset over the ocean` and the bot will start a threaded conversation and post an image based on your prompt. You can then respond in the same message thread with: - more messages, to add more criteria to your prompt. - a message saying `again`, to generate one more image with the current prompt. #### 🎨 Editing images Simply send a command like `%command_prefix% image edit Turn the following image into an anime-style drawing` and the bot will start a threaded conversation asking for more details. You can then respond in the same message thread with: - more messages, to add more criteria to your prompt. - one or more images, to provide the images that the bot will operate on. - a message saying `go`, to start the image generation process. - a message saying `again`, to prompt the bot to generate one more image edit with the current prompt. #### 🫵 Creating stickers A variation of **creating images** is creating "sticker images". To create a sticker, send a command like `%command_prefix% sticker A huge bowl of steaming ramen with a mountain of beansprouts on top`. The difference from **creating images** is that the bot will: - create a smaller-resolution image (as small as the model allows) - smaller/quicker, but still good enough for a sticker - potentially switch to a different (cheaper or otherwise more suitable) model, if available - post the image directly to the room (as a reply to your message), without starting a threaded conversation "#; message.replace("%command_prefix%", command_prefix) } ================================================ FILE: src/utils/base64.rs ================================================ use base64::{Engine as _, engine::general_purpose::STANDARD}; pub(crate) fn base64_decode(base64_string: &str) -> Result, base64::DecodeError> { STANDARD.decode(base64_string) } pub(crate) fn base64_encode(data: &[u8]) -> String { STANDARD.encode(data) } ================================================ FILE: src/utils/mime.rs ================================================ use mxlink::mime; pub fn get_file_extension(mime_type: &mime::Mime) -> String { match (mime_type.type_(), mime_type.subtype()) { (mime::AUDIO, mime::BASIC) => "au", (mime::AUDIO, mime::MPEG) => "mp3", (mime::AUDIO, mime::MP4) => "m4a", (mime::AUDIO, mime::OGG) => "ogg", (mime::IMAGE, mime::BMP) => "bmp", (mime::IMAGE, mime::GIF) => "gif", (mime::IMAGE, mime::JPEG) => "jpg", (mime::IMAGE, mime::PNG) => "png", (mime::IMAGE, mime::SVG) => "svg", _ => "bin", } .to_string() } pub fn get_mime_type_from_file_name(file_name: &str) -> mime::Mime { mime_guess::from_path(file_name) .first() .unwrap_or(mime::APPLICATION_OCTET_STREAM) } ================================================ FILE: src/utils/mod.rs ================================================ pub(crate) mod base64; pub(crate) mod mime; pub mod status; pub mod text; pub mod text_to_speech; ================================================ FILE: src/utils/status.rs ================================================ pub fn create_error_message_text(text: &str) -> String { format!("⚠️ Error: {}", text) } pub fn create_success_message_text(text: &str) -> String { format!("✅ {}", text) } pub fn create_tooltip_message_text(text: &str) -> String { format!("💡 {}", text) } ================================================ FILE: src/utils/text.rs ================================================ pub fn block_quote(text: &str) -> String { text.lines() .map(|line| format!("> {}", line)) .collect::>() .join("\n") } pub fn block_unquote(text: &str) -> String { text.lines() .map(|line| { if let Some(stripped) = line.strip_prefix("> ") { stripped.to_string() } else { line.to_string() } }) .collect::>() .join("\n") } ================================================ FILE: src/utils/text_to_speech.rs ================================================ use crate::agent::AgentPurpose; use super::text::{block_quote, block_unquote}; /// Creates a text message which is based on transcribed audio. /// This text message is prefixed with an emoji and blockquoted, to indicate that it is a transcription. /// To reverse the process, use `parse_transcribed_message_text()`. /// /// It should be noted that in certain cases (Transcribe-only mode), transcriptions are posted as regular notice messages which do not include /// the `> 🦻` prefixing. That is, not every transcribed message will pass through here (intentionally). pub fn create_transcribed_message_text(text: &str) -> String { block_quote(&format!("{} {}", AgentPurpose::SpeechToText.emoji(), text)) } /// Parses a transcribed message text, reversing the process done by `create_transcribed_message_text()`. /// If the provided text string does not match the expected format, None is returned. /// /// It should be noted that in certain cases (Transcribe-only mode), transcriptions are posted as regular notice messages which do not include /// the `> 🦻` prefix. This function will not handle these properly. pub fn parse_transcribed_message_text(text: &str) -> Option { if !text.starts_with("> ") { return None; } let unquoted = block_unquote(text); let emoji_prefix = format!("{} ", AgentPurpose::SpeechToText.emoji()); if let Some(original) = unquoted.strip_prefix(&emoji_prefix) { return Some(original.to_string()); } None } pub mod test { #[test] fn test_transcribed_message_text_creation() { let text = "Hello there!\nHow are you?"; let expected = format!( "> {} Hello there!\n> How are you?", crate::agent::AgentPurpose::SpeechToText.emoji() ); assert_eq!(expected, super::create_transcribed_message_text(text)); } #[test] fn test_transcribed_message_text_parsing() { // All good let text = format!( "> {} Hello there!\n> How are you?", crate::agent::AgentPurpose::SpeechToText.emoji() ); let expected = "Hello there!\nHow are you?"; assert_eq!( Some(expected.to_owned()), super::parse_transcribed_message_text(&text) ); // No blockquote let text = format!( "{} Hello there!\nHow are you?", crate::agent::AgentPurpose::SpeechToText.emoji() ); assert_eq!(None, super::parse_transcribed_message_text(&text)); // No emoji let text = "> Hello there!\n> How are you?"; assert_eq!(None, super::parse_transcribed_message_text(text)); // Different emoji let text = "> 🌸 Hello there!\n> How are you?"; assert_eq!(None, super::parse_transcribed_message_text(text)); } }