Showing preview only (2,730K chars total). Download the full file or copy to clipboard to get everything.
Repository: yichuan-w/LEANN
Branch: main
Commit: 3cab4ef40dcf
Files: 221
Total size: 2.6 MB
Directory structure:
gitextract_xaekqrq1/
├── .devcontainer/
│ └── devcontainer.json
├── .github/
│ ├── ISSUE_TEMPLATE/
│ │ ├── bug_report.yml
│ │ ├── config.yml
│ │ └── feature_request.yml
│ ├── pull_request_template.md
│ └── workflows/
│ ├── build-and-publish.yml
│ ├── build-reusable.yml
│ ├── link-check.yml
│ └── release-manual.yml
├── .gitignore
├── .gitmodules
├── .pre-commit-config.yaml
├── .python-version
├── .vscode/
│ ├── extensions.json
│ └── settings.json
├── CLAUDE.md
├── LICENSE
├── README.md
├── apps/
│ ├── __init__.py
│ ├── base_rag_example.py
│ ├── browser_rag.py
│ ├── chatgpt_data/
│ │ ├── __init__.py
│ │ └── chatgpt_reader.py
│ ├── chatgpt_rag.py
│ ├── chunking/
│ │ └── __init__.py
│ ├── claude_data/
│ │ ├── __init__.py
│ │ └── claude_reader.py
│ ├── claude_rag.py
│ ├── code_rag.py
│ ├── colqwen_rag.py
│ ├── document_rag.py
│ ├── email_data/
│ │ ├── LEANN_email_reader.py
│ │ └── email.py
│ ├── email_rag.py
│ ├── gemini_data/
│ │ ├── __init__.py
│ │ └── gemini_reader.py
│ ├── gemini_rag.py
│ ├── history_data/
│ │ ├── __init__.py
│ │ ├── history.py
│ │ └── wechat_history.py
│ ├── image_rag.py
│ ├── imessage_data/
│ │ ├── __init__.py
│ │ └── imessage_reader.py
│ ├── imessage_rag.py
│ ├── multimodal/
│ │ └── vision-based-pdf-multi-vector/
│ │ ├── README.md
│ │ ├── colqwen_forward.py
│ │ ├── leann_multi_vector.py
│ │ ├── multi-vector-leann-paper-example.py
│ │ ├── multi-vector-leann-similarity-map.py
│ │ ├── vidore_v1_benchmark.py
│ │ └── vidore_v2_benchmark.py
│ ├── qwen_data/
│ │ ├── __init__.py
│ │ └── qwen_reader.py
│ ├── qwen_rag.py
│ ├── semantic_file_search/
│ │ ├── leann-plus-temporal-search.py
│ │ ├── leann_index_builder.py
│ │ └── spotlight_index_dump.py
│ ├── slack_data/
│ │ ├── __init__.py
│ │ └── slack_mcp_reader.py
│ ├── slack_rag.py
│ ├── twitter_data/
│ │ ├── __init__.py
│ │ └── twitter_mcp_reader.py
│ ├── twitter_rag.py
│ └── wechat_rag.py
├── benchmarks/
│ ├── README.md
│ ├── __init__.py
│ ├── benchmark_embeddings.py
│ ├── benchmark_no_recompute.py
│ ├── bm25_diskann_baselines/
│ │ ├── README.md
│ │ ├── run_bm25.py
│ │ └── run_diskann.py
│ ├── compare_faiss_vs_leann.py
│ ├── diskann_vs_hnsw_speed_comparison.py
│ ├── enron_emails/
│ │ ├── README.md
│ │ ├── data/
│ │ │ └── .gitignore
│ │ ├── evaluate_enron_emails.py
│ │ └── setup_enron_emails.py
│ ├── faiss_only.py
│ ├── financebench/
│ │ ├── README.md
│ │ ├── evaluate_financebench.py
│ │ ├── setup_financebench.py
│ │ └── verify_recall.py
│ ├── issue_159.py
│ ├── laion/
│ │ ├── .gitignore
│ │ ├── README.md
│ │ ├── evaluate_laion.py
│ │ └── setup_laion.py
│ ├── llm_utils.py
│ ├── micro_tpt.py
│ ├── run_evaluation.py
│ ├── simple_mac_tpt_test.py
│ └── update/
│ ├── README.md
│ ├── __init__.py
│ ├── bench_hnsw_rng_recompute.py
│ ├── bench_update_vs_offline_search.py
│ └── plot_bench_results.py
├── data/
│ ├── PrideandPrejudice.txt
│ └── huawei_pangu.md
├── demo.ipynb
├── docker/
│ ├── Dockerfile
│ ├── Dockerfile.cpu
│ ├── Dockerfile.dev
│ └── README.md
├── docs/
│ ├── COLQWEN_GUIDE.md
│ ├── CONTRIBUTING.md
│ ├── RELEASE.md
│ ├── THINKING_BUDGET_FEATURE.md
│ ├── ast_chunking_guide.md
│ ├── code/
│ │ └── embedding_model_compare.py
│ ├── configuration-guide.md
│ ├── faq.md
│ ├── features.md
│ ├── grep_search.md
│ ├── metadata_filtering.md
│ ├── normalized_embeddings.md
│ ├── openclaw-setup.md
│ ├── react_agent.md
│ ├── roadmap.md
│ ├── slack-setup-guide.md
│ └── ultimate_goal.md
├── examples/
│ ├── __init__.py
│ ├── basic_demo.py
│ ├── dynamic_update_no_recompute.py
│ ├── grep_search_example.py
│ ├── mcp_integration_demo.py
│ ├── mlx_demo.py
│ └── spoiler_free_book_rag.py
├── llms.txt
├── packages/
│ ├── __init__.py
│ ├── leann/
│ │ ├── README.md
│ │ ├── __init__.py
│ │ └── pyproject.toml
│ ├── leann-backend-diskann/
│ │ ├── __init__.py
│ │ ├── leann_backend_diskann/
│ │ │ ├── __init__.py
│ │ │ ├── diskann_backend.py
│ │ │ ├── diskann_embedding_server.py
│ │ │ ├── embedding_pb2.py
│ │ │ └── graph_partition.py
│ │ ├── pyproject.toml
│ │ └── third_party/
│ │ ├── embedding.pb.cc
│ │ └── embedding.proto
│ ├── leann-backend-hnsw/
│ │ ├── CMakeLists.txt
│ │ ├── leann_backend_hnsw/
│ │ │ ├── __init__.py
│ │ │ ├── convert_to_csr.py
│ │ │ ├── hnsw_backend.py
│ │ │ └── hnsw_embedding_server.py
│ │ └── pyproject.toml
│ ├── leann-backend-ivf/
│ │ ├── README.md
│ │ ├── leann_backend_ivf/
│ │ │ ├── __init__.py
│ │ │ └── ivf_backend.py
│ │ └── pyproject.toml
│ ├── leann-core/
│ │ ├── pyproject.toml
│ │ └── src/
│ │ └── leann/
│ │ ├── __init__.py
│ │ ├── api.py
│ │ ├── chat.py
│ │ ├── chunking_utils.py
│ │ ├── cli.py
│ │ ├── embedding_compute.py
│ │ ├── embedding_server_manager.py
│ │ ├── interactive_utils.py
│ │ ├── interface.py
│ │ ├── mcp.py
│ │ ├── metadata_filter.py
│ │ ├── react_agent.py
│ │ ├── registry.py
│ │ ├── searcher_base.py
│ │ ├── server.py
│ │ ├── settings.py
│ │ └── sync.py
│ ├── leann-mcp/
│ │ └── README.md
│ └── wechat-exporter/
│ ├── __init__.py
│ ├── main.py
│ └── wechattweak-cli
├── pyproject.toml
├── scripts/
│ └── hf_upload.py
├── skills/
│ └── leann-memory/
│ ├── README.md
│ ├── claw.json
│ └── instructions.md
├── sky/
│ └── leann-build.yaml
└── tests/
├── README.md
├── openclaw/
│ ├── .gitignore
│ ├── __init__.py
│ ├── conftest.py
│ ├── docker-compose.yml
│ ├── fixtures/
│ │ ├── MEMORY.md
│ │ └── memory/
│ │ ├── 2026-02-15.md
│ │ ├── 2026-02-20.md
│ │ └── 2026-02-25.md
│ ├── run_docker_test.sh
│ ├── test_build_and_search.py
│ ├── test_mcp_e2e.py
│ ├── test_mcp_protocol.py
│ ├── test_openclaw_e2e.py
│ └── test_skill_manifest.py
├── support/
│ └── fake_embedding_server_module.py
├── test_astchunk_integration.py
├── test_basic.py
├── test_ci_minimal.py
├── test_cli_ask.py
├── test_cli_daemon_workflow.py
├── test_cli_prompt_template.py
├── test_cli_verbosity.py
├── test_cpu_only_install.py
├── test_diskann_partition.py
├── test_document_rag.py
├── test_embedding_prompt_template.py
├── test_embedding_server_cli_flags.py
├── test_embedding_server_manager.py
├── test_embedding_server_manager_e2e.py
├── test_hybrid_search.py
├── test_incremental_build.py
├── test_lmstudio_bridge.py
├── test_mcp_integration.py
├── test_mcp_standalone.py
├── test_metadata_filtering.py
├── test_minimax_provider.py
├── test_prompt_template_e2e.py
├── test_prompt_template_persistence.py
├── test_readme_examples.py
├── test_sync.py
└── test_token_truncation.py
================================================
FILE CONTENTS
================================================
================================================
FILE: .devcontainer/devcontainer.json
================================================
{
"name": "LEANN Dev",
"build": {
"context": "..",
"dockerfile": "../docker/Dockerfile.dev",
"args": {
"PYTHON_VERSION": "3.12"
}
},
"workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}",
"remoteUser": "root",
"overrideCommand": true,
"postCreateCommand": "uv sync --group lint --group test",
"customizations": {
"vscode": {
"extensions": [
"ms-python.python",
"ms-python.vscode-pylance",
"charliermarsh.ruff",
"ms-azuretools.vscode-docker",
"ms-toolsai.jupyter",
"tamasfe.even-better-toml",
"eamodio.gitlens",
"EditorConfig.EditorConfig",
"DavidAnson.vscode-markdownlint"
],
"settings": {
"python.defaultInterpreterPath": "/workspaces/${localWorkspaceFolderBasename}/.venv/bin/python",
"python.terminal.activateEnvironment": true,
"python.testing.pytestEnabled": true,
"python.testing.unittestEnabled": false,
"python.testing.pytestArgs": [
"tests"
],
"editor.formatOnSave": true,
"editor.codeActionsOnSave": {
"source.organizeImports.ruff": "explicit",
"source.fixAll.ruff": "explicit"
}
}
}
}
}
================================================
FILE: .github/ISSUE_TEMPLATE/bug_report.yml
================================================
name: Bug Report
description: Report a bug in LEANN
labels: ["bug"]
body:
- type: textarea
id: description
attributes:
label: What happened?
description: A clear description of the bug
validations:
required: true
- type: textarea
id: reproduce
attributes:
label: How to reproduce
placeholder: |
1. Install with...
2. Run command...
3. See error
validations:
required: true
- type: textarea
id: error
attributes:
label: Error message
description: Paste any error messages
render: shell
- type: input
id: version
attributes:
label: LEANN Version
placeholder: "0.1.0"
validations:
required: true
- type: dropdown
id: os
attributes:
label: Operating System
options:
- macOS
- Linux
- Windows
- Docker
validations:
required: true
================================================
FILE: .github/ISSUE_TEMPLATE/config.yml
================================================
blank_issues_enabled: true
contact_links:
- name: Documentation
url: https://github.com/LEANN-RAG/LEANN-RAG/tree/main/docs
about: Read the docs first
- name: Discussions
url: https://github.com/LEANN-RAG/LEANN-RAG/discussions
about: Ask questions and share ideas
================================================
FILE: .github/ISSUE_TEMPLATE/feature_request.yml
================================================
name: Feature Request
description: Suggest a new feature for LEANN
labels: ["enhancement"]
body:
- type: textarea
id: problem
attributes:
label: What problem does this solve?
description: Describe the problem or need
validations:
required: true
- type: textarea
id: solution
attributes:
label: Proposed solution
description: How would you like this to work?
validations:
required: true
- type: textarea
id: example
attributes:
label: Example usage
description: Show how the API might look
render: python
================================================
FILE: .github/pull_request_template.md
================================================
## What does this PR do?
<!-- Brief description of your changes -->
## Related Issues
Fixes #
## Checklist
- [ ] Tests pass (`uv run pytest`)
- [ ] Code formatted (`ruff format` and `ruff check`)
- [ ] Pre-commit hooks pass (`pre-commit run --all-files`)
================================================
FILE: .github/workflows/build-and-publish.yml
================================================
name: CI
on:
push:
branches: [ main ]
pull_request:
branches: [ main ]
workflow_dispatch:
jobs:
build:
uses: ./.github/workflows/build-reusable.yml
================================================
FILE: .github/workflows/build-reusable.yml
================================================
name: Reusable Build
on:
workflow_call:
inputs:
ref:
description: 'Git ref to build'
required: false
type: string
default: ''
jobs:
lint:
name: Lint and Format Check
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
ref: ${{ inputs.ref }}
submodules: recursive
- name: Install uv and Python
uses: astral-sh/setup-uv@v6
with:
python-version: '3.11'
- name: Run pre-commit with only lint group (no project deps)
run: |
uv run --only-group lint pre-commit run --all-files --show-diff-on-failure
type-check:
name: Type Check with ty
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
ref: ${{ inputs.ref }}
submodules: recursive
- name: Install uv and Python
uses: astral-sh/setup-uv@v6
with:
python-version: '3.11'
- name: Install ty
run: uv tool install ty==0.0.17
- name: Run ty type checker
run: |
# Run ty on core packages, apps, and tests
ty check packages/leann-core/src apps tests
build:
needs: [lint, type-check]
name: Build ${{ matrix.os }} Python ${{ matrix.python }}
defaults:
run:
shell: bash
strategy:
fail-fast: true
matrix:
include:
# Note: Python 3.9 dropped - uses PEP 604 union syntax (str | None)
# which requires Python 3.10+
- os: ubuntu-22.04
python: '3.10'
- os: ubuntu-22.04
python: '3.11'
- os: ubuntu-22.04
python: '3.12'
- os: ubuntu-22.04
python: '3.13'
# ARM64 Linux builds
- os: ubuntu-22.04-arm
python: '3.10'
- os: ubuntu-22.04-arm
python: '3.11'
- os: ubuntu-22.04-arm
python: '3.12'
- os: ubuntu-22.04-arm
python: '3.13'
- os: macos-14
python: '3.10'
- os: macos-14
python: '3.11'
- os: macos-14
python: '3.12'
- os: macos-14
python: '3.13'
- os: macos-15
python: '3.10'
- os: macos-15
python: '3.11'
- os: macos-15
python: '3.12'
- os: macos-15
python: '3.13'
# Intel Mac builds (x86_64) - replaces deprecated macos-13
# Note: Python 3.13 excluded - PyTorch has no wheels for macOS x86_64 + Python 3.13
# (PyTorch <=2.4.1 lacks cp313, PyTorch >=2.5.0 dropped Intel Mac support)
- os: macos-15-intel
python: '3.10'
- os: macos-15-intel
python: '3.11'
- os: macos-15-intel
python: '3.12'
# macOS 26 (beta) - arm64
- os: macos-26
python: '3.10'
- os: macos-26
python: '3.11'
- os: macos-26
python: '3.12'
- os: macos-26
python: '3.13'
# Windows validation (native HNSW + DiskANN build/install path)
- os: windows-2022
python: '3.11'
- os: windows-2022
python: '3.12'
- os: windows-2022
python: '3.13'
- os: windows-2022
python: '3.14'
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v5
with:
ref: ${{ inputs.ref }}
submodules: recursive
- name: Install uv and Python
uses: astral-sh/setup-uv@v6
with:
python-version: ${{ matrix.python }}
- name: Install system dependencies (Ubuntu)
if: runner.os == 'Linux'
run: |
sudo apt-get update
sudo apt-get install -y libomp-dev libboost-all-dev protobuf-compiler libzmq3-dev \
pkg-config libabsl-dev libaio-dev libprotobuf-dev \
patchelf
# Debug: Show system information
echo "🔍 System Information:"
echo "Architecture: $(uname -m)"
echo "OS: $(uname -a)"
echo "CPU info: $(lscpu | head -5)"
# Install math library based on architecture
ARCH=$(uname -m)
echo "🔍 Setting up math library for architecture: $ARCH"
if [[ "$ARCH" == "x86_64" ]]; then
# Install Intel MKL for DiskANN on x86_64
echo "📦 Installing Intel MKL for x86_64..."
wget -q https://registrationcenter-download.intel.com/akdlm/IRC_NAS/79153e0f-74d7-45af-b8c2-258941adf58a/intel-onemkl-2025.0.0.940.sh
sudo sh intel-onemkl-2025.0.0.940.sh -a --components intel.oneapi.lin.mkl.devel --action install --eula accept -s
source /opt/intel/oneapi/setvars.sh
echo "MKLROOT=/opt/intel/oneapi/mkl/latest" >> $GITHUB_ENV
echo "LD_LIBRARY_PATH=/opt/intel/oneapi/compiler/latest/linux/compiler/lib/intel64_lin" >> $GITHUB_ENV
echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/intel/oneapi/mkl/latest/lib/intel64" >> $GITHUB_ENV
echo "✅ Intel MKL installed for x86_64"
# Debug: Check MKL installation
echo "🔍 MKL Installation Check:"
ls -la /opt/intel/oneapi/mkl/latest/ || echo "MKL directory not found"
ls -la /opt/intel/oneapi/mkl/latest/lib/ || echo "MKL lib directory not found"
elif [[ "$ARCH" == "aarch64" ]]; then
# Use OpenBLAS for ARM64 (MKL installer not compatible with ARM64)
echo "📦 Installing OpenBLAS for ARM64..."
sudo apt-get install -y libopenblas-dev liblapack-dev liblapacke-dev
echo "✅ OpenBLAS installed for ARM64"
# Debug: Check OpenBLAS installation
echo "🔍 OpenBLAS Installation Check:"
dpkg -l | grep openblas || echo "OpenBLAS package not found"
ls -la /usr/lib/aarch64-linux-gnu/openblas/ || echo "OpenBLAS directory not found"
fi
# Debug: Show final library paths
echo "🔍 Final LD_LIBRARY_PATH: $LD_LIBRARY_PATH"
- name: Install system dependencies (macOS)
if: runner.os == 'macOS'
run: |
# Don't install LLVM, use system clang for better compatibility
brew install libomp boost protobuf zeromq
- name: Install system dependencies (Windows)
if: runner.os == 'Windows'
run: |
retry() {
local attempts=$1
shift
local n=1
while true; do
"$@" && break
if [[ $n -ge $attempts ]]; then
echo "Command failed after $n attempts: $*"
return 1
fi
echo "Command failed (attempt $n/$attempts). Retrying in 10s: $*"
sleep 10
n=$((n + 1))
done
}
retry 5 choco install swig -y --no-progress
retry 5 choco install nuget.commandline -y --no-progress
# pkgconfiglite via Chocolatey is flaky (exits 0 even on failure);
# verify the binary exists and fall back to direct download.
retry 3 choco install pkgconfiglite -y --no-progress || true
PKG_CONFIG_DIR="C:/ProgramData/chocolatey/bin"
if [[ ! -f "${PKG_CONFIG_DIR}/pkg-config.exe" ]]; then
echo "pkg-config.exe not found after choco, downloading directly..."
PKG_CONFIG_DIR="${RUNNER_TEMP}/pkg-config"
mkdir -p "${PKG_CONFIG_DIR}"
curl -fsSL -o "${RUNNER_TEMP}/pkgconfiglite.zip" \
"https://sourceforge.net/projects/pkgconfiglite/files/0.28-1/pkg-config-lite-0.28-1_bin-win32.zip/download"
unzip -q "${RUNNER_TEMP}/pkgconfiglite.zip" -d "${RUNNER_TEMP}/pkgconfiglite"
cp "${RUNNER_TEMP}/pkgconfiglite/pkg-config-lite-0.28-1/bin/"* "${PKG_CONFIG_DIR}/"
fi
echo "${PKG_CONFIG_DIR}" >> "$GITHUB_PATH"
echo "PKG_CONFIG_EXECUTABLE=${PKG_CONFIG_DIR}/pkg-config.exe" >> "$GITHUB_ENV"
if [[ -z "${VCPKG_INSTALLATION_ROOT:-}" ]]; then
echo "VCPKG_INSTALLATION_ROOT is not set on this runner"
exit 1
fi
retry 5 "${VCPKG_INSTALLATION_ROOT}/vcpkg" install zeromq:x64-windows
retry 5 "${VCPKG_INSTALLATION_ROOT}/vcpkg" install openblas:x64-windows
retry 5 "${VCPKG_INSTALLATION_ROOT}/vcpkg" install lapack:x64-windows
retry 5 "${VCPKG_INSTALLATION_ROOT}/vcpkg" install boost-program-options:x64-windows
retry 5 "${VCPKG_INSTALLATION_ROOT}/vcpkg" install protobuf:x64-windows
# DiskANN links against Intel OpenMP (libiomp5md) via NuGet during its
# CMake build. The NuGet packages end up in a temp build dir that is
# cleaned up by `uv build`, so delvewheel can't find the DLL later.
# Download it here to a persistent, known location.
NUGET_PKG_DIR="${RUNNER_TEMP}/nuget_pkgs"
retry 5 nuget install intelopenmp.redist.win -Version 2022.0.3.3747 \
-ExcludeVersion -OutputDirectory "${NUGET_PKG_DIR}"
echo "INTEL_OMP_BIN_DIR=${NUGET_PKG_DIR}/intelopenmp.redist.win/runtimes/win-x64/native" >> "$GITHUB_ENV"
echo "PKG_CONFIG_PATH=${VCPKG_INSTALLATION_ROOT}/installed/x64-windows/lib/pkgconfig" >> "$GITHUB_ENV"
echo "CMAKE_PREFIX_PATH=${VCPKG_INSTALLATION_ROOT}/installed/x64-windows" >> "$GITHUB_ENV"
echo "OPENBLAS_LIB=${VCPKG_INSTALLATION_ROOT}/installed/x64-windows/lib/openblas.lib" >> "$GITHUB_ENV"
echo "${VCPKG_INSTALLATION_ROOT}/installed/x64-windows/bin" >> "$GITHUB_PATH"
echo "${VCPKG_INSTALLATION_ROOT}/installed/x64-windows/debug/bin" >> "$GITHUB_PATH"
echo "${VCPKG_INSTALLATION_ROOT}/installed/x64-windows/tools/protobuf" >> "$GITHUB_PATH"
pkg-config --version || true
where pkg-config || true
pkg-config --modversion libzmq || true
- name: Install build dependencies
run: |
retry() {
local attempts=$1
shift
local n=1
while true; do
"$@" && break
if [[ $n -ge $attempts ]]; then
echo "Command failed after $n attempts: $*"
return 1
fi
echo "Command failed (attempt $n/$attempts). Retrying in 5s: $*"
sleep 5
n=$((n + 1))
done
}
retry 5 uv python install ${{ matrix.python }}
uv venv --python ${{ matrix.python }} .uv-build
if [[ "$RUNNER_OS" == "Windows" ]]; then
BUILD_PY=".uv-build\\Scripts\\python.exe"
else
BUILD_PY=".uv-build/bin/python"
fi
retry 5 uv pip install --python "$BUILD_PY" scikit-build-core numpy swig Cython pybind11
if [[ "$RUNNER_OS" == "Linux" ]]; then
retry 5 uv pip install --python "$BUILD_PY" auditwheel
elif [[ "$RUNNER_OS" == "macOS" ]]; then
retry 5 uv pip install --python "$BUILD_PY" delocate
else
retry 5 uv pip install --python "$BUILD_PY" delvewheel
fi
if [[ "$RUNNER_OS" == "Windows" ]]; then
echo "$(pwd)\\.uv-build\\Scripts" >> $GITHUB_PATH
else
echo "$(pwd)/.uv-build/bin" >> $GITHUB_PATH
fi
- name: Set macOS environment variables
if: runner.os == 'macOS'
run: |
# Use brew --prefix to automatically detect Homebrew installation path
HOMEBREW_PREFIX=$(brew --prefix)
echo "HOMEBREW_PREFIX=${HOMEBREW_PREFIX}" >> $GITHUB_ENV
echo "OpenMP_ROOT=${HOMEBREW_PREFIX}/opt/libomp" >> $GITHUB_ENV
# Set CMAKE_PREFIX_PATH to let CMake find all packages automatically
echo "CMAKE_PREFIX_PATH=${HOMEBREW_PREFIX}" >> $GITHUB_ENV
# Set compiler flags for OpenMP (required for both backends)
echo "LDFLAGS=-L${HOMEBREW_PREFIX}/opt/libomp/lib" >> $GITHUB_ENV
echo "CPPFLAGS=-I${HOMEBREW_PREFIX}/opt/libomp/include" >> $GITHUB_ENV
- name: Build packages
run: |
# Build core (platform independent)
cd packages/leann-core
uv build
cd ../..
# Build HNSW backend
cd packages/leann-backend-hnsw
if [[ "${{ matrix.os }}" == macos-* ]]; then
# Use system clang for better compatibility
export CC=clang
export CXX=clang++
# Set deployment target based on runner
# macos-15-intel runs macOS 15, so target 15.0 (system libraries require it)
if [[ "${{ matrix.os }}" == "macos-15-intel" ]]; then
export MACOSX_DEPLOYMENT_TARGET=15.0
elif [[ "${{ matrix.os }}" == macos-14* ]]; then
export MACOSX_DEPLOYMENT_TARGET=14.0
elif [[ "${{ matrix.os }}" == macos-15* ]]; then
export MACOSX_DEPLOYMENT_TARGET=15.0
elif [[ "${{ matrix.os }}" == macos-26* ]]; then
export MACOSX_DEPLOYMENT_TARGET=26.0
fi
uv build --wheel --python ${{ matrix.python }} --find-links ${GITHUB_WORKSPACE}/packages/leann-core/dist
else
uv build --wheel --python ${{ matrix.python }} --find-links ${GITHUB_WORKSPACE}/packages/leann-core/dist
fi
cd ../..
# Build DiskANN backend
cd packages/leann-backend-diskann
if [[ "${{ matrix.os }}" == macos-* ]]; then
# Use system clang for better compatibility
export CC=clang
export CXX=clang++
# Set deployment target based on runner
# macos-15-intel runs macOS 15, so target 15.0 (system libraries require it)
if [[ "${{ matrix.os }}" == "macos-15-intel" ]]; then
export MACOSX_DEPLOYMENT_TARGET=15.0
elif [[ "${{ matrix.os }}" == macos-14* ]]; then
export MACOSX_DEPLOYMENT_TARGET=14.0
elif [[ "${{ matrix.os }}" == macos-15* ]]; then
export MACOSX_DEPLOYMENT_TARGET=15.0
elif [[ "${{ matrix.os }}" == macos-26* ]]; then
export MACOSX_DEPLOYMENT_TARGET=26.0
fi
uv build --wheel --python ${{ matrix.python }} --find-links ${GITHUB_WORKSPACE}/packages/leann-core/dist
else
uv build --wheel --python ${{ matrix.python }} --find-links ${GITHUB_WORKSPACE}/packages/leann-core/dist
fi
cd ../..
# Build meta package (platform independent)
cd packages/leann
uv build
cd ../..
- name: Repair wheels (Linux)
if: runner.os == 'Linux'
run: |
# Repair HNSW wheel
cd packages/leann-backend-hnsw
if [ -d dist ]; then
auditwheel repair dist/*.whl -w dist_repaired
rm -rf dist
mv dist_repaired dist
fi
cd ../..
# Repair DiskANN wheel
cd packages/leann-backend-diskann
if [ -d dist ]; then
auditwheel repair dist/*.whl -w dist_repaired
rm -rf dist
mv dist_repaired dist
fi
cd ../..
- name: Repair wheels (macOS)
if: runner.os == 'macOS'
run: |
# Determine deployment target based on runner OS
# macos-15-intel runs macOS 15, so target 15.0 (system libraries require it)
if [[ "${{ matrix.os }}" == "macos-15-intel" ]]; then
HNSW_TARGET="15.0"
DISKANN_TARGET="15.0"
elif [[ "${{ matrix.os }}" == macos-14* ]]; then
HNSW_TARGET="14.0"
DISKANN_TARGET="14.0"
elif [[ "${{ matrix.os }}" == macos-15* ]]; then
HNSW_TARGET="15.0"
DISKANN_TARGET="15.0"
elif [[ "${{ matrix.os }}" == macos-26* ]]; then
HNSW_TARGET="26.0"
DISKANN_TARGET="26.0"
fi
# Repair HNSW wheel
cd packages/leann-backend-hnsw
if [ -d dist ]; then
export MACOSX_DEPLOYMENT_TARGET=$HNSW_TARGET
delocate-wheel -w dist_repaired -v --require-target-macos-version $HNSW_TARGET dist/*.whl
rm -rf dist
mv dist_repaired dist
fi
cd ../..
# Repair DiskANN wheel
cd packages/leann-backend-diskann
if [ -d dist ]; then
export MACOSX_DEPLOYMENT_TARGET=$DISKANN_TARGET
delocate-wheel -w dist_repaired -v --require-target-macos-version $DISKANN_TARGET dist/*.whl
rm -rf dist
mv dist_repaired dist
fi
cd ../..
- name: Repair wheels (Windows)
if: runner.os == 'Windows'
run: |
# Repair HNSW wheel
cd packages/leann-backend-hnsw
if [ -d dist ]; then
delvewheel repair dist/*.whl -w dist_repaired --add-path "${VCPKG_INSTALLATION_ROOT}/installed/x64-windows/bin"
rm -rf dist
mv dist_repaired dist
fi
cd ../..
# Repair DiskANN wheel.
# DiskANN's CMake install bundles diskann.dll and libiomp5md.dll inside
# the wheel, but delvewheel doesn't search inside the wheel for deps.
# Extract the wheel so delvewheel can discover them via --add-path.
cd packages/leann-backend-diskann
if [ -d dist ]; then
# Extract the wheel and build --add-path with native Windows paths.
# mktemp returns Git Bash paths (/tmp/...) that delvewheel.exe can't
# resolve, so use Python for the entire extract-and-repair flow.
python -c "
import zipfile, sys, glob, tempfile, subprocess, os, shutil
whl = glob.glob('dist/*.whl')[0]
tmpdir = tempfile.mkdtemp(prefix='diskann_whl_')
zipfile.ZipFile(whl).extractall(tmpdir)
add_paths = [os.environ.get('VCPKG_INSTALLATION_ROOT','') + '/installed/x64-windows/bin',
os.environ.get('INTEL_OMP_BIN_DIR','')]
for entry in os.listdir(tmpdir):
full = os.path.join(tmpdir, entry)
if os.path.isdir(full):
add_paths.append(full)
add_path_str = ';'.join(p for p in add_paths if p)
print(f'add-path: {add_path_str}')
rc = subprocess.call([sys.executable, '-m', 'delvewheel', 'repair', whl,
'-w', 'dist_repaired', '--add-path', add_path_str])
shutil.rmtree(tmpdir, ignore_errors=True)
sys.exit(rc)
"
rm -rf dist
mv dist_repaired dist
fi
cd ../..
- name: List built packages
run: |
echo "📦 Built packages:"
find packages/*/dist -name "*.whl" -o -name "*.tar.gz" | sort
- name: Install built packages for testing
run: |
retry() {
local attempts=$1
shift
local n=1
while true; do
"$@" && break
if [[ $n -ge $attempts ]]; then
echo "Command failed after $n attempts: $*"
return 1
fi
echo "Command failed (attempt $n/$attempts). Retrying in 5s: $*"
sleep 5
n=$((n + 1))
done
}
# Create uv-managed virtual environment with the requested interpreter
retry 5 uv python install ${{ matrix.python }}
uv venv --python ${{ matrix.python }}
source .venv/bin/activate || source .venv/Scripts/activate
if [[ "$RUNNER_OS" == "Windows" ]]; then
UV_PY=".venv\\Scripts\\python.exe"
else
UV_PY=".venv/bin/python"
fi
# Install test dependency group only (avoids reinstalling project package)
uv pip install --python "$UV_PY" --group test
# Install core wheel built in this job
CORE_WHL=$(find packages/leann-core/dist -maxdepth 1 -name "*.whl" -print -quit)
if [[ -n "$CORE_WHL" ]]; then
uv pip install --python "$UV_PY" "$CORE_WHL"
else
uv pip install --python "$UV_PY" packages/leann-core/dist/*.tar.gz
fi
PY_TAG=$($UV_PY -c "import sys; print(f'cp{sys.version_info[0]}{sys.version_info[1]}')")
if [[ "$RUNNER_OS" == "macOS" ]]; then
# macos-15-intel runs macOS 15, so target 15.0 (system libraries require it)
if [[ "${{ matrix.os }}" == "macos-15-intel" ]]; then
export MACOSX_DEPLOYMENT_TARGET=15.0
elif [[ "${{ matrix.os }}" == macos-14* ]]; then
export MACOSX_DEPLOYMENT_TARGET=14.0
elif [[ "${{ matrix.os }}" == macos-15* ]]; then
export MACOSX_DEPLOYMENT_TARGET=15.0
elif [[ "${{ matrix.os }}" == macos-26* ]]; then
export MACOSX_DEPLOYMENT_TARGET=26.0
fi
fi
HNSW_WHL=$(find packages/leann-backend-hnsw/dist -maxdepth 1 -name "*-${PY_TAG}-*.whl" -print -quit)
if [[ -z "$HNSW_WHL" ]]; then
HNSW_WHL=$(find packages/leann-backend-hnsw/dist -maxdepth 1 -name "*-py3-*.whl" -print -quit)
fi
if [[ -n "$HNSW_WHL" ]]; then
uv pip install --python "$UV_PY" "$HNSW_WHL"
else
uv pip install --python "$UV_PY" ./packages/leann-backend-hnsw
fi
DISKANN_WHL=$(find packages/leann-backend-diskann/dist -maxdepth 1 -name "*-${PY_TAG}-*.whl" -print -quit)
if [[ -z "$DISKANN_WHL" ]]; then
DISKANN_WHL=$(find packages/leann-backend-diskann/dist -maxdepth 1 -name "*-py3-*.whl" -print -quit)
fi
if [[ -n "$DISKANN_WHL" ]]; then
uv pip install --python "$UV_PY" "$DISKANN_WHL"
else
uv pip install --python "$UV_PY" ./packages/leann-backend-diskann
fi
LEANN_WHL=$(find packages/leann/dist -maxdepth 1 -name "*.whl" -print -quit)
if [[ -n "$LEANN_WHL" ]]; then
uv pip install --python "$UV_PY" "$LEANN_WHL"
else
uv pip install --python "$UV_PY" packages/leann/dist/*.tar.gz
fi
- name: Run tests with pytest
env:
CI: true
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
HF_HUB_DISABLE_SYMLINKS: 1
TOKENIZERS_PARALLELISM: false
PYTORCH_ENABLE_MPS_FALLBACK: 0
OMP_NUM_THREADS: 1
MKL_NUM_THREADS: 1
run: |
source .venv/bin/activate || source .venv/Scripts/activate
pytest tests/ -v --tb=short
- name: Run sanity checks (optional)
run: |
# Activate virtual environment
source .venv/bin/activate || source .venv/Scripts/activate
# Run distance function tests if available
if [ -f test/sanity_checks/test_distance_functions.py ]; then
echo "Running distance function sanity checks..."
python test/sanity_checks/test_distance_functions.py || echo "⚠️ Distance function test failed, continuing..."
fi
- name: Upload artifacts
uses: actions/upload-artifact@v4
with:
name: packages-${{ matrix.os }}-py${{ matrix.python }}
path: packages/*/dist/
arch-smoke:
name: Arch Linux smoke test (install & import)
needs: build
runs-on: ubuntu-latest
container:
image: archlinux:latest
steps:
- name: Prepare system
run: |
# Initialize pacman keyring to avoid "no secret key available" error
pacman-key --init
pacman -Syu --noconfirm
# Install build essentials (uv will manage Python version)
pacman -S --noconfirm gcc git zlib openssl
- name: Download ALL wheel artifacts from this run
uses: actions/download-artifact@v5
with:
# Don't specify name, download all artifacts
path: ./wheels
- name: Install uv
uses: astral-sh/setup-uv@v6
- name: Create virtual environment and install wheels
run: |
# Use Python 3.13 explicitly (Arch has Python 3.14 which PyO3/tokenizers doesn't support yet)
uv python install 3.13
uv venv --python 3.13
source .venv/bin/activate || source .venv/Scripts/activate
uv pip install --find-links wheels leann-core
uv pip install --find-links wheels leann-backend-hnsw
uv pip install --find-links wheels leann-backend-diskann
uv pip install --find-links wheels leann
- name: Import & tiny runtime check
env:
OMP_NUM_THREADS: 1
MKL_NUM_THREADS: 1
run: |
source .venv/bin/activate || source .venv/Scripts/activate
python - <<'PY'
import leann
import leann_backend_hnsw as h
import leann_backend_diskann as d
from leann import LeannBuilder, LeannSearcher
b = LeannBuilder(backend_name="hnsw")
b.add_text("hello arch")
b.build_index("arch_demo.leann")
s = LeannSearcher("arch_demo.leann")
print("search:", s.search("hello", top_k=1))
PY
================================================
FILE: .github/workflows/link-check.yml
================================================
name: Link Check
on:
push:
branches: [ main, master ]
pull_request:
schedule:
- cron: "0 3 * * 1"
jobs:
link-check:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: lycheeverse/lychee-action@v2
with:
args: >-
--no-progress --insecure
--user-agent 'curl/7.68.0'
--max-retries 3
--retry-wait-time 5
--exclude '.*star-history\.com.*'
--accept 200,201,202,203,204,205,206,207,208,226,300,301,302,303,304,305,306,307,308,503
README.md docs/ apps/ examples/ benchmarks/
fail: false
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
================================================
FILE: .github/workflows/release-manual.yml
================================================
name: Release
on:
workflow_dispatch:
inputs:
version:
description: 'Version to release (e.g., 0.1.2)'
required: true
type: string
jobs:
verify-ci:
name: Verify main CI
runs-on: ubuntu-latest
permissions:
actions: read
contents: read
steps:
- name: Require latest main CI success
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
set -euo pipefail
MAIN_SHA=$(gh api repos/${GITHUB_REPOSITORY}/commits/main --jq '.sha')
RUN_ID=$(gh api repos/${GITHUB_REPOSITORY}/actions/workflows/build-and-publish.yml/runs \
--method GET \
--field branch=main \
--field head_sha="${MAIN_SHA}" \
--field per_page=1 \
--jq '.workflow_runs[0].id')
if [ -z "${RUN_ID}" ] || [ "${RUN_ID}" = "null" ]; then
echo "❌ No CI run found for main @ ${MAIN_SHA}"
exit 1
fi
STATUS=$(gh api repos/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID} --jq '.status')
CONCLUSION=$(gh api repos/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID} --jq '.conclusion')
URL=$(gh api repos/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID} --jq '.html_url')
echo "CI run: ${URL}"
if [ "${STATUS}" != "completed" ] || [ "${CONCLUSION}" != "success" ]; then
echo "❌ CI not successful for main @ ${MAIN_SHA}"
echo "Status: ${STATUS}"
echo "Conclusion: ${CONCLUSION}"
exit 1
fi
echo "✅ CI succeeded for main @ ${MAIN_SHA}"
update-version:
name: Update Version
needs: verify-ci
runs-on: ubuntu-latest
permissions:
contents: write
outputs:
commit-sha: ${{ steps.push.outputs.commit-sha }}
steps:
- uses: actions/checkout@v4
- name: Validate version
run: |
# Remove 'v' prefix if present for validation
VERSION_CLEAN="${{ inputs.version }}"
VERSION_CLEAN="${VERSION_CLEAN#v}"
if ! [[ "$VERSION_CLEAN" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
echo "❌ Invalid version format. Expected format: X.Y.Z or vX.Y.Z"
exit 1
fi
echo "✅ Version format valid: ${{ inputs.version }}"
- name: Update versions and push
id: push
run: |
# Check current version
CURRENT_VERSION=$(grep "^version" packages/leann-core/pyproject.toml | cut -d'"' -f2)
echo "Current version: $CURRENT_VERSION"
echo "Target version: ${{ inputs.version }}"
if [ "$CURRENT_VERSION" = "${{ inputs.version }}" ]; then
echo "⚠️ Version is already ${{ inputs.version }}, skipping update"
COMMIT_SHA=$(git rev-parse HEAD)
else
./scripts/bump_version.sh ${{ inputs.version }}
git config user.name "GitHub Actions"
git config user.email "actions@github.com"
git add packages/*/pyproject.toml
git commit -m "chore: release v${{ inputs.version }}"
git push origin main
COMMIT_SHA=$(git rev-parse HEAD)
echo "✅ Pushed version update: $COMMIT_SHA"
fi
echo "commit-sha=$COMMIT_SHA" >> $GITHUB_OUTPUT
build-packages:
name: Build packages
needs: update-version
uses: ./.github/workflows/build-reusable.yml
with:
ref: 'main'
publish:
name: Publish and Release
needs: [update-version, build-packages]
if: always() && needs.update-version.result == 'success' && needs.build-packages.result == 'success'
runs-on: ubuntu-latest
permissions:
contents: write
steps:
- uses: actions/checkout@v4
with:
ref: 'main'
- name: Download all artifacts
uses: actions/download-artifact@v4
with:
path: dist-artifacts
- name: Collect packages
run: |
mkdir -p dist
find dist-artifacts -name "*.whl" -exec cp {} dist/ \;
find dist-artifacts -name "*.tar.gz" -exec cp {} dist/ \;
echo "📦 Packages to publish:"
ls -la dist/
- name: Publish to PyPI
env:
TWINE_USERNAME: __token__
TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
run: |
if [ -z "$TWINE_PASSWORD" ]; then
echo "❌ PYPI_API_TOKEN not configured!"
exit 1
fi
pip install twine
twine upload dist/* --skip-existing --verbose
echo "✅ Published to PyPI!"
- name: Create release
run: |
# Check if tag already exists
if git rev-parse "v${{ inputs.version }}" >/dev/null 2>&1; then
echo "⚠️ Tag v${{ inputs.version }} already exists, skipping tag creation"
else
git tag "v${{ inputs.version }}"
git push origin "v${{ inputs.version }}"
echo "✅ Created and pushed tag v${{ inputs.version }}"
fi
# Check if release already exists
if gh release view "v${{ inputs.version }}" >/dev/null 2>&1; then
echo "⚠️ Release v${{ inputs.version }} already exists, skipping release creation"
else
gh release create "v${{ inputs.version }}" \
--title "Release v${{ inputs.version }}" \
--notes "🚀 Released to PyPI: https://pypi.org/project/leann/${{ inputs.version }}/" \
--latest
echo "✅ Created GitHub release v${{ inputs.version }}"
fi
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
================================================
FILE: .gitignore
================================================
raw_data/
scaling_out/
scaling_out_old/
sanity_check/
demo/indices/
# .vscode/
*.log
*pycache*
outputs/
*.pkl
*.pdf
*.idx
*.map
.history/
lm_eval.egg-info/
demo/experiment_results/**/*.json
*.jsonl
*.eml
*.emlx
*.json
*.png
!.vscode/*.json
!.devcontainer/*.json
!skills/**/claw.json
*.sh
*.txt
!CMakeLists.txt
!llms.txt
latency_breakdown*.json
experiment_results/eval_results/diskann/*.json
aws/
.venv/
.cursor/rules/
*.egg-info/
skip_reorder_comparison/
analysis_results/
build/
.cache/
nprobe_logs/
micro/results
micro/contriever-INT8
data/*
!data/2501.14312v1 (1).pdf
!data/2506.08276v1.pdf
!data/PrideandPrejudice.txt
!data/huawei_pangu.md
!data/ground_truth/
!data/indices/
!data/queries/
!data/.gitattributes
*.qdstrm
benchmark_results/
results/
frac_*.png
final_in_*.png
embedding_comparison_results/
*.ind
*.gz
*.fvecs
*.ivecs
*.index
*.bin
*.old
read_graph
analyze_diskann_graph
degree_distribution.png
micro/degree_distribution.png
policy_results_*
results_*/
experiment_results/
.DS_Store
# The above are inherited from old Power RAG repo
# Python-generated files
__pycache__/
*.py[oc]
build/
dist/
wheels/
*.egg-info
# Virtual environments
.venv
.env
test_indices*/
test_*.py
!tests/**
# Re-ignore common generated artifacts globally (especially after allowlist rules).
**/.DS_Store
**/__pycache__/
**/*.cpython-*.pyc
**/*.cpython-*.pyo
packages/leann-backend-diskann/third_party/DiskANN/_deps/
*.meta.json
*.passages.json
*.npy
*.db
batchtest.py
tests/__pytest_cache__/
tests/__pycache__/
benchmarks/data/
## multi vector
apps/multimodal/vision-based-pdf-multi-vector/multi-vector-colpali-native-weaviate.py
# Ignore all PDFs (keep data exceptions above) and do not track demo PDFs
# If you need to commit a specific demo PDF, remove this negation locally.
# The following line used to force-add a large demo PDF; remove it to satisfy pre-commit:
# !apps/multimodal/vision-based-pdf-multi-vector/pdfs/2004.12832v2.pdf
!apps/multimodal/vision-based-pdf-multi-vector/fig/*
# AUR build directory (Arch Linux)
paru-bin/
merkle-tree-test/
test-code/
localtestmcp/
*.csv
*.pickle
================================================
FILE: .gitmodules
================================================
[submodule "packages/leann-backend-diskann/third_party/DiskANN"]
path = packages/leann-backend-diskann/third_party/DiskANN
url = https://github.com/yichuan-w/DiskANN.git
[submodule "packages/leann-backend-hnsw/third_party/faiss"]
path = packages/leann-backend-hnsw/third_party/faiss
url = https://github.com/yichuan-w/faiss.git
[submodule "packages/leann-backend-hnsw/third_party/msgpack-c"]
path = packages/leann-backend-hnsw/third_party/msgpack-c
url = https://github.com/msgpack/msgpack-c.git
branch = cpp_master
[submodule "packages/leann-backend-hnsw/third_party/cppzmq"]
path = packages/leann-backend-hnsw/third_party/cppzmq
url = https://github.com/zeromq/cppzmq.git
[submodule "packages/leann-backend-hnsw/third_party/libzmq"]
path = packages/leann-backend-hnsw/third_party/libzmq
url = https://github.com/zeromq/libzmq.git
[submodule "packages/astchunk-leann"]
path = packages/astchunk-leann
url = https://github.com/yichuan-w/astchunk-leann.git
================================================
FILE: .pre-commit-config.yaml
================================================
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v5.0.0
hooks:
- id: trailing-whitespace
- id: end-of-file-fixer
- id: check-yaml
- id: check-added-large-files
- id: check-merge-conflict
- id: debug-statements
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.12.7 # Fixed version to match pyproject.toml
hooks:
- id: ruff
args: [--fix, --exit-non-zero-on-fix]
- id: ruff-format
================================================
FILE: .python-version
================================================
3.11
================================================
FILE: .vscode/extensions.json
================================================
{
"recommendations": [
"charliermarsh.ruff",
]
}
================================================
FILE: .vscode/settings.json
================================================
{
"python.defaultInterpreterPath": ".venv/bin/python",
"python.terminal.activateEnvironment": true,
"[python]": {
"editor.defaultFormatter": "charliermarsh.ruff",
"editor.formatOnSave": true,
"editor.codeActionsOnSave": {
"source.organizeImports": "explicit",
"source.fixAll": "explicit"
},
"editor.insertSpaces": true,
"editor.tabSize": 4
},
"ruff.enable": true,
"files.watcherExclude": {
"**/.venv/**": true,
"**/__pycache__/**": true,
"**/*.egg-info/**": true,
"**/build/**": true,
"**/dist/**": true
}
}
================================================
FILE: CLAUDE.md
================================================
# CLAUDE.md
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
## Project Overview
LEANN is a lightweight vector database and RAG (Retrieval-Augmented Generation) system that achieves 97% storage reduction compared to traditional vector databases through graph-based selective recomputation. It enables semantic search across various data sources (emails, browser history, chat history, code, documents) on a single laptop without cloud dependencies.
## Build & Development Commands
### Quick install (pip)
```bash
pip install leann
```
### Development setup (from source)
```bash
# Install uv first (required package manager)
curl -LsSf https://astral.sh/uv/install.sh | sh
git submodule update --init --recursive
# macOS
brew install libomp boost protobuf zeromq pkgconf
uv sync
# Ubuntu/Debian
sudo apt-get install libomp-dev libboost-all-dev protobuf-compiler \
libabsl-dev libmkl-full-dev libaio-dev libzmq3-dev
uv sync
# Windows (requires VS 2022 Build Tools with C++ workload, vcpkg, chocolatey)
choco install cmake swig pkgconfiglite nuget.commandline -y
vcpkg install zeromq:x64-windows openblas:x64-windows lapack:x64-windows boost-program-options:x64-windows protobuf:x64-windows
# Set CMAKE_PREFIX_PATH, PKG_CONFIG_PATH, OPENBLAS_LIB to vcpkg paths (see README)
uv sync --extra diskann
# Install lint tools
uv sync --group lint
# Install test tools
uv sync --group test
```
## Code Quality
```bash
# Format code
ruff format
# Lint with auto-fix
ruff check --fix
# Pre-commit hooks (install once)
pre-commit install
# Run pre-commit manually
uv run pre-commit run --all-files
```
## Architecture
### Core API Layer (`packages/leann-core/src/leann/`)
- `api.py`: Main APIs - `LeannBuilder`, `LeannSearcher`, `LeannChat`
- `react_agent.py`: `ReActAgent` for multi-turn reasoning
- `cli.py`: CLI implementation (`leann build`, `leann search`, `leann ask`)
- `chat.py`: LLM provider integrations (OpenAI, Ollama, HuggingFace, Anthropic)
- `embedding_compute.py`: Embedding computation (sentence-transformers, MLX, OpenAI)
- `metadata_filter.py`: Search result filtering by metadata
### Backend Layer (`packages/`)
- `leann-backend-hnsw/`: Default backend using FAISS HNSW for fast in-memory search
- `leann-backend-ivf/`: IVF backend (FAISS IndexIVFFlat + DirectMap.Hashtable) supporting in-place add/remove without rebuild
- `leann-backend-diskann/`: DiskANN backend for larger-than-memory datasets
- `leann-mcp/`: MCP server for Claude Code integration
Backends are auto-discovered via `leann-backend-*` naming convention and registered in `registry.py`.
### RAG Applications (`apps/`)
Example applications demonstrating RAG on various data sources:
- `document_rag.py`: PDF/TXT/MD documents
- `email_rag.py`: Apple Mail
- `browser_rag.py`: Chrome browser history
- `wechat_rag.py`, `imessage_rag.py`: Chat history
- `code_rag.py`: Codebase search with AST-aware chunking
- `slack_rag.py`, `twitter_rag.py`: MCP-based live data
## Key Design Patterns
### Incremental Update (IVF backend)
The IVF backend supports in-place updates and deletes without rebuilding the entire index:
- `add_vectors(index_path, embeddings, passage_ids)`: Append new vectors to an existing index.
- `remove_ids(index_path, passage_ids)`: Remove vectors by passage ID using FAISS DirectMap.Hashtable.
- `LeannBuilder.update_index()`: High-level API that orchestrates remove-then-add for changed files, compacts `passages.jsonl`, and updates the offset map.
`leann build` is idempotent — re-running it on an existing index automatically performs an incremental update instead of a full rebuild. It detects new, modified, and removed files and applies the minimal set of changes:
- **IVF**: Supports add, remove, and modify incrementally (remove old chunks then re-insert).
- **HNSW** (non-compact): Supports add-only incremental updates; modified/removed files trigger a full rebuild.
- Use `--force` / `-f` to force a full rebuild regardless.
### Index Structure
A LEANN index consists of:
- `<name>.meta.json`: Metadata (backend, embedding model, dimensions)
- `<name>.passages.jsonl`: Raw text chunks with metadata
- `<name>.passages.idx`: Offset map for fast passage lookup
- `<name>.index`: Backend-specific vector index
### Embedding Recomputation
The core storage optimization: instead of storing embeddings, LEANN stores a pruned graph and recomputes embeddings on-demand during search via ZMQ server communication.
## CLI Usage
```bash
# Build index
leann build my-docs --docs ./documents/
# Search
leann search my-docs "query"
# Interactive chat
leann ask my-docs --interactive
# List indexes
leann list
# Remove index
leann remove my-docs
```
## Common Development Tasks
Running example RAG applications:
```bash
# Document RAG (easiest to test)
python -m apps.document_rag --query "What is LEANN?"
# Code RAG
python -m apps.code_rag --repo-dir ./src --query "How does search work?"
```
## Python Version
Requires Python 3.10+ (uses PEP 604 union syntax `X | Y`).
# Agent Coding Guidelines
## General
- Voice input may contain typos — interpret intent, not literal text.
- When you encounter a problem, fix it immediately and keep going until there are no more problems.
- Do not ask about ordering or sequencing — figure it out. If something is unclear, note it and skip it; only escalate when all paths are blocked.
- Obvious bugs: fix silently without reporting.
- No fallbacks or compatibility shims. One correct implementation per feature — no redundancy.
## Roadmap
- Public roadmap: `docs/roadmap.md` — tracks P0/P1 priorities, completed milestones, and timeline.
- Long-term vision: `docs/ultimate_goal.md` — the north star for where LEANN is headed.
- Keep in sync with [GitHub issue #237](https://github.com/yichuan-w/LEANN/issues/237).
- Welcome everyone to add more, and the craziest feature you want to put here! If people want some feature, all put there.
## Changelog (for contributors)
- Maintain `docs/CHANGELOG.md` — append-only log of major changes (new features, breaking changes, important fixes).
- Format: `## YYYY-MM-DD: <short summary>` followed by bullet points.
- Update the changelog when merging significant PRs or completing notable work.
- See `docs/CONTRIBUTING.md` for full contributor workflow (conventional commits, PR process, CI).
## Personal Dev Notes (gitignored)
- `docs/dev/` is gitignored for personal development notes (TODO, progress, experiments).
- Use `docs/dev/TODO.md` for in-progress tasks, `docs/dev/PROGRESS.md` for completed work.
- These are private scratch space — but must follow the Self-Contained Principle below.
## Documentation — Self-Contained Principle
All dev docs (`PROGRESS.md`, `STATES.md`, `EXPERIMENTS.md`, `TODO.md`) must be fully understandable from the document alone, with no reliance on conversation context or implied knowledge.
Requirements:
1. **Every technique/approach must be explained on first use.** Not "switched to IVF backend" — write "switched to IVF backend (FAISS IndexIVFFlat + DirectMap.Hashtable, supports in-place add/remove without full index rebuild)."
2. **Never assume the reader knows any abbreviation.** On first use: full name + one-sentence explanation. E.g., "HNSW (Hierarchical Navigable Small World — a graph-based ANN index used as LEANN's default backend)."
3. **Benchmark results must include full context.** Not "recall improved to 0.95" — write "recall@10 improved from 0.91 to 0.95 after switching from flat chunking (512 tokens, no overlap) to AST-aware chunking (function-level splits with 64-token overlap)."
4. **Numbers must have reference points.** Not "build time: 12s" — write "build time: 12s (vs. 45s before incremental update support, on 10k-document corpus)."
5. **Include the causal chain — not just conclusions.** Not "duplicate chunks appeared after incremental build" — write "Duplicate chunks appeared after incremental build because `passages.jsonl` was appended without first removing stale entries for modified files. The IVF index had correct vectors (remove-then-add), but the passage store was append-only, causing the same text to appear at multiple offsets."
6. **`docs/dev/STATES.md` top section maintains a glossary** of all key terms (backends, index files, chunking strategies, embedding models). Other docs reference it at the top.
Bad examples (forbidden):
- "Fixed the chunking bug" → Which bug? What was the symptom? What was the root cause?
- "Improved search quality" → By what metric? From what baseline? What change caused it?
- "Used nprobe=32" → What is nprobe? Why 32? What was it before and what effect did the change have?
## Doc Maintenance
- Maintain `docs/dev/PROGRESS.md` — completed work only (with key script/log/config paths). No plans.
- Maintain `docs/dev/TODO.md` — incomplete/in-progress/next-steps only (aim for one-command reproducibility). When done: remove from TODO, write result to PROGRESS, update STATES/EXPERIMENTS if needed.
- Both files: **append-only, chronological order** (oldest first). Use `tail -n 80 docs/dev/PROGRESS.md` to read recent entries; increase range or grep by date/keyword if needed.
- Keep TODO clean — either do items or remove them. Ask the user when unsure how to handle a TODO item.
- Maintain `docs/dev/STATES.md` — tracks all currently useful state (index configs, backend choices, known limitations); does NOT grow indefinitely (delete stale entries).
- Maintain `docs/dev/EXPERIMENTS.md` — benchmarks, A/B comparisons, parameter sweeps (recall@k, latency, storage size). Experimental content goes here, not in STATES.md.
## Commits
Commit when: (1) a complete feature is finished and tested, or (2) a destructive change is unavoidable.
```bash
git add <specific files>
git commit -m “feat: ...” # follow conventional commits
```
- When correcting errors: fix directly with no trace of the error.
- If you write a correct new version of a file, delete the wrong version. No duplicate implementations.
================================================
FILE: LICENSE
================================================
MIT License
Copyright (c) 2025 LEANN Contributors
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
================================================
FILE: README.md
================================================
<p align="center">
<img src="assets/logo-text.png" alt="LEANN Logo" width="400">
</p>
<p align="center">
<a href="https://trendshift.io/repositories/15049" target="_blank">
<img src="https://trendshift.io/api/badge/repositories/15049" alt="yichuan-w/LEANN | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/>
</a>
</p>
<p align="center">
<img src="https://img.shields.io/badge/Python-3.9%20%7C%203.10%20%7C%203.11%20%7C%203.12%20%7C%203.13-blue.svg" alt="Python Versions">
<img src="https://github.com/yichuan-w/LEANN/actions/workflows/build-and-publish.yml/badge.svg" alt="CI Status">
<img src="https://img.shields.io/badge/Platform-Ubuntu%20%26%20Arch%20%26%20WSL%20%7C%20macOS%20(ARM64%2FIntel)%20%7C%20Windows-lightgrey" alt="Platform">
<img src="https://img.shields.io/badge/License-MIT-green.svg" alt="MIT License">
<img src="https://img.shields.io/badge/MCP-Native%20Integration-blue" alt="MCP Integration">
<a href="https://join.slack.com/t/leann-e2u9779/shared_invite/zt-3ol2ww9ic-Eg_kB8omwe6xmYVd0epr4Q">
<img src="https://img.shields.io/badge/Slack-Join-4A154B?logo=slack&logoColor=white" alt="Join Slack">
</a>
</p>
<div align="center">
<a href="https://forms.gle/rDbZf864gMNxhpTq8">
<img src="https://img.shields.io/badge/📣_Community_Survey-Help_Shape_v0.4-007ec6?style=for-the-badge&logo=google-forms&logoColor=white" alt="Take Survey">
</a>
<p>
We track <b>zero telemetry</b>. This survey is the ONLY way to tell us if you want <br>
<b>GPU Acceleration</b> or <b>More Integrations</b> next.<br>
👉 <a href="https://forms.gle/rDbZf864gMNxhpTq8"><b>Click here to cast your vote (2 mins)</b></a>
</p>
</div>
<div align="center">
<h3>💬 Join our Slack community!</h3>
<p>
We'd love for you to be part of the LEANN community!<br>
👉 <a href="https://join.slack.com/t/leann-e2u9779/shared_invite/zt-3ol2ww9ic-Eg_kB8omwe6xmYVd0epr4Q"><b>Join LEANN Slack</b></a><br>
If the invite link has expired or you have trouble joining, please <a href="https://github.com/yichuan-w/LEANN/issues">open an issue</a> and we'll help you get in!
</p>
</div>
<h2 align="center" tabindex="-1" class="heading-element" dir="auto">
The smallest vector index in the world. RAG Everything with LEANN!
</h2>
LEANN is an innovative vector database that democratizes personal AI. Transform your laptop into a powerful RAG system that can index and search through millions of documents while using **97% less storage** than traditional solutions **without accuracy loss**.
LEANN achieves this through *graph-based selective recomputation* with *high-degree preserving pruning*, computing embeddings on-demand instead of storing them all. [Illustration Fig →](#️-architecture--how-it-works) | [Paper →](https://arxiv.org/abs/2506.08276)
**Ready to RAG Everything?** Transform your laptop into a personal AI assistant that can semantic search your **[file system](#-personal-data-manager-process-any-documents-pdf-txt-md)**, **[emails](#-your-personal-email-secretary-rag-on-apple-mail)**, **[browser history](#-time-machine-for-the-web-rag-your-entire-browser-history)**, **[chat history](#-wechat-detective-unlock-your-golden-memories)** ([WeChat](#-wechat-detective-unlock-your-golden-memories), [iMessage](#-imessage-history-your-personal-conversation-archive)), **[agent memory](#-chatgpt-chat-history-your-personal-ai-conversation-archive)** ([ChatGPT](#-chatgpt-chat-history-your-personal-ai-conversation-archive), [Claude](#-claude-chat-history-your-personal-ai-conversation-archive)), **[live data](#mcp-integration-rag-on-live-data-from-any-platform)** ([Slack](#slack-messages-search-your-team-conversations), [Twitter](#-twitter-bookmarks-your-personal-tweet-library)), **[codebase](#-claude-code-integration-transform-your-development-workflow)**\* , or external knowledge bases (i.e., 60M documents) - all on your laptop, with zero cloud costs and complete privacy.
\* Claude Code only supports basic `grep`-style keyword search. **LEANN** is a drop-in **semantic search MCP service fully compatible with Claude Code**, unlocking intelligent retrieval without changing your workflow. 🔥 Check out [the easy setup →](packages/leann-mcp/README.md)
## Why LEANN?
<p align="center">
<img src="assets/effects.png" alt="LEANN vs Traditional Vector DB Storage Comparison" width="70%">
</p>
> **The numbers speak for themselves:** Index 60 million text chunks in just 6GB instead of 201GB. From emails to browser history, everything fits on your laptop. [See detailed benchmarks for different applications below ↓](#-storage-comparison)
🔒 **Privacy:** Your data never leaves your laptop. No OpenAI, no cloud, no "terms of service".
🪶 **Lightweight:** Graph-based recomputation eliminates heavy embedding storage, while smart graph pruning and CSR format minimize graph storage overhead. Always less storage, less memory usage!
📦 **Portable:** Transfer your entire knowledge base between devices (even with others) with minimal cost - your personal AI memory travels with you.
📈 **Scalability:** Handle messy personal data that would crash traditional vector DBs, easily managing your growing personalized data and agent generated memory!
✨ **No Accuracy Loss:** Maintain the same search quality as heavyweight solutions while using 97% less storage.
## Installation
### 📦 Prerequisites: Install uv
[Install uv](https://docs.astral.sh/uv/getting-started/installation/#installation-methods) first if you don't have it. Typically, you can install it with:
```bash
curl -LsSf https://astral.sh/uv/install.sh | sh
```
### 🚀 Quick Install
Clone the repository to access all examples and try amazing applications,
```bash
git clone https://github.com/yichuan-w/LEANN.git leann
cd leann
```
and install LEANN from [PyPI](https://pypi.org/project/leann/) to run them immediately:
```bash
uv venv
source .venv/bin/activate
uv pip install leann
# CPU-only (Linux): use the `cpu` extra (e.g. `leann[cpu]`)
```
<!--
> Low-resource? See "Low-resource setups" in the [Configuration Guide](docs/configuration-guide.md#low-resource-setups). -->
<details>
<summary>
<strong>🔧 Build from Source (Recommended for development)</strong>
</summary>
```bash
git clone https://github.com/yichuan-w/LEANN.git leann
cd leann
git submodule update --init --recursive
```
**macOS:**
Note: DiskANN requires MacOS 13.3 or later.
```bash
brew install libomp boost protobuf zeromq pkgconf
uv sync --extra diskann
```
**Linux (Ubuntu/Debian):**
Note: On Ubuntu 20.04, you may need to build a newer Abseil and pin Protobuf (e.g., v3.20.x) for building DiskANN. See [Issue #30](https://github.com/yichuan-w/LEANN/issues/30) for a step-by-step note.
You can manually install [Intel oneAPI MKL](https://www.intel.com/content/www/us/en/developer/tools/oneapi/onemkl.html) instead of `libmkl-full-dev` for DiskANN. You can also use `libopenblas-dev` for building HNSW only, by removing `--extra diskann` in the command below.
```bash
sudo apt-get update && sudo apt-get install -y \
libomp-dev libboost-all-dev protobuf-compiler libzmq3-dev \
pkg-config libabsl-dev libaio-dev libprotobuf-dev \
libmkl-full-dev
uv sync --extra diskann
```
**Linux (Arch Linux):**
```bash
sudo pacman -Syu && sudo pacman -S --needed base-devel cmake pkgconf git gcc \
boost boost-libs protobuf abseil-cpp libaio zeromq
# For MKL in DiskANN
sudo pacman -S --needed base-devel git
git clone https://aur.archlinux.org/paru-bin.git
cd paru-bin && makepkg -si
paru -S intel-oneapi-mkl intel-oneapi-compiler
source /opt/intel/oneapi/setvars.sh
uv sync --extra diskann
```
**Linux (RHEL / CentOS Stream / Oracle / Rocky / AlmaLinux):**
See [Issue #50](https://github.com/yichuan-w/LEANN/issues/50) for more details.
```bash
sudo dnf groupinstall -y "Development Tools"
sudo dnf install -y libomp-devel boost-devel protobuf-compiler protobuf-devel \
abseil-cpp-devel libaio-devel zeromq-devel pkgconf-pkg-config
# For MKL in DiskANN
sudo dnf install -y intel-oneapi-mkl intel-oneapi-mkl-devel \
intel-oneapi-openmp || sudo dnf install -y intel-oneapi-compiler
source /opt/intel/oneapi/setvars.sh
uv sync --extra diskann
```
**Windows:**
Requires [Visual Studio 2022 Build Tools](https://visualstudio.microsoft.com/downloads/#build-tools-for-visual-studio-2022) with the **C++ desktop development** workload, and [vcpkg](https://github.com/microsoft/vcpkg).
```powershell
# Install toolchain (if not already present)
choco install cmake swig pkgconfiglite nuget.commandline -y
# Install C++ dependencies via vcpkg
vcpkg install zeromq:x64-windows openblas:x64-windows lapack:x64-windows `
boost-program-options:x64-windows protobuf:x64-windows
# Set environment variables (adjust VCPKG_ROOT to your vcpkg path)
$env:CMAKE_PREFIX_PATH = "$env:VCPKG_ROOT\installed\x64-windows"
$env:PKG_CONFIG_PATH = "$env:VCPKG_ROOT\installed\x64-windows\lib\pkgconfig"
$env:PKG_CONFIG_EXECUTABLE = "C:\ProgramData\chocolatey\bin\pkg-config.exe"
$env:OPENBLAS_LIB = "$env:VCPKG_ROOT\installed\x64-windows\lib\openblas.lib"
$env:PATH += ";$env:VCPKG_ROOT\installed\x64-windows\bin"
$env:PATH += ";$env:VCPKG_ROOT\installed\x64-windows\tools\protobuf"
uv sync --extra diskann
```
</details>
## Quick Start
Our declarative API makes RAG as easy as writing a config file.
Check out [demo.ipynb](demo.ipynb) or [](https://colab.research.google.com/github/yichuan-w/LEANN/blob/main/demo.ipynb)
```python
from leann import LeannBuilder, LeannSearcher, LeannChat
from pathlib import Path
INDEX_PATH = str(Path("./").resolve() / "demo.leann")
# Build an index
builder = LeannBuilder(backend_name="hnsw")
builder.add_text("LEANN saves 97% storage compared to traditional vector databases.")
builder.add_text("Tung Tung Tung Sahur called—they need their banana‑crocodile hybrid back")
builder.build_index(INDEX_PATH)
# Search
searcher = LeannSearcher(INDEX_PATH)
results = searcher.search("fantastical AI-generated creatures", top_k=1)
# Chat with your data
chat = LeannChat(INDEX_PATH, llm_config={"type": "hf", "model": "Qwen/Qwen3-0.6B"})
response = chat.ask("How much storage does LEANN save?", top_k=1)
```
## RAG on Everything!
LEANN supports RAG on various data sources including documents (`.pdf`, `.txt`, `.md`), Apple Mail, Google Search History, WeChat, ChatGPT conversations, Claude conversations, iMessage conversations, and **live data from any platform through MCP (Model Context Protocol) servers** - including Slack, Twitter, and more.
### Generation Model Setup
#### LLM Backend
LEANN supports many LLM providers for text generation (HuggingFace, Ollama, Anthropic, and Any OpenAI compatible API).
<details>
<summary><strong>🔑 OpenAI API Setup (Default)</strong></summary>
Set your OpenAI API key as an environment variable:
```bash
export OPENAI_API_KEY="your-api-key-here"
```
Make sure to use `--llm openai` flag when using the CLI.
You can also specify the model name with `--llm-model <model-name>` flag.
</details>
<details>
<summary><strong>🛠️ Supported LLM & Embedding Providers (via OpenAI Compatibility)</strong></summary>
Thanks to the widespread adoption of the OpenAI API format, LEANN is compatible out-of-the-box with a vast array of LLM and embedding providers. Simply set the `OPENAI_BASE_URL` and `OPENAI_API_KEY` environment variables to connect to your preferred service.
```sh
export OPENAI_API_KEY="xxx"
export OPENAI_BASE_URL="http://localhost:1234/v1" # base url of the provider
```
To use OpenAI compatible endpoint with the CLI interface:
If you are using it for text generation, make sure to use `--llm openai` flag and specify the model name with `--llm-model <model-name>` flag.
If you are using it for embedding, set the `--embedding-mode openai` flag and specify the model name with `--embedding-model <MODEL>`.
-----
Below is a list of base URLs for common providers to get you started.
### 🖥️ Local Inference Engines (Recommended for full privacy)
| Provider | Sample Base URL |
| ---------------- | --------------------------- |
| **Ollama** | `http://localhost:11434/v1` |
| **LM Studio** | `http://localhost:1234/v1` |
| **vLLM** | `http://localhost:8000/v1` |
| **llama.cpp** | `http://localhost:8080/v1` |
| **SGLang** | `http://localhost:30000/v1` |
| **LiteLLM** | `http://localhost:4000` |
-----
### ☁️ Cloud Providers
> **🚨 A Note on Privacy:** Before choosing a cloud provider, carefully review their privacy and data retention policies. Depending on their terms, your data may be used for their own purposes, including but not limited to human reviews and model training, which can lead to serious consequences if not handled properly.
| Provider | Base URL |
| ---------------- | ---------------------------------------------------------- |
| **OpenAI** | `https://api.openai.com/v1` |
| **OpenRouter** | `https://openrouter.ai/api/v1` |
| **Gemini** | `https://generativelanguage.googleapis.com/v1beta/openai/` |
| **x.AI (Grok)** | `https://api.x.ai/v1` |
| **Groq AI** | `https://api.groq.com/openai/v1` |
| **DeepSeek** | `https://api.deepseek.com/v1` |
| **SiliconFlow** | `https://api.siliconflow.cn/v1` |
| **Zhipu (BigModel)** | `https://open.bigmodel.cn/api/paas/v4/` |
| **Mistral AI** | `https://api.mistral.ai/v1` |
| **Anthropic** | `https://api.anthropic.com/v1` |
| **Jina AI** (Embeddings) | `https://api.jina.ai/v1` |
> **💡 Tip: Separate Embedding Provider**
>
> To use a different provider for embeddings (e.g., Jina AI) while using another for LLM, use `--embedding-api-base` and `--embedding-api-key`:
> ```bash
> leann build my-index --docs ./docs \
> --embedding-mode openai \
> --embedding-model jina-embeddings-v3 \
> --embedding-api-base https://api.jina.ai/v1 \
> --embedding-api-key $JINA_API_KEY
> ```
If your provider isn't on this list, don't worry! Check their documentation for an OpenAI-compatible endpoint—chances are, it's OpenAI Compatible too!
</details>
<details>
<summary><strong>🔧 Ollama Setup (Recommended for full privacy)</strong></summary>
**macOS:**
First, [download Ollama for macOS](https://ollama.com/download/mac).
```bash
# Pull a lightweight model (recommended for consumer hardware)
ollama pull llama3.2:1b
```
**Linux:**
```bash
# Install Ollama
curl -fsSL https://ollama.ai/install.sh | sh
# Start Ollama service manually
ollama serve &
# Pull a lightweight model (recommended for consumer hardware)
ollama pull llama3.2:1b
```
</details>
## ⭐ Flexible Configuration
LEANN provides flexible parameters for embedding models, search strategies, and data processing to fit your specific needs.
📚 **Need configuration best practices?** Check our [Configuration Guide](docs/configuration-guide.md) for detailed optimization tips, model selection advice, and solutions to common issues like slow embeddings or poor search quality.
<details>
<summary><strong>📋 Click to expand: Common Parameters (Available in All Examples)</strong></summary>
All RAG examples share these common parameters. **Interactive mode** is available in all examples - simply run without `--query` to start a continuous Q&A session where you can ask multiple questions. Type 'quit' to exit.
```bash
# Environment Variables (GPU Device Selection)
LEANN_EMBEDDING_DEVICE # GPU for embedding model (e.g., cuda:0, cuda:1, cpu)
LEANN_LLM_DEVICE # GPU for HFChat LLM (e.g., cuda:1, or "cuda" for multi-GPU auto)
# Core Parameters (General preprocessing for all examples)
--index-dir DIR # Directory to store the index (default: current directory)
--query "YOUR QUESTION" # Single query mode. Omit for interactive chat (type 'quit' to exit), and now you can play with your index interactively
--max-items N # Limit data preprocessing (default: -1, process all data)
--force-rebuild # Force rebuild index even if it exists
# Embedding Parameters
--embedding-model MODEL # e.g., facebook/contriever, text-embedding-3-small, mlx-community/Qwen3-Embedding-0.6B-8bit or nomic-embed-text
--embedding-mode MODE # sentence-transformers, openai, mlx, or ollama
# LLM Parameters (Text generation models)
--llm TYPE # LLM backend: openai, ollama, hf, or anthropic (default: openai)
--llm-model MODEL # Model name (default: gpt-4o) e.g., gpt-4o-mini, llama3.2:1b, Qwen/Qwen2.5-1.5B-Instruct
--thinking-budget LEVEL # Thinking budget for reasoning models: low/medium/high (supported by o3, o3-mini, GPT-Oss:20b, and other reasoning models)
# Search Parameters
--top-k N # Number of results to retrieve (default: 20)
--search-complexity N # Search complexity for graph traversal (default: 32)
# Chunking Parameters
--chunk-size N # Size of text chunks (default varies by source: 256 for most, 192 for WeChat)
--chunk-overlap N # Overlap between chunks (default varies: 25-128 depending on source)
# Index Building Parameters
--backend-name NAME # Backend to use: hnsw or diskann (default: hnsw)
--graph-degree N # Graph degree for index construction (default: 32)
--build-complexity N # Build complexity for index construction (default: 64)
--compact / --no-compact # Use compact storage (default: true). Must be `no-compact` for `no-recompute` build.
--recompute / --no-recompute # Enable/disable embedding recomputation (default: enabled). Should not do a `no-recompute` search in a `recompute` build.
```
</details>
### 📄 Personal Data Manager: Process Any Documents (`.pdf`, `.txt`, `.md`)!
Ask questions directly about your personal PDFs, documents, and any directory containing your files!
<p align="center">
<img src="videos/paper_clear.gif" alt="LEANN Document Search Demo" width="600">
</p>
The example below asks a question about summarizing our paper (uses default data in `data/`, which is a directory with diverse data sources: two papers, Pride and Prejudice, and a Technical report about LLM in Huawei in Chinese), and this is the **easiest example** to run here:
```bash
source .venv/bin/activate # Don't forget to activate the virtual environment
python -m apps.document_rag --query "What are the main techniques LEANN explores?"
```
<details>
<summary><strong>📋 Click to expand: Document-Specific Arguments</strong></summary>
#### Parameters
```bash
--data-dir DIR # Directory containing documents to process (default: data)
--file-types .ext .ext # Filter by specific file types (optional - all LlamaIndex supported types if omitted)
```
#### Example Commands
```bash
# Process all documents with larger chunks for academic papers
python -m apps.document_rag --data-dir "~/Documents/Papers" --chunk-size 1024
# Filter only markdown and Python files with smaller chunks
python -m apps.document_rag --data-dir "./docs" --chunk-size 256 --file-types .md .py
# Enable AST-aware chunking for code files
python -m apps.document_rag --enable-code-chunking --data-dir "./my_project"
# Or use the specialized code RAG for better code understanding
python -m apps.code_rag --repo-dir "./my_codebase" --query "How does authentication work?"
```
</details>
### 🎨 ColQwen: Multimodal PDF Retrieval with Vision-Language Models
Search through PDFs using both text and visual understanding with ColQwen2/ColPali models. Perfect for research papers, technical documents, and any PDFs with complex layouts, figures, or diagrams.
> **🍎 Mac Users**: ColQwen is optimized for Apple Silicon with MPS acceleration for faster inference!
```bash
# Build index from PDFs
python -m apps.colqwen_rag build --pdfs ./my_papers/ --index research_papers
# Search with text queries
python -m apps.colqwen_rag search research_papers "How does attention mechanism work?"
# Interactive Q&A
python -m apps.colqwen_rag ask research_papers --interactive
```
<details>
<summary><strong>📋 Click to expand: ColQwen Setup & Usage</strong></summary>
#### Prerequisites
```bash
# Install dependencies
uv pip install colpali_engine pdf2image pillow matplotlib qwen_vl_utils einops seaborn
brew install poppler # macOS only, for PDF processing
```
#### Build Index
```bash
python -m apps.colqwen_rag build \
--pdfs ./pdf_directory/ \
--index my_index \
--model colqwen2 # or colpali
```
#### Search
```bash
python -m apps.colqwen_rag search my_index "your question here" --top-k 5
```
#### Models
- **ColQwen2** (`colqwen2`): Latest vision-language model with improved performance
- **ColPali** (`colpali`): Proven multimodal retriever
For detailed usage, see the [ColQwen Guide](docs/COLQWEN_GUIDE.md).
</details>
### 📧 Your Personal Email Secretary: RAG on Apple Mail!
> **Note:** The examples below currently support macOS only. Windows support coming soon.
<p align="center">
<img src="videos/mail_clear.gif" alt="LEANN Email Search Demo" width="600">
</p>
Before running the example below, you need to grant full disk access to your terminal/VS Code in System Preferences → Privacy & Security → Full Disk Access.
```bash
python -m apps.email_rag --query "What's the food I ordered by DoorDash or Uber Eats mostly?"
```
**780K email chunks → 78MB storage.** Finally, search your email like you search Google.
<details>
<summary><strong>📋 Click to expand: Email-Specific Arguments</strong></summary>
#### Parameters
```bash
--mail-path PATH # Path to specific mail directory (auto-detects if omitted)
--include-html # Include HTML content in processing (useful for newsletters)
```
#### Example Commands
```bash
# Search work emails from a specific account
python -m apps.email_rag --mail-path "~/Library/Mail/V10/WORK_ACCOUNT"
# Find all receipts and order confirmations (includes HTML)
python -m apps.email_rag --query "receipt order confirmation invoice" --include-html
```
</details>
<details>
<summary><strong>📋 Click to expand: Example queries you can try</strong></summary>
Once the index is built, you can ask questions like:
- "Find emails from my boss about deadlines"
- "What did John say about the project timeline?"
- "Show me emails about travel expenses"
</details>
### 🔍 Time Machine for the Web: RAG Your Entire Chrome Browser History!
<p align="center">
<img src="videos/google_clear.gif" alt="LEANN Browser History Search Demo" width="600">
</p>
```bash
python -m apps.browser_rag --query "Tell me my browser history about machine learning?"
```
**38K browser entries → 6MB storage.** Your browser history becomes your personal search engine.
<details>
<summary><strong>📋 Click to expand: Browser-Specific Arguments</strong></summary>
#### Parameters
```bash
--chrome-profile PATH # Path to Chrome profile directory (auto-detects if omitted)
```
#### Example Commands
```bash
# Search academic research from your browsing history
python -m apps.browser_rag --query "arxiv papers machine learning transformer architecture"
# Track competitor analysis across work profile
python -m apps.browser_rag --chrome-profile "~/Library/Application Support/Google/Chrome/Work Profile" --max-items 5000
```
</details>
<details>
<summary><strong>📋 Click to expand: How to find your Chrome profile</strong></summary>
The default Chrome profile path is configured for a typical macOS setup. If you need to find your specific Chrome profile:
1. Open Terminal
2. Run: `ls ~/Library/Application\ Support/Google/Chrome/`
3. Look for folders like "Default", "Profile 1", "Profile 2", etc.
4. Use the full path as your `--chrome-profile` argument
**Common Chrome profile locations:**
- macOS: `~/Library/Application Support/Google/Chrome/Default`
- Linux: `~/.config/google-chrome/Default`
</details>
<details>
<summary><strong>💬 Click to expand: Example queries you can try</strong></summary>
Once the index is built, you can ask questions like:
- "What websites did I visit about machine learning?"
- "Find my search history about programming"
- "What YouTube videos did I watch recently?"
- "Show me websites I visited about travel planning"
</details>
### 💬 WeChat Detective: Unlock Your Golden Memories!
<p align="center">
<img src="videos/wechat_clear.gif" alt="LEANN WeChat Search Demo" width="600">
</p>
```bash
python -m apps.wechat_rag --query "Show me all group chats about weekend plans"
```
**400K messages → 64MB storage** Search years of chat history in any language.
<details>
<summary><strong>🔧 Click to expand: Installation Requirements</strong></summary>
First, you need to install the [WeChat exporter](https://github.com/sunnyyoung/WeChatTweak-CLI),
```bash
brew install sunnyyoung/repo/wechattweak-cli
```
or install it manually (if you have issues with Homebrew):
```bash
sudo packages/wechat-exporter/wechattweak-cli install
```
**Troubleshooting:**
- **Installation issues**: Check the [WeChatTweak-CLI issues page](https://github.com/sunnyyoung/WeChatTweak-CLI/issues/41)
- **Export errors**: If you encounter the error below, try restarting WeChat
```bash
Failed to export WeChat data. Please ensure WeChat is running and WeChatTweak is installed.
Failed to find or export WeChat data. Exiting.
```
</details>
<details>
<summary><strong>📋 Click to expand: WeChat-Specific Arguments</strong></summary>
#### Parameters
```bash
--export-dir DIR # Directory to store exported WeChat data (default: wechat_export_direct)
--force-export # Force re-export even if data exists
```
#### Example Commands
```bash
# Search for travel plans discussed in group chats
python -m apps.wechat_rag --query "travel plans" --max-items 10000
# Re-export and search recent chats (useful after new messages)
python -m apps.wechat_rag --force-export --query "work schedule"
```
</details>
<details>
<summary><strong>💬 Click to expand: Example queries you can try</strong></summary>
Once the index is built, you can ask questions like:
- "我想买魔术师约翰逊的球衣，给我一些对应聊天记录?" (Chinese: Show me chat records about buying Magic Johnson's jersey)
</details>
### 🤖 ChatGPT Chat History: Your Personal AI Conversation Archive!
Transform your ChatGPT conversations into a searchable knowledge base! Search through all your ChatGPT discussions about coding, research, brainstorming, and more.
```bash
python -m apps.chatgpt_rag --export-path chatgpt_export.html --query "How do I create a list in Python?"
```
**Unlock your AI conversation history.** Never lose track of valuable insights from your ChatGPT discussions again.
<details>
<summary><strong>📋 Click to expand: How to Export ChatGPT Data</strong></summary>
**Step-by-step export process:**
1. **Sign in to ChatGPT**
2. **Click your profile icon** in the top right corner
3. **Navigate to Settings** → **Data Controls**
4. **Click "Export"** under Export Data
5. **Confirm the export** request
6. **Download the ZIP file** from the email link (expires in 24 hours)
7. **Extract or use directly** with LEANN
**Supported formats:**
- `.html` files from ChatGPT exports
- `.zip` archives from ChatGPT
- Directories with multiple export files
</details>
<details>
<summary><strong>📋 Click to expand: ChatGPT-Specific Arguments</strong></summary>
#### Parameters
```bash
--export-path PATH # Path to ChatGPT export file (.html/.zip) or directory (default: ./chatgpt_export)
--separate-messages # Process each message separately instead of concatenated conversations
--chunk-size N # Text chunk size (default: 512)
--chunk-overlap N # Overlap between chunks (default: 128)
```
#### Example Commands
```bash
# Basic usage with HTML export
python -m apps.chatgpt_rag --export-path conversations.html
# Process ZIP archive from ChatGPT
python -m apps.chatgpt_rag --export-path chatgpt_export.zip
# Search with specific query
python -m apps.chatgpt_rag --export-path chatgpt_data.html --query "Python programming help"
# Process individual messages for fine-grained search
python -m apps.chatgpt_rag --separate-messages --export-path chatgpt_export.html
# Process directory containing multiple exports
python -m apps.chatgpt_rag --export-path ./chatgpt_exports/ --max-items 1000
```
</details>
<details>
<summary><strong>💡 Click to expand: Example queries you can try</strong></summary>
Once your ChatGPT conversations are indexed, you can search with queries like:
- "What did I ask ChatGPT about Python programming?"
- "Show me conversations about machine learning algorithms"
- "Find discussions about web development frameworks"
- "What coding advice did ChatGPT give me?"
- "Search for conversations about debugging techniques"
- "Find ChatGPT's recommendations for learning resources"
</details>
### 🤖 Claude Chat History: Your Personal AI Conversation Archive!
Transform your Claude conversations into a searchable knowledge base! Search through all your Claude discussions about coding, research, brainstorming, and more.
```bash
python -m apps.claude_rag --export-path claude_export.json --query "What did I ask about Python dictionaries?"
```
**Unlock your AI conversation history.** Never lose track of valuable insights from your Claude discussions again.
<details>
<summary><strong>📋 Click to expand: How to Export Claude Data</strong></summary>
**Step-by-step export process:**
1. **Open Claude** in your browser
2. **Navigate to Settings** (look for gear icon or settings menu)
3. **Find Export/Download** options in your account settings
4. **Download conversation data** (usually in JSON format)
5. **Place the file** in your project directory
*Note: Claude export methods may vary depending on the interface you're using. Check Claude's help documentation for the most current export instructions.*
**Supported formats:**
- `.json` files (recommended)
- `.zip` archives containing JSON data
- Directories with multiple export files
</details>
<details>
<summary><strong>📋 Click to expand: Claude-Specific Arguments</strong></summary>
#### Parameters
```bash
--export-path PATH # Path to Claude export file (.json/.zip) or directory (default: ./claude_export)
--separate-messages # Process each message separately instead of concatenated conversations
--chunk-size N # Text chunk size (default: 512)
--chunk-overlap N # Overlap between chunks (default: 128)
```
#### Example Commands
```bash
# Basic usage with JSON export
python -m apps.claude_rag --export-path my_claude_conversations.json
# Process ZIP archive from Claude
python -m apps.claude_rag --export-path claude_export.zip
# Search with specific query
python -m apps.claude_rag --export-path claude_data.json --query "machine learning advice"
# Process individual messages for fine-grained search
python -m apps.claude_rag --separate-messages --export-path claude_export.json
# Process directory containing multiple exports
python -m apps.claude_rag --export-path ./claude_exports/ --max-items 1000
```
</details>
<details>
<summary><strong>💡 Click to expand: Example queries you can try</strong></summary>
Once your Claude conversations are indexed, you can search with queries like:
- "What did I ask Claude about Python programming?"
- "Show me conversations about machine learning algorithms"
- "Find discussions about software architecture patterns"
- "What debugging advice did Claude give me?"
- "Search for conversations about data structures"
- "Find Claude's recommendations for learning resources"
</details>
### 💬 iMessage History: Your Personal Conversation Archive!
Transform your iMessage conversations into a searchable knowledge base! Search through all your text messages, group chats, and conversations with friends, family, and colleagues.
```bash
python -m apps.imessage_rag --query "What did we discuss about the weekend plans?"
```
**Unlock your message history.** Never lose track of important conversations, shared links, or memorable moments from your iMessage history.
<details>
<summary><strong>📋 Click to expand: How to Access iMessage Data</strong></summary>
**iMessage data location:**
iMessage conversations are stored in a SQLite database on your Mac at:
```
~/Library/Messages/chat.db
```
**Important setup requirements:**
1. **Grant Full Disk Access** to your terminal or IDE:
- Open **System Preferences** → **Security & Privacy** → **Privacy**
- Select **Full Disk Access** from the left sidebar
- Click the **+** button and add your terminal app (Terminal, iTerm2) or IDE (VS Code, etc.)
- Restart your terminal/IDE after granting access
2. **Alternative: Use a backup database**
- If you have Time Machine backups or manual copies of the database
- Use `--db-path` to specify a custom location
**Supported formats:**
- Direct access to `~/Library/Messages/chat.db` (default)
- Custom database path with `--db-path`
- Works with backup copies of the database
</details>
<details>
<summary><strong>📋 Click to expand: iMessage-Specific Arguments</strong></summary>
#### Parameters
```bash
--db-path PATH # Path to chat.db file (default: ~/Library/Messages/chat.db)
--concatenate-conversations # Group messages by conversation (default: True)
--no-concatenate-conversations # Process each message individually
--chunk-size N # Text chunk size (default: 1000)
--chunk-overlap N # Overlap between chunks (default: 200)
```
#### Example Commands
```bash
# Basic usage (requires Full Disk Access)
python -m apps.imessage_rag
# Search with specific query
python -m apps.imessage_rag --query "family dinner plans"
# Use custom database path
python -m apps.imessage_rag --db-path /path/to/backup/chat.db
# Process individual messages instead of conversations
python -m apps.imessage_rag --no-concatenate-conversations
# Limit processing for testing
python -m apps.imessage_rag --max-items 100 --query "weekend"
```
</details>
<details>
<summary><strong>💡 Click to expand: Example queries you can try</strong></summary>
Once your iMessage conversations are indexed, you can search with queries like:
- "What did we discuss about vacation plans?"
- "Find messages about restaurant recommendations"
- "Show me conversations with John about the project"
- "Search for shared links about technology"
- "Find group chat discussions about weekend events"
- "What did mom say about the family gathering?"
</details>
### MCP Integration: RAG on Live Data from Any Platform
Connect to live data sources through the Model Context Protocol (MCP). LEANN now supports real-time RAG on platforms like Slack, Twitter, and more through standardized MCP servers.
**Key Benefits:**
- **Live Data Access**: Fetch real-time data without manual exports
- **Standardized Protocol**: Use any MCP-compatible server
- **Easy Extension**: Add new platforms with minimal code
- **Secure Access**: MCP servers handle authentication
#### 💬 Slack Messages: Search Your Team Conversations
Transform your Slack workspace into a searchable knowledge base! Find discussions, decisions, and shared knowledge across all your channels.
```bash
# Test MCP server connection
python -m apps.slack_rag --mcp-server "slack-mcp-server" --test-connection
# Index and search Slack messages
python -m apps.slack_rag \
--mcp-server "slack-mcp-server" \
--workspace-name "my-team" \
--channels general dev-team random \
--query "What did we decide about the product launch?"
```
**📖 Comprehensive Setup Guide**: For detailed setup instructions, troubleshooting common issues (like "users cache is not ready yet"), and advanced configuration options, see our [**Slack Setup Guide**](docs/slack-setup-guide.md).
**Quick Setup:**
1. Install a Slack MCP server (e.g., `npm install -g slack-mcp-server`)
2. Create a Slack App and get API credentials (see detailed guide above)
3. Set environment variables:
```bash
export SLACK_BOT_TOKEN="xoxb-your-bot-token"
export SLACK_APP_TOKEN="xapp-your-app-token" # Optional
```
4. Test connection with `--test-connection` flag
**Arguments:**
- `--mcp-server`: Command to start the Slack MCP server
- `--workspace-name`: Slack workspace name for organization
- `--channels`: Specific channels to index (optional)
- `--concatenate-conversations`: Group messages by channel (default: true)
- `--max-messages-per-channel`: Limit messages per channel (default: 100)
- `--max-retries`: Maximum retries for cache sync issues (default: 5)
- `--retry-delay`: Initial delay between retries in seconds (default: 2.0)
#### 🐦 Twitter Bookmarks: Your Personal Tweet Library
Search through your Twitter bookmarks! Find that perfect article, thread, or insight you saved for later.
```bash
# Test MCP server connection
python -m apps.twitter_rag --mcp-server "twitter-mcp-server" --test-connection
# Index and search Twitter bookmarks
python -m apps.twitter_rag \
--mcp-server "twitter-mcp-server" \
--max-bookmarks 1000 \
--query "What AI articles did I bookmark about machine learning?"
```
**Setup Requirements:**
1. Install a Twitter MCP server (e.g., `npm install -g twitter-mcp-server`)
2. Get Twitter API credentials:
- Apply for a Twitter Developer Account at [developer.twitter.com](https://developer.twitter.com)
- Create a new app in the Twitter Developer Portal
- Generate API keys and access tokens with "Read" permissions
- For bookmarks access, you may need Twitter API v2 with appropriate scopes
```bash
export TWITTER_API_KEY="your-api-key"
export TWITTER_API_SECRET="your-api-secret"
export TWITTER_ACCESS_TOKEN="your-access-token"
export TWITTER_ACCESS_TOKEN_SECRET="your-access-token-secret"
```
3. Test connection with `--test-connection` flag
**Arguments:**
- `--mcp-server`: Command to start the Twitter MCP server
- `--username`: Filter bookmarks by username (optional)
- `--max-bookmarks`: Maximum bookmarks to fetch (default: 1000)
- `--no-tweet-content`: Exclude tweet content, only metadata
- `--no-metadata`: Exclude engagement metadata
</details>
<details>
<summary><strong>💡 Click to expand: Example queries you can try</strong></summary>
**Slack Queries:**
- "What did the team discuss about the project deadline?"
- "Find messages about the new feature launch"
- "Show me conversations about budget planning"
- "What decisions were made in the dev-team channel?"
**Twitter Queries:**
- "What AI articles did I bookmark last month?"
- "Find tweets about machine learning techniques"
- "Show me bookmarked threads about startup advice"
- "What Python tutorials did I save?"
</details>
<summary><strong>🔧 Using MCP with CLI Commands</strong></summary>
**Want to use MCP data with regular LEANN CLI?** You can combine MCP apps with CLI commands:
```bash
# Step 1: Use MCP app to fetch and index data
python -m apps.slack_rag --mcp-server "slack-mcp-server" --workspace-name "my-team"
# Step 2: The data is now indexed and available via CLI
leann search slack_messages "project deadline"
leann ask slack_messages "What decisions were made about the product launch?"
# Same for Twitter bookmarks
python -m apps.twitter_rag --mcp-server "twitter-mcp-server"
leann search twitter_bookmarks "machine learning articles"
```
**MCP vs Manual Export:**
- **MCP**: Live data, automatic updates, requires server setup
- **Manual Export**: One-time setup, works offline, requires manual data export
</details>
<details>
<summary><strong>🔧 Adding New MCP Platforms</strong></summary>
Want to add support for other platforms? LEANN's MCP integration is designed for easy extension:
1. **Find or create an MCP server** for your platform
2. **Create a reader class** following the pattern in `apps/slack_data/slack_mcp_reader.py`
3. **Create a RAG application** following the pattern in `apps/slack_rag.py`
4. **Test and contribute** back to the community!
**Popular MCP servers to explore:**
- GitHub repositories and issues
- Discord messages
- Notion pages
- Google Drive documents
- And many more in the MCP ecosystem!
</details>
### 🚀 Claude Code Integration: Transform Your Development Workflow!
<details>
<summary><strong>AST‑Aware Code Chunking</strong></summary>
LEANN features intelligent code chunking that preserves semantic boundaries (functions, classes, methods) for Python, Java, C#, and TypeScript, improving code understanding compared to text-based chunking.
📖 Read the [AST Chunking Guide →](docs/ast_chunking_guide.md)
</details>
**The future of code assistance is here.** Transform your development workflow with LEANN's native MCP integration for Claude Code. Index your entire codebase and get intelligent code assistance directly in your IDE.
**Key features:**
- 🔍 **Semantic code search** across your entire project, fully local index and lightweight
- 🧠 **AST-aware chunking** preserves code structure (functions, classes)
- 📚 **Context-aware assistance** for debugging and development
- 🚀 **Zero-config setup** with automatic language detection
```bash
# Install LEANN globally for MCP integration
uv tool install leann-core --with leann
claude mcp add --scope user leann-server -- leann_mcp
# Setup is automatic - just start using Claude Code!
```
Try our fully agentic pipeline with auto query rewriting, semantic search planning, and more:
**🔥 Ready to supercharge your coding?** [Complete Setup Guide →](packages/leann-mcp/README.md)
## Command Line Interface
LEANN includes a powerful CLI for document processing and search. Perfect for quick document indexing and interactive chat.
### Installation
If you followed the Quick Start, `leann` is already installed in your virtual environment:
```bash
source .venv/bin/activate
leann --help
```
**To make it globally available:**
```bash
# Install the LEANN CLI globally using uv tool
uv tool install leann-core --with leann
# Now you can use leann from anywhere without activating venv
leann --help
```
> **Note**: Global installation is required for Claude Code integration. The `leann_mcp` server depends on the globally available `leann` command.
### Usage Examples
```bash
# build from a specific directory, and my_docs is the index name(Here you can also build from multiple dict or multiple files)
leann build my-docs --docs ./your_documents
# Search your documents
leann search my-docs "machine learning concepts"
# Interactive chat with your documents
leann ask my-docs --interactive
# Ask a single question (non-interactive)
leann ask my-docs "Where are prompts configured?"
# Detect file changes since last build/watch checkpoint
leann watch my-docs
# List all your indexes
leann list
# Remove an index
leann remove my-docs
```
**Key CLI features:**
- Auto-detects document formats (PDF, TXT, MD, DOCX, PPTX + code files)
- **🧠 AST-aware chunking** for Python, Java, C#, TypeScript files
- Smart text chunking with overlap for all other content
- **📂 File change detection** via Merkle tree snapshots (`leann watch`)
- Multiple LLM providers (Ollama, OpenAI, HuggingFace)
- Organized index storage in `.leann/indexes/` (project-local)
- Support for advanced search parameters
<details>
<summary><strong>📋 Click to expand: Complete CLI Reference</strong></summary>
You can use `leann --help`, or `leann build --help`, `leann search --help`, `leann watch --help`, `leann ask --help`, `leann list --help`, `leann remove --help` to get the complete CLI reference.
**Build Command:**
```bash
leann build INDEX_NAME --docs DIRECTORY|FILE [DIRECTORY|FILE ...] [OPTIONS]
Options:
--backend {hnsw,diskann} Backend to use (default: hnsw)
--embedding-model MODEL Embedding model (default: facebook/contriever)
--graph-degree N Graph degree (default: 32)
--complexity N Build complexity (default: 64)
--force Force rebuild existing index
--compact / --no-compact Use compact storage (default: true). Must be `no-compact` for `no-recompute` build.
--recompute / --no-recompute Enable recomputation (default: true)
```
**Search Command:**
```bash
leann search INDEX_NAME QUERY [OPTIONS]
Options:
--top-k N Number of results (default: 5)
--complexity N Search complexity (default: 64)
--recompute / --no-recompute Enable/disable embedding recomputation (default: enabled). Should not do a `no-recompute` search in a `recompute` build.
--pruning-strategy {global,local,proportional}
```
**Watch Command:**
```bash
leann watch INDEX_NAME
# Compares the current file system state against the last checkpoint (Merkle tree snapshot)
# and reports which files have been added, removed, or modified, along with their chunk IDs.
#
# - Automatically saves a new checkpoint after detecting changes
# - Each subsequent run compares against the most recent checkpoint
# - File change detection uses SHA-256 content hashing via a Merkle tree
#
# Example output:
# === Changes since last checkpoint ===
# modified (1):
# - /path/to/file.py
# chunks: 42, 43, 44
```
**Ask Command:**
```bash
leann ask INDEX_NAME [OPTIONS]
Options:
--llm {ollama,openai,hf,anthropic} LLM provider (default: ollama)
--model MODEL Model name (default: qwen3:8b)
--interactive Interactive chat mode
--top-k N Retrieval count (default: 20)
```
**List Command:**
```bash
leann list
# Lists all indexes across all projects with status indicators:
# ✅ - Index is complete and ready to use
# ❌ - Index is incomplete or corrupted
# 📁 - CLI-created index (in .leann/indexes/)
# 📄 - App-created index (*.leann.meta.json files)
```
**Remove Command:**
```bash
leann remove INDEX_NAME [OPTIONS]
Options:
--force, -f Force removal without confirmation
# Smart removal: automatically finds and safely removes indexes
# - Shows all matching indexes across projects
# - Requires confirmation for cross-project removal
# - Interactive selection when multiple matches found
# - Supports both CLI and app-created indexes
```
</details>
## 🚀 Advanced Features
### 🎯 Metadata Filtering
LEANN supports a simple metadata filtering system to enable sophisticated use cases like document filtering by date/type, code search by file extension, and content management based on custom criteria.
```python
# Add metadata during indexing
builder.add_text(
"def authenticate_user(token): ...",
metadata={"file_extension": ".py", "lines_of_code": 25}
)
# Search with filters
results = searcher.search(
query="authentication function",
metadata_filters={
"file_extension": {"==": ".py"},
"lines_of_code": {"<": 100}
}
)
```
**Supported operators**: `==`, `!=`, `<`, `<=`, `>`, `>=`, `in`, `not_in`, `contains`, `starts_with`, `ends_with`, `is_true`, `is_false`
📖 **[Complete Metadata filtering guide →](docs/metadata_filtering.md)**
### 🔍 Grep Search
For exact text matching instead of semantic search, use the `use_grep` parameter:
```python
# Exact text search
results = searcher.search("banana‑crocodile", use_grep=True, top_k=1)
```
**Use cases**: Finding specific code patterns, error messages, function names, or exact phrases where semantic similarity isn't needed.
📖 **[Complete grep search guide →](docs/grep_search.md)**
## 🏗️ Architecture & How It Works
<p align="center">
<img src="assets/arch.png" alt="LEANN Architecture" width="800">
</p>
**The magic:** Most vector DBs store every single embedding (expensive). LEANN stores a pruned graph structure (cheap) and recomputes embeddings only when needed (fast).
**Core techniques:**
- **Graph-based selective recomputation:** Only compute embeddings for nodes in the search path
- **High-degree preserving pruning:** Keep important "hub" nodes while removing redundant connections
- **Dynamic batching:** Efficiently batch embedding computations for GPU utilization
- **Two-level search:** Smart graph traversal that prioritizes promising nodes
**Backends:**
- **HNSW** (default): Ideal for most datasets with maximum storage savings through full recomputation
- **DiskANN**: Advanced option with superior search performance, using PQ-based graph traversal with real-time reranking for the best speed-accuracy trade-off
## Benchmarks
**[DiskANN vs HNSW Performance Comparison →](benchmarks/diskann_vs_hnsw_speed_comparison.py)** - Compare search performance between both backends
**[Simple Example: Compare LEANN vs FAISS →](benchmarks/compare_faiss_vs_leann.py)** - See storage savings in action
### 📊 Storage Comparison
| System | DPR (2.1M) | Wiki (60M) | Chat (400K) | Email (780K) | Browser (38K) |
|--------|-------------|------------|-------------|--------------|---------------|
| Traditional vector database (e.g., FAISS) | 3.8 GB | 201 GB | 1.8 GB | 2.4 GB | 130 MB |
| LEANN | 324 MB | 6 GB | 64 MB | 79 MB | 6.4 MB |
| Savings| 91% | 97% | 97% | 97% | 95% |
## Reproduce Our Results
```bash
uv run benchmarks/run_evaluation.py # Will auto-download evaluation data and run benchmarks
uv run benchmarks/run_evaluation.py benchmarks/data/indices/rpj_wiki/rpj_wiki --num-queries 2000 # After downloading data, you can run the benchmark with our biggest index
```
The evaluation script downloads data automatically on first run. The last three results were tested with partial personal data, and you can reproduce them with your own data!
## 🔬 Paper
If you find Leann useful, please cite:
**[LEANN: A Low-Storage Vector Index](https://arxiv.org/abs/2506.08276)**
```bibtex
@misc{wang2025leannlowstoragevectorindex,
title={LEANN: A Low-Storage Vector Index},
author={Yichuan Wang and Shu Liu and Zhifei Li and Yongji Wu and Ziming Mao and Yilong Zhao and Xiao Yan and Zhiying Xu and Yang Zhou and Ion Stoica and Sewon Min and Matei Zaharia and Joseph E. Gonzalez},
year={2025},
eprint={2506.08276},
archivePrefix={arXiv},
primaryClass={cs.DB},
url={https://arxiv.org/abs/2506.08276},
}
```
## ✨ [Detailed Features →](docs/features.md)
## 🤝 [CONTRIBUTING →](docs/CONTRIBUTING.md)
## ❓ [FAQ →](docs/faq.md)
## 📈 [Roadmap →](docs/roadmap.md)
## 📄 License
MIT License - see [LICENSE](LICENSE) for details.
## 🙏 Acknowledgments
Core Contributors: [Yichuan Wang](https://yichuan-w.github.io/) & [Zhifei Li](https://github.com/andylizf).
Active Contributors: [Gabriel Dehan](https://github.com/gabriel-dehan), [Aakash Suresh](https://github.com/ASuresh0524)
We welcome more contributors! Feel free to open issues or submit PRs.
This work is done at [**Berkeley Sky Computing Lab**](https://sky.cs.berkeley.edu/).
## Star History
[](https://www.star-history.com/#yichuan-w/LEANN&Date)
<p align="center">
<strong>⭐ Star us on GitHub if Leann is useful for your research or applications!</strong>
</p>
<p align="center">
Made with ❤️ by the Leann team
</p>
## 🤖 Explore LEANN with AI
LEANN is indexed on [DeepWiki](https://deepwiki.com/yichuan-w/LEANN), so you can ask questions to LLMs using Deep Research to explore the codebase and get help to add new features.
================================================
FILE: apps/__init__.py
================================================
================================================
FILE: apps/base_rag_example.py
================================================
"""
Base class for unified RAG examples interface.
Provides common parameters and functionality for all RAG examples.
"""
import argparse
from abc import ABC, abstractmethod
from pathlib import Path
from typing import Any
import dotenv
from leann.api import LeannBuilder, LeannChat
# Optional import: older PyPI builds may not include interactive_utils
try:
from leann.interactive_utils import create_rag_session
except ImportError:
def create_rag_session(app_name: str, data_description: str):
class _SimpleSession:
def run_interactive_loop(self, handler):
print(f"Interactive session for {app_name}: {data_description}")
print("Interactive mode not available in this build")
return _SimpleSession()
from leann.registry import register_project_directory
# Optional import: older PyPI builds may not include settings
try:
from leann.settings import resolve_ollama_host, resolve_openai_api_key, resolve_openai_base_url
except ImportError:
# Minimal fallbacks if settings helpers are unavailable
import os
def resolve_ollama_host(value: str | None) -> str | None:
return value or os.getenv("LEANN_OLLAMA_HOST") or os.getenv("OLLAMA_HOST")
def resolve_openai_api_key(value: str | None) -> str | None:
return value or os.getenv("OPENAI_API_KEY")
def resolve_openai_base_url(value: str | None) -> str | None:
return value or os.getenv("OPENAI_BASE_URL")
dotenv.load_dotenv()
class BaseRAGExample(ABC):
"""Base class for all RAG examples with unified interface."""
def __init__(
self,
name: str,
description: str,
default_index_name: str,
):
self.name = name
self.description = description
self.default_index_name = default_index_name
self.parser = self._create_parser()
def _create_parser(self) -> argparse.ArgumentParser:
"""Create argument parser with common parameters."""
parser = argparse.ArgumentParser(
description=self.description, formatter_class=argparse.RawDescriptionHelpFormatter
)
# Core parameters (all examples share these)
core_group = parser.add_argument_group("Core Parameters")
core_group.add_argument(
"--index-dir",
type=str,
default=f"./{self.default_index_name}",
help=f"Directory to store the index (default: ./{self.default_index_name})",
)
core_group.add_argument(
"--query",
type=str,
default=None,
help="Query to run (if not provided, will run in interactive mode)",
)
# Allow subclasses to override default max_items
max_items_default = getattr(self, "max_items_default", -1)
core_group.add_argument(
"--max-items",
type=int,
default=max_items_default,
help="Maximum number of items to process -1 for all, means index all documents, and you should set it to a reasonable number if you have a large dataset and try at the first time)",
)
core_group.add_argument(
"--force-rebuild", action="store_true", help="Force rebuild index even if it exists"
)
# Embedding parameters
embedding_group = parser.add_argument_group("Embedding Parameters")
# Allow subclasses to override default embedding_model
embedding_model_default = getattr(self, "embedding_model_default", "facebook/contriever")
embedding_group.add_argument(
"--embedding-model",
type=str,
default=embedding_model_default,
help=f"Embedding model to use (default: {embedding_model_default}), we provide facebook/contriever, text-embedding-3-small,mlx-community/Qwen3-Embedding-0.6B-8bit or nomic-embed-text",
)
embedding_group.add_argument(
"--embedding-mode",
type=str,
default="sentence-transformers",
choices=["sentence-transformers", "openai", "mlx", "ollama"],
help="Embedding backend mode (default: sentence-transformers), we provide sentence-transformers, openai, mlx, or ollama",
)
embedding_group.add_argument(
"--embedding-host",
type=str,
default=None,
help="Override Ollama-compatible embedding host",
)
embedding_group.add_argument(
"--embedding-api-base",
type=str,
default=None,
help="Base URL for OpenAI-compatible embedding services",
)
embedding_group.add_argument(
"--embedding-api-key",
type=str,
default=None,
help="API key for embedding service (defaults to OPENAI_API_KEY)",
)
# LLM parameters
llm_group = parser.add_argument_group("LLM Parameters")
llm_group.add_argument(
"--llm",
type=str,
default="openai",
choices=["openai", "ollama", "hf", "simulated"],
help="LLM backend: openai, ollama, or hf (default: openai)",
)
llm_group.add_argument(
"--llm-model",
type=str,
default=None,
help="Model name (default: gpt-4o) e.g., gpt-4o-mini, llama3.2:1b, Qwen/Qwen2.5-1.5B-Instruct",
)
llm_group.add_argument(
"--llm-host",
type=str,
default=None,
help="Host for Ollama-compatible APIs (defaults to LEANN_OLLAMA_HOST/OLLAMA_HOST)",
)
llm_group.add_argument(
"--thinking-budget",
type=str,
choices=["low", "medium", "high"],
default=None,
help="Thinking budget for reasoning models (low/medium/high). Supported by GPT-Oss:20b and other reasoning models.",
)
llm_group.add_argument(
"--llm-api-base",
type=str,
default=None,
help="Base URL for OpenAI-compatible APIs",
)
llm_group.add_argument(
"--llm-api-key",
type=str,
default=None,
help="API key for OpenAI-compatible APIs (defaults to OPENAI_API_KEY)",
)
# AST Chunking parameters
ast_group = parser.add_argument_group("AST Chunking Parameters")
ast_group.add_argument(
"--use-ast-chunking",
action="store_true",
help="Enable AST-aware chunking for code files (requires astchunk)",
)
ast_group.add_argument(
"--ast-chunk-size",
type=int,
default=300,
help="Maximum CHARACTERS per AST chunk (default: 300). Final chunks may be larger due to overlap. For 512 token models: recommended 300 chars",
)
ast_group.add_argument(
"--ast-chunk-overlap",
type=int,
default=64,
help="Overlap between AST chunks in CHARACTERS (default: 64). Added to chunk size, not included in it",
)
ast_group.add_argument(
"--code-file-extensions",
nargs="+",
default=None,
help="Additional code file extensions to process with AST chunking (e.g., .py .java .cs .ts)",
)
ast_group.add_argument(
"--ast-fallback-traditional",
action="store_true",
default=True,
help="Fall back to traditional chunking if AST chunking fails (default: True)",
)
# Search parameters
search_group = parser.add_argument_group("Search Parameters")
search_group.add_argument(
"--top-k", type=int, default=20, help="Number of results to retrieve (default: 20)"
)
search_group.add_argument(
"--search-complexity",
type=int,
default=32,
help="Search complexity for graph traversal (default: 64)",
)
# Index building parameters
index_group = parser.add_argument_group("Index Building Parameters")
index_group.add_argument(
"--backend-name",
type=str,
default="hnsw",
choices=["hnsw", "diskann"],
help="Backend to use for index (default: hnsw)",
)
index_group.add_argument(
"--graph-degree",
type=int,
default=32,
help="Graph degree for index construction (default: 32)",
)
index_group.add_argument(
"--build-complexity",
type=int,
default=64,
help="Build complexity for index construction (default: 64)",
)
index_group.add_argument(
"--no-compact",
action="store_true",
help="Disable compact index storage",
)
index_group.add_argument(
"--no-recompute",
action="store_true",
help="Disable embedding recomputation",
)
# Add source-specific parameters
self._add_specific_arguments(parser)
return parser
@abstractmethod
def _add_specific_arguments(self, parser: argparse.ArgumentParser):
"""Add source-specific arguments. Override in subclasses."""
pass
@abstractmethod
async def load_data(self, args) -> list[dict[str, Any]]:
"""Load data from the source. Returns list of text chunks as dicts with 'text' and 'metadata' keys."""
pass
def get_llm_config(self, args) -> dict[str, Any]:
"""Get LLM configuration based on arguments."""
config = {"type": args.llm}
if args.llm == "openai":
config["model"] = args.llm_model or "gpt-4o"
config["base_url"] = resolve_openai_base_url(args.llm_api_base)
resolved_key = resolve_openai_api_key(args.llm_api_key)
if resolved_key:
config["api_key"] = resolved_key
elif args.llm == "ollama":
config["model"] = args.llm_model or "llama3.2:1b"
config["host"] = resolve_ollama_host(args.llm_host)
elif args.llm == "hf":
config["model"] = args.llm_model or "Qwen/Qwen2.5-1.5B-Instruct"
elif args.llm == "simulated":
# Simulated LLM doesn't need additional configuration
pass
return config
async def build_index(self, args, texts: list[dict[str, Any]]) -> str:
"""Build LEANN index from text chunks (dicts with 'text' and 'metadata' keys)."""
index_path = str(Path(args.index_dir) / f"{self.default_index_name}.leann")
print(f"\n[Building Index] Creating {self.name} index...")
print(f"Total text chunks: {len(texts)}")
embedding_options: dict[str, Any] = {}
if args.embedding_mode == "ollama":
embedding_options["host"] = resolve_ollama_host(args.embedding_host)
elif args.embedding_mode == "openai":
embedding_options["base_url"] = resolve_openai_base_url(args.embedding_api_base)
resolved_embedding_key = resolve_openai_api_key(args.embedding_api_key)
if resolved_embedding_key:
embedding_options["api_key"] = resolved_embedding_key
builder = LeannBuilder(
backend_name=args.backend_name,
embedding_model=args.embedding_model,
embedding_mode=args.embedding_mode,
embedding_options=embedding_options or None,
graph_degree=args.graph_degree,
complexity=args.build_complexity,
is_compact=not args.no_compact,
is_recompute=not args.no_recompute,
num_threads=1, # Force single-threaded mode
)
# Add texts in batches for better progress tracking
batch_size = 1000
for i in range(0, len(texts), batch_size):
batch = texts[i : i + batch_size]
for item in batch:
# Handle both dict format (from create_text_chunks) and plain strings
if isinstance(item, dict):
text = item.get("text", "")
metadata = item.get("metadata")
builder.add_text(text, metadata)
else:
builder.add_text(item)
print(f"Added {min(i + batch_size, len(texts))}/{len(texts)} texts...")
print("Building index structure...")
builder.build_index(index_path)
print(f"Index saved to: {index_path}")
# Register project directory so leann list can discover this index
# The index is saved as args.index_dir/index_name.leann
# We want to register the current working directory where the app is run
register_project_directory(Path.cwd())
return index_path
async def run_interactive_chat(self, args, index_path: str):
"""Run interactive chat with the index."""
chat = LeannChat(
index_path,
llm_config=self.get_llm_config(args),
system_prompt=f"You are a helpful assistant that answers questions about {self.name} data.",
complexity=args.search_complexity,
)
# Create interactive session
session = create_rag_session(
app_name=self.name.lower().replace(" ", "_"), data_description=self.name
)
def handle_query(query: str):
# Prepare LLM kwargs with thinking budget if specified
llm_kwargs = {}
if hasattr(args, "thinking_budget") and args.thinking_budget:
llm_kwargs["thinking_budget"] = args.thinking_budget
response = chat.ask(
query,
top_k=args.top_k,
complexity=args.search_complexity,
llm_kwargs=llm_kwargs,
)
print(f"\nAssistant: {response}\n")
session.run_interactive_loop(handle_query)
async def run_single_query(self, args, index_path: str, query: str):
"""Run a single query against the index."""
chat = LeannChat(
index_path,
llm_config=self.get_llm_config(args),
complexity=args.search_complexity,
)
print(f"\n[Query]: \033[36m{query}\033[0m")
# Prepare LLM kwargs with thinking budget if specified
llm_kwargs = {}
if hasattr(args, "thinking_budget") and args.thinking_budget:
llm_kwargs["thinking_budget"] = args.thinking_budget
response = chat.ask(
query, top_k=args.top_k, complexity=args.search_complexity, llm_kwargs=llm_kwargs
)
print(f"\n[Response]: \033[36m{response}\033[0m")
async def run(self):
"""Main entry point for the example."""
args = self.parser.parse_args()
# Check if index exists
index_path = str(Path(args.index_dir) / f"{self.default_index_name}.leann")
index_exists = Path(f"{index_path}.meta.json").exists()
if not index_exists or args.force_rebuild:
# Load data and build index
print(f"\n{'Rebuilding' if index_exists else 'Building'} index...")
texts = await self.load_data(args)
if not texts:
print("No data found to index!")
return
index_path = await self.build_index(args, texts)
else:
print(f"\nUsing existing index in {args.index_dir}")
# Run query or interactive mode
if args.query:
await self.run_single_query(args, index_path, args.query)
else:
await self.run_interactive_chat(args, index_path)
================================================
FILE: apps/browser_rag.py
================================================
"""
Browser History RAG example using the unified interface.
Supports Chrome browser history.
"""
import os
import sys
from pathlib import Path
from typing import Any
# Add parent directory to path for imports
sys.path.insert(0, str(Path(__file__).parent))
from base_rag_example import BaseRAGExample
from chunking import create_text_chunks
from .history_data.history import ChromeHistoryReader
class BrowserRAG(BaseRAGExample):
"""RAG example for Chrome browser history."""
def __init__(self):
# Set default values BEFORE calling super().__init__
self.embedding_model_default = (
"sentence-transformers/all-MiniLM-L6-v2" # Fast 384-dim model
)
super().__init__(
name="Browser History",
description="Process and query Chrome browser history with LEANN",
default_index_name="google_history_index",
)
def _add_specific_arguments(self, parser):
"""Add browser-specific arguments."""
browser_group = parser.add_argument_group("Browser Parameters")
browser_group.add_argument(
"--chrome-profile",
type=str,
default=None,
help="Path to Chrome profile directory (auto-detected if not specified)",
)
browser_group.add_argument(
"--auto-find-profiles",
action="store_true",
default=True,
help="Automatically find all Chrome profiles (default: True)",
)
browser_group.add_argument(
"--chunk-size", type=int, default=256, help="Text chunk size (default: 256)"
)
browser_group.add_argument(
"--chunk-overlap", type=int, default=128, help="Text chunk overlap (default: 128)"
)
def _get_chrome_base_path(self) -> Path:
"""Get the base Chrome profile path based on OS."""
if sys.platform == "darwin":
return Path.home() / "Library" / "Application Support" / "Google" / "Chrome"
elif sys.platform.startswith("linux"):
return Path.home() / ".config" / "google-chrome"
elif sys.platform == "win32":
return Path(os.environ["LOCALAPPDATA"]) / "Google" / "Chrome" / "User Data"
else:
raise ValueError(f"Unsupported platform: {sys.platform}")
def _find_chrome_profiles(self) -> list[Path]:
"""Auto-detect all Chrome profiles."""
base_path = self._get_chrome_base_path()
if not base_path.exists():
return []
profiles = []
# Check Default profile
default_profile = base_path / "Default"
if default_profile.exists() and (default_profile / "History").exists():
profiles.append(default_profile)
# Check numbered profiles
for item in base_path.iterdir():
if item.is_dir() and item.name.startswith("Profile "):
if (item / "History").exists():
profiles.append(item)
return profiles
async def load_data(self, args) -> list[dict[str, Any]]:
"""Load browser history and convert to text chunks."""
# Determine Chrome profiles
if args.chrome_profile and not args.auto_find_profiles:
profile_dirs = [Path(args.chrome_profile)]
else:
print("Auto-detecting Chrome profiles...")
profile_dirs = self._find_chrome_profiles()
# If specific profile given, filter to just that one
if args.chrome_profile:
profile_path = Path(args.chrome_profile)
profile_dirs = [p for p in profile_dirs if p == profile_path]
if not profile_dirs:
print("No Chrome profiles found!")
print("Please specify --chrome-profile manually")
return []
print(f"Found {len(profile_dirs)} Chrome profiles")
# Create reader
reader = ChromeHistoryReader()
# Process each profile
all_documents = []
total_processed = 0
for i, profile_dir in enumerate(profile_dirs):
print(f"\nProcessing profile {i + 1}/{len(profile_dirs)}: {profile_dir.name}")
try:
# Apply max_items limit per profile
max_per_profile = -1
if args.max_items > 0:
remaining = args.max_items - total_processed
if remaining <= 0:
break
max_per_profile = remaining
# Load history
documents = reader.load_data(
chrome_profile_path=str(profile_dir),
max_count=max_per_profile,
)
if documents:
all_documents.extend(documents)
total_processed += len(documents)
print(f"Processed {len(documents)} history entries from this profile")
except Exception as e:
print(f"Error processing {profile_dir}: {e}")
continue
if not all_documents:
print("No browser history found to process!")
return []
print(f"\nTotal history entries processed: {len(all_documents)}")
# Convert to text chunks
all_texts = create_text_chunks(
all_documents, chunk_size=args.chunk_size, chunk_overlap=args.chunk_overlap
)
return all_texts
if __name__ == "__main__":
import asyncio
# Example queries for browser history RAG
print("\n🌐 Browser History RAG Example")
print("=" * 50)
print("\nExample queries you can try:")
print("- 'What websites did I visit about machine learning?'")
print("- 'Find my search history about programming'")
print("- 'What YouTube videos did I watch recently?'")
print("- 'Show me websites about travel planning'")
print("\nNote: Make sure Chrome is closed before running\n")
rag = BrowserRAG()
asyncio.run(rag.run())
================================================
FILE: apps/chatgpt_data/__init__.py
================================================
================================================
FILE: apps/chatgpt_data/chatgpt_reader.py
================================================
"""
ChatGPT export data reader.
Reads and processes ChatGPT export data from chat.html files.
"""
import re
from pathlib import Path
from typing import Any
from zipfile import ZipFile
from bs4 import BeautifulSoup
from llama_index.core import Document
from llama_index.core.readers.base import BaseReader
class ChatGPTReader(BaseReader):
"""
ChatGPT export data reader.
Reads ChatGPT conversation data from exported chat.html files or zip archives.
Processes conversations into structured documents with metadata.
"""
def __init__(self, concatenate_conversations: bool = True) -> None:
"""
Initialize.
Args:
concatenate_conversations: Whether to concatenate messages within conversations for better context
"""
try:
from bs4 import BeautifulSoup # noqa
except ImportError:
raise ImportError("`beautifulsoup4` package not found: `pip install beautifulsoup4`")
self.concatenate_conversations = concatenate_conversations
def _extract_html_from_zip(self, zip_path: Path) -> str | None:
"""
Extract chat.html from ChatGPT export zip file.
Args:
zip_path: Path to the ChatGPT export zip file
Returns:
HTML content as string, or None if not found
"""
try:
with ZipFile(zip_path, "r") as zip_file:
# Look for chat.html or conversations.html
html_files = [
f
for f in zip_file.namelist()
if f.endswith(".html") and ("chat" in f.lower() or "conversation" in f.lower())
]
if not html_files:
print(f"No HTML chat file found in {zip_path}")
return None
# Use the first HTML file found
html_file = html_files[0]
print(f"Found HTML file: {html_file}")
with zip_file.open(html_file) as f:
return f.read().decode("utf-8", errors="ignore")
except Exception as e:
print(f"Error extracting HTML from zip {zip_path}: {e}")
return None
def _parse_chatgpt_html(self, html_content: str) -> list[dict]:
"""
Parse ChatGPT HTML export to extract conversations.
Args:
html_content: HTML content from ChatGPT export
Returns:
List of conversation dictionaries
"""
soup = BeautifulSoup(html_content, "html.parser")
conversations = []
# Try different possible structures for ChatGPT exports
# Structure 1: Look for conversation containers
conversation_containers = soup.find_all(
["div", "section"], class_=re.compile(r"conversation|chat", re.I)
)
if not conversation_containers:
# Structure 2: Look for message containers directly
conversation_containers = [soup] # Use the entire document as one conversation
for container in conversation_containers:
conversation = self._extract_conversation_from_container(container)
if conversation and conversation.get("messages"):
conversations.append(conversation)
# If no structured conversations found, try to extract all text as one conversation
if not conversations:
all_text = soup.get_text(separator="\n", strip=True)
if all_text:
conversations.append(
{
"title": "ChatGPT Conversation",
"messages": [{"role": "mixed", "content": all_text, "timestamp": None}],
"timestamp": None,
}
)
return conversations
def _extract_conversation_from_container(self, container) -> dict | None:
"""
Extract conversation data from a container element.
Args:
container: BeautifulSoup element containing conversation
Returns:
Dictionary with conversation data or None
"""
messages = []
# Look for message elements with various possible structures
message_selectors = ['[class*="message"]', '[class*="chat"]', "[data-message]", "p", "div"]
for selector in message_selectors:
message_elements = container.select(selector)
if message_elements:
break
else:
message_elements = []
# If no structured messages found, treat the entire container as one message
if not message_elements:
text_content = container.get_text(separator="\n", strip=True)
if text_content:
messages.append({"role": "mixed", "content": text_content, "timestamp": None})
else:
for element in message_elements:
message = self._extract_message_from_element(element)
if message:
messages.append(message)
if not messages:
return None
# Try to extract conversation title
title_element = container.find(["h1", "h2", "h3", "title"])
title = title_element.get_text(strip=True) if title_element else "ChatGPT Conversation"
# Try to extract timestamp from various possible locations
timestamp = self._extract_timestamp_from_container(container)
return {"title": title, "messages": messages, "timestamp": timestamp}
def _extract_message_from_element(self, element) -> dict | None:
"""
Extract message data from an element.
Args:
element: BeautifulSoup element containing message
Returns:
Dictionary with message data or None
"""
text_content = element.get_text(separator=" ", strip=True)
# Skip empty or very short messages
if not text_content or len(text_content.strip()) < 3:
return None
# Try to determine role (user/assistant) from class names or content
role = "mixed" # Default role
class_names = " ".join(element.get("class", [])).lower()
if "user" in class_names or "human" in class_names:
role = "user"
elif "assistant" in class_names or "ai" in class_names or "gpt" in class_names:
role = "assistant"
elif text_content.lower().startswith(("you:", "user:", "me:")):
role = "user"
text_content = re.sub(r"^(you|user|me):\s*", "", text_content, flags=re.IGNORECASE)
elif text_content.lower().startswith(("chatgpt:", "assistant:", "ai:")):
role = "assistant"
text_content = re.sub(
r"^(chatgpt|assistant|ai):\s*", "", text_content, flags=re.IGNORECASE
)
# Try to extract timestamp
timestamp = self._extract_timestamp_from_element(element)
return {"role": role, "content": text_content, "timestamp": timestamp}
def _extract_timestamp_from_element(self, element) -> str | None:
"""Extract timestamp from element."""
# Look for timestamp in various attributes and child elements
timestamp_attrs = ["data-timestamp", "timestamp", "datetime"]
for attr in timestamp_attrs:
if element.get(attr):
return element.get(attr)
# Look for time elements
time_element = element.find("time")
if time_element:
return time_element.get("datetime") or time_element.get_text(strip=True)
# Look for date-like text patterns
text = element.get_text()
date_patterns = [r"\d{4}-\d{2}-\d{2}", r"\d{1,2}/\d{1,2}/\d{4}", r"\w+ \d{1,2}, \d{4}"]
for pattern in date_patterns:
match = re.search(pattern, text)
if match:
return match.group()
return None
def _extract_timestamp_from_container(self, container) -> str | None:
"""Extract timestamp from conversation container."""
return self._extract_timestamp_from_element(container)
def _create_concatenated_content(self, conversation: dict) -> str:
"""
Create concatenated content from conversation messages.
Args:
conversation: Dictionary containing conversation data
Returns:
Formatted concatenated content
"""
title = conversation.get("title", "ChatGPT Conversation")
messages = conversation.get("messages", [])
timestamp = conversation.get("timestamp", "Unknown")
# Build message content
message_parts = []
for message in messages:
role = message.get("role", "mixed")
content = message.get("content", "")
msg_timestamp = message.get("timestamp", "")
if role == "user":
prefix = "[You]"
elif role == "assistant":
prefix = "[ChatGPT]"
else:
prefix = "[Message]"
# Add timestamp if available
if msg_timestamp:
prefix += f" ({msg_timestamp})"
message_parts.append(f"{prefix}: {content}")
concatenated_text = "\n\n".join(message_parts)
# Create final document content
doc_content = f"""Conversation: {title}
Date: {timestamp}
Messages ({len(messages)} messages):
{concatenated_text}
"""
return doc_content
def load_data(self, input_dir: str | None = None, **load_kwargs: Any) -> list[Document]:
"""
Load ChatGPT export data.
Args:
input_dir: Directory containing ChatGPT export files or path to specific file
**load_kwargs:
max_count (int): Maximum number of conversations to process
chatgpt_export_path (str): Specific path to ChatGPT export file/directory
include_metadata (bool): Whether to include metadata in documents
"""
docs: list[Document] = []
max_count = load_kwargs.get("max_count", -1)
chatgpt_export_path = load_kwargs.get("chatgpt_export_path", input_dir)
include_metadata = load_kwargs.get("include_metadata", True)
if not chatgpt_export_path:
print("No ChatGPT export path provided")
return docs
export_path = Path(chatgpt_export_path)
if not export_path.exists():
print(f"ChatGPT export path not found: {export_path}")
return docs
html_content = None
# Handle different input types
if export_path.is_file():
if export_path.suffix.lower() == ".zip":
# Extract HTML from zip file
html_content = self._extract_html_from_zip(export_path)
elif export_path.suffix.lower() == ".html":
# Read HTML file directly
try:
with open(export_path, encoding="utf-8", errors="ignore") as f:
html_content = f.read()
except Exception as e:
print(f"Error reading HTML file {export_path}: {e}")
return docs
else:
print(f"Unsupported file type: {export_path.suffix}")
return docs
elif export_path.is_dir():
# Look for HTML files in directory
html_files = list(export_path.glob("*.html"))
zip_files = list(export_path.glob("*.zip"))
if html_files:
# Use first HTML file found
html_file = html_files[0]
print(f"Found HTML file: {html_file}")
try:
with open(html_file, encoding="utf-8", errors="ignore") as f:
html_content = f.read()
except Exception as e:
print(f"Error reading HTML file {html_file}: {e}")
return docs
elif zip_files:
# Use first zip file found
zip_file = zip_files[0]
print(f"Found zip file: {zip_file}")
html_content = self._extract_html_from_zip(zip_file)
else:
print(f"No HTML or zip files found in {export_path}")
return docs
if not html_content:
print("No HTML content found to process")
return docs
# Parse conversations from HTML
print("Parsing ChatGPT conversations from HTML...")
conversations = self._parse_chatgpt_html(html_content)
if not conversations:
print("No conversations found in HTML content")
return docs
print(f"Found {len(conversations)} conversations")
# Process conversations into documents
count = 0
for conversation in conversations:
if max_count > 0 and count >= max_count:
break
if self.concatenate_conversations:
# Create one document per conversation with concatenated messages
doc_content = self._create_concatenated_content(conversation)
metadata = {}
if include_metadata:
metadata = {
"title": conversation.get("title", "ChatGPT Conversation"),
"timestamp": conversation.get("timestamp", "Unknown"),
"message_count": len(conversation.get("messages", [])),
"source": "ChatGPT Export",
}
doc = Document(text=doc_content, metadata=metadata)
docs.append(doc)
count += 1
else:
# Create separate documents for each message
for message in conversation.get("messages", []):
if max_count > 0 and count >= max_count:
break
role = message.get("role", "mixed")
content = message.get("content", "")
msg_timestamp = message.get("timestamp", "")
if not content.strip():
continue
# Create document content with context
doc_content = f"""Conversation: {conversation.get("title", "ChatGPT Conversation")}
Role: {role}
Timestamp: {msg_timestamp or conversation.get("timestamp", "Unknown")}
Message: {content}
"""
metadata = {}
if include_metadata:
metadata = {
"conversation_title": conversation.get("title", "ChatGPT Conversation"),
"role": role,
"timestamp": msg_timestamp or conversation.get("timestamp", "Unknown"),
"source": "ChatGPT Export",
}
doc = Document(text=doc_content, metadata=metadata)
docs.append(doc)
count += 1
print(f"Created {len(docs)} documents from ChatGPT export")
return docs
================================================
FILE: apps/chatgpt_rag.py
================================================
"""
ChatGPT RAG example using the unified interface.
Supports ChatGPT export data from chat.html files.
"""
import sys
from pathlib import Path
from typing import Any
# Add parent directory to path for imports
sys.path.insert(0, str(Path(__file__).parent))
from base_rag_example import BaseRAGExample
from chunking import create_text_chunks
from .chatgpt_data.chatgpt_reader import ChatGPTReader
class ChatGPTRAG(BaseRAGExample):
"""RAG example for ChatGPT conversation data."""
def __init__(self):
# Set default values BEFORE calling super().__init__
self.max_items_default = -1 # Process all conversations by default
self.embedding_model_default = (
"sentence-transformers/all-MiniLM-L6-v2" # Fast 384-dim model
)
super().__init__(
name="ChatGPT",
description="Process and query ChatGPT conversation exports with LEANN",
default_index_name="chatgpt_conversations_index",
)
def _add_specific_arguments(self, parser):
"""Add ChatGPT-specific arguments."""
chatgpt_group = parser.add_argument_group("ChatGPT Parameters")
chatgpt_group.add_argument(
"--export-path",
type=str,
default="./chatgpt_export",
help="Path to ChatGPT export file (.zip or .html) or directory containing exports (default: ./chatgpt_export)",
)
chatgpt_group.add_argument(
"--concatenate-conversations",
action="store_true",
default=True,
help="Concatenate messages within conversations for better context (default: True)",
)
chatgpt_group.add_argument(
"--separate-messages",
action="store_true",
help="Process each message as a separate document (overrides --concatenate-conversations)",
)
chatgpt_group.add_argument(
"--chunk-size", type=int, default=512, help="Text chunk size (default: 512)"
)
chatgpt_group.add_argument(
"--chunk-overlap", type=int, default=128, help="Text chunk overlap (default: 128)"
)
def _find_chatgpt_exports(self, export_path: Path) -> list[Path]:
"""
Find ChatGPT export files in the given path.
Args:
export_path: Path to search for exports
Returns:
List of paths to ChatGPT export files
"""
export_files = []
if export_path.is_file():
if export_path.suffix.lower() in [".zip", ".html"]:
export_files.append(export_path)
elif export_path.is_dir():
# Look for zip and html files
export_files.extend(export_path.glob("*.zip"))
export_files.extend(export_path.glob("*.html"))
return export_files
async def load_data(self, args) -> list[dict[str, Any]]:
"""Load ChatGPT export data and convert to text chunks."""
export_path = Path(args.export_path)
if not export_path.exists():
print(f"ChatGPT export path not found: {export_path}")
print(
"Please ensure you have exported your ChatGPT data and placed it in the correct location."
)
print("\nTo export your ChatGPT data:")
print("1. Sign in to ChatGPT")
print("2. Click on your profile icon → Settings → Data Controls")
print("3. Click 'Export' under Export Data")
print("4. Download the zip file from the email link")
print("5. Extract or place the file/directory at the specified path")
return []
# Find export files
export_files = self._find_chatgpt_exports(export_path)
if not export_files:
print(f"No ChatGPT export files (.zip or .html) found in: {export_path}")
return []
print(f"Found {len(export_files)} ChatGPT export files")
# Create reader with appropriate settings
concatenate = args.concatenate_conversations and not args.separate_messages
reader = ChatGPTReader(concatenate_conversations=concatenate)
# Process each export file
all_documents = []
total_processed = 0
for i, export_file in enumerate(export_files):
print(f"\nProcessing export file {i + 1}/{len(export_files)}: {export_file.name}")
try:
# Apply max_items limit per file
max_per_file = -1
if args.max_items > 0:
remaining = args.max_items - total_processed
if remaining <= 0:
break
max_per_file = remaining
# Load conversations
documents = reader.load_data(
chatgpt_export_path=str(export_file),
max_count=max_per_file,
include_metadata=True,
)
if documents:
all_documents.extend(documents)
total_processed += len(documents)
print(f"Processed {len(documents)} conversations from this file")
else:
print(f"No conversations loaded from {export_file}")
except Exception as e:
print(f"Error processing {export_file}: {e}")
continue
if not all_documents:
print("No conversations found to process!")
print("\nTroubleshooting:")
print("- Ensure the export file is a valid ChatGPT export")
print("- Check that the HTML file contains conversation data")
print("- Try extracting the zip file and pointing to the HTML file directly")
return []
print(f"\nTotal conversations processed: {len(all_documents)}")
print("Now starting to split into text chunks... this may take some time")
# Convert to text chunks
all_texts = create_text_chunks(
all_documents, chunk_size=args.chunk_size, chunk_overlap=args.chunk_overlap
)
print(f"Created {len(all_texts)} text chunks from {len(all_documents)} conversations")
return all_texts
if __name__ == "__main__":
import asyncio
# Example queries for ChatGPT RAG
print("\n🤖 ChatGPT RAG Example")
print("=" * 50)
print("\nExample queries you can try:")
print("- 'What did I ask about Python programming?'")
print("- 'Show me conversations about machine learning'")
print("- 'Find discussions about travel planning'")
print("- 'What advice did ChatGPT give me about career development?'")
print("- 'Search for conversations about cooking recipes'")
print("\nTo get started:")
print("1. Export your ChatGPT data from Settings → Data Controls → Export")
print("2. Place the downloaded zip file or extracted HTML in ./chatgpt_export/")
print("3. Run this script to build your personal ChatGPT knowledge base!")
print("\nOr run without --query for interactive mode\n")
rag = ChatGPTRAG()
asyncio.run(rag.run())
================================================
FILE: apps/chunking/__init__.py
================================================
"""Unified chunking utilities facade.
This module re-exports the packaged utilities from `leann.chunking_utils` so
that both repo apps (importing `chunking`) and installed wheels share one
single implementation. When running from the repo without installation, it
adds the `packages/leann-core/src` directory to `sys.path` as a fallback.
"""
import sys
from pathlib import Path
try:
from leann.chunking_utils import (
CODE_EXTENSIONS,
_traditional_chunks_as_dicts,
create_ast_chunks,
create_text_chunks,
create_traditional_chunks,
detect_code_files,
get_language_from_extension,
)
except Exception: # pragma: no cover - best-effort fallback for dev environment
repo_root = Path(__file__).resolve().parents[2]
leann_src = repo_root / "packages" / "leann-core" / "src"
if leann_src.exists():
sys.path.insert(0, str(leann_src))
from leann.chunking_utils import (
CODE_EXTENSIONS,
_traditional_chunks_as_dicts,
create_ast_chunks,
create_text_chunks,
create_traditional_chunks,
detect_code_files,
get_language_from_extension,
)
else:
raise
__all__ = [
"CODE_EXTENSIONS",
"_traditional_chunks_as_dicts",
"create_ast_chunks",
"create_text_chunks",
"create_traditional_chunks",
"detect_code_files",
"get_language_from_extension",
]
================================================
FILE: apps/claude_data/__init__.py
================================================
================================================
FILE: apps/claude_data/claude_reader.py
================================================
"""
Claude export data reader.
Reads and processes Claude conversation data from exported JSON files.
"""
import json
from pathlib import Path
from typing import Any
from zipfile import ZipFile
from llama_index.core import Document
from llama_index.core.readers.base import BaseReader
class ClaudeReader(BaseReader):
"""
Claude export data reader.
Reads Claude conversation data from exported JSON files or zip archives.
Processes conversations into structured documents with metadata.
"""
def __init__(self, concatenate_conversations: bool = True) -> None:
"""
Initialize.
Args:
concatenate_conversations: Whether to concatenate messages within conversations for better context
"""
self.concatenate_conversations = concatenate_conversations
def _extract_json_from_zip(self, zip_path: Path) -> list[str]:
"""
Extract JSON files from Claude export zip file.
Args:
zip_path: Path to the Claude export zip file
Returns:
List of JSON content strings, or empty list if not found
"""
json_contents = []
try:
with ZipFile(zip_path, "r") as zip_file:
# Look for JSON files
json_files = [f for f in zip_file.namelist() if f.endswith(".json")]
if not json_files:
print(f"No JSON files found in {zip_path}")
return []
print(f"Found {len(json_files)} JSON files in archive")
for json_file in json_files:
with zip_file.open(json_file) as f:
content = f.read().decode("utf-8", errors="ignore")
json_contents.append(content)
except Exception as e:
print(f"Error extracting JSON from zip {zip_path}: {e}")
return json_contents
def _parse_claude_json(self, json_content: str) -> list[dict]:
"""
Parse Claude JSON export to extract conversations.
Args:
json_content: JSON content from Claude export
Returns:
List of conversation dictionaries
"""
try:
data = json.loads(json_content)
except json.JSONDecodeError as e:
print(f"Error parsing JSON: {e}")
return []
conversations = []
# Handle different possible JSON structures
if isinstance(data, list):
# If data is a list of conversations
for item in data:
conversation = self._extract_conversation_from_json(item)
if conversation:
conversations.append(conversation)
elif isinstance(data, dict):
# Check for common structures
if "conversations" in data:
# Structure: {"conversations": [...]}
for item in data["conversations"]:
conversation = self._extract_conversation_from_json(item)
if conversation:
conversations.append(conversation)
elif "messages" in data:
# Single conversation with messages
conversation = self._extract_conversation_from_json(data)
if conversation:
conversations.append(conversation)
else:
# Try to treat the whole object as a conversation
conversation = self._extract_conversation_from_json(data)
if conversation:
conversations.append(conversation)
return conversations
def _extract_conversation_from_json(self, conv_data: dict) -> dict | None:
"""
Extract conversation data from a JSON object.
Args:
conv_data: Dictionary containing conversation data
Returns:
Dictionary with conversation data or None
"""
if not isinstance(conv_data, dict):
return None
messages = []
# Look for messages in various possible structures
message_sources = []
if "messages" in conv_data:
message_sources = conv_data["messages"]
elif "chat" in conv_data:
message_sources = conv_data["chat"]
elif "conversation" in conv_data:
message_sources = conv_data["conversation"]
else:
# If no clear message structure, try to extract from the object itself
if "content" in conv_data and "role" in conv_data:
message_sources = [conv_data]
for msg_data in message_sources:
message = self._extract_message_from_json(msg_data)
if message:
messages.append(message)
if not messages:
return None
# Extract conversation metadata
title = self._extract_title_from_conversation(conv_data, messages)
timestamp = self._extract_timestamp_from_conversation(conv_data)
return {"title": title, "messages": messages, "timestamp": timestamp}
def _extract_message_from_json(self, msg_data: dict) -> dict | None:
"""
Extract message data from a JSON message object.
Args:
msg_data: Dictionary containing message data
Returns:
Dictionary with message data or None
"""
if not isinstance(msg_data, dict):
return None
# Extract content from various possible fields
content = ""
content_fields = ["content", "text", "message", "body"]
for field in content_fields:
if msg_data.get(field):
content = str(msg_data[field])
break
if not content or len(content.strip()) < 3:
return None
# Extract role (user/assistant/human/ai/claude)
role = "mixed" # Default role
role_fields = ["role", "sender", "from", "author", "type"]
for field in role_fields:
if msg_data.get(field):
role_value = str(msg_data[field]).lower()
if role_value in ["user", "human", "person"]:
role = "user"
elif role_value in ["assistant", "ai", "claude", "bot"]:
role = "assistant"
break
# Extract timestamp
timestamp = self._extract_timestamp_from_message(msg_data)
return {"role": role, "content": content, "timestamp": timestamp}
def _extract_timestamp_from_message(self, msg_data: dict) -> str | None:
"""Extract timestamp from message data."""
timestamp_fields = ["timestamp", "created_at", "date", "time"]
for field in timestamp_fields:
if msg_data.get(field):
return str(msg_data[field])
return None
def _extract_timestamp_from_conversation(self, conv_data: dict) -> str | None:
"""Extract timestamp from conversation data."""
timestamp_fields = ["timestamp", "created_at", "date", "updated_at", "last_updated"]
for field in timestamp_fields:
if conv_data.get(field):
return str(conv_data[field])
return None
def _extract_title_from_conversation(self, conv_data: dict, messages: list) -> str:
"""Extract or generate title for conversation."""
# Try to find explicit title
title_fields = ["title", "name", "subject", "topic"]
for field in title_fields:
if conv_data.get(field):
return str(conv_data[field])
# Generate title from first user message
for message in messages:
if message.get("role") == "user":
content = message.get("content", "")
if content:
# Use first 50 characters as title
title = content[:50].strip()
if len(content) > 50:
title += "..."
return title
return "Claude Conversation"
def _create_concatenated_content(self, conversation: dict) -> str:
"""
Create concatenated content from conversation messages.
Args:
conversation: Dictionary containing conversation data
Returns:
Formatted concatenated content
"""
title = conversation.get("title", "Claude Conversation")
messages = conversation.get("messages", [])
timestamp = conversation.get("timestamp", "Unknown")
# Build message content
message_parts = []
for message in messages:
role = message.get("role", "mixed")
content = message.get("content", "")
msg_timestamp = message.get("timestamp", "")
if role == "user":
prefix = "[You]"
elif role == "assistant":
prefix = "[Claude]"
else:
prefix = "[Message]"
# Add timestamp if available
if msg_timestamp:
prefix += f" ({msg_timestamp})"
message_parts.append(f"{prefix}: {content}")
concatenated_text = "\n\n".join(message_parts)
# Create final document content
doc_content = f"""Conversation: {title}
Date: {timestamp}
Messages ({len(messages)} messages):
{concatenated_text}
"""
return doc_content
def load_data(self, input_dir: str | None = None, **load_kwargs: Any) -> list[Document]:
"""
Load Claude export data.
Args:
input_dir: Directory containing Claude export files or path to specific file
**load_kwargs:
max_count (int): Maximum number of conversations to process
claude_export_path (str): Specific path to Claude export file/directory
include_metadata (bool): Whether to include metadata in documents
"""
docs: list[Document] = []
max_count = load_kwargs.get("max_count", -1)
claude_export_path = load_kwargs.get("claude_export_path", input_dir)
include_metadata = load_kwargs.get("include_metadata", True)
if not claude_export_path:
print("No Claude export path provided")
return docs
export_path = Path(claude_export_path)
if not export_path.exists():
print(f"Claude export path not found: {export_path}")
return docs
json_contents = []
# Handle different input types
if export_path.is_file():
if export_path.suffix.lower() == ".zip":
# Extract JSON from zip file
json_contents = self._extract_json_from_zip(export_path)
elif export_path.suffix.lower() == ".json":
# Read JSON file directly
try:
with open(export_path, encoding="utf-8", errors="ignore") as f:
json_contents.append(f.read())
except Exception as e:
print(f"Error reading JSON file {export_path}: {e}")
return docs
else:
print(f"Unsupported file type: {export_path.suffix}")
return docs
elif export_path.is_dir():
# Look for JSON files in directory
json_files = list(export_path.glob("*.json"))
zip_files = list(export_path.glob("*.zip"))
if json_files:
print(f"Found {len(json_files)} JSON files in directory")
for json_file in json_files:
try:
with open(json_file, encoding="utf-8", errors="ignore") as f:
json_contents.append(f.read())
except Exception as e:
print(f"Error reading JSON file {json_file}: {e}")
continue
if zip_files:
print(f"Found {len(zip_files)} ZIP files in directory")
for zip_file in zip_files:
zip_contents = self._extract_json_from_zip(zip_file)
json_contents.extend(zip_contents)
if not json_files and not zip_files:
print(f"No JSON or ZIP files found in {export_path}")
return docs
if not json_contents:
print("No JSON content found to process")
return docs
# Parse conversations from JSON content
print("Parsing Claude conversations from JSON...")
all_conversations = []
for json_content in json_contents:
conversations = self._parse_claude_json(json_content)
all_conversations.extend(conversations)
if not all_conversations:
print("No conversations found in JSON content")
return docs
print(f"Found {len(all_conversations)} conversations")
# Process conversations into documents
count = 0
for conversation in all_conversations:
if max_count > 0 and count >= max_count:
break
if self.concatenate_conversations:
# Create one document per conversation with concatenated messages
doc_content = self._create_concatenated_content(conversation)
metadata = {}
if include_metadata:
metadata = {
"title": conversation.get("title", "Claude Conversation"),
"timestamp": conversation.get("timestamp", "Unknown"),
"message_count": len(conversation.get("messages", [])),
"source": "Claude Export",
}
doc = Document(text=doc_content, metadata=metadata)
docs.append(doc)
count += 1
else:
# Create separate documents for each message
for message in conversation.get("messages", []):
if max_count > 0 and count >= max_count:
break
role = message.get("role", "mixed")
content = message.get("content", "")
msg_timestamp = message.get("timestamp", "")
if not content.strip():
continue
# Create document content with context
doc_content = f"""Conversation: {conversation.get("title", "Claude Conversation")}
Role: {role}
Timestamp: {msg_timestamp or conversation.get("timestamp", "Unknown")}
Message: {content}
"""
metadata = {}
if include_metadata:
metadata = {
"conversation_title": conversation.get("title", "Claude Conversation"),
"role": role,
"timestamp": msg_timestamp or conversation.get("timestamp", "Unknown"),
"source": "Claude Export",
}
doc = Document(text=doc_content, metadata=metadata)
docs.append(doc)
count += 1
print(f"Created {len(docs)} documents from Claude export")
return docs
================================================
FILE: apps/claude_rag.py
================================================
"""
Claude RAG example using the unified interface.
Supports Claude export data from JSON files.
"""
import sys
from pathlib import Path
from typing import Any
# Add parent directory to path for imports
sys.path.insert(0, str(Path(__file__).parent))
from base_rag_example import BaseRAGExample
from chunking import create_text_chunks
from .claude_data.claude_reader import ClaudeReader
class ClaudeRAG(BaseRAGExample):
"""RAG example for Claude conversation data."""
def __init__(self):
# Set default values BEFORE calling super().__init__
self.max_items_default = -1 # Process all conversations by default
self.embedding_model_default = (
"sentence-transformers/all-MiniLM-L6-v2" # Fast 384-dim model
)
super().__init__(
name="Claude",
description="Process and query Claude conversation exports with LEANN",
default_index_name="claude_conversations_index",
)
def _add_specific_arguments(self, parser):
"""Add Claude-specific arguments."""
claude_group = parser.add_argument_group("Claude Parameters")
claude_group.add_argument(
"--export-path",
type=str,
default="./claude_export",
help="Path to Claude export file (.json or .zip) or directory containing exports (default: ./claude_export)",
)
claude_group.add_argument(
"--concatenate-conversations",
action="store_true",
default=True,
help="Concatenate messages within conversations for better context (default: True)",
)
claude_group.add_argument(
"--separate-messages",
action="store_true",
help="Process each message as a separate document (overrides --concatenate-conversations)",
)
claude_group.add_argument(
"--chunk-size", type=int, default=512, help="Text chunk size (default: 512)"
)
claude_group.add_argument(
"--chunk-overlap", type=int, default=128, help="Text chunk overlap (default: 128)"
)
def _find_claude_exports(self, export_path: Path) -> list[Path]:
"""
Find Claude export files in the given path.
Args:
export_path: Path to search for exports
Returns:
List of paths to Claude export files
"""
export_files = []
if export_path.is_file():
if export_path.suffix.lower() in [".zip", ".json"]:
export_files.append(export_path)
elif export_path.is_dir():
# Look for zip and json files
export_files.extend(export_path.glob("*.zip"))
export_files.extend(export_path.glob("*.json"))
return export_files
async def load_data(self, args) -> list[dict[str, Any]]:
"""Load Claude export data and convert to text chunks."""
export_path = Path(args.export_path)
if not export_path.exists():
print(f"Claude export path not found: {export_path}")
print(
"Please ensure you have exported your Claude data and placed it in the correct location."
)
print("\nTo export your Claude data:")
print("1. Open Claude in your browser")
print("2. Look for export/download options in settings or conversation menu")
print("3. Download the conversation data (usually in JSON format)")
print("4. Place the file/directory at the specified path")
print(
"\nNote: Claude export methods may vary. Check Claude's help documentation for current instructions."
)
return []
# Find export files
export_files = self._find_claude_exports(export_path)
if not export_files:
print(f"No Claude export files (.json or .zip) found in: {export_path}")
return []
print(f"Found {len(export_files)} Claude export files")
# Create reader with appropriate settings
concatenate = args.concatenate_conversations and not args.separate_messages
reader = ClaudeReader(concatenate_conversations=concatenate)
# Process each export file
all_documents = []
total_processed = 0
for i, export_file in enumerate(export_files):
print(f"\nProcessing export file {i + 1}/{len(export_files)}: {export_file.name}")
try:
# Apply max_items limit per file
max_per_file = -1
if args.max_items > 0:
remaining = args.max_items - total_processed
if remaining <= 0:
break
max_per_file = remaining
# Load conversations
documents = reader.load_data(
claude_export_path=str(export_file),
max_count=max_per_file,
include_metadata=True,
)
if documents:
all_documents.extend(documents)
total_processed += len(documents)
print(f"Processed {len(documents)} conversations from this file")
else:
print(f"No conversations loaded from {export_file}")
except Exception as e:
print(f"Error processing {export_file}: {e}")
continue
if not all_documents:
print("No conversations found to process!")
print("\nTroubleshooting:")
print("- Ensure the export file is a valid Claude export")
print("- Check that the JSON file contains conversation data")
print("- Try using a different export format or method")
print("- Check Claude's documentation for current export procedures")
return []
print(f"\nTotal conversations processed: {len(all_documents)}")
print("Now starting to split into text chunks... this may take some time")
# Convert to text chunks
all_texts = create_text_chunks(
all_documents, chunk_size=args.chunk_size, chunk_overlap=args.chunk_overlap
)
print(f"Created {len(all_texts)} text chunks from {len(all_documents)} conversations")
return all_texts
if __name__ == "__main__":
import asyncio
# Example queries for Claude RAG
print("\n🤖 Claude RAG Example")
print("=" * 50)
print("\nExample queries you can try:")
print("- 'What did I ask Claude about Python programming?'")
print("- 'Show me conversations about machine learning'")
print("- 'Find discussions about code optimization'")
print("- 'What advice did Claude give me about software design?'")
print("- 'Search for conversations about debugging techniques'")
print("\nTo get started:")
print("1. Export your Claude conversation data")
print("2. Place the JSON/ZIP file in ./claude_export/")
print("3. Run this script to build your personal Claude knowledge base!")
print("\nOr run without --query for interactive mode\n")
rag = ClaudeRAG()
asyncio.run(rag.run())
================================================
FILE: apps/code_rag.py
================================================
"""
Code RAG example using AST-aware chunking for optimal code understanding.
Specialized for code repositories with automatic language detection and
optimized chunking parameters.
"""
import sys
from pathlib import Path
from typing import Any
# Add parent directory to path for imports
sys.path.insert(0, str(Path(__file__).parent))
from base_rag_example import BaseRAGExample
from chunking import CODE_EXTENSIONS, create_text_chunks
from llama_index.core import SimpleDirectoryReader
class CodeRAG(BaseRAGExample):
"""Specialized RAG example for code repositories with AST-aware chunking."""
def __init__(self):
super().__init__(
name="Code",
description="Process and query code repositories with AST-aware chunking",
default_index_name="code_index",
)
# Override defaults for code-specific usage
self.embedding_model_default = "facebook/contriever" # Good for code
self.max_items_default = -1 # Process all code files by default
def _add_specific_arguments(self, parser):
"""Add code-specific arguments."""
code_group = parser.add_argument_group("Code Repository Parameters")
code_group.add_argument(
"--repo-dir",
type=str,
default=".",
help="Code repository directory to index (default: current directory)",
)
code_group.add_argument(
"--include-extensions",
nargs="+",
default=list(CODE_EXTENSIONS.keys()),
help="File extensions to include (default: supported code extensions)",
)
code_group.add_argument(
"--exclude-dirs",
nargs="+",
default=[
".git",
"__pycache__",
"node_modules",
"venv",
".venv",
"build",
"dist",
"target",
],
help="Directories to exclude from indexing",
)
code_group.add_argument(
"--max-file-size",
type=int,
default=1000000, # 1MB
help="Maximum file size in bytes to process (default: 1MB)",
)
code_group.add_argument(
"--include-comments",
action="store_true",
help="Include comments in chunking (useful for documentation)",
)
code_group.add_argument(
"--preserve-imports",
action="store_true",
default=True,
help="Try to preserve import statements in chunks (default: True)",
)
async def load_data(self, args) -> list[dict[str, Any]]:
"""Load code files and convert to AST-aware chunks."""
print(f"🔍 Scanning code repository: {args.repo_dir}")
print(f"📁 Including extensions: {args.include_extensions}")
print(f"🚫 Excluding directories: {args.exclude_dirs}")
# Check if repository directory exists
repo_path = Path(args.repo_dir)
if not repo_path.exists():
raise ValueError(f"Repository directory not found: {args.repo_dir}")
# Create exclusion filter
def file_filter(file_path: str) -> bool:
"""Filter out unwanted files and directories."""
path = Path(file_path)
# Check file size
try:
if path.stat().st_size > args.max_file_size:
print(f"⚠️ Skipping large file: {path.name} ({path.stat().st_size} bytes)")
return False
except Exception:
return False
# Check if in excluded directory
for exclude_dir in args.exclude_dirs:
if exclude_dir in path.parts:
return False
return True
try:
# Load documents with file filtering
documents = SimpleDirectoryReader(
args.repo_dir,
file_extractor=None,
recursive=True,
encoding="utf-8",
required_exts=args.include_extensions,
exclude_hidden=True,
).load_data(show_progress=True)
# Apply custom filtering
filtered_docs = []
for doc in documents:
file_path = doc.metadata.get("file_path", "")
if file_filter(file_path):
filtered_docs.append(doc)
documents = filtered_docs
except Exception as e:
print(f"❌ Error loading code files: {e}")
return []
if not documents:
print(
f"❌ No code files found in {args.repo_dir} with extensions {args.include_extensions}"
)
return []
print(f"✅ Loaded {len(documents)} code files")
# Show breakdown by language/extension
ext_counts = {}
for doc in documents:
file_path = doc.metadata.get("file_path", "")
if file_path:
ext = Path(file_path).suffix.lower()
ext_counts[ext] = ext_counts.get(ext, 0) + 1
print("📊 Files by extension:")
for ext, count in sorted(ext_counts.items()):
print(f" {ext}: {count} files")
# Use AST-aware chunking by default for code
print(
f"🧠 Using AST-aware chunking (chunk_size: {args.ast_chunk_size}, overlap: {args.ast_chunk_overlap})"
)
all_texts = create_text_chunks(
documents,
chunk_size=256, # Fallback for non-code files
chunk_overlap=64,
use_ast_chunking=True, # Always use AST for code RAG
ast_chunk_size=args.ast_chunk_size,
ast_chunk_overlap=args.ast_chunk_overlap,
code_file_extensions=args.include_extensions,
ast_fallback_traditional=True,
)
# Apply max_items limit if specified
if args.max_items > 0 and len(all_texts) > args.max_items:
print(f"⏳ Limiting to {args.max_items} chunks (from {len(all_texts)})")
all_texts = all_texts[: args.max_items]
print(f"✅ Generated {len(all_texts)} code chunks")
return all_texts
if __name__ == "__main__":
import asyncio
# Example queries for code RAG
print("\n💻 Code RAG Example")
print("=" * 50)
print("\nExample queries you can try:")
print("- 'How does the embedding computation work?'")
print("- 'What are the main classes in this codebase?'")
print("- 'Show me the search implementation'")
print("- 'How is error handling implemented?'")
print("- 'What design patterns are used?'")
print("- 'Explain the chunking logic'")
print("\n🚀 Features:")
print("- ✅ AST-aware chunking preserves code structure")
print("- ✅ Automatic language detection")
print("- ✅ Smart filtering of large files and common excludes")
print("- ✅ Optimized for code understanding")
print("\nUsage examples:")
print(" python -m apps.code_rag --repo-dir ./my_project")
print(
" python -m apps.code_rag --include-extensions .py .js --query 'How does authentication work?'"
)
print("\nOr run without --query for interactive mode\n")
rag = CodeRAG()
asyncio.run(rag.run())
================================================
FILE: apps/colqwen_rag.py
================================================
#!/usr/bin/env python3
"""
ColQwen RAG - Easy-to-use multimodal PDF retrieval with ColQwen2/ColPali
Usage:
python -m apps.colqwen_rag build --pdfs ./my_pdfs/ --index my_index
python -m apps.colqwen_rag search my_index "How does attention work?"
python -m apps.colqwen_rag ask my_index --interactive
"""
import argparse
import os
import sys
from pathlib import Path
from typing import Any, Optional, cast
# Add LEANN packages to path
_repo_root = Path(__file__).resolve().parents[1]
_leann_core_src = _repo_root / "packages" / "leann-core" / "src"
_leann_hnsw_pkg = _repo_root / "packages" / "leann-backend-hnsw"
if str(_leann_core_src) not in sys.path:
sys.path.append(str(_leann_core_src))
if str(_leann_hnsw_pkg) not in sys.path:
sys.path.append(str(_leann_hnsw_pkg))
import torch # noqa: E402
from pdf2image import convert_from_path # noqa: E402
from PIL import Image # noqa: E402
from torch.utils.data import DataLoader # noqa: E402
from tqdm import tqdm # noqa: E402
# Import the existing multi-vector implementation
sys.path.append(str(_repo_root / "apps" / "multimodal" / "vision-based-pdf-multi-vector"))
from leann_multi_vector import LeannMultiVector # noqa: E402
class ColQwenRAG:
"""Easy-to-use ColQwen RAG system for multimodal PDF retrieval."""
def __init__(self, model_type: str = "colpali"):
"""
Initialize ColQwen RAG system.
Args:
model_type: "colqwen2" or "colpali"
"""
self._assert_supported_transformers()
self.model_type = model_type
self.device = self._get_device()
# Use float32 on MPS to avoid memory issues, float16 on CUDA, bfloat16 on CPU
if self.device.type == "mps":
self.dtype = torch.float32
elif self.device.type == "cuda":
self.dtype = torch.float16
else:
self.dtype = torch.bfloat16
print(f"🚀 Initializing {model_type.upper()} on {self.device} with {self.dtype}")
# Load model and processor with MPS-optimized settings
try:
from colpali_engine import (
ColPali,
ColPaliProcessor,
ColQwen2,
ColQwen2Processor,
)
from colpali_engine.utils.torch_utils import ListDataset
self._list_dataset_cls: type[Any] = ListDataset
if model_type == "colqwen2":
self.model_name = "vidore/colqwen2-v1.0"
if self.device.type == "mps":
# For MPS, load on CPU first then move to avoid memory allocation issues
self.model = ColQwen2.from_pretrained(
self.model_name,
torch_dtype=self.dtype,
device_map="cpu",
low_cpu_mem_usage=True,
).eval()
self.model = self.model.to(self.device)
else:
self.model = ColQwen2.from_pretrained(
self.model_name,
torch_dtype=self.dtype,
device_map=self.device,
low_cpu_mem_usage=True,
).eval()
self.processor = ColQwen2Processor.from_pretrained(self.model_name)
else: # colpali
self.model_name = "vidore/colpali-v1.2"
if self.device.type == "mps":
# For MPS, load on CPU first then move to avoid memory allocation issues
self.model = ColPali.from_pretrained(
self.model_name,
torch_dtype=self.dtype,
device_map="cpu",
low_cpu_mem_usage=True,
).eval()
self.model = self.model.to(self.device)
else:
self.model = ColPali.from_pretrained(
self.model_name,
torch_dtype=self.dtype,
device_map=self.device,
low_cpu_mem_usage=True,
).eval()
self.processor = ColPaliProcessor.from_pretrained(self.model_name)
except Exception as e:
if "memory" in str(e).lower() or "offload" in str(e).lower():
print(f"⚠️ Memory constraint on {self.device}, using CPU with optimizations...")
self.device = torch.device("cpu")
self.dtype = torch.float32
if model_type == "colqwen2":
self.model = ColQwen2.from_pretrained(
self.model_name,
torch_dtype=self.dtype,
device_map="cpu",
low_cpu_mem_usage=True,
).eval()
self.processor = ColQwen2Processor.from_pretrained(self.model_name)
else:
self.model = ColPali.from_pretrained(
self.model_name,
torch_dtype=self.dtype,
device_map="cpu",
low_cpu_mem_usage=True,
).eval()
self.processor = ColPaliProcessor.from_pretrained(self.model_name)
else:
raise
def _assert_supported_transformers(self) -> None:
"""Fail fast on unsupported transformers versions."""
from importlib.metadata import PackageNotFoundError, version
try:
transformers_version = version("transformers")
except PackageNotFoundError:
return
def _parse_semver(value: str) -> tuple[int, int, int]:
parts = value.split(".")
numbers: list[int] = []
for part in parts[:3]:
digits = []
for ch in part:
if ch.isdigit():
digits.append(ch)
else:
break
numbers.append(int("".join(digits)) if digits else 0)
while len(numbers) < 3:
numbers.append(0)
return tuple(numbers) # type: ignore[return-value]
if _parse_semver(transformers_version) >= (4, 46, 0):
raise RuntimeError(
"Unsupported transformers version detected. "
"LEANN currently requires transformers<4.46 due to typing changes "
"and transformers 5.x removing symbols such as HybridCache. "
"Please install a compatible version, e.g. "
'`pip install "transformers<4.46"`.'
)
def _get_device(self):
"""Auto-select best available device."""
if torch.cuda.is_available():
return torch.device("cuda")
elif hasattr(torch.backends, "mps") and torch.backends.mps.is_available():
return torch.device("mps")
else:
return torch.device("cpu")
def build_index(self, pdf_paths: list[str], index_name: str, pages_dir: Optional[str] = None):
"""
Build multimodal index from PDF files.
Args:
pdf_paths: List of PDF file paths
index_name: Name for the index
pages_dir: Directory to save page images (optional)
"""
print(f"Building index '{index_name}' from {len(pdf_paths)} PDFs...")
# Convert PDFs to images
all_images = []
all_metadata = []
if pages_dir:
os.makedirs(pages_dir, exist_ok=True)
for pdf_path in tqdm(pdf_paths, desc="Converting PDFs"):
try:
images = convert_from_path(pdf_path, dpi=150)
pdf_name = Path(pdf_path).stem
for i, image in enumerate(images):
# Save image if pages_dir specified
if pages_dir:
image_path = Path(pages_dir) / f"{pdf_name}_page_{i + 1}.png"
image.save(image_path)
all_images.append(image)
all_metadata.append(
{
"pdf_path": pdf_path,
"pdf_name": pdf_name,
"page_number": i + 1,
"image_path": str(image_path) if pages_dir else None,
}
)
except Exception as e:
print(f"❌ Error processing {pdf_path}: {e}")
continue
print(f"📄 Converted {len(all_images)} pages from {len(pdf_paths)} PDFs")
if len(all_images) == 0:
raise RuntimeError(
"No PDF pages were converted to images, so there is nothing to embed.\n"
"Common causes:\n"
"- `poppler`/`pdftoppm` is missing (required by `pdf2image`)\n"
"- The input PDFs are encrypted/corrupt or have zero pages\n\n"
"Try:\n"
"- Install poppler (macOS: `brew install poppler`, Ubuntu: `apt-get install poppler-utils`)\n"
"- Re-run with a known-good PDF\n"
)
# Generate embeddings
print("🧠 Generating embeddings...")
embeddings = self._embed_images(all_images)
# Build LEANN index
print("🔍 Building LEANN index...")
leann_mv = LeannMultiVector(
index_path=index_name,
dim=embeddings.shape[-1],
embedding_model_name=self.model_type,
)
# Create collection and insert data
leann_mv.create_collection()
for i, (embedding, metadata) in enumerate(zip(embeddings, all_metadata)):
data = {
"doc_id": i,
"filepath": metadata.get("image_path", ""),
"colbert_vecs": embedding.numpy(), # Convert tensor to numpy
}
leann_mv.insert(data)
# Build the index
leann_mv.create_index()
print(f"✅ Index '{index_name}' built successfully!")
return leann_mv
def search(self, index_name: str, query: str, top_k: int = 5):
"""
Search the index with a text query.
Args:
index_name: Name of the index to search
query: Text query
top_k: Number of results to return
"""
print(f"🔍 Searching '{index_name}' for: '{query}'")
# Load index
leann_mv = LeannMultiVector(
index_path=index_name,
dim=128, # Will be updated when loading
embedding_model_name=self.model_type,
)
# Generate query embedding
query_embedding = self._embed_query(query)
# Search (returns list of (score, doc_id) tuples)
search_results = leann_mv.search(query_embedding.numpy(), topk=top_k)
# Display results
print(f"\n📋 Top {len(search_results)} results:")
for i, (score, doc_id) in enumerate(search_results, 1):
# Get metadata for this doc_id (we need to load the metadata)
print(f"{i}. Score: {score:.3f} | Doc ID: {doc_id}")
return search_results
def ask(self, index_name: str, interactive: bool = False):
"""
Interactive Q&A with the indexed documents.
Args:
index_name: Name of the index to query
interactive: Whether to run in interactive mode
"""
print(f"💬 ColQwen Chat with '{index_name}'")
if interactive:
print("Type 'quit' to exit, 'help' for commands")
while True:
try:
query = input("\n🤔 Your question: ").strip()
if query.lower() in ["quit", "exit", "q"]:
break
elif query.lower() == "help":
print("Commands: quit/exit/q (exit), help (this message)")
continue
elif not query:
continue
self.search(index_name, query, top_k=3)
# TODO: Add answer generation with Qwen-VL
print("\n💡 For detailed answers, we can integrate Qwen-VL here!")
except KeyboardInterrupt:
print("\n👋 Goodbye!")
break
else:
query = input("🤔 Your question: ").strip()
if query:
self.search(index_name, query)
def _embed_images(self, images: list[Image.Image]) -> torch.Tensor:
"""Generate embeddings for a list of images."""
if not images:
raise RuntimeError("No images provided for embedding.")
dataset = self._list_dataset_cls(images)
dataloader = DataLoader(dataset, batch_size=1, shuffle=False, collate_fn=lambda x: x)
embeddings = []
with torch.no_grad():
for batch in tqdm(dataloader, desc="Embedding images"):
batch_images = cast(list, batch)
batch_inputs = self.processor.process_images(batch_images).to(self.device)
batch_embeddings = self.model(**batch_inputs)
embeddings.append(batch_embeddings.cpu())
if not embeddings:
raise RuntimeError(
"Image embedding produced no tensors (empty embedding list). "
"This usually indicates that no images were processed successfully."
)
return torch.cat(embeddings, dim=0)
def _embed_query(self, query: str) -> torch.Tensor:
"""Generate embedding for a text query."""
with torch.no_grad():
query_inputs = self.processor.process_queries([query]).to(self.device)
query_embedding = self.model(**query_inputs)
return query_embedding.cpu()
def main():
parser = argparse.ArgumentParser(description="ColQwen RAG - Easy multimodal PDF retrieval")
subparsers = parser.add_subparsers(dest="command", help="Available commands")
# Build command
build_parser = subparsers.add_parser("build", help="Build index from PDFs")
build_parser.add_argument("--pdfs", required=True, help="Directory containing PDF files")
build_parser.add_argument("--index", required=True, help="Index name")
build_parser.add_argument(
"--model", choices=["colqwen2", "colpali"], default="colqwen2", help="Model to use"
)
build_parser.add_argument("--pages-dir", help="Directory to save page images")
# Search command
search_parser = subparsers.add_parser("search", help="Search the index")
search_parser.add_argument("index", help="Index name")
search_parser.add_argument("query", help="Search query")
search_parser.add_argument("--top-k", type=int, default=5, help="Number of results")
search_parser.add_argument(
"--model", choices=["colqwen2", "colpali"], default="colqwen2", help="Model to use"
)
# Ask command
ask_parser = subparsers.add_parser("ask", help="Interactive Q&A")
ask_parser.add_argument("index", help="Index name")
ask_parser.add_argument("--interactive", action="store_true", help="Interactive mode")
ask_parser.add_argument(
"--model", choices=["colqwen2", "colpali"], default="colqwen2", help="Model to use"
)
args = parser.parse_args()
if not args.command:
parser.print_help()
return
# Initialize ColQwen RAG
if args.command == "build":
colqwen = ColQwenRAG(args.model)
# Get PDF files
pdf_dir = Path(args.pdfs)
if pdf_dir.is_file() and pdf_dir.suffix.lower() == ".pdf":
pdf_paths = [str(pdf_dir)]
elif pdf_dir.is_dir():
pdf_paths = [str(p) for p in pdf_dir.glob("*.pdf")]
else:
print(f"❌ Invalid PDF path: {args.pdfs}")
return
if not pdf_paths:
print(f"❌ No PDF files found in {args.pdfs}")
return
colqwen.build_index(pdf_paths, args.index, args.pages_dir)
elif args.command == "search":
colqwen = ColQwenRAG(args.model)
colqwen.search(args.index, args.query, args.top_k)
elif args.command == "ask":
colqwen = ColQwenRAG(args.model)
colqwen.ask(args.index, args.interactive)
if __name__ == "__main__":
main()
================================================
FILE: apps/document_rag.py
================================================
"""
Document RAG example using the unified interface.
Supports PDF, TXT, MD, and other document formats.
"""
import sys
from pathlib import Path
from typing import Any
# Add parent directory to path for imports
sys.path.insert(0, str(Path(__file__).parent))
from base_rag_example import BaseRAGExample
from chunking import create_text_chunks
from llama_index.core import SimpleDirectoryReader
class DocumentRAG(BaseRAGExample):
"""RAG example for document processing (PDF, TXT, MD, etc.)."""
def __init__(self):
super().__init__(
name="Document",
description="Process and query documents (PDF, TXT, MD, etc.) with LEANN",
default_index_name="test_doc_files",
)
def _add_specific_arguments(self, parser):
"""Add document-specific arguments."""
doc_group = parser.add_argument_group("Document Parameters")
doc_group.add_argument(
"--data-dir",
type=str,
default="data",
help="Directory containing documents to index (default: data)",
)
doc_group.add_argument(
"--file-types",
nargs="+",
default=None,
help="Filter by file types (e.g., .pdf .txt .md). If not specified, all supported types are processed",
)
doc_group.add_argument(
"--chunk-size", type=int, default=256, help="Text chunk size (default: 256)"
)
doc_group.add_argument(
"--chunk-overlap", type=int, default=128, help="Text chunk overlap (default: 128)"
)
doc_group.add_argument(
"--enable-code-chunking",
action="store_true",
help="Enable AST-aware chunking for code files in the data directory",
)
async def load_data(self, args) -> list[dict[str, Any]]:
"""Load documents and convert to text chunks."""
print(f"Loading documents from: {args.data_dir}")
if args.file_types:
print(f"Filtering by file types: {args.file_types}")
else:
print("Processing all supported file types")
# Check if data directory exists
data_path = Path(args.data_dir)
gitextract_xaekqrq1/
├── .devcontainer/
│ └── devcontainer.json
├── .github/
│ ├── ISSUE_TEMPLATE/
│ │ ├── bug_report.yml
│ │ ├── config.yml
│ │ └── feature_request.yml
│ ├── pull_request_template.md
│ └── workflows/
│ ├── build-and-publish.yml
│ ├── build-reusable.yml
│ ├── link-check.yml
│ └── release-manual.yml
├── .gitignore
├── .gitmodules
├── .pre-commit-config.yaml
├── .python-version
├── .vscode/
│ ├── extensions.json
│ └── settings.json
├── CLAUDE.md
├── LICENSE
├── README.md
├── apps/
│ ├── __init__.py
│ ├── base_rag_example.py
│ ├── browser_rag.py
│ ├── chatgpt_data/
│ │ ├── __init__.py
│ │ └── chatgpt_reader.py
│ ├── chatgpt_rag.py
│ ├── chunking/
│ │ └── __init__.py
│ ├── claude_data/
│ │ ├── __init__.py
│ │ └── claude_reader.py
│ ├── claude_rag.py
│ ├── code_rag.py
│ ├── colqwen_rag.py
│ ├── document_rag.py
│ ├── email_data/
│ │ ├── LEANN_email_reader.py
│ │ └── email.py
│ ├── email_rag.py
│ ├── gemini_data/
│ │ ├── __init__.py
│ │ └── gemini_reader.py
│ ├── gemini_rag.py
│ ├── history_data/
│ │ ├── __init__.py
│ │ ├── history.py
│ │ └── wechat_history.py
│ ├── image_rag.py
│ ├── imessage_data/
│ │ ├── __init__.py
│ │ └── imessage_reader.py
│ ├── imessage_rag.py
│ ├── multimodal/
│ │ └── vision-based-pdf-multi-vector/
│ │ ├── README.md
│ │ ├── colqwen_forward.py
│ │ ├── leann_multi_vector.py
│ │ ├── multi-vector-leann-paper-example.py
│ │ ├── multi-vector-leann-similarity-map.py
│ │ ├── vidore_v1_benchmark.py
│ │ └── vidore_v2_benchmark.py
│ ├── qwen_data/
│ │ ├── __init__.py
│ │ └── qwen_reader.py
│ ├── qwen_rag.py
│ ├── semantic_file_search/
│ │ ├── leann-plus-temporal-search.py
│ │ ├── leann_index_builder.py
│ │ └── spotlight_index_dump.py
│ ├── slack_data/
│ │ ├── __init__.py
│ │ └── slack_mcp_reader.py
│ ├── slack_rag.py
│ ├── twitter_data/
│ │ ├── __init__.py
│ │ └── twitter_mcp_reader.py
│ ├── twitter_rag.py
│ └── wechat_rag.py
├── benchmarks/
│ ├── README.md
│ ├── __init__.py
│ ├── benchmark_embeddings.py
│ ├── benchmark_no_recompute.py
│ ├── bm25_diskann_baselines/
│ │ ├── README.md
│ │ ├── run_bm25.py
│ │ └── run_diskann.py
│ ├── compare_faiss_vs_leann.py
│ ├── diskann_vs_hnsw_speed_comparison.py
│ ├── enron_emails/
│ │ ├── README.md
│ │ ├── data/
│ │ │ └── .gitignore
│ │ ├── evaluate_enron_emails.py
│ │ └── setup_enron_emails.py
│ ├── faiss_only.py
│ ├── financebench/
│ │ ├── README.md
│ │ ├── evaluate_financebench.py
│ │ ├── setup_financebench.py
│ │ └── verify_recall.py
│ ├── issue_159.py
│ ├── laion/
│ │ ├── .gitignore
│ │ ├── README.md
│ │ ├── evaluate_laion.py
│ │ └── setup_laion.py
│ ├── llm_utils.py
│ ├── micro_tpt.py
│ ├── run_evaluation.py
│ ├── simple_mac_tpt_test.py
│ └── update/
│ ├── README.md
│ ├── __init__.py
│ ├── bench_hnsw_rng_recompute.py
│ ├── bench_update_vs_offline_search.py
│ └── plot_bench_results.py
├── data/
│ ├── PrideandPrejudice.txt
│ └── huawei_pangu.md
├── demo.ipynb
├── docker/
│ ├── Dockerfile
│ ├── Dockerfile.cpu
│ ├── Dockerfile.dev
│ └── README.md
├── docs/
│ ├── COLQWEN_GUIDE.md
│ ├── CONTRIBUTING.md
│ ├── RELEASE.md
│ ├── THINKING_BUDGET_FEATURE.md
│ ├── ast_chunking_guide.md
│ ├── code/
│ │ └── embedding_model_compare.py
│ ├── configuration-guide.md
│ ├── faq.md
│ ├── features.md
│ ├── grep_search.md
│ ├── metadata_filtering.md
│ ├── normalized_embeddings.md
│ ├── openclaw-setup.md
│ ├── react_agent.md
│ ├── roadmap.md
│ ├── slack-setup-guide.md
│ └── ultimate_goal.md
├── examples/
│ ├── __init__.py
│ ├── basic_demo.py
│ ├── dynamic_update_no_recompute.py
│ ├── grep_search_example.py
│ ├── mcp_integration_demo.py
│ ├── mlx_demo.py
│ └── spoiler_free_book_rag.py
├── llms.txt
├── packages/
│ ├── __init__.py
│ ├── leann/
│ │ ├── README.md
│ │ ├── __init__.py
│ │ └── pyproject.toml
│ ├── leann-backend-diskann/
│ │ ├── __init__.py
│ │ ├── leann_backend_diskann/
│ │ │ ├── __init__.py
│ │ │ ├── diskann_backend.py
│ │ │ ├── diskann_embedding_server.py
│ │ │ ├── embedding_pb2.py
│ │ │ └── graph_partition.py
│ │ ├── pyproject.toml
│ │ └── third_party/
│ │ ├── embedding.pb.cc
│ │ └── embedding.proto
│ ├── leann-backend-hnsw/
│ │ ├── CMakeLists.txt
│ │ ├── leann_backend_hnsw/
│ │ │ ├── __init__.py
│ │ │ ├── convert_to_csr.py
│ │ │ ├── hnsw_backend.py
│ │ │ └── hnsw_embedding_server.py
│ │ └── pyproject.toml
│ ├── leann-backend-ivf/
│ │ ├── README.md
│ │ ├── leann_backend_ivf/
│ │ │ ├── __init__.py
│ │ │ └── ivf_backend.py
│ │ └── pyproject.toml
│ ├── leann-core/
│ │ ├── pyproject.toml
│ │ └── src/
│ │ └── leann/
│ │ ├── __init__.py
│ │ ├── api.py
│ │ ├── chat.py
│ │ ├── chunking_utils.py
│ │ ├── cli.py
│ │ ├── embedding_compute.py
│ │ ├── embedding_server_manager.py
│ │ ├── interactive_utils.py
│ │ ├── interface.py
│ │ ├── mcp.py
│ │ ├── metadata_filter.py
│ │ ├── react_agent.py
│ │ ├── registry.py
│ │ ├── searcher_base.py
│ │ ├── server.py
│ │ ├── settings.py
│ │ └── sync.py
│ ├── leann-mcp/
│ │ └── README.md
│ └── wechat-exporter/
│ ├── __init__.py
│ ├── main.py
│ └── wechattweak-cli
├── pyproject.toml
├── scripts/
│ └── hf_upload.py
├── skills/
│ └── leann-memory/
│ ├── README.md
│ ├── claw.json
│ └── instructions.md
├── sky/
│ └── leann-build.yaml
└── tests/
├── README.md
├── openclaw/
│ ├── .gitignore
│ ├── __init__.py
│ ├── conftest.py
│ ├── docker-compose.yml
│ ├── fixtures/
│ │ ├── MEMORY.md
│ │ └── memory/
│ │ ├── 2026-02-15.md
│ │ ├── 2026-02-20.md
│ │ └── 2026-02-25.md
│ ├── run_docker_test.sh
│ ├── test_build_and_search.py
│ ├── test_mcp_e2e.py
│ ├── test_mcp_protocol.py
│ ├── test_openclaw_e2e.py
│ └── test_skill_manifest.py
├── support/
│ └── fake_embedding_server_module.py
├── test_astchunk_integration.py
├── test_basic.py
├── test_ci_minimal.py
├── test_cli_ask.py
├── test_cli_daemon_workflow.py
├── test_cli_prompt_template.py
├── test_cli_verbosity.py
├── test_cpu_only_install.py
├── test_diskann_partition.py
├── test_document_rag.py
├── test_embedding_prompt_template.py
├── test_embedding_server_cli_flags.py
├── test_embedding_server_manager.py
├── test_embedding_server_manager_e2e.py
├── test_hybrid_search.py
├── test_incremental_build.py
├── test_lmstudio_bridge.py
├── test_mcp_integration.py
├── test_mcp_standalone.py
├── test_metadata_filtering.py
├── test_minimax_provider.py
├── test_prompt_template_e2e.py
├── test_prompt_template_persistence.py
├── test_readme_examples.py
├── test_sync.py
└── test_token_truncation.py
SYMBOL INDEX (1240 symbols across 124 files)
FILE: apps/base_rag_example.py
function create_rag_session (line 19) | def create_rag_session(app_name: str, data_description: str):
function resolve_ollama_host (line 37) | def resolve_ollama_host(value: str | None) -> str | None:
function resolve_openai_api_key (line 40) | def resolve_openai_api_key(value: str | None) -> str | None:
function resolve_openai_base_url (line 43) | def resolve_openai_base_url(value: str | None) -> str | None:
class BaseRAGExample (line 50) | class BaseRAGExample(ABC):
method __init__ (line 53) | def __init__(
method _create_parser (line 64) | def _create_parser(self) -> argparse.ArgumentParser:
method _add_specific_arguments (line 255) | def _add_specific_arguments(self, parser: argparse.ArgumentParser):
method load_data (line 260) | async def load_data(self, args) -> list[dict[str, Any]]:
method get_llm_config (line 264) | def get_llm_config(self, args) -> dict[str, Any]:
method build_index (line 285) | async def build_index(self, args, texts: list[dict[str, Any]]) -> str:
method run_interactive_chat (line 338) | async def run_interactive_chat(self, args, index_path: str):
method run_single_query (line 368) | async def run_single_query(self, args, index_path: str, query: str):
method run (line 388) | async def run(self):
FILE: apps/browser_rag.py
class BrowserRAG (line 20) | class BrowserRAG(BaseRAGExample):
method __init__ (line 23) | def __init__(self):
method _add_specific_arguments (line 35) | def _add_specific_arguments(self, parser):
method _get_chrome_base_path (line 57) | def _get_chrome_base_path(self) -> Path:
method _find_chrome_profiles (line 68) | def _find_chrome_profiles(self) -> list[Path]:
method load_data (line 89) | async def load_data(self, args) -> list[dict[str, Any]]:
FILE: apps/chatgpt_data/chatgpt_reader.py
class ChatGPTReader (line 17) | class ChatGPTReader(BaseReader):
method __init__ (line 25) | def __init__(self, concatenate_conversations: bool = True) -> None:
method _extract_html_from_zip (line 39) | def _extract_html_from_zip(self, zip_path: Path) -> str | None:
method _parse_chatgpt_html (line 73) | def _parse_chatgpt_html(self, html_content: str) -> list[dict]:
method _extract_conversation_from_container (line 115) | def _extract_conversation_from_container(self, container) -> dict | None:
method _extract_message_from_element (line 160) | def _extract_message_from_element(self, element) -> dict | None:
method _extract_timestamp_from_element (line 198) | def _extract_timestamp_from_element(self, element) -> str | None:
method _extract_timestamp_from_container (line 222) | def _extract_timestamp_from_container(self, container) -> str | None:
method _create_concatenated_content (line 226) | def _create_concatenated_content(self, conversation: dict) -> str:
method load_data (line 271) | def load_data(self, input_dir: str | None = None, **load_kwargs: Any) ...
FILE: apps/chatgpt_rag.py
class ChatGPTRAG (line 19) | class ChatGPTRAG(BaseRAGExample):
method __init__ (line 22) | def __init__(self):
method _add_specific_arguments (line 35) | def _add_specific_arguments(self, parser):
method _find_chatgpt_exports (line 62) | def _find_chatgpt_exports(self, export_path: Path) -> list[Path]:
method load_data (line 84) | async def load_data(self, args) -> list[dict[str, Any]]:
FILE: apps/claude_data/claude_reader.py
class ClaudeReader (line 16) | class ClaudeReader(BaseReader):
method __init__ (line 24) | def __init__(self, concatenate_conversations: bool = True) -> None:
method _extract_json_from_zip (line 33) | def _extract_json_from_zip(self, zip_path: Path) -> list[str]:
method _parse_claude_json (line 65) | def _parse_claude_json(self, json_content: str) -> list[dict]:
method _extract_conversation_from_json (line 111) | def _extract_conversation_from_json(self, conv_data: dict) -> dict | N...
method _extract_message_from_json (line 153) | def _extract_message_from_json(self, msg_data: dict) -> dict | None:
method _extract_timestamp_from_message (line 194) | def _extract_timestamp_from_message(self, msg_data: dict) -> str | None:
method _extract_timestamp_from_conversation (line 202) | def _extract_timestamp_from_conversation(self, conv_data: dict) -> str...
method _extract_title_from_conversation (line 210) | def _extract_title_from_conversation(self, conv_data: dict, messages: ...
method _create_concatenated_content (line 231) | def _create_concatenated_content(self, conversation: dict) -> str:
method load_data (line 276) | def load_data(self, input_dir: str | None = None, **load_kwargs: Any) ...
FILE: apps/claude_rag.py
class ClaudeRAG (line 19) | class ClaudeRAG(BaseRAGExample):
method __init__ (line 22) | def __init__(self):
method _add_specific_arguments (line 35) | def _add_specific_arguments(self, parser):
method _find_claude_exports (line 62) | def _find_claude_exports(self, export_path: Path) -> list[Path]:
method load_data (line 84) | async def load_data(self, args) -> list[dict[str, Any]]:
FILE: apps/code_rag.py
class CodeRAG (line 19) | class CodeRAG(BaseRAGExample):
method __init__ (line 22) | def __init__(self):
method _add_specific_arguments (line 32) | def _add_specific_arguments(self, parser):
method load_data (line 81) | async def load_data(self, args) -> list[dict[str, Any]]:
FILE: apps/colqwen_rag.py
class ColQwenRAG (line 37) | class ColQwenRAG:
method __init__ (line 40) | def __init__(self, model_type: str = "colpali"):
method _assert_supported_transformers (line 135) | def _assert_supported_transformers(self) -> None:
method _get_device (line 168) | def _get_device(self):
method build_index (line 177) | def build_index(self, pdf_paths: list[str], index_name: str, pages_dir...
method search (line 260) | def search(self, index_name: str, query: str, top_k: int = 5):
method ask (line 292) | def ask(self, index_name: str, interactive: bool = False):
method _embed_images (line 328) | def _embed_images(self, images: list[Image.Image]) -> torch.Tensor:
method _embed_query (line 352) | def _embed_query(self, query: str) -> torch.Tensor:
function main (line 360) | def main():
FILE: apps/document_rag.py
class DocumentRAG (line 18) | class DocumentRAG(BaseRAGExample):
method __init__ (line 21) | def __init__(self):
method _add_specific_arguments (line 28) | def _add_specific_arguments(self, parser):
method load_data (line 55) | async def load_data(self, args) -> list[dict[str, Any]]:
FILE: apps/email_data/LEANN_email_reader.py
function find_all_messages_directories (line 10) | def find_all_messages_directories(root: str | None = None) -> list[Path]:
class EmlxReader (line 27) | class EmlxReader(BaseReader):
method __init__ (line 34) | def __init__(self, include_html: bool = False) -> None:
method _payload_to_text (line 43) | def _payload_to_text(self, payload: object) -> str:
method load_data (line 50) | def load_data(self, input_dir: str, **load_kwargs: Any) -> list[Docume...
FILE: apps/email_data/email.py
class MboxReader (line 19) | class MboxReader(BaseReader):
method __init__ (line 33) | def __init__(
method _payload_to_text (line 50) | def _payload_to_text(self, payload: object) -> str:
method load_data (line 57) | def load_data(
class EmlxMboxReader (line 126) | class EmlxMboxReader(MboxReader):
method load_data (line 136) | def load_data(
FILE: apps/email_rag.py
class EmailRAG (line 19) | class EmailRAG(BaseRAGExample):
method __init__ (line 22) | def __init__(self):
method _add_specific_arguments (line 35) | def _add_specific_arguments(self, parser):
method _find_mail_directories (line 54) | def _find_mail_directories(self) -> list[Path]:
method load_data (line 68) | async def load_data(self, args) -> list[dict[str, Any]]:
FILE: apps/gemini_data/gemini_reader.py
class GeminiReader (line 9) | class GeminiReader:
method __init__ (line 12) | def __init__(self):
method load_data (line 15) | def load_data(self, history_dir: str, max_count: int = -1) -> list[dic...
method _parse_json_session (line 91) | def _parse_json_session(self, file_path: Path) -> str:
method _parse_jsonl_session (line 123) | def _parse_jsonl_session(self, file_path: Path) -> str:
FILE: apps/gemini_rag.py
class GeminiRAG (line 19) | class GeminiRAG(BaseRAGExample):
method __init__ (line 22) | def __init__(self):
method _add_specific_arguments (line 29) | def _add_specific_arguments(self, parser):
method load_data (line 39) | async def load_data(self, args) -> list[dict[str, Any]]:
FILE: apps/history_data/history.py
class ChromeHistoryReader (line 10) | class ChromeHistoryReader(BaseReader):
method __init__ (line 18) | def __init__(self) -> None:
method load_data (line 22) | def load_data(self, input_dir: str | None = None, **load_kwargs: Any) ...
method find_chrome_profiles (line 110) | def find_chrome_profiles() -> list[Path]:
method export_history_to_file (line 136) | def export_history_to_file(
FILE: apps/history_data/wechat_history.py
class WeChatHistoryReader (line 14) | class WeChatHistoryReader(BaseReader):
method __init__ (line 24) | def __init__(self) -> None:
method check_wechat_running (line 30) | def check_wechat_running(self) -> bool:
method install_wechattweak (line 38) | def install_wechattweak(self) -> bool:
method restart_wechat (line 69) | def restart_wechat(self):
method check_api_available (line 80) | def check_api_available(self) -> bool:
method _extract_readable_text (line 93) | def _extract_readable_text(self, content: str) -> str:
method _is_text_message (line 139) | def _is_text_message(self, content: str) -> bool:
method _concatenate_messages (line 200) | def _concatenate_messages(
method _create_concatenated_content (line 317) | def _create_concatenated_content(
method load_data (line 396) | def load_data(self, input_dir: str | None = None, **load_kwargs: Any) ...
method find_wechat_export_dirs (line 556) | def find_wechat_export_dirs() -> list[Path]:
method export_chat_to_file (line 586) | def export_chat_to_file(
method export_wechat_chat_history (line 664) | def export_wechat_chat_history(self, export_dir: str = "./wechat_expor...
method find_or_export_wechat_data (line 736) | def find_or_export_wechat_data(self, export_dir: str = "./wechat_expor...
FILE: apps/image_rag.py
class ImageRAG (line 27) | class ImageRAG(BaseRAGExample):
method __init__ (line 35) | def __init__(self):
method _add_specific_arguments (line 46) | def _add_specific_arguments(self, parser: argparse.ArgumentParser):
method load_data (line 69) | async def load_data(self, args) -> list[dict[str, Any]]:
method _load_images_and_embeddings (line 74) | def _load_images_and_embeddings(self, args) -> list[dict]:
method build_index (line 172) | async def build_index(self, args, texts: list[dict[str, Any]]) -> str:
function main (line 210) | def main():
FILE: apps/imessage_data/imessage_reader.py
class IMessageReader (line 16) | class IMessageReader(BaseReader):
method __init__ (line 24) | def __init__(self, concatenate_conversations: bool = True) -> None:
method _get_default_chat_db_path (line 33) | def _get_default_chat_db_path(self) -> Path:
method _convert_cocoa_timestamp (line 43) | def _convert_cocoa_timestamp(self, cocoa_timestamp: int) -> str:
method _get_contact_name (line 66) | def _get_contact_name(self, handle_id: str) -> str:
method _read_messages_from_db (line 94) | def _read_messages_from_db(self, db_path: Path) -> list[dict]:
method _group_messages_by_chat (line 175) | def _group_messages_by_chat(self, messages: list[dict]) -> dict[int, l...
method _create_concatenated_content (line 194) | def _create_concatenated_content(self, chat_id: int, messages: list[di...
method _create_individual_content (line 241) | def _create_individual_content(self, message: dict) -> str:
method load_data (line 264) | def load_data(self, input_dir: str | None = None, **load_kwargs: Any) ...
FILE: apps/imessage_rag.py
class IMessageRAG (line 17) | class IMessageRAG(BaseRAGExample):
method __init__ (line 20) | def __init__(self):
method _add_specific_arguments (line 27) | def _add_specific_arguments(self, parser):
method load_data (line 60) | async def load_data(self, args) -> list[dict[str, Any]]:
function main (line 119) | async def main():
FILE: apps/multimodal/vision-based-pdf-multi-vector/colqwen_forward.py
function create_test_image (line 22) | def create_test_image():
function load_test_image_from_file (line 29) | def load_test_image_from_file():
function main (line 52) | def main():
FILE: apps/multimodal/vision-based-pdf-multi-vector/leann_multi_vector.py
function _ensure_repo_paths_importable (line 19) | def _ensure_repo_paths_importable(current_file: str) -> None:
function _find_backend_module_file (line 30) | def _find_backend_module_file() -> Optional[Path]:
function _get_backend_leann_multi_vector (line 63) | def _get_backend_leann_multi_vector() -> type:
function _natural_sort_key (line 98) | def _natural_sort_key(name: str) -> int:
function _load_images_from_dir (line 103) | def _load_images_from_dir(
function _maybe_convert_pdf_to_images (line 162) | def _maybe_convert_pdf_to_images(pdf_path: Optional[str], pages_dir: str...
function _select_device_and_dtype (line 177) | def _select_device_and_dtype():
function _load_colvision (line 208) | def _load_colvision(model_choice: str):
function _embed_images (line 306) | def _embed_images(model, processor, images: list[Image.Image]) -> list[A...
function _embed_queries (line 338) | def _embed_queries(model, processor, queries: list[str]) -> list[Any]:
function _build_index (line 383) | def _build_index(
function _load_retriever_if_index_exists (line 402) | def _load_retriever_if_index_exists(index_path: str) -> Optional[Any]:
function _build_fast_plaid_index (line 420) | def _build_fast_plaid_index(
function _fast_plaid_index_exists (line 515) | def _fast_plaid_index_exists(index_path: str) -> bool:
function _load_fast_plaid_index_if_exists (line 546) | def _load_fast_plaid_index_if_exists(index_path: str) -> Optional[Any]:
function _search_fast_plaid (line 579) | def _search_fast_plaid(
function _get_fast_plaid_image (line 633) | def _get_fast_plaid_image(index_path: str, doc_id: int) -> Optional[Imag...
function _get_fast_plaid_metadata (line 671) | def _get_fast_plaid_metadata(index_path: str, doc_id: int) -> Optional[d...
function _generate_similarity_map (line 693) | def _generate_similarity_map(
class QwenVL (line 753) | class QwenVL:
method __init__ (line 754) | def __init__(self, device: str):
method answer (line 772) | def answer(self, query: str, images: list[Image.Image], max_new_tokens...
class LeannMultiVector (line 811) | class LeannMultiVector:
method __init__ (line 812) | def __init__(
method _meta_dict (line 837) | def _meta_dict(self) -> dict:
method create_collection (line 850) | def create_collection(self) -> None:
method insert (line 854) | def insert(self, data: dict) -> None:
method _labels_path (line 864) | def _labels_path(self) -> Path:
method _meta_path (line 868) | def _meta_path(self) -> Path:
method _embeddings_path (line 872) | def _embeddings_path(self) -> Path:
method _images_dir_path (line 876) | def _images_dir_path(self) -> Path:
method create_index (line 881) | def create_index(self) -> None:
method _load_labels_meta_if_needed (line 940) | def _load_labels_meta_if_needed(self) -> None:
method _build_docid_to_indices_if_needed (line 950) | def _build_docid_to_indices_if_needed(self) -> None:
method search (line 963) | def search(
method search_exact (line 1011) | def search_exact(
method search_exact_all (line 1102) | def search_exact_all(
method get_image (line 1160) | def get_image(self, doc_id: int) -> Optional[Image.Image]:
method get_metadata (line 1181) | def get_metadata(self, doc_id: int) -> Optional[dict]:
class ViDoReBenchmarkEvaluator (line 1203) | class ViDoReBenchmarkEvaluator:
method __init__ (line 1209) | def __init__(
method _load_model_if_needed (line 1238) | def _load_model_if_needed(self):
method build_index_from_corpus (line 1247) | def build_index_from_corpus(
method search_queries (line 1301) | def search_queries(
method evaluate_results (line 1367) | def evaluate_results(
FILE: apps/multimodal/vision-based-pdf-multi-vector/multi-vector-leann-paper-example.py
function _page_sort_key (line 77) | def _page_sort_key(name: str) -> int:
FILE: apps/multimodal/vision-based-pdf-multi-vector/multi-vector-leann-similarity-map.py
function safe_get (line 383) | def safe_get(field_name, default=None):
FILE: apps/multimodal/vision-based-pdf-multi-vector/vidore_v1_benchmark.py
function normalize_task_name (line 108) | def normalize_task_name(task_name: str) -> str:
function get_safe_model_name (line 122) | def get_safe_model_name(model_name: str) -> str:
function load_vidore_v1_data (line 139) | def load_vidore_v1_data(
function evaluate_task (line 213) | def evaluate_task(
function main (line 329) | def main():
FILE: apps/multimodal/vision-based-pdf-multi-vector/vidore_v2_benchmark.py
function load_vidore_v2_data (line 78) | def load_vidore_v2_data(
function evaluate_task (line 186) | def evaluate_task(
function main (line 313) | def main():
FILE: apps/qwen_data/qwen_reader.py
class QwenReader (line 9) | class QwenReader:
method __init__ (line 12) | def __init__(self):
method load_data (line 15) | def load_data(self, history_dir: str, max_count: int = -1) -> list[dic...
method _parse_json_session (line 92) | def _parse_json_session(self, file_path: Path) -> str:
method _parse_jsonl_session (line 124) | def _parse_jsonl_session(self, file_path: Path) -> str:
FILE: apps/qwen_rag.py
class QwenRAG (line 19) | class QwenRAG(BaseRAGExample):
method __init__ (line 22) | def __init__(self):
method _add_specific_arguments (line 29) | def _add_specific_arguments(self, parser):
method load_data (line 39) | async def load_data(self, args) -> list[dict[str, Any]]:
FILE: apps/semantic_file_search/leann-plus-temporal-search.py
class TimeParser (line 12) | class TimeParser:
method __init__ (line 13) | def __init__(self):
method clean_text (line 42) | def clean_text(self, text):
method parse (line 48) | def parse(self, text):
method calculate_range (line 71) | def calculate_range(self, number, unit, is_fuzzy):
function search_files (line 96) | def search_files(query, top_k=15):
FILE: apps/semantic_file_search/leann_index_builder.py
function process_json_items (line 9) | def process_json_items(json_file_path):
FILE: apps/semantic_file_search/spotlight_index_dump.py
function convert_to_serializable (line 37) | def convert_to_serializable(obj):
function dump_spotlight_data (line 57) | def dump_spotlight_data(max_items=10, output_file="spotlight_dump.json"):
function main (line 246) | def main():
FILE: apps/slack_data/slack_mcp_reader.py
class SlackMCPReader (line 19) | class SlackMCPReader:
method __init__ (line 27) | def __init__(
method start_mcp_server (line 55) | async def start_mcp_server(self):
method stop_mcp_server (line 69) | async def stop_mcp_server(self):
method send_mcp_request (line 76) | async def send_mcp_request(self, request: dict[str, Any]) -> dict[str,...
method initialize_mcp_connection (line 94) | async def initialize_mcp_connection(self):
method list_available_tools (line 113) | async def list_available_tools(self) -> list[dict[str, Any]]:
method _is_cache_sync_error (line 123) | def _is_cache_sync_error(self, error: dict) -> bool:
method _retry_with_backoff (line 132) | async def _retry_with_backoff(self, func, *args, **kwargs):
method fetch_slack_messages (line 187) | async def fetch_slack_messages(
method _fetch_slack_messages_impl (line 202) | async def _fetch_slack_messages_impl(
method _parse_csv_messages (line 287) | def _parse_csv_messages(self, csv_text: str, channel: str) -> list[dic...
method _format_message (line 337) | def _format_message(self, message: dict[str, Any]) -> str:
method _create_concatenated_content (line 374) | def _create_concatenated_content(self, messages: list[dict[str, Any]],...
method get_all_channels (line 408) | async def get_all_channels(self) -> list[str]:
method read_slack_data (line 442) | async def read_slack_data(self, channels: Optional[list[str]] = None) ...
method __aenter__ (line 511) | async def __aenter__(self):
method __aexit__ (line 517) | async def __aexit__(self, exc_type, exc_val, exc_tb):
FILE: apps/slack_rag.py
class SlackMCPRAG (line 20) | class SlackMCPRAG(BaseRAGExample):
method __init__ (line 28) | def __init__(self):
method _add_specific_arguments (line 35) | def _add_specific_arguments(self, parser: argparse.ArgumentParser):
method test_mcp_connection (line 96) | async def test_mcp_connection(self, args) -> bool:
method load_data (line 143) | async def load_data(self, args) -> list[dict[str, Any]]:
method run (line 204) | async def run(self):
function main (line 222) | async def main():
FILE: apps/twitter_data/twitter_mcp_reader.py
class TwitterMCPReader (line 18) | class TwitterMCPReader:
method __init__ (line 26) | def __init__(
method start_mcp_server (line 51) | async def start_mcp_server(self):
method stop_mcp_server (line 65) | async def stop_mcp_server(self):
method send_mcp_request (line 72) | async def send_mcp_request(self, request: dict[str, Any]) -> dict[str,...
method initialize_mcp_connection (line 90) | async def initialize_mcp_connection(self):
method list_available_tools (line 109) | async def list_available_tools(self) -> list[dict[str, Any]]:
method fetch_twitter_bookmarks (line 119) | async def fetch_twitter_bookmarks(self, limit: Optional[int] = None) -...
method _format_bookmark (line 177) | def _format_bookmark(self, bookmark: dict[str, Any]) -> str:
method read_twitter_bookmarks (line 248) | async def read_twitter_bookmarks(self) -> list[str]:
method __aenter__ (line 290) | async def __aenter__(self):
method __aexit__ (line 296) | async def __aexit__(self, exc_type, exc_val, exc_tb):
FILE: apps/twitter_rag.py
class TwitterMCPRAG (line 20) | class TwitterMCPRAG(BaseRAGExample):
method __init__ (line 28) | def __init__(self):
method _add_specific_arguments (line 35) | def _add_specific_arguments(self, parser: argparse.ArgumentParser):
method test_mcp_connection (line 73) | async def test_mcp_connection(self, args) -> bool:
method load_data (line 120) | async def load_data(self, args) -> list[dict[str, Any]]:
method run (line 172) | async def run(self):
function main (line 190) | async def main():
FILE: apps/wechat_rag.py
class WeChatRAG (line 19) | class WeChatRAG(BaseRAGExample):
method __init__ (line 22) | def __init__(self):
method _add_specific_arguments (line 35) | def _add_specific_arguments(self, parser):
method _export_wechat_data (line 56) | def _export_wechat_data(self, export_dir: Path) -> bool:
method load_data (line 95) | async def load_data(self, args) -> list[dict[str, Any]]:
FILE: benchmarks/benchmark_embeddings.py
function benchmark_torch (line 23) | def benchmark_torch(model, sentences):
function benchmark_mlx (line 30) | def benchmark_mlx(model, tokenizer, sentences):
function main (line 71) | def main():
FILE: benchmarks/benchmark_no_recompute.py
function _meta_exists (line 9) | def _meta_exists(index_path: str) -> bool:
function ensure_index (line 14) | def ensure_index(index_path: str, backend_name: str, num_docs: int, is_r...
function _bench_group (line 37) | def _bench_group(
function main (line 77) | def main():
FILE: benchmarks/bm25_diskann_baselines/run_bm25.py
function load_queries (line 23) | def load_queries(path: str, limit: int | None) -> list[str]:
function percentile (line 54) | def percentile(values: list[float], p: float) -> float:
function main (line 66) | def main():
FILE: benchmarks/bm25_diskann_baselines/run_diskann.py
function load_queries (line 15) | def load_queries(path: Path, limit: int | None) -> list[str]:
function main (line 26) | def main() -> None:
FILE: benchmarks/compare_faiss_vs_leann.py
function get_memory_usage (line 22) | def get_memory_usage():
function print_memory_stats (line 28) | def print_memory_stats(stage: str, start_mem: float):
class MemoryTracker (line 36) | class MemoryTracker:
method __init__ (line 37) | def __init__(self, name: str):
method checkpoint (line 42) | def checkpoint(self, stage: str):
method summary (line 47) | def summary(self):
function test_faiss_hnsw (line 57) | def test_faiss_hnsw():
function test_leann_hnsw (line 98) | def test_leann_hnsw():
function main (line 250) | def main():
FILE: benchmarks/diskann_vs_hnsw_speed_comparison.py
function create_test_texts (line 28) | def create_test_texts(n_docs: int) -> list[str]:
function benchmark_backend (line 58) | def benchmark_backend(
function run_comparison (line 141) | def run_comparison(n_docs: int = 500, n_queries: int = 10):
FILE: benchmarks/enron_emails/evaluate_enron_emails.py
class RecallEvaluator (line 27) | class RecallEvaluator:
method __init__ (line 30) | def __init__(self, index_path: str, baseline_dir: str):
method evaluate_recall_at_3 (line 46) | def evaluate_recall_at_3(
method cleanup (line 103) | def cleanup(self):
class EnronEvaluator (line 108) | class EnronEvaluator:
method __init__ (line 109) | def __init__(self, index_path: str):
method load_queries (line 113) | def load_queries(self, queries_file: str) -> list[str]:
method cleanup (line 125) | def cleanup(self):
method analyze_index_sizes (line 129) | def analyze_index_sizes(self) -> dict:
method create_non_compact_index_for_comparison (line 159) | def create_non_compact_index_for_comparison(self, non_compact_index_pa...
method compare_index_performance (line 229) | def compare_index_performance(
method evaluate_complexity (line 288) | def evaluate_complexity(
function main (line 366) | def main():
FILE: benchmarks/enron_emails/setup_enron_emails.py
class EnronSetup (line 20) | class EnronSetup:
method __init__ (line 21) | def __init__(self, data_dir: str = "data"):
method ensure_emails_csv (line 34) | def ensure_emails_csv(self, emails_csv: Optional[str]) -> str:
method _extract_message_id (line 74) | def _extract_message_id(raw_email: str) -> str:
method _split_header_body (line 82) | def _split_header_body(raw_email: str) -> tuple[str, str]:
method _split_fixed_words (line 93) | def _split_fixed_words(text: str, chunk_words: int, keep_last: bool) -...
method _iter_passages_from_csv (line 110) | def _iter_passages_from_csv(
method build_leann_index (line 166) | def build_leann_index(
method build_faiss_flat_baseline (line 204) | def build_faiss_flat_baseline(self, index_path: str, output_dir: str =...
method prepare_queries (line 273) | def prepare_queries(self, min_realism: float = 0.85) -> Path:
function main (line 307) | def main():
FILE: benchmarks/faiss_only.py
function get_memory_usage (line 11) | def get_memory_usage():
class MemoryTracker (line 16) | class MemoryTracker:
method __init__ (line 17) | def __init__(self, name: str):
method checkpoint (line 22) | def checkpoint(self, stage: str):
method summary (line 29) | def summary(self):
function main (line 35) | def main():
FILE: benchmarks/financebench/evaluate_financebench.py
class RecallEvaluator (line 27) | class RecallEvaluator:
method __init__ (line 30) | def __init__(self, index_path: str, baseline_dir: str):
method evaluate_recall_at_3 (line 44) | def evaluate_recall_at_3(
method cleanup (line 106) | def cleanup(self):
class FinanceBenchEvaluator (line 112) | class FinanceBenchEvaluator:
method __init__ (line 113) | def __init__(self, index_path: str, openai_api_key: Optional[str] = No...
method load_dataset (line 120) | def load_dataset(self, dataset_path: str = "data/financebench_merged.j...
method analyze_index_sizes (line 131) | def analyze_index_sizes(self) -> dict:
method create_compact_index_for_comparison (line 172) | def create_compact_index_for_comparison(self, compact_index_path: str)...
method create_non_compact_index_for_comparison (line 227) | def create_non_compact_index_for_comparison(self, non_compact_index_pa...
method compare_index_performance (line 286) | def compare_index_performance(
method evaluate_timing_breakdown (line 358) | def evaluate_timing_breakdown(
method _check_answer_accuracy (line 468) | def _check_answer_accuracy(
method _print_results (line 505) | def _print_results(self, timing_metrics: dict):
method cleanup (line 574) | def cleanup(self):
function main (line 580) | def main():
FILE: benchmarks/financebench/setup_financebench.py
class FinanceBenchSetup (line 21) | class FinanceBenchSetup:
method __init__ (line 22) | def __init__(self, data_dir: str = "data"):
method download_dataset (line 30) | def download_dataset(self):
method get_pdf_list (line 49) | def get_pdf_list(self):
method download_single_pdf (line 62) | def download_single_pdf(self, pdf_info, position):
method download_all_pdfs (line 86) | def download_all_pdfs(self, max_workers: int = 5):
method build_leann_index (line 126) | def build_leann_index(
method build_faiss_flat_baseline (line 190) | def build_faiss_flat_baseline(self, index_path: str, output_dir: str =...
method extract_pdf_text (line 271) | def extract_pdf_text(self, pdf_path: Path) -> list[dict]:
method extract_year_from_filename (line 338) | def extract_year_from_filename(self, filename: str) -> str:
method verify_setup (line 345) | def verify_setup(self, index_path: str):
function main (line 372) | def main():
FILE: benchmarks/financebench/verify_recall.py
function compute_embeddings_direct (line 28) | def compute_embeddings_direct(chunks: list[str], model_name: str) -> np....
function load_financebench_queries (line 48) | def load_financebench_queries(dataset_path: str, max_queries: int = 200)...
function load_passages_from_leann_index (line 61) | def load_passages_from_leann_index(index_path: str) -> tuple[list[str], ...
function build_faiss_indexes (line 90) | def build_faiss_indexes(embeddings: np.ndarray) -> tuple[faiss.Index, fa...
function evaluate_recall_at_k (line 110) | def evaluate_recall_at_k(
function main (line 152) | def main():
FILE: benchmarks/issue_159.py
function generate_test_data (line 25) | def generate_test_data(num_chunks=90000, chunk_size=2000):
function test_search_performance (line 41) | def test_search_performance():
FILE: benchmarks/laion/evaluate_laion.py
class RecallEvaluator (line 26) | class RecallEvaluator:
method __init__ (line 29) | def __init__(self, index_path: str, baseline_dir: str):
method evaluate_recall_at_3 (line 46) | def evaluate_recall_at_3(
method cleanup (line 104) | def cleanup(self):
class LAIONEvaluator (line 110) | class LAIONEvaluator:
method __init__ (line 111) | def __init__(self, index_path: str):
method load_queries (line 115) | def load_queries(self, queries_file: str) -> list[str]:
method analyze_index_sizes (line 127) | def analyze_index_sizes(self) -> dict:
method create_non_compact_index_for_comparison (line 169) | def create_non_compact_index_for_comparison(self, non_compact_index_pa...
method compare_index_performance (line 248) | def compare_index_performance(
method _print_results (line 316) | def _print_results(self, timing_metrics: dict):
method cleanup (line 385) | def cleanup(self):
function main (line 391) | def main():
FILE: benchmarks/laion/setup_laion.py
class LAIONSetup (line 24) | class LAIONSetup:
method __init__ (line 25) | def __init__(self, data_dir: str = "data"):
method download_single_image (line 34) | async def download_single_image(self, session, sample_data, semaphore,...
method download_laion_subset (line 68) | def download_laion_subset(self, num_samples: int = 1000):
method generate_clip_image_embeddings (line 145) | def generate_clip_image_embeddings(self, samples: list[dict]):
method build_faiss_baseline (line 186) | def build_faiss_baseline(
method create_leann_passages (line 230) | def create_leann_passages(self, samples: list[dict]):
method build_compact_index (line 255) | def build_compact_index(
method build_non_compact_index (line 319) | def build_non_compact_index(
method _add_passages_with_embeddings (line 377) | def _add_passages_with_embeddings(self, builder, passages_file: Path, ...
method _analyze_index_size (line 392) | def _analyze_index_size(self, index_path: str):
method create_evaluation_queries (line 437) | def create_evaluation_queries(self, samples: list[dict], num_queries: ...
function main (line 462) | def main():
FILE: benchmarks/llm_utils.py
function is_qwen3_model (line 23) | def is_qwen3_model(model_name):
function is_qwen_vl_model (line 28) | def is_qwen_vl_model(model_name):
function apply_qwen3_chat_template (line 33) | def apply_qwen3_chat_template(tokenizer, prompt):
function extract_thinking_answer (line 44) | def extract_thinking_answer(response):
function load_hf_model (line 57) | def load_hf_model(model_name="Qwen/Qwen3-8B", trust_remote_code=False):
function load_vllm_model (line 84) | def load_vllm_model(model_name="Qwen/Qwen3-8B", trust_remote_code=False):
function generate_hf (line 115) | def generate_hf(tokenizer, model, prompt, max_tokens=None):
function generate_vllm (line 145) | def generate_vllm(llm, sampling_params, prompt):
function create_prompt (line 157) | def create_prompt(context, query, domain="default"):
function evaluate_rag (line 169) | def evaluate_rag(searcher, llm_func, queries, domain="default", top_k=3,...
function load_qwen_vl_model (line 203) | def load_qwen_vl_model(model_name="Qwen/Qwen2.5-VL-7B-Instruct", trust_r...
function generate_qwen_vl (line 257) | def generate_qwen_vl(processor, model, prompt, image_path=None, max_toke...
function create_multimodal_prompt (line 281) | def create_multimodal_prompt(context, query, image_descriptions, task_ty...
function evaluate_multimodal_rag (line 296) | def evaluate_multimodal_rag(searcher, queries, processor=None, model=Non...
FILE: benchmarks/micro_tpt.py
class BenchmarkConfig (line 16) | class BenchmarkConfig:
class GraphContainer (line 29) | class GraphContainer:
method __init__ (line 32) | def __init__(self, model: nn.Module, seq_length: int):
method get_or_create (line 37) | def get_or_create(self, batch_size: int) -> "GraphWrapper":
class GraphWrapper (line 43) | class GraphWrapper:
method __init__ (line 46) | def __init__(self, model: nn.Module, batch_size: int, seq_length: int):
method _get_device (line 70) | def _get_device(self) -> str:
method _create_random_batch (line 78) | def _create_random_batch(self, batch_size: int, seq_length: int) -> to...
method _warmup (line 83) | def _warmup(self, num_warmup: int = 3):
method __call__ (line 91) | def __call__(self, input_ids: torch.Tensor, attention_mask: torch.Tens...
class ModelOptimizer (line 102) | class ModelOptimizer:
method optimize (line 106) | def optimize(model: nn.Module, config: BenchmarkConfig) -> nn.Module:
class Timer (line 173) | class Timer:
method __init__ (line 176) | def __init__(self):
method timing (line 189) | def timing(self):
method elapsed_time (line 201) | def elapsed_time(self) -> float:
class Benchmark (line 208) | class Benchmark:
method __init__ (line 211) | def __init__(self, config: BenchmarkConfig):
method _load_model (line 228) | def _load_model(self) -> nn.Module:
method _create_random_batch (line 426) | def _create_random_batch(self, batch_size: int) -> torch.Tensor:
method _run_inference (line 442) | def _run_inference(
method run (line 455) | def run(self) -> dict[int, dict[str, float]]:
function main (line 531) | def main():
FILE: benchmarks/run_evaluation.py
function download_data_if_needed (line 18) | def download_data_if_needed(data_root: Path, download_embeddings: bool =...
function download_embeddings_if_needed (line 63) | def download_embeddings_if_needed(data_root: Path, dataset_type: str | N...
function get_golden_texts (line 101) | def get_golden_texts(searcher: LeannSearcher, golden_ids: list[int]) -> ...
function load_queries (line 117) | def load_queries(file_path: Path) -> list[str]:
function build_index_from_embeddings (line 126) | def build_index_from_embeddings(embeddings_file: str, output_path: str, ...
function main (line 168) | def main():
FILE: benchmarks/simple_mac_tpt_test.py
class BenchmarkConfig (line 22) | class BenchmarkConfig:
method __post_init__ (line 35) | def __post_init__(self):
class MLXBenchmark (line 40) | class MLXBenchmark:
method __init__ (line 43) | def __init__(self, config: BenchmarkConfig):
method _load_model (line 47) | def _load_model(self):
method _create_random_batch (line 58) | def _create_random_batch(self, batch_size: int):
method _run_inference (line 62) | def _run_inference(self, input_ids: torch.Tensor) -> float:
method run (line 88) | def run(self) -> dict[int, dict[str, float]]:
class Benchmark (line 147) | class Benchmark:
method __init__ (line 148) | def __init__(self, config: BenchmarkConfig):
method _load_model (line 159) | def _load_model(self) -> nn.Module:
method _create_random_batch (line 171) | def _create_random_batch(self, batch_size: int) -> torch.Tensor:
method _run_inference (line 180) | def _run_inference(self, input_ids: torch.Tensor) -> float:
method run (line 196) | def run(self) -> dict[int, dict[str, float]]:
function run_benchmark (line 243) | def run_benchmark():
function run_mlx_benchmark (line 265) | def run_mlx_benchmark():
FILE: benchmarks/update/__init__.py
function find_repo_root (line 7) | def find_repo_root() -> Path:
FILE: benchmarks/update/bench_hnsw_rng_recompute.py
function _find_repo_root (line 50) | def _find_repo_root() -> Path:
function load_chunks_from_files (line 75) | def load_chunks_from_files(paths: list[Path], limit: int | None = None) ...
function ensure_index_dir (line 105) | def ensure_index_dir(index_path: Path) -> None:
function cleanup_index_files (line 109) | def cleanup_index_files(index_path: Path) -> None:
function build_initial_index (line 119) | def build_initial_index(
function prepare_new_chunks (line 146) | def prepare_new_chunks(paragraphs: list[str]) -> list[dict[str, Any]]:
function benchmark_update_with_mode (line 150) | def benchmark_update_with_mode(
function _total_zmq_nodes (line 334) | def _total_zmq_nodes(log_path: Path) -> int:
function _warmup_embedding_server (line 342) | def _warmup_embedding_server(port: int) -> None:
function main (line 362) | def main() -> None:
FILE: benchmarks/update/bench_update_vs_offline_search.py
function _find_repo_root (line 59) | def _find_repo_root() -> Path:
function load_chunks_from_files (line 82) | def load_chunks_from_files(paths: list[Path], limit: int | None = None) ...
function ensure_index_dir (line 112) | def ensure_index_dir(index_path: Path) -> None:
function cleanup_index_files (line 116) | def cleanup_index_files(index_path: Path) -> None:
function build_initial_index (line 126) | def build_initial_index(
function _maybe_norm_cosine (line 153) | def _maybe_norm_cosine(vecs: np.ndarray, metric: str) -> np.ndarray:
function _read_index_for_search (line 162) | def _read_index_for_search(index_path: Path) -> Any:
function _append_passages_for_updates (line 199) | def _append_passages_for_updates(
function _search (line 251) | def _search(index: Any, q: np.ndarray, k: int) -> tuple[np.ndarray, np.n...
function _score_for_metric (line 265) | def _score_for_metric(dist: float, metric: str) -> float:
function _merge_results (line 273) | def _merge_results(
class ScenarioResult (line 290) | class ScenarioResult:
function main (line 297) | def main() -> None:
FILE: benchmarks/update/plot_bench_results.py
function load_latest_run (line 56) | def load_latest_run(csv_path: Path):
function aggregate_latency (line 75) | def aggregate_latency(rows):
function _auto_cap (line 88) | def _auto_cap(values: list[float]) -> float | None:
function _add_break_marker (line 103) | def _add_break_marker(ax, y, rel_x0=0.02, rel_x1=0.98, size=0.02):
function _fmt_ms (line 110) | def _fmt_ms(v: float) -> str:
function main (line 116) | def main():
FILE: docs/code/embedding_model_compare.py
function cosine_similarity (line 25) | def cosine_similarity(a, b):
function analyze_embeddings (line 29) | def analyze_embeddings(embeddings, model_name):
FILE: examples/basic_demo.py
function main (line 11) | def main():
FILE: examples/dynamic_update_no_recompute.py
function load_chunks_from_files (line 54) | def load_chunks_from_files(paths: list[Path]) -> list[str]:
function run_search (line 81) | def run_search(index_path: Path, query: str, top_k: int, *, recompute_em...
function print_results (line 94) | def print_results(title: str, results: Iterable) -> None:
function build_initial_index (line 106) | def build_initial_index(
function update_index (line 125) | def update_index(
function ensure_index_dir (line 145) | def ensure_index_dir(index_path: Path) -> None:
function cleanup_index_files (line 149) | def cleanup_index_files(index_path: Path) -> None:
function index_file_size (line 161) | def index_file_size(index_path: Path) -> int:
function load_metadata_snapshot (line 168) | def load_metadata_snapshot(index_path: Path) -> dict[str, Any] | None:
function run_workflow (line 178) | def run_workflow(
function main (line 247) | def main() -> None:
FILE: examples/mcp_integration_demo.py
function demo_slack_mcp (line 22) | async def demo_slack_mcp():
function demo_twitter_mcp (line 63) | async def demo_twitter_mcp():
function show_mcp_server_setup (line 98) | async def show_mcp_server_setup():
function show_integration_benefits (line 133) | async def show_integration_benefits():
function main (line 153) | async def main():
FILE: examples/spoiler_free_book_rag.py
function chunk_book_with_metadata (line 23) | def chunk_book_with_metadata(book_title: str = "Sample Book") -> list[di...
function build_spoiler_free_index (line 111) | def build_spoiler_free_index(book_chunks: list[dict[str, Any]], index_na...
function spoiler_free_search (line 141) | def spoiler_free_search(
function demo_spoiler_free_rag (line 173) | def demo_spoiler_free_rag():
FILE: packages/leann-backend-diskann/leann_backend_diskann/__init__.py
function _configure_windows_dll_search_path (line 7) | def _configure_windows_dll_search_path() -> None:
FILE: packages/leann-backend-diskann/leann_backend_diskann/diskann_backend.py
function suppress_cpp_output_if_needed (line 23) | def suppress_cpp_output_if_needed():
function _get_diskann_metrics (line 65) | def _get_diskann_metrics():
function chdir (line 76) | def chdir(path):
function _write_vectors_to_bin (line 85) | def _write_vectors_to_bin(data: np.ndarray, file_path: Path):
function _calculate_smart_memory_config (line 93) | def _calculate_smart_memory_config(data: np.ndarray) -> tuple[float, flo...
class DiskannBackend (line 131) | class DiskannBackend(LeannBackendFactoryInterface):
method builder (line 133) | def builder(**kwargs) -> LeannBackendBuilderInterface:
method searcher (line 137) | def searcher(index_path: str, **kwargs) -> LeannBackendSearcherInterface:
class DiskannBuilder (line 141) | class DiskannBuilder(LeannBackendBuilderInterface):
method __init__ (line 142) | def __init__(self, **kwargs):
method _safe_cleanup_after_partition (line 145) | def _safe_cleanup_after_partition(self, index_dir: Path, index_prefix:...
method build (line 210) | def build(self, data: np.ndarray, ids: list[str], index_path: str, **k...
class DiskannSearcher (line 300) | class DiskannSearcher(BaseSearcher):
method __init__ (line 301) | def __init__(self, index_path: str, **kwargs):
method close (line 362) | def close(self):
method _ensure_index_loaded (line 375) | def _ensure_index_loaded(self, zmq_port: int):
method search (line 397) | def search(
FILE: packages/leann-backend-diskann/leann_backend_diskann/diskann_embedding_server.py
function create_diskann_embedding_server (line 45) | def create_diskann_embedding_server(
FILE: packages/leann-backend-diskann/leann_backend_diskann/graph_partition.py
class GraphPartitioner (line 18) | class GraphPartitioner:
method __init__ (line 26) | def __init__(self, build_type: str = "release"):
method _get_executable_path (line 36) | def _get_executable_path(self, name: str) -> str:
method _ensure_executables (line 49) | def _ensure_executables(self):
method _build_executables (line 59) | def _build_executables(self):
method partition_graph (line 89) | def partition_graph(
method get_partition_info (line 241) | def get_partition_info(self, partition_bin_path: str) -> dict:
function partition_graph (line 264) | def partition_graph(
FILE: packages/leann-backend-diskann/third_party/embedding.pb.cc
type protoembedding (line 17) | namespace protoembedding {
class NodeEmbeddingRequestDefaultTypeInternal (line 18) | class NodeEmbeddingRequestDefaultTypeInternal {
class NodeEmbeddingResponseDefaultTypeInternal (line 22) | class NodeEmbeddingResponseDefaultTypeInternal {
class NodeEmbeddingRequest::_Internal (line 114) | class NodeEmbeddingRequest::_Internal {
function NodeEmbeddingRequest (line 154) | const NodeEmbeddingRequest& NodeEmbeddingRequest::default_instance() {
class NodeEmbeddingResponse::_Internal (line 323) | class NodeEmbeddingResponse::_Internal {
function NodeEmbeddingResponse (line 373) | const NodeEmbeddingResponse& NodeEmbeddingResponse::default_instance() {
function InitDefaultsscc_info_NodeEmbeddingRequest_embedding_2eproto (line 27) | static void InitDefaultsscc_info_NodeEmbeddingRequest_embedding_2eproto() {
function InitDefaultsscc_info_NodeEmbeddingResponse_embedding_2eproto (line 41) | static void InitDefaultsscc_info_NodeEmbeddingResponse_embedding_2eproto...
type protoembedding (line 108) | namespace protoembedding {
class NodeEmbeddingRequestDefaultTypeInternal (line 18) | class NodeEmbeddingRequestDefaultTypeInternal {
class NodeEmbeddingResponseDefaultTypeInternal (line 22) | class NodeEmbeddingResponseDefaultTypeInternal {
class NodeEmbeddingRequest::_Internal (line 114) | class NodeEmbeddingRequest::_Internal {
function NodeEmbeddingRequest (line 154) | const NodeEmbeddingRequest& NodeEmbeddingRequest::default_instance() {
class NodeEmbeddingResponse::_Internal (line 323) | class NodeEmbeddingResponse::_Internal {
function NodeEmbeddingResponse (line 373) | const NodeEmbeddingResponse& NodeEmbeddingResponse::default_instance() {
function PROTOBUF_NAMESPACE_OPEN (line 603) | PROTOBUF_NAMESPACE_OPEN
FILE: packages/leann-backend-hnsw/leann_backend_hnsw/__init__.py
function _configure_windows_dll_search_path (line 7) | def _configure_windows_dll_search_path() -> None:
FILE: packages/leann-backend-hnsw/leann_backend_hnsw/convert_to_csr.py
function read_struct (line 32) | def read_struct(f, fmt):
function read_vector_raw (line 43) | def read_vector_raw(f, element_fmt_char):
function read_numpy_vector (line 81) | def read_numpy_vector(f, np_dtype, struct_fmt_char):
function write_numpy_vector (line 118) | def write_numpy_vector(f, arr, struct_fmt_char):
function write_list_vector (line 138) | def write_list_vector(f, lst, struct_fmt_char):
function get_cum_neighbors (line 174) | def get_cum_neighbors(cum_nneighbor_per_level_np, level):
function write_compact_format (line 184) | def write_compact_format(
class HNSWComponents (line 243) | class HNSWComponents:
function _read_hnsw_structure (line 258) | def _read_hnsw_structure(f) -> HNSWComponents:
function _read_hnsw_structure_from_file (line 362) | def _read_hnsw_structure_from_file(path: str) -> HNSWComponents:
function write_original_format (line 367) | def write_original_format(
function prune_hnsw_embeddings (line 408) | def prune_hnsw_embeddings(input_filename: str, output_filename: str) -> ...
function convert_hnsw_graph_to_csr (line 527) | def convert_hnsw_graph_to_csr(input_filename, output_filename, prune_emb...
function prune_hnsw_embeddings_inplace (line 987) | def prune_hnsw_embeddings_inplace(index_filename: str) -> bool:
FILE: packages/leann-backend-hnsw/leann_backend_hnsw/hnsw_backend.py
function get_metric_map (line 22) | def get_metric_map():
function normalize_l2 (line 32) | def normalize_l2(data: np.ndarray) -> np.ndarray:
class HNSWBackend (line 39) | class HNSWBackend(LeannBackendFactoryInterface):
method builder (line 41) | def builder(**kwargs) -> LeannBackendBuilderInterface:
method searcher (line 45) | def searcher(index_path: str, **kwargs) -> LeannBackendSearcherInterface:
class HNSWBuilder (line 49) | class HNSWBuilder(LeannBackendBuilderInterface):
method __init__ (line 50) | def __init__(self, **kwargs):
method build (line 66) | def build(self, data: np.ndarray, ids: list[str], index_path: str, **k...
method _convert_to_csr (line 107) | def _convert_to_csr(self, index_file: Path):
class HNSWSearcher (line 131) | class HNSWSearcher(BaseSearcher):
method __init__ (line 132) | def __init__(self, index_path: str, **kwargs):
method search (line 174) | def search(
FILE: packages/leann-backend-hnsw/leann_backend_hnsw/hnsw_embedding_server.py
function create_hnsw_embedding_server (line 58) | def create_hnsw_embedding_server(
FILE: packages/leann-backend-ivf/leann_backend_ivf/ivf_backend.py
function _check_faiss (line 33) | def _check_faiss():
function _get_metric_map (line 40) | def _get_metric_map():
function _normalize_l2 (line 49) | def _normalize_l2(data: np.ndarray) -> np.ndarray:
function _load_id_map (line 55) | def _load_id_map(index_dir: Path, index_prefix: str) -> tuple[dict[int, ...
function _save_id_map (line 68) | def _save_id_map(
class IVFBackend (line 86) | class IVFBackend(LeannBackendFactoryInterface):
method builder (line 88) | def builder(**kwargs) -> LeannBackendBuilderInterface:
method searcher (line 92) | def searcher(index_path: str, **kwargs) -> LeannBackendSearcherInterface:
class IVFBuilder (line 96) | class IVFBuilder(LeannBackendBuilderInterface):
method __init__ (line 97) | def __init__(self, **kwargs):
method build (line 104) | def build(self, data: np.ndarray, ids: list[str], index_path: str, **k...
class IVFSearcher (line 139) | class IVFSearcher(BaseSearcher):
method __init__ (line 140) | def __init__(self, index_path: str, **kwargs):
method search (line 161) | def search(
method compute_query_embedding (line 185) | def compute_query_embedding(
function add_vectors (line 200) | def add_vectors(index_path: str, embeddings: np.ndarray, passage_ids: li...
function remove_ids (line 238) | def remove_ids(index_path: str, passage_ids: list[str]) -> int:
FILE: packages/leann-core/src/leann/api.py
function get_registered_backends (line 34) | def get_registered_backends() -> list[str]:
function compute_embeddings (line 39) | def compute_embeddings(
function compute_embeddings_via_server (line 84) | def compute_embeddings_via_server(chunks: list[str], model_name: str, po...
class SearchResult (line 121) | class SearchResult:
class PassageManager (line 128) | class PassageManager:
method __init__ (line 129) | def __init__(
method get_passage (line 212) | def get_passage(self, passage_id: str) -> dict[str, Any]:
method filter_search_results (line 226) | def filter_search_results(
method __len__ (line 276) | def __len__(self) -> int:
class BM25Scorer (line 280) | class BM25Scorer:
method __init__ (line 281) | def __init__(self, k1: float = 1.2, b: float = 0.75):
method _tokenize (line 291) | def _tokenize(self, text: str) -> list[str]:
method fit (line 294) | def fit(self, documents: list[dict[str, Any]]):
method score (line 320) | def score(self, query_words: list[str], document_id: str) -> float:
method search (line 346) | def search(self, query: str, top_k: int = 5) -> list[SearchResult]:
class LeannBuilder (line 356) | class LeannBuilder:
method __init__ (line 357) | def __init__(
method add_text (line 460) | def add_text(self, text: str, metadata: Optional[dict[str, Any]] = None):
method build_index (line 467) | def build_index(self, index_path: str):
method build_index_from_embeddings (line 584) | def build_index_from_embeddings(self, index_path: str, embeddings_file...
method _compact_passages (line 722) | def _compact_passages(
method update_index (line 746) | def update_index(self, index_path: str, remove_passage_ids: Optional[l...
class LeannSearcher (line 1064) | class LeannSearcher:
method __init__ (line 1065) | def __init__(
method warmup (line 1129) | def warmup(self) -> None:
method search (line 1139) | def search(
method _init_bm25 (line 1368) | def _init_bm25(self) -> None:
method _bm25_search (line 1381) | def _bm25_search(self, query: str, top_k: int = 5) -> list[SearchResult]:
method _find_jsonl_file (line 1391) | def _find_jsonl_file(self) -> Optional[str]:
method _grep_search (line 1404) | def _grep_search(self, query: str, top_k: int = 5) -> list[SearchResult]:
method _python_regex_search (line 1451) | def _python_regex_search(self, query: str, top_k: int = 5) -> list[Sea...
method cleanup (line 1479) | def cleanup(self):
method __enter__ (line 1494) | def __enter__(self):
method __exit__ (line 1497) | def __exit__(self, exc_type, exc, tb):
method __del__ (line 1503) | def __del__(self):
class LeannChat (line 1511) | class LeannChat:
method __init__ (line 1512) | def __init__(
method ask (line 1528) | def ask(
method start_interactive (line 1590) | def start_interactive(self):
method cleanup (line 1600) | def cleanup(self):
method __enter__ (line 1612) | def __enter__(self):
method __exit__ (line 1615) | def __exit__(self, exc_type, exc, tb):
method __del__ (line 1621) | def __del__(self):
FILE: packages/leann-core/src/leann/chat.py
function check_ollama_models (line 30) | def check_ollama_models(host: str) -> list[str]:
function check_ollama_model_exists_remotely (line 44) | def check_ollama_model_exists_remotely(model_name: str) -> tuple[bool, l...
function search_ollama_models_fuzzy (line 107) | def search_ollama_models_fuzzy(query: str, available_models: list[str]) ...
function suggest_similar_models (line 182) | def suggest_similar_models(invalid_model: str, available_models: list[st...
function check_hf_model_exists (line 192) | def check_hf_model_exists(model_name: str) -> bool:
function get_popular_hf_models (line 203) | def get_popular_hf_models() -> list[str]:
function _get_fallback_hf_models (line 235) | def _get_fallback_hf_models() -> list[str]:
function search_hf_models_fuzzy (line 251) | def search_hf_models_fuzzy(query: str, limit: int = 10) -> list[str]:
function search_hf_models (line 317) | def search_hf_models(query: str, limit: int = 10) -> list[str]:
function validate_model_and_suggest (line 322) | def validate_model_and_suggest(
class LLMInterface (line 424) | class LLMInterface(ABC):
method ask (line 428) | def ask(self, prompt: str, **kwargs) -> str:
class OllamaChat (line 468) | class OllamaChat(LLMInterface):
method __init__ (line 471) | def __init__(self, model: str = "llama3:8b", host: Optional[str] = None):
method ask (line 499) | def ask(self, prompt: str, **kwargs) -> str:
class HFChat (line 556) | class HFChat(LLMInterface):
method __init__ (line 566) | def __init__(
method ask (line 665) | def ask(self, prompt: str, **kwargs) -> str:
class GeminiChat (line 731) | class GeminiChat(LLMInterface):
method __init__ (line 734) | def __init__(self, model: str = "gemini-2.5-flash", api_key: Optional[...
method ask (line 754) | def ask(self, prompt: str, **kwargs) -> str:
class OpenAIChat (line 785) | class OpenAIChat(LLMInterface):
method __init__ (line 788) | def __init__(
method ask (line 818) | def ask(self, prompt: str, **kwargs) -> str:
class AnthropicChat (line 870) | class AnthropicChat(LLMInterface):
method __init__ (line 873) | def __init__(
method ask (line 907) | def ask(self, prompt: str, **kwargs) -> str:
class MiniMaxChat (line 945) | class MiniMaxChat(LLMInterface):
method __init__ (line 955) | def __init__(
method ask (line 985) | def ask(self, prompt: str, **kwargs) -> str:
class SimulatedChat (line 1013) | class SimulatedChat(LLMInterface):
method ask (line 1016) | def ask(self, prompt: str, **kwargs) -> str:
function get_llm (line 1022) | def get_llm(llm_config: Optional[dict[str, Any]] = None) -> LLMInterface:
FILE: packages/leann-core/src/leann/chunking_utils.py
function estimate_token_count (line 18) | def estimate_token_count(text: str) -> int:
function calculate_safe_chunk_size (line 41) | def calculate_safe_chunk_size(
function validate_chunk_token_limits (line 73) | def validate_chunk_token_limits(chunks: list[str], max_tokens: int = 512...
function detect_code_files (line 142) | def detect_code_files(documents, code_extensions=None) -> tuple[list, li...
function get_language_from_extension (line 169) | def get_language_from_extension(file_path: str) -> Optional[str]:
function create_ast_chunks (line 175) | def create_ast_chunks(
function create_traditional_chunks (line 290) | def create_traditional_chunks(
function _traditional_chunks_as_dicts (line 340) | def _traditional_chunks_as_dicts(
function create_text_chunks (line 350) | def create_text_chunks(
FILE: packages/leann-core/src/leann/cli.py
function _normalize_path (line 34) | def _normalize_path(path: str) -> str:
function suppress_cpp_output (line 42) | def suppress_cpp_output(suppress: bool = True):
function extract_pdf_text_with_pymupdf (line 99) | def extract_pdf_text_with_pymupdf(file_path: str) -> str | None:
function extract_pdf_text_with_pdfplumber (line 115) | def extract_pdf_text_with_pdfplumber(file_path: str) -> str | None:
class LeannCLI (line 130) | class LeannCLI:
method __init__ (line 131) | def __init__(self):
method get_index_path (line 149) | def get_index_path(self, index_name: str) -> str:
method index_exists (line 153) | def index_exists(self, index_name: str) -> bool:
method create_parser (line 158) | def create_parser(self) -> argparse.ArgumentParser:
method register_project_dir (line 618) | def register_project_dir(self):
method _build_gitignore_parser (line 622) | def _build_gitignore_parser(self, docs_dir: str):
method _should_exclude_file (line 649) | def _should_exclude_file(self, file_path: Path, gitignore_matches) -> ...
method _is_git_submodule (line 661) | def _is_git_submodule(self, path: Path) -> bool:
method list_indexes (line 687) | def list_indexes(self):
method _discover_indexes_in_project (line 796) | def _discover_indexes_in_project(
method remove_index (line 890) | def remove_index(self, index_name: str, force: bool = False):
method _find_all_matching_indexes (line 906) | def _find_all_matching_indexes(self, index_name: str):
method _remove_single_match (line 1011) | def _remove_single_match(self, match, index_name: str, force: bool):
method _remove_from_multiple_matches (line 1058) | def _remove_from_multiple_matches(self, matches, index_name: str, forc...
method _delete_index_directory (line 1156) | def _delete_index_directory(
method load_documents (line 1218) | def load_documents(
method _parse_file_types (line 1630) | def _parse_file_types(self, custom_file_types: Optional[str]) -> Optio...
method _sync_ignore_patterns (line 1636) | def _sync_ignore_patterns(self, include_hidden: bool) -> Optional[list...
method _build_embedding_options (line 1641) | def _build_embedding_options(self, args) -> dict[str, Any]:
method _resolve_sync_roots (line 1659) | def _resolve_sync_roots(self, docs_paths: list[str]) -> list[str]:
method _create_synchronizers (line 1669) | def _create_synchronizers(
method _build_synchronizers (line 1693) | def _build_synchronizers(
method _detect_build_changes (line 1706) | def _detect_build_changes(
method _commit_synchronizers (line 1721) | def _commit_synchronizers(self, synchronizers: list[FileSynchronizer])...
method _assign_chunk_ids (line 1727) | def _assign_chunk_ids(chunks: list[dict]) -> None:
method _assign_unique_chunk_ids (line 1742) | def _assign_unique_chunk_ids(chunks: list[dict]) -> None:
method _chunks_for_paths (line 1749) | def _chunks_for_paths(self, all_texts: list[dict], paths: set[str]) ->...
method _make_incremental_builder (line 1760) | def _make_incremental_builder(self, args) -> "LeannBuilder":
method _incremental_add_only (line 1773) | def _incremental_add_only(
method _incremental_ivf_remove_only (line 1795) | def _incremental_ivf_remove_only(
method _path_lookup_keys (line 1827) | def _path_lookup_keys(self, path: str, roots: list[str]) -> list[str]:
method _incremental_ivf_update (line 1836) | def _incremental_ivf_update(
method _log_rebuild_reason (line 1909) | def _log_rebuild_reason(
method _write_sync_config (line 1950) | def _write_sync_config(
method _load_sync_roots (line 1966) | def _load_sync_roots(self, index_dir: Path) -> list[str]:
method _resolve_index_for_watch (line 1978) | def _resolve_index_for_watch(self, index_name: str) -> Optional[dict[s...
method _load_chunk_ids_by_file (line 2015) | def _load_chunk_ids_by_file(
method build_index (line 2048) | async def build_index(self, args):
method _watch_check_changes (line 2281) | def _watch_check_changes(self, index_name: str) -> tuple[set[str], set...
method _watch_report_changes (line 2307) | def _watch_report_changes(
method _watch_trigger_build (line 2340) | async def _watch_trigger_build(self, index_name: str) -> None:
method watch_index (line 2384) | async def watch_index(self, args):
method _resolve_index_path (line 2425) | def _resolve_index_path(
method search_documents (line 2504) | async def search_documents(self, args):
method warmup_index (line 2591) | async def warmup_index(self, args):
method daemon_command (line 2613) | async def daemon_command(self, args):
method ask_questions (line 2690) | async def ask_questions(self, args):
method react_agent (line 2763) | async def react_agent(self, args):
method serve_api (line 2839) | async def serve_api(self, args):
method run (line 2868) | async def run(self, args=None):
function main (line 2911) | def main():
FILE: packages/leann-core/src/leann/embedding_compute.py
class _SentenceTransformerLike (line 27) | class _SentenceTransformerLike(Protocol):
method eval (line 28) | def eval(self) -> Any: ...
method parameters (line 29) | def parameters(self) -> Any: ...
method encode (line 30) | def encode(self, *args: Any, **kwargs: Any) -> Any: ...
method half (line 31) | def half(self) -> Any: ...
function get_model_token_limit (line 59) | def get_model_token_limit(
function truncate_to_token_limit (line 134) | def truncate_to_token_limit(texts: list[str], token_limit: int) -> list[...
function _query_ollama_context_limit (line 198) | def _query_ollama_context_limit(model_name: str, base_url: str) -> Optio...
function _query_lmstudio_context_limit (line 231) | def _query_lmstudio_context_limit(model_name: str, base_url: str) -> Opt...
function compute_embeddings (line 320) | def compute_embeddings(
function compute_embeddings_sentence_transformers (line 399) | def compute_embeddings_sentence_transformers(
function compute_embeddings_openai (line 743) | def compute_embeddings_openai(
function compute_embeddings_mlx (line 857) | def compute_embeddings_mlx(chunks: list[str], model_name: str, batch_siz...
function compute_embeddings_ollama (line 931) | def compute_embeddings_ollama(
function compute_embeddings_gemini (line 1254) | def compute_embeddings_gemini(
FILE: packages/leann-core/src/leann/embedding_server_manager.py
function _flock_acquire (line 32) | def _flock_acquire(lock_file) -> None: # type: ignore[type-arg]
function _flock_release (line 73) | def _flock_release(lock_file) -> None: # type: ignore[type-arg]
function _is_colab_environment (line 96) | def _is_colab_environment() -> bool:
function _get_available_port (line 101) | def _get_available_port(start_port: int = 5557) -> int:
function _check_port (line 114) | def _check_port(port: int) -> bool:
function _pid_is_alive (line 120) | def _pid_is_alive(pid: int) -> bool:
function _safe_resolve (line 134) | def _safe_resolve(path: Path) -> str:
function _safe_stat_signature (line 142) | def _safe_stat_signature(path: Path) -> dict:
function _build_passages_signature (line 157) | def _build_passages_signature(passages_file: Optional[str]) -> Optional[...
class EmbeddingServerManager (line 213) | class EmbeddingServerManager:
method __init__ (line 218) | def __init__(self, backend_module_name: str):
method start_server (line 242) | def start_server(
method _build_config_signature (line 349) | def _build_config_signature(
method _start_server_colab (line 371) | def _start_server_colab(
method _start_new_server (line 417) | def _start_new_server(
method _build_server_command (line 451) | def _build_server_command(
method _launch_server_process (line 482) | def _launch_server_process(
method _wait_for_server_ready (line 563) | def _wait_for_server_ready(self, port: int) -> tuple[bool, int]:
method stop_server (line 581) | def stop_server(self):
method _finalize_process (line 648) | def _finalize_process(self) -> None:
method _adopt_existing_server (line 655) | def _adopt_existing_server(self, *args, **kwargs) -> None:
method _launch_server_process_colab (line 659) | def _launch_server_process_colab(
method _wait_for_server_ready_colab (line 701) | def _wait_for_server_ready_colab(self, port: int) -> tuple[bool, int]:
method _registry_dir (line 725) | def _registry_dir() -> Path:
method _registry_lock (line 729) | def _registry_lock(self, config_signature: dict[str, Any]):
method _recover_stale_lock_info (line 768) | def _recover_stale_lock_info(self, lock_info_path: Path) -> None:
method _write_lock_info (line 783) | def _write_lock_info(self, lock_info_path: Path) -> None:
method _registry_key (line 789) | def _registry_key(self, config_signature: dict[str, Any]) -> str:
method _write_registry_record (line 797) | def _write_registry_record(
method _adopt_registered_server (line 820) | def _adopt_registered_server(self, config_signature: dict[str, Any]) -...
method list_daemons (line 853) | def list_daemons(cls) -> list[dict[str, Any]]:
method stop_daemons (line 878) | def stop_daemons(
FILE: packages/leann-core/src/leann/interactive_utils.py
class InteractiveSession (line 26) | class InteractiveSession:
method __init__ (line 29) | def __init__(
method setup_readline (line 53) | def setup_readline(self):
method _show_help (line 87) | def _show_help(self):
method _show_history (line 95) | def _show_history(self):
method get_user_input (line 115) | def get_user_input(self) -> Optional[str]:
method run_interactive_loop (line 131) | def run_interactive_loop(self, handler_func: Callable[[str], None]):
function create_cli_session (line 172) | def create_cli_session(index_name: str) -> InteractiveSession:
function create_api_session (line 182) | def create_api_session() -> InteractiveSession:
function create_rag_session (line 192) | def create_rag_session(app_name: str, data_description: str) -> Interact...
FILE: packages/leann-core/src/leann/interface.py
class LeannBackendBuilderInterface (line 7) | class LeannBackendBuilderInterface(ABC):
method build (line 11) | def build(self, data: np.ndarray, ids: list[str], index_path: str, **k...
class LeannBackendSearcherInterface (line 23) | class LeannBackendSearcherInterface(ABC):
method __init__ (line 27) | def __init__(self, index_path: str, **kwargs):
method _ensure_server_running (line 37) | def _ensure_server_running(
method search (line 44) | def search(
method compute_query_embedding (line 75) | def compute_query_embedding(
class LeannBackendFactoryInterface (line 96) | class LeannBackendFactoryInterface(ABC):
method builder (line 101) | def builder(**kwargs) -> LeannBackendBuilderInterface:
method searcher (line 107) | def searcher(index_path: str, **kwargs) -> LeannBackendSearcherInterface:
FILE: packages/leann-core/src/leann/mcp.py
function handle_request (line 8) | def handle_request(request):
function main (line 142) | def main():
FILE: packages/leann-core/src/leann/metadata_filter.py
class MetadataFilterEngine (line 20) | class MetadataFilterEngine:
method __init__ (line 31) | def __init__(self):
method apply_filters (line 49) | def apply_filters(
method _evaluate_filters (line 77) | def _evaluate_filters(self, result: dict[str, Any], filters: MetadataF...
method _evaluate_field_filter (line 95) | def _evaluate_field_filter(
method _equals (line 143) | def _equals(self, field_value: Any, expected_value: Any) -> bool:
method _not_equals (line 147) | def _not_equals(self, field_value: Any, expected_value: Any) -> bool:
method _less_than (line 151) | def _less_than(self, field_value: Any, expected_value: Any) -> bool:
method _less_than_or_equal (line 155) | def _less_than_or_equal(self, field_value: Any, expected_value: Any) -...
method _greater_than (line 159) | def _greater_than(self, field_value: Any, expected_value: Any) -> bool:
method _greater_than_or_equal (line 163) | def _greater_than_or_equal(self, field_value: Any, expected_value: Any...
method _in (line 168) | def _in(self, field_value: Any, expected_value: Any) -> bool:
method _not_in (line 174) | def _not_in(self, field_value: Any, expected_value: Any) -> bool:
method _contains (line 181) | def _contains(self, field_value: Any, expected_value: Any) -> bool:
method _starts_with (line 187) | def _starts_with(self, field_value: Any, expected_value: Any) -> bool:
method _ends_with (line 193) | def _ends_with(self, field_value: Any, expected_value: Any) -> bool:
method _is_true (line 200) | def _is_true(self, field_value: Any, expected_value: Any) -> bool:
method _is_false (line 204) | def _is_false(self, field_value: Any, expected_value: Any) -> bool:
method _numeric_compare (line 209) | def _numeric_compare(self, field_value: Any, expected_value: Any, comp...
FILE: packages/leann-core/src/leann/react_agent.py
class ReActAgent (line 24) | class ReActAgent:
method __init__ (line 35) | def __init__(
method _format_search_results (line 59) | def _format_search_results(self, results: list[SearchResult]) -> str:
method _create_react_prompt (line 70) | def _create_react_prompt(
method _parse_llm_response (line 102) | def _parse_llm_response(self, response: str) -> tuple[str, str | None]:
method search (line 161) | def search(self, query: str, top_k: int = 5) -> list[SearchResult]:
method run (line 176) | def run(self, question: str, top_k: int = 5) -> str:
function create_react_agent (line 269) | def create_react_agent(
FILE: packages/leann-core/src/leann/registry.py
function register_backend (line 19) | def register_backend(name: str):
function autodiscover_backends (line 30) | def autodiscover_backends():
function register_project_directory (line 52) | def register_project_directory(project_dir: Optional[Union[str, Path]] =...
FILE: packages/leann-core/src/leann/searcher_base.py
class BaseSearcher (line 12) | class BaseSearcher(LeannBackendSearcherInterface, ABC):
method __init__ (line 18) | def __init__(self, index_path: str, backend_module_name: str, **kwargs):
method _load_meta (line 53) | def _load_meta(self) -> dict[str, Any]:
method _ensure_server_running (line 62) | def _ensure_server_running(
method compute_query_embedding (line 104) | def compute_query_embedding(
method _compute_embedding_via_server (line 164) | def _compute_embedding_via_server(self, chunks: list, zmq_port: int) -...
method search (line 197) | def search(
method __del__ (line 228) | def __del__(self):
FILE: packages/leann-core/src/leann/server.py
function _ensure_fastapi (line 25) | def _ensure_fastapi():
function _resolve_index_path (line 40) | def _resolve_index_path(index_name: str) -> str:
function _list_current_project_indexes (line 59) | def _list_current_project_indexes() -> list[dict[str, Any]]:
function create_app (line 85) | def create_app():
function main (line 170) | def main() -> None:
FILE: packages/leann-core/src/leann/settings.py
function _clean_url (line 16) | def _clean_url(value: str) -> str:
function resolve_ollama_host (line 22) | def resolve_ollama_host(explicit: str | None = None) -> str:
function resolve_openai_base_url (line 40) | def resolve_openai_base_url(explicit: str | None = None) -> str:
function resolve_anthropic_base_url (line 57) | def resolve_anthropic_base_url(explicit: str | None = None) -> str:
function resolve_openai_api_key (line 74) | def resolve_openai_api_key(explicit: str | None = None) -> str | None:
function resolve_anthropic_api_key (line 83) | def resolve_anthropic_api_key(explicit: str | None = None) -> str | None:
function resolve_minimax_base_url (line 92) | def resolve_minimax_base_url(explicit: str | None = None) -> str:
function resolve_minimax_api_key (line 108) | def resolve_minimax_api_key(explicit: str | None = None) -> str | None:
function encode_provider_options (line 117) | def encode_provider_options(options: dict[str, Any] | None) -> str | None:
FILE: packages/leann-core/src/leann/sync.py
function hash_data (line 13) | def hash_data(data: str | bytes):
class MerkleTreeNode (line 20) | class MerkleTreeNode:
class MerkleTree (line 28) | class MerkleTree:
method __init__ (line 29) | def __init__(self):
method add_node (line 33) | def add_node(self, data: str, parent_id=None, hash: Optional[str] = No...
method compare_with (line 46) | def compare_with(self, other: "MerkleTree"):
class FileSynchronizer (line 74) | class FileSynchronizer:
method __init__ (line 75) | def __init__(
method generate_file_hashes (line 94) | def generate_file_hashes(self):
method build_merkle_tree (line 123) | def build_merkle_tree(self, file_hashes):
method detect_changes (line 139) | def detect_changes(self) -> tuple[list[str], list[str], list[str]]:
method commit (line 150) | def commit(self):
method create_snapshot (line 157) | def create_snapshot(self):
method check_for_changes (line 163) | def check_for_changes(self) -> tuple[list[str], list[str], list[str]]:
method snapshot_path (line 170) | def snapshot_path(self):
method save_snapshot (line 175) | def save_snapshot(self):
method load_snapshot (line 181) | def load_snapshot(self):
FILE: packages/wechat-exporter/main.py
function get_safe_path (line 14) | def get_safe_path(s: str) -> str:
function process_history (line 26) | def process_history(history: str):
function get_message (line 45) | def get_message(history: dict | str):
function export_chathistory (line 53) | def export_chathistory(user_id: str):
function export_all (line 65) | def export_all(dest: Annotated[Path, typer.Argument(help="Destination pa...
function export_sqlite (line 101) | def export_sqlite(
function main (line 139) | def main():
FILE: scripts/hf_upload.py
function _enable_transfer_accel_if_available (line 26) | def _enable_transfer_accel_if_available() -> None:
function parse_args (line 51) | def parse_args() -> argparse.Namespace:
function main (line 88) | def main() -> None:
FILE: tests/openclaw/conftest.py
function memory_fixtures (line 13) | def memory_fixtures(tmp_path):
function leann_index_dir (line 21) | def leann_index_dir(tmp_path):
function skill_dir (line 29) | def skill_dir():
function claw_manifest (line 35) | def claw_manifest(skill_dir):
FILE: tests/openclaw/test_build_and_search.py
function cli_instance (line 22) | def cli_instance(leann_index_dir):
function _parse_args (line 47) | def _parse_args(cli, argv: list[str]):
function _build_argv (line 52) | def _build_argv(docs_dir: str) -> list[str]:
function test_build_memory_index (line 56) | def test_build_memory_index(cli_instance, memory_fixtures):
function _build_and_search (line 71) | def _build_and_search(cli_instance, memory_fixtures, capsys, query, top_...
function test_search_returns_json (line 88) | def test_search_returns_json(cli_instance, memory_fixtures, capsys):
function test_search_relevance (line 96) | def test_search_relevance(cli_instance, memory_fixtures, capsys):
function test_idempotent_rebuild (line 105) | def test_idempotent_rebuild(cli_instance, memory_fixtures, capsys):
function test_incremental_add (line 118) | def test_incremental_add(cli_instance, memory_fixtures, capsys):
FILE: tests/openclaw/test_mcp_e2e.py
function mcp_index (line 30) | def mcp_index(tmp_path_factory):
function _project_root (line 66) | def _project_root(mcp_index):
function test_mcp_search_via_subprocess (line 75) | def test_mcp_search_via_subprocess(mcp_index):
function test_mcp_search_relevance (line 102) | def test_mcp_search_relevance(mcp_index):
function test_mcp_list (line 127) | def test_mcp_list(mcp_index):
function test_mcp_stdio_protocol (line 142) | def test_mcp_stdio_protocol(mcp_index):
FILE: tests/openclaw/test_mcp_protocol.py
function test_initialize (line 11) | def test_initialize():
function test_tools_list (line 22) | def test_tools_list():
function test_tools_list_search_schema (line 32) | def test_tools_list_search_schema():
function test_search_missing_params (line 44) | def test_search_missing_params():
function test_search_missing_query (line 57) | def test_search_missing_query():
function test_jsonrpc_envelope (line 70) | def test_jsonrpc_envelope():
FILE: tests/openclaw/test_openclaw_e2e.py
function _docker_running (line 25) | def _docker_running() -> bool:
function _openclaw_agent (line 38) | def _openclaw_agent(message: str, timeout: int = 300) -> dict:
function _leann_cmd (line 63) | def _leann_cmd(
function openclaw_ready (line 82) | def openclaw_ready():
function openclaw_memory (line 89) | def openclaw_memory(openclaw_ready):
function leann_index (line 126) | def leann_index(openclaw_memory):
class TestOpenClawMemoryFormation (line 154) | class TestOpenClawMemoryFormation:
method test_memory_dir_exists (line 157) | def test_memory_dir_exists(self, openclaw_memory):
method test_daily_log_created (line 160) | def test_daily_log_created(self, openclaw_memory):
method test_daily_log_content (line 166) | def test_daily_log_content(self, openclaw_memory):
method test_memory_md_exists (line 173) | def test_memory_md_exists(self, openclaw_memory):
method test_memory_md_content (line 177) | def test_memory_md_content(self, openclaw_memory):
class TestLeannIndexOnRealMemory (line 187) | class TestLeannIndexOnRealMemory:
method test_index_built (line 190) | def test_index_built(self, leann_index):
method test_search_returns_results (line 195) | def test_search_returns_results(self, leann_index):
method test_search_relevance_bug_fix (line 211) | def test_search_relevance_bug_fix(self, leann_index):
method test_search_relevance_project_info (line 230) | def test_search_relevance_project_info(self, leann_index):
method test_search_score_is_native_float (line 249) | def test_search_score_is_native_float(self, leann_index):
method test_search_metadata_has_file_path (line 266) | def test_search_metadata_has_file_path(self, leann_index):
FILE: tests/openclaw/test_skill_manifest.py
function test_claw_json_required_fields (line 4) | def test_claw_json_required_fields(claw_manifest):
function test_claw_json_name (line 11) | def test_claw_json_name(claw_manifest):
function test_claw_json_permissions (line 15) | def test_claw_json_permissions(claw_manifest):
function test_claw_json_entry_exists (line 20) | def test_claw_json_entry_exists(skill_dir, claw_manifest):
function test_claw_json_tags (line 26) | def test_claw_json_tags(claw_manifest):
function test_claw_json_models (line 33) | def test_claw_json_models(claw_manifest):
function test_instructions_contains_build_command (line 39) | def test_instructions_contains_build_command(skill_dir, claw_manifest):
function test_instructions_contains_search_command (line 45) | def test_instructions_contains_search_command(skill_dir, claw_manifest):
function test_instructions_contains_install_check (line 52) | def test_instructions_contains_install_check(skill_dir, claw_manifest):
function test_readme_exists (line 58) | def test_readme_exists(skill_dir):
FILE: tests/support/fake_embedding_server_module.py
function main (line 7) | def main() -> None:
FILE: tests/test_astchunk_integration.py
class MockDocument (line 29) | class MockDocument:
method __init__ (line 32) | def __init__(self, content: str, file_path: str = "", metadata: Option...
method get_content (line 38) | def get_content(self) -> str:
class TestCodeFileDetection (line 42) | class TestCodeFileDetection:
method test_detect_code_files_python (line 45) | def test_detect_code_files_python(self):
method test_detect_code_files_multiple_languages (line 60) | def test_detect_code_files_multiple_languages(self):
method test_detect_code_files_no_file_path (line 81) | def test_detect_code_files_no_file_path(self):
method test_get_language_from_extension (line 95) | def test_get_language_from_extension(self):
class TestChunkingFunctions (line 105) | class TestChunkingFunctions:
method test_create_traditional_chunks (line 108) | def test_create_traditional_chunks(self):
method test_create_traditional_chunks_empty_docs (line 124) | def test_create_traditional_chunks_empty_docs(self):
method test_create_ast_chunks_with_astchunk_available (line 133) | def test_create_ast_chunks_with_astchunk_available(self):
method test_create_ast_chunks_fallback_to_traditional (line 187) | def test_create_ast_chunks_fallback_to_traditional(self):
method test_create_text_chunks_traditional_mode (line 200) | def test_create_text_chunks_traditional_mode(self):
method test_create_text_chunks_ast_mode (line 216) | def test_create_text_chunks_ast_mode(self):
method test_create_text_chunks_custom_extensions (line 239) | def test_create_text_chunks_custom_extensions(self):
class TestIntegrationWithDocumentRAG (line 259) | class TestIntegrationWithDocumentRAG:
method temp_code_dir (line 263) | def temp_code_dir(self):
method test_document_rag_with_ast_chunking (line 296) | def test_document_rag_with_ast_chunking(self, temp_code_dir):
method test_code_rag_application (line 344) | def test_code_rag_application(self, temp_code_dir):
class TestASTContentExtraction (line 379) | class TestASTContentExtraction:
method test_extract_content_from_astchunk_dict (line 386) | def test_extract_content_from_astchunk_dict(self):
method test_extract_text_key_fallback (line 459) | def test_extract_text_key_fallback(self):
method test_handles_string_chunks (line 499) | def test_handles_string_chunks(self):
method test_multiple_chunks_with_mixed_formats (line 538) | def test_multiple_chunks_with_mixed_formats(self):
method test_empty_content_value_handling (line 591) | def test_empty_content_value_handling(self):
class TestASTMetadataPreservation (line 629) | class TestASTMetadataPreservation:
method test_ast_chunks_preserve_file_metadata (line 640) | def test_ast_chunks_preserve_file_metadata(self):
method test_ast_chunks_include_astchunk_metadata (line 743) | def test_ast_chunks_include_astchunk_metadata(self):
method test_traditional_chunks_as_dicts_helper (line 846) | def test_traditional_chunks_as_dicts_helper(self):
class TestErrorHandling (line 925) | class TestErrorHandling:
method test_text_chunking_empty_documents (line 928) | def test_text_chunking_empty_documents(self):
method test_text_chunking_invalid_parameters (line 933) | def test_text_chunking_invalid_parameters(self):
method test_create_ast_chunks_no_language (line 945) | def test_create_ast_chunks_no_language(self):
method test_create_ast_chunks_empty_content (line 955) | def test_create_ast_chunks_empty_content(self):
FILE: tests/test_basic.py
function test_imports (line 12) | def test_imports():
function test_backend_basic (line 22) | def test_backend_basic(backend_name):
function test_large_index (line 64) | def test_large_index():
FILE: tests/test_ci_minimal.py
function test_package_imports (line 9) | def test_package_imports():
function test_cli_help (line 20) | def test_cli_help():
function test_backend_registration (line 31) | def test_backend_registration():
function test_version_info (line 40) | def test_version_info():
FILE: tests/test_cli_ask.py
function test_cli_ask_accepts_positional_query (line 4) | def test_cli_ask_accepts_positional_query(tmp_path, monkeypatch):
FILE: tests/test_cli_daemon_workflow.py
function test_search_passes_daemon_flags_to_searcher (line 9) | def test_search_passes_daemon_flags_to_searcher(monkeypatch):
function test_warmup_command_calls_searcher_warmup (line 53) | def test_warmup_command_calls_searcher_warmup(monkeypatch):
function test_daemon_status_filters_by_index (line 82) | def test_daemon_status_filters_by_index(monkeypatch, capsys):
function test_daemon_stop_by_index_calls_stop_daemons (line 120) | def test_daemon_stop_by_index_calls_stop_daemons(monkeypatch):
function test_daemon_start_calls_searcher_warmup (line 141) | def test_daemon_start_calls_searcher_warmup(monkeypatch):
function test_daemon_status_all_lists_records (line 173) | def test_daemon_status_all_lists_records(monkeypatch, capsys):
function test_daemon_stop_all_calls_manager (line 200) | def test_daemon_stop_all_calls_manager(monkeypatch):
FILE: tests/test_cli_prompt_template.py
class TestCLIPromptTemplateArgument (line 16) | class TestCLIPromptTemplateArgument:
method test_commands_accept_prompt_template_argument (line 19) | def test_commands_accept_prompt_template_argument(self):
method test_commands_default_to_none (line 52) | def test_commands_default_to_none(self):
class TestBuildCommandPromptTemplateArgumentExtras (line 76) | class TestBuildCommandPromptTemplateArgumentExtras:
method test_build_command_prompt_template_with_multiword_value (line 79) | def test_build_command_prompt_template_with_multiword_value(self):
class TestPromptTemplateStoredInEmbeddingOptions (line 104) | class TestPromptTemplateStoredInEmbeddingOptions:
method test_prompt_template_stored_in_embedding_options_on_build (line 108) | def test_prompt_template_stored_in_embedding_options_on_build(
method test_prompt_template_not_in_options_when_not_provided (line 164) | def test_prompt_template_not_in_options_when_not_provided(self, mock_b...
method test_build_stores_separate_templates (line 208) | def test_build_stores_separate_templates(self, mock_builder_class, tmp...
method test_build_backward_compat_single_template (line 287) | def test_build_backward_compat_single_template(self, mock_builder_clas...
method test_build_no_templates (line 357) | def test_build_no_templates(self, mock_builder_class, tmp_path):
class TestPromptTemplateFlowsToComputeEmbeddings (line 407) | class TestPromptTemplateFlowsToComputeEmbeddings:
method test_prompt_template_flows_to_compute_embeddings_via_provider_options (line 411) | def test_prompt_template_flows_to_compute_embeddings_via_provider_opti...
class TestPromptTemplateArgumentHelp (line 478) | class TestPromptTemplateArgumentHelp:
method test_build_command_prompt_template_has_help_text (line 481) | def test_build_command_prompt_template_has_help_text(self):
method test_search_command_prompt_template_has_help_text (line 513) | def test_search_command_prompt_template_has_help_text(self):
FILE: tests/test_cli_verbosity.py
class TestSuppressCppOutput (line 13) | class TestSuppressCppOutput:
method test_suppress_cpp_output_captures_stdout (line 16) | def test_suppress_cpp_output_captures_stdout(self):
method test_suppress_cpp_output_captures_stderr (line 27) | def test_suppress_cpp_output_captures_stderr(self):
method test_suppress_cpp_output_restores_fds (line 35) | def test_suppress_cpp_output_restores_fds(self):
method test_suppress_cpp_output_disabled (line 55) | def test_suppress_cpp_output_disabled(self):
class TestCliVerbosityArgs (line 65) | class TestCliVerbosityArgs:
method test_verbose_flag_parsed (line 68) | def test_verbose_flag_parsed(self):
method test_quiet_flag_parsed (line 79) | def test_quiet_flag_parsed(self):
method test_verbose_short_flag (line 90) | def test_verbose_short_flag(self):
method test_verbose_and_quiet_mutually_exclusive (line 100) | def test_verbose_and_quiet_mutually_exclusive(self):
method test_default_is_quiet (line 110) | def test_default_is_quiet(self):
class TestVerbosityIntegration (line 123) | class TestVerbosityIntegration:
method test_list_command_does_not_suppress (line 126) | def test_list_command_does_not_suppress(self):
method test_verbose_flag_with_list (line 139) | def test_verbose_flag_with_list(self):
FILE: tests/test_cpu_only_install.py
function _load_leann_pyproject (line 13) | def _load_leann_pyproject():
function _load_leann_core_pyproject (line 18) | def _load_leann_core_pyproject():
function test_leann_base_dependencies_include_diskann (line 25) | def test_leann_base_dependencies_include_diskann():
function test_leann_core_numpy_pinned_below_2 (line 34) | def test_leann_core_numpy_pinned_below_2():
function test_leann_core_cpu_extra_pins_cpu_torch (line 41) | def test_leann_core_cpu_extra_pins_cpu_torch():
function test_leann_cpu_extra_defined (line 56) | def test_leann_cpu_extra_defined():
FILE: tests/test_diskann_partition.py
function test_diskann_without_partition (line 20) | def test_diskann_without_partition():
function test_diskann_with_partition (line 84) | def test_diskann_with_partition():
function test_diskann_partition_search_functionality (line 149) | def test_diskann_partition_search_functionality():
function test_diskann_medoid_and_norm_files (line 208) | def test_diskann_medoid_and_norm_files():
function test_diskann_vs_hnsw_performance (line 272) | def test_diskann_vs_hnsw_performance():
FILE: tests/test_document_rag.py
function test_data_dir (line 15) | def test_data_dir():
function test_document_rag_simulated (line 23) | def test_document_rag_simulated(test_data_dir):
function test_document_rag_with_ast_chunking (line 64) | def test_document_rag_with_ast_chunking(test_data_dir):
function test_document_rag_openai (line 109) | def test_document_rag_openai(test_data_dir):
function test_document_rag_error_handling (line 150) | def test_document_rag_error_handling(test_data_dir):
FILE: tests/test_embedding_prompt_template.py
class TestPromptTemplatePrepending (line 23) | class TestPromptTemplatePrepending:
method mock_openai_client (line 27) | def mock_openai_client(self):
method mock_openai_module (line 42) | def mock_openai_module(self, mock_openai_client, monkeypatch):
method test_prompt_template_prepended_to_all_texts (line 51) | def test_prompt_template_prepended_to_all_texts(self, mock_openai_modu...
method test_template_not_applied_when_missing_or_empty (line 91) | def test_template_not_applied_when_missing_or_empty(
method test_prompt_template_with_multiple_batches (line 160) | def test_prompt_template_with_multiple_batches(self, mock_openai_modul...
method test_gemini_openai_compat_caps_batch_size_to_100 (line 200) | def test_gemini_openai_compat_caps_batch_size_to_100(
method test_prompt_template_with_special_characters (line 225) | def test_prompt_template_with_special_characters(self, mock_openai_mod...
method test_prompt_template_integration_with_existing_validation (line 252) | def test_prompt_template_integration_with_existing_validation(
method test_prompt_template_with_api_key_and_base_url (line 274) | def test_prompt_template_with_api_key_and_base_url(
FILE: tests/test_embedding_server_cli_flags.py
function _run_help (line 7) | def _run_help(module_name: str) -> str:
function test_hnsw_server_help_has_daemon_and_warmup_flags (line 28) | def test_hnsw_server_help_has_daemon_and_warmup_flags():
function test_diskann_server_help_has_daemon_and_warmup_flags (line 35) | def test_diskann_server_help_has_daemon_and_warmup_flags():
FILE: tests/test_embedding_server_manager.py
class DummyProcess (line 11) | class DummyProcess:
method __init__ (line 12) | def __init__(self, pid=12345):
method poll (line 16) | def poll(self):
method terminate (line 19) | def terminate(self):
method kill (line 22) | def kill(self):
method wait (line 25) | def wait(self, timeout=None):
function embedding_manager (line 31) | def embedding_manager(monkeypatch):
function _write_meta (line 69) | def _write_meta(meta_path, passages_name, index_name, total):
function test_server_restarts_when_metadata_changes (line 92) | def test_server_restarts_when_metadata_changes(tmp_path, embedding_manag...
function test_list_daemons_ignores_stale_records (line 146) | def test_list_daemons_ignores_stale_records(tmp_path, monkeypatch):
function test_stop_daemons_filters_by_backend_and_passages (line 169) | def test_stop_daemons_filters_by_backend_and_passages(tmp_path, monkeypa...
function test_daemon_registry_reuse_across_manager_instances (line 220) | def test_daemon_registry_reuse_across_manager_instances(tmp_path, monkey...
function test_stale_registry_falls_back_to_fresh_start (line 266) | def test_stale_registry_falls_back_to_fresh_start(tmp_path, monkeypatch):
function test_build_server_command_includes_daemon_and_warmup_flags (line 323) | def test_build_server_command_includes_daemon_and_warmup_flags():
function test_corrupted_registry_file_is_recovered_on_start (line 351) | def test_corrupted_registry_file_is_recovered_on_start(tmp_path, monkeyp...
function test_stop_server_detaches_when_daemon_mode (line 393) | def test_stop_server_detaches_when_daemon_mode(monkeypatch):
function test_concurrent_daemon_start_only_spawns_once (line 417) | def test_concurrent_daemon_start_only_spawns_once(tmp_path, monkeypatch):
function test_registry_record_write_is_atomic (line 462) | def test_registry_record_write_is_atomic(tmp_path, monkeypatch):
function test_stale_lock_info_removed_when_pid_dead (line 492) | def test_stale_lock_info_removed_when_pid_dead(tmp_path, monkeypatch):
FILE: tests/test_embedding_server_manager_e2e.py
function _configure_fake_module_env (line 8) | def _configure_fake_module_env(monkeypatch):
function _wait_until (line 16) | def _wait_until(predicate, timeout=5.0, interval=0.05):
function test_daemon_reuse_with_real_subprocess (line 25) | def test_daemon_reuse_with_real_subprocess(tmp_path, monkeypatch):
function test_daemon_ttl_expiry_with_real_subprocess (line 64) | def test_daemon_ttl_expiry_with_real_subprocess(tmp_path, monkeypatch):
FILE: tests/test_hybrid_search.py
class TestHybridSearch (line 18) | class TestHybridSearch:
method sample_index (line 22) | def sample_index(self):
method test_pure_vector_search (line 62) | def test_pure_vector_search(self, sample_index):
method test_pure_keyword_search (line 77) | def test_pure_keyword_search(self, sample_index):
method test_hybrid_search_balanced (line 91) | def test_hybrid_search_balanced(self, sample_index):
method test_hybrid_search_vector_heavy (line 104) | def test_hybrid_search_vector_heavy(self, sample_index):
method test_hybrid_search_keyword_heavy (line 119) | def test_hybrid_search_keyword_heavy(self, sample_index):
method test_hybrid_search_score_combination (line 136) | def test_hybrid_search_score_combination(self, sample_index):
method test_hybrid_search_with_metadata_filters (line 154) | def test_hybrid_search_with_metadata_filters(self, sample_index):
FILE: tests/test_incremental_build.py
function test_normalize_path (line 17) | def test_normalize_path():
function test_file_synchronizer_detect_changes (line 25) | def test_file_synchronizer_detect_changes(tmp_path):
function test_file_synchronizer_no_changes_after_commit (line 38) | def test_file_synchronizer_no_changes_after_commit(tmp_path):
function test_file_synchronizer_detects_new_file (line 53) | def test_file_synchronizer_detects_new_file(tmp_path):
function test_file_synchronizer_detects_modification (line 71) | def test_file_synchronizer_detects_modification(tmp_path):
function test_file_synchronizer_touch_no_false_positive (line 88) | def test_file_synchronizer_touch_no_false_positive(tmp_path):
function test_incremental_build_adds_only_new_files (line 109) | def test_incremental_build_adds_only_new_files(tmp_path):
function test_ivf_incremental_add_then_remove_searchable (line 181) | def test_ivf_incremental_add_then_remove_searchable(tmp_path):
function test_ivf_multiple_incremental_no_duplicates (line 249) | def test_ivf_multiple_incremental_no_duplicates(tmp_path):
FILE: tests/test_lmstudio_bridge.py
function _query_lmstudio_context_limit (line 36) | def _query_lmstudio_context_limit(*args, **kwargs):
class TestLMStudioBridge (line 42) | class TestLMStudioBridge:
method test_query_lmstudio_success (line 45) | def test_query_lmstudio_success(self, monkeypatch):
method test_query_lmstudio_nodejs_not_found (line 77) | def test_query_lmstudio_nodejs_not_found(self, monkeypatch):
method test_query_lmstudio_sdk_not_installed (line 95) | def test_query_lmstudio_sdk_not_installed(self, monkeypatch):
method test_query_lmstudio_timeout (line 120) | def test_query_lmstudio_timeout(self, monkeypatch):
method test_query_lmstudio_invalid_json (line 139) | def test_query_lmstudio_invalid_json(self, monkeypatch):
method test_query_lmstudio_missing_context_length_field (line 162) | def test_query_lmstudio_missing_context_length_field(self, monkeypatch):
method test_query_lmstudio_null_context_length (line 184) | def test_query_lmstudio_null_context_length(self, monkeypatch):
method test_query_lmstudio_zero_context_length (line 206) | def test_query_lmstudio_zero_context_length(self, monkeypatch):
method test_query_lmstudio_with_custom_port (line 228) | def test_query_lmstudio_with_custom_port(self, monkeypatch):
method test_query_lmstudio_various_context_lengths (line 265) | def test_query_lmstudio_various_context_lengths(self, monkeypatch, con...
method test_query_lmstudio_logs_at_debug_level (line 287) | def test_query_lmstudio_logs_at_debug_level(self, monkeypatch, caplog):
FILE: tests/test_mcp_integration.py
function test_slack_reader_initialization (line 21) | def test_slack_reader_initialization():
function test_twitter_reader_initialization (line 45) | def test_twitter_reader_initialization():
function test_slack_message_formatting (line 72) | def test_slack_message_formatting():
function test_twitter_bookmark_formatting (line 100) | def test_twitter_bookmark_formatting():
function test_slack_rag_initialization (line 134) | def test_slack_rag_initialization():
function test_twitter_rag_initialization (line 145) | def test_twitter_rag_initialization():
function test_concatenated_content_creation (line 156) | def test_concatenated_content_creation():
function main (line 180) | def main():
FILE: tests/test_mcp_standalone.py
function test_slack_reader_basic (line 17) | def test_slack_reader_basic():
function test_twitter_reader_basic (line 56) | def test_twitter_reader_basic():
function test_mcp_request_format (line 99) | def test_mcp_request_format():
function test_data_processing (line 132) | def test_data_processing():
function main (line 180) | def main():
FILE: tests/test_metadata_filtering.py
class TestMetadataFilterEngine (line 20) | class TestMetadataFilterEngine:
method setup_method (line 23) | def setup_method(self):
method test_engine_initialization (line 83) | def test_engine_initialization(self):
method test_direct_instantiation (line 91) | def test_direct_instantiation(self):
method test_no_filters_returns_all_results (line 96) | def test_no_filters_returns_all_results(self):
method test_equals_filter (line 107) | def test_equals_filter(self):
method test_not_equals_filter (line 114) | def test_not_equals_filter(self):
method test_less_than_filter (line 121) | def test_less_than_filter(self):
method test_less_than_or_equal_filter (line 129) | def test_less_than_or_equal_filter(self):
method test_greater_than_filter (line 137) | def test_greater_than_filter(self):
method test_greater_than_or_equal_filter (line 145) | def test_greater_than_or_equal_filter(self):
method test_in_filter (line 154) | def test_in_filter(self):
method test_not_in_filter (line 162) | def test_not_in_filter(self):
method test_contains_filter (line 170) | def test_contains_filter(self):
method test_starts_with_filter (line 176) | def test_starts_with_filter(self):
method test_ends_with_filter (line 183) | def test_ends_with_filter(self):
method test_is_true_filter (line 190) | def test_is_true_filter(self):
method test_is_false_filter (line 197) | def test_is_false_filter(self):
method test_compound_filters (line 205) | def test_compound_filters(self):
method test_multiple_operators_same_field (line 214) | def test_multiple_operators_same_field(self):
method test_missing_field_fails_filter (line 224) | def test_missing_field_fails_filter(self):
method test_invalid_operator (line 230) | def test_invalid_operator(self):
method test_type_coercion_numeric (line 236) | def test_type_coercion_numeric(self):
method test_list_membership_with_nested_tags (line 257) | def test_list_membership_with_nested_tags(self):
method test_empty_results_list (line 266) | def test_empty_results_list(self):
class TestPassageManagerFiltering (line 273) | class TestPassageManagerFiltering:
method setup_method (line 276) | def setup_method(self):
method test_search_result_filtering (line 304) | def test_search_result_filtering(self):
method test_filter_search_results_no_filters (line 318) | def test_filter_search_results_no_filters(self):
method test_filter_maintains_search_result_type (line 326) | def test_filter_maintains_search_result_type(self):
FILE: tests/test_minimax_provider.py
class TestMiniMaxSettings (line 37) | class TestMiniMaxSettings:
method test_resolve_minimax_api_key_explicit (line 40) | def test_resolve_minimax_api_key_explicit(self):
method test_resolve_minimax_api_key_from_env (line 43) | def test_resolve_minimax_api_key_from_env(self):
method test_resolve_minimax_api_key_none (line 47) | def test_resolve_minimax_api_key_none(self):
method test_resolve_minimax_base_url_default (line 51) | def test_resolve_minimax_base_url_default(self):
method test_resolve_minimax_base_url_explicit (line 55) | def test_resolve_minimax_base_url_explicit(self):
method test_resolve_minimax_base_url_from_env (line 58) | def test_resolve_minimax_base_url_from_env(self):
method test_resolve_minimax_base_url_leann_env (line 62) | def test_resolve_minimax_base_url_leann_env(self):
method test_resolve_minimax_base_url_strips_trailing_slash (line 66) | def test_resolve_minimax_base_url_strips_trailing_slash(self):
class TestMiniMaxChat (line 70) | class TestMiniMaxChat:
method test_init_requires_api_key (line 73) | def test_init_requires_api_key(self):
method test_init_with_api_key (line 81) | def test_init_with_api_key(self, mock_openai_cls):
method test_init_custom_model (line 93) | def test_init_custom_model(self, mock_openai_cls):
method test_init_custom_base_url (line 100) | def test_init_custom_base_url(self, mock_openai_cls):
method test_ask_returns_response (line 107) | def test_ask_returns_response(self, mock_openai_cls):
method test_ask_with_kwargs (line 130) | def test_ask_with_kwargs(self, mock_openai_cls):
method test_ask_handles_error (line 154) | def test_ask_handles_error(self, mock_openai_cls):
class TestGetLLMFactory (line 168) | class TestGetLLMFactory:
method test_get_llm_minimax (line 172) | def test_get_llm_minimax(self, mock_openai_cls):
method test_get_llm_minimax_custom_model (line 180) | def test_get_llm_minimax_custom_model(self, mock_openai_cls):
method test_get_llm_minimax_custom_base_url (line 188) | def test_get_llm_minimax_custom_base_url(self, mock_openai_cls):
class TestMiniMaxLiveAPI (line 206) | class TestMiniMaxLiveAPI:
method test_minimax_m25_live (line 209) | def test_minimax_m25_live(self):
method test_minimax_m25_highspeed_live (line 217) | def test_minimax_m25_highspeed_live(self):
method test_minimax_via_get_llm_live (line 225) | def test_minimax_via_get_llm_live(self):
FILE: tests/test_prompt_template_e2e.py
function check_service_available (line 36) | def check_service_available(host: str, port: int, timeout: float = 2.0) ...
function check_ollama_available (line 48) | def check_ollama_available() -> bool:
function check_lmstudio_available (line 59) | def check_lmstudio_available() -> bool:
function get_lmstudio_first_model (line 70) | def get_lmstudio_first_model() -> str | None:
class TestPromptTemplateOpenAI (line 83) | class TestPromptTemplateOpenAI:
method test_lmstudio_embedding_with_prompt_template (line 89) | def test_lmstudio_embedding_with_prompt_template(self):
method test_lmstudio_prompt_template_affects_embeddings (line 119) | def test_lmstudio_prompt_template_affects_embeddings(self):
class TestPromptTemplateOllama (line 154) | class TestPromptTemplateOllama:
method test_ollama_embedding_with_prompt_template (line 160) | def test_ollama_embedding_with_prompt_template(self):
method test_ollama_prompt_template_affects_embeddings (line 205) | def test_ollama_prompt_template_affects_embeddings(self):
class TestLMStudioSDK (line 249) | class TestLMStudioSDK:
method test_lmstudio_model_listing (line 253) | def test_lmstudio_model_listing(self):
method test_lmstudio_sdk_context_length_detection (line 271) | def test_lmstudio_sdk_context_length_detection(self):
class TestOllamaTokenLimit (line 304) | class TestOllamaTokenLimit:
method test_ollama_token_limit_detection (line 308) | def test_ollama_token_limit_detection(self):
class TestHybridTokenLimit (line 337) | class TestHybridTokenLimit:
method test_hybrid_discovery_registry_fallback (line 340) | def test_hybrid_discovery_registry_fallback(self):
method test_hybrid_discovery_default_fallback (line 352) | def test_hybrid_discovery_default_fallback(self):
method test_hybrid_discovery_ollama_dynamic_first (line 365) | def test_hybrid_discovery_ollama_dynamic_first(self):
FILE: tests/test_prompt_template_persistence.py
class TestPromptTemplateMetadataPersistence (line 30) | class TestPromptTemplateMetadataPersistence:
method temp_index_dir (line 34) | def temp_index_dir(self):
method mock_embeddings (line 40) | def mock_embeddings(self):
method test_prompt_template_saved_to_metadata (line 47) | def test_prompt_template_saved_to_metadata(self, temp_index_dir, mock_...
method test_prompt_template_absent_when_not_provided (line 100) | def test_prompt_template_absent_when_not_provided(self, temp_index_dir...
class TestPromptTemplateAutoLoadOnSearch (line 137) | class TestPromptTemplateAutoLoadOnSearch:
method temp_index_dir (line 146) | def temp_index_dir(self):
method mock_embeddings (line 152) | def mock_embeddings(self):
method test_search_without_template_in_metadata (line 158) | def test_search_without_template_in_metadata(self, temp_index_dir, moc...
class TestQueryPromptTemplateAutoLoad (line 190) | class TestQueryPromptTemplateAutoLoad:
method temp_index_dir (line 202) | def temp_index_dir(self):
method mock_compute_embeddings (line 208) | def mock_compute_embeddings(self):
method test_search_auto_loads_query_template (line 214) | def test_search_auto_loads_query_template(self, temp_index_dir, mock_c...
method test_search_backward_compat_single_template (line 270) | def test_search_backward_compat_single_template(self, temp_index_dir, ...
method test_search_backward_compat_no_template (line 316) | def test_search_backward_compat_no_template(self, temp_index_dir, mock...
method test_search_override_via_provider_options (line 362) | def test_search_override_via_provider_options(self, temp_index_dir, mo...
class TestPromptTemplateReuseInChat (line 419) | class TestPromptTemplateReuseInChat:
method temp_index_dir (line 423) | def temp_index_dir(self):
method mock_embeddings (line 429) | def mock_embeddings(self):
method mock_embedding_server_manager (line 436) | def mock_embedding_server_manager(self):
method index_with_template (line 445) | def index_with_template(self, temp_index_dir, mock_embeddings):
class TestPromptTemplateIntegrationWithEmbeddingModes (line 463) | class TestPromptTemplateIntegrationWithEmbeddingModes:
method temp_index_dir (line 467) | def temp_index_dir(self):
method test_prompt_template_metadata_with_embedding_modes (line 485) | def test_prompt_template_metadata_with_embedding_modes(
class TestQueryTemplateApplicationInComputeEmbedding (line 523) | class TestQueryTemplateApplicationInComputeEmbedding:
method temp_index_with_template (line 544) | def temp_index_with_template(self):
method test_query_template_applied_in_fallback_path (line 578) | def test_query_template_applied_in_fallback_path(self, temp_index_with...
method test_query_template_applied_in_server_path (line 636) | def test_query_template_applied_in_server_path(self, temp_index_with_t...
method test_query_template_without_template_parameter (line 697) | def test_query_template_without_template_parameter(self, temp_index_wi...
method test_query_template_consistency_between_paths (line 750) | def test_query_template_consistency_between_paths(self, temp_index_wit...
method test_query_template_with_empty_string (line 828) | def test_query_template_with_empty_string(self, temp_index_with_templa...
FILE: tests/test_readme_examples.py
function test_readme_basic_example (line 14) | def test_readme_basic_example(backend_name):
function test_readme_imports (line 67) | def test_readme_imports():
function test_backend_options (line 78) | def test_backend_options():
function test_llm_config_simulated (line 119) | def test_llm_config_simulated(backend_name):
function test_llm_config_hf (line 153) | def test_llm_config_hf():
FILE: tests/test_sync.py
class TestMerkleTreeCompare (line 9) | class TestMerkleTreeCompare(unittest.TestCase):
method test_no_changes_if_root_hash_same (line 10) | def test_no_changes_if_root_hash_same(self):
method test_added_removed_modified (line 23) | def test_added_removed_modified(self):
class TestFileSynchronizer (line 59) | class TestFileSynchronizer(unittest.TestCase):
method test_generate_file_hashes (line 60) | def test_generate_file_hashes(self):
method test_build_merkle_tree (line 80) | def test_build_merkle_tree(self):
method test_check_for_changes_detected (line 103) | def test_check_for_changes_detected(self):
FILE: tests/test_token_truncation.py
class TestModelTokenLimits (line 23) | class TestModelTokenLimits:
method test_get_model_token_limit_known_model (line 26) | def test_get_model_token_limit_known_model(self):
method test_get_model_token_limit_unknown_model (line 48) | def test_get_model_token_limit_unknown_model(self):
method test_get_model_token_limit_custom_default (line 62) | def test_get_model_token_limit_custom_default(self):
method test_embedding_model_limits_dictionary_exists (line 75) | def test_embedding_model_limits_dictionary_exists(self):
class TestTokenTruncation (line 93) | class TestTokenTruncation:
method tokenizer (line 97) | def tokenizer(self):
method test_truncate_single_text_under_limit (line 101) | def test_truncate_single_text_under_limit(self, tokenizer):
method test_truncate_single_text_over_limit (line 117) | def test_truncate_single_text_over_limit(self, tokenizer):
method test_truncate_multiple_texts_mixed_lengths (line 143) | def test_truncate_multiple_texts_mixed_lengths(self, tokenizer):
method test_truncate_empty_list (line 185) | def test_truncate_empty_list(self):
method test_truncate_preserves_order (line 193) | def test_truncate_preserves_order(self, tokenizer):
method test_truncate_extremely_long_text (line 213) | def test_truncate_extremely_long_text(self, tokenizer):
method test_truncate_exact_token_limit (line 234) | def test_truncate_exact_token_limit(self, tokenizer):
class TestLMStudioHybridDiscovery (line 271) | class TestLMStudioHybridDiscovery:
method test_get_model_token_limit_lmstudio_success (line 288) | def test_get_model_token_limit_lmstudio_success(self, monkeypatch):
method test_get_model_token_limit_lmstudio_fallback_to_registry (line 316) | def test_get_model_token_limit_lmstudio_fallback_to_registry(self, mon...
method test_get_model_token_limit_lmstudio_port_detection (line 340) | def test_get_model_token_limit_lmstudio_port_detection(self, monkeypat...
method test_get_model_token_limit_lmstudio_url_keyword_detection (line 371) | def test_get_model_token_limit_lmstudio_url_keyword_detection(
method test_get_model_token_limit_protocol_conversion (line 403) | def test_get_model_token_limit_protocol_conversion(
method test_get_model_token_limit_lmstudio_executes_after_ollama (line 436) | def test_get_model_token_limit_lmstudio_executes_after_ollama(self, mo...
method test_get_model_token_limit_lmstudio_not_detected_for_non_lmstudio_urls (line 478) | def test_get_model_token_limit_lmstudio_not_detected_for_non_lmstudio_...
method test_get_model_token_limit_lmstudio_case_insensitive_detection (line 508) | def test_get_model_token_limit_lmstudio_case_insensitive_detection(sel...
class TestTokenLimitCaching (line 541) | class TestTokenLimitCaching:
method setup_method (line 553) | def setup_method(self):
method test_registry_lookup_is_cached (line 559) | def test_registry_lookup_is_cached(self):
method test_default_fallback_is_cached (line 576) | def test_default_fallback_is_cached(self):
method test_different_urls_create_separate_cache_entries (line 593) | def test_different_urls_create_separate_cache_entries(self):
method test_cache_prevents_repeated_lookups (line 613) | def test_cache_prevents_repeated_lookups(self):
method test_versioned_model_names_cached_correctly (line 632) | def test_versioned_model_names_cached_correctly(self):
Condensed preview — 221 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (2,824K chars).
[
{
"path": ".devcontainer/devcontainer.json",
"chars": 1255,
"preview": "{\n \"name\": \"LEANN Dev\",\n \"build\": {\n \"context\": \"..\",\n \"dockerfile\": \"../docker/Dockerfile.dev\",\n \"args\": {\n "
},
{
"path": ".github/ISSUE_TEMPLATE/bug_report.yml",
"chars": 942,
"preview": "name: Bug Report\ndescription: Report a bug in LEANN\nlabels: [\"bug\"]\n\nbody:\n - type: textarea\n id: description\n at"
},
{
"path": ".github/ISSUE_TEMPLATE/config.yml",
"chars": 283,
"preview": "blank_issues_enabled: true\ncontact_links:\n - name: Documentation\n url: https://github.com/LEANN-RAG/LEANN-RAG/tree/m"
},
{
"path": ".github/ISSUE_TEMPLATE/feature_request.yml",
"chars": 599,
"preview": "name: Feature Request\ndescription: Suggest a new feature for LEANN\nlabels: [\"enhancement\"]\n\nbody:\n - type: textarea\n "
},
{
"path": ".github/pull_request_template.md",
"chars": 260,
"preview": "## What does this PR do?\n\n<!-- Brief description of your changes -->\n\n## Related Issues\n\nFixes #\n\n## Checklist\n\n- [ ] Te"
},
{
"path": ".github/workflows/build-and-publish.yml",
"chars": 170,
"preview": "name: CI\n\non:\n push:\n branches: [ main ]\n pull_request:\n branches: [ main ]\n workflow_dispatch:\n\njobs:\n build:"
},
{
"path": ".github/workflows/build-reusable.yml",
"chars": 25376,
"preview": "name: Reusable Build\n\non:\n workflow_call:\n inputs:\n ref:\n description: 'Git ref to build'\n requir"
},
{
"path": ".github/workflows/link-check.yml",
"chars": 708,
"preview": "name: Link Check\n\non:\n push:\n branches: [ main, master ]\n pull_request:\n schedule:\n - cron: \"0 3 * * 1\"\n\njobs:\n"
},
{
"path": ".github/workflows/release-manual.yml",
"chars": 5634,
"preview": "name: Release\n\non:\n workflow_dispatch:\n inputs:\n version:\n description: 'Version to release (e.g., 0.1.2"
},
{
"path": ".gitignore",
"chars": 2097,
"preview": "raw_data/\nscaling_out/\nscaling_out_old/\nsanity_check/\ndemo/indices/\n# .vscode/\n*.log\n*pycache*\noutputs/\n*.pkl\n*.pdf\n*.id"
},
{
"path": ".gitmodules",
"chars": 968,
"preview": "[submodule \"packages/leann-backend-diskann/third_party/DiskANN\"]\n\tpath = packages/leann-backend-diskann/third_party/Disk"
},
{
"path": ".pre-commit-config.yaml",
"chars": 503,
"preview": "repos:\r\n - repo: https://github.com/pre-commit/pre-commit-hooks\r\n rev: v5.0.0\r\n hooks:\r\n - id: trailing-whit"
},
{
"path": ".python-version",
"chars": 5,
"preview": "3.11\n"
},
{
"path": ".vscode/extensions.json",
"chars": 65,
"preview": "{\n \"recommendations\": [\n \"charliermarsh.ruff\",\n ]\n}\n"
},
{
"path": ".vscode/settings.json",
"chars": 578,
"preview": "{\n \"python.defaultInterpreterPath\": \".venv/bin/python\",\n \"python.terminal.activateEnvironment\": true,\n \"[python]\": {\n"
},
{
"path": "CLAUDE.md",
"chars": 10012,
"preview": "# CLAUDE.md\n\nThis file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.\n\n## "
},
{
"path": "LICENSE",
"chars": 1075,
"preview": "MIT License\n\nCopyright (c) 2025 LEANN Contributors\n\nPermission is hereby granted, free of charge, to any person obtainin"
},
{
"path": "README.md",
"chars": 51555,
"preview": "<p align=\"center\">\n <img src=\"assets/logo-text.png\" alt=\"LEANN Logo\" width=\"400\">\n</p>\n\n<p align=\"center\">\n <a href=\"h"
},
{
"path": "apps/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "apps/base_rag_example.py",
"chars": 15696,
"preview": "\"\"\"\nBase class for unified RAG examples interface.\nProvides common parameters and functionality for all RAG examples.\n\"\""
},
{
"path": "apps/browser_rag.py",
"chars": 6002,
"preview": "\"\"\"\nBrowser History RAG example using the unified interface.\nSupports Chrome browser history.\n\"\"\"\n\nimport os\nimport sys\n"
},
{
"path": "apps/chatgpt_data/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "apps/chatgpt_data/chatgpt_reader.py",
"chars": 15153,
"preview": "\"\"\"\nChatGPT export data reader.\n\nReads and processes ChatGPT export data from chat.html files.\n\"\"\"\n\nimport re\nfrom pathl"
},
{
"path": "apps/chatgpt_rag.py",
"chars": 7131,
"preview": "\"\"\"\nChatGPT RAG example using the unified interface.\nSupports ChatGPT export data from chat.html files.\n\"\"\"\n\nimport sys\n"
},
{
"path": "apps/chunking/__init__.py",
"chars": 1453,
"preview": "\"\"\"Unified chunking utilities facade.\n\nThis module re-exports the packaged utilities from `leann.chunking_utils` so\nthat"
},
{
"path": "apps/claude_data/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "apps/claude_data/claude_reader.py",
"chars": 15293,
"preview": "\"\"\"\nClaude export data reader.\n\nReads and processes Claude conversation data from exported JSON files.\n\"\"\"\n\nimport json\n"
},
{
"path": "apps/claude_rag.py",
"chars": 7225,
"preview": "\"\"\"\nClaude RAG example using the unified interface.\nSupports Claude export data from JSON files.\n\"\"\"\n\nimport sys\nfrom pa"
},
{
"path": "apps/code_rag.py",
"chars": 7328,
"preview": "\"\"\"\nCode RAG example using AST-aware chunking for optimal code understanding.\nSpecialized for code repositories with aut"
},
{
"path": "apps/colqwen_rag.py",
"chars": 16357,
"preview": "#!/usr/bin/env python3\n\"\"\"\nColQwen RAG - Easy-to-use multimodal PDF retrieval with ColQwen2/ColPali\n\nUsage:\n python -"
},
{
"path": "apps/document_rag.py",
"chars": 4613,
"preview": "\"\"\"\nDocument RAG example using the unified interface.\nSupports PDF, TXT, MD, and other document formats.\n\"\"\"\n\nimport sys"
},
{
"path": "apps/email_data/LEANN_email_reader.py",
"chars": 7340,
"preview": "import email\nimport os\nfrom pathlib import Path\nfrom typing import Any\n\nfrom llama_index.core import Document\nfrom llama"
},
{
"path": "apps/email_data/email.py",
"chars": 6946,
"preview": "\"\"\"\nMbox parser.\n\nContains simple parser for mbox files.\n\n\"\"\"\n\nimport logging\nfrom pathlib import Path\nfrom typing impor"
},
{
"path": "apps/email_rag.py",
"chars": 5425,
"preview": "\"\"\"\nEmail RAG example using the unified interface.\nSupports Apple Mail on macOS.\n\"\"\"\n\nimport sys\nfrom pathlib import Pat"
},
{
"path": "apps/gemini_data/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "apps/gemini_data/gemini_reader.py",
"chars": 5425,
"preview": "import json\nimport logging\nfrom pathlib import Path\nfrom typing import Any\n\nlogger = logging.getLogger(__name__)\n\n\nclass"
},
{
"path": "apps/gemini_rag.py",
"chars": 1973,
"preview": "\"\"\"\nGemini CLI RAG example.\nIndexes and searches Gemini CLI history (~/.gemini).\n\"\"\"\n\nimport sys\nfrom pathlib import Pat"
},
{
"path": "apps/history_data/__init__.py",
"chars": 76,
"preview": "from .history import ChromeHistoryReader\n\n__all__ = [\"ChromeHistoryReader\"]\n"
},
{
"path": "apps/history_data/history.py",
"chars": 6474,
"preview": "import os\nimport sqlite3\nfrom pathlib import Path\nfrom typing import Any\n\nfrom llama_index.core import Document\nfrom lla"
},
{
"path": "apps/history_data/wechat_history.py",
"chars": 30500,
"preview": "import json\nimport os\nimport re\nimport subprocess\nimport time\nfrom datetime import datetime\nfrom pathlib import Path\nfro"
},
{
"path": "apps/image_rag.py",
"chars": 7972,
"preview": "#!/usr/bin/env python3\n\"\"\"\nCLIP Image RAG Application\n\nThis application enables RAG (Retrieval-Augmented Generation) on "
},
{
"path": "apps/imessage_data/__init__.py",
"chars": 39,
"preview": "\"\"\"iMessage data processing module.\"\"\"\n"
},
{
"path": "apps/imessage_data/imessage_reader.py",
"chars": 11096,
"preview": "\"\"\"\niMessage data reader.\n\nReads and processes iMessage conversation data from the macOS Messages database.\n\"\"\"\n\nimport "
},
{
"path": "apps/imessage_rag.py",
"chars": 4251,
"preview": "\"\"\"\niMessage RAG Example.\n\nThis example demonstrates how to build a RAG system on your iMessage conversation history.\n\"\""
},
{
"path": "apps/multimodal/vision-based-pdf-multi-vector/README.md",
"chars": 3964,
"preview": "## Vision-based PDF Multi-Vector Demos (macOS/MPS)\n\nThis folder contains two demos to index PDF pages as images and run "
},
{
"path": "apps/multimodal/vision-based-pdf-multi-vector/colqwen_forward.py",
"chars": 4133,
"preview": "#!/usr/bin/env python3\n\"\"\"Simple test script to test colqwen2 forward pass with a single image.\"\"\"\n\nimport os\nimport sys"
},
{
"path": "apps/multimodal/vision-based-pdf-multi-vector/leann_multi_vector.py",
"chars": 52825,
"preview": "import concurrent.futures\nimport glob\nimport json\nimport logging\nimport os\nimport re\nimport sys\nimport time\nfrom pathlib"
},
{
"path": "apps/multimodal/vision-based-pdf-multi-vector/multi-vector-leann-paper-example.py",
"chars": 3596,
"preview": "# pip install pdf2image\n# pip install pymilvus\n# pip install colpali_engine\n# pip install tqdm\n# pip install pillow\n\nimp"
},
{
"path": "apps/multimodal/vision-based-pdf-multi-vector/multi-vector-leann-similarity-map.py",
"chars": 30339,
"preview": "## Jupyter-style notebook script\n# %%\n# uv pip install matplotlib qwen_vl_utils\nimport argparse\nimport faulthandler\nimpo"
},
{
"path": "apps/multimodal/vision-based-pdf-multi-vector/vidore_v1_benchmark.py",
"chars": 15756,
"preview": "#!/usr/bin/env python3\n\"\"\"\nModular script to reproduce NDCG results for ViDoRe v1 benchmark.\n\nThis script uses the inter"
},
{
"path": "apps/multimodal/vision-based-pdf-multi-vector/vidore_v2_benchmark.py",
"chars": 15113,
"preview": "#!/usr/bin/env python3\n\"\"\"\nModular script to reproduce NDCG results for ViDoRe v2 benchmark.\n\nThis script uses the inter"
},
{
"path": "apps/qwen_data/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "apps/qwen_data/qwen_reader.py",
"chars": 5573,
"preview": "import json\nimport logging\nfrom pathlib import Path\nfrom typing import Any\n\nlogger = logging.getLogger(__name__)\n\n\nclass"
},
{
"path": "apps/qwen_rag.py",
"chars": 1966,
"preview": "\"\"\"\nQwen Code RAG example.\nIndexes and searches Qwen Code CLI history (~/.qwen-code).\n\"\"\"\n\nimport sys\nfrom pathlib impor"
},
{
"path": "apps/semantic_file_search/leann-plus-temporal-search.py",
"chars": 5975,
"preview": "#!/usr/bin/env python3\nimport re\nimport sys\nfrom datetime import datetime, timedelta\nfrom pathlib import Path\n\nfrom lean"
},
{
"path": "apps/semantic_file_search/leann_index_builder.py",
"chars": 2622,
"preview": "#!/usr/bin/env python3\nimport json\nimport sys\nfrom pathlib import Path\n\nfrom leann import LeannBuilder\n\n\ndef process_jso"
},
{
"path": "apps/semantic_file_search/spotlight_index_dump.py",
"chars": 8409,
"preview": "#!/usr/bin/env python3\n\"\"\"\nSpotlight Metadata Dumper for Vector DB\nExtracts only essential metadata for semantic search "
},
{
"path": "apps/slack_data/__init__.py",
"chars": 39,
"preview": "# Slack MCP data integration for LEANN\n"
},
{
"path": "apps/slack_data/slack_mcp_reader.py",
"chars": 21400,
"preview": "#!/usr/bin/env python3\n\"\"\"\nSlack MCP Reader for LEANN\n\nThis module provides functionality to connect to Slack MCP server"
},
{
"path": "apps/slack_rag.py",
"chars": 8236,
"preview": "#!/usr/bin/env python3\n\"\"\"\nSlack RAG Application with MCP Support\n\nThis application enables RAG (Retrieval-Augmented Gen"
},
{
"path": "apps/twitter_data/__init__.py",
"chars": 41,
"preview": "# Twitter MCP data integration for LEANN\n"
},
{
"path": "apps/twitter_data/twitter_mcp_reader.py",
"chars": 10712,
"preview": "#!/usr/bin/env python3\n\"\"\"\nTwitter MCP Reader for LEANN\n\nThis module provides functionality to connect to Twitter MCP se"
},
{
"path": "apps/twitter_rag.py",
"chars": 7263,
"preview": "#!/usr/bin/env python3\n\"\"\"\nTwitter RAG Application with MCP Support\n\nThis application enables RAG (Retrieval-Augmented G"
},
{
"path": "apps/wechat_rag.py",
"chars": 7182,
"preview": "\"\"\"\nWeChat History RAG example using the unified interface.\nSupports WeChat chat history export and search.\n\"\"\"\n\nimport "
},
{
"path": "benchmarks/README.md",
"chars": 3876,
"preview": "# 🧪 LEANN Benchmarks & Testing\n\nThis directory contains performance benchmarks and comprehensive tests for the LEANN sys"
},
{
"path": "benchmarks/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "benchmarks/benchmark_embeddings.py",
"chars": 4399,
"preview": "import time\n\nimport matplotlib.pyplot as plt\nimport mlx.core as mx\nimport numpy as np\nimport torch\nfrom mlx_lm import lo"
},
{
"path": "benchmarks/benchmark_no_recompute.py",
"chars": 4758,
"preview": "import argparse\nimport os\nimport time\nfrom pathlib import Path\n\nfrom leann import LeannBuilder, LeannSearcher\n\n\ndef _met"
},
{
"path": "benchmarks/bm25_diskann_baselines/README.md",
"chars": 1280,
"preview": "BM25 vs DiskANN Baselines\n\n```bash\naws s3 sync s3://powerrag-diskann-rpj-wiki-20250824-224037-194d640c/bm25_rpj_wiki/ind"
},
{
"path": "benchmarks/bm25_diskann_baselines/run_bm25.py",
"chars": 5960,
"preview": "# /// script\n# dependencies = [\n# \"pyserini\"\n# ]\n# ///\n# sudo pacman -S jdk21-openjdk\n# export JAVA_HOME=/usr/lib/jvm/"
},
{
"path": "benchmarks/bm25_diskann_baselines/run_diskann.py",
"chars": 3819,
"preview": "# /// script\n# dependencies = [\n# \"leann-backend-diskann\"\n# ]\n# ///\n\nimport argparse\nimport json\nimport time\nfrom path"
},
{
"path": "benchmarks/compare_faiss_vs_leann.py",
"chars": 10234,
"preview": "#!/usr/bin/env python3\n\"\"\"\nMemory comparison between Faiss HNSW and LEANN HNSW backend\n\"\"\"\n\nimport gc\nimport logging\nimp"
},
{
"path": "benchmarks/diskann_vs_hnsw_speed_comparison.py",
"chars": 9851,
"preview": "#!/usr/bin/env python3\n\"\"\"\nDiskANN vs HNSW Search Performance Comparison\n\nThis benchmark compares search performance bet"
},
{
"path": "benchmarks/enron_emails/README.md",
"chars": 6787,
"preview": "# Enron Emails Benchmark\n\nA comprehensive RAG benchmark for evaluating LEANN search and generation on the Enron email co"
},
{
"path": "benchmarks/enron_emails/data/.gitignore",
"chars": 11,
"preview": "downloads/\n"
},
{
"path": "benchmarks/enron_emails/evaluate_enron_emails.py",
"chars": 24525,
"preview": "\"\"\"\nEnron Emails Benchmark Evaluation - Retrieval Recall@3 (Stages 2/3/4)\nFollows the style of FinanceBench/LAION: Stage"
},
{
"path": "benchmarks/enron_emails/setup_enron_emails.py",
"chars": 13264,
"preview": "\"\"\"\nEnron Emails Benchmark Setup Script\nPrepares passages from emails.csv, builds LEANN index, and FAISS Flat baseline\n\""
},
{
"path": "benchmarks/faiss_only.py",
"chars": 5020,
"preview": "#!/usr/bin/env python3\n\"\"\"Test only Faiss HNSW\"\"\"\n\nimport os\nimport sys\nimport time\n\nimport psutil\n\n\ndef get_memory_usag"
},
{
"path": "benchmarks/financebench/README.md",
"chars": 3970,
"preview": "# FinanceBench Benchmark for LEANN-RAG\n\nFinanceBench is a benchmark for evaluating retrieval-augmented generation (RAG) "
},
{
"path": "benchmarks/financebench/evaluate_financebench.py",
"chars": 38793,
"preview": "\"\"\"\nFinanceBench Evaluation Script - Modular Recall-based Evaluation\n\"\"\"\n\nimport argparse\nimport json\nimport logging\nimp"
},
{
"path": "benchmarks/financebench/setup_financebench.py",
"chars": 17025,
"preview": "#!/usr/bin/env python3\n\"\"\"\nFinanceBench Complete Setup Script\nDownloads all PDFs and builds full LEANN datastore\n\"\"\"\n\nim"
},
{
"path": "benchmarks/financebench/verify_recall.py",
"chars": 6856,
"preview": "#!/usr/bin/env python3\n# /// script\n# requires-python = \">=3.9\"\n# dependencies = [\n# \"faiss-cpu\",\n# \"numpy\",\n# "
},
{
"path": "benchmarks/issue_159.py",
"chars": 3044,
"preview": "#!/usr/bin/env python3\n\"\"\"\nTest script to reproduce issue #159: Slow search performance\nConfiguration:\n- GPU: A10\n- embe"
},
{
"path": "benchmarks/laion/.gitignore",
"chars": 6,
"preview": "data/\n"
},
{
"path": "benchmarks/laion/README.md",
"chars": 5989,
"preview": "# LAION Multimodal Benchmark\n\nA multimodal benchmark for evaluating image retrieval and generation performance using LEA"
},
{
"path": "benchmarks/laion/evaluate_laion.py",
"chars": 31745,
"preview": "\"\"\"\nLAION Multimodal Benchmark Evaluation Script - Modular Recall-based Evaluation\n\"\"\"\n\nimport argparse\nimport json\nimpo"
},
{
"path": "benchmarks/laion/setup_laion.py",
"chars": 23574,
"preview": "\"\"\"\nLAION Multimodal Benchmark Setup Script\nDownloads LAION subset and builds LEANN index with sentence embeddings\n\"\"\"\n\n"
},
{
"path": "benchmarks/llm_utils.py",
"chars": 11107,
"preview": "\"\"\"\nLLM utils for RAG benchmarks with Qwen3-8B and Qwen2.5-VL (multimodal)\n\"\"\"\n\nimport time\n\ntry:\n import torch\n f"
},
{
"path": "benchmarks/micro_tpt.py",
"chars": 23647,
"preview": "# python embedd_micro.py --use_int8 Fastest\n\nimport argparse\nimport time\nfrom contextlib import contextmanager\nfrom data"
},
{
"path": "benchmarks/run_evaluation.py",
"chars": 14793,
"preview": "#!/usr/bin/env python3\n\"\"\"\nThis script runs a recall evaluation on a given LEANN index.\nIt correctly compares results by"
},
{
"path": "benchmarks/simple_mac_tpt_test.py",
"chars": 10553,
"preview": "import time\nfrom dataclasses import dataclass\n\nimport numpy as np\nimport torch\nfrom torch import nn\nfrom tqdm import tqd"
},
{
"path": "benchmarks/update/README.md",
"chars": 6109,
"preview": "# Update Benchmarks\n\nThis directory hosts two benchmark suites that exercise LEANN’s HNSW “update +\nsearch” pipeline und"
},
{
"path": "benchmarks/update/__init__.py",
"chars": 448,
"preview": "\"\"\"Benchmarks for LEANN update workflows.\"\"\"\n\n# Expose helper to locate repository root for other modules that need it.\n"
},
{
"path": "benchmarks/update/bench_hnsw_rng_recompute.py",
"chars": 29229,
"preview": "\"\"\"Benchmark incremental HNSW add() under different RNG pruning modes with real\nembedding recomputation.\n\nThis script cl"
},
{
"path": "benchmarks/update/bench_update_vs_offline_search.py",
"chars": 25466,
"preview": "\"\"\"\nCompare two latency models for small incremental updates vs. search:\n\nScenario A (sequential update then search):\n "
},
{
"path": "benchmarks/update/plot_bench_results.py",
"chars": 23910,
"preview": "#!/usr/bin/env python3\n\"\"\"\nPlot latency bars from the benchmark CSV produced by\nbenchmarks/update/bench_hnsw_rng_recompu"
},
{
"path": "data/PrideandPrejudice.txt",
"chars": 763003,
"preview": "The Project Gutenberg eBook of Pride and Prejudice\r\n\r\nThis ebook is for the use of anyone anywhere in the United States"
},
{
"path": "data/huawei_pangu.md",
"chars": 6740,
"preview": "# 盘古之殇：华为诺亚盘古大模型研发历程的心酸与黑暗\n\n各位好，\n\n我是一名盘古大模型团队，华为诺亚方舟实验室的员工。\n\n首先为自证身份，列举一些细节：\n\n1. 现诺亚主任，前算法应用部部长，后改名为小模型实验室的主任王云鹤。前诺亚主任：姚"
},
{
"path": "demo.ipynb",
"chars": 3500,
"preview": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"# Quick Start \\n\",\n \"\\n\",\n \"*"
},
{
"path": "docker/Dockerfile",
"chars": 705,
"preview": "ARG PYTHON_VERSION=3.11\nFROM python:${PYTHON_VERSION}-slim\n\nARG LEANN_VERSION=0.3.6\n\nENV PIP_NO_CACHE_DIR=1 \\\n PIP_DI"
},
{
"path": "docker/Dockerfile.cpu",
"chars": 824,
"preview": "ARG PYTHON_VERSION=3.11\nFROM python:${PYTHON_VERSION}-slim\n\nARG LEANN_VERSION=0.3.6\n\nENV PIP_NO_CACHE_DIR=1 \\\n PIP_DI"
},
{
"path": "docker/Dockerfile.dev",
"chars": 930,
"preview": "ARG PYTHON_VERSION=3.11\nFROM python:${PYTHON_VERSION}-slim\n\nENV DEBIAN_FRONTEND=noninteractive \\\n PIP_NO_CACHE_DIR=1 "
},
{
"path": "docker/README.md",
"chars": 998,
"preview": "# Docker Setup\n\nThis folder provides reference Docker images for LEANN.\n\n## Build (default)\n\nDefault image (no forced CP"
},
{
"path": "docs/COLQWEN_GUIDE.md",
"chars": 5486,
"preview": "# ColQwen Integration Guide\n\nEasy-to-use multimodal PDF retrieval with ColQwen2/ColPali models.\n\n## Quick Start\n\n> **🍎 M"
},
{
"path": "docs/CONTRIBUTING.md",
"chars": 5154,
"preview": "# 🤝 Contributing\n\nWe welcome contributions! Leann is built by the community, for the community.\n\n## Ways to Contribute\n\n"
},
{
"path": "docs/RELEASE.md",
"chars": 609,
"preview": "# Release Guide\n\n## Setup (One-time)\n\nAdd `PYPI_API_TOKEN` to GitHub Secrets:\n1. Get token: https://pypi.org/manage/acco"
},
{
"path": "docs/THINKING_BUDGET_FEATURE.md",
"chars": 4456,
"preview": "# Thinking Budget Feature Implementation\n\n## Overview\n\nThis document describes the implementation of the **thinking budg"
},
{
"path": "docs/ast_chunking_guide.md",
"chars": 3960,
"preview": "# AST-Aware Code chunking guide\n\n## Overview\n\nThis guide covers best practices for using AST-aware code chunking in LEAN"
},
{
"path": "docs/code/embedding_model_compare.py",
"chars": 3709,
"preview": "\"\"\"\nComparison between Sentence Transformers and OpenAI embeddings\n\nThis example shows how different embedding models ha"
},
{
"path": "docs/configuration-guide.md",
"chars": 22425,
"preview": "# LEANN Configuration Guide\n\nThis guide helps you optimize LEANN for different use cases and understand the trade-offs b"
},
{
"path": "docs/faq.md",
"chars": 2284,
"preview": "# FAQ\n\n## 1. My building time seems long\n\nYou can speed up the process by using a lightweight embedding model. Add this "
},
{
"path": "docs/features.md",
"chars": 1675,
"preview": "# ✨ Detailed Features\n\n## 🔥 Core Features\n\n- **🔄 Real-time Embeddings** - Eliminate heavy embedding storage with dynamic"
},
{
"path": "docs/grep_search.md",
"chars": 3878,
"preview": "# LEANN Grep Search Usage Guide\n\n## Overview\n\nLEANN's grep search functionality provides exact text matching for finding"
},
{
"path": "docs/metadata_filtering.md",
"chars": 8040,
"preview": "# LEANN Metadata Filtering Usage Guide\n\n## Overview\n\nLeann possesses metadata filtering capabilities that allow you to f"
},
{
"path": "docs/normalized_embeddings.md",
"chars": 2544,
"preview": "# Normalized Embeddings Support in LEANN\n\nLEANN now automatically detects normalized embedding models and sets the appro"
},
{
"path": "docs/openclaw-setup.md",
"chars": 4237,
"preview": "# OpenClaw + LEANN Setup Guide\n\nTwo ways to connect LEANN to your OpenClaw agent: **MCP server** (recommended)\nor **Claw"
},
{
"path": "docs/react_agent.md",
"chars": 7427,
"preview": "# LEANN ReAct Agent Guide\n\n## Overview\n\nThe LEANN ReAct (Reasoning + Acting) Agent enables **multiturn retrieval and rea"
},
{
"path": "docs/roadmap.md",
"chars": 2805,
"preview": "# LEANN Roadmap\n\nLEANN aims to be a **personal knowledge layer** — not just a storage-efficient vector database, but a u"
},
{
"path": "docs/slack-setup-guide.md",
"chars": 12344,
"preview": "# Slack Integration Setup Guide\n\nThis guide provides step-by-step instructions for setting up Slack integration with LEA"
},
{
"path": "docs/ultimate_goal.md",
"chars": 3309,
"preview": "# LEANN — Long-Term Vision\n\n## The Best Personal Data Management Platform\n\nLEANN's ultimate goal is to be the **unified "
},
{
"path": "examples/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "examples/basic_demo.py",
"chars": 3062,
"preview": "\"\"\"\nSimple demo showing basic leann usage\nRun: uv run python examples/basic_demo.py\n\"\"\"\n\nimport argparse\n\nfrom leann imp"
},
{
"path": "examples/dynamic_update_no_recompute.py",
"chars": 13876,
"preview": "\"\"\"Dynamic HNSW update demo without compact storage.\n\nThis script reproduces the minimal scenario we used while debuggin"
},
{
"path": "examples/grep_search_example.py",
"chars": 1107,
"preview": "\"\"\"\nGrep Search Example\n\nShows how to use grep-based text search instead of semantic search.\nUseful when you need exact "
},
{
"path": "examples/mcp_integration_demo.py",
"chars": 6228,
"preview": "#!/usr/bin/env python3\n\"\"\"\nMCP Integration Examples for LEANN\n\nThis script demonstrates how to use LEANN with different "
},
{
"path": "examples/mlx_demo.py",
"chars": 1546,
"preview": "import os\n\nfrom leann.api import LeannBuilder, LeannChat\n\n# Define the path for our new MLX-based index\nINDEX_PATH = \"./"
},
{
"path": "examples/spoiler_free_book_rag.py",
"chars": 8599,
"preview": "#!/usr/bin/env python3\n\"\"\"\nSpoiler-Free Book RAG Example using LEANN Metadata Filtering\n\nThis example demonstrates how t"
},
{
"path": "llms.txt",
"chars": 990,
"preview": "# llms.txt — LEANN MCP and Agent Integration\nproduct: LEANN\nhomepage: https://github.com/yichuan-w/LEANN\ncontact: https:"
},
{
"path": "packages/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "packages/leann/README.md",
"chars": 1456,
"preview": "# LEANN - The smallest vector index in the world\n\nLEANN is a revolutionary vector database that democratizes personal AI"
},
{
"path": "packages/leann/__init__.py",
"chars": 317,
"preview": "\"\"\"\nLEANN - Low-storage Embedding Approximation for Neural Networks\n\nA revolutionary vector database that democratizes p"
},
{
"path": "packages/leann/pyproject.toml",
"chars": 1127,
"preview": "[build-system]\nrequires = [\"setuptools>=61.0\"]\nbuild-backend = \"setuptools.build_meta\"\n\n[project]\nname = \"leann\"\nversion"
},
{
"path": "packages/leann-backend-diskann/__init__.py",
"chars": 49,
"preview": "# This file makes the directory a Python package\n"
},
{
"path": "packages/leann-backend-diskann/leann_backend_diskann/__init__.py",
"chars": 2285,
"preview": "import os\nfrom pathlib import Path\n\n_DLL_DIR_HANDLES: list[object] = []\n\n\ndef _configure_windows_dll_search_path() -> No"
},
{
"path": "packages/leann-backend-diskann/leann_backend_diskann/diskann_backend.py",
"chars": 19528,
"preview": "import contextlib\nimport logging\nimport os\nimport struct\nimport sys\nfrom pathlib import Path\nfrom typing import Any, Lit"
},
{
"path": "packages/leann-backend-diskann/leann_backend_diskann/diskann_embedding_server.py",
"chars": 21003,
"preview": "\"\"\"\nDiskANN-specific embedding server\n\"\"\"\n\nimport argparse\nimport json\nimport logging\nimport os\nimport sys\nimport thread"
},
{
"path": "packages/leann-backend-diskann/leann_backend_diskann/embedding_pb2.py",
"chars": 1255,
"preview": "# Generated by the protocol buffer compiler. DO NOT EDIT!\n# source: embedding.proto\n# ruff: noqa\n\"\"\"Generated protocol "
},
{
"path": "packages/leann-backend-diskann/leann_backend_diskann/graph_partition.py",
"chars": 11185,
"preview": "#!/usr/bin/env python3\n\"\"\"\nGraph Partition Module for LEANN DiskANN Backend\n\nThis module provides Python bindings for th"
},
{
"path": "packages/leann-backend-diskann/pyproject.toml",
"chars": 714,
"preview": "[build-system]\nrequires = [\"scikit-build-core>=0.10\", \"pybind11>=2.12.0\", \"numpy\", \"cmake>=3.30\"]\nbuild-backend = \"sciki"
},
{
"path": "packages/leann-backend-diskann/third_party/embedding.pb.cc",
"chars": 25279,
"preview": "// Generated by the protocol buffer compiler. DO NOT EDIT!\n// source: embedding.proto\n\n#include \"embedding.pb.h\"\n\n#incl"
},
{
"path": "packages/leann-backend-diskann/third_party/embedding.proto",
"chars": 339,
"preview": "syntax = \"proto3\";\n\npackage protoembedding;\n\nmessage NodeEmbeddingRequest {\n repeated uint32 node_ids = 1;\n}\n\nmessage N"
},
{
"path": "packages/leann-backend-hnsw/CMakeLists.txt",
"chars": 4335,
"preview": "cmake_minimum_required(VERSION 3.24)\nproject(leann_backend_hnsw_wrapper)\nset(CMAKE_C_COMPILER_WORKS 1)\nset(CMAKE_CXX_COM"
},
{
"path": "packages/leann-backend-hnsw/leann_backend_hnsw/__init__.py",
"chars": 2032,
"preview": "import os\nfrom pathlib import Path\n\n_DLL_DIR_HANDLES: list[object] = []\n\n\ndef _configure_windows_dll_search_path() -> No"
},
{
"path": "packages/leann-backend-hnsw/leann_backend_hnsw/convert_to_csr.py",
"chars": 44066,
"preview": "import argparse\nimport gc # Import garbage collector interface\nimport logging\nimport os\nimport struct\nimport sys\nimport"
},
{
"path": "packages/leann-backend-hnsw/leann_backend_hnsw/hnsw_backend.py",
"chars": 11565,
"preview": "import logging\nimport os\nimport shutil\nimport time\nfrom pathlib import Path\nfrom typing import Any, Literal, Optional\n\ni"
},
{
"path": "packages/leann-backend-hnsw/leann_backend_hnsw/hnsw_embedding_server.py",
"chars": 20970,
"preview": "\"\"\"\nHNSW-specific embedding server\n\"\"\"\n\nimport argparse\nimport json\nimport logging\nimport os\nimport sys\nimport threading"
},
{
"path": "packages/leann-backend-hnsw/pyproject.toml",
"chars": 716,
"preview": "# packages/leann-backend-hnsw/pyproject.toml\n\n[build-system]\nrequires = [\"scikit-build-core>=0.10\", \"numpy\", \"swig\"]\nbui"
},
{
"path": "packages/leann-backend-ivf/README.md",
"chars": 2204,
"preview": "# LEANN IVF Backend\n\nFAISS **IndexIVFFlat** backend for LEANN with **add** and **delete** APIs for incremental updates ("
},
{
"path": "packages/leann-backend-ivf/leann_backend_ivf/__init__.py",
"chars": 309,
"preview": "\"\"\"LEANN IVF backend: FAISS IndexIVFFlat with add/delete APIs for incremental updates.\"\"\"\n\nfrom .ivf_backend import (\n "
},
{
"path": "packages/leann-backend-ivf/leann_backend_ivf/ivf_backend.py",
"chars": 10080,
"preview": "\"\"\"\nIVF backend: FAISS IndexIVFFlat with DirectMap.Hashtable for add/remove by passage id.\n\nUses IndexIVFFlat + DirectMa"
},
{
"path": "packages/leann-backend-ivf/pyproject.toml",
"chars": 561,
"preview": "[project]\nname = \"leann-backend-ivf\"\nversion = \"0.3.6\"\ndescription = \"FAISS IVF backend for LEANN with add/delete suppor"
},
{
"path": "packages/leann-core/pyproject.toml",
"chars": 2028,
"preview": "[build-system]\nrequires = [\"setuptools>=61.0\"]\nbuild-backend = \"setuptools.build_meta\"\n\n[project]\nname = \"leann-core\"\nve"
},
{
"path": "packages/leann-core/src/leann/__init__.py",
"chars": 1393,
"preview": "# packages/leann-core/src/leann/__init__.py\nimport os\nimport platform\n\n# ruff: noqa: E402 (env vars must be set before "
},
{
"path": "packages/leann-core/src/leann/api.py",
"chars": 67277,
"preview": "\"\"\"\nThis file contains the core API for the LEANN project, now definitively updated\nwith the correct, original embedding"
},
{
"path": "packages/leann-core/src/leann/chat.py",
"chars": 41868,
"preview": "#!/usr/bin/env python3\n\"\"\"\nThis file contains the chat generation logic for the LEANN project,\nsupporting different back"
},
{
"path": "packages/leann-core/src/leann/chunking_utils.py",
"chars": 16360,
"preview": "\"\"\"\nEnhanced chunking utilities with AST-aware code chunking support.\nPackaged within leann-core so installed wheels can"
},
{
"path": "packages/leann-core/src/leann/cli.py",
"chars": 117741,
"preview": "import argparse\nimport asyncio\nimport contextlib\nimport hashlib\nimport io\nimport json\nimport os\nimport pickle\nimport sys"
},
{
"path": "packages/leann-core/src/leann/embedding_compute.py",
"chars": 51182,
"preview": "\"\"\"\nUnified embedding computation module\nConsolidates all embedding computation logic using SentenceTransformer\nPreserve"
},
{
"path": "packages/leann-core/src/leann/embedding_server_manager.py",
"chars": 33691,
"preview": "import atexit\nimport contextlib\nimport hashlib\nimport json\nimport logging\nimport os\nimport socket\nimport subprocess\nimpo"
},
{
"path": "packages/leann-core/src/leann/interactive_utils.py",
"chars": 6300,
"preview": "\"\"\"\nInteractive session utilities for LEANN applications.\n\nProvides shared readline functionality and command handling a"
},
{
"path": "packages/leann-core/src/leann/interface.py",
"chars": 3458,
"preview": "from abc import ABC, abstractmethod\nfrom typing import Any, Literal, Optional\n\nimport numpy as np\n\n\nclass LeannBackendBu"
},
{
"path": "packages/leann-core/src/leann/mcp.py",
"chars": 6486,
"preview": "#!/usr/bin/env python3\n\nimport json\nimport subprocess\nimport sys\n\n\ndef handle_request(request):\n if request.get(\"meth"
},
{
"path": "packages/leann-core/src/leann/metadata_filter.py",
"chars": 9526,
"preview": "\"\"\"\nMetadata filtering engine for LEANN search results.\n\nThis module provides generic metadata filtering capabilities th"
},
{
"path": "packages/leann-core/src/leann/react_agent.py",
"chars": 9988,
"preview": "\"\"\"\nSimple ReAct agent for multiturn retrieval with LEANN.\n\nThis implements a basic ReAct (Reasoning + Acting) agent pat"
},
{
"path": "packages/leann-core/src/leann/registry.py",
"chars": 3330,
"preview": "# packages/leann-core/src/leann/registry.py\n\nimport importlib\nimport importlib.metadata\nimport json\nimport logging\nfrom "
},
{
"path": "packages/leann-core/src/leann/searcher_base.py",
"chars": 9537,
"preview": "import json\nfrom abc import ABC, abstractmethod\nfrom pathlib import Path\nfrom typing import Any, Literal, Optional\n\nimpo"
},
{
"path": "packages/leann-core/src/leann/server.py",
"chars": 5958,
"preview": "\"\"\"\nMinimal HTTP API server for LEANN.\n\nThis exposes LEANN indexes over HTTP so clients can:\n- List available indexes\n- "
},
{
"path": "packages/leann-core/src/leann/settings.py",
"chars": 3339,
"preview": "\"\"\"Runtime configuration helpers for LEANN.\"\"\"\n\nfrom __future__ import annotations\n\nimport json\nimport os\nfrom typing im"
},
{
"path": "packages/leann-core/src/leann/sync.py",
"chars": 6089,
"preview": "import logging\nimport os\nimport pickle\nfrom dataclasses import dataclass, field\nfrom hashlib import sha256\nfrom typing i"
},
{
"path": "packages/leann-mcp/README.md",
"chars": 6070,
"preview": "# 🔥 LEANN Claude Code Integration\n\nTransform your development workflow with intelligent code assistance using LEANN's se"
},
{
"path": "packages/wechat-exporter/__init__.py",
"chars": 13,
"preview": "__all__ = []\n"
},
{
"path": "packages/wechat-exporter/main.py",
"chars": 4677,
"preview": "import json\nimport sqlite3\nimport xml.etree.ElementTree as ElementTree\nfrom pathlib import Path\nfrom typing import Annot"
},
{
"path": "pyproject.toml",
"chars": 5629,
"preview": "[build-system]\nrequires = [\"setuptools>=61.0\", \"cmake>=3.24\"]\nbuild-backend = \"setuptools.build_meta\"\n\n[project]\nname = "
},
{
"path": "scripts/hf_upload.py",
"chars": 3466,
"preview": "#!/usr/bin/env python3\n\"\"\"\nUpload local evaluation data to Hugging Face Hub, excluding diskann_rpj_wiki.\n\nDefaults:\n- re"
},
{
"path": "skills/leann-memory/README.md",
"chars": 1815,
"preview": "# LEANN Memory Search for OpenClaw\n\n97% storage-compressed semantic memory search with free local embeddings.\n\n## Why\n\nE"
},
{
"path": "skills/leann-memory/claw.json",
"chars": 602,
"preview": "{\n \"name\": \"leann-memory\",\n \"version\": \"1.0.0\",\n \"description\": \"97% storage-compressed semantic memory search with f"
},
{
"path": "skills/leann-memory/instructions.md",
"chars": 2480,
"preview": "# LEANN Memory Search\n\nYou have access to LEANN, a high-performance semantic search engine with 97%\nstorage compression."
},
{
"path": "sky/leann-build.yaml",
"chars": 2411,
"preview": "name: leann-build\n\nresources:\n # Choose a GPU for fast embeddings (examples: L4, A10G, A100). CPU also works but is slo"
},
{
"path": "tests/README.md",
"chars": 4725,
"preview": "# LEANN Tests\n\nThis directory contains automated tests for the LEANN project using pytest.\n\n## Test Files\n\n### `test_rea"
},
{
"path": "tests/openclaw/.gitignore",
"chars": 13,
"preview": "docker-data/\n"
},
{
"path": "tests/openclaw/__init__.py",
"chars": 0,
"preview": ""
},
{
"path": "tests/openclaw/conftest.py",
"chars": 1067,
"preview": "\"\"\"Shared fixtures for OpenClaw integration tests.\"\"\"\n\nimport json\nimport shutil\nfrom pathlib import Path\n\nimport pytest"
},
{
"path": "tests/openclaw/docker-compose.yml",
"chars": 552,
"preview": "services:\n openclaw-test:\n image: ghcr.io/phioranex/openclaw-docker:latest\n container_name: openclaw-leann-test\n "
},
{
"path": "tests/openclaw/fixtures/MEMORY.md",
"chars": 223,
"preview": "# Core Memory\n\n- Name: Alice Chen\n- Role: Senior backend engineer at Acme Corp\n- Tech stack: Python, PostgreSQL, Docker,"
},
{
"path": "tests/openclaw/fixtures/memory/2026-02-15.md",
"chars": 364,
"preview": "# 2026-02-15 Saturday\n\n- Discussed migrating the user-service from REST to gRPC with Bob.\n- Alice prefers Protocol Buffe"
},
{
"path": "tests/openclaw/fixtures/memory/2026-02-20.md",
"chars": 396,
"preview": "# 2026-02-20 Thursday\n\n- Deployed the v2.1.0 hotfix for the payment gateway timeout issue.\n- Root cause: connection pool"
},
{
"path": "tests/openclaw/fixtures/memory/2026-02-25.md",
"chars": 448,
"preview": "# 2026-02-25 Wednesday\n\n- Met with the ML team about integrating the recommendation engine into the product catalog.\n- T"
},
{
"path": "tests/openclaw/run_docker_test.sh",
"chars": 3321,
"preview": "#!/usr/bin/env bash\n#\n# End-to-end test: spin up an isolated OpenClaw Docker container,\n# install the leann-memory skill"
},
{
"path": "tests/openclaw/test_build_and_search.py",
"chars": 5503,
"preview": "\"\"\"\nEnd-to-end test: build a LEANN index on OpenClaw-style memory files,\nthen search with --json and verify results.\n\nRe"
},
{
"path": "tests/openclaw/test_mcp_e2e.py",
"chars": 5663,
"preview": "\"\"\"\nEnd-to-end test for the MCP server: build a real index, then invoke\nhandle_request(tools/call → leann_search) and ve"
},
{
"path": "tests/openclaw/test_mcp_protocol.py",
"chars": 2687,
"preview": "\"\"\"Test the LEANN MCP server JSON-RPC protocol handling.\"\"\"\n\nimport json\nimport sys\nfrom pathlib import Path\n\nsys.path.i"
},
{
"path": "tests/openclaw/test_openclaw_e2e.py",
"chars": 9125,
"preview": "\"\"\"End-to-end integration test: real OpenClaw instance → memory formation → LEANN indexing → search.\n\nRequirements:\n "
},
{
"path": "tests/openclaw/test_skill_manifest.py",
"chars": 2390,
"preview": "\"\"\"Validate the ClawHub skill manifest and instructions.\"\"\"\n\n\ndef test_claw_json_required_fields(claw_manifest):\n \"\"\""
},
{
"path": "tests/support/fake_embedding_server_module.py",
"chars": 2095,
"preview": "import argparse\nimport signal\nimport socket\nimport time\n\n\ndef main() -> None:\n parser = argparse.ArgumentParser(descr"
},
{
"path": "tests/test_astchunk_integration.py",
"chars": 37265,
"preview": "\"\"\"\nTest suite for astchunk integration with LEANN.\nTests AST-aware chunking functionality, language detection, and fall"
},
{
"path": "tests/test_basic.py",
"chars": 2705,
"preview": "\"\"\"\nBasic functionality tests for CI pipeline using pytest.\n\"\"\"\n\nimport os\nimport tempfile\nfrom pathlib import Path\n\nimp"
},
{
"path": "tests/test_ci_minimal.py",
"chars": 1333,
"preview": "\"\"\"\nMinimal tests for CI that don't require model loading or significant memory.\n\"\"\"\n\nimport subprocess\nimport sys\n\n\ndef"
},
{
"path": "tests/test_cli_ask.py",
"chars": 400,
"preview": "from leann.cli import LeannCLI\n\n\ndef test_cli_ask_accepts_positional_query(tmp_path, monkeypatch):\n monkeypatch.chdir"
},
{
"path": "tests/test_cli_daemon_workflow.py",
"chars": 6295,
"preview": "import argparse\nimport asyncio\nfrom pathlib import Path\nfrom typing import Any\n\nfrom leann.cli import LeannCLI\n\n\ndef tes"
},
{
"path": "tests/test_cli_prompt_template.py",
"chars": 20143,
"preview": "\"\"\"\nTests for CLI argument integration of --embedding-prompt-template.\n\nThese tests verify that:\n1. The --embedding-prom"
}
]
// ... and 21 more files (download for full content)
About this extraction
This page contains the full source code of the yichuan-w/LEANN GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 221 files (2.6 MB), approximately 683.6k tokens, and a symbol index with 1240 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.