[ { "path": ".devcontainer/devcontainer.json", "content": "{\n \"name\": \"LEANN Dev\",\n \"build\": {\n \"context\": \"..\",\n \"dockerfile\": \"../docker/Dockerfile.dev\",\n \"args\": {\n \"PYTHON_VERSION\": \"3.12\"\n }\n },\n \"workspaceFolder\": \"/workspaces/${localWorkspaceFolderBasename}\",\n \"remoteUser\": \"root\",\n \"overrideCommand\": true,\n \"postCreateCommand\": \"uv sync --group lint --group test\",\n \"customizations\": {\n \"vscode\": {\n \"extensions\": [\n \"ms-python.python\",\n \"ms-python.vscode-pylance\",\n \"charliermarsh.ruff\",\n \"ms-azuretools.vscode-docker\",\n \"ms-toolsai.jupyter\",\n \"tamasfe.even-better-toml\",\n \"eamodio.gitlens\",\n \"EditorConfig.EditorConfig\",\n \"DavidAnson.vscode-markdownlint\"\n ],\n \"settings\": {\n \"python.defaultInterpreterPath\": \"/workspaces/${localWorkspaceFolderBasename}/.venv/bin/python\",\n \"python.terminal.activateEnvironment\": true,\n \"python.testing.pytestEnabled\": true,\n \"python.testing.unittestEnabled\": false,\n \"python.testing.pytestArgs\": [\n \"tests\"\n ],\n \"editor.formatOnSave\": true,\n \"editor.codeActionsOnSave\": {\n \"source.organizeImports.ruff\": \"explicit\",\n \"source.fixAll.ruff\": \"explicit\"\n }\n }\n }\n }\n}\n" }, { "path": ".github/ISSUE_TEMPLATE/bug_report.yml", "content": "name: Bug Report\ndescription: Report a bug in LEANN\nlabels: [\"bug\"]\n\nbody:\n - type: textarea\n id: description\n attributes:\n label: What happened?\n description: A clear description of the bug\n validations:\n required: true\n\n - type: textarea\n id: reproduce\n attributes:\n label: How to reproduce\n placeholder: |\n 1. Install with...\n 2. Run command...\n 3. See error\n validations:\n required: true\n\n - type: textarea\n id: error\n attributes:\n label: Error message\n description: Paste any error messages\n render: shell\n\n - type: input\n id: version\n attributes:\n label: LEANN Version\n placeholder: \"0.1.0\"\n validations:\n required: true\n\n - type: dropdown\n id: os\n attributes:\n label: Operating System\n options:\n - macOS\n - Linux\n - Windows\n - Docker\n validations:\n required: true\n" }, { "path": ".github/ISSUE_TEMPLATE/config.yml", "content": "blank_issues_enabled: true\ncontact_links:\n - name: Documentation\n url: https://github.com/LEANN-RAG/LEANN-RAG/tree/main/docs\n about: Read the docs first\n - name: Discussions\n url: https://github.com/LEANN-RAG/LEANN-RAG/discussions\n about: Ask questions and share ideas\n" }, { "path": ".github/ISSUE_TEMPLATE/feature_request.yml", "content": "name: Feature Request\ndescription: Suggest a new feature for LEANN\nlabels: [\"enhancement\"]\n\nbody:\n - type: textarea\n id: problem\n attributes:\n label: What problem does this solve?\n description: Describe the problem or need\n validations:\n required: true\n\n - type: textarea\n id: solution\n attributes:\n label: Proposed solution\n description: How would you like this to work?\n validations:\n required: true\n\n - type: textarea\n id: example\n attributes:\n label: Example usage\n description: Show how the API might look\n render: python\n" }, { "path": ".github/pull_request_template.md", "content": "## What does this PR do?\n\n\n\n## Related Issues\n\nFixes #\n\n## Checklist\n\n- [ ] Tests pass (`uv run pytest`)\n- [ ] Code formatted (`ruff format` and `ruff check`)\n- [ ] Pre-commit hooks pass (`pre-commit run --all-files`)\n" }, { "path": ".github/workflows/build-and-publish.yml", "content": "name: CI\n\non:\n push:\n branches: [ main ]\n pull_request:\n branches: [ main ]\n workflow_dispatch:\n\njobs:\n build:\n uses: ./.github/workflows/build-reusable.yml\n" }, { "path": ".github/workflows/build-reusable.yml", "content": "name: Reusable Build\n\non:\n workflow_call:\n inputs:\n ref:\n description: 'Git ref to build'\n required: false\n type: string\n default: ''\n\njobs:\n lint:\n name: Lint and Format Check\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v4\n with:\n ref: ${{ inputs.ref }}\n submodules: recursive\n\n - name: Install uv and Python\n uses: astral-sh/setup-uv@v6\n with:\n python-version: '3.11'\n\n - name: Run pre-commit with only lint group (no project deps)\n run: |\n uv run --only-group lint pre-commit run --all-files --show-diff-on-failure\n\n type-check:\n name: Type Check with ty\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v4\n with:\n ref: ${{ inputs.ref }}\n submodules: recursive\n\n - name: Install uv and Python\n uses: astral-sh/setup-uv@v6\n with:\n python-version: '3.11'\n\n - name: Install ty\n run: uv tool install ty==0.0.17\n\n - name: Run ty type checker\n run: |\n # Run ty on core packages, apps, and tests\n ty check packages/leann-core/src apps tests\n\n build:\n needs: [lint, type-check]\n name: Build ${{ matrix.os }} Python ${{ matrix.python }}\n defaults:\n run:\n shell: bash\n strategy:\n fail-fast: true\n matrix:\n include:\n # Note: Python 3.9 dropped - uses PEP 604 union syntax (str | None)\n # which requires Python 3.10+\n - os: ubuntu-22.04\n python: '3.10'\n - os: ubuntu-22.04\n python: '3.11'\n - os: ubuntu-22.04\n python: '3.12'\n - os: ubuntu-22.04\n python: '3.13'\n # ARM64 Linux builds\n - os: ubuntu-22.04-arm\n python: '3.10'\n - os: ubuntu-22.04-arm\n python: '3.11'\n - os: ubuntu-22.04-arm\n python: '3.12'\n - os: ubuntu-22.04-arm\n python: '3.13'\n - os: macos-14\n python: '3.10'\n - os: macos-14\n python: '3.11'\n - os: macos-14\n python: '3.12'\n - os: macos-14\n python: '3.13'\n - os: macos-15\n python: '3.10'\n - os: macos-15\n python: '3.11'\n - os: macos-15\n python: '3.12'\n - os: macos-15\n python: '3.13'\n # Intel Mac builds (x86_64) - replaces deprecated macos-13\n # Note: Python 3.13 excluded - PyTorch has no wheels for macOS x86_64 + Python 3.13\n # (PyTorch <=2.4.1 lacks cp313, PyTorch >=2.5.0 dropped Intel Mac support)\n - os: macos-15-intel\n python: '3.10'\n - os: macos-15-intel\n python: '3.11'\n - os: macos-15-intel\n python: '3.12'\n # macOS 26 (beta) - arm64\n - os: macos-26\n python: '3.10'\n - os: macos-26\n python: '3.11'\n - os: macos-26\n python: '3.12'\n - os: macos-26\n python: '3.13'\n # Windows validation (native HNSW + DiskANN build/install path)\n - os: windows-2022\n python: '3.11'\n - os: windows-2022\n python: '3.12'\n - os: windows-2022\n python: '3.13'\n - os: windows-2022\n python: '3.14'\n runs-on: ${{ matrix.os }}\n\n steps:\n - uses: actions/checkout@v5\n with:\n ref: ${{ inputs.ref }}\n submodules: recursive\n\n - name: Install uv and Python\n uses: astral-sh/setup-uv@v6\n with:\n python-version: ${{ matrix.python }}\n\n - name: Install system dependencies (Ubuntu)\n if: runner.os == 'Linux'\n run: |\n sudo apt-get update\n sudo apt-get install -y libomp-dev libboost-all-dev protobuf-compiler libzmq3-dev \\\n pkg-config libabsl-dev libaio-dev libprotobuf-dev \\\n patchelf\n\n # Debug: Show system information\n echo \"🔍 System Information:\"\n echo \"Architecture: $(uname -m)\"\n echo \"OS: $(uname -a)\"\n echo \"CPU info: $(lscpu | head -5)\"\n\n # Install math library based on architecture\n ARCH=$(uname -m)\n echo \"🔍 Setting up math library for architecture: $ARCH\"\n\n if [[ \"$ARCH\" == \"x86_64\" ]]; then\n # Install Intel MKL for DiskANN on x86_64\n echo \"📦 Installing Intel MKL for x86_64...\"\n wget -q https://registrationcenter-download.intel.com/akdlm/IRC_NAS/79153e0f-74d7-45af-b8c2-258941adf58a/intel-onemkl-2025.0.0.940.sh\n sudo sh intel-onemkl-2025.0.0.940.sh -a --components intel.oneapi.lin.mkl.devel --action install --eula accept -s\n source /opt/intel/oneapi/setvars.sh\n echo \"MKLROOT=/opt/intel/oneapi/mkl/latest\" >> $GITHUB_ENV\n echo \"LD_LIBRARY_PATH=/opt/intel/oneapi/compiler/latest/linux/compiler/lib/intel64_lin\" >> $GITHUB_ENV\n echo \"LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/intel/oneapi/mkl/latest/lib/intel64\" >> $GITHUB_ENV\n echo \"✅ Intel MKL installed for x86_64\"\n\n # Debug: Check MKL installation\n echo \"🔍 MKL Installation Check:\"\n ls -la /opt/intel/oneapi/mkl/latest/ || echo \"MKL directory not found\"\n ls -la /opt/intel/oneapi/mkl/latest/lib/ || echo \"MKL lib directory not found\"\n\n elif [[ \"$ARCH\" == \"aarch64\" ]]; then\n # Use OpenBLAS for ARM64 (MKL installer not compatible with ARM64)\n echo \"📦 Installing OpenBLAS for ARM64...\"\n sudo apt-get install -y libopenblas-dev liblapack-dev liblapacke-dev\n echo \"✅ OpenBLAS installed for ARM64\"\n\n # Debug: Check OpenBLAS installation\n echo \"🔍 OpenBLAS Installation Check:\"\n dpkg -l | grep openblas || echo \"OpenBLAS package not found\"\n ls -la /usr/lib/aarch64-linux-gnu/openblas/ || echo \"OpenBLAS directory not found\"\n fi\n\n # Debug: Show final library paths\n echo \"🔍 Final LD_LIBRARY_PATH: $LD_LIBRARY_PATH\"\n\n - name: Install system dependencies (macOS)\n if: runner.os == 'macOS'\n run: |\n # Don't install LLVM, use system clang for better compatibility\n brew install libomp boost protobuf zeromq\n\n - name: Install system dependencies (Windows)\n if: runner.os == 'Windows'\n run: |\n retry() {\n local attempts=$1\n shift\n local n=1\n while true; do\n \"$@\" && break\n if [[ $n -ge $attempts ]]; then\n echo \"Command failed after $n attempts: $*\"\n return 1\n fi\n echo \"Command failed (attempt $n/$attempts). Retrying in 10s: $*\"\n sleep 10\n n=$((n + 1))\n done\n }\n\n retry 5 choco install swig -y --no-progress\n retry 5 choco install nuget.commandline -y --no-progress\n # pkgconfiglite via Chocolatey is flaky (exits 0 even on failure);\n # verify the binary exists and fall back to direct download.\n retry 3 choco install pkgconfiglite -y --no-progress || true\n PKG_CONFIG_DIR=\"C:/ProgramData/chocolatey/bin\"\n if [[ ! -f \"${PKG_CONFIG_DIR}/pkg-config.exe\" ]]; then\n echo \"pkg-config.exe not found after choco, downloading directly...\"\n PKG_CONFIG_DIR=\"${RUNNER_TEMP}/pkg-config\"\n mkdir -p \"${PKG_CONFIG_DIR}\"\n curl -fsSL -o \"${RUNNER_TEMP}/pkgconfiglite.zip\" \\\n \"https://sourceforge.net/projects/pkgconfiglite/files/0.28-1/pkg-config-lite-0.28-1_bin-win32.zip/download\"\n unzip -q \"${RUNNER_TEMP}/pkgconfiglite.zip\" -d \"${RUNNER_TEMP}/pkgconfiglite\"\n cp \"${RUNNER_TEMP}/pkgconfiglite/pkg-config-lite-0.28-1/bin/\"* \"${PKG_CONFIG_DIR}/\"\n fi\n echo \"${PKG_CONFIG_DIR}\" >> \"$GITHUB_PATH\"\n echo \"PKG_CONFIG_EXECUTABLE=${PKG_CONFIG_DIR}/pkg-config.exe\" >> \"$GITHUB_ENV\"\n if [[ -z \"${VCPKG_INSTALLATION_ROOT:-}\" ]]; then\n echo \"VCPKG_INSTALLATION_ROOT is not set on this runner\"\n exit 1\n fi\n retry 5 \"${VCPKG_INSTALLATION_ROOT}/vcpkg\" install zeromq:x64-windows\n retry 5 \"${VCPKG_INSTALLATION_ROOT}/vcpkg\" install openblas:x64-windows\n retry 5 \"${VCPKG_INSTALLATION_ROOT}/vcpkg\" install lapack:x64-windows\n retry 5 \"${VCPKG_INSTALLATION_ROOT}/vcpkg\" install boost-program-options:x64-windows\n retry 5 \"${VCPKG_INSTALLATION_ROOT}/vcpkg\" install protobuf:x64-windows\n\n # DiskANN links against Intel OpenMP (libiomp5md) via NuGet during its\n # CMake build. The NuGet packages end up in a temp build dir that is\n # cleaned up by `uv build`, so delvewheel can't find the DLL later.\n # Download it here to a persistent, known location.\n NUGET_PKG_DIR=\"${RUNNER_TEMP}/nuget_pkgs\"\n retry 5 nuget install intelopenmp.redist.win -Version 2022.0.3.3747 \\\n -ExcludeVersion -OutputDirectory \"${NUGET_PKG_DIR}\"\n echo \"INTEL_OMP_BIN_DIR=${NUGET_PKG_DIR}/intelopenmp.redist.win/runtimes/win-x64/native\" >> \"$GITHUB_ENV\"\n\n echo \"PKG_CONFIG_PATH=${VCPKG_INSTALLATION_ROOT}/installed/x64-windows/lib/pkgconfig\" >> \"$GITHUB_ENV\"\n echo \"CMAKE_PREFIX_PATH=${VCPKG_INSTALLATION_ROOT}/installed/x64-windows\" >> \"$GITHUB_ENV\"\n echo \"OPENBLAS_LIB=${VCPKG_INSTALLATION_ROOT}/installed/x64-windows/lib/openblas.lib\" >> \"$GITHUB_ENV\"\n echo \"${VCPKG_INSTALLATION_ROOT}/installed/x64-windows/bin\" >> \"$GITHUB_PATH\"\n echo \"${VCPKG_INSTALLATION_ROOT}/installed/x64-windows/debug/bin\" >> \"$GITHUB_PATH\"\n echo \"${VCPKG_INSTALLATION_ROOT}/installed/x64-windows/tools/protobuf\" >> \"$GITHUB_PATH\"\n pkg-config --version || true\n where pkg-config || true\n pkg-config --modversion libzmq || true\n\n - name: Install build dependencies\n run: |\n retry() {\n local attempts=$1\n shift\n local n=1\n while true; do\n \"$@\" && break\n if [[ $n -ge $attempts ]]; then\n echo \"Command failed after $n attempts: $*\"\n return 1\n fi\n echo \"Command failed (attempt $n/$attempts). Retrying in 5s: $*\"\n sleep 5\n n=$((n + 1))\n done\n }\n\n retry 5 uv python install ${{ matrix.python }}\n uv venv --python ${{ matrix.python }} .uv-build\n if [[ \"$RUNNER_OS\" == \"Windows\" ]]; then\n BUILD_PY=\".uv-build\\\\Scripts\\\\python.exe\"\n else\n BUILD_PY=\".uv-build/bin/python\"\n fi\n retry 5 uv pip install --python \"$BUILD_PY\" scikit-build-core numpy swig Cython pybind11\n if [[ \"$RUNNER_OS\" == \"Linux\" ]]; then\n retry 5 uv pip install --python \"$BUILD_PY\" auditwheel\n elif [[ \"$RUNNER_OS\" == \"macOS\" ]]; then\n retry 5 uv pip install --python \"$BUILD_PY\" delocate\n else\n retry 5 uv pip install --python \"$BUILD_PY\" delvewheel\n fi\n\n if [[ \"$RUNNER_OS\" == \"Windows\" ]]; then\n echo \"$(pwd)\\\\.uv-build\\\\Scripts\" >> $GITHUB_PATH\n else\n echo \"$(pwd)/.uv-build/bin\" >> $GITHUB_PATH\n fi\n\n - name: Set macOS environment variables\n if: runner.os == 'macOS'\n run: |\n # Use brew --prefix to automatically detect Homebrew installation path\n HOMEBREW_PREFIX=$(brew --prefix)\n echo \"HOMEBREW_PREFIX=${HOMEBREW_PREFIX}\" >> $GITHUB_ENV\n echo \"OpenMP_ROOT=${HOMEBREW_PREFIX}/opt/libomp\" >> $GITHUB_ENV\n\n # Set CMAKE_PREFIX_PATH to let CMake find all packages automatically\n echo \"CMAKE_PREFIX_PATH=${HOMEBREW_PREFIX}\" >> $GITHUB_ENV\n\n # Set compiler flags for OpenMP (required for both backends)\n echo \"LDFLAGS=-L${HOMEBREW_PREFIX}/opt/libomp/lib\" >> $GITHUB_ENV\n echo \"CPPFLAGS=-I${HOMEBREW_PREFIX}/opt/libomp/include\" >> $GITHUB_ENV\n\n - name: Build packages\n run: |\n # Build core (platform independent)\n cd packages/leann-core\n uv build\n cd ../..\n\n # Build HNSW backend\n cd packages/leann-backend-hnsw\n if [[ \"${{ matrix.os }}\" == macos-* ]]; then\n # Use system clang for better compatibility\n export CC=clang\n export CXX=clang++\n # Set deployment target based on runner\n # macos-15-intel runs macOS 15, so target 15.0 (system libraries require it)\n if [[ \"${{ matrix.os }}\" == \"macos-15-intel\" ]]; then\n export MACOSX_DEPLOYMENT_TARGET=15.0\n elif [[ \"${{ matrix.os }}\" == macos-14* ]]; then\n export MACOSX_DEPLOYMENT_TARGET=14.0\n elif [[ \"${{ matrix.os }}\" == macos-15* ]]; then\n export MACOSX_DEPLOYMENT_TARGET=15.0\n elif [[ \"${{ matrix.os }}\" == macos-26* ]]; then\n export MACOSX_DEPLOYMENT_TARGET=26.0\n fi\n uv build --wheel --python ${{ matrix.python }} --find-links ${GITHUB_WORKSPACE}/packages/leann-core/dist\n else\n uv build --wheel --python ${{ matrix.python }} --find-links ${GITHUB_WORKSPACE}/packages/leann-core/dist\n fi\n cd ../..\n\n # Build DiskANN backend\n cd packages/leann-backend-diskann\n if [[ \"${{ matrix.os }}\" == macos-* ]]; then\n # Use system clang for better compatibility\n export CC=clang\n export CXX=clang++\n # Set deployment target based on runner\n # macos-15-intel runs macOS 15, so target 15.0 (system libraries require it)\n if [[ \"${{ matrix.os }}\" == \"macos-15-intel\" ]]; then\n export MACOSX_DEPLOYMENT_TARGET=15.0\n elif [[ \"${{ matrix.os }}\" == macos-14* ]]; then\n export MACOSX_DEPLOYMENT_TARGET=14.0\n elif [[ \"${{ matrix.os }}\" == macos-15* ]]; then\n export MACOSX_DEPLOYMENT_TARGET=15.0\n elif [[ \"${{ matrix.os }}\" == macos-26* ]]; then\n export MACOSX_DEPLOYMENT_TARGET=26.0\n fi\n uv build --wheel --python ${{ matrix.python }} --find-links ${GITHUB_WORKSPACE}/packages/leann-core/dist\n else\n uv build --wheel --python ${{ matrix.python }} --find-links ${GITHUB_WORKSPACE}/packages/leann-core/dist\n fi\n cd ../..\n\n # Build meta package (platform independent)\n cd packages/leann\n uv build\n cd ../..\n\n - name: Repair wheels (Linux)\n if: runner.os == 'Linux'\n run: |\n # Repair HNSW wheel\n cd packages/leann-backend-hnsw\n if [ -d dist ]; then\n auditwheel repair dist/*.whl -w dist_repaired\n rm -rf dist\n mv dist_repaired dist\n fi\n cd ../..\n\n # Repair DiskANN wheel\n cd packages/leann-backend-diskann\n if [ -d dist ]; then\n auditwheel repair dist/*.whl -w dist_repaired\n rm -rf dist\n mv dist_repaired dist\n fi\n cd ../..\n\n - name: Repair wheels (macOS)\n if: runner.os == 'macOS'\n run: |\n # Determine deployment target based on runner OS\n # macos-15-intel runs macOS 15, so target 15.0 (system libraries require it)\n if [[ \"${{ matrix.os }}\" == \"macos-15-intel\" ]]; then\n HNSW_TARGET=\"15.0\"\n DISKANN_TARGET=\"15.0\"\n elif [[ \"${{ matrix.os }}\" == macos-14* ]]; then\n HNSW_TARGET=\"14.0\"\n DISKANN_TARGET=\"14.0\"\n elif [[ \"${{ matrix.os }}\" == macos-15* ]]; then\n HNSW_TARGET=\"15.0\"\n DISKANN_TARGET=\"15.0\"\n elif [[ \"${{ matrix.os }}\" == macos-26* ]]; then\n HNSW_TARGET=\"26.0\"\n DISKANN_TARGET=\"26.0\"\n fi\n\n # Repair HNSW wheel\n cd packages/leann-backend-hnsw\n if [ -d dist ]; then\n export MACOSX_DEPLOYMENT_TARGET=$HNSW_TARGET\n delocate-wheel -w dist_repaired -v --require-target-macos-version $HNSW_TARGET dist/*.whl\n rm -rf dist\n mv dist_repaired dist\n fi\n cd ../..\n\n # Repair DiskANN wheel\n cd packages/leann-backend-diskann\n if [ -d dist ]; then\n export MACOSX_DEPLOYMENT_TARGET=$DISKANN_TARGET\n delocate-wheel -w dist_repaired -v --require-target-macos-version $DISKANN_TARGET dist/*.whl\n rm -rf dist\n mv dist_repaired dist\n fi\n cd ../..\n\n - name: Repair wheels (Windows)\n if: runner.os == 'Windows'\n run: |\n # Repair HNSW wheel\n cd packages/leann-backend-hnsw\n if [ -d dist ]; then\n delvewheel repair dist/*.whl -w dist_repaired --add-path \"${VCPKG_INSTALLATION_ROOT}/installed/x64-windows/bin\"\n rm -rf dist\n mv dist_repaired dist\n fi\n cd ../..\n\n # Repair DiskANN wheel.\n # DiskANN's CMake install bundles diskann.dll and libiomp5md.dll inside\n # the wheel, but delvewheel doesn't search inside the wheel for deps.\n # Extract the wheel so delvewheel can discover them via --add-path.\n cd packages/leann-backend-diskann\n if [ -d dist ]; then\n # Extract the wheel and build --add-path with native Windows paths.\n # mktemp returns Git Bash paths (/tmp/...) that delvewheel.exe can't\n # resolve, so use Python for the entire extract-and-repair flow.\n python -c \"\n import zipfile, sys, glob, tempfile, subprocess, os, shutil\n whl = glob.glob('dist/*.whl')[0]\n tmpdir = tempfile.mkdtemp(prefix='diskann_whl_')\n zipfile.ZipFile(whl).extractall(tmpdir)\n\n add_paths = [os.environ.get('VCPKG_INSTALLATION_ROOT','') + '/installed/x64-windows/bin',\n os.environ.get('INTEL_OMP_BIN_DIR','')]\n for entry in os.listdir(tmpdir):\n full = os.path.join(tmpdir, entry)\n if os.path.isdir(full):\n add_paths.append(full)\n add_path_str = ';'.join(p for p in add_paths if p)\n\n print(f'add-path: {add_path_str}')\n rc = subprocess.call([sys.executable, '-m', 'delvewheel', 'repair', whl,\n '-w', 'dist_repaired', '--add-path', add_path_str])\n shutil.rmtree(tmpdir, ignore_errors=True)\n sys.exit(rc)\n \"\n rm -rf dist\n mv dist_repaired dist\n fi\n cd ../..\n\n - name: List built packages\n run: |\n echo \"📦 Built packages:\"\n find packages/*/dist -name \"*.whl\" -o -name \"*.tar.gz\" | sort\n\n\n - name: Install built packages for testing\n run: |\n retry() {\n local attempts=$1\n shift\n local n=1\n while true; do\n \"$@\" && break\n if [[ $n -ge $attempts ]]; then\n echo \"Command failed after $n attempts: $*\"\n return 1\n fi\n echo \"Command failed (attempt $n/$attempts). Retrying in 5s: $*\"\n sleep 5\n n=$((n + 1))\n done\n }\n\n # Create uv-managed virtual environment with the requested interpreter\n retry 5 uv python install ${{ matrix.python }}\n uv venv --python ${{ matrix.python }}\n source .venv/bin/activate || source .venv/Scripts/activate\n\n if [[ \"$RUNNER_OS\" == \"Windows\" ]]; then\n UV_PY=\".venv\\\\Scripts\\\\python.exe\"\n else\n UV_PY=\".venv/bin/python\"\n fi\n\n # Install test dependency group only (avoids reinstalling project package)\n uv pip install --python \"$UV_PY\" --group test\n\n # Install core wheel built in this job\n CORE_WHL=$(find packages/leann-core/dist -maxdepth 1 -name \"*.whl\" -print -quit)\n if [[ -n \"$CORE_WHL\" ]]; then\n uv pip install --python \"$UV_PY\" \"$CORE_WHL\"\n else\n uv pip install --python \"$UV_PY\" packages/leann-core/dist/*.tar.gz\n fi\n\n PY_TAG=$($UV_PY -c \"import sys; print(f'cp{sys.version_info[0]}{sys.version_info[1]}')\")\n\n if [[ \"$RUNNER_OS\" == \"macOS\" ]]; then\n # macos-15-intel runs macOS 15, so target 15.0 (system libraries require it)\n if [[ \"${{ matrix.os }}\" == \"macos-15-intel\" ]]; then\n export MACOSX_DEPLOYMENT_TARGET=15.0\n elif [[ \"${{ matrix.os }}\" == macos-14* ]]; then\n export MACOSX_DEPLOYMENT_TARGET=14.0\n elif [[ \"${{ matrix.os }}\" == macos-15* ]]; then\n export MACOSX_DEPLOYMENT_TARGET=15.0\n elif [[ \"${{ matrix.os }}\" == macos-26* ]]; then\n export MACOSX_DEPLOYMENT_TARGET=26.0\n fi\n fi\n\n HNSW_WHL=$(find packages/leann-backend-hnsw/dist -maxdepth 1 -name \"*-${PY_TAG}-*.whl\" -print -quit)\n if [[ -z \"$HNSW_WHL\" ]]; then\n HNSW_WHL=$(find packages/leann-backend-hnsw/dist -maxdepth 1 -name \"*-py3-*.whl\" -print -quit)\n fi\n if [[ -n \"$HNSW_WHL\" ]]; then\n uv pip install --python \"$UV_PY\" \"$HNSW_WHL\"\n else\n uv pip install --python \"$UV_PY\" ./packages/leann-backend-hnsw\n fi\n\n DISKANN_WHL=$(find packages/leann-backend-diskann/dist -maxdepth 1 -name \"*-${PY_TAG}-*.whl\" -print -quit)\n if [[ -z \"$DISKANN_WHL\" ]]; then\n DISKANN_WHL=$(find packages/leann-backend-diskann/dist -maxdepth 1 -name \"*-py3-*.whl\" -print -quit)\n fi\n if [[ -n \"$DISKANN_WHL\" ]]; then\n uv pip install --python \"$UV_PY\" \"$DISKANN_WHL\"\n else\n uv pip install --python \"$UV_PY\" ./packages/leann-backend-diskann\n fi\n\n LEANN_WHL=$(find packages/leann/dist -maxdepth 1 -name \"*.whl\" -print -quit)\n if [[ -n \"$LEANN_WHL\" ]]; then\n uv pip install --python \"$UV_PY\" \"$LEANN_WHL\"\n else\n uv pip install --python \"$UV_PY\" packages/leann/dist/*.tar.gz\n fi\n\n - name: Run tests with pytest\n env:\n CI: true\n OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}\n HF_HUB_DISABLE_SYMLINKS: 1\n TOKENIZERS_PARALLELISM: false\n PYTORCH_ENABLE_MPS_FALLBACK: 0\n OMP_NUM_THREADS: 1\n MKL_NUM_THREADS: 1\n run: |\n source .venv/bin/activate || source .venv/Scripts/activate\n pytest tests/ -v --tb=short\n\n - name: Run sanity checks (optional)\n run: |\n # Activate virtual environment\n source .venv/bin/activate || source .venv/Scripts/activate\n\n # Run distance function tests if available\n if [ -f test/sanity_checks/test_distance_functions.py ]; then\n echo \"Running distance function sanity checks...\"\n python test/sanity_checks/test_distance_functions.py || echo \"⚠️ Distance function test failed, continuing...\"\n fi\n\n - name: Upload artifacts\n uses: actions/upload-artifact@v4\n with:\n name: packages-${{ matrix.os }}-py${{ matrix.python }}\n path: packages/*/dist/\n\n\n arch-smoke:\n name: Arch Linux smoke test (install & import)\n needs: build\n runs-on: ubuntu-latest\n container:\n image: archlinux:latest\n\n steps:\n - name: Prepare system\n run: |\n # Initialize pacman keyring to avoid \"no secret key available\" error\n pacman-key --init\n pacman -Syu --noconfirm\n # Install build essentials (uv will manage Python version)\n pacman -S --noconfirm gcc git zlib openssl\n\n - name: Download ALL wheel artifacts from this run\n uses: actions/download-artifact@v5\n with:\n # Don't specify name, download all artifacts\n path: ./wheels\n\n - name: Install uv\n uses: astral-sh/setup-uv@v6\n\n - name: Create virtual environment and install wheels\n run: |\n # Use Python 3.13 explicitly (Arch has Python 3.14 which PyO3/tokenizers doesn't support yet)\n uv python install 3.13\n uv venv --python 3.13\n source .venv/bin/activate || source .venv/Scripts/activate\n uv pip install --find-links wheels leann-core\n uv pip install --find-links wheels leann-backend-hnsw\n uv pip install --find-links wheels leann-backend-diskann\n uv pip install --find-links wheels leann\n\n - name: Import & tiny runtime check\n env:\n OMP_NUM_THREADS: 1\n MKL_NUM_THREADS: 1\n run: |\n source .venv/bin/activate || source .venv/Scripts/activate\n python - <<'PY'\n import leann\n import leann_backend_hnsw as h\n import leann_backend_diskann as d\n from leann import LeannBuilder, LeannSearcher\n b = LeannBuilder(backend_name=\"hnsw\")\n b.add_text(\"hello arch\")\n b.build_index(\"arch_demo.leann\")\n s = LeannSearcher(\"arch_demo.leann\")\n print(\"search:\", s.search(\"hello\", top_k=1))\n PY\n" }, { "path": ".github/workflows/link-check.yml", "content": "name: Link Check\n\non:\n push:\n branches: [ main, master ]\n pull_request:\n schedule:\n - cron: \"0 3 * * 1\"\n\njobs:\n link-check:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v4\n - uses: lycheeverse/lychee-action@v2\n with:\n args: >-\n --no-progress --insecure\n --user-agent 'curl/7.68.0'\n --max-retries 3\n --retry-wait-time 5\n --exclude '.*star-history\\.com.*'\n --accept 200,201,202,203,204,205,206,207,208,226,300,301,302,303,304,305,306,307,308,503\n README.md docs/ apps/ examples/ benchmarks/\n fail: false\n env:\n GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n" }, { "path": ".github/workflows/release-manual.yml", "content": "name: Release\n\non:\n workflow_dispatch:\n inputs:\n version:\n description: 'Version to release (e.g., 0.1.2)'\n required: true\n type: string\n\njobs:\n verify-ci:\n name: Verify main CI\n runs-on: ubuntu-latest\n permissions:\n actions: read\n contents: read\n\n steps:\n - name: Require latest main CI success\n env:\n GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n run: |\n set -euo pipefail\n\n MAIN_SHA=$(gh api repos/${GITHUB_REPOSITORY}/commits/main --jq '.sha')\n RUN_ID=$(gh api repos/${GITHUB_REPOSITORY}/actions/workflows/build-and-publish.yml/runs \\\n --method GET \\\n --field branch=main \\\n --field head_sha=\"${MAIN_SHA}\" \\\n --field per_page=1 \\\n --jq '.workflow_runs[0].id')\n\n if [ -z \"${RUN_ID}\" ] || [ \"${RUN_ID}\" = \"null\" ]; then\n echo \"❌ No CI run found for main @ ${MAIN_SHA}\"\n exit 1\n fi\n\n STATUS=$(gh api repos/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID} --jq '.status')\n CONCLUSION=$(gh api repos/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID} --jq '.conclusion')\n URL=$(gh api repos/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID} --jq '.html_url')\n\n echo \"CI run: ${URL}\"\n if [ \"${STATUS}\" != \"completed\" ] || [ \"${CONCLUSION}\" != \"success\" ]; then\n echo \"❌ CI not successful for main @ ${MAIN_SHA}\"\n echo \"Status: ${STATUS}\"\n echo \"Conclusion: ${CONCLUSION}\"\n exit 1\n fi\n\n echo \"✅ CI succeeded for main @ ${MAIN_SHA}\"\n\n update-version:\n name: Update Version\n needs: verify-ci\n runs-on: ubuntu-latest\n permissions:\n contents: write\n outputs:\n commit-sha: ${{ steps.push.outputs.commit-sha }}\n\n steps:\n - uses: actions/checkout@v4\n\n - name: Validate version\n run: |\n # Remove 'v' prefix if present for validation\n VERSION_CLEAN=\"${{ inputs.version }}\"\n VERSION_CLEAN=\"${VERSION_CLEAN#v}\"\n if ! [[ \"$VERSION_CLEAN\" =~ ^[0-9]+\\.[0-9]+\\.[0-9]+$ ]]; then\n echo \"❌ Invalid version format. Expected format: X.Y.Z or vX.Y.Z\"\n exit 1\n fi\n echo \"✅ Version format valid: ${{ inputs.version }}\"\n\n - name: Update versions and push\n id: push\n run: |\n # Check current version\n CURRENT_VERSION=$(grep \"^version\" packages/leann-core/pyproject.toml | cut -d'\"' -f2)\n echo \"Current version: $CURRENT_VERSION\"\n echo \"Target version: ${{ inputs.version }}\"\n\n if [ \"$CURRENT_VERSION\" = \"${{ inputs.version }}\" ]; then\n echo \"⚠️ Version is already ${{ inputs.version }}, skipping update\"\n COMMIT_SHA=$(git rev-parse HEAD)\n else\n ./scripts/bump_version.sh ${{ inputs.version }}\n git config user.name \"GitHub Actions\"\n git config user.email \"actions@github.com\"\n git add packages/*/pyproject.toml\n git commit -m \"chore: release v${{ inputs.version }}\"\n git push origin main\n COMMIT_SHA=$(git rev-parse HEAD)\n echo \"✅ Pushed version update: $COMMIT_SHA\"\n fi\n\n echo \"commit-sha=$COMMIT_SHA\" >> $GITHUB_OUTPUT\n\n build-packages:\n name: Build packages\n needs: update-version\n uses: ./.github/workflows/build-reusable.yml\n with:\n ref: 'main'\n\n publish:\n name: Publish and Release\n needs: [update-version, build-packages]\n if: always() && needs.update-version.result == 'success' && needs.build-packages.result == 'success'\n runs-on: ubuntu-latest\n permissions:\n contents: write\n\n steps:\n - uses: actions/checkout@v4\n with:\n ref: 'main'\n\n - name: Download all artifacts\n uses: actions/download-artifact@v4\n with:\n path: dist-artifacts\n\n - name: Collect packages\n run: |\n mkdir -p dist\n find dist-artifacts -name \"*.whl\" -exec cp {} dist/ \\;\n find dist-artifacts -name \"*.tar.gz\" -exec cp {} dist/ \\;\n\n echo \"📦 Packages to publish:\"\n ls -la dist/\n\n - name: Publish to PyPI\n env:\n TWINE_USERNAME: __token__\n TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}\n run: |\n if [ -z \"$TWINE_PASSWORD\" ]; then\n echo \"❌ PYPI_API_TOKEN not configured!\"\n exit 1\n fi\n\n pip install twine\n twine upload dist/* --skip-existing --verbose\n\n echo \"✅ Published to PyPI!\"\n\n - name: Create release\n run: |\n # Check if tag already exists\n if git rev-parse \"v${{ inputs.version }}\" >/dev/null 2>&1; then\n echo \"⚠️ Tag v${{ inputs.version }} already exists, skipping tag creation\"\n else\n git tag \"v${{ inputs.version }}\"\n git push origin \"v${{ inputs.version }}\"\n echo \"✅ Created and pushed tag v${{ inputs.version }}\"\n fi\n\n # Check if release already exists\n if gh release view \"v${{ inputs.version }}\" >/dev/null 2>&1; then\n echo \"⚠️ Release v${{ inputs.version }} already exists, skipping release creation\"\n else\n gh release create \"v${{ inputs.version }}\" \\\n --title \"Release v${{ inputs.version }}\" \\\n --notes \"🚀 Released to PyPI: https://pypi.org/project/leann/${{ inputs.version }}/\" \\\n --latest\n echo \"✅ Created GitHub release v${{ inputs.version }}\"\n fi\n env:\n GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n" }, { "path": ".gitignore", "content": "raw_data/\nscaling_out/\nscaling_out_old/\nsanity_check/\ndemo/indices/\n# .vscode/\n*.log\n*pycache*\noutputs/\n*.pkl\n*.pdf\n*.idx\n*.map\n.history/\nlm_eval.egg-info/\ndemo/experiment_results/**/*.json\n*.jsonl\n*.eml\n*.emlx\n*.json\n*.png\n!.vscode/*.json\n!.devcontainer/*.json\n!skills/**/claw.json\n*.sh\n*.txt\n!CMakeLists.txt\n!llms.txt\nlatency_breakdown*.json\nexperiment_results/eval_results/diskann/*.json\naws/\n.venv/\n.cursor/rules/\n*.egg-info/\nskip_reorder_comparison/\nanalysis_results/\nbuild/\n.cache/\nnprobe_logs/\nmicro/results\nmicro/contriever-INT8\ndata/*\n!data/2501.14312v1 (1).pdf\n!data/2506.08276v1.pdf\n!data/PrideandPrejudice.txt\n!data/huawei_pangu.md\n!data/ground_truth/\n!data/indices/\n!data/queries/\n!data/.gitattributes\n*.qdstrm\nbenchmark_results/\nresults/\nfrac_*.png\nfinal_in_*.png\nembedding_comparison_results/\n*.ind\n*.gz\n*.fvecs\n*.ivecs\n*.index\n*.bin\n*.old\n\nread_graph\nanalyze_diskann_graph\ndegree_distribution.png\nmicro/degree_distribution.png\n\npolicy_results_*\nresults_*/\nexperiment_results/\n.DS_Store\n\n# The above are inherited from old Power RAG repo\n\n# Python-generated files\n__pycache__/\n*.py[oc]\nbuild/\ndist/\nwheels/\n*.egg-info\n\n# Virtual environments\n.venv\n.env\n\ntest_indices*/\ntest_*.py\n!tests/**\n# Re-ignore common generated artifacts globally (especially after allowlist rules).\n**/.DS_Store\n**/__pycache__/\n**/*.cpython-*.pyc\n**/*.cpython-*.pyo\npackages/leann-backend-diskann/third_party/DiskANN/_deps/\n\n*.meta.json\n*.passages.json\n*.npy\n*.db\nbatchtest.py\ntests/__pytest_cache__/\ntests/__pycache__/\nbenchmarks/data/\n\n## multi vector\napps/multimodal/vision-based-pdf-multi-vector/multi-vector-colpali-native-weaviate.py\n\n# Ignore all PDFs (keep data exceptions above) and do not track demo PDFs\n# If you need to commit a specific demo PDF, remove this negation locally.\n# The following line used to force-add a large demo PDF; remove it to satisfy pre-commit:\n# !apps/multimodal/vision-based-pdf-multi-vector/pdfs/2004.12832v2.pdf\n!apps/multimodal/vision-based-pdf-multi-vector/fig/*\n\n# AUR build directory (Arch Linux)\nparu-bin/\nmerkle-tree-test/\ntest-code/\nlocaltestmcp/\n*.csv\n*.pickle\n" }, { "path": ".gitmodules", "content": "[submodule \"packages/leann-backend-diskann/third_party/DiskANN\"]\n\tpath = packages/leann-backend-diskann/third_party/DiskANN\n\turl = https://github.com/yichuan-w/DiskANN.git\n[submodule \"packages/leann-backend-hnsw/third_party/faiss\"]\n\tpath = packages/leann-backend-hnsw/third_party/faiss\n\turl = https://github.com/yichuan-w/faiss.git\n[submodule \"packages/leann-backend-hnsw/third_party/msgpack-c\"]\n\tpath = packages/leann-backend-hnsw/third_party/msgpack-c\n\turl = https://github.com/msgpack/msgpack-c.git\n\tbranch = cpp_master\n[submodule \"packages/leann-backend-hnsw/third_party/cppzmq\"]\n\tpath = packages/leann-backend-hnsw/third_party/cppzmq\n\turl = https://github.com/zeromq/cppzmq.git\n[submodule \"packages/leann-backend-hnsw/third_party/libzmq\"]\n\tpath = packages/leann-backend-hnsw/third_party/libzmq\n\turl = https://github.com/zeromq/libzmq.git\n[submodule \"packages/astchunk-leann\"]\n\tpath = packages/astchunk-leann\n\turl = https://github.com/yichuan-w/astchunk-leann.git\n" }, { "path": ".pre-commit-config.yaml", "content": "repos:\r\n - repo: https://github.com/pre-commit/pre-commit-hooks\r\n rev: v5.0.0\r\n hooks:\r\n - id: trailing-whitespace\r\n - id: end-of-file-fixer\r\n - id: check-yaml\r\n - id: check-added-large-files\r\n - id: check-merge-conflict\r\n - id: debug-statements\r\n\r\n - repo: https://github.com/astral-sh/ruff-pre-commit\r\n rev: v0.12.7 # Fixed version to match pyproject.toml\r\n hooks:\r\n - id: ruff\r\n args: [--fix, --exit-non-zero-on-fix]\r\n - id: ruff-format\r\n" }, { "path": ".python-version", "content": "3.11\n" }, { "path": ".vscode/extensions.json", "content": "{\n \"recommendations\": [\n \"charliermarsh.ruff\",\n ]\n}\n" }, { "path": ".vscode/settings.json", "content": "{\n \"python.defaultInterpreterPath\": \".venv/bin/python\",\n \"python.terminal.activateEnvironment\": true,\n \"[python]\": {\n \"editor.defaultFormatter\": \"charliermarsh.ruff\",\n \"editor.formatOnSave\": true,\n \"editor.codeActionsOnSave\": {\n \"source.organizeImports\": \"explicit\",\n \"source.fixAll\": \"explicit\"\n },\n \"editor.insertSpaces\": true,\n \"editor.tabSize\": 4\n },\n \"ruff.enable\": true,\n \"files.watcherExclude\": {\n \"**/.venv/**\": true,\n \"**/__pycache__/**\": true,\n \"**/*.egg-info/**\": true,\n \"**/build/**\": true,\n \"**/dist/**\": true\n }\n}\n" }, { "path": "CLAUDE.md", "content": "# CLAUDE.md\n\nThis file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.\n\n## Project Overview\n\nLEANN is a lightweight vector database and RAG (Retrieval-Augmented Generation) system that achieves 97% storage reduction compared to traditional vector databases through graph-based selective recomputation. It enables semantic search across various data sources (emails, browser history, chat history, code, documents) on a single laptop without cloud dependencies.\n\n## Build & Development Commands\n\n### Quick install (pip)\n\n```bash\npip install leann\n```\n\n### Development setup (from source)\n\n```bash\n# Install uv first (required package manager)\ncurl -LsSf https://astral.sh/uv/install.sh | sh\n\ngit submodule update --init --recursive\n\n# macOS\nbrew install libomp boost protobuf zeromq pkgconf\nuv sync\n\n# Ubuntu/Debian\nsudo apt-get install libomp-dev libboost-all-dev protobuf-compiler \\\n libabsl-dev libmkl-full-dev libaio-dev libzmq3-dev\nuv sync\n\n# Windows (requires VS 2022 Build Tools with C++ workload, vcpkg, chocolatey)\nchoco install cmake swig pkgconfiglite nuget.commandline -y\nvcpkg install zeromq:x64-windows openblas:x64-windows lapack:x64-windows boost-program-options:x64-windows protobuf:x64-windows\n# Set CMAKE_PREFIX_PATH, PKG_CONFIG_PATH, OPENBLAS_LIB to vcpkg paths (see README)\nuv sync --extra diskann\n\n# Install lint tools\nuv sync --group lint\n\n# Install test tools\nuv sync --group test\n```\n\n## Code Quality\n\n```bash\n# Format code\nruff format\n\n# Lint with auto-fix\nruff check --fix\n\n# Pre-commit hooks (install once)\npre-commit install\n\n# Run pre-commit manually\nuv run pre-commit run --all-files\n```\n\n## Architecture\n\n### Core API Layer (`packages/leann-core/src/leann/`)\n\n- `api.py`: Main APIs - `LeannBuilder`, `LeannSearcher`, `LeannChat`\n- `react_agent.py`: `ReActAgent` for multi-turn reasoning\n- `cli.py`: CLI implementation (`leann build`, `leann search`, `leann ask`)\n- `chat.py`: LLM provider integrations (OpenAI, Ollama, HuggingFace, Anthropic)\n- `embedding_compute.py`: Embedding computation (sentence-transformers, MLX, OpenAI)\n- `metadata_filter.py`: Search result filtering by metadata\n\n### Backend Layer (`packages/`)\n\n- `leann-backend-hnsw/`: Default backend using FAISS HNSW for fast in-memory search\n- `leann-backend-ivf/`: IVF backend (FAISS IndexIVFFlat + DirectMap.Hashtable) supporting in-place add/remove without rebuild\n- `leann-backend-diskann/`: DiskANN backend for larger-than-memory datasets\n- `leann-mcp/`: MCP server for Claude Code integration\n\nBackends are auto-discovered via `leann-backend-*` naming convention and registered in `registry.py`.\n\n### RAG Applications (`apps/`)\n\nExample applications demonstrating RAG on various data sources:\n- `document_rag.py`: PDF/TXT/MD documents\n- `email_rag.py`: Apple Mail\n- `browser_rag.py`: Chrome browser history\n- `wechat_rag.py`, `imessage_rag.py`: Chat history\n- `code_rag.py`: Codebase search with AST-aware chunking\n- `slack_rag.py`, `twitter_rag.py`: MCP-based live data\n\n## Key Design Patterns\n\n### Incremental Update (IVF backend)\n\nThe IVF backend supports in-place updates and deletes without rebuilding the entire index:\n- `add_vectors(index_path, embeddings, passage_ids)`: Append new vectors to an existing index.\n- `remove_ids(index_path, passage_ids)`: Remove vectors by passage ID using FAISS DirectMap.Hashtable.\n- `LeannBuilder.update_index()`: High-level API that orchestrates remove-then-add for changed files, compacts `passages.jsonl`, and updates the offset map.\n\n`leann build` is idempotent — re-running it on an existing index automatically performs an incremental update instead of a full rebuild. It detects new, modified, and removed files and applies the minimal set of changes:\n- **IVF**: Supports add, remove, and modify incrementally (remove old chunks then re-insert).\n- **HNSW** (non-compact): Supports add-only incremental updates; modified/removed files trigger a full rebuild.\n- Use `--force` / `-f` to force a full rebuild regardless.\n\n### Index Structure\n\nA LEANN index consists of:\n- `.meta.json`: Metadata (backend, embedding model, dimensions)\n- `.passages.jsonl`: Raw text chunks with metadata\n- `.passages.idx`: Offset map for fast passage lookup\n- `.index`: Backend-specific vector index\n\n### Embedding Recomputation\n\nThe core storage optimization: instead of storing embeddings, LEANN stores a pruned graph and recomputes embeddings on-demand during search via ZMQ server communication.\n\n## CLI Usage\n\n```bash\n# Build index\nleann build my-docs --docs ./documents/\n\n# Search\nleann search my-docs \"query\"\n\n# Interactive chat\nleann ask my-docs --interactive\n\n# List indexes\nleann list\n\n# Remove index\nleann remove my-docs\n```\n\n## Common Development Tasks\n\nRunning example RAG applications:\n```bash\n# Document RAG (easiest to test)\npython -m apps.document_rag --query \"What is LEANN?\"\n\n# Code RAG\npython -m apps.code_rag --repo-dir ./src --query \"How does search work?\"\n```\n\n## Python Version\n\nRequires Python 3.10+ (uses PEP 604 union syntax `X | Y`).\n\n\n\n\n# Agent Coding Guidelines\n\n## General\n- Voice input may contain typos — interpret intent, not literal text.\n- When you encounter a problem, fix it immediately and keep going until there are no more problems.\n- Do not ask about ordering or sequencing — figure it out. If something is unclear, note it and skip it; only escalate when all paths are blocked.\n- Obvious bugs: fix silently without reporting.\n- No fallbacks or compatibility shims. One correct implementation per feature — no redundancy.\n\n## Roadmap\n- Public roadmap: `docs/roadmap.md` — tracks P0/P1 priorities, completed milestones, and timeline.\n- Long-term vision: `docs/ultimate_goal.md` — the north star for where LEANN is headed.\n- Keep in sync with [GitHub issue #237](https://github.com/yichuan-w/LEANN/issues/237).\n- Welcome everyone to add more, and the craziest feature you want to put here! If people want some feature, all put there.\n\n## Changelog (for contributors)\n- Maintain `docs/CHANGELOG.md` — append-only log of major changes (new features, breaking changes, important fixes).\n- Format: `## YYYY-MM-DD: ` followed by bullet points.\n- Update the changelog when merging significant PRs or completing notable work.\n- See `docs/CONTRIBUTING.md` for full contributor workflow (conventional commits, PR process, CI).\n\n## Personal Dev Notes (gitignored)\n- `docs/dev/` is gitignored for personal development notes (TODO, progress, experiments).\n- Use `docs/dev/TODO.md` for in-progress tasks, `docs/dev/PROGRESS.md` for completed work.\n- These are private scratch space — but must follow the Self-Contained Principle below.\n\n## Documentation — Self-Contained Principle\n\nAll dev docs (`PROGRESS.md`, `STATES.md`, `EXPERIMENTS.md`, `TODO.md`) must be fully understandable from the document alone, with no reliance on conversation context or implied knowledge.\n\nRequirements:\n1. **Every technique/approach must be explained on first use.** Not \"switched to IVF backend\" — write \"switched to IVF backend (FAISS IndexIVFFlat + DirectMap.Hashtable, supports in-place add/remove without full index rebuild).\"\n2. **Never assume the reader knows any abbreviation.** On first use: full name + one-sentence explanation. E.g., \"HNSW (Hierarchical Navigable Small World — a graph-based ANN index used as LEANN's default backend).\"\n3. **Benchmark results must include full context.** Not \"recall improved to 0.95\" — write \"recall@10 improved from 0.91 to 0.95 after switching from flat chunking (512 tokens, no overlap) to AST-aware chunking (function-level splits with 64-token overlap).\"\n4. **Numbers must have reference points.** Not \"build time: 12s\" — write \"build time: 12s (vs. 45s before incremental update support, on 10k-document corpus).\"\n5. **Include the causal chain — not just conclusions.** Not \"duplicate chunks appeared after incremental build\" — write \"Duplicate chunks appeared after incremental build because `passages.jsonl` was appended without first removing stale entries for modified files. The IVF index had correct vectors (remove-then-add), but the passage store was append-only, causing the same text to appear at multiple offsets.\"\n6. **`docs/dev/STATES.md` top section maintains a glossary** of all key terms (backends, index files, chunking strategies, embedding models). Other docs reference it at the top.\n\nBad examples (forbidden):\n- \"Fixed the chunking bug\" → Which bug? What was the symptom? What was the root cause?\n- \"Improved search quality\" → By what metric? From what baseline? What change caused it?\n- \"Used nprobe=32\" → What is nprobe? Why 32? What was it before and what effect did the change have?\n\n## Doc Maintenance\n- Maintain `docs/dev/PROGRESS.md` — completed work only (with key script/log/config paths). No plans.\n- Maintain `docs/dev/TODO.md` — incomplete/in-progress/next-steps only (aim for one-command reproducibility). When done: remove from TODO, write result to PROGRESS, update STATES/EXPERIMENTS if needed.\n- Both files: **append-only, chronological order** (oldest first). Use `tail -n 80 docs/dev/PROGRESS.md` to read recent entries; increase range or grep by date/keyword if needed.\n- Keep TODO clean — either do items or remove them. Ask the user when unsure how to handle a TODO item.\n- Maintain `docs/dev/STATES.md` — tracks all currently useful state (index configs, backend choices, known limitations); does NOT grow indefinitely (delete stale entries).\n- Maintain `docs/dev/EXPERIMENTS.md` — benchmarks, A/B comparisons, parameter sweeps (recall@k, latency, storage size). Experimental content goes here, not in STATES.md.\n\n## Commits\nCommit when: (1) a complete feature is finished and tested, or (2) a destructive change is unavoidable.\n```bash\ngit add \ngit commit -m “feat: ...” # follow conventional commits\n```\n- When correcting errors: fix directly with no trace of the error.\n- If you write a correct new version of a file, delete the wrong version. No duplicate implementations.\n" }, { "path": "LICENSE", "content": "MIT License\n\nCopyright (c) 2025 LEANN Contributors\n\nPermission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \"Software\"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n" }, { "path": "README.md", "content": "

\n \"LEANN\n

\n\n

\n \n \"yichuan-w/LEANN\n \n

\n\n

\n \"Python\n \"CI\n \"Platform\"\n \"MIT\n \"MCP\n \n \"Join\n \n\n

\n\n
\n \n \"Take\n \n

\n We track zero telemetry. This survey is the ONLY way to tell us if you want
\n GPU Acceleration or More Integrations next.
\n 👉 Click here to cast your vote (2 mins)\n

\n
\n\n
\n

💬 Join our Slack community!

\n

\n We'd love for you to be part of the LEANN community!
\n 👉 Join LEANN Slack
\n If the invite link has expired or you have trouble joining, please open an issue and we'll help you get in!\n

\n
\n\n

\n The smallest vector index in the world. RAG Everything with LEANN!\n

\n\nLEANN is an innovative vector database that democratizes personal AI. Transform your laptop into a powerful RAG system that can index and search through millions of documents while using **97% less storage** than traditional solutions **without accuracy loss**.\n\n\nLEANN achieves this through *graph-based selective recomputation* with *high-degree preserving pruning*, computing embeddings on-demand instead of storing them all. [Illustration Fig →](#️-architecture--how-it-works) | [Paper →](https://arxiv.org/abs/2506.08276)\n\n**Ready to RAG Everything?** Transform your laptop into a personal AI assistant that can semantic search your **[file system](#-personal-data-manager-process-any-documents-pdf-txt-md)**, **[emails](#-your-personal-email-secretary-rag-on-apple-mail)**, **[browser history](#-time-machine-for-the-web-rag-your-entire-browser-history)**, **[chat history](#-wechat-detective-unlock-your-golden-memories)** ([WeChat](#-wechat-detective-unlock-your-golden-memories), [iMessage](#-imessage-history-your-personal-conversation-archive)), **[agent memory](#-chatgpt-chat-history-your-personal-ai-conversation-archive)** ([ChatGPT](#-chatgpt-chat-history-your-personal-ai-conversation-archive), [Claude](#-claude-chat-history-your-personal-ai-conversation-archive)), **[live data](#mcp-integration-rag-on-live-data-from-any-platform)** ([Slack](#slack-messages-search-your-team-conversations), [Twitter](#-twitter-bookmarks-your-personal-tweet-library)), **[codebase](#-claude-code-integration-transform-your-development-workflow)**\\* , or external knowledge bases (i.e., 60M documents) - all on your laptop, with zero cloud costs and complete privacy.\n\n\n\\* Claude Code only supports basic `grep`-style keyword search. **LEANN** is a drop-in **semantic search MCP service fully compatible with Claude Code**, unlocking intelligent retrieval without changing your workflow. 🔥 Check out [the easy setup →](packages/leann-mcp/README.md)\n\n\n\n## Why LEANN?\n\n

\n \"LEANN\n

\n\n> **The numbers speak for themselves:** Index 60 million text chunks in just 6GB instead of 201GB. From emails to browser history, everything fits on your laptop. [See detailed benchmarks for different applications below ↓](#-storage-comparison)\n\n\n🔒 **Privacy:** Your data never leaves your laptop. No OpenAI, no cloud, no \"terms of service\".\n\n🪶 **Lightweight:** Graph-based recomputation eliminates heavy embedding storage, while smart graph pruning and CSR format minimize graph storage overhead. Always less storage, less memory usage!\n\n📦 **Portable:** Transfer your entire knowledge base between devices (even with others) with minimal cost - your personal AI memory travels with you.\n\n📈 **Scalability:** Handle messy personal data that would crash traditional vector DBs, easily managing your growing personalized data and agent generated memory!\n\n✨ **No Accuracy Loss:** Maintain the same search quality as heavyweight solutions while using 97% less storage.\n\n## Installation\n\n### 📦 Prerequisites: Install uv\n\n[Install uv](https://docs.astral.sh/uv/getting-started/installation/#installation-methods) first if you don't have it. Typically, you can install it with:\n\n```bash\ncurl -LsSf https://astral.sh/uv/install.sh | sh\n```\n\n### 🚀 Quick Install\n\nClone the repository to access all examples and try amazing applications,\n\n```bash\ngit clone https://github.com/yichuan-w/LEANN.git leann\ncd leann\n```\n\nand install LEANN from [PyPI](https://pypi.org/project/leann/) to run them immediately:\n\n```bash\nuv venv\nsource .venv/bin/activate\nuv pip install leann\n\n# CPU-only (Linux): use the `cpu` extra (e.g. `leann[cpu]`)\n```\n\n\n\n
\n\n🔧 Build from Source (Recommended for development)\n\n\n\n\n```bash\ngit clone https://github.com/yichuan-w/LEANN.git leann\ncd leann\ngit submodule update --init --recursive\n```\n\n**macOS:**\n\nNote: DiskANN requires MacOS 13.3 or later.\n\n```bash\nbrew install libomp boost protobuf zeromq pkgconf\nuv sync --extra diskann\n```\n\n**Linux (Ubuntu/Debian):**\n\nNote: On Ubuntu 20.04, you may need to build a newer Abseil and pin Protobuf (e.g., v3.20.x) for building DiskANN. See [Issue #30](https://github.com/yichuan-w/LEANN/issues/30) for a step-by-step note.\n\nYou can manually install [Intel oneAPI MKL](https://www.intel.com/content/www/us/en/developer/tools/oneapi/onemkl.html) instead of `libmkl-full-dev` for DiskANN. You can also use `libopenblas-dev` for building HNSW only, by removing `--extra diskann` in the command below.\n\n```bash\nsudo apt-get update && sudo apt-get install -y \\\n libomp-dev libboost-all-dev protobuf-compiler libzmq3-dev \\\n pkg-config libabsl-dev libaio-dev libprotobuf-dev \\\n libmkl-full-dev\n\nuv sync --extra diskann\n```\n\n**Linux (Arch Linux):**\n\n```bash\nsudo pacman -Syu && sudo pacman -S --needed base-devel cmake pkgconf git gcc \\\n boost boost-libs protobuf abseil-cpp libaio zeromq\n\n# For MKL in DiskANN\nsudo pacman -S --needed base-devel git\ngit clone https://aur.archlinux.org/paru-bin.git\ncd paru-bin && makepkg -si\nparu -S intel-oneapi-mkl intel-oneapi-compiler\nsource /opt/intel/oneapi/setvars.sh\n\nuv sync --extra diskann\n```\n\n**Linux (RHEL / CentOS Stream / Oracle / Rocky / AlmaLinux):**\n\nSee [Issue #50](https://github.com/yichuan-w/LEANN/issues/50) for more details.\n\n```bash\nsudo dnf groupinstall -y \"Development Tools\"\nsudo dnf install -y libomp-devel boost-devel protobuf-compiler protobuf-devel \\\n abseil-cpp-devel libaio-devel zeromq-devel pkgconf-pkg-config\n\n# For MKL in DiskANN\nsudo dnf install -y intel-oneapi-mkl intel-oneapi-mkl-devel \\\n intel-oneapi-openmp || sudo dnf install -y intel-oneapi-compiler\nsource /opt/intel/oneapi/setvars.sh\n\nuv sync --extra diskann\n```\n\n**Windows:**\n\nRequires [Visual Studio 2022 Build Tools](https://visualstudio.microsoft.com/downloads/#build-tools-for-visual-studio-2022) with the **C++ desktop development** workload, and [vcpkg](https://github.com/microsoft/vcpkg).\n\n```powershell\n# Install toolchain (if not already present)\nchoco install cmake swig pkgconfiglite nuget.commandline -y\n\n# Install C++ dependencies via vcpkg\nvcpkg install zeromq:x64-windows openblas:x64-windows lapack:x64-windows `\n boost-program-options:x64-windows protobuf:x64-windows\n\n# Set environment variables (adjust VCPKG_ROOT to your vcpkg path)\n$env:CMAKE_PREFIX_PATH = \"$env:VCPKG_ROOT\\installed\\x64-windows\"\n$env:PKG_CONFIG_PATH = \"$env:VCPKG_ROOT\\installed\\x64-windows\\lib\\pkgconfig\"\n$env:PKG_CONFIG_EXECUTABLE = \"C:\\ProgramData\\chocolatey\\bin\\pkg-config.exe\"\n$env:OPENBLAS_LIB = \"$env:VCPKG_ROOT\\installed\\x64-windows\\lib\\openblas.lib\"\n$env:PATH += \";$env:VCPKG_ROOT\\installed\\x64-windows\\bin\"\n$env:PATH += \";$env:VCPKG_ROOT\\installed\\x64-windows\\tools\\protobuf\"\n\nuv sync --extra diskann\n```\n\n
\n\n\n## Quick Start\n\nOur declarative API makes RAG as easy as writing a config file.\n\nCheck out [demo.ipynb](demo.ipynb) or [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/yichuan-w/LEANN/blob/main/demo.ipynb)\n\n```python\nfrom leann import LeannBuilder, LeannSearcher, LeannChat\nfrom pathlib import Path\nINDEX_PATH = str(Path(\"./\").resolve() / \"demo.leann\")\n\n# Build an index\nbuilder = LeannBuilder(backend_name=\"hnsw\")\nbuilder.add_text(\"LEANN saves 97% storage compared to traditional vector databases.\")\nbuilder.add_text(\"Tung Tung Tung Sahur called—they need their banana‑crocodile hybrid back\")\nbuilder.build_index(INDEX_PATH)\n\n# Search\nsearcher = LeannSearcher(INDEX_PATH)\nresults = searcher.search(\"fantastical AI-generated creatures\", top_k=1)\n\n# Chat with your data\nchat = LeannChat(INDEX_PATH, llm_config={\"type\": \"hf\", \"model\": \"Qwen/Qwen3-0.6B\"})\nresponse = chat.ask(\"How much storage does LEANN save?\", top_k=1)\n```\n\n## RAG on Everything!\n\nLEANN supports RAG on various data sources including documents (`.pdf`, `.txt`, `.md`), Apple Mail, Google Search History, WeChat, ChatGPT conversations, Claude conversations, iMessage conversations, and **live data from any platform through MCP (Model Context Protocol) servers** - including Slack, Twitter, and more.\n\n\n\n### Generation Model Setup\n\n#### LLM Backend\n\nLEANN supports many LLM providers for text generation (HuggingFace, Ollama, Anthropic, and Any OpenAI compatible API).\n\n\n
\n🔑 OpenAI API Setup (Default)\n\nSet your OpenAI API key as an environment variable:\n\n```bash\nexport OPENAI_API_KEY=\"your-api-key-here\"\n```\n\nMake sure to use `--llm openai` flag when using the CLI.\nYou can also specify the model name with `--llm-model ` flag.\n\n
\n\n
\n🛠️ Supported LLM & Embedding Providers (via OpenAI Compatibility)\n\nThanks to the widespread adoption of the OpenAI API format, LEANN is compatible out-of-the-box with a vast array of LLM and embedding providers. Simply set the `OPENAI_BASE_URL` and `OPENAI_API_KEY` environment variables to connect to your preferred service.\n\n```sh\nexport OPENAI_API_KEY=\"xxx\"\nexport OPENAI_BASE_URL=\"http://localhost:1234/v1\" # base url of the provider\n```\n\nTo use OpenAI compatible endpoint with the CLI interface:\n\nIf you are using it for text generation, make sure to use `--llm openai` flag and specify the model name with `--llm-model ` flag.\n\nIf you are using it for embedding, set the `--embedding-mode openai` flag and specify the model name with `--embedding-model `.\n\n-----\n\n\nBelow is a list of base URLs for common providers to get you started.\n\n\n### 🖥️ Local Inference Engines (Recommended for full privacy)\n\n| Provider | Sample Base URL |\n| ---------------- | --------------------------- |\n| **Ollama** | `http://localhost:11434/v1` |\n| **LM Studio** | `http://localhost:1234/v1` |\n| **vLLM** | `http://localhost:8000/v1` |\n| **llama.cpp** | `http://localhost:8080/v1` |\n| **SGLang** | `http://localhost:30000/v1` |\n| **LiteLLM** | `http://localhost:4000` |\n\n-----\n\n### ☁️ Cloud Providers\n\n> **🚨 A Note on Privacy:** Before choosing a cloud provider, carefully review their privacy and data retention policies. Depending on their terms, your data may be used for their own purposes, including but not limited to human reviews and model training, which can lead to serious consequences if not handled properly.\n\n\n| Provider | Base URL |\n| ---------------- | ---------------------------------------------------------- |\n| **OpenAI** | `https://api.openai.com/v1` |\n| **OpenRouter** | `https://openrouter.ai/api/v1` |\n| **Gemini** | `https://generativelanguage.googleapis.com/v1beta/openai/` |\n| **x.AI (Grok)** | `https://api.x.ai/v1` |\n| **Groq AI** | `https://api.groq.com/openai/v1` |\n| **DeepSeek** | `https://api.deepseek.com/v1` |\n| **SiliconFlow** | `https://api.siliconflow.cn/v1` |\n| **Zhipu (BigModel)** | `https://open.bigmodel.cn/api/paas/v4/` |\n| **Mistral AI** | `https://api.mistral.ai/v1` |\n| **Anthropic** | `https://api.anthropic.com/v1` |\n| **Jina AI** (Embeddings) | `https://api.jina.ai/v1` |\n\n> **💡 Tip: Separate Embedding Provider**\n>\n> To use a different provider for embeddings (e.g., Jina AI) while using another for LLM, use `--embedding-api-base` and `--embedding-api-key`:\n> ```bash\n> leann build my-index --docs ./docs \\\n> --embedding-mode openai \\\n> --embedding-model jina-embeddings-v3 \\\n> --embedding-api-base https://api.jina.ai/v1 \\\n> --embedding-api-key $JINA_API_KEY\n> ```\n\nIf your provider isn't on this list, don't worry! Check their documentation for an OpenAI-compatible endpoint—chances are, it's OpenAI Compatible too!\n\n
\n\n
\n🔧 Ollama Setup (Recommended for full privacy)\n\n**macOS:**\n\nFirst, [download Ollama for macOS](https://ollama.com/download/mac).\n\n```bash\n# Pull a lightweight model (recommended for consumer hardware)\nollama pull llama3.2:1b\n```\n\n**Linux:**\n\n```bash\n# Install Ollama\ncurl -fsSL https://ollama.ai/install.sh | sh\n\n# Start Ollama service manually\nollama serve &\n\n# Pull a lightweight model (recommended for consumer hardware)\nollama pull llama3.2:1b\n```\n\n
\n\n\n## ⭐ Flexible Configuration\n\nLEANN provides flexible parameters for embedding models, search strategies, and data processing to fit your specific needs.\n\n📚 **Need configuration best practices?** Check our [Configuration Guide](docs/configuration-guide.md) for detailed optimization tips, model selection advice, and solutions to common issues like slow embeddings or poor search quality.\n\n
\n📋 Click to expand: Common Parameters (Available in All Examples)\n\nAll RAG examples share these common parameters. **Interactive mode** is available in all examples - simply run without `--query` to start a continuous Q&A session where you can ask multiple questions. Type 'quit' to exit.\n\n```bash\n# Environment Variables (GPU Device Selection)\nLEANN_EMBEDDING_DEVICE # GPU for embedding model (e.g., cuda:0, cuda:1, cpu)\nLEANN_LLM_DEVICE # GPU for HFChat LLM (e.g., cuda:1, or \"cuda\" for multi-GPU auto)\n\n# Core Parameters (General preprocessing for all examples)\n--index-dir DIR # Directory to store the index (default: current directory)\n--query \"YOUR QUESTION\" # Single query mode. Omit for interactive chat (type 'quit' to exit), and now you can play with your index interactively\n--max-items N # Limit data preprocessing (default: -1, process all data)\n--force-rebuild # Force rebuild index even if it exists\n\n# Embedding Parameters\n--embedding-model MODEL # e.g., facebook/contriever, text-embedding-3-small, mlx-community/Qwen3-Embedding-0.6B-8bit or nomic-embed-text\n--embedding-mode MODE # sentence-transformers, openai, mlx, or ollama\n\n# LLM Parameters (Text generation models)\n--llm TYPE # LLM backend: openai, ollama, hf, or anthropic (default: openai)\n--llm-model MODEL # Model name (default: gpt-4o) e.g., gpt-4o-mini, llama3.2:1b, Qwen/Qwen2.5-1.5B-Instruct\n--thinking-budget LEVEL # Thinking budget for reasoning models: low/medium/high (supported by o3, o3-mini, GPT-Oss:20b, and other reasoning models)\n\n# Search Parameters\n--top-k N # Number of results to retrieve (default: 20)\n--search-complexity N # Search complexity for graph traversal (default: 32)\n\n# Chunking Parameters\n--chunk-size N # Size of text chunks (default varies by source: 256 for most, 192 for WeChat)\n--chunk-overlap N # Overlap between chunks (default varies: 25-128 depending on source)\n\n# Index Building Parameters\n--backend-name NAME # Backend to use: hnsw or diskann (default: hnsw)\n--graph-degree N # Graph degree for index construction (default: 32)\n--build-complexity N # Build complexity for index construction (default: 64)\n--compact / --no-compact # Use compact storage (default: true). Must be `no-compact` for `no-recompute` build.\n--recompute / --no-recompute # Enable/disable embedding recomputation (default: enabled). Should not do a `no-recompute` search in a `recompute` build.\n```\n\n
\n\n### 📄 Personal Data Manager: Process Any Documents (`.pdf`, `.txt`, `.md`)!\n\nAsk questions directly about your personal PDFs, documents, and any directory containing your files!\n\n

\n \"LEANN\n

\n\nThe example below asks a question about summarizing our paper (uses default data in `data/`, which is a directory with diverse data sources: two papers, Pride and Prejudice, and a Technical report about LLM in Huawei in Chinese), and this is the **easiest example** to run here:\n\n```bash\nsource .venv/bin/activate # Don't forget to activate the virtual environment\npython -m apps.document_rag --query \"What are the main techniques LEANN explores?\"\n```\n\n
\n📋 Click to expand: Document-Specific Arguments\n\n#### Parameters\n```bash\n--data-dir DIR # Directory containing documents to process (default: data)\n--file-types .ext .ext # Filter by specific file types (optional - all LlamaIndex supported types if omitted)\n```\n\n#### Example Commands\n```bash\n# Process all documents with larger chunks for academic papers\npython -m apps.document_rag --data-dir \"~/Documents/Papers\" --chunk-size 1024\n\n# Filter only markdown and Python files with smaller chunks\npython -m apps.document_rag --data-dir \"./docs\" --chunk-size 256 --file-types .md .py\n\n# Enable AST-aware chunking for code files\npython -m apps.document_rag --enable-code-chunking --data-dir \"./my_project\"\n\n# Or use the specialized code RAG for better code understanding\npython -m apps.code_rag --repo-dir \"./my_codebase\" --query \"How does authentication work?\"\n```\n\n
\n\n### 🎨 ColQwen: Multimodal PDF Retrieval with Vision-Language Models\n\nSearch through PDFs using both text and visual understanding with ColQwen2/ColPali models. Perfect for research papers, technical documents, and any PDFs with complex layouts, figures, or diagrams.\n\n> **🍎 Mac Users**: ColQwen is optimized for Apple Silicon with MPS acceleration for faster inference!\n\n```bash\n# Build index from PDFs\npython -m apps.colqwen_rag build --pdfs ./my_papers/ --index research_papers\n\n# Search with text queries\npython -m apps.colqwen_rag search research_papers \"How does attention mechanism work?\"\n\n# Interactive Q&A\npython -m apps.colqwen_rag ask research_papers --interactive\n```\n\n
\n📋 Click to expand: ColQwen Setup & Usage\n\n#### Prerequisites\n```bash\n# Install dependencies\nuv pip install colpali_engine pdf2image pillow matplotlib qwen_vl_utils einops seaborn\nbrew install poppler # macOS only, for PDF processing\n```\n\n#### Build Index\n```bash\npython -m apps.colqwen_rag build \\\n --pdfs ./pdf_directory/ \\\n --index my_index \\\n --model colqwen2 # or colpali\n```\n\n#### Search\n```bash\npython -m apps.colqwen_rag search my_index \"your question here\" --top-k 5\n```\n\n#### Models\n- **ColQwen2** (`colqwen2`): Latest vision-language model with improved performance\n- **ColPali** (`colpali`): Proven multimodal retriever\n\nFor detailed usage, see the [ColQwen Guide](docs/COLQWEN_GUIDE.md).\n\n
\n\n### 📧 Your Personal Email Secretary: RAG on Apple Mail!\n\n> **Note:** The examples below currently support macOS only. Windows support coming soon.\n\n\n

\n \"LEANN\n

\n\nBefore running the example below, you need to grant full disk access to your terminal/VS Code in System Preferences → Privacy & Security → Full Disk Access.\n\n```bash\npython -m apps.email_rag --query \"What's the food I ordered by DoorDash or Uber Eats mostly?\"\n```\n**780K email chunks → 78MB storage.** Finally, search your email like you search Google.\n\n
\n📋 Click to expand: Email-Specific Arguments\n\n#### Parameters\n```bash\n--mail-path PATH # Path to specific mail directory (auto-detects if omitted)\n--include-html # Include HTML content in processing (useful for newsletters)\n```\n\n#### Example Commands\n```bash\n# Search work emails from a specific account\npython -m apps.email_rag --mail-path \"~/Library/Mail/V10/WORK_ACCOUNT\"\n\n# Find all receipts and order confirmations (includes HTML)\npython -m apps.email_rag --query \"receipt order confirmation invoice\" --include-html\n```\n\n
\n\n
\n📋 Click to expand: Example queries you can try\n\nOnce the index is built, you can ask questions like:\n- \"Find emails from my boss about deadlines\"\n- \"What did John say about the project timeline?\"\n- \"Show me emails about travel expenses\"\n
\n\n### 🔍 Time Machine for the Web: RAG Your Entire Chrome Browser History!\n\n

\n \"LEANN\n

\n\n```bash\npython -m apps.browser_rag --query \"Tell me my browser history about machine learning?\"\n```\n**38K browser entries → 6MB storage.** Your browser history becomes your personal search engine.\n\n
\n📋 Click to expand: Browser-Specific Arguments\n\n#### Parameters\n```bash\n--chrome-profile PATH # Path to Chrome profile directory (auto-detects if omitted)\n```\n\n#### Example Commands\n```bash\n# Search academic research from your browsing history\npython -m apps.browser_rag --query \"arxiv papers machine learning transformer architecture\"\n\n# Track competitor analysis across work profile\npython -m apps.browser_rag --chrome-profile \"~/Library/Application Support/Google/Chrome/Work Profile\" --max-items 5000\n```\n\n
\n\n
\n📋 Click to expand: How to find your Chrome profile\n\nThe default Chrome profile path is configured for a typical macOS setup. If you need to find your specific Chrome profile:\n\n1. Open Terminal\n2. Run: `ls ~/Library/Application\\ Support/Google/Chrome/`\n3. Look for folders like \"Default\", \"Profile 1\", \"Profile 2\", etc.\n4. Use the full path as your `--chrome-profile` argument\n\n**Common Chrome profile locations:**\n- macOS: `~/Library/Application Support/Google/Chrome/Default`\n- Linux: `~/.config/google-chrome/Default`\n\n
\n\n
\n💬 Click to expand: Example queries you can try\n\nOnce the index is built, you can ask questions like:\n\n- \"What websites did I visit about machine learning?\"\n- \"Find my search history about programming\"\n- \"What YouTube videos did I watch recently?\"\n- \"Show me websites I visited about travel planning\"\n\n
\n\n### 💬 WeChat Detective: Unlock Your Golden Memories!\n\n

\n \"LEANN\n

\n\n```bash\npython -m apps.wechat_rag --query \"Show me all group chats about weekend plans\"\n```\n**400K messages → 64MB storage** Search years of chat history in any language.\n\n\n
\n🔧 Click to expand: Installation Requirements\n\nFirst, you need to install the [WeChat exporter](https://github.com/sunnyyoung/WeChatTweak-CLI),\n\n```bash\nbrew install sunnyyoung/repo/wechattweak-cli\n```\n\nor install it manually (if you have issues with Homebrew):\n\n```bash\nsudo packages/wechat-exporter/wechattweak-cli install\n```\n\n**Troubleshooting:**\n- **Installation issues**: Check the [WeChatTweak-CLI issues page](https://github.com/sunnyyoung/WeChatTweak-CLI/issues/41)\n- **Export errors**: If you encounter the error below, try restarting WeChat\n ```bash\n Failed to export WeChat data. Please ensure WeChat is running and WeChatTweak is installed.\n Failed to find or export WeChat data. Exiting.\n ```\n
\n\n
\n📋 Click to expand: WeChat-Specific Arguments\n\n#### Parameters\n```bash\n--export-dir DIR # Directory to store exported WeChat data (default: wechat_export_direct)\n--force-export # Force re-export even if data exists\n```\n\n#### Example Commands\n```bash\n# Search for travel plans discussed in group chats\npython -m apps.wechat_rag --query \"travel plans\" --max-items 10000\n\n# Re-export and search recent chats (useful after new messages)\npython -m apps.wechat_rag --force-export --query \"work schedule\"\n```\n\n
\n\n
\n💬 Click to expand: Example queries you can try\n\nOnce the index is built, you can ask questions like:\n\n- \"我想买魔术师约翰逊的球衣，给我一些对应聊天记录?\" (Chinese: Show me chat records about buying Magic Johnson's jersey)\n\n
\n\n### 🤖 ChatGPT Chat History: Your Personal AI Conversation Archive!\n\nTransform your ChatGPT conversations into a searchable knowledge base! Search through all your ChatGPT discussions about coding, research, brainstorming, and more.\n\n```bash\npython -m apps.chatgpt_rag --export-path chatgpt_export.html --query \"How do I create a list in Python?\"\n```\n\n**Unlock your AI conversation history.** Never lose track of valuable insights from your ChatGPT discussions again.\n\n
\n📋 Click to expand: How to Export ChatGPT Data\n\n**Step-by-step export process:**\n\n1. **Sign in to ChatGPT**\n2. **Click your profile icon** in the top right corner\n3. **Navigate to Settings** → **Data Controls**\n4. **Click \"Export\"** under Export Data\n5. **Confirm the export** request\n6. **Download the ZIP file** from the email link (expires in 24 hours)\n7. **Extract or use directly** with LEANN\n\n**Supported formats:**\n- `.html` files from ChatGPT exports\n- `.zip` archives from ChatGPT\n- Directories with multiple export files\n\n
\n\n
\n📋 Click to expand: ChatGPT-Specific Arguments\n\n#### Parameters\n```bash\n--export-path PATH # Path to ChatGPT export file (.html/.zip) or directory (default: ./chatgpt_export)\n--separate-messages # Process each message separately instead of concatenated conversations\n--chunk-size N # Text chunk size (default: 512)\n--chunk-overlap N # Overlap between chunks (default: 128)\n```\n\n#### Example Commands\n```bash\n# Basic usage with HTML export\npython -m apps.chatgpt_rag --export-path conversations.html\n\n# Process ZIP archive from ChatGPT\npython -m apps.chatgpt_rag --export-path chatgpt_export.zip\n\n# Search with specific query\npython -m apps.chatgpt_rag --export-path chatgpt_data.html --query \"Python programming help\"\n\n# Process individual messages for fine-grained search\npython -m apps.chatgpt_rag --separate-messages --export-path chatgpt_export.html\n\n# Process directory containing multiple exports\npython -m apps.chatgpt_rag --export-path ./chatgpt_exports/ --max-items 1000\n```\n\n
\n\n
\n💡 Click to expand: Example queries you can try\n\nOnce your ChatGPT conversations are indexed, you can search with queries like:\n- \"What did I ask ChatGPT about Python programming?\"\n- \"Show me conversations about machine learning algorithms\"\n- \"Find discussions about web development frameworks\"\n- \"What coding advice did ChatGPT give me?\"\n- \"Search for conversations about debugging techniques\"\n- \"Find ChatGPT's recommendations for learning resources\"\n\n
\n\n### 🤖 Claude Chat History: Your Personal AI Conversation Archive!\n\nTransform your Claude conversations into a searchable knowledge base! Search through all your Claude discussions about coding, research, brainstorming, and more.\n\n```bash\npython -m apps.claude_rag --export-path claude_export.json --query \"What did I ask about Python dictionaries?\"\n```\n\n**Unlock your AI conversation history.** Never lose track of valuable insights from your Claude discussions again.\n\n
\n📋 Click to expand: How to Export Claude Data\n\n**Step-by-step export process:**\n\n1. **Open Claude** in your browser\n2. **Navigate to Settings** (look for gear icon or settings menu)\n3. **Find Export/Download** options in your account settings\n4. **Download conversation data** (usually in JSON format)\n5. **Place the file** in your project directory\n\n*Note: Claude export methods may vary depending on the interface you're using. Check Claude's help documentation for the most current export instructions.*\n\n**Supported formats:**\n- `.json` files (recommended)\n- `.zip` archives containing JSON data\n- Directories with multiple export files\n\n
\n\n
\n📋 Click to expand: Claude-Specific Arguments\n\n#### Parameters\n```bash\n--export-path PATH # Path to Claude export file (.json/.zip) or directory (default: ./claude_export)\n--separate-messages # Process each message separately instead of concatenated conversations\n--chunk-size N # Text chunk size (default: 512)\n--chunk-overlap N # Overlap between chunks (default: 128)\n```\n\n#### Example Commands\n```bash\n# Basic usage with JSON export\npython -m apps.claude_rag --export-path my_claude_conversations.json\n\n# Process ZIP archive from Claude\npython -m apps.claude_rag --export-path claude_export.zip\n\n# Search with specific query\npython -m apps.claude_rag --export-path claude_data.json --query \"machine learning advice\"\n\n# Process individual messages for fine-grained search\npython -m apps.claude_rag --separate-messages --export-path claude_export.json\n\n# Process directory containing multiple exports\npython -m apps.claude_rag --export-path ./claude_exports/ --max-items 1000\n```\n\n
\n\n
\n💡 Click to expand: Example queries you can try\n\nOnce your Claude conversations are indexed, you can search with queries like:\n- \"What did I ask Claude about Python programming?\"\n- \"Show me conversations about machine learning algorithms\"\n- \"Find discussions about software architecture patterns\"\n- \"What debugging advice did Claude give me?\"\n- \"Search for conversations about data structures\"\n- \"Find Claude's recommendations for learning resources\"\n\n
\n\n### 💬 iMessage History: Your Personal Conversation Archive!\n\nTransform your iMessage conversations into a searchable knowledge base! Search through all your text messages, group chats, and conversations with friends, family, and colleagues.\n\n```bash\npython -m apps.imessage_rag --query \"What did we discuss about the weekend plans?\"\n```\n\n**Unlock your message history.** Never lose track of important conversations, shared links, or memorable moments from your iMessage history.\n\n
\n📋 Click to expand: How to Access iMessage Data\n\n**iMessage data location:**\n\niMessage conversations are stored in a SQLite database on your Mac at:\n```\n~/Library/Messages/chat.db\n```\n\n**Important setup requirements:**\n\n1. **Grant Full Disk Access** to your terminal or IDE:\n - Open **System Preferences** → **Security & Privacy** → **Privacy**\n - Select **Full Disk Access** from the left sidebar\n - Click the **+** button and add your terminal app (Terminal, iTerm2) or IDE (VS Code, etc.)\n - Restart your terminal/IDE after granting access\n\n2. **Alternative: Use a backup database**\n - If you have Time Machine backups or manual copies of the database\n - Use `--db-path` to specify a custom location\n\n**Supported formats:**\n- Direct access to `~/Library/Messages/chat.db` (default)\n- Custom database path with `--db-path`\n- Works with backup copies of the database\n\n
\n\n
\n📋 Click to expand: iMessage-Specific Arguments\n\n#### Parameters\n```bash\n--db-path PATH # Path to chat.db file (default: ~/Library/Messages/chat.db)\n--concatenate-conversations # Group messages by conversation (default: True)\n--no-concatenate-conversations # Process each message individually\n--chunk-size N # Text chunk size (default: 1000)\n--chunk-overlap N # Overlap between chunks (default: 200)\n```\n\n#### Example Commands\n```bash\n# Basic usage (requires Full Disk Access)\npython -m apps.imessage_rag\n\n# Search with specific query\npython -m apps.imessage_rag --query \"family dinner plans\"\n\n# Use custom database path\npython -m apps.imessage_rag --db-path /path/to/backup/chat.db\n\n# Process individual messages instead of conversations\npython -m apps.imessage_rag --no-concatenate-conversations\n\n# Limit processing for testing\npython -m apps.imessage_rag --max-items 100 --query \"weekend\"\n```\n\n
\n\n
\n💡 Click to expand: Example queries you can try\n\nOnce your iMessage conversations are indexed, you can search with queries like:\n- \"What did we discuss about vacation plans?\"\n- \"Find messages about restaurant recommendations\"\n- \"Show me conversations with John about the project\"\n- \"Search for shared links about technology\"\n- \"Find group chat discussions about weekend events\"\n- \"What did mom say about the family gathering?\"\n\n
\n\n### MCP Integration: RAG on Live Data from Any Platform\n\nConnect to live data sources through the Model Context Protocol (MCP). LEANN now supports real-time RAG on platforms like Slack, Twitter, and more through standardized MCP servers.\n\n**Key Benefits:**\n- **Live Data Access**: Fetch real-time data without manual exports\n- **Standardized Protocol**: Use any MCP-compatible server\n- **Easy Extension**: Add new platforms with minimal code\n- **Secure Access**: MCP servers handle authentication\n\n#### 💬 Slack Messages: Search Your Team Conversations\n\nTransform your Slack workspace into a searchable knowledge base! Find discussions, decisions, and shared knowledge across all your channels.\n\n```bash\n# Test MCP server connection\npython -m apps.slack_rag --mcp-server \"slack-mcp-server\" --test-connection\n\n# Index and search Slack messages\npython -m apps.slack_rag \\\n --mcp-server \"slack-mcp-server\" \\\n --workspace-name \"my-team\" \\\n --channels general dev-team random \\\n --query \"What did we decide about the product launch?\"\n```\n\n**📖 Comprehensive Setup Guide**: For detailed setup instructions, troubleshooting common issues (like \"users cache is not ready yet\"), and advanced configuration options, see our [**Slack Setup Guide**](docs/slack-setup-guide.md).\n\n**Quick Setup:**\n1. Install a Slack MCP server (e.g., `npm install -g slack-mcp-server`)\n2. Create a Slack App and get API credentials (see detailed guide above)\n3. Set environment variables:\n ```bash\n export SLACK_BOT_TOKEN=\"xoxb-your-bot-token\"\n export SLACK_APP_TOKEN=\"xapp-your-app-token\" # Optional\n ```\n4. Test connection with `--test-connection` flag\n\n**Arguments:**\n- `--mcp-server`: Command to start the Slack MCP server\n- `--workspace-name`: Slack workspace name for organization\n- `--channels`: Specific channels to index (optional)\n- `--concatenate-conversations`: Group messages by channel (default: true)\n- `--max-messages-per-channel`: Limit messages per channel (default: 100)\n- `--max-retries`: Maximum retries for cache sync issues (default: 5)\n- `--retry-delay`: Initial delay between retries in seconds (default: 2.0)\n\n#### 🐦 Twitter Bookmarks: Your Personal Tweet Library\n\nSearch through your Twitter bookmarks! Find that perfect article, thread, or insight you saved for later.\n\n```bash\n# Test MCP server connection\npython -m apps.twitter_rag --mcp-server \"twitter-mcp-server\" --test-connection\n\n# Index and search Twitter bookmarks\npython -m apps.twitter_rag \\\n --mcp-server \"twitter-mcp-server\" \\\n --max-bookmarks 1000 \\\n --query \"What AI articles did I bookmark about machine learning?\"\n```\n\n**Setup Requirements:**\n1. Install a Twitter MCP server (e.g., `npm install -g twitter-mcp-server`)\n2. Get Twitter API credentials:\n - Apply for a Twitter Developer Account at [developer.twitter.com](https://developer.twitter.com)\n - Create a new app in the Twitter Developer Portal\n - Generate API keys and access tokens with \"Read\" permissions\n - For bookmarks access, you may need Twitter API v2 with appropriate scopes\n ```bash\n export TWITTER_API_KEY=\"your-api-key\"\n export TWITTER_API_SECRET=\"your-api-secret\"\n export TWITTER_ACCESS_TOKEN=\"your-access-token\"\n export TWITTER_ACCESS_TOKEN_SECRET=\"your-access-token-secret\"\n ```\n3. Test connection with `--test-connection` flag\n\n**Arguments:**\n- `--mcp-server`: Command to start the Twitter MCP server\n- `--username`: Filter bookmarks by username (optional)\n- `--max-bookmarks`: Maximum bookmarks to fetch (default: 1000)\n- `--no-tweet-content`: Exclude tweet content, only metadata\n- `--no-metadata`: Exclude engagement metadata\n\n\n\n
\n💡 Click to expand: Example queries you can try\n\n**Slack Queries:**\n- \"What did the team discuss about the project deadline?\"\n- \"Find messages about the new feature launch\"\n- \"Show me conversations about budget planning\"\n- \"What decisions were made in the dev-team channel?\"\n\n**Twitter Queries:**\n- \"What AI articles did I bookmark last month?\"\n- \"Find tweets about machine learning techniques\"\n- \"Show me bookmarked threads about startup advice\"\n- \"What Python tutorials did I save?\"\n\n
\n🔧 Using MCP with CLI Commands\n\n**Want to use MCP data with regular LEANN CLI?** You can combine MCP apps with CLI commands:\n\n```bash\n# Step 1: Use MCP app to fetch and index data\npython -m apps.slack_rag --mcp-server \"slack-mcp-server\" --workspace-name \"my-team\"\n\n# Step 2: The data is now indexed and available via CLI\nleann search slack_messages \"project deadline\"\nleann ask slack_messages \"What decisions were made about the product launch?\"\n\n# Same for Twitter bookmarks\npython -m apps.twitter_rag --mcp-server \"twitter-mcp-server\"\nleann search twitter_bookmarks \"machine learning articles\"\n```\n\n**MCP vs Manual Export:**\n- **MCP**: Live data, automatic updates, requires server setup\n- **Manual Export**: One-time setup, works offline, requires manual data export\n\n\n\n
\n🔧 Adding New MCP Platforms\n\nWant to add support for other platforms? LEANN's MCP integration is designed for easy extension:\n\n1. **Find or create an MCP server** for your platform\n2. **Create a reader class** following the pattern in `apps/slack_data/slack_mcp_reader.py`\n3. **Create a RAG application** following the pattern in `apps/slack_rag.py`\n4. **Test and contribute** back to the community!\n\n**Popular MCP servers to explore:**\n- GitHub repositories and issues\n- Discord messages\n- Notion pages\n- Google Drive documents\n- And many more in the MCP ecosystem!\n\n
\n\n### 🚀 Claude Code Integration: Transform Your Development Workflow!\n\n
\nAST‑Aware Code Chunking\n\nLEANN features intelligent code chunking that preserves semantic boundaries (functions, classes, methods) for Python, Java, C#, and TypeScript, improving code understanding compared to text-based chunking.\n\n📖 Read the [AST Chunking Guide →](docs/ast_chunking_guide.md)\n\n
\n\n**The future of code assistance is here.** Transform your development workflow with LEANN's native MCP integration for Claude Code. Index your entire codebase and get intelligent code assistance directly in your IDE.\n\n**Key features:**\n- 🔍 **Semantic code search** across your entire project, fully local index and lightweight\n- 🧠 **AST-aware chunking** preserves code structure (functions, classes)\n- 📚 **Context-aware assistance** for debugging and development\n- 🚀 **Zero-config setup** with automatic language detection\n\n```bash\n# Install LEANN globally for MCP integration\nuv tool install leann-core --with leann\nclaude mcp add --scope user leann-server -- leann_mcp\n# Setup is automatic - just start using Claude Code!\n```\nTry our fully agentic pipeline with auto query rewriting, semantic search planning, and more:\n\n![LEANN MCP Integration](assets/mcp_leann.png)\n\n**🔥 Ready to supercharge your coding?** [Complete Setup Guide →](packages/leann-mcp/README.md)\n\n## Command Line Interface\n\nLEANN includes a powerful CLI for document processing and search. Perfect for quick document indexing and interactive chat.\n\n### Installation\n\nIf you followed the Quick Start, `leann` is already installed in your virtual environment:\n```bash\nsource .venv/bin/activate\nleann --help\n```\n\n**To make it globally available:**\n```bash\n# Install the LEANN CLI globally using uv tool\nuv tool install leann-core --with leann\n\n\n# Now you can use leann from anywhere without activating venv\nleann --help\n```\n\n> **Note**: Global installation is required for Claude Code integration. The `leann_mcp` server depends on the globally available `leann` command.\n\n\n\n### Usage Examples\n\n```bash\n# build from a specific directory, and my_docs is the index name(Here you can also build from multiple dict or multiple files)\nleann build my-docs --docs ./your_documents\n\n# Search your documents\nleann search my-docs \"machine learning concepts\"\n\n# Interactive chat with your documents\nleann ask my-docs --interactive\n\n# Ask a single question (non-interactive)\nleann ask my-docs \"Where are prompts configured?\"\n\n# Detect file changes since last build/watch checkpoint\nleann watch my-docs\n\n# List all your indexes\nleann list\n\n# Remove an index\nleann remove my-docs\n```\n\n**Key CLI features:**\n- Auto-detects document formats (PDF, TXT, MD, DOCX, PPTX + code files)\n- **🧠 AST-aware chunking** for Python, Java, C#, TypeScript files\n- Smart text chunking with overlap for all other content\n- **📂 File change detection** via Merkle tree snapshots (`leann watch`)\n- Multiple LLM providers (Ollama, OpenAI, HuggingFace)\n- Organized index storage in `.leann/indexes/` (project-local)\n- Support for advanced search parameters\n\n
\n📋 Click to expand: Complete CLI Reference\n\nYou can use `leann --help`, or `leann build --help`, `leann search --help`, `leann watch --help`, `leann ask --help`, `leann list --help`, `leann remove --help` to get the complete CLI reference.\n\n**Build Command:**\n```bash\nleann build INDEX_NAME --docs DIRECTORY|FILE [DIRECTORY|FILE ...] [OPTIONS]\n\nOptions:\n --backend {hnsw,diskann} Backend to use (default: hnsw)\n --embedding-model MODEL Embedding model (default: facebook/contriever)\n --graph-degree N Graph degree (default: 32)\n --complexity N Build complexity (default: 64)\n --force Force rebuild existing index\n --compact / --no-compact Use compact storage (default: true). Must be `no-compact` for `no-recompute` build.\n --recompute / --no-recompute Enable recomputation (default: true)\n```\n\n**Search Command:**\n```bash\nleann search INDEX_NAME QUERY [OPTIONS]\n\nOptions:\n --top-k N Number of results (default: 5)\n --complexity N Search complexity (default: 64)\n --recompute / --no-recompute Enable/disable embedding recomputation (default: enabled). Should not do a `no-recompute` search in a `recompute` build.\n --pruning-strategy {global,local,proportional}\n```\n\n**Watch Command:**\n```bash\nleann watch INDEX_NAME\n\n# Compares the current file system state against the last checkpoint (Merkle tree snapshot)\n# and reports which files have been added, removed, or modified, along with their chunk IDs.\n#\n# - Automatically saves a new checkpoint after detecting changes\n# - Each subsequent run compares against the most recent checkpoint\n# - File change detection uses SHA-256 content hashing via a Merkle tree\n#\n# Example output:\n# === Changes since last checkpoint ===\n# modified (1):\n# - /path/to/file.py\n# chunks: 42, 43, 44\n```\n\n**Ask Command:**\n```bash\nleann ask INDEX_NAME [OPTIONS]\n\nOptions:\n --llm {ollama,openai,hf,anthropic} LLM provider (default: ollama)\n --model MODEL Model name (default: qwen3:8b)\n --interactive Interactive chat mode\n --top-k N Retrieval count (default: 20)\n```\n\n**List Command:**\n```bash\nleann list\n\n# Lists all indexes across all projects with status indicators:\n# ✅ - Index is complete and ready to use\n# ❌ - Index is incomplete or corrupted\n# 📁 - CLI-created index (in .leann/indexes/)\n# 📄 - App-created index (*.leann.meta.json files)\n```\n\n**Remove Command:**\n```bash\nleann remove INDEX_NAME [OPTIONS]\n\nOptions:\n --force, -f Force removal without confirmation\n\n# Smart removal: automatically finds and safely removes indexes\n# - Shows all matching indexes across projects\n# - Requires confirmation for cross-project removal\n# - Interactive selection when multiple matches found\n# - Supports both CLI and app-created indexes\n```\n\n
\n\n## 🚀 Advanced Features\n\n### 🎯 Metadata Filtering\n\nLEANN supports a simple metadata filtering system to enable sophisticated use cases like document filtering by date/type, code search by file extension, and content management based on custom criteria.\n\n```python\n# Add metadata during indexing\nbuilder.add_text(\n \"def authenticate_user(token): ...\",\n metadata={\"file_extension\": \".py\", \"lines_of_code\": 25}\n)\n\n# Search with filters\nresults = searcher.search(\n query=\"authentication function\",\n metadata_filters={\n \"file_extension\": {\"==\": \".py\"},\n \"lines_of_code\": {\"<\": 100}\n }\n)\n```\n\n**Supported operators**: `==`, `!=`, `<`, `<=`, `>`, `>=`, `in`, `not_in`, `contains`, `starts_with`, `ends_with`, `is_true`, `is_false`\n\n📖 **[Complete Metadata filtering guide →](docs/metadata_filtering.md)**\n\n### 🔍 Grep Search\n\nFor exact text matching instead of semantic search, use the `use_grep` parameter:\n\n```python\n# Exact text search\nresults = searcher.search(\"banana‑crocodile\", use_grep=True, top_k=1)\n```\n\n**Use cases**: Finding specific code patterns, error messages, function names, or exact phrases where semantic similarity isn't needed.\n\n📖 **[Complete grep search guide →](docs/grep_search.md)**\n\n## 🏗️ Architecture & How It Works\n\n

\n \"LEANN\n

\n\n**The magic:** Most vector DBs store every single embedding (expensive). LEANN stores a pruned graph structure (cheap) and recomputes embeddings only when needed (fast).\n\n**Core techniques:**\n- **Graph-based selective recomputation:** Only compute embeddings for nodes in the search path\n- **High-degree preserving pruning:** Keep important \"hub\" nodes while removing redundant connections\n- **Dynamic batching:** Efficiently batch embedding computations for GPU utilization\n- **Two-level search:** Smart graph traversal that prioritizes promising nodes\n\n**Backends:**\n- **HNSW** (default): Ideal for most datasets with maximum storage savings through full recomputation\n- **DiskANN**: Advanced option with superior search performance, using PQ-based graph traversal with real-time reranking for the best speed-accuracy trade-off\n\n## Benchmarks\n\n**[DiskANN vs HNSW Performance Comparison →](benchmarks/diskann_vs_hnsw_speed_comparison.py)** - Compare search performance between both backends\n\n**[Simple Example: Compare LEANN vs FAISS →](benchmarks/compare_faiss_vs_leann.py)** - See storage savings in action\n\n### 📊 Storage Comparison\n\n| System | DPR (2.1M) | Wiki (60M) | Chat (400K) | Email (780K) | Browser (38K) |\n|--------|-------------|------------|-------------|--------------|---------------|\n| Traditional vector database (e.g., FAISS) | 3.8 GB | 201 GB | 1.8 GB | 2.4 GB | 130 MB |\n| LEANN | 324 MB | 6 GB | 64 MB | 79 MB | 6.4 MB |\n| Savings| 91% | 97% | 97% | 97% | 95% |\n\n\n\n## Reproduce Our Results\n\n```bash\nuv run benchmarks/run_evaluation.py # Will auto-download evaluation data and run benchmarks\nuv run benchmarks/run_evaluation.py benchmarks/data/indices/rpj_wiki/rpj_wiki --num-queries 2000 # After downloading data, you can run the benchmark with our biggest index\n```\n\nThe evaluation script downloads data automatically on first run. The last three results were tested with partial personal data, and you can reproduce them with your own data!\n## 🔬 Paper\n\nIf you find Leann useful, please cite:\n\n**[LEANN: A Low-Storage Vector Index](https://arxiv.org/abs/2506.08276)**\n\n```bibtex\n@misc{wang2025leannlowstoragevectorindex,\n title={LEANN: A Low-Storage Vector Index},\n author={Yichuan Wang and Shu Liu and Zhifei Li and Yongji Wu and Ziming Mao and Yilong Zhao and Xiao Yan and Zhiying Xu and Yang Zhou and Ion Stoica and Sewon Min and Matei Zaharia and Joseph E. Gonzalez},\n year={2025},\n eprint={2506.08276},\n archivePrefix={arXiv},\n primaryClass={cs.DB},\n url={https://arxiv.org/abs/2506.08276},\n}\n```\n\n## ✨ [Detailed Features →](docs/features.md)\n\n## 🤝 [CONTRIBUTING →](docs/CONTRIBUTING.md)\n\n\n## ❓ [FAQ →](docs/faq.md)\n\n\n## 📈 [Roadmap →](docs/roadmap.md)\n\n## 📄 License\n\nMIT License - see [LICENSE](LICENSE) for details.\n\n## 🙏 Acknowledgments\n\nCore Contributors: [Yichuan Wang](https://yichuan-w.github.io/) & [Zhifei Li](https://github.com/andylizf).\n\nActive Contributors: [Gabriel Dehan](https://github.com/gabriel-dehan), [Aakash Suresh](https://github.com/ASuresh0524)\n\n\nWe welcome more contributors! Feel free to open issues or submit PRs.\n\nThis work is done at [**Berkeley Sky Computing Lab**](https://sky.cs.berkeley.edu/).\n\n## Star History\n\n[![Star History Chart](https://api.star-history.com/svg?repos=yichuan-w/LEANN&type=Date)](https://www.star-history.com/#yichuan-w/LEANN&Date)\n

\n ⭐ Star us on GitHub if Leann is useful for your research or applications!\n

\n\n

\n Made with ❤️ by the Leann team\n

\n\n## 🤖 Explore LEANN with AI\n\nLEANN is indexed on [DeepWiki](https://deepwiki.com/yichuan-w/LEANN), so you can ask questions to LLMs using Deep Research to explore the codebase and get help to add new features.\n" }, { "path": "apps/__init__.py", "content": "" }, { "path": "apps/base_rag_example.py", "content": "\"\"\"\nBase class for unified RAG examples interface.\nProvides common parameters and functionality for all RAG examples.\n\"\"\"\n\nimport argparse\nfrom abc import ABC, abstractmethod\nfrom pathlib import Path\nfrom typing import Any\n\nimport dotenv\nfrom leann.api import LeannBuilder, LeannChat\n\n# Optional import: older PyPI builds may not include interactive_utils\ntry:\n from leann.interactive_utils import create_rag_session\nexcept ImportError:\n\n def create_rag_session(app_name: str, data_description: str):\n class _SimpleSession:\n def run_interactive_loop(self, handler):\n print(f\"Interactive session for {app_name}: {data_description}\")\n print(\"Interactive mode not available in this build\")\n\n return _SimpleSession()\n\n\nfrom leann.registry import register_project_directory\n\n# Optional import: older PyPI builds may not include settings\ntry:\n from leann.settings import resolve_ollama_host, resolve_openai_api_key, resolve_openai_base_url\nexcept ImportError:\n # Minimal fallbacks if settings helpers are unavailable\n import os\n\n def resolve_ollama_host(value: str | None) -> str | None:\n return value or os.getenv(\"LEANN_OLLAMA_HOST\") or os.getenv(\"OLLAMA_HOST\")\n\n def resolve_openai_api_key(value: str | None) -> str | None:\n return value or os.getenv(\"OPENAI_API_KEY\")\n\n def resolve_openai_base_url(value: str | None) -> str | None:\n return value or os.getenv(\"OPENAI_BASE_URL\")\n\n\ndotenv.load_dotenv()\n\n\nclass BaseRAGExample(ABC):\n \"\"\"Base class for all RAG examples with unified interface.\"\"\"\n\n def __init__(\n self,\n name: str,\n description: str,\n default_index_name: str,\n ):\n self.name = name\n self.description = description\n self.default_index_name = default_index_name\n self.parser = self._create_parser()\n\n def _create_parser(self) -> argparse.ArgumentParser:\n \"\"\"Create argument parser with common parameters.\"\"\"\n parser = argparse.ArgumentParser(\n description=self.description, formatter_class=argparse.RawDescriptionHelpFormatter\n )\n\n # Core parameters (all examples share these)\n core_group = parser.add_argument_group(\"Core Parameters\")\n core_group.add_argument(\n \"--index-dir\",\n type=str,\n default=f\"./{self.default_index_name}\",\n help=f\"Directory to store the index (default: ./{self.default_index_name})\",\n )\n core_group.add_argument(\n \"--query\",\n type=str,\n default=None,\n help=\"Query to run (if not provided, will run in interactive mode)\",\n )\n # Allow subclasses to override default max_items\n max_items_default = getattr(self, \"max_items_default\", -1)\n core_group.add_argument(\n \"--max-items\",\n type=int,\n default=max_items_default,\n help=\"Maximum number of items to process -1 for all, means index all documents, and you should set it to a reasonable number if you have a large dataset and try at the first time)\",\n )\n core_group.add_argument(\n \"--force-rebuild\", action=\"store_true\", help=\"Force rebuild index even if it exists\"\n )\n\n # Embedding parameters\n embedding_group = parser.add_argument_group(\"Embedding Parameters\")\n # Allow subclasses to override default embedding_model\n embedding_model_default = getattr(self, \"embedding_model_default\", \"facebook/contriever\")\n embedding_group.add_argument(\n \"--embedding-model\",\n type=str,\n default=embedding_model_default,\n help=f\"Embedding model to use (default: {embedding_model_default}), we provide facebook/contriever, text-embedding-3-small,mlx-community/Qwen3-Embedding-0.6B-8bit or nomic-embed-text\",\n )\n embedding_group.add_argument(\n \"--embedding-mode\",\n type=str,\n default=\"sentence-transformers\",\n choices=[\"sentence-transformers\", \"openai\", \"mlx\", \"ollama\"],\n help=\"Embedding backend mode (default: sentence-transformers), we provide sentence-transformers, openai, mlx, or ollama\",\n )\n embedding_group.add_argument(\n \"--embedding-host\",\n type=str,\n default=None,\n help=\"Override Ollama-compatible embedding host\",\n )\n embedding_group.add_argument(\n \"--embedding-api-base\",\n type=str,\n default=None,\n help=\"Base URL for OpenAI-compatible embedding services\",\n )\n embedding_group.add_argument(\n \"--embedding-api-key\",\n type=str,\n default=None,\n help=\"API key for embedding service (defaults to OPENAI_API_KEY)\",\n )\n\n # LLM parameters\n llm_group = parser.add_argument_group(\"LLM Parameters\")\n llm_group.add_argument(\n \"--llm\",\n type=str,\n default=\"openai\",\n choices=[\"openai\", \"ollama\", \"hf\", \"simulated\"],\n help=\"LLM backend: openai, ollama, or hf (default: openai)\",\n )\n llm_group.add_argument(\n \"--llm-model\",\n type=str,\n default=None,\n help=\"Model name (default: gpt-4o) e.g., gpt-4o-mini, llama3.2:1b, Qwen/Qwen2.5-1.5B-Instruct\",\n )\n llm_group.add_argument(\n \"--llm-host\",\n type=str,\n default=None,\n help=\"Host for Ollama-compatible APIs (defaults to LEANN_OLLAMA_HOST/OLLAMA_HOST)\",\n )\n llm_group.add_argument(\n \"--thinking-budget\",\n type=str,\n choices=[\"low\", \"medium\", \"high\"],\n default=None,\n help=\"Thinking budget for reasoning models (low/medium/high). Supported by GPT-Oss:20b and other reasoning models.\",\n )\n llm_group.add_argument(\n \"--llm-api-base\",\n type=str,\n default=None,\n help=\"Base URL for OpenAI-compatible APIs\",\n )\n llm_group.add_argument(\n \"--llm-api-key\",\n type=str,\n default=None,\n help=\"API key for OpenAI-compatible APIs (defaults to OPENAI_API_KEY)\",\n )\n\n # AST Chunking parameters\n ast_group = parser.add_argument_group(\"AST Chunking Parameters\")\n ast_group.add_argument(\n \"--use-ast-chunking\",\n action=\"store_true\",\n help=\"Enable AST-aware chunking for code files (requires astchunk)\",\n )\n ast_group.add_argument(\n \"--ast-chunk-size\",\n type=int,\n default=300,\n help=\"Maximum CHARACTERS per AST chunk (default: 300). Final chunks may be larger due to overlap. For 512 token models: recommended 300 chars\",\n )\n ast_group.add_argument(\n \"--ast-chunk-overlap\",\n type=int,\n default=64,\n help=\"Overlap between AST chunks in CHARACTERS (default: 64). Added to chunk size, not included in it\",\n )\n ast_group.add_argument(\n \"--code-file-extensions\",\n nargs=\"+\",\n default=None,\n help=\"Additional code file extensions to process with AST chunking (e.g., .py .java .cs .ts)\",\n )\n ast_group.add_argument(\n \"--ast-fallback-traditional\",\n action=\"store_true\",\n default=True,\n help=\"Fall back to traditional chunking if AST chunking fails (default: True)\",\n )\n\n # Search parameters\n search_group = parser.add_argument_group(\"Search Parameters\")\n search_group.add_argument(\n \"--top-k\", type=int, default=20, help=\"Number of results to retrieve (default: 20)\"\n )\n search_group.add_argument(\n \"--search-complexity\",\n type=int,\n default=32,\n help=\"Search complexity for graph traversal (default: 64)\",\n )\n\n # Index building parameters\n index_group = parser.add_argument_group(\"Index Building Parameters\")\n index_group.add_argument(\n \"--backend-name\",\n type=str,\n default=\"hnsw\",\n choices=[\"hnsw\", \"diskann\"],\n help=\"Backend to use for index (default: hnsw)\",\n )\n index_group.add_argument(\n \"--graph-degree\",\n type=int,\n default=32,\n help=\"Graph degree for index construction (default: 32)\",\n )\n index_group.add_argument(\n \"--build-complexity\",\n type=int,\n default=64,\n help=\"Build complexity for index construction (default: 64)\",\n )\n index_group.add_argument(\n \"--no-compact\",\n action=\"store_true\",\n help=\"Disable compact index storage\",\n )\n index_group.add_argument(\n \"--no-recompute\",\n action=\"store_true\",\n help=\"Disable embedding recomputation\",\n )\n\n # Add source-specific parameters\n self._add_specific_arguments(parser)\n\n return parser\n\n @abstractmethod\n def _add_specific_arguments(self, parser: argparse.ArgumentParser):\n \"\"\"Add source-specific arguments. Override in subclasses.\"\"\"\n pass\n\n @abstractmethod\n async def load_data(self, args) -> list[dict[str, Any]]:\n \"\"\"Load data from the source. Returns list of text chunks as dicts with 'text' and 'metadata' keys.\"\"\"\n pass\n\n def get_llm_config(self, args) -> dict[str, Any]:\n \"\"\"Get LLM configuration based on arguments.\"\"\"\n config = {\"type\": args.llm}\n\n if args.llm == \"openai\":\n config[\"model\"] = args.llm_model or \"gpt-4o\"\n config[\"base_url\"] = resolve_openai_base_url(args.llm_api_base)\n resolved_key = resolve_openai_api_key(args.llm_api_key)\n if resolved_key:\n config[\"api_key\"] = resolved_key\n elif args.llm == \"ollama\":\n config[\"model\"] = args.llm_model or \"llama3.2:1b\"\n config[\"host\"] = resolve_ollama_host(args.llm_host)\n elif args.llm == \"hf\":\n config[\"model\"] = args.llm_model or \"Qwen/Qwen2.5-1.5B-Instruct\"\n elif args.llm == \"simulated\":\n # Simulated LLM doesn't need additional configuration\n pass\n\n return config\n\n async def build_index(self, args, texts: list[dict[str, Any]]) -> str:\n \"\"\"Build LEANN index from text chunks (dicts with 'text' and 'metadata' keys).\"\"\"\n index_path = str(Path(args.index_dir) / f\"{self.default_index_name}.leann\")\n\n print(f\"\\n[Building Index] Creating {self.name} index...\")\n print(f\"Total text chunks: {len(texts)}\")\n\n embedding_options: dict[str, Any] = {}\n if args.embedding_mode == \"ollama\":\n embedding_options[\"host\"] = resolve_ollama_host(args.embedding_host)\n elif args.embedding_mode == \"openai\":\n embedding_options[\"base_url\"] = resolve_openai_base_url(args.embedding_api_base)\n resolved_embedding_key = resolve_openai_api_key(args.embedding_api_key)\n if resolved_embedding_key:\n embedding_options[\"api_key\"] = resolved_embedding_key\n\n builder = LeannBuilder(\n backend_name=args.backend_name,\n embedding_model=args.embedding_model,\n embedding_mode=args.embedding_mode,\n embedding_options=embedding_options or None,\n graph_degree=args.graph_degree,\n complexity=args.build_complexity,\n is_compact=not args.no_compact,\n is_recompute=not args.no_recompute,\n num_threads=1, # Force single-threaded mode\n )\n\n # Add texts in batches for better progress tracking\n batch_size = 1000\n for i in range(0, len(texts), batch_size):\n batch = texts[i : i + batch_size]\n for item in batch:\n # Handle both dict format (from create_text_chunks) and plain strings\n if isinstance(item, dict):\n text = item.get(\"text\", \"\")\n metadata = item.get(\"metadata\")\n builder.add_text(text, metadata)\n else:\n builder.add_text(item)\n print(f\"Added {min(i + batch_size, len(texts))}/{len(texts)} texts...\")\n\n print(\"Building index structure...\")\n builder.build_index(index_path)\n print(f\"Index saved to: {index_path}\")\n\n # Register project directory so leann list can discover this index\n # The index is saved as args.index_dir/index_name.leann\n # We want to register the current working directory where the app is run\n register_project_directory(Path.cwd())\n\n return index_path\n\n async def run_interactive_chat(self, args, index_path: str):\n \"\"\"Run interactive chat with the index.\"\"\"\n chat = LeannChat(\n index_path,\n llm_config=self.get_llm_config(args),\n system_prompt=f\"You are a helpful assistant that answers questions about {self.name} data.\",\n complexity=args.search_complexity,\n )\n\n # Create interactive session\n session = create_rag_session(\n app_name=self.name.lower().replace(\" \", \"_\"), data_description=self.name\n )\n\n def handle_query(query: str):\n # Prepare LLM kwargs with thinking budget if specified\n llm_kwargs = {}\n if hasattr(args, \"thinking_budget\") and args.thinking_budget:\n llm_kwargs[\"thinking_budget\"] = args.thinking_budget\n\n response = chat.ask(\n query,\n top_k=args.top_k,\n complexity=args.search_complexity,\n llm_kwargs=llm_kwargs,\n )\n print(f\"\\nAssistant: {response}\\n\")\n\n session.run_interactive_loop(handle_query)\n\n async def run_single_query(self, args, index_path: str, query: str):\n \"\"\"Run a single query against the index.\"\"\"\n chat = LeannChat(\n index_path,\n llm_config=self.get_llm_config(args),\n complexity=args.search_complexity,\n )\n\n print(f\"\\n[Query]: \\033[36m{query}\\033[0m\")\n\n # Prepare LLM kwargs with thinking budget if specified\n llm_kwargs = {}\n if hasattr(args, \"thinking_budget\") and args.thinking_budget:\n llm_kwargs[\"thinking_budget\"] = args.thinking_budget\n\n response = chat.ask(\n query, top_k=args.top_k, complexity=args.search_complexity, llm_kwargs=llm_kwargs\n )\n print(f\"\\n[Response]: \\033[36m{response}\\033[0m\")\n\n async def run(self):\n \"\"\"Main entry point for the example.\"\"\"\n args = self.parser.parse_args()\n\n # Check if index exists\n index_path = str(Path(args.index_dir) / f\"{self.default_index_name}.leann\")\n index_exists = Path(f\"{index_path}.meta.json\").exists()\n\n if not index_exists or args.force_rebuild:\n # Load data and build index\n print(f\"\\n{'Rebuilding' if index_exists else 'Building'} index...\")\n texts = await self.load_data(args)\n\n if not texts:\n print(\"No data found to index!\")\n return\n\n index_path = await self.build_index(args, texts)\n else:\n print(f\"\\nUsing existing index in {args.index_dir}\")\n\n # Run query or interactive mode\n if args.query:\n await self.run_single_query(args, index_path, args.query)\n else:\n await self.run_interactive_chat(args, index_path)\n" }, { "path": "apps/browser_rag.py", "content": "\"\"\"\nBrowser History RAG example using the unified interface.\nSupports Chrome browser history.\n\"\"\"\n\nimport os\nimport sys\nfrom pathlib import Path\nfrom typing import Any\n\n# Add parent directory to path for imports\nsys.path.insert(0, str(Path(__file__).parent))\n\nfrom base_rag_example import BaseRAGExample\nfrom chunking import create_text_chunks\n\nfrom .history_data.history import ChromeHistoryReader\n\n\nclass BrowserRAG(BaseRAGExample):\n \"\"\"RAG example for Chrome browser history.\"\"\"\n\n def __init__(self):\n # Set default values BEFORE calling super().__init__\n self.embedding_model_default = (\n \"sentence-transformers/all-MiniLM-L6-v2\" # Fast 384-dim model\n )\n\n super().__init__(\n name=\"Browser History\",\n description=\"Process and query Chrome browser history with LEANN\",\n default_index_name=\"google_history_index\",\n )\n\n def _add_specific_arguments(self, parser):\n \"\"\"Add browser-specific arguments.\"\"\"\n browser_group = parser.add_argument_group(\"Browser Parameters\")\n browser_group.add_argument(\n \"--chrome-profile\",\n type=str,\n default=None,\n help=\"Path to Chrome profile directory (auto-detected if not specified)\",\n )\n browser_group.add_argument(\n \"--auto-find-profiles\",\n action=\"store_true\",\n default=True,\n help=\"Automatically find all Chrome profiles (default: True)\",\n )\n browser_group.add_argument(\n \"--chunk-size\", type=int, default=256, help=\"Text chunk size (default: 256)\"\n )\n browser_group.add_argument(\n \"--chunk-overlap\", type=int, default=128, help=\"Text chunk overlap (default: 128)\"\n )\n\n def _get_chrome_base_path(self) -> Path:\n \"\"\"Get the base Chrome profile path based on OS.\"\"\"\n if sys.platform == \"darwin\":\n return Path.home() / \"Library\" / \"Application Support\" / \"Google\" / \"Chrome\"\n elif sys.platform.startswith(\"linux\"):\n return Path.home() / \".config\" / \"google-chrome\"\n elif sys.platform == \"win32\":\n return Path(os.environ[\"LOCALAPPDATA\"]) / \"Google\" / \"Chrome\" / \"User Data\"\n else:\n raise ValueError(f\"Unsupported platform: {sys.platform}\")\n\n def _find_chrome_profiles(self) -> list[Path]:\n \"\"\"Auto-detect all Chrome profiles.\"\"\"\n base_path = self._get_chrome_base_path()\n if not base_path.exists():\n return []\n\n profiles = []\n\n # Check Default profile\n default_profile = base_path / \"Default\"\n if default_profile.exists() and (default_profile / \"History\").exists():\n profiles.append(default_profile)\n\n # Check numbered profiles\n for item in base_path.iterdir():\n if item.is_dir() and item.name.startswith(\"Profile \"):\n if (item / \"History\").exists():\n profiles.append(item)\n\n return profiles\n\n async def load_data(self, args) -> list[dict[str, Any]]:\n \"\"\"Load browser history and convert to text chunks.\"\"\"\n # Determine Chrome profiles\n if args.chrome_profile and not args.auto_find_profiles:\n profile_dirs = [Path(args.chrome_profile)]\n else:\n print(\"Auto-detecting Chrome profiles...\")\n profile_dirs = self._find_chrome_profiles()\n\n # If specific profile given, filter to just that one\n if args.chrome_profile:\n profile_path = Path(args.chrome_profile)\n profile_dirs = [p for p in profile_dirs if p == profile_path]\n\n if not profile_dirs:\n print(\"No Chrome profiles found!\")\n print(\"Please specify --chrome-profile manually\")\n return []\n\n print(f\"Found {len(profile_dirs)} Chrome profiles\")\n\n # Create reader\n reader = ChromeHistoryReader()\n\n # Process each profile\n all_documents = []\n total_processed = 0\n\n for i, profile_dir in enumerate(profile_dirs):\n print(f\"\\nProcessing profile {i + 1}/{len(profile_dirs)}: {profile_dir.name}\")\n\n try:\n # Apply max_items limit per profile\n max_per_profile = -1\n if args.max_items > 0:\n remaining = args.max_items - total_processed\n if remaining <= 0:\n break\n max_per_profile = remaining\n\n # Load history\n documents = reader.load_data(\n chrome_profile_path=str(profile_dir),\n max_count=max_per_profile,\n )\n\n if documents:\n all_documents.extend(documents)\n total_processed += len(documents)\n print(f\"Processed {len(documents)} history entries from this profile\")\n\n except Exception as e:\n print(f\"Error processing {profile_dir}: {e}\")\n continue\n\n if not all_documents:\n print(\"No browser history found to process!\")\n return []\n\n print(f\"\\nTotal history entries processed: {len(all_documents)}\")\n\n # Convert to text chunks\n all_texts = create_text_chunks(\n all_documents, chunk_size=args.chunk_size, chunk_overlap=args.chunk_overlap\n )\n\n return all_texts\n\n\nif __name__ == \"__main__\":\n import asyncio\n\n # Example queries for browser history RAG\n print(\"\\n🌐 Browser History RAG Example\")\n print(\"=\" * 50)\n print(\"\\nExample queries you can try:\")\n print(\"- 'What websites did I visit about machine learning?'\")\n print(\"- 'Find my search history about programming'\")\n print(\"- 'What YouTube videos did I watch recently?'\")\n print(\"- 'Show me websites about travel planning'\")\n print(\"\\nNote: Make sure Chrome is closed before running\\n\")\n\n rag = BrowserRAG()\n asyncio.run(rag.run())\n" }, { "path": "apps/chatgpt_data/__init__.py", "content": "" }, { "path": "apps/chatgpt_data/chatgpt_reader.py", "content": "\"\"\"\nChatGPT export data reader.\n\nReads and processes ChatGPT export data from chat.html files.\n\"\"\"\n\nimport re\nfrom pathlib import Path\nfrom typing import Any\nfrom zipfile import ZipFile\n\nfrom bs4 import BeautifulSoup\nfrom llama_index.core import Document\nfrom llama_index.core.readers.base import BaseReader\n\n\nclass ChatGPTReader(BaseReader):\n \"\"\"\n ChatGPT export data reader.\n\n Reads ChatGPT conversation data from exported chat.html files or zip archives.\n Processes conversations into structured documents with metadata.\n \"\"\"\n\n def __init__(self, concatenate_conversations: bool = True) -> None:\n \"\"\"\n Initialize.\n\n Args:\n concatenate_conversations: Whether to concatenate messages within conversations for better context\n \"\"\"\n try:\n from bs4 import BeautifulSoup # noqa\n except ImportError:\n raise ImportError(\"`beautifulsoup4` package not found: `pip install beautifulsoup4`\")\n\n self.concatenate_conversations = concatenate_conversations\n\n def _extract_html_from_zip(self, zip_path: Path) -> str | None:\n \"\"\"\n Extract chat.html from ChatGPT export zip file.\n\n Args:\n zip_path: Path to the ChatGPT export zip file\n\n Returns:\n HTML content as string, or None if not found\n \"\"\"\n try:\n with ZipFile(zip_path, \"r\") as zip_file:\n # Look for chat.html or conversations.html\n html_files = [\n f\n for f in zip_file.namelist()\n if f.endswith(\".html\") and (\"chat\" in f.lower() or \"conversation\" in f.lower())\n ]\n\n if not html_files:\n print(f\"No HTML chat file found in {zip_path}\")\n return None\n\n # Use the first HTML file found\n html_file = html_files[0]\n print(f\"Found HTML file: {html_file}\")\n\n with zip_file.open(html_file) as f:\n return f.read().decode(\"utf-8\", errors=\"ignore\")\n\n except Exception as e:\n print(f\"Error extracting HTML from zip {zip_path}: {e}\")\n return None\n\n def _parse_chatgpt_html(self, html_content: str) -> list[dict]:\n \"\"\"\n Parse ChatGPT HTML export to extract conversations.\n\n Args:\n html_content: HTML content from ChatGPT export\n\n Returns:\n List of conversation dictionaries\n \"\"\"\n soup = BeautifulSoup(html_content, \"html.parser\")\n conversations = []\n\n # Try different possible structures for ChatGPT exports\n # Structure 1: Look for conversation containers\n conversation_containers = soup.find_all(\n [\"div\", \"section\"], class_=re.compile(r\"conversation|chat\", re.I)\n )\n\n if not conversation_containers:\n # Structure 2: Look for message containers directly\n conversation_containers = [soup] # Use the entire document as one conversation\n\n for container in conversation_containers:\n conversation = self._extract_conversation_from_container(container)\n if conversation and conversation.get(\"messages\"):\n conversations.append(conversation)\n\n # If no structured conversations found, try to extract all text as one conversation\n if not conversations:\n all_text = soup.get_text(separator=\"\\n\", strip=True)\n if all_text:\n conversations.append(\n {\n \"title\": \"ChatGPT Conversation\",\n \"messages\": [{\"role\": \"mixed\", \"content\": all_text, \"timestamp\": None}],\n \"timestamp\": None,\n }\n )\n\n return conversations\n\n def _extract_conversation_from_container(self, container) -> dict | None:\n \"\"\"\n Extract conversation data from a container element.\n\n Args:\n container: BeautifulSoup element containing conversation\n\n Returns:\n Dictionary with conversation data or None\n \"\"\"\n messages = []\n\n # Look for message elements with various possible structures\n message_selectors = ['[class*=\"message\"]', '[class*=\"chat\"]', \"[data-message]\", \"p\", \"div\"]\n\n for selector in message_selectors:\n message_elements = container.select(selector)\n if message_elements:\n break\n else:\n message_elements = []\n\n # If no structured messages found, treat the entire container as one message\n if not message_elements:\n text_content = container.get_text(separator=\"\\n\", strip=True)\n if text_content:\n messages.append({\"role\": \"mixed\", \"content\": text_content, \"timestamp\": None})\n else:\n for element in message_elements:\n message = self._extract_message_from_element(element)\n if message:\n messages.append(message)\n\n if not messages:\n return None\n\n # Try to extract conversation title\n title_element = container.find([\"h1\", \"h2\", \"h3\", \"title\"])\n title = title_element.get_text(strip=True) if title_element else \"ChatGPT Conversation\"\n\n # Try to extract timestamp from various possible locations\n timestamp = self._extract_timestamp_from_container(container)\n\n return {\"title\": title, \"messages\": messages, \"timestamp\": timestamp}\n\n def _extract_message_from_element(self, element) -> dict | None:\n \"\"\"\n Extract message data from an element.\n\n Args:\n element: BeautifulSoup element containing message\n\n Returns:\n Dictionary with message data or None\n \"\"\"\n text_content = element.get_text(separator=\" \", strip=True)\n\n # Skip empty or very short messages\n if not text_content or len(text_content.strip()) < 3:\n return None\n\n # Try to determine role (user/assistant) from class names or content\n role = \"mixed\" # Default role\n\n class_names = \" \".join(element.get(\"class\", [])).lower()\n if \"user\" in class_names or \"human\" in class_names:\n role = \"user\"\n elif \"assistant\" in class_names or \"ai\" in class_names or \"gpt\" in class_names:\n role = \"assistant\"\n elif text_content.lower().startswith((\"you:\", \"user:\", \"me:\")):\n role = \"user\"\n text_content = re.sub(r\"^(you|user|me):\\s*\", \"\", text_content, flags=re.IGNORECASE)\n elif text_content.lower().startswith((\"chatgpt:\", \"assistant:\", \"ai:\")):\n role = \"assistant\"\n text_content = re.sub(\n r\"^(chatgpt|assistant|ai):\\s*\", \"\", text_content, flags=re.IGNORECASE\n )\n\n # Try to extract timestamp\n timestamp = self._extract_timestamp_from_element(element)\n\n return {\"role\": role, \"content\": text_content, \"timestamp\": timestamp}\n\n def _extract_timestamp_from_element(self, element) -> str | None:\n \"\"\"Extract timestamp from element.\"\"\"\n # Look for timestamp in various attributes and child elements\n timestamp_attrs = [\"data-timestamp\", \"timestamp\", \"datetime\"]\n for attr in timestamp_attrs:\n if element.get(attr):\n return element.get(attr)\n\n # Look for time elements\n time_element = element.find(\"time\")\n if time_element:\n return time_element.get(\"datetime\") or time_element.get_text(strip=True)\n\n # Look for date-like text patterns\n text = element.get_text()\n date_patterns = [r\"\\d{4}-\\d{2}-\\d{2}\", r\"\\d{1,2}/\\d{1,2}/\\d{4}\", r\"\\w+ \\d{1,2}, \\d{4}\"]\n\n for pattern in date_patterns:\n match = re.search(pattern, text)\n if match:\n return match.group()\n\n return None\n\n def _extract_timestamp_from_container(self, container) -> str | None:\n \"\"\"Extract timestamp from conversation container.\"\"\"\n return self._extract_timestamp_from_element(container)\n\n def _create_concatenated_content(self, conversation: dict) -> str:\n \"\"\"\n Create concatenated content from conversation messages.\n\n Args:\n conversation: Dictionary containing conversation data\n\n Returns:\n Formatted concatenated content\n \"\"\"\n title = conversation.get(\"title\", \"ChatGPT Conversation\")\n messages = conversation.get(\"messages\", [])\n timestamp = conversation.get(\"timestamp\", \"Unknown\")\n\n # Build message content\n message_parts = []\n for message in messages:\n role = message.get(\"role\", \"mixed\")\n content = message.get(\"content\", \"\")\n msg_timestamp = message.get(\"timestamp\", \"\")\n\n if role == \"user\":\n prefix = \"[You]\"\n elif role == \"assistant\":\n prefix = \"[ChatGPT]\"\n else:\n prefix = \"[Message]\"\n\n # Add timestamp if available\n if msg_timestamp:\n prefix += f\" ({msg_timestamp})\"\n\n message_parts.append(f\"{prefix}: {content}\")\n\n concatenated_text = \"\\n\\n\".join(message_parts)\n\n # Create final document content\n doc_content = f\"\"\"Conversation: {title}\nDate: {timestamp}\nMessages ({len(messages)} messages):\n\n{concatenated_text}\n\"\"\"\n return doc_content\n\n def load_data(self, input_dir: str | None = None, **load_kwargs: Any) -> list[Document]:\n \"\"\"\n Load ChatGPT export data.\n\n Args:\n input_dir: Directory containing ChatGPT export files or path to specific file\n **load_kwargs:\n max_count (int): Maximum number of conversations to process\n chatgpt_export_path (str): Specific path to ChatGPT export file/directory\n include_metadata (bool): Whether to include metadata in documents\n \"\"\"\n docs: list[Document] = []\n max_count = load_kwargs.get(\"max_count\", -1)\n chatgpt_export_path = load_kwargs.get(\"chatgpt_export_path\", input_dir)\n include_metadata = load_kwargs.get(\"include_metadata\", True)\n\n if not chatgpt_export_path:\n print(\"No ChatGPT export path provided\")\n return docs\n\n export_path = Path(chatgpt_export_path)\n\n if not export_path.exists():\n print(f\"ChatGPT export path not found: {export_path}\")\n return docs\n\n html_content = None\n\n # Handle different input types\n if export_path.is_file():\n if export_path.suffix.lower() == \".zip\":\n # Extract HTML from zip file\n html_content = self._extract_html_from_zip(export_path)\n elif export_path.suffix.lower() == \".html\":\n # Read HTML file directly\n try:\n with open(export_path, encoding=\"utf-8\", errors=\"ignore\") as f:\n html_content = f.read()\n except Exception as e:\n print(f\"Error reading HTML file {export_path}: {e}\")\n return docs\n else:\n print(f\"Unsupported file type: {export_path.suffix}\")\n return docs\n\n elif export_path.is_dir():\n # Look for HTML files in directory\n html_files = list(export_path.glob(\"*.html\"))\n zip_files = list(export_path.glob(\"*.zip\"))\n\n if html_files:\n # Use first HTML file found\n html_file = html_files[0]\n print(f\"Found HTML file: {html_file}\")\n try:\n with open(html_file, encoding=\"utf-8\", errors=\"ignore\") as f:\n html_content = f.read()\n except Exception as e:\n print(f\"Error reading HTML file {html_file}: {e}\")\n return docs\n\n elif zip_files:\n # Use first zip file found\n zip_file = zip_files[0]\n print(f\"Found zip file: {zip_file}\")\n html_content = self._extract_html_from_zip(zip_file)\n\n else:\n print(f\"No HTML or zip files found in {export_path}\")\n return docs\n\n if not html_content:\n print(\"No HTML content found to process\")\n return docs\n\n # Parse conversations from HTML\n print(\"Parsing ChatGPT conversations from HTML...\")\n conversations = self._parse_chatgpt_html(html_content)\n\n if not conversations:\n print(\"No conversations found in HTML content\")\n return docs\n\n print(f\"Found {len(conversations)} conversations\")\n\n # Process conversations into documents\n count = 0\n for conversation in conversations:\n if max_count > 0 and count >= max_count:\n break\n\n if self.concatenate_conversations:\n # Create one document per conversation with concatenated messages\n doc_content = self._create_concatenated_content(conversation)\n\n metadata = {}\n if include_metadata:\n metadata = {\n \"title\": conversation.get(\"title\", \"ChatGPT Conversation\"),\n \"timestamp\": conversation.get(\"timestamp\", \"Unknown\"),\n \"message_count\": len(conversation.get(\"messages\", [])),\n \"source\": \"ChatGPT Export\",\n }\n\n doc = Document(text=doc_content, metadata=metadata)\n docs.append(doc)\n count += 1\n\n else:\n # Create separate documents for each message\n for message in conversation.get(\"messages\", []):\n if max_count > 0 and count >= max_count:\n break\n\n role = message.get(\"role\", \"mixed\")\n content = message.get(\"content\", \"\")\n msg_timestamp = message.get(\"timestamp\", \"\")\n\n if not content.strip():\n continue\n\n # Create document content with context\n doc_content = f\"\"\"Conversation: {conversation.get(\"title\", \"ChatGPT Conversation\")}\nRole: {role}\nTimestamp: {msg_timestamp or conversation.get(\"timestamp\", \"Unknown\")}\nMessage: {content}\n\"\"\"\n\n metadata = {}\n if include_metadata:\n metadata = {\n \"conversation_title\": conversation.get(\"title\", \"ChatGPT Conversation\"),\n \"role\": role,\n \"timestamp\": msg_timestamp or conversation.get(\"timestamp\", \"Unknown\"),\n \"source\": \"ChatGPT Export\",\n }\n\n doc = Document(text=doc_content, metadata=metadata)\n docs.append(doc)\n count += 1\n\n print(f\"Created {len(docs)} documents from ChatGPT export\")\n return docs\n" }, { "path": "apps/chatgpt_rag.py", "content": "\"\"\"\nChatGPT RAG example using the unified interface.\nSupports ChatGPT export data from chat.html files.\n\"\"\"\n\nimport sys\nfrom pathlib import Path\nfrom typing import Any\n\n# Add parent directory to path for imports\nsys.path.insert(0, str(Path(__file__).parent))\n\nfrom base_rag_example import BaseRAGExample\nfrom chunking import create_text_chunks\n\nfrom .chatgpt_data.chatgpt_reader import ChatGPTReader\n\n\nclass ChatGPTRAG(BaseRAGExample):\n \"\"\"RAG example for ChatGPT conversation data.\"\"\"\n\n def __init__(self):\n # Set default values BEFORE calling super().__init__\n self.max_items_default = -1 # Process all conversations by default\n self.embedding_model_default = (\n \"sentence-transformers/all-MiniLM-L6-v2\" # Fast 384-dim model\n )\n\n super().__init__(\n name=\"ChatGPT\",\n description=\"Process and query ChatGPT conversation exports with LEANN\",\n default_index_name=\"chatgpt_conversations_index\",\n )\n\n def _add_specific_arguments(self, parser):\n \"\"\"Add ChatGPT-specific arguments.\"\"\"\n chatgpt_group = parser.add_argument_group(\"ChatGPT Parameters\")\n chatgpt_group.add_argument(\n \"--export-path\",\n type=str,\n default=\"./chatgpt_export\",\n help=\"Path to ChatGPT export file (.zip or .html) or directory containing exports (default: ./chatgpt_export)\",\n )\n chatgpt_group.add_argument(\n \"--concatenate-conversations\",\n action=\"store_true\",\n default=True,\n help=\"Concatenate messages within conversations for better context (default: True)\",\n )\n chatgpt_group.add_argument(\n \"--separate-messages\",\n action=\"store_true\",\n help=\"Process each message as a separate document (overrides --concatenate-conversations)\",\n )\n chatgpt_group.add_argument(\n \"--chunk-size\", type=int, default=512, help=\"Text chunk size (default: 512)\"\n )\n chatgpt_group.add_argument(\n \"--chunk-overlap\", type=int, default=128, help=\"Text chunk overlap (default: 128)\"\n )\n\n def _find_chatgpt_exports(self, export_path: Path) -> list[Path]:\n \"\"\"\n Find ChatGPT export files in the given path.\n\n Args:\n export_path: Path to search for exports\n\n Returns:\n List of paths to ChatGPT export files\n \"\"\"\n export_files = []\n\n if export_path.is_file():\n if export_path.suffix.lower() in [\".zip\", \".html\"]:\n export_files.append(export_path)\n elif export_path.is_dir():\n # Look for zip and html files\n export_files.extend(export_path.glob(\"*.zip\"))\n export_files.extend(export_path.glob(\"*.html\"))\n\n return export_files\n\n async def load_data(self, args) -> list[dict[str, Any]]:\n \"\"\"Load ChatGPT export data and convert to text chunks.\"\"\"\n export_path = Path(args.export_path)\n\n if not export_path.exists():\n print(f\"ChatGPT export path not found: {export_path}\")\n print(\n \"Please ensure you have exported your ChatGPT data and placed it in the correct location.\"\n )\n print(\"\\nTo export your ChatGPT data:\")\n print(\"1. Sign in to ChatGPT\")\n print(\"2. Click on your profile icon → Settings → Data Controls\")\n print(\"3. Click 'Export' under Export Data\")\n print(\"4. Download the zip file from the email link\")\n print(\"5. Extract or place the file/directory at the specified path\")\n return []\n\n # Find export files\n export_files = self._find_chatgpt_exports(export_path)\n\n if not export_files:\n print(f\"No ChatGPT export files (.zip or .html) found in: {export_path}\")\n return []\n\n print(f\"Found {len(export_files)} ChatGPT export files\")\n\n # Create reader with appropriate settings\n concatenate = args.concatenate_conversations and not args.separate_messages\n reader = ChatGPTReader(concatenate_conversations=concatenate)\n\n # Process each export file\n all_documents = []\n total_processed = 0\n\n for i, export_file in enumerate(export_files):\n print(f\"\\nProcessing export file {i + 1}/{len(export_files)}: {export_file.name}\")\n\n try:\n # Apply max_items limit per file\n max_per_file = -1\n if args.max_items > 0:\n remaining = args.max_items - total_processed\n if remaining <= 0:\n break\n max_per_file = remaining\n\n # Load conversations\n documents = reader.load_data(\n chatgpt_export_path=str(export_file),\n max_count=max_per_file,\n include_metadata=True,\n )\n\n if documents:\n all_documents.extend(documents)\n total_processed += len(documents)\n print(f\"Processed {len(documents)} conversations from this file\")\n else:\n print(f\"No conversations loaded from {export_file}\")\n\n except Exception as e:\n print(f\"Error processing {export_file}: {e}\")\n continue\n\n if not all_documents:\n print(\"No conversations found to process!\")\n print(\"\\nTroubleshooting:\")\n print(\"- Ensure the export file is a valid ChatGPT export\")\n print(\"- Check that the HTML file contains conversation data\")\n print(\"- Try extracting the zip file and pointing to the HTML file directly\")\n return []\n\n print(f\"\\nTotal conversations processed: {len(all_documents)}\")\n print(\"Now starting to split into text chunks... this may take some time\")\n\n # Convert to text chunks\n all_texts = create_text_chunks(\n all_documents, chunk_size=args.chunk_size, chunk_overlap=args.chunk_overlap\n )\n\n print(f\"Created {len(all_texts)} text chunks from {len(all_documents)} conversations\")\n return all_texts\n\n\nif __name__ == \"__main__\":\n import asyncio\n\n # Example queries for ChatGPT RAG\n print(\"\\n🤖 ChatGPT RAG Example\")\n print(\"=\" * 50)\n print(\"\\nExample queries you can try:\")\n print(\"- 'What did I ask about Python programming?'\")\n print(\"- 'Show me conversations about machine learning'\")\n print(\"- 'Find discussions about travel planning'\")\n print(\"- 'What advice did ChatGPT give me about career development?'\")\n print(\"- 'Search for conversations about cooking recipes'\")\n print(\"\\nTo get started:\")\n print(\"1. Export your ChatGPT data from Settings → Data Controls → Export\")\n print(\"2. Place the downloaded zip file or extracted HTML in ./chatgpt_export/\")\n print(\"3. Run this script to build your personal ChatGPT knowledge base!\")\n print(\"\\nOr run without --query for interactive mode\\n\")\n\n rag = ChatGPTRAG()\n asyncio.run(rag.run())\n" }, { "path": "apps/chunking/__init__.py", "content": "\"\"\"Unified chunking utilities facade.\n\nThis module re-exports the packaged utilities from `leann.chunking_utils` so\nthat both repo apps (importing `chunking`) and installed wheels share one\nsingle implementation. When running from the repo without installation, it\nadds the `packages/leann-core/src` directory to `sys.path` as a fallback.\n\"\"\"\n\nimport sys\nfrom pathlib import Path\n\ntry:\n from leann.chunking_utils import (\n CODE_EXTENSIONS,\n _traditional_chunks_as_dicts,\n create_ast_chunks,\n create_text_chunks,\n create_traditional_chunks,\n detect_code_files,\n get_language_from_extension,\n )\nexcept Exception: # pragma: no cover - best-effort fallback for dev environment\n repo_root = Path(__file__).resolve().parents[2]\n leann_src = repo_root / \"packages\" / \"leann-core\" / \"src\"\n if leann_src.exists():\n sys.path.insert(0, str(leann_src))\n from leann.chunking_utils import (\n CODE_EXTENSIONS,\n _traditional_chunks_as_dicts,\n create_ast_chunks,\n create_text_chunks,\n create_traditional_chunks,\n detect_code_files,\n get_language_from_extension,\n )\n else:\n raise\n\n__all__ = [\n \"CODE_EXTENSIONS\",\n \"_traditional_chunks_as_dicts\",\n \"create_ast_chunks\",\n \"create_text_chunks\",\n \"create_traditional_chunks\",\n \"detect_code_files\",\n \"get_language_from_extension\",\n]\n" }, { "path": "apps/claude_data/__init__.py", "content": "" }, { "path": "apps/claude_data/claude_reader.py", "content": "\"\"\"\nClaude export data reader.\n\nReads and processes Claude conversation data from exported JSON files.\n\"\"\"\n\nimport json\nfrom pathlib import Path\nfrom typing import Any\nfrom zipfile import ZipFile\n\nfrom llama_index.core import Document\nfrom llama_index.core.readers.base import BaseReader\n\n\nclass ClaudeReader(BaseReader):\n \"\"\"\n Claude export data reader.\n\n Reads Claude conversation data from exported JSON files or zip archives.\n Processes conversations into structured documents with metadata.\n \"\"\"\n\n def __init__(self, concatenate_conversations: bool = True) -> None:\n \"\"\"\n Initialize.\n\n Args:\n concatenate_conversations: Whether to concatenate messages within conversations for better context\n \"\"\"\n self.concatenate_conversations = concatenate_conversations\n\n def _extract_json_from_zip(self, zip_path: Path) -> list[str]:\n \"\"\"\n Extract JSON files from Claude export zip file.\n\n Args:\n zip_path: Path to the Claude export zip file\n\n Returns:\n List of JSON content strings, or empty list if not found\n \"\"\"\n json_contents = []\n try:\n with ZipFile(zip_path, \"r\") as zip_file:\n # Look for JSON files\n json_files = [f for f in zip_file.namelist() if f.endswith(\".json\")]\n\n if not json_files:\n print(f\"No JSON files found in {zip_path}\")\n return []\n\n print(f\"Found {len(json_files)} JSON files in archive\")\n\n for json_file in json_files:\n with zip_file.open(json_file) as f:\n content = f.read().decode(\"utf-8\", errors=\"ignore\")\n json_contents.append(content)\n\n except Exception as e:\n print(f\"Error extracting JSON from zip {zip_path}: {e}\")\n\n return json_contents\n\n def _parse_claude_json(self, json_content: str) -> list[dict]:\n \"\"\"\n Parse Claude JSON export to extract conversations.\n\n Args:\n json_content: JSON content from Claude export\n\n Returns:\n List of conversation dictionaries\n \"\"\"\n try:\n data = json.loads(json_content)\n except json.JSONDecodeError as e:\n print(f\"Error parsing JSON: {e}\")\n return []\n\n conversations = []\n\n # Handle different possible JSON structures\n if isinstance(data, list):\n # If data is a list of conversations\n for item in data:\n conversation = self._extract_conversation_from_json(item)\n if conversation:\n conversations.append(conversation)\n elif isinstance(data, dict):\n # Check for common structures\n if \"conversations\" in data:\n # Structure: {\"conversations\": [...]}\n for item in data[\"conversations\"]:\n conversation = self._extract_conversation_from_json(item)\n if conversation:\n conversations.append(conversation)\n elif \"messages\" in data:\n # Single conversation with messages\n conversation = self._extract_conversation_from_json(data)\n if conversation:\n conversations.append(conversation)\n else:\n # Try to treat the whole object as a conversation\n conversation = self._extract_conversation_from_json(data)\n if conversation:\n conversations.append(conversation)\n\n return conversations\n\n def _extract_conversation_from_json(self, conv_data: dict) -> dict | None:\n \"\"\"\n Extract conversation data from a JSON object.\n\n Args:\n conv_data: Dictionary containing conversation data\n\n Returns:\n Dictionary with conversation data or None\n \"\"\"\n if not isinstance(conv_data, dict):\n return None\n\n messages = []\n\n # Look for messages in various possible structures\n message_sources = []\n if \"messages\" in conv_data:\n message_sources = conv_data[\"messages\"]\n elif \"chat\" in conv_data:\n message_sources = conv_data[\"chat\"]\n elif \"conversation\" in conv_data:\n message_sources = conv_data[\"conversation\"]\n else:\n # If no clear message structure, try to extract from the object itself\n if \"content\" in conv_data and \"role\" in conv_data:\n message_sources = [conv_data]\n\n for msg_data in message_sources:\n message = self._extract_message_from_json(msg_data)\n if message:\n messages.append(message)\n\n if not messages:\n return None\n\n # Extract conversation metadata\n title = self._extract_title_from_conversation(conv_data, messages)\n timestamp = self._extract_timestamp_from_conversation(conv_data)\n\n return {\"title\": title, \"messages\": messages, \"timestamp\": timestamp}\n\n def _extract_message_from_json(self, msg_data: dict) -> dict | None:\n \"\"\"\n Extract message data from a JSON message object.\n\n Args:\n msg_data: Dictionary containing message data\n\n Returns:\n Dictionary with message data or None\n \"\"\"\n if not isinstance(msg_data, dict):\n return None\n\n # Extract content from various possible fields\n content = \"\"\n content_fields = [\"content\", \"text\", \"message\", \"body\"]\n for field in content_fields:\n if msg_data.get(field):\n content = str(msg_data[field])\n break\n\n if not content or len(content.strip()) < 3:\n return None\n\n # Extract role (user/assistant/human/ai/claude)\n role = \"mixed\" # Default role\n role_fields = [\"role\", \"sender\", \"from\", \"author\", \"type\"]\n for field in role_fields:\n if msg_data.get(field):\n role_value = str(msg_data[field]).lower()\n if role_value in [\"user\", \"human\", \"person\"]:\n role = \"user\"\n elif role_value in [\"assistant\", \"ai\", \"claude\", \"bot\"]:\n role = \"assistant\"\n break\n\n # Extract timestamp\n timestamp = self._extract_timestamp_from_message(msg_data)\n\n return {\"role\": role, \"content\": content, \"timestamp\": timestamp}\n\n def _extract_timestamp_from_message(self, msg_data: dict) -> str | None:\n \"\"\"Extract timestamp from message data.\"\"\"\n timestamp_fields = [\"timestamp\", \"created_at\", \"date\", \"time\"]\n for field in timestamp_fields:\n if msg_data.get(field):\n return str(msg_data[field])\n return None\n\n def _extract_timestamp_from_conversation(self, conv_data: dict) -> str | None:\n \"\"\"Extract timestamp from conversation data.\"\"\"\n timestamp_fields = [\"timestamp\", \"created_at\", \"date\", \"updated_at\", \"last_updated\"]\n for field in timestamp_fields:\n if conv_data.get(field):\n return str(conv_data[field])\n return None\n\n def _extract_title_from_conversation(self, conv_data: dict, messages: list) -> str:\n \"\"\"Extract or generate title for conversation.\"\"\"\n # Try to find explicit title\n title_fields = [\"title\", \"name\", \"subject\", \"topic\"]\n for field in title_fields:\n if conv_data.get(field):\n return str(conv_data[field])\n\n # Generate title from first user message\n for message in messages:\n if message.get(\"role\") == \"user\":\n content = message.get(\"content\", \"\")\n if content:\n # Use first 50 characters as title\n title = content[:50].strip()\n if len(content) > 50:\n title += \"...\"\n return title\n\n return \"Claude Conversation\"\n\n def _create_concatenated_content(self, conversation: dict) -> str:\n \"\"\"\n Create concatenated content from conversation messages.\n\n Args:\n conversation: Dictionary containing conversation data\n\n Returns:\n Formatted concatenated content\n \"\"\"\n title = conversation.get(\"title\", \"Claude Conversation\")\n messages = conversation.get(\"messages\", [])\n timestamp = conversation.get(\"timestamp\", \"Unknown\")\n\n # Build message content\n message_parts = []\n for message in messages:\n role = message.get(\"role\", \"mixed\")\n content = message.get(\"content\", \"\")\n msg_timestamp = message.get(\"timestamp\", \"\")\n\n if role == \"user\":\n prefix = \"[You]\"\n elif role == \"assistant\":\n prefix = \"[Claude]\"\n else:\n prefix = \"[Message]\"\n\n # Add timestamp if available\n if msg_timestamp:\n prefix += f\" ({msg_timestamp})\"\n\n message_parts.append(f\"{prefix}: {content}\")\n\n concatenated_text = \"\\n\\n\".join(message_parts)\n\n # Create final document content\n doc_content = f\"\"\"Conversation: {title}\nDate: {timestamp}\nMessages ({len(messages)} messages):\n\n{concatenated_text}\n\"\"\"\n return doc_content\n\n def load_data(self, input_dir: str | None = None, **load_kwargs: Any) -> list[Document]:\n \"\"\"\n Load Claude export data.\n\n Args:\n input_dir: Directory containing Claude export files or path to specific file\n **load_kwargs:\n max_count (int): Maximum number of conversations to process\n claude_export_path (str): Specific path to Claude export file/directory\n include_metadata (bool): Whether to include metadata in documents\n \"\"\"\n docs: list[Document] = []\n max_count = load_kwargs.get(\"max_count\", -1)\n claude_export_path = load_kwargs.get(\"claude_export_path\", input_dir)\n include_metadata = load_kwargs.get(\"include_metadata\", True)\n\n if not claude_export_path:\n print(\"No Claude export path provided\")\n return docs\n\n export_path = Path(claude_export_path)\n\n if not export_path.exists():\n print(f\"Claude export path not found: {export_path}\")\n return docs\n\n json_contents = []\n\n # Handle different input types\n if export_path.is_file():\n if export_path.suffix.lower() == \".zip\":\n # Extract JSON from zip file\n json_contents = self._extract_json_from_zip(export_path)\n elif export_path.suffix.lower() == \".json\":\n # Read JSON file directly\n try:\n with open(export_path, encoding=\"utf-8\", errors=\"ignore\") as f:\n json_contents.append(f.read())\n except Exception as e:\n print(f\"Error reading JSON file {export_path}: {e}\")\n return docs\n else:\n print(f\"Unsupported file type: {export_path.suffix}\")\n return docs\n\n elif export_path.is_dir():\n # Look for JSON files in directory\n json_files = list(export_path.glob(\"*.json\"))\n zip_files = list(export_path.glob(\"*.zip\"))\n\n if json_files:\n print(f\"Found {len(json_files)} JSON files in directory\")\n for json_file in json_files:\n try:\n with open(json_file, encoding=\"utf-8\", errors=\"ignore\") as f:\n json_contents.append(f.read())\n except Exception as e:\n print(f\"Error reading JSON file {json_file}: {e}\")\n continue\n\n if zip_files:\n print(f\"Found {len(zip_files)} ZIP files in directory\")\n for zip_file in zip_files:\n zip_contents = self._extract_json_from_zip(zip_file)\n json_contents.extend(zip_contents)\n\n if not json_files and not zip_files:\n print(f\"No JSON or ZIP files found in {export_path}\")\n return docs\n\n if not json_contents:\n print(\"No JSON content found to process\")\n return docs\n\n # Parse conversations from JSON content\n print(\"Parsing Claude conversations from JSON...\")\n all_conversations = []\n for json_content in json_contents:\n conversations = self._parse_claude_json(json_content)\n all_conversations.extend(conversations)\n\n if not all_conversations:\n print(\"No conversations found in JSON content\")\n return docs\n\n print(f\"Found {len(all_conversations)} conversations\")\n\n # Process conversations into documents\n count = 0\n for conversation in all_conversations:\n if max_count > 0 and count >= max_count:\n break\n\n if self.concatenate_conversations:\n # Create one document per conversation with concatenated messages\n doc_content = self._create_concatenated_content(conversation)\n\n metadata = {}\n if include_metadata:\n metadata = {\n \"title\": conversation.get(\"title\", \"Claude Conversation\"),\n \"timestamp\": conversation.get(\"timestamp\", \"Unknown\"),\n \"message_count\": len(conversation.get(\"messages\", [])),\n \"source\": \"Claude Export\",\n }\n\n doc = Document(text=doc_content, metadata=metadata)\n docs.append(doc)\n count += 1\n\n else:\n # Create separate documents for each message\n for message in conversation.get(\"messages\", []):\n if max_count > 0 and count >= max_count:\n break\n\n role = message.get(\"role\", \"mixed\")\n content = message.get(\"content\", \"\")\n msg_timestamp = message.get(\"timestamp\", \"\")\n\n if not content.strip():\n continue\n\n # Create document content with context\n doc_content = f\"\"\"Conversation: {conversation.get(\"title\", \"Claude Conversation\")}\nRole: {role}\nTimestamp: {msg_timestamp or conversation.get(\"timestamp\", \"Unknown\")}\nMessage: {content}\n\"\"\"\n\n metadata = {}\n if include_metadata:\n metadata = {\n \"conversation_title\": conversation.get(\"title\", \"Claude Conversation\"),\n \"role\": role,\n \"timestamp\": msg_timestamp or conversation.get(\"timestamp\", \"Unknown\"),\n \"source\": \"Claude Export\",\n }\n\n doc = Document(text=doc_content, metadata=metadata)\n docs.append(doc)\n count += 1\n\n print(f\"Created {len(docs)} documents from Claude export\")\n return docs\n" }, { "path": "apps/claude_rag.py", "content": "\"\"\"\nClaude RAG example using the unified interface.\nSupports Claude export data from JSON files.\n\"\"\"\n\nimport sys\nfrom pathlib import Path\nfrom typing import Any\n\n# Add parent directory to path for imports\nsys.path.insert(0, str(Path(__file__).parent))\n\nfrom base_rag_example import BaseRAGExample\nfrom chunking import create_text_chunks\n\nfrom .claude_data.claude_reader import ClaudeReader\n\n\nclass ClaudeRAG(BaseRAGExample):\n \"\"\"RAG example for Claude conversation data.\"\"\"\n\n def __init__(self):\n # Set default values BEFORE calling super().__init__\n self.max_items_default = -1 # Process all conversations by default\n self.embedding_model_default = (\n \"sentence-transformers/all-MiniLM-L6-v2\" # Fast 384-dim model\n )\n\n super().__init__(\n name=\"Claude\",\n description=\"Process and query Claude conversation exports with LEANN\",\n default_index_name=\"claude_conversations_index\",\n )\n\n def _add_specific_arguments(self, parser):\n \"\"\"Add Claude-specific arguments.\"\"\"\n claude_group = parser.add_argument_group(\"Claude Parameters\")\n claude_group.add_argument(\n \"--export-path\",\n type=str,\n default=\"./claude_export\",\n help=\"Path to Claude export file (.json or .zip) or directory containing exports (default: ./claude_export)\",\n )\n claude_group.add_argument(\n \"--concatenate-conversations\",\n action=\"store_true\",\n default=True,\n help=\"Concatenate messages within conversations for better context (default: True)\",\n )\n claude_group.add_argument(\n \"--separate-messages\",\n action=\"store_true\",\n help=\"Process each message as a separate document (overrides --concatenate-conversations)\",\n )\n claude_group.add_argument(\n \"--chunk-size\", type=int, default=512, help=\"Text chunk size (default: 512)\"\n )\n claude_group.add_argument(\n \"--chunk-overlap\", type=int, default=128, help=\"Text chunk overlap (default: 128)\"\n )\n\n def _find_claude_exports(self, export_path: Path) -> list[Path]:\n \"\"\"\n Find Claude export files in the given path.\n\n Args:\n export_path: Path to search for exports\n\n Returns:\n List of paths to Claude export files\n \"\"\"\n export_files = []\n\n if export_path.is_file():\n if export_path.suffix.lower() in [\".zip\", \".json\"]:\n export_files.append(export_path)\n elif export_path.is_dir():\n # Look for zip and json files\n export_files.extend(export_path.glob(\"*.zip\"))\n export_files.extend(export_path.glob(\"*.json\"))\n\n return export_files\n\n async def load_data(self, args) -> list[dict[str, Any]]:\n \"\"\"Load Claude export data and convert to text chunks.\"\"\"\n export_path = Path(args.export_path)\n\n if not export_path.exists():\n print(f\"Claude export path not found: {export_path}\")\n print(\n \"Please ensure you have exported your Claude data and placed it in the correct location.\"\n )\n print(\"\\nTo export your Claude data:\")\n print(\"1. Open Claude in your browser\")\n print(\"2. Look for export/download options in settings or conversation menu\")\n print(\"3. Download the conversation data (usually in JSON format)\")\n print(\"4. Place the file/directory at the specified path\")\n print(\n \"\\nNote: Claude export methods may vary. Check Claude's help documentation for current instructions.\"\n )\n return []\n\n # Find export files\n export_files = self._find_claude_exports(export_path)\n\n if not export_files:\n print(f\"No Claude export files (.json or .zip) found in: {export_path}\")\n return []\n\n print(f\"Found {len(export_files)} Claude export files\")\n\n # Create reader with appropriate settings\n concatenate = args.concatenate_conversations and not args.separate_messages\n reader = ClaudeReader(concatenate_conversations=concatenate)\n\n # Process each export file\n all_documents = []\n total_processed = 0\n\n for i, export_file in enumerate(export_files):\n print(f\"\\nProcessing export file {i + 1}/{len(export_files)}: {export_file.name}\")\n\n try:\n # Apply max_items limit per file\n max_per_file = -1\n if args.max_items > 0:\n remaining = args.max_items - total_processed\n if remaining <= 0:\n break\n max_per_file = remaining\n\n # Load conversations\n documents = reader.load_data(\n claude_export_path=str(export_file),\n max_count=max_per_file,\n include_metadata=True,\n )\n\n if documents:\n all_documents.extend(documents)\n total_processed += len(documents)\n print(f\"Processed {len(documents)} conversations from this file\")\n else:\n print(f\"No conversations loaded from {export_file}\")\n\n except Exception as e:\n print(f\"Error processing {export_file}: {e}\")\n continue\n\n if not all_documents:\n print(\"No conversations found to process!\")\n print(\"\\nTroubleshooting:\")\n print(\"- Ensure the export file is a valid Claude export\")\n print(\"- Check that the JSON file contains conversation data\")\n print(\"- Try using a different export format or method\")\n print(\"- Check Claude's documentation for current export procedures\")\n return []\n\n print(f\"\\nTotal conversations processed: {len(all_documents)}\")\n print(\"Now starting to split into text chunks... this may take some time\")\n\n # Convert to text chunks\n all_texts = create_text_chunks(\n all_documents, chunk_size=args.chunk_size, chunk_overlap=args.chunk_overlap\n )\n\n print(f\"Created {len(all_texts)} text chunks from {len(all_documents)} conversations\")\n return all_texts\n\n\nif __name__ == \"__main__\":\n import asyncio\n\n # Example queries for Claude RAG\n print(\"\\n🤖 Claude RAG Example\")\n print(\"=\" * 50)\n print(\"\\nExample queries you can try:\")\n print(\"- 'What did I ask Claude about Python programming?'\")\n print(\"- 'Show me conversations about machine learning'\")\n print(\"- 'Find discussions about code optimization'\")\n print(\"- 'What advice did Claude give me about software design?'\")\n print(\"- 'Search for conversations about debugging techniques'\")\n print(\"\\nTo get started:\")\n print(\"1. Export your Claude conversation data\")\n print(\"2. Place the JSON/ZIP file in ./claude_export/\")\n print(\"3. Run this script to build your personal Claude knowledge base!\")\n print(\"\\nOr run without --query for interactive mode\\n\")\n\n rag = ClaudeRAG()\n asyncio.run(rag.run())\n" }, { "path": "apps/code_rag.py", "content": "\"\"\"\nCode RAG example using AST-aware chunking for optimal code understanding.\nSpecialized for code repositories with automatic language detection and\noptimized chunking parameters.\n\"\"\"\n\nimport sys\nfrom pathlib import Path\nfrom typing import Any\n\n# Add parent directory to path for imports\nsys.path.insert(0, str(Path(__file__).parent))\n\nfrom base_rag_example import BaseRAGExample\nfrom chunking import CODE_EXTENSIONS, create_text_chunks\nfrom llama_index.core import SimpleDirectoryReader\n\n\nclass CodeRAG(BaseRAGExample):\n \"\"\"Specialized RAG example for code repositories with AST-aware chunking.\"\"\"\n\n def __init__(self):\n super().__init__(\n name=\"Code\",\n description=\"Process and query code repositories with AST-aware chunking\",\n default_index_name=\"code_index\",\n )\n # Override defaults for code-specific usage\n self.embedding_model_default = \"facebook/contriever\" # Good for code\n self.max_items_default = -1 # Process all code files by default\n\n def _add_specific_arguments(self, parser):\n \"\"\"Add code-specific arguments.\"\"\"\n code_group = parser.add_argument_group(\"Code Repository Parameters\")\n\n code_group.add_argument(\n \"--repo-dir\",\n type=str,\n default=\".\",\n help=\"Code repository directory to index (default: current directory)\",\n )\n code_group.add_argument(\n \"--include-extensions\",\n nargs=\"+\",\n default=list(CODE_EXTENSIONS.keys()),\n help=\"File extensions to include (default: supported code extensions)\",\n )\n code_group.add_argument(\n \"--exclude-dirs\",\n nargs=\"+\",\n default=[\n \".git\",\n \"__pycache__\",\n \"node_modules\",\n \"venv\",\n \".venv\",\n \"build\",\n \"dist\",\n \"target\",\n ],\n help=\"Directories to exclude from indexing\",\n )\n code_group.add_argument(\n \"--max-file-size\",\n type=int,\n default=1000000, # 1MB\n help=\"Maximum file size in bytes to process (default: 1MB)\",\n )\n code_group.add_argument(\n \"--include-comments\",\n action=\"store_true\",\n help=\"Include comments in chunking (useful for documentation)\",\n )\n code_group.add_argument(\n \"--preserve-imports\",\n action=\"store_true\",\n default=True,\n help=\"Try to preserve import statements in chunks (default: True)\",\n )\n\n async def load_data(self, args) -> list[dict[str, Any]]:\n \"\"\"Load code files and convert to AST-aware chunks.\"\"\"\n print(f\"🔍 Scanning code repository: {args.repo_dir}\")\n print(f\"📁 Including extensions: {args.include_extensions}\")\n print(f\"🚫 Excluding directories: {args.exclude_dirs}\")\n\n # Check if repository directory exists\n repo_path = Path(args.repo_dir)\n if not repo_path.exists():\n raise ValueError(f\"Repository directory not found: {args.repo_dir}\")\n\n # Create exclusion filter\n def file_filter(file_path: str) -> bool:\n \"\"\"Filter out unwanted files and directories.\"\"\"\n path = Path(file_path)\n\n # Check file size\n try:\n if path.stat().st_size > args.max_file_size:\n print(f\"⚠️ Skipping large file: {path.name} ({path.stat().st_size} bytes)\")\n return False\n except Exception:\n return False\n\n # Check if in excluded directory\n for exclude_dir in args.exclude_dirs:\n if exclude_dir in path.parts:\n return False\n\n return True\n\n try:\n # Load documents with file filtering\n documents = SimpleDirectoryReader(\n args.repo_dir,\n file_extractor=None,\n recursive=True,\n encoding=\"utf-8\",\n required_exts=args.include_extensions,\n exclude_hidden=True,\n ).load_data(show_progress=True)\n\n # Apply custom filtering\n filtered_docs = []\n for doc in documents:\n file_path = doc.metadata.get(\"file_path\", \"\")\n if file_filter(file_path):\n filtered_docs.append(doc)\n\n documents = filtered_docs\n\n except Exception as e:\n print(f\"❌ Error loading code files: {e}\")\n return []\n\n if not documents:\n print(\n f\"❌ No code files found in {args.repo_dir} with extensions {args.include_extensions}\"\n )\n return []\n\n print(f\"✅ Loaded {len(documents)} code files\")\n\n # Show breakdown by language/extension\n ext_counts = {}\n for doc in documents:\n file_path = doc.metadata.get(\"file_path\", \"\")\n if file_path:\n ext = Path(file_path).suffix.lower()\n ext_counts[ext] = ext_counts.get(ext, 0) + 1\n\n print(\"📊 Files by extension:\")\n for ext, count in sorted(ext_counts.items()):\n print(f\" {ext}: {count} files\")\n\n # Use AST-aware chunking by default for code\n print(\n f\"🧠 Using AST-aware chunking (chunk_size: {args.ast_chunk_size}, overlap: {args.ast_chunk_overlap})\"\n )\n\n all_texts = create_text_chunks(\n documents,\n chunk_size=256, # Fallback for non-code files\n chunk_overlap=64,\n use_ast_chunking=True, # Always use AST for code RAG\n ast_chunk_size=args.ast_chunk_size,\n ast_chunk_overlap=args.ast_chunk_overlap,\n code_file_extensions=args.include_extensions,\n ast_fallback_traditional=True,\n )\n\n # Apply max_items limit if specified\n if args.max_items > 0 and len(all_texts) > args.max_items:\n print(f\"⏳ Limiting to {args.max_items} chunks (from {len(all_texts)})\")\n all_texts = all_texts[: args.max_items]\n\n print(f\"✅ Generated {len(all_texts)} code chunks\")\n return all_texts\n\n\nif __name__ == \"__main__\":\n import asyncio\n\n # Example queries for code RAG\n print(\"\\n💻 Code RAG Example\")\n print(\"=\" * 50)\n print(\"\\nExample queries you can try:\")\n print(\"- 'How does the embedding computation work?'\")\n print(\"- 'What are the main classes in this codebase?'\")\n print(\"- 'Show me the search implementation'\")\n print(\"- 'How is error handling implemented?'\")\n print(\"- 'What design patterns are used?'\")\n print(\"- 'Explain the chunking logic'\")\n print(\"\\n🚀 Features:\")\n print(\"- ✅ AST-aware chunking preserves code structure\")\n print(\"- ✅ Automatic language detection\")\n print(\"- ✅ Smart filtering of large files and common excludes\")\n print(\"- ✅ Optimized for code understanding\")\n print(\"\\nUsage examples:\")\n print(\" python -m apps.code_rag --repo-dir ./my_project\")\n print(\n \" python -m apps.code_rag --include-extensions .py .js --query 'How does authentication work?'\"\n )\n print(\"\\nOr run without --query for interactive mode\\n\")\n\n rag = CodeRAG()\n asyncio.run(rag.run())\n" }, { "path": "apps/colqwen_rag.py", "content": "#!/usr/bin/env python3\n\"\"\"\nColQwen RAG - Easy-to-use multimodal PDF retrieval with ColQwen2/ColPali\n\nUsage:\n python -m apps.colqwen_rag build --pdfs ./my_pdfs/ --index my_index\n python -m apps.colqwen_rag search my_index \"How does attention work?\"\n python -m apps.colqwen_rag ask my_index --interactive\n\"\"\"\n\nimport argparse\nimport os\nimport sys\nfrom pathlib import Path\nfrom typing import Any, Optional, cast\n\n# Add LEANN packages to path\n_repo_root = Path(__file__).resolve().parents[1]\n_leann_core_src = _repo_root / \"packages\" / \"leann-core\" / \"src\"\n_leann_hnsw_pkg = _repo_root / \"packages\" / \"leann-backend-hnsw\"\nif str(_leann_core_src) not in sys.path:\n sys.path.append(str(_leann_core_src))\nif str(_leann_hnsw_pkg) not in sys.path:\n sys.path.append(str(_leann_hnsw_pkg))\n\nimport torch # noqa: E402\nfrom pdf2image import convert_from_path # noqa: E402\nfrom PIL import Image # noqa: E402\nfrom torch.utils.data import DataLoader # noqa: E402\nfrom tqdm import tqdm # noqa: E402\n\n# Import the existing multi-vector implementation\nsys.path.append(str(_repo_root / \"apps\" / \"multimodal\" / \"vision-based-pdf-multi-vector\"))\nfrom leann_multi_vector import LeannMultiVector # noqa: E402\n\n\nclass ColQwenRAG:\n \"\"\"Easy-to-use ColQwen RAG system for multimodal PDF retrieval.\"\"\"\n\n def __init__(self, model_type: str = \"colpali\"):\n \"\"\"\n Initialize ColQwen RAG system.\n\n Args:\n model_type: \"colqwen2\" or \"colpali\"\n \"\"\"\n self._assert_supported_transformers()\n self.model_type = model_type\n self.device = self._get_device()\n # Use float32 on MPS to avoid memory issues, float16 on CUDA, bfloat16 on CPU\n if self.device.type == \"mps\":\n self.dtype = torch.float32\n elif self.device.type == \"cuda\":\n self.dtype = torch.float16\n else:\n self.dtype = torch.bfloat16\n\n print(f\"🚀 Initializing {model_type.upper()} on {self.device} with {self.dtype}\")\n\n # Load model and processor with MPS-optimized settings\n try:\n from colpali_engine import (\n ColPali,\n ColPaliProcessor,\n ColQwen2,\n ColQwen2Processor,\n )\n from colpali_engine.utils.torch_utils import ListDataset\n\n self._list_dataset_cls: type[Any] = ListDataset\n\n if model_type == \"colqwen2\":\n self.model_name = \"vidore/colqwen2-v1.0\"\n if self.device.type == \"mps\":\n # For MPS, load on CPU first then move to avoid memory allocation issues\n self.model = ColQwen2.from_pretrained(\n self.model_name,\n torch_dtype=self.dtype,\n device_map=\"cpu\",\n low_cpu_mem_usage=True,\n ).eval()\n self.model = self.model.to(self.device)\n else:\n self.model = ColQwen2.from_pretrained(\n self.model_name,\n torch_dtype=self.dtype,\n device_map=self.device,\n low_cpu_mem_usage=True,\n ).eval()\n self.processor = ColQwen2Processor.from_pretrained(self.model_name)\n else: # colpali\n self.model_name = \"vidore/colpali-v1.2\"\n if self.device.type == \"mps\":\n # For MPS, load on CPU first then move to avoid memory allocation issues\n self.model = ColPali.from_pretrained(\n self.model_name,\n torch_dtype=self.dtype,\n device_map=\"cpu\",\n low_cpu_mem_usage=True,\n ).eval()\n self.model = self.model.to(self.device)\n else:\n self.model = ColPali.from_pretrained(\n self.model_name,\n torch_dtype=self.dtype,\n device_map=self.device,\n low_cpu_mem_usage=True,\n ).eval()\n self.processor = ColPaliProcessor.from_pretrained(self.model_name)\n except Exception as e:\n if \"memory\" in str(e).lower() or \"offload\" in str(e).lower():\n print(f\"⚠️ Memory constraint on {self.device}, using CPU with optimizations...\")\n self.device = torch.device(\"cpu\")\n self.dtype = torch.float32\n\n if model_type == \"colqwen2\":\n self.model = ColQwen2.from_pretrained(\n self.model_name,\n torch_dtype=self.dtype,\n device_map=\"cpu\",\n low_cpu_mem_usage=True,\n ).eval()\n self.processor = ColQwen2Processor.from_pretrained(self.model_name)\n else:\n self.model = ColPali.from_pretrained(\n self.model_name,\n torch_dtype=self.dtype,\n device_map=\"cpu\",\n low_cpu_mem_usage=True,\n ).eval()\n self.processor = ColPaliProcessor.from_pretrained(self.model_name)\n else:\n raise\n\n def _assert_supported_transformers(self) -> None:\n \"\"\"Fail fast on unsupported transformers versions.\"\"\"\n from importlib.metadata import PackageNotFoundError, version\n\n try:\n transformers_version = version(\"transformers\")\n except PackageNotFoundError:\n return\n\n def _parse_semver(value: str) -> tuple[int, int, int]:\n parts = value.split(\".\")\n numbers: list[int] = []\n for part in parts[:3]:\n digits = []\n for ch in part:\n if ch.isdigit():\n digits.append(ch)\n else:\n break\n numbers.append(int(\"\".join(digits)) if digits else 0)\n while len(numbers) < 3:\n numbers.append(0)\n return tuple(numbers) # type: ignore[return-value]\n\n if _parse_semver(transformers_version) >= (4, 46, 0):\n raise RuntimeError(\n \"Unsupported transformers version detected. \"\n \"LEANN currently requires transformers<4.46 due to typing changes \"\n \"and transformers 5.x removing symbols such as HybridCache. \"\n \"Please install a compatible version, e.g. \"\n '`pip install \"transformers<4.46\"`.'\n )\n\n def _get_device(self):\n \"\"\"Auto-select best available device.\"\"\"\n if torch.cuda.is_available():\n return torch.device(\"cuda\")\n elif hasattr(torch.backends, \"mps\") and torch.backends.mps.is_available():\n return torch.device(\"mps\")\n else:\n return torch.device(\"cpu\")\n\n def build_index(self, pdf_paths: list[str], index_name: str, pages_dir: Optional[str] = None):\n \"\"\"\n Build multimodal index from PDF files.\n\n Args:\n pdf_paths: List of PDF file paths\n index_name: Name for the index\n pages_dir: Directory to save page images (optional)\n \"\"\"\n print(f\"Building index '{index_name}' from {len(pdf_paths)} PDFs...\")\n\n # Convert PDFs to images\n all_images = []\n all_metadata = []\n\n if pages_dir:\n os.makedirs(pages_dir, exist_ok=True)\n\n for pdf_path in tqdm(pdf_paths, desc=\"Converting PDFs\"):\n try:\n images = convert_from_path(pdf_path, dpi=150)\n pdf_name = Path(pdf_path).stem\n\n for i, image in enumerate(images):\n # Save image if pages_dir specified\n if pages_dir:\n image_path = Path(pages_dir) / f\"{pdf_name}_page_{i + 1}.png\"\n image.save(image_path)\n\n all_images.append(image)\n all_metadata.append(\n {\n \"pdf_path\": pdf_path,\n \"pdf_name\": pdf_name,\n \"page_number\": i + 1,\n \"image_path\": str(image_path) if pages_dir else None,\n }\n )\n\n except Exception as e:\n print(f\"❌ Error processing {pdf_path}: {e}\")\n continue\n\n print(f\"📄 Converted {len(all_images)} pages from {len(pdf_paths)} PDFs\")\n if len(all_images) == 0:\n raise RuntimeError(\n \"No PDF pages were converted to images, so there is nothing to embed.\\n\"\n \"Common causes:\\n\"\n \"- `poppler`/`pdftoppm` is missing (required by `pdf2image`)\\n\"\n \"- The input PDFs are encrypted/corrupt or have zero pages\\n\\n\"\n \"Try:\\n\"\n \"- Install poppler (macOS: `brew install poppler`, Ubuntu: `apt-get install poppler-utils`)\\n\"\n \"- Re-run with a known-good PDF\\n\"\n )\n\n # Generate embeddings\n print(\"🧠 Generating embeddings...\")\n embeddings = self._embed_images(all_images)\n\n # Build LEANN index\n print(\"🔍 Building LEANN index...\")\n leann_mv = LeannMultiVector(\n index_path=index_name,\n dim=embeddings.shape[-1],\n embedding_model_name=self.model_type,\n )\n\n # Create collection and insert data\n leann_mv.create_collection()\n for i, (embedding, metadata) in enumerate(zip(embeddings, all_metadata)):\n data = {\n \"doc_id\": i,\n \"filepath\": metadata.get(\"image_path\", \"\"),\n \"colbert_vecs\": embedding.numpy(), # Convert tensor to numpy\n }\n leann_mv.insert(data)\n\n # Build the index\n leann_mv.create_index()\n print(f\"✅ Index '{index_name}' built successfully!\")\n\n return leann_mv\n\n def search(self, index_name: str, query: str, top_k: int = 5):\n \"\"\"\n Search the index with a text query.\n\n Args:\n index_name: Name of the index to search\n query: Text query\n top_k: Number of results to return\n \"\"\"\n print(f\"🔍 Searching '{index_name}' for: '{query}'\")\n\n # Load index\n leann_mv = LeannMultiVector(\n index_path=index_name,\n dim=128, # Will be updated when loading\n embedding_model_name=self.model_type,\n )\n\n # Generate query embedding\n query_embedding = self._embed_query(query)\n\n # Search (returns list of (score, doc_id) tuples)\n search_results = leann_mv.search(query_embedding.numpy(), topk=top_k)\n\n # Display results\n print(f\"\\n📋 Top {len(search_results)} results:\")\n for i, (score, doc_id) in enumerate(search_results, 1):\n # Get metadata for this doc_id (we need to load the metadata)\n print(f\"{i}. Score: {score:.3f} | Doc ID: {doc_id}\")\n\n return search_results\n\n def ask(self, index_name: str, interactive: bool = False):\n \"\"\"\n Interactive Q&A with the indexed documents.\n\n Args:\n index_name: Name of the index to query\n interactive: Whether to run in interactive mode\n \"\"\"\n print(f\"💬 ColQwen Chat with '{index_name}'\")\n\n if interactive:\n print(\"Type 'quit' to exit, 'help' for commands\")\n while True:\n try:\n query = input(\"\\n🤔 Your question: \").strip()\n if query.lower() in [\"quit\", \"exit\", \"q\"]:\n break\n elif query.lower() == \"help\":\n print(\"Commands: quit/exit/q (exit), help (this message)\")\n continue\n elif not query:\n continue\n\n self.search(index_name, query, top_k=3)\n\n # TODO: Add answer generation with Qwen-VL\n print(\"\\n💡 For detailed answers, we can integrate Qwen-VL here!\")\n\n except KeyboardInterrupt:\n print(\"\\n👋 Goodbye!\")\n break\n else:\n query = input(\"🤔 Your question: \").strip()\n if query:\n self.search(index_name, query)\n\n def _embed_images(self, images: list[Image.Image]) -> torch.Tensor:\n \"\"\"Generate embeddings for a list of images.\"\"\"\n if not images:\n raise RuntimeError(\"No images provided for embedding.\")\n\n dataset = self._list_dataset_cls(images)\n dataloader = DataLoader(dataset, batch_size=1, shuffle=False, collate_fn=lambda x: x)\n\n embeddings = []\n with torch.no_grad():\n for batch in tqdm(dataloader, desc=\"Embedding images\"):\n batch_images = cast(list, batch)\n batch_inputs = self.processor.process_images(batch_images).to(self.device)\n batch_embeddings = self.model(**batch_inputs)\n embeddings.append(batch_embeddings.cpu())\n\n if not embeddings:\n raise RuntimeError(\n \"Image embedding produced no tensors (empty embedding list). \"\n \"This usually indicates that no images were processed successfully.\"\n )\n\n return torch.cat(embeddings, dim=0)\n\n def _embed_query(self, query: str) -> torch.Tensor:\n \"\"\"Generate embedding for a text query.\"\"\"\n with torch.no_grad():\n query_inputs = self.processor.process_queries([query]).to(self.device)\n query_embedding = self.model(**query_inputs)\n return query_embedding.cpu()\n\n\ndef main():\n parser = argparse.ArgumentParser(description=\"ColQwen RAG - Easy multimodal PDF retrieval\")\n subparsers = parser.add_subparsers(dest=\"command\", help=\"Available commands\")\n\n # Build command\n build_parser = subparsers.add_parser(\"build\", help=\"Build index from PDFs\")\n build_parser.add_argument(\"--pdfs\", required=True, help=\"Directory containing PDF files\")\n build_parser.add_argument(\"--index\", required=True, help=\"Index name\")\n build_parser.add_argument(\n \"--model\", choices=[\"colqwen2\", \"colpali\"], default=\"colqwen2\", help=\"Model to use\"\n )\n build_parser.add_argument(\"--pages-dir\", help=\"Directory to save page images\")\n\n # Search command\n search_parser = subparsers.add_parser(\"search\", help=\"Search the index\")\n search_parser.add_argument(\"index\", help=\"Index name\")\n search_parser.add_argument(\"query\", help=\"Search query\")\n search_parser.add_argument(\"--top-k\", type=int, default=5, help=\"Number of results\")\n search_parser.add_argument(\n \"--model\", choices=[\"colqwen2\", \"colpali\"], default=\"colqwen2\", help=\"Model to use\"\n )\n\n # Ask command\n ask_parser = subparsers.add_parser(\"ask\", help=\"Interactive Q&A\")\n ask_parser.add_argument(\"index\", help=\"Index name\")\n ask_parser.add_argument(\"--interactive\", action=\"store_true\", help=\"Interactive mode\")\n ask_parser.add_argument(\n \"--model\", choices=[\"colqwen2\", \"colpali\"], default=\"colqwen2\", help=\"Model to use\"\n )\n\n args = parser.parse_args()\n\n if not args.command:\n parser.print_help()\n return\n\n # Initialize ColQwen RAG\n if args.command == \"build\":\n colqwen = ColQwenRAG(args.model)\n\n # Get PDF files\n pdf_dir = Path(args.pdfs)\n if pdf_dir.is_file() and pdf_dir.suffix.lower() == \".pdf\":\n pdf_paths = [str(pdf_dir)]\n elif pdf_dir.is_dir():\n pdf_paths = [str(p) for p in pdf_dir.glob(\"*.pdf\")]\n else:\n print(f\"❌ Invalid PDF path: {args.pdfs}\")\n return\n\n if not pdf_paths:\n print(f\"❌ No PDF files found in {args.pdfs}\")\n return\n\n colqwen.build_index(pdf_paths, args.index, args.pages_dir)\n\n elif args.command == \"search\":\n colqwen = ColQwenRAG(args.model)\n colqwen.search(args.index, args.query, args.top_k)\n\n elif args.command == \"ask\":\n colqwen = ColQwenRAG(args.model)\n colqwen.ask(args.index, args.interactive)\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "apps/document_rag.py", "content": "\"\"\"\nDocument RAG example using the unified interface.\nSupports PDF, TXT, MD, and other document formats.\n\"\"\"\n\nimport sys\nfrom pathlib import Path\nfrom typing import Any\n\n# Add parent directory to path for imports\nsys.path.insert(0, str(Path(__file__).parent))\n\nfrom base_rag_example import BaseRAGExample\nfrom chunking import create_text_chunks\nfrom llama_index.core import SimpleDirectoryReader\n\n\nclass DocumentRAG(BaseRAGExample):\n \"\"\"RAG example for document processing (PDF, TXT, MD, etc.).\"\"\"\n\n def __init__(self):\n super().__init__(\n name=\"Document\",\n description=\"Process and query documents (PDF, TXT, MD, etc.) with LEANN\",\n default_index_name=\"test_doc_files\",\n )\n\n def _add_specific_arguments(self, parser):\n \"\"\"Add document-specific arguments.\"\"\"\n doc_group = parser.add_argument_group(\"Document Parameters\")\n doc_group.add_argument(\n \"--data-dir\",\n type=str,\n default=\"data\",\n help=\"Directory containing documents to index (default: data)\",\n )\n doc_group.add_argument(\n \"--file-types\",\n nargs=\"+\",\n default=None,\n help=\"Filter by file types (e.g., .pdf .txt .md). If not specified, all supported types are processed\",\n )\n doc_group.add_argument(\n \"--chunk-size\", type=int, default=256, help=\"Text chunk size (default: 256)\"\n )\n doc_group.add_argument(\n \"--chunk-overlap\", type=int, default=128, help=\"Text chunk overlap (default: 128)\"\n )\n doc_group.add_argument(\n \"--enable-code-chunking\",\n action=\"store_true\",\n help=\"Enable AST-aware chunking for code files in the data directory\",\n )\n\n async def load_data(self, args) -> list[dict[str, Any]]:\n \"\"\"Load documents and convert to text chunks.\"\"\"\n print(f\"Loading documents from: {args.data_dir}\")\n if args.file_types:\n print(f\"Filtering by file types: {args.file_types}\")\n else:\n print(\"Processing all supported file types\")\n\n # Check if data directory exists\n data_path = Path(args.data_dir)\n if not data_path.exists():\n raise ValueError(f\"Data directory not found: {args.data_dir}\")\n\n # Load documents\n documents = SimpleDirectoryReader(\n args.data_dir,\n recursive=True,\n encoding=\"utf-8\",\n required_exts=args.file_types if args.file_types else None,\n ).load_data(show_progress=True)\n\n if not documents:\n print(f\"No documents found in {args.data_dir} with extensions {args.file_types}\")\n return []\n\n print(f\"Loaded {len(documents)} documents\")\n\n # Determine chunking strategy\n use_ast = args.enable_code_chunking or getattr(args, \"use_ast_chunking\", False)\n\n if use_ast:\n print(\"Using AST-aware chunking for code files\")\n\n # Convert to text chunks with optional AST support\n all_texts = create_text_chunks(\n documents,\n chunk_size=args.chunk_size,\n chunk_overlap=args.chunk_overlap,\n use_ast_chunking=use_ast,\n ast_chunk_size=getattr(args, \"ast_chunk_size\", 512),\n ast_chunk_overlap=getattr(args, \"ast_chunk_overlap\", 64),\n code_file_extensions=getattr(args, \"code_file_extensions\", None),\n ast_fallback_traditional=getattr(args, \"ast_fallback_traditional\", True),\n )\n\n # Apply max_items limit if specified\n if args.max_items > 0 and len(all_texts) > args.max_items:\n print(f\"Limiting to {args.max_items} chunks (from {len(all_texts)})\")\n all_texts = all_texts[: args.max_items]\n\n return all_texts\n\n\nif __name__ == \"__main__\":\n import asyncio\n\n # Example queries for document RAG\n print(\"\\nDocument RAG Example\")\n print(\"=\" * 50)\n print(\"\\nExample queries you can try:\")\n print(\"- 'What are the main techniques LEANN uses?'\")\n print(\"- 'What is the technique DLPM?'\")\n print(\"- 'Who does Elizabeth Bennet marry?'\")\n print(\"- 'What challenges did Huawei face while developing the Pangu model?'\")\n print(\"\\nNEW: Code-aware chunking available!\")\n print(\"- Use --enable-code-chunking to enable AST-aware chunking for code files\")\n print(\"- Supports Python, Java, C#, TypeScript files\")\n print(\"- Better semantic understanding of code structure\")\n print(\"\\nOr run without --query for interactive mode\\n\")\n\n rag = DocumentRAG()\n asyncio.run(rag.run())\n" }, { "path": "apps/email_data/LEANN_email_reader.py", "content": "import email\nimport os\nfrom pathlib import Path\nfrom typing import Any\n\nfrom llama_index.core import Document\nfrom llama_index.core.readers.base import BaseReader\n\n\ndef find_all_messages_directories(root: str | None = None) -> list[Path]:\n \"\"\"\n Recursively find all 'Messages' directories under the given root.\n Returns a list of Path objects.\n \"\"\"\n if root is None:\n # Auto-detect user's mail path\n home_dir = os.path.expanduser(\"~\")\n root = os.path.join(home_dir, \"Library\", \"Mail\")\n\n messages_dirs = []\n for dirpath, _dirnames, _filenames in os.walk(root):\n if os.path.basename(dirpath) == \"Messages\":\n messages_dirs.append(Path(dirpath))\n return messages_dirs\n\n\nclass EmlxReader(BaseReader):\n \"\"\"\n Apple Mail .emlx file reader with embedded metadata.\n\n Reads individual .emlx files from Apple Mail's storage format.\n \"\"\"\n\n def __init__(self, include_html: bool = False) -> None:\n \"\"\"\n Initialize.\n\n Args:\n include_html: Whether to include HTML content in the email body (default: False)\n \"\"\"\n self.include_html = include_html\n\n def _payload_to_text(self, payload: object) -> str:\n if isinstance(payload, bytes):\n return payload.decode(\"utf-8\", errors=\"ignore\")\n if isinstance(payload, str):\n return payload\n return \"\"\n\n def load_data(self, input_dir: str, **load_kwargs: Any) -> list[Document]:\n \"\"\"\n Load data from the input directory containing .emlx files.\n\n Args:\n input_dir: Directory containing .emlx files\n **load_kwargs:\n max_count (int): Maximum amount of messages to read.\n \"\"\"\n docs: list[Document] = []\n max_count = load_kwargs.get(\"max_count\", 1000)\n count = 0\n total_files = 0\n successful_files = 0\n failed_files = 0\n\n print(f\"Starting to process directory: {input_dir}\")\n\n # Walk through the directory recursively\n for dirpath, dirnames, filenames in os.walk(input_dir):\n # Skip hidden directories\n dirnames[:] = [d for d in dirnames if not d.startswith(\".\")]\n\n for filename in filenames:\n # Check if we've reached the max count (skip if max_count == -1)\n if max_count > 0 and count >= max_count:\n break\n\n if filename.endswith(\".emlx\"):\n total_files += 1\n filepath = os.path.join(dirpath, filename)\n try:\n # Read the .emlx file\n with open(filepath, encoding=\"utf-8\", errors=\"ignore\") as f:\n content = f.read()\n\n # .emlx files have a length prefix followed by the email content\n # The first line contains the length, followed by the email\n lines = content.split(\"\\n\", 1)\n if len(lines) >= 2:\n email_content = lines[1]\n\n # Parse the email using Python's email module\n try:\n msg = email.message_from_string(email_content)\n\n # Extract email metadata\n subject = msg.get(\"Subject\", \"No Subject\")\n from_addr = msg.get(\"From\", \"Unknown\")\n to_addr = msg.get(\"To\", \"Unknown\")\n date = msg.get(\"Date\", \"Unknown\")\n\n # Extract email body\n body = \"\"\n if msg.is_multipart():\n for part in msg.walk():\n if (\n part.get_content_type() == \"text/plain\"\n or part.get_content_type() == \"text/html\"\n ):\n if (\n part.get_content_type() == \"text/html\"\n and not self.include_html\n ):\n continue\n try:\n payload = part.get_payload(decode=True)\n if payload:\n body += self._payload_to_text(payload)\n except Exception as e:\n print(f\"Error decoding payload: {e}\")\n continue\n else:\n try:\n payload = msg.get_payload(decode=True)\n if payload:\n body = self._payload_to_text(payload)\n except Exception as e:\n print(f\"Error decoding single part payload: {e}\")\n body = \"\"\n\n # Only create document if we have some content\n if body.strip() or subject != \"No Subject\":\n # Create document content with metadata embedded in text\n doc_content = f\"\"\"\n[File]: {filename}\n[From]: {from_addr}\n[To]: {to_addr}\n[Subject]: {subject}\n[Date]: {date}\n[EMAIL BODY Start]:\n{body}\n\"\"\"\n\n # No separate metadata - everything is in the text\n doc = Document(text=doc_content, metadata={})\n docs.append(doc)\n count += 1\n successful_files += 1\n\n # Print first few successful files for debugging\n if successful_files <= 3:\n print(\n f\"Successfully loaded: {filename} - Subject: {subject[:50]}...\"\n )\n\n except Exception as e:\n failed_files += 1\n if failed_files <= 5: # Only print first few errors\n print(f\"Error parsing email from {filepath}: {e}\")\n continue\n\n except Exception as e:\n failed_files += 1\n if failed_files <= 5: # Only print first few errors\n print(f\"Error reading file {filepath}: {e}\")\n continue\n\n print(\"Processing summary:\")\n print(f\" Total .emlx files found: {total_files}\")\n print(f\" Successfully loaded: {successful_files}\")\n print(f\" Failed to load: {failed_files}\")\n print(f\" Final documents: {len(docs)}\")\n\n return docs\n" }, { "path": "apps/email_data/email.py", "content": "\"\"\"\nMbox parser.\n\nContains simple parser for mbox files.\n\n\"\"\"\n\nimport logging\nfrom pathlib import Path\nfrom typing import Any\n\nfrom fsspec import AbstractFileSystem\nfrom llama_index.core.readers.base import BaseReader\nfrom llama_index.core.schema import Document\n\nlogger = logging.getLogger(__name__)\n\n\nclass MboxReader(BaseReader):\n \"\"\"\n Mbox parser.\n\n Extract messages from mailbox files.\n Returns string including date, subject, sender, receiver and\n content for each message.\n\n \"\"\"\n\n DEFAULT_MESSAGE_FORMAT: str = (\n \"Date: {_date}\\nFrom: {_from}\\nTo: {_to}\\nSubject: {_subject}\\nContent: {_content}\"\n )\n\n def __init__(\n self,\n *args: Any,\n max_count: int = 0,\n message_format: str = DEFAULT_MESSAGE_FORMAT,\n **kwargs: Any,\n ) -> None:\n \"\"\"Init params.\"\"\"\n try:\n from bs4 import BeautifulSoup # noqa\n except ImportError:\n raise ImportError(\"`beautifulsoup4` package not found: `pip install beautifulsoup4`\")\n\n super().__init__(*args, **kwargs)\n self.max_count = max_count\n self.message_format = message_format\n\n def _payload_to_text(self, payload: object) -> str:\n if isinstance(payload, bytes):\n return payload.decode(\"utf-8\", errors=\"ignore\")\n if isinstance(payload, str):\n return payload\n return \"\"\n\n def load_data(\n self,\n file: Path,\n extra_info: dict | None = None,\n fs: AbstractFileSystem | None = None,\n ) -> list[Document]:\n \"\"\"Parse file into string.\"\"\"\n # Import required libraries\n import mailbox\n from email.parser import BytesParser\n from email.policy import default\n\n from bs4 import BeautifulSoup\n\n if fs:\n logger.warning(\n \"fs was specified but MboxReader doesn't support loading \"\n \"from fsspec filesystems. Will load from local filesystem instead.\"\n )\n\n i = 0\n results: list[str] = []\n # Load file using mailbox\n bytes_parser = BytesParser(policy=default).parse\n mbox = mailbox.mbox(file, factory=bytes_parser) # type: ignore\n\n # Iterate through all messages\n for _, _msg in enumerate(mbox):\n try:\n msg: mailbox.mboxMessage = _msg\n # Parse multipart messages\n if msg.is_multipart():\n for part in msg.walk():\n ctype = part.get_content_type()\n cdispo = str(part.get(\"Content-Disposition\"))\n if \"attachment\" in cdispo:\n print(f\"Attachment found: {part.get_filename()}\")\n if ctype == \"text/plain\" and \"attachment\" not in cdispo:\n content = part.get_payload(decode=True) # decode\n break\n # Get plain message payload for non-multipart messages\n else:\n content = msg.get_payload(decode=True)\n\n # Parse message HTML content and remove unneeded whitespace\n content_text = self._payload_to_text(content)\n soup = BeautifulSoup(content_text)\n stripped_content = \" \".join(soup.get_text().split())\n # Format message to include date, sender, receiver and subject\n msg_string = self.message_format.format(\n _date=msg[\"date\"],\n _from=msg[\"from\"],\n _to=msg[\"to\"],\n _subject=msg[\"subject\"],\n _content=stripped_content,\n )\n # Add message string to results\n results.append(msg_string)\n except Exception as e:\n logger.warning(f\"Failed to parse message:\\n{_msg}\\n with exception {e}\")\n\n # Increment counter and return if max count is met\n i += 1\n if self.max_count > 0 and i >= self.max_count:\n break\n\n return [Document(text=result, metadata=extra_info or {}) for result in results]\n\n\nclass EmlxMboxReader(MboxReader):\n \"\"\"\n EmlxMboxReader - Modified MboxReader that handles directories of .emlx files.\n\n Extends MboxReader to work with Apple Mail's .emlx format by:\n 1. Reading .emlx files from a directory\n 2. Converting them to mbox format in memory\n 3. Using the parent MboxReader's parsing logic\n \"\"\"\n\n def load_data(\n self,\n file: Path, # Note: for EmlxMboxReader, this is actually a directory\n extra_info: dict | None = None,\n fs: AbstractFileSystem | None = None,\n ) -> list[Document]:\n \"\"\"Parse .emlx files from directory into strings using MboxReader logic.\"\"\"\n directory = file # Rename for clarity - this is a directory of .emlx files\n import os\n import tempfile\n\n if fs:\n logger.warning(\n \"fs was specified but EmlxMboxReader doesn't support loading \"\n \"from fsspec filesystems. Will load from local filesystem instead.\"\n )\n\n # Find all .emlx files in the directory\n emlx_files = list(directory.glob(\"*.emlx\"))\n logger.info(f\"Found {len(emlx_files)} .emlx files in {directory}\")\n\n if not emlx_files:\n logger.warning(f\"No .emlx files found in {directory}\")\n return []\n\n # Create a temporary mbox file\n with tempfile.NamedTemporaryFile(mode=\"w\", suffix=\".mbox\", delete=False) as temp_mbox:\n temp_mbox_path = temp_mbox.name\n\n # Convert .emlx files to mbox format\n for emlx_file in emlx_files:\n try:\n # Read the .emlx file\n with open(emlx_file, encoding=\"utf-8\", errors=\"ignore\") as f:\n content = f.read()\n\n # .emlx format: first line is length, rest is email content\n lines = content.split(\"\\n\", 1)\n if len(lines) >= 2:\n email_content = lines[1] # Skip the length line\n\n # Write to mbox format (each message starts with \"From \" and ends with blank line)\n temp_mbox.write(f\"From {emlx_file.name} {email_content}\\n\\n\")\n\n except Exception as e:\n logger.warning(f\"Failed to process {emlx_file}: {e}\")\n continue\n\n # Close the temporary file so MboxReader can read it\n temp_mbox.close()\n\n try:\n # Use the parent MboxReader's logic to parse the mbox file\n return super().load_data(Path(temp_mbox_path), extra_info, fs)\n finally:\n # Clean up temporary file\n try:\n os.unlink(temp_mbox_path)\n except OSError:\n pass\n" }, { "path": "apps/email_rag.py", "content": "\"\"\"\nEmail RAG example using the unified interface.\nSupports Apple Mail on macOS.\n\"\"\"\n\nimport sys\nfrom pathlib import Path\nfrom typing import Any\n\n# Add parent directory to path for imports\nsys.path.insert(0, str(Path(__file__).parent))\n\nfrom base_rag_example import BaseRAGExample\nfrom chunking import create_text_chunks\n\nfrom .email_data.LEANN_email_reader import EmlxReader\n\n\nclass EmailRAG(BaseRAGExample):\n \"\"\"RAG example for Apple Mail processing.\"\"\"\n\n def __init__(self):\n # Set default values BEFORE calling super().__init__\n self.max_items_default = -1 # Process all emails by default\n self.embedding_model_default = (\n \"sentence-transformers/all-MiniLM-L6-v2\" # Fast 384-dim model\n )\n\n super().__init__(\n name=\"Email\",\n description=\"Process and query Apple Mail emails with LEANN\",\n default_index_name=\"mail_index\",\n )\n\n def _add_specific_arguments(self, parser):\n \"\"\"Add email-specific arguments.\"\"\"\n email_group = parser.add_argument_group(\"Email Parameters\")\n email_group.add_argument(\n \"--mail-path\",\n type=str,\n default=None,\n help=\"Path to Apple Mail directory (auto-detected if not specified)\",\n )\n email_group.add_argument(\n \"--include-html\", action=\"store_true\", help=\"Include HTML content in email processing\"\n )\n email_group.add_argument(\n \"--chunk-size\", type=int, default=256, help=\"Text chunk size (default: 256)\"\n )\n email_group.add_argument(\n \"--chunk-overlap\", type=int, default=25, help=\"Text chunk overlap (default: 25)\"\n )\n\n def _find_mail_directories(self) -> list[Path]:\n \"\"\"Auto-detect all Apple Mail directories.\"\"\"\n mail_base = Path.home() / \"Library\" / \"Mail\"\n if not mail_base.exists():\n return []\n\n # Find all Messages directories\n messages_dirs = []\n for item in mail_base.rglob(\"Messages\"):\n if item.is_dir():\n messages_dirs.append(item)\n\n return messages_dirs\n\n async def load_data(self, args) -> list[dict[str, Any]]:\n \"\"\"Load emails and convert to text chunks.\"\"\"\n # Determine mail directories\n if args.mail_path:\n messages_dirs = [Path(args.mail_path)]\n else:\n print(\"Auto-detecting Apple Mail directories...\")\n messages_dirs = self._find_mail_directories()\n\n if not messages_dirs:\n print(\"No Apple Mail directories found!\")\n print(\"Please specify --mail-path manually\")\n return []\n\n print(f\"Found {len(messages_dirs)} mail directories\")\n\n # Create reader\n reader = EmlxReader(include_html=args.include_html)\n\n # Process each directory\n all_documents = []\n total_processed = 0\n\n for i, messages_dir in enumerate(messages_dirs):\n print(f\"\\nProcessing directory {i + 1}/{len(messages_dirs)}: {messages_dir}\")\n\n try:\n # Count emlx files\n emlx_files = list(messages_dir.glob(\"*.emlx\"))\n print(f\"Found {len(emlx_files)} email files\")\n\n # Apply max_items limit per directory\n max_per_dir = -1 # Default to process all\n if args.max_items > 0:\n remaining = args.max_items - total_processed\n if remaining <= 0:\n break\n max_per_dir = remaining\n # If args.max_items == -1, max_per_dir stays -1 (process all)\n\n # Load emails - fix the parameter passing\n documents = reader.load_data(\n input_dir=str(messages_dir),\n max_count=max_per_dir,\n )\n\n if documents:\n all_documents.extend(documents)\n total_processed += len(documents)\n print(f\"Processed {len(documents)} emails from this directory\")\n\n except Exception as e:\n print(f\"Error processing {messages_dir}: {e}\")\n continue\n\n if not all_documents:\n print(\"No emails found to process!\")\n return []\n\n print(f\"\\nTotal emails processed: {len(all_documents)}\")\n print(\"now starting to split into text chunks ... take some time\")\n\n # Convert to text chunks\n # Email reader uses chunk_overlap=25 as in original\n all_texts = create_text_chunks(\n all_documents, chunk_size=args.chunk_size, chunk_overlap=args.chunk_overlap\n )\n\n return all_texts\n\n\nif __name__ == \"__main__\":\n import asyncio\n\n # Check platform\n if sys.platform != \"darwin\":\n print(\"\\n⚠️ Warning: This example is designed for macOS (Apple Mail)\")\n print(\" Windows/Linux support coming soon!\\n\")\n\n # Example queries for email RAG\n print(\"\\n📧 Email RAG Example\")\n print(\"=\" * 50)\n print(\"\\nExample queries you can try:\")\n print(\"- 'What did my boss say about deadlines?'\")\n print(\"- 'Find emails about travel expenses'\")\n print(\"- 'Show me emails from last month about the project'\")\n print(\"- 'What food did I order from DoorDash?'\")\n print(\"\\nNote: You may need to grant Full Disk Access to your terminal\\n\")\n\n rag = EmailRAG()\n asyncio.run(rag.run())\n" }, { "path": "apps/gemini_data/__init__.py", "content": "" }, { "path": "apps/gemini_data/gemini_reader.py", "content": "import json\nimport logging\nfrom pathlib import Path\nfrom typing import Any\n\nlogger = logging.getLogger(__name__)\n\n\nclass GeminiReader:\n \"\"\"Reader for Gemini CLI history files.\"\"\"\n\n def __init__(self):\n pass\n\n def load_data(self, history_dir: str, max_count: int = -1) -> list[dict[str, Any]]:\n \"\"\"\n Load data from Gemini history directory.\n\n Args:\n history_dir: Path to .gemini directory\n max_count: Max number of conversations to load\n\n Returns:\n List of dictionaries with 'text' and 'metadata' keys\n \"\"\"\n history_path = Path(history_dir).expanduser()\n if not history_path.exists():\n print(f\"Gemini history directory not found: {history_path}\")\n return []\n\n documents = []\n\n # 1. Load Memory (GEMINI.md)\n memory_file = history_path / \"GEMINI.md\"\n if memory_file.exists():\n try:\n text = memory_file.read_text(encoding=\"utf-8\")\n if text.strip():\n documents.append(\n {\n \"text\": f\"Gemini Memory:\\n{text}\",\n \"metadata\": {\"source\": str(memory_file), \"type\": \"memory\"},\n }\n )\n except Exception as e:\n print(f\"Error reading memory file: {e}\")\n\n # 2. Find Session Files\n # Legacy JSON sessions\n session_files = list(history_path.glob(\"session-*.json\"))\n # New JSONL sessions\n session_files.extend(list(history_path.glob(\"session-*.jsonl\")))\n # Checkpoints\n session_files.extend(list(history_path.glob(\"checkpoint-*.json\")))\n\n # Sort by modification time (newest first)\n session_files.sort(key=lambda x: x.stat().st_mtime, reverse=True)\n\n print(f\"Found {len(session_files)} session files.\")\n\n count = 0\n for file_path in session_files:\n if max_count > 0 and count >= max_count:\n break\n\n try:\n content = \"\"\n if file_path.suffix == \".jsonl\":\n content = self._parse_jsonl_session(file_path)\n elif file_path.suffix == \".json\":\n content = self._parse_json_session(file_path)\n\n if content:\n documents.append(\n {\n \"text\": content,\n \"metadata\": {\n \"source\": str(file_path),\n \"type\": \"session\",\n \"filename\": file_path.name,\n },\n }\n )\n count += 1\n except Exception as e:\n print(f\"Error reading {file_path.name}: {e}\")\n\n print(f\"Successfully loaded {len(documents)} items from Gemini history.\")\n return documents\n\n def _parse_json_session(self, file_path: Path) -> str:\n \"\"\"Parse legacy JSON session file.\"\"\"\n data = json.loads(file_path.read_text(encoding=\"utf-8\"))\n\n # Handle dict format (standard session)\n messages = []\n if isinstance(data, dict):\n # Check for 'messages' key (standard format)\n if \"messages\" in data:\n for msg in data[\"messages\"]:\n role = msg.get(\"role\", \"unknown\")\n content = msg.get(\"content\", \"\")\n if content:\n messages.append(f\"{role.upper()}: {content}\")\n # Check for 'parts' key (checkpoint format sometimes)\n elif \"parts\" in data:\n messages.append(f\"Saved Session Content: {data['parts']}\")\n\n # Handle list format (some older array-based sessions)\n elif isinstance(data, list):\n for item in data:\n if isinstance(item, dict):\n role = item.get(\"role\", \"unknown\")\n content = item.get(\"content\", \"\") or item.get(\"parts\", \"\")\n if content:\n messages.append(f\"{role.upper()}: {content}\")\n\n if not messages:\n return \"\"\n\n return f\"File: {file_path.name}\\n\\n\" + \"\\n\\n\".join(messages)\n\n def _parse_jsonl_session(self, file_path: Path) -> str:\n \"\"\"Parse JSONL session file.\"\"\"\n messages = []\n try:\n with open(file_path, encoding=\"utf-8\") as f:\n for line in f:\n line = line.strip()\n if not line:\n continue\n try:\n data = json.loads(line)\n # Skip metadata lines if they don't have content\n if \"role\" in data and \"content\" in data:\n messages.append(f\"{data['role'].upper()}: {data['content']}\")\n elif \"parts\" in data: # sometimes parts is used\n messages.append(\n f\"{data.get('role', 'unknown').upper()}: {data['parts']}\"\n )\n except json.JSONDecodeError:\n continue\n except Exception:\n return \"\"\n\n if not messages:\n return \"\"\n\n return f\"File: {file_path.name}\\n\\n\" + \"\\n\\n\".join(messages)\n" }, { "path": "apps/gemini_rag.py", "content": "\"\"\"\nGemini CLI RAG example.\nIndexes and searches Gemini CLI history (~/.gemini).\n\"\"\"\n\nimport sys\nfrom pathlib import Path\nfrom typing import Any\n\n# Add parent directory to path for imports\nsys.path.insert(0, str(Path(__file__).parent))\n\nfrom base_rag_example import BaseRAGExample\nfrom chunking import create_text_chunks\n\nfrom .gemini_data.gemini_reader import GeminiReader\n\n\nclass GeminiRAG(BaseRAGExample):\n \"\"\"RAG example for Gemini CLI history.\"\"\"\n\n def __init__(self):\n super().__init__(\n name=\"Gemini CLI\",\n description=\"Process and query Gemini CLI history with LEANN\",\n default_index_name=\"gemini_index\",\n )\n\n def _add_specific_arguments(self, parser):\n \"\"\"Add Gemini-specific arguments.\"\"\"\n group = parser.add_argument_group(\"Gemini Parameters\")\n group.add_argument(\n \"--gemini-path\",\n type=str,\n default=\"~/.gemini\",\n help=\"Path to .gemini directory (default: ~/.gemini)\",\n )\n\n async def load_data(self, args) -> list[dict[str, Any]]:\n \"\"\"Load Gemini history and convert to text chunks.\"\"\"\n print(f\"Loading Gemini history from: {args.gemini_path}\")\n\n reader = GeminiReader()\n documents = reader.load_data(history_dir=args.gemini_path, max_count=args.max_items)\n\n if not documents:\n print(\"No documents found! Check if ~/.gemini exists and has history.\")\n return []\n\n # Convert dicts to Document objects for chunking\n from llama_index.core import Document\n\n docs = [Document(text=d[\"text\"], metadata=d[\"metadata\"]) for d in documents]\n\n # Convert to text chunks\n print(f\"splitting {len(documents)} documents into chunks...\")\n chunks = create_text_chunks(docs)\n\n return chunks\n\n\nif __name__ == \"__main__\":\n import asyncio\n\n print(\"\\n✨ Gemini CLI RAG\")\n print(\"=\" * 50)\n\n rag = GeminiRAG()\n asyncio.run(rag.run())\n" }, { "path": "apps/history_data/__init__.py", "content": "from .history import ChromeHistoryReader\n\n__all__ = [\"ChromeHistoryReader\"]\n" }, { "path": "apps/history_data/history.py", "content": "import os\nimport sqlite3\nfrom pathlib import Path\nfrom typing import Any\n\nfrom llama_index.core import Document\nfrom llama_index.core.readers.base import BaseReader\n\n\nclass ChromeHistoryReader(BaseReader):\n \"\"\"\n Chrome browser history reader that extracts browsing data from SQLite database.\n\n Reads Chrome history from the default Chrome profile location and creates documents\n with embedded metadata similar to the email reader structure.\n \"\"\"\n\n def __init__(self) -> None:\n \"\"\"Initialize.\"\"\"\n pass\n\n def load_data(self, input_dir: str | None = None, **load_kwargs: Any) -> list[Document]:\n \"\"\"\n Load Chrome history data from the default Chrome profile location.\n\n Args:\n input_dir: Not used for Chrome history (kept for compatibility)\n **load_kwargs:\n max_count (int): Maximum amount of history entries to read.\n chrome_profile_path (str): Custom path to Chrome profile directory.\n \"\"\"\n docs: list[Document] = []\n max_count = load_kwargs.get(\"max_count\", 1000)\n chrome_profile_path = load_kwargs.get(\"chrome_profile_path\", None)\n\n # Default Chrome profile path on macOS\n if chrome_profile_path is None:\n chrome_profile_path = os.path.expanduser(\n \"~/Library/Application Support/Google/Chrome/Default\"\n )\n\n history_db_path = os.path.join(chrome_profile_path, \"History\")\n\n if not os.path.exists(history_db_path):\n print(f\"Chrome history database not found at: {history_db_path}\")\n return docs\n\n try:\n # Connect to the Chrome history database\n print(f\"Connecting to database: {history_db_path}\")\n conn = sqlite3.connect(history_db_path)\n cursor = conn.cursor()\n\n # Query to get browsing history with metadata (removed created_time column)\n query = \"\"\"\n SELECT\n datetime(last_visit_time/1000000-11644473600,'unixepoch','localtime') as last_visit,\n url,\n title,\n visit_count,\n typed_count,\n hidden\n FROM urls\n ORDER BY last_visit_time DESC\n \"\"\"\n\n print(f\"Executing query on database: {history_db_path}\")\n cursor.execute(query)\n rows = cursor.fetchall()\n print(f\"Query returned {len(rows)} rows\")\n\n count = 0\n for row in rows:\n if count >= max_count and max_count > 0:\n break\n\n last_visit, url, title, visit_count, typed_count, _hidden = row\n\n # Create document content with metadata embedded in text\n doc_content = f\"\"\"\n[Title]: {title}\n[URL of the page]: {url}\n[Last visited time]: {last_visit}\n[Visit times]: {visit_count}\n[Typed times]: {typed_count}\n\"\"\"\n\n # Create document with embedded metadata\n doc = Document(text=doc_content, metadata={\"title\": title[0:150]})\n # if len(title) > 150:\n # print(f\"Title is too long: {title}\")\n docs.append(doc)\n count += 1\n\n conn.close()\n print(f\"Loaded {len(docs)} Chrome history documents\")\n\n except Exception as e:\n print(f\"Error reading Chrome history: {e}\")\n # add you may need to close your browser to make the database file available\n # also highlight in red\n print(\n \"\\033[91mYou may need to close your browser to make the database file available\\033[0m\"\n )\n return docs\n\n return docs\n\n @staticmethod\n def find_chrome_profiles() -> list[Path]:\n \"\"\"\n Find all Chrome profile directories.\n\n Returns:\n List of Path objects pointing to Chrome profile directories\n \"\"\"\n chrome_base_path = Path(os.path.expanduser(\"~/Library/Application Support/Google/Chrome\"))\n profile_dirs = []\n\n if not chrome_base_path.exists():\n print(f\"Chrome directory not found at: {chrome_base_path}\")\n return profile_dirs\n\n # Find all profile directories\n for profile_dir in chrome_base_path.iterdir():\n if profile_dir.is_dir() and profile_dir.name != \"System Profile\":\n history_path = profile_dir / \"History\"\n if history_path.exists():\n profile_dirs.append(profile_dir)\n print(f\"Found Chrome profile: {profile_dir}\")\n\n print(f\"Found {len(profile_dirs)} Chrome profiles\")\n return profile_dirs\n\n @staticmethod\n def export_history_to_file(\n output_file: str = \"chrome_history_export.txt\", max_count: int = 1000\n ):\n \"\"\"\n Export Chrome history to a text file using the same SQL query format.\n\n Args:\n output_file: Path to the output file\n max_count: Maximum number of entries to export\n \"\"\"\n chrome_profile_path = os.path.expanduser(\n \"~/Library/Application Support/Google/Chrome/Default\"\n )\n history_db_path = os.path.join(chrome_profile_path, \"History\")\n\n if not os.path.exists(history_db_path):\n print(f\"Chrome history database not found at: {history_db_path}\")\n return\n\n try:\n conn = sqlite3.connect(history_db_path)\n cursor = conn.cursor()\n\n query = \"\"\"\n SELECT\n datetime(last_visit_time/1000000-11644473600,'unixepoch','localtime') as last_visit,\n url,\n title,\n visit_count,\n typed_count,\n hidden\n FROM urls\n ORDER BY last_visit_time DESC\n LIMIT ?\n \"\"\"\n\n cursor.execute(query, (max_count,))\n rows = cursor.fetchall()\n\n with open(output_file, \"w\", encoding=\"utf-8\") as f:\n for row in rows:\n last_visit, url, title, visit_count, typed_count, hidden = row\n f.write(\n f\"{last_visit}\\t{url}\\t{title}\\t{visit_count}\\t{typed_count}\\t{hidden}\\n\"\n )\n\n conn.close()\n print(f\"Exported {len(rows)} history entries to {output_file}\")\n\n except Exception as e:\n print(f\"Error exporting Chrome history: {e}\")\n" }, { "path": "apps/history_data/wechat_history.py", "content": "import json\nimport os\nimport re\nimport subprocess\nimport time\nfrom datetime import datetime\nfrom pathlib import Path\nfrom typing import Any\n\nfrom llama_index.core import Document\nfrom llama_index.core.readers.base import BaseReader\n\n\nclass WeChatHistoryReader(BaseReader):\n \"\"\"\n WeChat chat history reader that extracts chat data from exported JSON files.\n\n Reads WeChat chat history from exported JSON files (from wechat-exporter tool)\n and creates documents with embedded metadata similar to the Chrome history reader structure.\n\n Also includes utilities for automatic WeChat chat history export.\n \"\"\"\n\n def __init__(self) -> None:\n \"\"\"Initialize.\"\"\"\n self.packages_dir = Path(__file__).parent.parent.parent / \"packages\"\n self.wechat_exporter_dir = self.packages_dir / \"wechat-exporter\"\n self.wechat_decipher_dir = self.packages_dir / \"wechat-decipher-macos\"\n\n def check_wechat_running(self) -> bool:\n \"\"\"Check if WeChat is currently running.\"\"\"\n try:\n result = subprocess.run([\"pgrep\", \"-f\", \"WeChat\"], capture_output=True, text=True)\n return result.returncode == 0\n except Exception:\n return False\n\n def install_wechattweak(self) -> bool:\n \"\"\"Install WeChatTweak CLI tool.\"\"\"\n try:\n # Create wechat-exporter directory if it doesn't exist\n self.wechat_exporter_dir.mkdir(parents=True, exist_ok=True)\n\n wechattweak_path = self.wechat_exporter_dir / \"wechattweak-cli\"\n if not wechattweak_path.exists():\n print(\"Downloading WeChatTweak CLI...\")\n subprocess.run(\n [\n \"curl\",\n \"-L\",\n \"-o\",\n str(wechattweak_path),\n \"https://github.com/JettChenT/WeChatTweak-CLI/releases/latest/download/wechattweak-cli\",\n ],\n check=True,\n )\n\n # Make executable\n wechattweak_path.chmod(0o755)\n\n # Install WeChatTweak\n print(\"Installing WeChatTweak...\")\n subprocess.run([\"sudo\", str(wechattweak_path), \"install\"], check=True)\n return True\n except Exception as e:\n print(f\"Error installing WeChatTweak: {e}\")\n return False\n\n def restart_wechat(self):\n \"\"\"Restart WeChat to apply WeChatTweak.\"\"\"\n try:\n print(\"Restarting WeChat...\")\n subprocess.run([\"pkill\", \"-f\", \"WeChat\"], check=False)\n time.sleep(2)\n subprocess.run([\"open\", \"-a\", \"WeChat\"], check=True)\n time.sleep(5) # Wait for WeChat to start\n except Exception as e:\n print(f\"Error restarting WeChat: {e}\")\n\n def check_api_available(self) -> bool:\n \"\"\"Check if WeChatTweak API is available.\"\"\"\n try:\n result = subprocess.run(\n [\"curl\", \"-s\", \"http://localhost:48065/wechat/allcontacts\"],\n capture_output=True,\n text=True,\n timeout=5,\n )\n return result.returncode == 0 and bool(result.stdout.strip())\n except Exception:\n return False\n\n def _extract_readable_text(self, content: str) -> str:\n \"\"\"\n Extract readable text from message content, removing XML and system messages.\n\n Args:\n content: The raw message content (can be string or dict)\n\n Returns:\n Cleaned, readable text\n \"\"\"\n if not content:\n return \"\"\n\n # Handle dictionary content (like quoted messages)\n if isinstance(content, dict):\n # Extract text from dictionary structure\n text_parts = []\n if \"title\" in content:\n text_parts.append(str(content[\"title\"]))\n if \"quoted\" in content:\n text_parts.append(str(content[\"quoted\"]))\n if \"content\" in content:\n text_parts.append(str(content[\"content\"]))\n if \"text\" in content:\n text_parts.append(str(content[\"text\"]))\n\n if text_parts:\n return \" | \".join(text_parts)\n else:\n # If we can't extract meaningful text from dict, return empty\n return \"\"\n\n # Handle string content\n if not isinstance(content, str):\n return \"\"\n\n # Remove common prefixes like \"wxid_xxx:\\n\"\n clean_content = re.sub(r\"^wxid_[^:]+:\\s*\", \"\", content)\n clean_content = re.sub(r\"^[^:]+:\\s*\", \"\", clean_content)\n\n # If it's just XML or system message, return empty\n if clean_content.strip().startswith(\"<\") or \"recalled a message\" in clean_content:\n return \"\"\n\n return clean_content.strip()\n\n def _is_text_message(self, content: str) -> bool:\n \"\"\"\n Check if a message contains readable text content.\n\n Args:\n content: The message content (can be string or dict)\n\n Returns:\n True if the message contains readable text, False otherwise\n \"\"\"\n if not content:\n return False\n\n # Handle dictionary content\n if isinstance(content, dict):\n # Check if dict has any readable text fields\n text_fields = [\"title\", \"quoted\", \"content\", \"text\"]\n for field in text_fields:\n if content.get(field):\n return True\n return False\n\n # Handle string content\n if not isinstance(content, str):\n return False\n\n # Skip image messages (contain XML with img tags)\n if \" 0 and not clean_content.strip().startswith(\"<\"):\n return True\n\n return False\n\n def _concatenate_messages(\n self,\n messages: list[dict],\n max_length: int = 128,\n time_window_minutes: int = 30,\n overlap_messages: int = 0,\n ) -> list[dict]:\n \"\"\"\n Concatenate messages based on length and time rules.\n\n Args:\n messages: List of message dictionaries\n max_length: Maximum length for concatenated message groups. Use -1 to disable length constraint.\n time_window_minutes: Time window in minutes to group messages together. Use -1 to disable time constraint.\n overlap_messages: Number of messages to overlap between consecutive groups\n\n Returns:\n List of concatenated message groups\n \"\"\"\n if not messages:\n return []\n\n concatenated_groups = []\n current_group = []\n current_length = 0\n last_timestamp = None\n\n for message in messages:\n # Extract message info\n content = message.get(\"content\", \"\")\n message_text = message.get(\"message\", \"\")\n create_time = message.get(\"createTime\", 0)\n message.get(\"fromUser\", \"\")\n message.get(\"toUser\", \"\")\n message.get(\"isSentFromSelf\", False)\n\n # Extract readable text\n readable_text = self._extract_readable_text(content)\n if not readable_text:\n readable_text = message_text\n\n # Skip empty messages\n if not readable_text.strip():\n continue\n\n # Check time window constraint (only if time_window_minutes != -1)\n if time_window_minutes != -1 and last_timestamp is not None and create_time > 0:\n time_diff_minutes = (create_time - last_timestamp) / 60\n if time_diff_minutes > time_window_minutes:\n # Time gap too large, start new group\n if current_group:\n concatenated_groups.append(\n {\n \"messages\": current_group,\n \"total_length\": current_length,\n \"start_time\": current_group[0].get(\"createTime\", 0),\n \"end_time\": current_group[-1].get(\"createTime\", 0),\n }\n )\n # Keep last few messages for overlap\n if overlap_messages > 0 and len(current_group) > overlap_messages:\n current_group = current_group[-overlap_messages:]\n current_length = sum(\n len(\n self._extract_readable_text(msg.get(\"content\", \"\"))\n or msg.get(\"message\", \"\")\n )\n for msg in current_group\n )\n else:\n current_group = []\n current_length = 0\n\n # Check length constraint (only if max_length != -1)\n message_length = len(readable_text)\n if max_length != -1 and current_length + message_length > max_length and current_group:\n # Current group would exceed max length, save it and start new\n concatenated_groups.append(\n {\n \"messages\": current_group,\n \"total_length\": current_length,\n \"start_time\": current_group[0].get(\"createTime\", 0),\n \"end_time\": current_group[-1].get(\"createTime\", 0),\n }\n )\n # Keep last few messages for overlap\n if overlap_messages > 0 and len(current_group) > overlap_messages:\n current_group = current_group[-overlap_messages:]\n current_length = sum(\n len(\n self._extract_readable_text(msg.get(\"content\", \"\"))\n or msg.get(\"message\", \"\")\n )\n for msg in current_group\n )\n else:\n current_group = []\n current_length = 0\n\n # Add message to current group\n current_group.append(message)\n current_length += message_length\n last_timestamp = create_time\n\n # Add the last group if it exists\n if current_group:\n concatenated_groups.append(\n {\n \"messages\": current_group,\n \"total_length\": current_length,\n \"start_time\": current_group[0].get(\"createTime\", 0),\n \"end_time\": current_group[-1].get(\"createTime\", 0),\n }\n )\n\n return concatenated_groups\n\n def _create_concatenated_content(\n self, message_group: dict, contact_name: str\n ) -> tuple[str, str]:\n \"\"\"\n Create concatenated content from a group of messages.\n\n Args:\n message_group: Dictionary containing messages and metadata\n contact_name: Name of the contact\n\n Returns:\n Formatted concatenated content\n \"\"\"\n messages = message_group[\"messages\"]\n start_time = message_group[\"start_time\"]\n end_time = message_group[\"end_time\"]\n\n # Format timestamps\n if start_time:\n try:\n start_timestamp = datetime.fromtimestamp(start_time)\n start_time_str = start_timestamp.strftime(\"%Y-%m-%d %H:%M:%S\")\n except (ValueError, OSError):\n start_time_str = str(start_time)\n else:\n start_time_str = \"Unknown\"\n\n if end_time:\n try:\n end_timestamp = datetime.fromtimestamp(end_time)\n end_time_str = end_timestamp.strftime(\"%Y-%m-%d %H:%M:%S\")\n except (ValueError, OSError):\n end_time_str = str(end_time)\n else:\n end_time_str = \"Unknown\"\n\n # Build concatenated message content\n message_parts = []\n for message in messages:\n content = message.get(\"content\", \"\")\n message_text = message.get(\"message\", \"\")\n create_time = message.get(\"createTime\", 0)\n is_sent_from_self = message.get(\"isSentFromSelf\", False)\n\n # Extract readable text\n readable_text = self._extract_readable_text(content)\n if not readable_text:\n readable_text = message_text\n\n # Format individual message\n if create_time:\n try:\n timestamp = datetime.fromtimestamp(create_time)\n # change to YYYY-MM-DD HH:MM:SS\n time_str = timestamp.strftime(\"%Y-%m-%d %H:%M:%S\")\n except (ValueError, OSError):\n time_str = str(create_time)\n else:\n time_str = \"Unknown\"\n\n sender = \"[Me]\" if is_sent_from_self else \"[Contact]\"\n message_parts.append(f\"({time_str}) {sender}: {readable_text}\")\n\n concatenated_text = \"\\n\".join(message_parts)\n\n # Create final document content\n doc_content = f\"\"\"\nContact: {contact_name}\nTime Range: {start_time_str} - {end_time_str}\nMessages ({len(messages)} messages, {message_group[\"total_length\"]} chars):\n\n{concatenated_text}\n\"\"\"\n # TODO @yichuan give better format and rich info here!\n doc_content = f\"\"\"\n{concatenated_text}\n\"\"\"\n return doc_content, contact_name\n\n def load_data(self, input_dir: str | None = None, **load_kwargs: Any) -> list[Document]:\n \"\"\"\n Load WeChat chat history data from exported JSON files.\n\n Args:\n input_dir: Directory containing exported WeChat JSON files\n **load_kwargs:\n max_count (int): Maximum amount of chat entries to read.\n wechat_export_dir (str): Custom path to WeChat export directory.\n include_non_text (bool): Whether to include non-text messages (images, emojis, etc.)\n concatenate_messages (bool): Whether to concatenate messages based on length rules.\n max_length (int): Maximum length for concatenated message groups (default: 1000).\n time_window_minutes (int): Time window in minutes to group messages together (default: 30).\n overlap_messages (int): Number of messages to overlap between consecutive groups (default: 2).\n \"\"\"\n docs: list[Document] = []\n max_count = load_kwargs.get(\"max_count\", 1000)\n wechat_export_dir = load_kwargs.get(\"wechat_export_dir\", None)\n include_non_text = load_kwargs.get(\"include_non_text\", False)\n concatenate_messages = load_kwargs.get(\"concatenate_messages\", False)\n max_length = load_kwargs.get(\"max_length\", 1000)\n time_window_minutes = load_kwargs.get(\"time_window_minutes\", 30)\n\n # Default WeChat export path\n if wechat_export_dir is None:\n wechat_export_dir = \"./wechat_export_test\"\n\n if not os.path.exists(wechat_export_dir):\n print(f\"WeChat export directory not found at: {wechat_export_dir}\")\n return docs\n\n try:\n # Find all JSON files in the export directory\n json_files = list(Path(wechat_export_dir).glob(\"*.json\"))\n print(f\"Found {len(json_files)} WeChat chat history files\")\n\n count = 0\n for json_file in json_files:\n if count >= max_count and max_count > 0:\n break\n\n try:\n with open(json_file, encoding=\"utf-8\") as f:\n chat_data = json.load(f)\n\n # Extract contact name from filename\n contact_name = json_file.stem\n\n if concatenate_messages:\n # Filter messages to only include readable text messages\n readable_messages = []\n for message in chat_data:\n try:\n content = message.get(\"content\", \"\")\n if not include_non_text and not self._is_text_message(content):\n continue\n\n readable_text = self._extract_readable_text(content)\n if not readable_text and not include_non_text:\n continue\n\n readable_messages.append(message)\n except Exception as e:\n print(f\"Error processing message in {json_file}: {e}\")\n continue\n\n # Concatenate messages based on rules\n message_groups = self._concatenate_messages(\n readable_messages,\n max_length=max_length,\n time_window_minutes=time_window_minutes,\n overlap_messages=0, # No overlap between groups\n )\n\n # Create documents from concatenated groups\n for message_group in message_groups:\n if count >= max_count and max_count > 0:\n break\n\n doc_content, contact_name = self._create_concatenated_content(\n message_group, contact_name\n )\n doc = Document(\n text=doc_content,\n metadata={\"contact_name\": contact_name},\n )\n docs.append(doc)\n count += 1\n\n print(\n f\"Created {len(message_groups)} concatenated message groups for {contact_name}\"\n )\n\n else:\n # Original single-message processing\n for message in chat_data:\n if count >= max_count and max_count > 0:\n break\n\n # Extract message information\n message.get(\"fromUser\", \"\")\n message.get(\"toUser\", \"\")\n content = message.get(\"content\", \"\")\n message_text = message.get(\"message\", \"\")\n create_time = message.get(\"createTime\", 0)\n is_sent_from_self = message.get(\"isSentFromSelf\", False)\n\n # Handle content that might be dict or string\n try:\n # Check if this is a readable text message\n if not include_non_text and not self._is_text_message(content):\n continue\n\n # Extract readable text\n readable_text = self._extract_readable_text(content)\n if not readable_text and not include_non_text:\n continue\n except Exception as e:\n # Skip messages that cause processing errors\n print(f\"Error processing message in {json_file}: {e}\")\n continue\n\n # Convert timestamp to readable format\n if create_time:\n try:\n timestamp = datetime.fromtimestamp(create_time)\n time_str = timestamp.strftime(\"%Y-%m-%d %H:%M:%S\")\n except (ValueError, OSError):\n time_str = str(create_time)\n else:\n time_str = \"Unknown\"\n\n # Create document content with metadata header and contact info\n doc_content = f\"\"\"\nContact: {contact_name}\nIs sent from self: {is_sent_from_self}\nTime: {time_str}\nMessage: {readable_text if readable_text else message_text}\n\"\"\"\n\n # Create document with embedded metadata\n doc = Document(\n text=doc_content, metadata={\"contact_name\": contact_name}\n )\n docs.append(doc)\n count += 1\n\n except Exception as e:\n print(f\"Error reading {json_file}: {e}\")\n continue\n\n print(f\"Loaded {len(docs)} WeChat chat documents\")\n\n except Exception as e:\n print(f\"Error reading WeChat history: {e}\")\n return docs\n\n return docs\n\n @staticmethod\n def find_wechat_export_dirs() -> list[Path]:\n \"\"\"\n Find all WeChat export directories.\n\n Returns:\n List of Path objects pointing to WeChat export directories\n \"\"\"\n export_dirs = []\n\n # Look for common export directory names\n possible_dirs = [\n Path(\"./wechat_export\"),\n Path(\"./wechat_export_direct\"),\n Path(\"./wechat_chat_history\"),\n Path(\"./chat_export\"),\n ]\n\n for export_dir in possible_dirs:\n if export_dir.exists() and export_dir.is_dir():\n json_files = list(export_dir.glob(\"*.json\"))\n if json_files:\n export_dirs.append(export_dir)\n print(\n f\"Found WeChat export directory: {export_dir} with {len(json_files)} files\"\n )\n\n print(f\"Found {len(export_dirs)} WeChat export directories\")\n return export_dirs\n\n @staticmethod\n def export_chat_to_file(\n output_file: str = \"wechat_chat_export.txt\",\n max_count: int = 1000,\n export_dir: str | None = None,\n include_non_text: bool = False,\n ):\n \"\"\"\n Export WeChat chat history to a text file.\n\n Args:\n output_file: Path to the output file\n max_count: Maximum number of entries to export\n export_dir: Directory containing WeChat JSON files\n include_non_text: Whether to include non-text messages\n \"\"\"\n if export_dir is None:\n export_dir = \"./wechat_export_test\"\n\n if not os.path.exists(export_dir):\n print(f\"WeChat export directory not found at: {export_dir}\")\n return\n\n try:\n json_files = list(Path(export_dir).glob(\"*.json\"))\n\n with open(output_file, \"w\", encoding=\"utf-8\") as f:\n count = 0\n for json_file in json_files:\n if count >= max_count and max_count > 0:\n break\n\n try:\n with open(json_file, encoding=\"utf-8\") as json_f:\n chat_data = json.load(json_f)\n\n contact_name = json_file.stem\n f.write(f\"\\n=== Chat with {contact_name} ===\\n\")\n\n for message in chat_data:\n if count >= max_count and max_count > 0:\n break\n\n from_user = message.get(\"fromUser\", \"\")\n content = message.get(\"content\", \"\")\n message_text = message.get(\"message\", \"\")\n create_time = message.get(\"createTime\", 0)\n\n # Skip non-text messages unless requested\n if not include_non_text:\n reader = WeChatHistoryReader()\n if not reader._is_text_message(content):\n continue\n readable_text = reader._extract_readable_text(content)\n if not readable_text:\n continue\n message_text = readable_text\n\n if create_time:\n try:\n timestamp = datetime.fromtimestamp(create_time)\n time_str = timestamp.strftime(\"%Y-%m-%d %H:%M:%S\")\n except (ValueError, OSError):\n time_str = str(create_time)\n else:\n time_str = \"Unknown\"\n\n f.write(f\"[{time_str}] {from_user}: {message_text}\\n\")\n count += 1\n\n except Exception as e:\n print(f\"Error processing {json_file}: {e}\")\n continue\n\n print(f\"Exported {count} chat entries to {output_file}\")\n\n except Exception as e:\n print(f\"Error exporting WeChat chat history: {e}\")\n\n def export_wechat_chat_history(self, export_dir: str = \"./wechat_export_direct\") -> Path | None:\n \"\"\"\n Export WeChat chat history using wechat-exporter tool.\n\n Args:\n export_dir: Directory to save exported chat history\n\n Returns:\n Path to export directory if successful, None otherwise\n \"\"\"\n try:\n import subprocess\n import sys\n\n # Create export directory\n export_path = Path(export_dir)\n export_path.mkdir(exist_ok=True)\n\n print(f\"Exporting WeChat chat history to {export_path}...\")\n\n # Check if wechat-exporter directory exists\n if not self.wechat_exporter_dir.exists():\n print(f\"wechat-exporter directory not found at: {self.wechat_exporter_dir}\")\n return None\n\n # Install requirements if needed\n requirements_file = self.wechat_exporter_dir / \"requirements.txt\"\n if requirements_file.exists():\n print(\"Installing wechat-exporter requirements...\")\n subprocess.run([\"uv\", \"pip\", \"install\", \"-r\", str(requirements_file)], check=True)\n\n # Run the export command\n print(\"Running wechat-exporter...\")\n result = subprocess.run(\n [\n sys.executable,\n str(self.wechat_exporter_dir / \"main.py\"),\n \"export-all\",\n str(export_path),\n ],\n capture_output=True,\n text=True,\n check=True,\n )\n\n print(\"Export command output:\")\n print(result.stdout)\n if result.stderr:\n print(\"Export errors:\")\n print(result.stderr)\n\n # Check if export was successful\n if export_path.exists() and any(export_path.glob(\"*.json\")):\n json_files = list(export_path.glob(\"*.json\"))\n print(\n f\"Successfully exported {len(json_files)} chat history files to {export_path}\"\n )\n return export_path\n else:\n print(\"Export completed but no JSON files found\")\n return None\n\n except subprocess.CalledProcessError as e:\n print(f\"Export command failed: {e}\")\n print(f\"Command output: {e.stdout}\")\n print(f\"Command errors: {e.stderr}\")\n return None\n except Exception as e:\n print(f\"Export failed: {e}\")\n print(\"Please ensure WeChat is running and WeChatTweak is installed.\")\n return None\n\n def find_or_export_wechat_data(self, export_dir: str = \"./wechat_export_direct\") -> list[Path]:\n \"\"\"\n Find existing WeChat exports or create new ones.\n\n Args:\n export_dir: Directory to save exported chat history if needed\n\n Returns:\n List of Path objects pointing to WeChat export directories\n \"\"\"\n export_dirs = []\n\n # Look for existing exports in common locations\n possible_export_dirs = [\n Path(\"./wechat_database_export\"),\n Path(\"./wechat_export_test\"),\n Path(\"./wechat_export\"),\n Path(\"./wechat_export_direct\"),\n Path(\"./wechat_chat_history\"),\n Path(\"./chat_export\"),\n ]\n\n for export_dir_path in possible_export_dirs:\n if export_dir_path.exists() and any(export_dir_path.glob(\"*.json\")):\n export_dirs.append(export_dir_path)\n print(f\"Found existing export: {export_dir_path}\")\n\n # If no existing exports, try to export automatically\n if not export_dirs:\n print(\"No existing WeChat exports found. Starting direct export...\")\n\n # Try to export using wechat-exporter\n exported_path = self.export_wechat_chat_history(export_dir)\n if exported_path:\n export_dirs = [exported_path]\n else:\n print(\n \"Failed to export WeChat data. Please ensure WeChat is running and WeChatTweak is installed.\"\n )\n\n return export_dirs\n" }, { "path": "apps/image_rag.py", "content": "#!/usr/bin/env python3\n\"\"\"\nCLIP Image RAG Application\n\nThis application enables RAG (Retrieval-Augmented Generation) on images using CLIP embeddings.\nYou can index a directory of images and search them using text queries.\n\nUsage:\n python -m apps.image_rag --image-dir ./my_images/ --query \"a sunset over mountains\"\n python -m apps.image_rag --image-dir ./my_images/ --interactive\n\"\"\"\n\nimport argparse\nimport pickle\nimport tempfile\nfrom pathlib import Path\nfrom typing import Any\n\nimport numpy as np\nfrom PIL import Image\nfrom sentence_transformers import SentenceTransformer\nfrom tqdm import tqdm\n\nfrom apps.base_rag_example import BaseRAGExample\n\n\nclass ImageRAG(BaseRAGExample):\n \"\"\"\n RAG application for images using CLIP embeddings.\n\n This class provides a complete RAG pipeline for image data, including\n CLIP embedding generation, indexing, and text-based image search.\n \"\"\"\n\n def __init__(self):\n super().__init__(\n name=\"Image RAG\",\n description=\"RAG application for images using CLIP embeddings\",\n default_index_name=\"image_index\",\n )\n # Override default embedding model to use CLIP\n self.embedding_model_default = \"clip-ViT-L-14\"\n self.embedding_mode_default = \"sentence-transformers\"\n self._image_data: list[dict] = []\n\n def _add_specific_arguments(self, parser: argparse.ArgumentParser):\n \"\"\"Add image-specific arguments.\"\"\"\n image_group = parser.add_argument_group(\"Image Parameters\")\n image_group.add_argument(\n \"--image-dir\",\n type=str,\n required=True,\n help=\"Directory containing images to index\",\n )\n image_group.add_argument(\n \"--image-extensions\",\n type=str,\n nargs=\"+\",\n default=[\".jpg\", \".jpeg\", \".png\", \".gif\", \".bmp\", \".webp\"],\n help=\"Image file extensions to process (default: .jpg .jpeg .png .gif .bmp .webp)\",\n )\n image_group.add_argument(\n \"--batch-size\",\n type=int,\n default=32,\n help=\"Batch size for CLIP embedding generation (default: 32)\",\n )\n\n async def load_data(self, args) -> list[dict[str, Any]]:\n \"\"\"Load images, generate CLIP embeddings, and return text descriptions.\"\"\"\n self._image_data = self._load_images_and_embeddings(args)\n return [entry[\"text\"] for entry in self._image_data]\n\n def _load_images_and_embeddings(self, args) -> list[dict]:\n \"\"\"Helper to process images and produce embeddings/metadata.\"\"\"\n image_dir = Path(args.image_dir)\n if not image_dir.exists():\n raise ValueError(f\"Image directory does not exist: {image_dir}\")\n\n print(f\"📸 Loading images from {image_dir}...\")\n\n # Find all image files\n image_files = []\n for ext in args.image_extensions:\n image_files.extend(image_dir.rglob(f\"*{ext}\"))\n image_files.extend(image_dir.rglob(f\"*{ext.upper()}\"))\n\n if not image_files:\n raise ValueError(\n f\"No images found in {image_dir} with extensions {args.image_extensions}\"\n )\n\n print(f\"✅ Found {len(image_files)} images\")\n\n # Limit if max_items is set\n if args.max_items > 0:\n image_files = image_files[: args.max_items]\n print(f\"📊 Processing {len(image_files)} images (limited by --max-items)\")\n\n # Load CLIP model\n print(\"🔍 Loading CLIP model...\")\n model = SentenceTransformer(self.embedding_model_default)\n\n # Process images and generate embeddings\n print(\"🖼️ Processing images and generating embeddings...\")\n image_data = []\n batch_images = []\n batch_paths = []\n\n for image_path in tqdm(image_files, desc=\"Processing images\"):\n try:\n image = Image.open(image_path).convert(\"RGB\")\n batch_images.append(image)\n batch_paths.append(image_path)\n\n # Process in batches\n if len(batch_images) >= args.batch_size:\n embeddings = model.encode(\n batch_images,\n convert_to_numpy=True,\n normalize_embeddings=True,\n batch_size=args.batch_size,\n show_progress_bar=False,\n )\n\n for img_path, embedding in zip(batch_paths, embeddings):\n image_data.append(\n {\n \"text\": f\"Image: {img_path.name}\\nPath: {img_path}\",\n \"metadata\": {\n \"image_path\": str(img_path),\n \"image_name\": img_path.name,\n \"image_dir\": str(image_dir),\n },\n \"embedding\": embedding.astype(np.float32),\n }\n )\n\n batch_images = []\n batch_paths = []\n\n except Exception as e:\n print(f\"⚠️ Failed to process {image_path}: {e}\")\n continue\n\n # Process remaining images\n if batch_images:\n embeddings = model.encode(\n batch_images,\n convert_to_numpy=True,\n normalize_embeddings=True,\n batch_size=len(batch_images),\n show_progress_bar=False,\n )\n\n for img_path, embedding in zip(batch_paths, embeddings):\n image_data.append(\n {\n \"text\": f\"Image: {img_path.name}\\nPath: {img_path}\",\n \"metadata\": {\n \"image_path\": str(img_path),\n \"image_name\": img_path.name,\n \"image_dir\": str(image_dir),\n },\n \"embedding\": embedding.astype(np.float32),\n }\n )\n\n print(f\"✅ Processed {len(image_data)} images\")\n return image_data\n\n async def build_index(self, args, texts: list[dict[str, Any]]) -> str:\n \"\"\"Build index using pre-computed CLIP embeddings.\"\"\"\n from leann.api import LeannBuilder\n\n if not self._image_data or len(self._image_data) != len(texts):\n raise RuntimeError(\"No image data found. Make sure load_data() ran successfully.\")\n\n print(\"🔨 Building LEANN index with CLIP embeddings...\")\n builder = LeannBuilder(\n backend_name=args.backend_name,\n embedding_model=self.embedding_model_default,\n embedding_mode=self.embedding_mode_default,\n is_recompute=False,\n distance_metric=\"cosine\",\n graph_degree=args.graph_degree,\n build_complexity=args.build_complexity,\n is_compact=not args.no_compact,\n )\n\n for text, data in zip(texts, self._image_data):\n builder.add_text(text=text, metadata=data[\"metadata\"])\n\n ids = [str(i) for i in range(len(self._image_data))]\n embeddings = np.array([data[\"embedding\"] for data in self._image_data], dtype=np.float32)\n\n with tempfile.NamedTemporaryFile(mode=\"wb\", suffix=\".pkl\", delete=False) as f:\n pickle.dump((ids, embeddings), f)\n pkl_path = f.name\n\n try:\n index_path = str(Path(args.index_dir) / f\"{self.default_index_name}.leann\")\n builder.build_index_from_embeddings(index_path, pkl_path)\n print(f\"✅ Index built successfully at {index_path}\")\n return index_path\n finally:\n Path(pkl_path).unlink()\n\n\ndef main():\n \"\"\"Main entry point for the image RAG application.\"\"\"\n import asyncio\n\n app = ImageRAG()\n asyncio.run(app.run())\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "apps/imessage_data/__init__.py", "content": "\"\"\"iMessage data processing module.\"\"\"\n" }, { "path": "apps/imessage_data/imessage_reader.py", "content": "\"\"\"\niMessage data reader.\n\nReads and processes iMessage conversation data from the macOS Messages database.\n\"\"\"\n\nimport sqlite3\nfrom datetime import datetime\nfrom pathlib import Path\nfrom typing import Any\n\nfrom llama_index.core import Document\nfrom llama_index.core.readers.base import BaseReader\n\n\nclass IMessageReader(BaseReader):\n \"\"\"\n iMessage data reader.\n\n Reads iMessage conversation data from the macOS Messages database (chat.db).\n Processes conversations into structured documents with metadata.\n \"\"\"\n\n def __init__(self, concatenate_conversations: bool = True) -> None:\n \"\"\"\n Initialize.\n\n Args:\n concatenate_conversations: Whether to concatenate messages within conversations for better context\n \"\"\"\n self.concatenate_conversations = concatenate_conversations\n\n def _get_default_chat_db_path(self) -> Path:\n \"\"\"\n Get the default path to the iMessage chat database.\n\n Returns:\n Path to the chat.db file\n \"\"\"\n home = Path.home()\n return home / \"Library\" / \"Messages\" / \"chat.db\"\n\n def _convert_cocoa_timestamp(self, cocoa_timestamp: int) -> str:\n \"\"\"\n Convert Cocoa timestamp to readable format.\n\n Args:\n cocoa_timestamp: Timestamp in Cocoa format (nanoseconds since 2001-01-01)\n\n Returns:\n Formatted timestamp string\n \"\"\"\n if cocoa_timestamp == 0:\n return \"Unknown\"\n\n try:\n # Cocoa timestamp is nanoseconds since 2001-01-01 00:00:00 UTC\n # Convert to seconds and add to Unix epoch\n cocoa_epoch = datetime(2001, 1, 1)\n unix_timestamp = cocoa_timestamp / 1_000_000_000 # Convert nanoseconds to seconds\n message_time = cocoa_epoch.timestamp() + unix_timestamp\n return datetime.fromtimestamp(message_time).strftime(\"%Y-%m-%d %H:%M:%S\")\n except (ValueError, OSError):\n return \"Unknown\"\n\n def _get_contact_name(self, handle_id: str) -> str:\n \"\"\"\n Get a readable contact name from handle ID.\n\n Args:\n handle_id: The handle ID (phone number or email)\n\n Returns:\n Formatted contact name\n \"\"\"\n if not handle_id:\n return \"Unknown\"\n\n # Clean up phone numbers and emails for display\n if \"@\" in handle_id:\n return handle_id # Email address\n elif handle_id.startswith(\"+\"):\n return handle_id # International phone number\n else:\n # Try to format as phone number\n digits = \"\".join(filter(str.isdigit, handle_id))\n if len(digits) == 10:\n return f\"({digits[:3]}) {digits[3:6]}-{digits[6:]}\"\n elif len(digits) == 11 and digits[0] == \"1\":\n return f\"+1 ({digits[1:4]}) {digits[4:7]}-{digits[7:]}\"\n else:\n return handle_id\n\n def _read_messages_from_db(self, db_path: Path) -> list[dict]:\n \"\"\"\n Read messages from the iMessage database.\n\n Args:\n db_path: Path to the chat.db file\n\n Returns:\n List of message dictionaries\n \"\"\"\n if not db_path.exists():\n print(f\"iMessage database not found at: {db_path}\")\n return []\n\n try:\n # Connect to the database\n conn = sqlite3.connect(str(db_path))\n cursor = conn.cursor()\n\n # Query to get messages with chat and handle information\n query = \"\"\"\n SELECT\n m.ROWID as message_id,\n m.text,\n m.date,\n m.is_from_me,\n m.service,\n c.chat_identifier,\n c.display_name as chat_display_name,\n h.id as handle_id,\n c.ROWID as chat_id\n FROM message m\n LEFT JOIN chat_message_join cmj ON m.ROWID = cmj.message_id\n LEFT JOIN chat c ON cmj.chat_id = c.ROWID\n LEFT JOIN handle h ON m.handle_id = h.ROWID\n WHERE m.text IS NOT NULL AND m.text != ''\n ORDER BY c.ROWID, m.date\n \"\"\"\n\n cursor.execute(query)\n rows = cursor.fetchall()\n\n messages = []\n for row in rows:\n (\n message_id,\n text,\n date,\n is_from_me,\n service,\n chat_identifier,\n chat_display_name,\n handle_id,\n chat_id,\n ) = row\n\n message = {\n \"message_id\": message_id,\n \"text\": text,\n \"timestamp\": self._convert_cocoa_timestamp(date),\n \"is_from_me\": bool(is_from_me),\n \"service\": service or \"iMessage\",\n \"chat_identifier\": chat_identifier or \"Unknown\",\n \"chat_display_name\": chat_display_name or \"Unknown Chat\",\n \"handle_id\": handle_id or \"Unknown\",\n \"contact_name\": self._get_contact_name(handle_id or \"\"),\n \"chat_id\": chat_id,\n }\n messages.append(message)\n\n conn.close()\n print(f\"Found {len(messages)} messages in database\")\n return messages\n\n except sqlite3.Error as e:\n print(f\"Error reading iMessage database: {e}\")\n return []\n except Exception as e:\n print(f\"Unexpected error reading iMessage database: {e}\")\n return []\n\n def _group_messages_by_chat(self, messages: list[dict]) -> dict[int, list[dict]]:\n \"\"\"\n Group messages by chat ID.\n\n Args:\n messages: List of message dictionaries\n\n Returns:\n Dictionary mapping chat_id to list of messages\n \"\"\"\n chats = {}\n for message in messages:\n chat_id = message[\"chat_id\"]\n if chat_id not in chats:\n chats[chat_id] = []\n chats[chat_id].append(message)\n\n return chats\n\n def _create_concatenated_content(self, chat_id: int, messages: list[dict]) -> str:\n \"\"\"\n Create concatenated content from chat messages.\n\n Args:\n chat_id: The chat ID\n messages: List of messages in the chat\n\n Returns:\n Concatenated text content\n \"\"\"\n if not messages:\n return \"\"\n\n # Get chat info from first message\n first_msg = messages[0]\n chat_name = first_msg[\"chat_display_name\"]\n chat_identifier = first_msg[\"chat_identifier\"]\n\n # Build message content\n message_parts = []\n for message in messages:\n timestamp = message[\"timestamp\"]\n is_from_me = message[\"is_from_me\"]\n text = message[\"text\"]\n contact_name = message[\"contact_name\"]\n\n if is_from_me:\n prefix = \"[You]\"\n else:\n prefix = f\"[{contact_name}]\"\n\n if timestamp != \"Unknown\":\n prefix += f\" ({timestamp})\"\n\n message_parts.append(f\"{prefix}: {text}\")\n\n concatenated_text = \"\\n\\n\".join(message_parts)\n\n doc_content = f\"\"\"Chat: {chat_name}\nIdentifier: {chat_identifier}\nMessages ({len(messages)} messages):\n\n{concatenated_text}\n\"\"\"\n return doc_content\n\n def _create_individual_content(self, message: dict) -> str:\n \"\"\"\n Create content for individual message.\n\n Args:\n message: Message dictionary\n\n Returns:\n Formatted message content\n \"\"\"\n timestamp = message[\"timestamp\"]\n is_from_me = message[\"is_from_me\"]\n text = message[\"text\"]\n contact_name = message[\"contact_name\"]\n chat_name = message[\"chat_display_name\"]\n\n sender = \"You\" if is_from_me else contact_name\n\n return f\"\"\"Message from {sender} in chat \"{chat_name}\"\nTime: {timestamp}\nContent: {text}\n\"\"\"\n\n def load_data(self, input_dir: str | None = None, **load_kwargs: Any) -> list[Document]:\n \"\"\"\n Load iMessage data and return as documents.\n\n Args:\n input_dir: Optional path to directory containing chat.db file.\n If not provided, uses default macOS location.\n **load_kwargs: Additional arguments (unused)\n\n Returns:\n List of Document objects containing iMessage data\n \"\"\"\n docs = []\n\n # Determine database path\n if input_dir:\n db_path = Path(input_dir) / \"chat.db\"\n else:\n db_path = self._get_default_chat_db_path()\n\n print(f\"Reading iMessage database from: {db_path}\")\n\n # Read messages from database\n messages = self._read_messages_from_db(db_path)\n if not messages:\n return docs\n\n if self.concatenate_conversations:\n # Group messages by chat and create concatenated documents\n chats = self._group_messages_by_chat(messages)\n\n for chat_id, chat_messages in chats.items():\n if not chat_messages:\n continue\n\n content = self._create_concatenated_content(chat_id, chat_messages)\n\n # Create metadata\n first_msg = chat_messages[0]\n last_msg = chat_messages[-1]\n\n metadata = {\n \"source\": \"iMessage\",\n \"chat_id\": chat_id,\n \"chat_name\": first_msg[\"chat_display_name\"],\n \"chat_identifier\": first_msg[\"chat_identifier\"],\n \"message_count\": len(chat_messages),\n \"first_message_date\": first_msg[\"timestamp\"],\n \"last_message_date\": last_msg[\"timestamp\"],\n \"participants\": list(\n {msg[\"contact_name\"] for msg in chat_messages if not msg[\"is_from_me\"]}\n ),\n }\n\n doc = Document(text=content, metadata=metadata)\n docs.append(doc)\n\n else:\n # Create individual documents for each message\n for message in messages:\n content = self._create_individual_content(message)\n\n metadata = {\n \"source\": \"iMessage\",\n \"message_id\": message[\"message_id\"],\n \"chat_id\": message[\"chat_id\"],\n \"chat_name\": message[\"chat_display_name\"],\n \"chat_identifier\": message[\"chat_identifier\"],\n \"timestamp\": message[\"timestamp\"],\n \"is_from_me\": message[\"is_from_me\"],\n \"contact_name\": message[\"contact_name\"],\n \"service\": message[\"service\"],\n }\n\n doc = Document(text=content, metadata=metadata)\n docs.append(doc)\n\n print(f\"Created {len(docs)} documents from iMessage data\")\n return docs\n" }, { "path": "apps/imessage_rag.py", "content": "\"\"\"\niMessage RAG Example.\n\nThis example demonstrates how to build a RAG system on your iMessage conversation history.\n\"\"\"\n\nimport asyncio\nfrom pathlib import Path\nfrom typing import Any\n\nfrom leann.chunking_utils import create_text_chunks\n\nfrom apps.base_rag_example import BaseRAGExample\nfrom apps.imessage_data.imessage_reader import IMessageReader\n\n\nclass IMessageRAG(BaseRAGExample):\n \"\"\"RAG example for iMessage conversation history.\"\"\"\n\n def __init__(self):\n super().__init__(\n name=\"iMessage\",\n description=\"RAG on your iMessage conversation history\",\n default_index_name=\"imessage_index\",\n )\n\n def _add_specific_arguments(self, parser):\n \"\"\"Add iMessage-specific arguments.\"\"\"\n imessage_group = parser.add_argument_group(\"iMessage Parameters\")\n imessage_group.add_argument(\n \"--db-path\",\n type=str,\n default=None,\n help=\"Path to iMessage chat.db file (default: ~/Library/Messages/chat.db)\",\n )\n imessage_group.add_argument(\n \"--concatenate-conversations\",\n action=\"store_true\",\n default=True,\n help=\"Concatenate messages within conversations for better context (default: True)\",\n )\n imessage_group.add_argument(\n \"--no-concatenate-conversations\",\n action=\"store_true\",\n help=\"Process each message individually instead of concatenating by conversation\",\n )\n imessage_group.add_argument(\n \"--chunk-size\",\n type=int,\n default=1000,\n help=\"Maximum characters per text chunk (default: 1000)\",\n )\n imessage_group.add_argument(\n \"--chunk-overlap\",\n type=int,\n default=200,\n help=\"Overlap between text chunks (default: 200)\",\n )\n\n async def load_data(self, args) -> list[dict[str, Any]]:\n \"\"\"Load iMessage history and convert to text chunks.\"\"\"\n print(\"Loading iMessage conversation history...\")\n\n # Determine concatenation setting\n concatenate = args.concatenate_conversations and not args.no_concatenate_conversations\n\n # Initialize iMessage reader\n reader = IMessageReader(concatenate_conversations=concatenate)\n\n # Load documents\n try:\n if args.db_path:\n # Use custom database path\n db_dir = str(Path(args.db_path).parent)\n documents = reader.load_data(input_dir=db_dir)\n else:\n # Use default macOS location\n documents = reader.load_data()\n\n except Exception as e:\n print(f\"Error loading iMessage data: {e}\")\n print(\"\\nTroubleshooting tips:\")\n print(\"1. Make sure you have granted Full Disk Access to your terminal/IDE\")\n print(\"2. Check that the iMessage database exists at ~/Library/Messages/chat.db\")\n print(\"3. Try specifying a custom path with --db-path if you have a backup\")\n return []\n\n if not documents:\n print(\"No iMessage conversations found!\")\n return []\n\n print(f\"Loaded {len(documents)} iMessage documents\")\n\n # Show some statistics\n total_messages = sum(doc.metadata.get(\"message_count\", 1) for doc in documents)\n print(f\"Total messages: {total_messages}\")\n\n if concatenate:\n # Show chat statistics\n chat_names = [doc.metadata.get(\"chat_name\", \"Unknown\") for doc in documents]\n unique_chats = len(set(chat_names))\n print(f\"Unique conversations: {unique_chats}\")\n\n # Convert to text chunks\n all_texts = create_text_chunks(\n documents,\n chunk_size=args.chunk_size,\n chunk_overlap=args.chunk_overlap,\n )\n\n # Apply max_items limit if specified\n if args.max_items > 0:\n all_texts = all_texts[: args.max_items]\n print(f\"Limited to {len(all_texts)} text chunks (max_items={args.max_items})\")\n\n return all_texts\n\n\nasync def main():\n \"\"\"Main entry point.\"\"\"\n app = IMessageRAG()\n await app.run()\n\n\nif __name__ == \"__main__\":\n asyncio.run(main())\n" }, { "path": "apps/multimodal/vision-based-pdf-multi-vector/README.md", "content": "## Vision-based PDF Multi-Vector Demos (macOS/MPS)\n\nThis folder contains two demos to index PDF pages as images and run multi-vector retrieval with ColPali/ColQwen2, plus optional similarity map visualization and answer generation.\n\n### What you’ll run\n- `multi-vector-leann-paper-example.py`: local PDF → pages → embed → build HNSW index → search.\n- `multi-vector-leann-similarity-map.py`: HF dataset (default) or local pages → embed → index → retrieve → similarity maps → optional Qwen-VL answer.\n\n## Prerequisites (macOS)\n\n### 1) Homebrew poppler (for pdf2image)\n```bash\nbrew install poppler\nwhich pdfinfo && pdfinfo -v\n```\n\n### 2) Python environment\nUse uv (recommended) or pip. Python 3.9+.\n\nUsing uv:\n```bash\nuv pip install \\\n colpali_engine \\\n pdf2image \\\n pillow \\\n matplotlib qwen_vl_utils \\\n einops \\\n seaborn\n```\n\nNotes:\n- On first run, models download from Hugging Face. Login/config if needed.\n- The scripts auto-select device: CUDA > MPS > CPU. Verify MPS:\n```bash\npython -c \"import torch; print('MPS available:', bool(getattr(torch.backends, 'mps', None) and torch.backends.mps.is_available()))\"\n```\n\n## Run the demos\n\n### A) Local PDF example\nConverts a local PDF into page images, embeds them, builds an index, and searches.\n\n```bash\ncd apps/multimodal/vision-based-pdf-multi-vector\n# If you don't have the sample PDF locally, download it (ignored by Git)\nmkdir -p pdfs\ncurl -L -o pdfs/2004.12832v2.pdf https://arxiv.org/pdf/2004.12832.pdf\nls pdfs/2004.12832v2.pdf\n# Ensure output dir exists\nmkdir -p pages\npython multi-vector-leann-paper-example.py\n```\nExpected:\n- Page images in `pages/`.\n- Console prints like `Using device=mps, dtype=...` and retrieved file paths for queries.\n\nTo use your own PDF: edit `pdf_path` near the top of the script.\n\n### B) Similarity map + answer demo\nUses HF dataset `weaviate/arXiv-AI-papers-multi-vector` by default; can switch to local pages.\n\n```bash\ncd apps/multimodal/vision-based-pdf-multi-vector\npython multi-vector-leann-similarity-map.py\n```\nArtifacts (when enabled):\n- Retrieved pages: `./figures/retrieved_page_rank{K}.png`\n- Similarity maps: `./figures/similarity_map_rank{K}.png`\n\nKey knobs in the script (top of file):\n- `QUERY`: your question\n- `MODEL`: `\"colqwen2\"` or `\"colpali\"`\n- `USE_HF_DATASET`: set `False` to use local pages\n- `PDF`, `PAGES_DIR`: for local mode\n- `INDEX_PATH`, `TOPK`, `FIRST_STAGE_K`, `REBUILD_INDEX`\n- `SIMILARITY_MAP`, `SIM_TOKEN_IDX`, `SIM_OUTPUT`\n- `ANSWER`, `MAX_NEW_TOKENS` (Qwen-VL)\n\n## Troubleshooting\n- pdf2image errors on macOS: ensure `brew install poppler` and `pdfinfo` works in terminal.\n- Slow or OOM on MPS: reduce dataset size (e.g., set `MAX_DOCS`) or switch to CPU.\n- NaNs on MPS: keep fp32 on MPS (default in similarity-map script); avoid fp16 there.\n- First-run model downloads can be large; ensure network access (HF mirrors if needed).\n\n## Notes\n- Index files are under `./indexes/`. Delete or set `REBUILD_INDEX=True` to rebuild.\n- For local PDFs, page images go to `./pages/`.\n\n\n### Retrieval and Visualization Example\n\nExample settings in `multi-vector-leann-similarity-map.py`:\n- `QUERY = \"How does DeepSeek-V2 compare against the LLaMA family of LLMs?\"`\n- `SIMILARITY_MAP = True` (to generate heatmaps)\n- `TOPK = 1` (save the top retrieved page and its similarity map)\n\nRun:\n```bash\ncd apps/multimodal/vision-based-pdf-multi-vector\npython multi-vector-leann-similarity-map.py\n```\n\nOutputs (by default):\n- Retrieved page: `./figures/retrieved_page_rank1.png`\n- Similarity map: `./figures/similarity_map_rank1.png`\n\nSample visualization (example result, and the query is \"QUERY = \"How does Vim model performance and efficiency compared to other models?\"\n\"):\n![Similarity map example](fig/image.png)\n\nNotes:\n- Set `SIM_TOKEN_IDX` to visualize a specific token index; set `-1` to auto-select the most salient token.\n- If you change `SIM_OUTPUT` to a file path (e.g., `./figures/my_map.png`), multiple ranks are saved as `my_map_rank{K}.png`.\n" }, { "path": "apps/multimodal/vision-based-pdf-multi-vector/colqwen_forward.py", "content": "#!/usr/bin/env python3\n\"\"\"Simple test script to test colqwen2 forward pass with a single image.\"\"\"\n\nimport os\nimport sys\nfrom pathlib import Path\n\n# Add the current directory to path to import leann_multi_vector\nsys.path.insert(0, str(Path(__file__).parent))\n\nimport torch\nfrom leann_multi_vector import _embed_images, _ensure_repo_paths_importable, _load_colvision\nfrom PIL import Image\n\n# Ensure repo paths are importable\n_ensure_repo_paths_importable(__file__)\n\n# Set environment variable\nos.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n\n\ndef create_test_image():\n \"\"\"Create a simple test image.\"\"\"\n # Create a simple RGB image (800x600)\n img = Image.new(\"RGB\", (800, 600), color=\"white\")\n return img\n\n\ndef load_test_image_from_file():\n \"\"\"Try to load an image from the indexes directory if available.\"\"\"\n # Try to find an existing image in the indexes directory\n indexes_dir = Path(__file__).parent / \"indexes\"\n\n # Look for images in common locations\n possible_paths = [\n indexes_dir / \"vidore_fastplaid\" / \"images\",\n indexes_dir / \"colvision_large.leann.images\",\n indexes_dir / \"colvision.leann.images\",\n ]\n\n for img_dir in possible_paths:\n if img_dir.exists():\n # Find first image file\n for ext in [\".png\", \".jpg\", \".jpeg\"]:\n for img_file in img_dir.glob(f\"*{ext}\"):\n print(f\"Loading test image from: {img_file}\")\n return Image.open(img_file)\n\n return None\n\n\ndef main():\n print(\"=\" * 60)\n print(\"Testing ColQwen2 Forward Pass\")\n print(\"=\" * 60)\n\n # Step 1: Load or create test image\n print(\"\\n[Step 1] Loading test image...\")\n test_image = load_test_image_from_file()\n if test_image is None:\n print(\"No existing image found, creating a simple test image...\")\n test_image = create_test_image()\n else:\n print(f\"✓ Loaded image: {test_image.size} ({test_image.mode})\")\n\n # Convert to RGB if needed\n if test_image.mode != \"RGB\":\n test_image = test_image.convert(\"RGB\")\n print(f\"✓ Converted to RGB: {test_image.size}\")\n\n # Step 2: Load model\n print(\"\\n[Step 2] Loading ColQwen2 model...\")\n try:\n model_name, model, processor, device_str, device, dtype = _load_colvision(\"colqwen2\")\n print(f\"✓ Model loaded: {model_name}\")\n print(f\"✓ Device: {device_str}, dtype: {dtype}\")\n\n # Print model info\n if hasattr(model, \"device\"):\n print(f\"✓ Model device: {model.device}\")\n if hasattr(model, \"dtype\"):\n print(f\"✓ Model dtype: {model.dtype}\")\n\n except Exception as e:\n print(f\"✗ Error loading model: {e}\")\n import traceback\n\n traceback.print_exc()\n return\n\n # Step 3: Test forward pass\n print(\"\\n[Step 3] Running forward pass...\")\n try:\n # Use the _embed_images function which handles batching and forward pass\n images = [test_image]\n print(f\"Processing {len(images)} image(s)...\")\n\n doc_vecs = _embed_images(model, processor, images)\n\n print(\"✓ Forward pass completed!\")\n print(f\"✓ Number of embeddings: {len(doc_vecs)}\")\n\n if len(doc_vecs) > 0:\n emb = doc_vecs[0]\n print(f\"✓ Embedding shape: {emb.shape}\")\n print(f\"✓ Embedding dtype: {emb.dtype}\")\n print(\"✓ Embedding stats:\")\n print(f\" - Min: {emb.min().item():.4f}\")\n print(f\" - Max: {emb.max().item():.4f}\")\n print(f\" - Mean: {emb.mean().item():.4f}\")\n print(f\" - Std: {emb.std().item():.4f}\")\n\n # Check for NaN or Inf\n if torch.isnan(emb).any():\n print(\"⚠ Warning: Embedding contains NaN values!\")\n if torch.isinf(emb).any():\n print(\"⚠ Warning: Embedding contains Inf values!\")\n\n except Exception as e:\n print(f\"✗ Error during forward pass: {e}\")\n import traceback\n\n traceback.print_exc()\n return\n\n print(\"\\n\" + \"=\" * 60)\n print(\"Test completed successfully!\")\n print(\"=\" * 60)\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "apps/multimodal/vision-based-pdf-multi-vector/leann_multi_vector.py", "content": "import concurrent.futures\nimport glob\nimport json\nimport logging\nimport os\nimport re\nimport sys\nimport time\nfrom pathlib import Path\nfrom typing import Any, Optional, cast\n\nimport numpy as np\nfrom PIL import Image\nfrom tqdm import tqdm\n\nlogger = logging.getLogger(__name__)\n\n\ndef _ensure_repo_paths_importable(current_file: str) -> None:\n \"\"\"Make local leann packages importable without installing (mirrors multi-vector-leann.py).\"\"\"\n _repo_root = Path(current_file).resolve().parents[3]\n _leann_core_src = _repo_root / \"packages\" / \"leann-core\" / \"src\"\n _leann_hnsw_pkg = _repo_root / \"packages\" / \"leann-backend-hnsw\"\n if str(_leann_core_src) not in sys.path:\n sys.path.append(str(_leann_core_src))\n if str(_leann_hnsw_pkg) not in sys.path:\n sys.path.append(str(_leann_hnsw_pkg))\n\n\ndef _find_backend_module_file() -> Optional[Path]:\n \"\"\"Best-effort locate the backend leann_multi_vector.py file, avoiding this file.\"\"\"\n this_file = Path(__file__).resolve()\n candidates: list[Path] = []\n\n # Common in-repo location\n repo_root = this_file.parents[3]\n candidates.append(repo_root / \"packages\" / \"leann-backend-hnsw\" / \"leann_multi_vector.py\")\n candidates.append(\n repo_root / \"packages\" / \"leann-backend-hnsw\" / \"src\" / \"leann_multi_vector.py\"\n )\n\n for cand in candidates:\n try:\n if cand.exists() and cand.resolve() != this_file:\n return cand.resolve()\n except Exception:\n pass\n\n # Fallback: scan sys.path for another leann_multi_vector.py different from this file\n for p in list(sys.path):\n try:\n cand = Path(p) / \"leann_multi_vector.py\"\n if cand.exists() and cand.resolve() != this_file:\n return cand.resolve()\n except Exception:\n continue\n return None\n\n\n_BACKEND_LEANN_CLASS: Optional[type] = None\n\n\ndef _get_backend_leann_multi_vector() -> type:\n \"\"\"Load backend LeannMultiVector class even if this file shadows its module name.\"\"\"\n global _BACKEND_LEANN_CLASS\n if _BACKEND_LEANN_CLASS is not None:\n return _BACKEND_LEANN_CLASS\n\n backend_path = _find_backend_module_file()\n if backend_path is None:\n # Fallback to local implementation in this module\n try:\n cls = LeannMultiVector\n _BACKEND_LEANN_CLASS = cls\n return cls\n except Exception as e:\n raise ImportError(\n \"Could not locate backend 'leann_multi_vector.py' and no local implementation found. \"\n \"Ensure the leann backend is available under packages/leann-backend-hnsw or installed.\"\n ) from e\n\n import importlib.util\n\n module_name = \"leann_hnsw_backend_module\"\n spec = importlib.util.spec_from_file_location(module_name, str(backend_path))\n if spec is None or spec.loader is None:\n raise ImportError(f\"Failed to create spec for backend module at {backend_path}\")\n backend_module = importlib.util.module_from_spec(spec)\n sys.modules[module_name] = backend_module\n spec.loader.exec_module(backend_module)\n\n if not hasattr(backend_module, \"LeannMultiVector\"):\n raise ImportError(f\"'LeannMultiVector' not found in backend module at {backend_path}\")\n _BACKEND_LEANN_CLASS = backend_module.LeannMultiVector\n return _BACKEND_LEANN_CLASS\n\n\ndef _natural_sort_key(name: str) -> int:\n m = re.search(r\"\\d+\", name)\n return int(m.group()) if m else 0\n\n\ndef _load_images_from_dir(\n pages_dir: str, recursive: bool = False\n) -> tuple[list[str], list[Image.Image]]:\n \"\"\"\n Load images from a directory.\n\n Args:\n pages_dir: Directory path containing images\n recursive: If True, recursively search subdirectories (default: False)\n\n Returns:\n Tuple of (filepaths, images)\n \"\"\"\n\n # Supported image extensions\n extensions = (\"*.png\", \"*.jpg\", \"*.jpeg\", \"*.PNG\", \"*.JPG\", \"*.JPEG\", \"*.webp\", \"*.WEBP\")\n\n if recursive:\n # Recursive search\n filepaths = []\n for ext in extensions:\n pattern = os.path.join(pages_dir, \"**\", ext)\n filepaths.extend(glob.glob(pattern, recursive=True))\n else:\n # Non-recursive search (only top-level directory)\n filepaths = []\n for ext in extensions:\n pattern = os.path.join(pages_dir, ext)\n filepaths.extend(glob.glob(pattern))\n\n # Sort files naturally\n filepaths = sorted(filepaths, key=lambda x: _natural_sort_key(os.path.basename(x)))\n\n # Load images with error handling\n images = []\n valid_filepaths = []\n failed_count = 0\n\n for filepath in filepaths:\n try:\n img = Image.open(filepath)\n # Convert to RGB if necessary (handles RGBA, P, etc.)\n if img.mode != \"RGB\":\n img = img.convert(\"RGB\")\n images.append(img)\n valid_filepaths.append(filepath)\n except Exception as e:\n failed_count += 1\n print(f\"Warning: Failed to load image {filepath}: {e}\")\n continue\n\n if failed_count > 0:\n print(\n f\"Warning: Failed to load {failed_count} image(s) out of {len(filepaths)} total files\"\n )\n\n return valid_filepaths, images\n\n\ndef _maybe_convert_pdf_to_images(pdf_path: Optional[str], pages_dir: str, dpi: int = 200) -> None:\n if not pdf_path:\n return\n os.makedirs(pages_dir, exist_ok=True)\n try:\n from pdf2image import convert_from_path\n except Exception as e:\n raise RuntimeError(\n \"pdf2image is required to convert PDF to images. Install via pip install pdf2image\"\n ) from e\n images = convert_from_path(pdf_path, dpi=dpi)\n for i, image in enumerate(images):\n image.save(os.path.join(pages_dir, f\"page_{i + 1}.png\"), \"PNG\")\n\n\ndef _select_device_and_dtype():\n import torch\n from colpali_engine.utils.torch_utils import get_torch_device\n\n device_str = (\n \"cuda\"\n if torch.cuda.is_available()\n else (\n \"mps\"\n if getattr(torch.backends, \"mps\", None) and torch.backends.mps.is_available()\n else \"cpu\"\n )\n )\n device = get_torch_device(device_str)\n # Stable dtype selection to avoid NaNs:\n # - CUDA: prefer bfloat16 if supported, else float16\n # - MPS: use float32 (fp16 on MPS can produce NaNs in some ops)\n # - CPU: float32\n if device_str == \"cuda\":\n dtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16\n try:\n torch.backends.cuda.matmul.allow_tf32 = True # Better stability/perf on Ampere+\n except Exception:\n pass\n elif device_str == \"mps\":\n dtype = torch.float32\n else:\n dtype = torch.float32\n return device_str, device, dtype\n\n\ndef _load_colvision(model_choice: str):\n import os\n\n import torch\n from colpali_engine.models import (\n ColPali,\n ColQwen2,\n ColQwen2_5,\n ColQwen2_5_Processor,\n ColQwen2Processor,\n )\n from colpali_engine.models.paligemma.colpali.processing_colpali import ColPaliProcessor\n from transformers.utils.import_utils import is_flash_attn_2_available\n\n # Force HuggingFace Hub to use HF endpoint, avoid Google Drive\n # Set environment variables to ensure models are downloaded from HuggingFace\n os.environ.setdefault(\"HF_ENDPOINT\", \"https://huggingface.co\")\n os.environ.setdefault(\"HF_HUB_ENABLE_HF_TRANSFER\", \"1\")\n\n # Log model loading info\n logger.info(f\"Loading ColVision model: {model_choice}\")\n logger.info(f\"HF_ENDPOINT: {os.environ.get('HF_ENDPOINT', 'not set')}\")\n logger.info(\"Models will be downloaded from HuggingFace Hub, not Google Drive\")\n\n device_str, device, dtype = _select_device_and_dtype()\n\n # Determine model name and type\n # IMPORTANT: Check colqwen2.5 BEFORE colqwen2 to avoid false matches\n model_choice_lower = model_choice.lower()\n if model_choice == \"colqwen2\":\n model_name = \"vidore/colqwen2-v1.0\"\n model_type = \"colqwen2\"\n elif model_choice == \"colqwen2.5\" or model_choice == \"colqwen25\":\n model_name = \"vidore/colqwen2.5-v0.2\"\n model_type = \"colqwen2.5\"\n elif model_choice == \"colpali\":\n model_name = \"vidore/colpali-v1.2\"\n model_type = \"colpali\"\n elif (\n \"colqwen2.5\" in model_choice_lower\n or \"colqwen25\" in model_choice_lower\n or \"colqwen2_5\" in model_choice_lower\n ):\n # Handle HuggingFace model names like \"vidore/colqwen2.5-v0.2\"\n model_name = model_choice\n model_type = \"colqwen2.5\"\n elif \"colqwen2\" in model_choice_lower and \"colqwen2-v1.0\" in model_choice_lower:\n # Handle HuggingFace model names like \"vidore/colqwen2-v1.0\" (but not colqwen2.5)\n model_name = model_choice\n model_type = \"colqwen2\"\n elif \"colpali\" in model_choice_lower:\n # Handle HuggingFace model names like \"vidore/colpali-v1.2\"\n model_name = model_choice\n model_type = \"colpali\"\n else:\n # Default to colpali for backward compatibility\n model_name = \"vidore/colpali-v1.2\"\n model_type = \"colpali\"\n\n # Load model based on type\n attn_implementation = (\n \"flash_attention_2\" if (device_str == \"cuda\" and is_flash_attn_2_available()) else \"eager\"\n )\n\n # Load model from HuggingFace Hub (not Google Drive)\n # Use local_files_only=False to ensure download from HF if not cached\n if model_type == \"colqwen2.5\":\n model = ColQwen2_5.from_pretrained(\n model_name,\n torch_dtype=torch.bfloat16,\n device_map=device,\n attn_implementation=attn_implementation,\n local_files_only=False, # Ensure download from HuggingFace Hub\n ).eval()\n processor = ColQwen2_5_Processor.from_pretrained(model_name, local_files_only=False)\n elif model_type == \"colqwen2\":\n model = ColQwen2.from_pretrained(\n model_name,\n torch_dtype=torch.bfloat16,\n device_map=device,\n attn_implementation=attn_implementation,\n local_files_only=False, # Ensure download from HuggingFace Hub\n ).eval()\n processor = ColQwen2Processor.from_pretrained(model_name, local_files_only=False)\n else: # colpali\n model = ColPali.from_pretrained(\n model_name,\n torch_dtype=torch.bfloat16,\n device_map=device,\n local_files_only=False, # Ensure download from HuggingFace Hub\n ).eval()\n processor = cast(\n ColPaliProcessor, ColPaliProcessor.from_pretrained(model_name, local_files_only=False)\n )\n\n return model_name, model, processor, device_str, device, dtype\n\n\ndef _embed_images(model, processor, images: list[Image.Image]) -> list[Any]:\n import torch\n from colpali_engine.utils.torch_utils import ListDataset\n from torch.utils.data import DataLoader\n\n # Ensure deterministic eval and autocast for stability\n model.eval()\n\n dataloader = DataLoader(\n dataset=ListDataset[Image.Image](images),\n batch_size=32,\n shuffle=False,\n collate_fn=lambda x: processor.process_images(x),\n )\n\n doc_vecs: list[Any] = []\n for batch_doc in tqdm(dataloader, desc=\"Embedding images\"):\n with torch.no_grad():\n batch_doc = {k: v.to(model.device) for k, v in batch_doc.items()}\n # autocast on CUDA for bf16/fp16; on CPU/MPS stay in fp32\n if model.device.type == \"cuda\":\n with torch.autocast(\n device_type=\"cuda\",\n dtype=model.dtype if model.dtype.is_floating_point else torch.bfloat16,\n ):\n embeddings_doc = model(**batch_doc)\n else:\n embeddings_doc = model(**batch_doc)\n doc_vecs.extend(list(torch.unbind(embeddings_doc.to(\"cpu\"))))\n return doc_vecs\n\n\ndef _embed_queries(model, processor, queries: list[str]) -> list[Any]:\n import torch\n\n model.eval()\n\n # Match MTEB's exact query processing from ColPaliEngineWrapper.get_text_embeddings:\n # 1. MTEB receives batch[\"text\"] which already includes instruction/prompt (from _combine_queries_with_instruction_text)\n # 2. Manually adds: query_prefix + text + query_augmentation_token * 10\n # 3. Calls processor.process_queries(batch) where batch is now a list of strings\n # 4. process_queries adds: query_prefix + text + suffix (suffix = query_augmentation_token * 10)\n #\n # This results in duplicate addition: query_prefix is added twice, query_augmentation_token * 20 total\n # We need to match this exactly to reproduce MTEB results\n\n all_embeds = []\n batch_size = 32 # Match MTEB's default batch_size\n\n with torch.no_grad():\n for i in tqdm(range(0, len(queries), batch_size), desc=\"Embedding queries\"):\n batch_queries = queries[i : i + batch_size]\n\n # Match MTEB: manually add query_prefix + text + query_augmentation_token * 10\n # Then process_queries will add them again (resulting in 20 augmentation tokens total)\n batch = [\n processor.query_prefix + t + processor.query_augmentation_token * 10\n for t in batch_queries\n ]\n inputs = processor.process_queries(batch)\n inputs = {k: v.to(model.device) for k, v in inputs.items()}\n\n if model.device.type == \"cuda\":\n with torch.autocast(\n device_type=\"cuda\",\n dtype=model.dtype if model.dtype.is_floating_point else torch.bfloat16,\n ):\n outs = model(**inputs)\n else:\n outs = model(**inputs)\n\n # Match MTEB: convert to float32 on CPU\n all_embeds.extend(list(torch.unbind(outs.cpu().to(torch.float32))))\n\n return all_embeds\n\n\ndef _build_index(\n index_path: str, doc_vecs: list[Any], filepaths: list[str], images: list[Image.Image]\n) -> Any:\n LeannMultiVector = _get_backend_leann_multi_vector()\n dim = int(doc_vecs[0].shape[-1])\n retriever = LeannMultiVector(index_path=index_path, dim=dim)\n retriever.create_collection()\n for i, vec in enumerate(doc_vecs):\n data = {\n \"colbert_vecs\": vec.float().numpy(),\n \"doc_id\": i,\n \"filepath\": filepaths[i],\n \"image\": images[i], # Include the original image\n }\n retriever.insert(data)\n retriever.create_index()\n return retriever\n\n\ndef _load_retriever_if_index_exists(index_path: str) -> Optional[Any]:\n LeannMultiVector = _get_backend_leann_multi_vector()\n index_base = Path(index_path)\n # Check for the actual HNSW index file written by the backend + our sidecar files\n index_file = index_base.parent / f\"{index_base.stem}.index\"\n meta = index_base.parent / f\"{index_base.name}.meta.json\"\n labels = index_base.parent / f\"{index_base.name}.labels.json\"\n if index_file.exists() and meta.exists() and labels.exists():\n try:\n with open(meta, encoding=\"utf-8\") as f:\n meta_json = json.load(f)\n dim = int(meta_json.get(\"dimensions\", 128))\n except Exception:\n dim = 128\n return LeannMultiVector(index_path=index_path, dim=dim)\n return None\n\n\ndef _build_fast_plaid_index(\n index_path: str,\n doc_vecs: list[Any],\n filepaths: list[str],\n images: list[Image.Image],\n) -> tuple[Any, float]:\n \"\"\"\n Build a Fast-Plaid index from document embeddings.\n\n Args:\n index_path: Path to save the Fast-Plaid index\n doc_vecs: List of document embeddings (each is a tensor with shape [num_tokens, embedding_dim])\n filepaths: List of filepath identifiers for each document\n images: List of PIL Images corresponding to each document\n\n Returns:\n Tuple of (FastPlaid index object, build_time_in_seconds)\n \"\"\"\n import torch\n from fast_plaid import search as fast_plaid_search\n\n print(f\" Preparing {len(doc_vecs)} document embeddings for Fast-Plaid...\")\n _t0 = time.perf_counter()\n\n # Convert doc_vecs to list of tensors\n documents_embeddings = []\n for i, vec in enumerate(doc_vecs):\n if i % 1000 == 0:\n print(f\" Converting embedding {i}/{len(doc_vecs)}...\")\n if not isinstance(vec, torch.Tensor):\n vec = (\n torch.tensor(vec)\n if isinstance(vec, np.ndarray)\n else torch.from_numpy(np.array(vec))\n )\n # Ensure float32 for Fast-Plaid\n if vec.dtype != torch.float32:\n vec = vec.float()\n documents_embeddings.append(vec)\n\n print(f\" Converted {len(documents_embeddings)} embeddings\")\n if len(documents_embeddings) > 0:\n print(f\" First embedding shape: {documents_embeddings[0].shape}\")\n print(f\" First embedding dtype: {documents_embeddings[0].dtype}\")\n\n # Prepare metadata for Fast-Plaid\n print(f\" Preparing metadata for {len(filepaths)} documents...\")\n metadata_list = []\n for i, filepath in enumerate(filepaths):\n metadata_list.append(\n {\n \"filepath\": filepath,\n \"index\": i,\n }\n )\n\n # Create Fast-Plaid index\n print(f\" Creating FastPlaid object with index path: {index_path}\")\n try:\n fast_plaid_index = fast_plaid_search.FastPlaid(index=index_path)\n print(\" FastPlaid object created successfully\")\n except Exception as e:\n print(f\" Error creating FastPlaid object: {type(e).__name__}: {e}\")\n import traceback\n\n traceback.print_exc()\n raise\n\n print(f\" Calling fast_plaid_index.create() with {len(documents_embeddings)} documents...\")\n try:\n fast_plaid_index.create(\n documents_embeddings=documents_embeddings,\n metadata=metadata_list,\n )\n print(\" Fast-Plaid index created successfully\")\n except Exception as e:\n print(f\" Error creating Fast-Plaid index: {type(e).__name__}: {e}\")\n import traceback\n\n traceback.print_exc()\n raise\n\n build_secs = time.perf_counter() - _t0\n\n # Save images separately (Fast-Plaid doesn't store images)\n print(f\" Saving {len(images)} images...\")\n images_dir = Path(index_path) / \"images\"\n images_dir.mkdir(parents=True, exist_ok=True)\n for i, img in enumerate(tqdm(images, desc=\"Saving images\")):\n img_path = images_dir / f\"doc_{i}.png\"\n img.save(str(img_path))\n\n return fast_plaid_index, build_secs\n\n\ndef _fast_plaid_index_exists(index_path: str) -> bool:\n \"\"\"\n Check if Fast-Plaid index exists by checking for key files.\n This avoids creating the FastPlaid object which may trigger memory allocation.\n\n Args:\n index_path: Path to the Fast-Plaid index\n\n Returns:\n True if index appears to exist, False otherwise\n \"\"\"\n index_path_obj = Path(index_path)\n if not index_path_obj.exists() or not index_path_obj.is_dir():\n return False\n\n # Fast-Plaid creates a SQLite database file for metadata\n # Check for metadata.db as the most reliable indicator\n metadata_db = index_path_obj / \"metadata.db\"\n if metadata_db.exists() and metadata_db.stat().st_size > 0:\n return True\n\n # Also check if directory has any files (might be incomplete index)\n try:\n if any(index_path_obj.iterdir()):\n return True\n except Exception:\n pass\n\n return False\n\n\ndef _load_fast_plaid_index_if_exists(index_path: str) -> Optional[Any]:\n \"\"\"\n Load Fast-Plaid index if it exists.\n First checks if index files exist, then creates the FastPlaid object.\n The actual index data loading happens lazily when search is called.\n\n Args:\n index_path: Path to the Fast-Plaid index\n\n Returns:\n FastPlaid index object if exists, None otherwise\n \"\"\"\n try:\n from fast_plaid import search as fast_plaid_search\n\n # First check if index files exist without creating the object\n if not _fast_plaid_index_exists(index_path):\n return None\n\n # Now try to create FastPlaid object\n # This may trigger some memory allocation, but the full index loading is deferred\n fast_plaid_index = fast_plaid_search.FastPlaid(index=index_path)\n return fast_plaid_index\n except ImportError:\n # fast-plaid not installed\n return None\n except Exception as e:\n # Any error (including memory errors from Rust backend) - return None\n # The error will be caught and index will be rebuilt\n print(f\"Warning: Could not load Fast-Plaid index: {type(e).__name__}: {e}\")\n return None\n\n\ndef _search_fast_plaid(\n fast_plaid_index: Any,\n query_vec: Any,\n top_k: int,\n) -> tuple[list[tuple[float, int]], float]:\n \"\"\"\n Search Fast-Plaid index with a query embedding.\n\n Args:\n fast_plaid_index: FastPlaid index object\n query_vec: Query embedding tensor with shape [num_tokens, embedding_dim]\n top_k: Number of top results to return\n\n Returns:\n Tuple of (results_list, search_time_in_seconds)\n results_list: List of (score, doc_id) tuples\n \"\"\"\n import torch\n\n _t0 = time.perf_counter()\n\n # Ensure query is a torch tensor\n if not isinstance(query_vec, torch.Tensor):\n q_vec_tensor = (\n torch.tensor(query_vec)\n if isinstance(query_vec, np.ndarray)\n else torch.from_numpy(np.array(query_vec))\n )\n else:\n q_vec_tensor = query_vec\n\n # Fast-Plaid expects shape [num_queries, num_tokens, embedding_dim]\n if q_vec_tensor.dim() == 2:\n q_vec_tensor = q_vec_tensor.unsqueeze(0) # [1, num_tokens, embedding_dim]\n\n # Perform search\n scores = fast_plaid_index.search(\n queries_embeddings=q_vec_tensor,\n top_k=top_k,\n show_progress=True,\n )\n\n search_secs = time.perf_counter() - _t0\n\n # Convert Fast-Plaid results to same format as LEANN: list of (score, doc_id) tuples\n results = []\n if scores and len(scores) > 0:\n query_results = scores[0]\n # Fast-Plaid returns (doc_id, score), convert to (score, doc_id) to match LEANN format\n results = [(float(score), int(doc_id)) for doc_id, score in query_results]\n\n return results, search_secs\n\n\ndef _get_fast_plaid_image(index_path: str, doc_id: int) -> Optional[Image.Image]:\n \"\"\"\n Retrieve image for a document from Fast-Plaid index.\n\n Args:\n index_path: Path to the Fast-Plaid index\n doc_id: Document ID returned by Fast-Plaid search\n\n Returns:\n PIL Image if found, None otherwise\n\n Note: Uses metadata['index'] to get the actual file index, as Fast-Plaid\n doc_id may differ from the file naming index.\n \"\"\"\n # First get metadata to find the actual index used for file naming\n metadata = _get_fast_plaid_metadata(index_path, doc_id)\n if metadata is None:\n # Fallback: try using doc_id directly\n file_index = doc_id\n else:\n # Use the 'index' field from metadata, which matches the file naming\n file_index = metadata.get(\"index\", doc_id)\n\n images_dir = Path(index_path) / \"images\"\n image_path = images_dir / f\"doc_{file_index}.png\"\n\n if image_path.exists():\n return Image.open(image_path)\n\n # If not found with index, try doc_id as fallback\n if file_index != doc_id:\n fallback_path = images_dir / f\"doc_{doc_id}.png\"\n if fallback_path.exists():\n return Image.open(fallback_path)\n\n return None\n\n\ndef _get_fast_plaid_metadata(index_path: str, doc_id: int) -> Optional[dict]:\n \"\"\"\n Retrieve metadata for a document from Fast-Plaid index.\n\n Args:\n index_path: Path to the Fast-Plaid index\n doc_id: Document ID\n\n Returns:\n Dictionary with metadata if found, None otherwise\n \"\"\"\n try:\n from fast_plaid import filtering\n\n metadata_list = filtering.get(index=index_path, subset=[doc_id])\n if metadata_list and len(metadata_list) > 0:\n return metadata_list[0]\n except Exception:\n pass\n return None\n\n\ndef _generate_similarity_map(\n model,\n processor,\n image: Image.Image,\n query: str,\n token_idx: Optional[int] = None,\n output_path: Optional[str] = None,\n) -> tuple[int, float]:\n import torch\n from colpali_engine.interpretability import (\n get_similarity_maps_from_embeddings,\n plot_similarity_map,\n )\n\n batch_images = processor.process_images([image]).to(model.device)\n batch_queries = processor.process_queries([query]).to(model.device)\n\n with torch.no_grad():\n image_embeddings = model.forward(**batch_images)\n query_embeddings = model.forward(**batch_queries)\n\n n_patches = processor.get_n_patches(\n image_size=image.size,\n spatial_merge_size=getattr(model, \"spatial_merge_size\", None),\n )\n image_mask = processor.get_image_mask(batch_images)\n\n batched_similarity_maps = get_similarity_maps_from_embeddings(\n image_embeddings=image_embeddings,\n query_embeddings=query_embeddings,\n n_patches=n_patches,\n image_mask=image_mask,\n )\n\n similarity_maps = batched_similarity_maps[0]\n\n # Determine token index if not provided: choose the token with highest max score\n if token_idx is None:\n per_token_max = similarity_maps.view(similarity_maps.shape[0], -1).max(dim=1).values\n token_idx = int(per_token_max.argmax().item())\n\n max_sim_score = similarity_maps[token_idx, :, :].max().item()\n\n if output_path:\n import matplotlib.pyplot as plt\n\n fig, ax = plot_similarity_map(\n image=image,\n similarity_map=similarity_maps[token_idx],\n figsize=(14, 14),\n show_colorbar=False,\n )\n ax.set_title(f\"Token #{token_idx}. MaxSim score: {max_sim_score:.2f}\", fontsize=12)\n os.makedirs(os.path.dirname(output_path), exist_ok=True)\n plt.savefig(output_path, bbox_inches=\"tight\")\n plt.close(fig)\n\n return token_idx, float(max_sim_score)\n\n\nclass QwenVL:\n def __init__(self, device: str):\n from transformers import AutoProcessor, Qwen2_5_VLForConditionalGeneration\n from transformers.utils.import_utils import is_flash_attn_2_available\n\n attn_implementation = \"flash_attention_2\" if is_flash_attn_2_available() else \"eager\"\n self.model = Qwen2_5_VLForConditionalGeneration.from_pretrained(\n \"Qwen/Qwen2.5-VL-3B-Instruct\",\n torch_dtype=\"auto\",\n device_map=device,\n attn_implementation=attn_implementation,\n )\n\n min_pixels = 256 * 28 * 28\n max_pixels = 1280 * 28 * 28\n self.processor = AutoProcessor.from_pretrained(\n \"Qwen/Qwen2.5-VL-3B-Instruct\", min_pixels=min_pixels, max_pixels=max_pixels\n )\n\n def answer(self, query: str, images: list[Image.Image], max_new_tokens: int = 128) -> str:\n import base64\n from io import BytesIO\n\n from qwen_vl_utils import process_vision_info\n\n content = []\n for img in images:\n buffer = BytesIO()\n img.save(buffer, format=\"jpeg\")\n img_base64 = base64.b64encode(buffer.getvalue()).decode(\"utf-8\")\n content.append({\"type\": \"image\", \"image\": f\"data:image;base64,{img_base64}\"})\n content.append({\"type\": \"text\", \"text\": query})\n messages = [{\"role\": \"user\", \"content\": content}]\n\n text = self.processor.apply_chat_template(\n messages, tokenize=False, add_generation_prompt=True\n )\n image_inputs, video_inputs = process_vision_info(messages)\n inputs = self.processor(\n text=[text], images=image_inputs, videos=video_inputs, padding=True, return_tensors=\"pt\"\n )\n inputs = inputs.to(self.model.device)\n\n generated_ids = self.model.generate(**inputs, max_new_tokens=max_new_tokens)\n generated_ids_trimmed = [\n out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)\n ]\n return self.processor.batch_decode(\n generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False\n )[0]\n\n\n# Ensure repo paths are importable for dynamic backend loading\n_ensure_repo_paths_importable(__file__)\n\nfrom leann_backend_hnsw.hnsw_backend import HNSWBuilder, HNSWSearcher # noqa: E402\n\n\nclass LeannMultiVector:\n def __init__(\n self,\n index_path: str,\n dim: int = 128,\n distance_metric: str = \"mips\",\n m: int = 16,\n ef_construction: int = 500,\n is_compact: bool = False,\n is_recompute: bool = False,\n embedding_model_name: str = \"colvision\",\n ) -> None:\n self.index_path = index_path\n self.dim = dim\n self.embedding_model_name = embedding_model_name\n self._pending_items: list[dict] = []\n self._backend_kwargs = {\n \"distance_metric\": distance_metric,\n \"M\": m,\n \"efConstruction\": ef_construction,\n \"is_compact\": is_compact,\n \"is_recompute\": is_recompute,\n }\n self._labels_meta: list[dict] = []\n self._docid_to_indices: dict[int, list[int]] | None = None\n\n def _meta_dict(self) -> dict:\n return {\n \"version\": \"1.0\",\n \"backend_name\": \"hnsw\",\n \"embedding_model\": self.embedding_model_name,\n \"embedding_mode\": \"custom\",\n \"dimensions\": self.dim,\n \"backend_kwargs\": self._backend_kwargs,\n \"is_compact\": self._backend_kwargs.get(\"is_compact\", True),\n \"is_pruned\": self._backend_kwargs.get(\"is_compact\", True)\n and self._backend_kwargs.get(\"is_recompute\", True),\n }\n\n def create_collection(self) -> None:\n path = Path(self.index_path)\n path.parent.mkdir(parents=True, exist_ok=True)\n\n def insert(self, data: dict) -> None:\n self._pending_items.append(\n {\n \"doc_id\": int(data[\"doc_id\"]),\n \"filepath\": data.get(\"filepath\", \"\"),\n \"colbert_vecs\": [np.asarray(v, dtype=np.float32) for v in data[\"colbert_vecs\"]],\n \"image\": data.get(\"image\"), # PIL Image object (optional)\n }\n )\n\n def _labels_path(self) -> Path:\n index_path_obj = Path(self.index_path)\n return index_path_obj.parent / f\"{index_path_obj.name}.labels.json\"\n\n def _meta_path(self) -> Path:\n index_path_obj = Path(self.index_path)\n return index_path_obj.parent / f\"{index_path_obj.name}.meta.json\"\n\n def _embeddings_path(self) -> Path:\n index_path_obj = Path(self.index_path)\n return index_path_obj.parent / f\"{index_path_obj.name}.emb.npy\"\n\n def _images_dir_path(self) -> Path:\n \"\"\"Directory where original images are stored.\"\"\"\n index_path_obj = Path(self.index_path)\n return index_path_obj.parent / f\"{index_path_obj.name}.images\"\n\n def create_index(self) -> None:\n if not self._pending_items:\n return\n\n embeddings: list[np.ndarray] = []\n labels_meta: list[dict] = []\n\n # Create images directory if needed\n images_dir = self._images_dir_path()\n images_dir.mkdir(parents=True, exist_ok=True)\n\n for item in self._pending_items:\n doc_id = int(item[\"doc_id\"])\n filepath = item.get(\"filepath\", \"\")\n colbert_vecs = item[\"colbert_vecs\"]\n image = item.get(\"image\")\n\n # Save image if provided\n image_path = \"\"\n if image is not None and isinstance(image, Image.Image):\n image_filename = f\"doc_{doc_id}.png\"\n image_path = str(images_dir / image_filename)\n image.save(image_path, \"PNG\")\n\n for seq_id, vec in enumerate(colbert_vecs):\n vec_np = np.asarray(vec, dtype=np.float32)\n embeddings.append(vec_np)\n labels_meta.append(\n {\n \"id\": f\"{doc_id}:{seq_id}\",\n \"doc_id\": doc_id,\n \"seq_id\": int(seq_id),\n \"filepath\": filepath,\n \"image_path\": image_path, # Store the path to the saved image\n }\n )\n\n if not embeddings:\n return\n\n embeddings_np = np.vstack(embeddings).astype(np.float32)\n print(embeddings_np.shape)\n\n builder = HNSWBuilder(**{**self._backend_kwargs, \"dimensions\": self.dim})\n ids = [str(i) for i in range(embeddings_np.shape[0])]\n builder.build(embeddings_np, ids, self.index_path)\n\n import json as _json\n\n with open(self._meta_path(), \"w\", encoding=\"utf-8\") as f:\n _json.dump(self._meta_dict(), f, indent=2)\n with open(self._labels_path(), \"w\", encoding=\"utf-8\") as f:\n _json.dump(labels_meta, f)\n\n # Persist embeddings for exact reranking\n np.save(self._embeddings_path(), embeddings_np)\n\n self._labels_meta = labels_meta\n\n def _load_labels_meta_if_needed(self) -> None:\n if self._labels_meta:\n return\n labels_path = self._labels_path()\n if labels_path.exists():\n import json as _json\n\n with open(labels_path, encoding=\"utf-8\") as f:\n self._labels_meta = _json.load(f)\n\n def _build_docid_to_indices_if_needed(self) -> None:\n if self._docid_to_indices is not None:\n return\n self._load_labels_meta_if_needed()\n mapping: dict[int, list[int]] = {}\n for idx, meta in enumerate(self._labels_meta):\n try:\n doc_id = int(meta[\"doc_id\"])\n except Exception:\n continue\n mapping.setdefault(doc_id, []).append(idx)\n self._docid_to_indices = mapping\n\n def search(\n self, data: np.ndarray, topk: int, first_stage_k: int = 50\n ) -> list[tuple[float, int]]:\n if data.ndim == 1:\n data = data.reshape(1, -1)\n if data.dtype != np.float32:\n data = data.astype(np.float32)\n\n self._load_labels_meta_if_needed()\n\n searcher = HNSWSearcher(self.index_path, meta=self._meta_dict())\n raw = searcher.search(\n data,\n first_stage_k,\n recompute_embeddings=False,\n complexity=128,\n beam_width=1,\n prune_ratio=0.0,\n batch_size=0,\n )\n\n labels = raw.get(\"labels\")\n distances = raw.get(\"distances\")\n if labels is None or distances is None:\n return []\n\n doc_scores: dict[int, float] = {}\n B = len(labels)\n for b in range(B):\n per_doc_best: dict[int, float] = {}\n for k, sid in enumerate(labels[b]):\n try:\n idx = int(sid)\n except Exception:\n continue\n if 0 <= idx < len(self._labels_meta):\n doc_id = int(self._labels_meta[idx][\"doc_id\"])\n else:\n continue\n score = float(distances[b][k])\n if (doc_id not in per_doc_best) or (score > per_doc_best[doc_id]):\n per_doc_best[doc_id] = score\n for doc_id, best_score in per_doc_best.items():\n doc_scores[doc_id] = doc_scores.get(doc_id, 0.0) + best_score\n\n scores = sorted(((v, k) for k, v in doc_scores.items()), key=lambda x: x[0], reverse=True)\n return scores[:topk] if len(scores) >= topk else scores\n\n def search_exact(\n self,\n data: np.ndarray,\n topk: int,\n *,\n first_stage_k: int = 200,\n max_workers: int = 32,\n ) -> list[tuple[float, int]]:\n \"\"\"\n High-precision MaxSim reranking over candidate documents.\n\n Steps:\n 1) Run a first-stage ANN to collect candidate doc_ids (using seq-level neighbors).\n 2) For each candidate doc, load all its token embeddings and compute\n MaxSim(query_tokens, doc_tokens) exactly: sum(max(dot(q_i, d_j))).\n\n Returns top-k list of (score, doc_id).\n \"\"\"\n # Normalize inputs\n if data.ndim == 1:\n data = data.reshape(1, -1)\n if data.dtype != np.float32:\n data = data.astype(np.float32)\n\n self._load_labels_meta_if_needed()\n self._build_docid_to_indices_if_needed()\n\n emb_path = self._embeddings_path()\n if not emb_path.exists():\n # Fallback to approximate if we don't have persisted embeddings\n return self.search(data, topk, first_stage_k=first_stage_k)\n\n # Memory-map embeddings to avoid loading all into RAM\n all_embeddings = np.load(emb_path, mmap_mode=\"r\")\n if all_embeddings.dtype != np.float32:\n all_embeddings = all_embeddings.astype(np.float32)\n\n # First-stage ANN to collect candidate doc_ids\n searcher = HNSWSearcher(self.index_path, meta=self._meta_dict())\n raw = searcher.search(\n data,\n first_stage_k,\n recompute_embeddings=False,\n complexity=128,\n beam_width=1,\n prune_ratio=0.0,\n batch_size=0,\n )\n labels = raw.get(\"labels\")\n if labels is None:\n return []\n candidate_doc_ids: set[int] = set()\n for batch in labels:\n for sid in batch:\n try:\n idx = int(sid)\n except Exception:\n continue\n if 0 <= idx < len(self._labels_meta):\n candidate_doc_ids.add(int(self._labels_meta[idx][\"doc_id\"]))\n\n # Exact scoring per doc (parallelized)\n docid_to_indices = self._docid_to_indices\n if docid_to_indices is None:\n return []\n\n def _score_one(doc_id: int) -> tuple[float, int]:\n token_indices = docid_to_indices.get(doc_id, [])\n if not token_indices:\n return (0.0, doc_id)\n doc_vecs = np.asarray(all_embeddings[token_indices], dtype=np.float32)\n # (Q, D) x (P, D)^T -> (Q, P) then MaxSim over P, sum over Q\n sim = np.dot(data, doc_vecs.T)\n # nan-safe\n sim = np.nan_to_num(sim, nan=-1e30, posinf=1e30, neginf=-1e30)\n score = sim.max(axis=2).sum(axis=1) if sim.ndim == 3 else sim.max(axis=1).sum()\n return (float(score), doc_id)\n\n scores: list[tuple[float, int]] = []\n # load and core time\n start_time = time.time()\n with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as ex:\n futures = [ex.submit(_score_one, doc_id) for doc_id in candidate_doc_ids]\n for fut in concurrent.futures.as_completed(futures):\n scores.append(fut.result())\n end_time = time.time()\n print(f\"Number of candidate doc ids: {len(candidate_doc_ids)}\")\n print(f\"Time taken in load and core time: {end_time - start_time} seconds\")\n scores.sort(key=lambda x: x[0], reverse=True)\n return scores[:topk] if len(scores) >= topk else scores\n\n def search_exact_all(\n self,\n data: np.ndarray,\n topk: int,\n *,\n max_workers: int = 32,\n ) -> list[tuple[float, int]]:\n \"\"\"\n Exact MaxSim over ALL documents (no ANN pre-filtering).\n\n This computes, for each document, sum_i max_j dot(q_i, d_j).\n It memory-maps the persisted token-embedding matrix for scalability.\n \"\"\"\n if data.ndim == 1:\n data = data.reshape(1, -1)\n if data.dtype != np.float32:\n data = data.astype(np.float32)\n\n self._load_labels_meta_if_needed()\n self._build_docid_to_indices_if_needed()\n\n emb_path = self._embeddings_path()\n if not emb_path.exists():\n return self.search(data, topk)\n all_embeddings = np.load(emb_path, mmap_mode=\"r\")\n if all_embeddings.dtype != np.float32:\n all_embeddings = all_embeddings.astype(np.float32)\n\n docid_to_indices = self._docid_to_indices\n if docid_to_indices is None:\n return []\n candidate_doc_ids = list(docid_to_indices.keys())\n\n def _score_one(doc_id: int, _all_embeddings=all_embeddings) -> tuple[float, int]:\n token_indices = docid_to_indices.get(doc_id, [])\n if not token_indices:\n return (0.0, doc_id)\n doc_vecs = np.asarray(_all_embeddings[token_indices], dtype=np.float32)\n sim = np.dot(data, doc_vecs.T)\n sim = np.nan_to_num(sim, nan=-1e30, posinf=1e30, neginf=-1e30)\n score = sim.max(axis=2).sum(axis=1) if sim.ndim == 3 else sim.max(axis=1).sum()\n return (float(score), doc_id)\n\n scores: list[tuple[float, int]] = []\n # load and core time\n start_time = time.time()\n with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as ex:\n futures = [ex.submit(_score_one, d) for d in candidate_doc_ids]\n for fut in concurrent.futures.as_completed(futures):\n scores.append(fut.result())\n end_time = time.time()\n # print number of candidate doc ids\n print(f\"Number of candidate doc ids: {len(candidate_doc_ids)}\")\n print(f\"Time taken in load and core time: {end_time - start_time} seconds\")\n scores.sort(key=lambda x: x[0], reverse=True)\n del all_embeddings\n return scores[:topk] if len(scores) >= topk else scores\n\n def get_image(self, doc_id: int) -> Optional[Image.Image]:\n \"\"\"\n Retrieve the original image for a given doc_id from the index.\n\n Args:\n doc_id: The document ID\n\n Returns:\n PIL Image object if found, None otherwise\n \"\"\"\n self._load_labels_meta_if_needed()\n\n # Find the image_path for this doc_id (all seq_ids for same doc share the same image_path)\n for meta in self._labels_meta:\n if meta.get(\"doc_id\") == doc_id:\n image_path = meta.get(\"image_path\", \"\")\n if image_path and Path(image_path).exists():\n return Image.open(image_path)\n break\n return None\n\n def get_metadata(self, doc_id: int) -> Optional[dict]:\n \"\"\"\n Retrieve metadata for a given doc_id.\n\n Args:\n doc_id: The document ID\n\n Returns:\n Dictionary with metadata (filepath, image_path, etc.) if found, None otherwise\n \"\"\"\n self._load_labels_meta_if_needed()\n\n for meta in self._labels_meta:\n if meta.get(\"doc_id\") == doc_id:\n return {\n \"doc_id\": doc_id,\n \"filepath\": meta.get(\"filepath\", \"\"),\n \"image_path\": meta.get(\"image_path\", \"\"),\n }\n return None\n\n\nclass ViDoReBenchmarkEvaluator:\n \"\"\"\n A reusable class for evaluating ViDoRe benchmarks (v1 and v2).\n This class encapsulates common functionality for building indexes, searching, and evaluating.\n \"\"\"\n\n def __init__(\n self,\n model_name: str,\n use_fast_plaid: bool = False,\n top_k: int = 100,\n first_stage_k: int = 500,\n k_values: Optional[list[int]] = None,\n ):\n \"\"\"\n Initialize the evaluator.\n\n Args:\n model_name: Model name (\"colqwen2\" or \"colpali\")\n use_fast_plaid: Whether to use Fast-Plaid instead of LEANN\n top_k: Top-k results to retrieve\n first_stage_k: First stage k for LEANN search\n k_values: List of k values for evaluation metrics\n \"\"\"\n self.model_name = model_name\n self.use_fast_plaid = use_fast_plaid\n self.top_k = top_k\n self.first_stage_k = first_stage_k\n self.k_values = k_values if k_values is not None else [1, 3, 5, 10, 100]\n\n # Load model once (can be reused across tasks)\n self._model = None\n self._processor = None\n self._model_name_actual = None\n\n def _load_model_if_needed(self):\n \"\"\"Lazy load the model.\"\"\"\n if self._model is None:\n print(f\"\\nLoading model: {self.model_name}\")\n self._model_name_actual, self._model, self._processor, _, _, _ = _load_colvision(\n self.model_name\n )\n print(f\"Model loaded: {self._model_name_actual}\")\n\n def build_index_from_corpus(\n self,\n corpus: dict[str, Image.Image],\n index_path: str,\n rebuild: bool = False,\n ) -> tuple[Any, list[str]]:\n \"\"\"\n Build index from corpus images.\n\n Args:\n corpus: dict mapping corpus_id to PIL Image\n index_path: Path to save/load the index\n rebuild: Whether to rebuild even if index exists\n\n Returns:\n tuple: (retriever or fast_plaid_index object, list of corpus_ids in order)\n \"\"\"\n self._load_model_if_needed()\n\n # Ensure consistent ordering\n corpus_ids = sorted(corpus.keys())\n images = [corpus[cid] for cid in corpus_ids]\n\n if self.use_fast_plaid:\n # Check if Fast-Plaid index exists\n if not rebuild and _load_fast_plaid_index_if_exists(index_path) is not None:\n print(f\"Fast-Plaid index already exists at {index_path}\")\n return _load_fast_plaid_index_if_exists(index_path), corpus_ids\n\n print(f\"Building Fast-Plaid index at {index_path}...\")\n print(\"Embedding images...\")\n doc_vecs = _embed_images(self._model, self._processor, images)\n\n fast_plaid_index, build_time = _build_fast_plaid_index(\n index_path, doc_vecs, corpus_ids, images\n )\n print(f\"Fast-Plaid index built in {build_time:.2f}s\")\n return fast_plaid_index, corpus_ids\n else:\n # Check if LEANN index exists\n if not rebuild:\n retriever = _load_retriever_if_index_exists(index_path)\n if retriever is not None:\n print(f\"LEANN index already exists at {index_path}\")\n return retriever, corpus_ids\n\n print(f\"Building LEANN index at {index_path}...\")\n print(\"Embedding images...\")\n doc_vecs = _embed_images(self._model, self._processor, images)\n\n retriever = _build_index(index_path, doc_vecs, corpus_ids, images)\n print(\"LEANN index built\")\n return retriever, corpus_ids\n\n def search_queries(\n self,\n queries: dict[str, str],\n corpus_ids: list[str],\n index_or_retriever: Any,\n fast_plaid_index_path: Optional[str] = None,\n task_prompt: Optional[dict[str, str]] = None,\n ) -> dict[str, dict[str, float]]:\n \"\"\"\n Search queries against the index.\n\n Args:\n queries: dict mapping query_id to query text\n corpus_ids: list of corpus_ids in the same order as the index\n index_or_retriever: index or retriever object\n fast_plaid_index_path: path to Fast-Plaid index (for metadata)\n task_prompt: Optional dict with prompt for query (e.g., {\"query\": \"...\"})\n\n Returns:\n results: dict mapping query_id to dict of {corpus_id: score}\n \"\"\"\n self._load_model_if_needed()\n\n print(f\"Searching {len(queries)} queries (top_k={self.top_k})...\")\n\n query_ids = list(queries.keys())\n query_texts = [queries[qid] for qid in query_ids]\n\n # Note: ColPaliEngineWrapper does NOT use task prompt from metadata\n # It uses query_prefix + text + query_augmentation_token (handled in _embed_queries)\n # So we don't append task_prompt here to match MTEB behavior\n\n # Embed queries\n print(\"Embedding queries...\")\n query_vecs = _embed_queries(self._model, self._processor, query_texts)\n\n results = {}\n\n for query_id, query_vec in zip(tqdm(query_ids, desc=\"Searching\"), query_vecs):\n if self.use_fast_plaid:\n # Fast-Plaid search\n search_results, _ = _search_fast_plaid(index_or_retriever, query_vec, self.top_k)\n query_results = {}\n for score, doc_id in search_results:\n if doc_id < len(corpus_ids):\n corpus_id = corpus_ids[doc_id]\n query_results[corpus_id] = float(score)\n else:\n # LEANN search\n import torch\n\n query_np = (\n query_vec.float().numpy() if isinstance(query_vec, torch.Tensor) else query_vec\n )\n search_results = index_or_retriever.search_exact(query_np, topk=self.top_k)\n query_results = {}\n for score, doc_id in search_results:\n if doc_id < len(corpus_ids):\n corpus_id = corpus_ids[doc_id]\n query_results[corpus_id] = float(score)\n\n results[query_id] = query_results\n\n return results\n\n @staticmethod\n def evaluate_results(\n results: dict[str, dict[str, float]],\n qrels: dict[str, dict[str, int]],\n k_values: Optional[list[int]] = None,\n ) -> dict[str, float]:\n \"\"\"\n Evaluate retrieval results using NDCG and other metrics.\n\n Args:\n results: dict mapping query_id to dict of {corpus_id: score}\n qrels: dict mapping query_id to dict of {corpus_id: relevance_score}\n k_values: List of k values for evaluation metrics\n\n Returns:\n Dictionary of metric scores\n \"\"\"\n try:\n from mteb._evaluators.retrieval_metrics import (\n calculate_retrieval_scores,\n make_score_dict,\n )\n except ImportError:\n raise ImportError(\n \"pytrec_eval is required for evaluation. Install with: pip install pytrec-eval\"\n )\n\n if k_values is None:\n k_values = [1, 3, 5, 10, 100]\n\n # Check if we have any queries to evaluate\n if len(results) == 0:\n print(\"Warning: No queries to evaluate. Returning zero scores.\")\n scores = {}\n for k in k_values:\n scores[f\"ndcg_at_{k}\"] = 0.0\n scores[f\"map_at_{k}\"] = 0.0\n scores[f\"recall_at_{k}\"] = 0.0\n scores[f\"precision_at_{k}\"] = 0.0\n scores[f\"mrr_at_{k}\"] = 0.0\n return scores\n\n print(f\"Evaluating results with k_values={k_values}...\")\n print(f\"Before filtering: {len(results)} results, {len(qrels)} qrels\")\n\n # Filter to ensure qrels and results have the same query set\n # This matches MTEB behavior: only evaluate queries that exist in both\n # pytrec_eval only evaluates queries in qrels, so we need to ensure\n # results contains all queries in qrels, and filter out queries not in qrels\n results_filtered = {qid: res for qid, res in results.items() if qid in qrels}\n qrels_filtered = {\n qid: rel_docs for qid, rel_docs in qrels.items() if qid in results_filtered\n }\n\n print(f\"After filtering: {len(results_filtered)} results, {len(qrels_filtered)} qrels\")\n\n if len(results_filtered) != len(qrels_filtered):\n print(\n f\"Warning: Mismatch between results ({len(results_filtered)}) and qrels ({len(qrels_filtered)}) queries\"\n )\n missing_in_results = set(qrels.keys()) - set(results.keys())\n if missing_in_results:\n print(f\"Queries in qrels but not in results: {len(missing_in_results)} queries\")\n print(f\"First 5 missing queries: {list(missing_in_results)[:5]}\")\n\n # Convert qrels to pytrec_eval format\n qrels_pytrec = {}\n for qid, rel_docs in qrels_filtered.items():\n qrels_pytrec[qid] = dict(rel_docs.items())\n\n # Evaluate\n eval_result = calculate_retrieval_scores(\n results=results_filtered,\n qrels=qrels_pytrec,\n k_values=k_values,\n )\n\n # Format scores\n scores = make_score_dict(\n ndcg=eval_result.ndcg,\n _map=eval_result.map,\n recall=eval_result.recall,\n precision=eval_result.precision,\n mrr=eval_result.mrr,\n naucs=eval_result.naucs,\n naucs_mrr=eval_result.naucs_mrr,\n cv_recall=eval_result.cv_recall,\n task_scores={},\n )\n\n return scores\n" }, { "path": "apps/multimodal/vision-based-pdf-multi-vector/multi-vector-leann-paper-example.py", "content": "# pip install pdf2image\n# pip install pymilvus\n# pip install colpali_engine\n# pip install tqdm\n# pip install pillow\n\nimport os\nimport re\nimport sys\nfrom pathlib import Path\nfrom typing import cast\n\nfrom PIL import Image\nfrom tqdm import tqdm\n\n# Ensure local leann packages are importable before importing them\n_repo_root = Path(__file__).resolve().parents[3]\n_leann_core_src = _repo_root / \"packages\" / \"leann-core\" / \"src\"\n_leann_hnsw_pkg = _repo_root / \"packages\" / \"leann-backend-hnsw\"\nif str(_leann_core_src) not in sys.path:\n sys.path.insert(0, str(_leann_core_src))\nif str(_leann_hnsw_pkg) not in sys.path:\n sys.path.insert(0, str(_leann_hnsw_pkg))\n\nfrom leann_multi_vector import LeannMultiVector\n\nimport torch\nfrom colpali_engine.models import ColPali\nfrom colpali_engine.models.paligemma.colpali.processing_colpali import ColPaliProcessor\nfrom colpali_engine.utils.torch_utils import ListDataset, get_torch_device\nfrom torch.utils.data import DataLoader\n\n# Auto-select device: CUDA > MPS (mac) > CPU\n_device_str = (\n \"cuda\"\n if torch.cuda.is_available()\n else (\n \"mps\"\n if getattr(torch.backends, \"mps\", None) and torch.backends.mps.is_available()\n else \"cpu\"\n )\n)\ndevice = get_torch_device(_device_str)\n# Prefer fp16 on GPU/MPS, bfloat16 on CPU\n_dtype = torch.float16 if _device_str in (\"cuda\", \"mps\") else torch.bfloat16\nmodel_name = \"vidore/colpali-v1.2\"\n\nmodel = ColPali.from_pretrained(\n model_name,\n torch_dtype=_dtype,\n device_map=device,\n).eval()\nprint(f\"Using device={_device_str}, dtype={_dtype}\")\n\nqueries = [\n \"How to end-to-end retrieval with ColBert\",\n \"Where is ColBERT performance Table, including text representation results?\",\n]\n\nprocessor = cast(ColPaliProcessor, ColPaliProcessor.from_pretrained(model_name))\n\ndataloader = DataLoader(\n dataset=ListDataset[str](queries),\n batch_size=1,\n shuffle=False,\n collate_fn=lambda x: processor.process_queries(x),\n)\n\nqs: list[torch.Tensor] = []\nfor batch_query in dataloader:\n with torch.no_grad():\n batch_query = {k: v.to(model.device) for k, v in batch_query.items()}\n embeddings_query = model(**batch_query)\n qs.extend(list(torch.unbind(embeddings_query.to(\"cpu\"))))\nprint(qs[0].shape)\n# %%\ndef _page_sort_key(name: str) -> int:\n match = re.search(r\"\\d+\", name)\n return int(match.group()) if match else -1\n\n\npage_filenames = sorted(os.listdir(\"./pages\"), key=_page_sort_key)\nimages = [Image.open(os.path.join(\"./pages\", name)) for name in page_filenames]\n\ndataloader = DataLoader(\n dataset=ListDataset[str](images),\n batch_size=1,\n shuffle=False,\n collate_fn=lambda x: processor.process_images(x),\n)\n\nds: list[torch.Tensor] = []\nfor batch_doc in tqdm(dataloader):\n with torch.no_grad():\n batch_doc = {k: v.to(model.device) for k, v in batch_doc.items()}\n embeddings_doc = model(**batch_doc)\n ds.extend(list(torch.unbind(embeddings_doc.to(\"cpu\"))))\n\nprint(ds[0].shape)\n\n# %%\n# Build HNSW index via LeannMultiVector primitives and run search\nindex_path = \"./indexes/colpali.leann\"\nretriever = LeannMultiVector(index_path=index_path, dim=int(ds[0].shape[-1]))\nretriever.create_collection()\nfilepaths = [os.path.join(\"./pages\", name) for name in page_filenames]\nfor i in range(len(filepaths)):\n data = {\n \"colbert_vecs\": ds[i].float().numpy(),\n \"doc_id\": i,\n \"filepath\": filepaths[i],\n }\n retriever.insert(data)\nretriever.create_index()\nfor query in qs:\n query_np = query.float().numpy()\n result = retriever.search(query_np, topk=1)\n print(filepaths[result[0][1]])\n" }, { "path": "apps/multimodal/vision-based-pdf-multi-vector/multi-vector-leann-similarity-map.py", "content": "## Jupyter-style notebook script\n# %%\n# uv pip install matplotlib qwen_vl_utils\nimport argparse\nimport faulthandler\nimport os\nimport time\nfrom typing import Any, Optional, cast\n\nimport numpy as np\nfrom PIL import Image\nfrom tqdm import tqdm\n\n# Enable faulthandler to get stack trace on segfault\nfaulthandler.enable()\n\n\nfrom leann_multi_vector import ( # utility functions/classes\n _ensure_repo_paths_importable,\n _load_images_from_dir,\n _maybe_convert_pdf_to_images,\n _load_colvision,\n _embed_images,\n _embed_queries,\n _build_index,\n _load_retriever_if_index_exists,\n _generate_similarity_map,\n _build_fast_plaid_index,\n _load_fast_plaid_index_if_exists,\n _search_fast_plaid,\n _get_fast_plaid_image,\n _get_fast_plaid_metadata,\n QwenVL,\n)\n\n_ensure_repo_paths_importable(__file__)\n\n# %%\n# Config\nos.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\nQUERY = \"The paper talk about the latent video generative model and data curation in the related work part?\"\nMODEL: str = \"colqwen2\" # \"colpali\" or \"colqwen2\"\n\n# Data source: set to True to use the Hugging Face dataset example (recommended)\nUSE_HF_DATASET: bool = True\n# Single dataset name (used when DATASET_NAMES is None)\nDATASET_NAME: str = \"weaviate/arXiv-AI-papers-multi-vector\"\n# Multiple datasets to combine (if provided, DATASET_NAME is ignored)\n# Can be:\n# - List of strings: [\"dataset1\", \"dataset2\"]\n# - List of tuples: [(\"dataset1\", \"config1\"), (\"dataset2\", None)] # None = no config needed\n# - Mixed: [\"dataset1\", (\"dataset2\", \"config2\")]\n#\n# Some potential datasets with images (may need IMAGE_FIELD_NAME adjustment):\n# - \"weaviate/arXiv-AI-papers-multi-vector\" (current, has \"page_image\" field)\n# - (\"lmms-lab/DocVQA\", \"DocVQA\") (has \"image\" field, document images, needs config)\n# - (\"lmms-lab/DocVQA\", \"InfographicVQA\") (has \"image\" field, infographic images)\n# - \"pixparse/arxiv-papers\" (if available, arXiv papers)\n# - \"allenai/ai2d\" (AI2D diagram dataset, has \"image\" field)\n# - \"huggingface/document-images\" (if available)\n# Note: Check dataset structure first - some may need IMAGE_FIELD_NAME specified\n# DATASET_NAMES: Optional[list[str | tuple[str, Optional[str]]]] = None\nDATASET_NAMES = [\n \"weaviate/arXiv-AI-papers-multi-vector\",\n # (\"lmms-lab/DocVQA\", \"DocVQA\"), # Specify config name for datasets with multiple configs\n]\n# Load multiple splits to get more data (e.g., [\"train\", \"test\", \"validation\"])\n# Set to None to try loading all available splits automatically\nDATASET_SPLITS: Optional[list[str]] = [\"train\", \"test\"] # None = auto-detect all splits\n# Image field name in the dataset (auto-detect if None)\n# Common names: \"page_image\", \"image\", \"images\", \"img\"\nIMAGE_FIELD_NAME: Optional[str] = None # None = auto-detect\nMAX_DOCS: Optional[int] = None # limit number of pages to index; None = all\n\n# Local pages (used when USE_HF_DATASET == False)\nPDF: Optional[str] = None # e.g., \"./pdfs/2004.12832v2.pdf\"\nPAGES_DIR: str = \"./pages\"\n# Custom folder path (takes precedence over USE_HF_DATASET and PAGES_DIR)\n# If set, images will be loaded directly from this folder\nCUSTOM_FOLDER_PATH: Optional[str] = None # e.g., \"/home/ubuntu/dr-tulu/agent/screenshots\"\n# Whether to recursively search subdirectories when loading from custom folder\nCUSTOM_FOLDER_RECURSIVE: bool = False # Set to True to search subdirectories\n\n# Index + retrieval settings\n# Use a different index path for larger dataset to avoid overwriting existing index\nINDEX_PATH: str = \"./indexes/colvision_large.leann\"\n# Fast-Plaid index settings (alternative to LEANN index)\n# These are now command-line arguments (see CLI overrides section)\nTOPK: int = 3\nFIRST_STAGE_K: int = 500\nREBUILD_INDEX: bool = False # Set to True to force rebuild even if index exists\n\n# Artifacts\nSAVE_TOP_IMAGE: Optional[str] = \"./figures/retrieved_page.png\"\nSIMILARITY_MAP: bool = True\nSIM_TOKEN_IDX: int = 13 # -1 means auto-select the most salient token\nSIM_OUTPUT: str = \"./figures/similarity_map.png\"\nANSWER: bool = True\nMAX_NEW_TOKENS: int = 1024\n\n\n# %%\n# CLI overrides\nparser = argparse.ArgumentParser(description=\"Multi-vector LEANN similarity map demo\")\nparser.add_argument(\n \"--search-method\",\n type=str,\n choices=[\"ann\", \"exact\", \"exact-all\"],\n default=\"ann\",\n help=\"Which search method to use: 'ann' (fast ANN), 'exact' (ANN + exact rerank), or 'exact-all' (exact over all docs).\",\n)\nparser.add_argument(\n \"--query\",\n type=str,\n default=QUERY,\n help=f\"Query string to search for. Default: '{QUERY}'\",\n)\nparser.add_argument(\n \"--use-fast-plaid\",\n action=\"store_true\",\n default=False,\n help=\"Set to True to use fast-plaid instead of LEANN. Default: False\",\n)\nparser.add_argument(\n \"--fast-plaid-index-path\",\n type=str,\n default=\"./indexes/colvision_fastplaid\",\n help=\"Path to the Fast-Plaid index. Default: './indexes/colvision_fastplaid'\",\n)\nparser.add_argument(\n \"--topk\",\n type=int,\n default=TOPK,\n help=f\"Number of top results to retrieve. Default: {TOPK}\",\n)\nparser.add_argument(\n \"--custom-folder\",\n type=str,\n default=None,\n help=\"Path to a custom folder containing images to search. Takes precedence over dataset loading. Default: None\",\n)\nparser.add_argument(\n \"--recursive\",\n action=\"store_true\",\n default=False,\n help=\"Recursively search subdirectories when loading images from custom folder. Default: False\",\n)\nparser.add_argument(\n \"--rebuild-index\",\n action=\"store_true\",\n default=False,\n help=\"Force rebuild the index even if it already exists. Default: False (reuse existing index if available)\",\n)\ncli_args, _unknown = parser.parse_known_args()\nSEARCH_METHOD: str = cli_args.search_method\nQUERY = cli_args.query # Override QUERY with CLI argument if provided\nUSE_FAST_PLAID: bool = cli_args.use_fast_plaid\nFAST_PLAID_INDEX_PATH: str = cli_args.fast_plaid_index_path\nTOPK: int = cli_args.topk # Override TOPK with CLI argument if provided\nCUSTOM_FOLDER_PATH = cli_args.custom_folder if cli_args.custom_folder else CUSTOM_FOLDER_PATH # Override with CLI argument if provided\nCUSTOM_FOLDER_RECURSIVE = cli_args.recursive if cli_args.recursive else CUSTOM_FOLDER_RECURSIVE # Override with CLI argument if provided\nREBUILD_INDEX = cli_args.rebuild_index # Override REBUILD_INDEX with CLI argument\n\n# %%\n\n# Step 1: Check if we can skip data loading (index already exists)\nretriever: Optional[Any] = None\nfast_plaid_index: Optional[Any] = None\nneed_to_build_index = REBUILD_INDEX\n\nif USE_FAST_PLAID:\n # Fast-Plaid index handling\n if not REBUILD_INDEX:\n try:\n fast_plaid_index = _load_fast_plaid_index_if_exists(FAST_PLAID_INDEX_PATH)\n if fast_plaid_index is not None:\n print(f\"✓ Fast-Plaid index found at {FAST_PLAID_INDEX_PATH}\")\n need_to_build_index = False\n else:\n print(f\"Fast-Plaid index not found, will build new index\")\n need_to_build_index = True\n except Exception as e:\n # If loading fails (e.g., memory error, corrupted index), rebuild\n print(f\"Warning: Failed to load Fast-Plaid index: {e}\")\n print(\"Will rebuild the index...\")\n need_to_build_index = True\n fast_plaid_index = None\n else:\n print(f\"REBUILD_INDEX=True, will rebuild Fast-Plaid index\")\n need_to_build_index = True\nelse:\n # Original LEANN index handling\n if not REBUILD_INDEX:\n retriever = _load_retriever_if_index_exists(INDEX_PATH)\n if retriever is not None:\n retriever_any = cast(Any, retriever)\n print(f\"✓ Index loaded from {INDEX_PATH}\")\n print(f\"✓ Images available at: {retriever_any._images_dir_path()}\")\n need_to_build_index = False\n else:\n print(f\"Index not found, will build new index\")\n need_to_build_index = True\n else:\n print(f\"REBUILD_INDEX=True, will rebuild index\")\n need_to_build_index = True\n\n# Step 2: Load data only if we need to build the index\nif need_to_build_index:\n print(\"Loading dataset...\")\n # Check for custom folder path first (takes precedence)\n if CUSTOM_FOLDER_PATH:\n if not os.path.isdir(CUSTOM_FOLDER_PATH):\n raise RuntimeError(f\"Custom folder path does not exist: {CUSTOM_FOLDER_PATH}\")\n print(f\"Loading images from custom folder: {CUSTOM_FOLDER_PATH}\")\n if CUSTOM_FOLDER_RECURSIVE:\n print(\" (recursive mode: searching subdirectories)\")\n filepaths, images = _load_images_from_dir(CUSTOM_FOLDER_PATH, recursive=CUSTOM_FOLDER_RECURSIVE)\n print(f\" Found {len(filepaths)} image files\")\n if not images:\n raise RuntimeError(\n f\"No images found in {CUSTOM_FOLDER_PATH}. Ensure the folder contains image files (.png, .jpg, .jpeg, .webp).\"\n )\n print(f\" Successfully loaded {len(images)} images\")\n # Use filenames as identifiers instead of full paths for cleaner metadata\n filepaths = [os.path.basename(fp) for fp in filepaths]\n elif USE_HF_DATASET:\n from datasets import Dataset, concatenate_datasets, load_dataset\n\n # Determine which datasets to load\n if DATASET_NAMES is not None:\n dataset_names_to_load = DATASET_NAMES\n print(f\"Loading {len(dataset_names_to_load)} datasets: {dataset_names_to_load}\")\n else:\n dataset_names_to_load = [DATASET_NAME]\n print(f\"Loading single dataset: {DATASET_NAME}\")\n\n # Load and combine datasets\n all_datasets_to_concat = []\n\n for dataset_entry in dataset_names_to_load:\n # Handle both string and tuple formats\n if isinstance(dataset_entry, tuple):\n dataset_name, config_name = dataset_entry\n else:\n dataset_name = dataset_entry\n config_name = None\n\n print(f\"\\nProcessing dataset: {dataset_name}\" + (f\" (config: {config_name})\" if config_name else \"\"))\n\n # Load dataset to check available splits\n # If config_name is provided, use it; otherwise try without config\n try:\n if config_name:\n dataset_dict = load_dataset(dataset_name, config_name)\n else:\n dataset_dict = load_dataset(dataset_name)\n except ValueError as e:\n if \"Config name is missing\" in str(e):\n # Try to get available configs and suggest\n from datasets import get_dataset_config_names\n try:\n available_configs = get_dataset_config_names(dataset_name)\n raise ValueError(\n f\"Dataset '{dataset_name}' requires a config name. \"\n f\"Available configs: {available_configs}. \"\n f\"Please specify as: ('{dataset_name}', 'config_name')\"\n ) from e\n except Exception:\n raise ValueError(\n f\"Dataset '{dataset_name}' requires a config name. \"\n f\"Please specify as: ('{dataset_name}', 'config_name')\"\n ) from e\n raise\n\n if not isinstance(dataset_dict, dict):\n dataset = cast(Dataset, dataset_dict)\n all_datasets_to_concat.append(dataset)\n continue\n\n # Determine which splits to load\n if DATASET_SPLITS is None:\n # Auto-detect: try to load all available splits\n available_splits = list(dataset_dict.keys())\n print(f\" Auto-detected splits: {available_splits}\")\n splits_to_load = available_splits\n else:\n splits_to_load = DATASET_SPLITS\n\n # Load and concatenate multiple splits for this dataset\n datasets_to_concat: list[Dataset] = []\n for split in splits_to_load:\n if split not in dataset_dict:\n print(\n f\" Warning: Split '{split}' not found in dataset. Available splits: {list(dataset_dict.keys())}\"\n )\n continue\n split_dataset = cast(Dataset, dataset_dict[split])\n print(f\" Loaded split '{split}': {len(split_dataset)} pages\")\n datasets_to_concat.append(split_dataset)\n\n if not datasets_to_concat:\n print(f\" Warning: No valid splits found for {dataset_name}. Skipping.\")\n continue\n\n # Concatenate splits for this dataset\n if len(datasets_to_concat) > 1:\n combined_dataset = concatenate_datasets(datasets_to_concat)\n print(f\" Concatenated {len(datasets_to_concat)} splits into {len(combined_dataset)} pages\")\n else:\n combined_dataset = datasets_to_concat[0]\n\n all_datasets_to_concat.append(combined_dataset)\n\n if not all_datasets_to_concat:\n raise RuntimeError(\"No valid datasets or splits found.\")\n\n # Concatenate all datasets\n if len(all_datasets_to_concat) > 1:\n dataset = concatenate_datasets(all_datasets_to_concat)\n print(f\"\\nConcatenated {len(all_datasets_to_concat)} datasets into {len(dataset)} total pages\")\n else:\n dataset = all_datasets_to_concat[0]\n\n # Apply MAX_DOCS limit if specified\n N = len(dataset) if MAX_DOCS is None else min(MAX_DOCS, len(dataset))\n if N < len(dataset):\n print(f\"Limiting to {N} pages (from {len(dataset)} total)\")\n dataset = dataset.select(range(N))\n\n # Auto-detect image field name if not specified\n if IMAGE_FIELD_NAME is None:\n # Check multiple samples to find the most common image field\n # (useful when datasets are merged and may have different field names)\n possible_image_fields = [\"page_image\", \"image\", \"images\", \"img\", \"page\", \"document_image\"]\n field_counts = {}\n\n # Check first few samples to find image fields\n num_samples_to_check = min(10, len(dataset))\n for sample_idx in range(num_samples_to_check):\n sample = dataset[sample_idx]\n for field in possible_image_fields:\n if field in sample and sample[field] is not None:\n value = sample[field]\n if isinstance(value, Image.Image) or (hasattr(value, 'size') and hasattr(value, 'mode')):\n field_counts[field] = field_counts.get(field, 0) + 1\n\n # Choose the most common field, or first found if tied\n if field_counts:\n image_field = max(field_counts.items(), key=lambda x: x[1])[0]\n print(f\"Auto-detected image field: '{image_field}' (found in {field_counts[image_field]}/{num_samples_to_check} samples)\")\n else:\n # Fallback: check first sample only\n sample = dataset[0]\n image_field = None\n for field in possible_image_fields:\n if field in sample:\n value = sample[field]\n if isinstance(value, Image.Image) or (hasattr(value, 'size') and hasattr(value, 'mode')):\n image_field = field\n break\n if image_field is None:\n raise RuntimeError(\n f\"Could not auto-detect image field. Available fields: {list(sample.keys())}. \"\n f\"Please specify IMAGE_FIELD_NAME manually.\"\n )\n print(f\"Auto-detected image field: '{image_field}'\")\n else:\n image_field = IMAGE_FIELD_NAME\n if image_field not in dataset[0]:\n raise RuntimeError(\n f\"Image field '{image_field}' not found. Available fields: {list(dataset[0].keys())}\"\n )\n\n filepaths: list[str] = []\n images: list[Image.Image] = []\n for i in tqdm(range(len(dataset)), desc=\"Loading dataset\", total=len(dataset)):\n p = dataset[i]\n # Try to compose a descriptive identifier\n # Handle different dataset structures\n identifier_parts = []\n\n # Helper function to safely get field value\n def safe_get(field_name, default=None):\n if field_name in p and p[field_name] is not None:\n return p[field_name]\n return default\n\n # Try to get various identifier fields\n if safe_get(\"paper_arxiv_id\"):\n identifier_parts.append(f\"arXiv:{p['paper_arxiv_id']}\")\n if safe_get(\"paper_title\"):\n identifier_parts.append(f\"title:{p['paper_title']}\")\n if safe_get(\"page_number\") is not None:\n try:\n identifier_parts.append(f\"page:{int(p['page_number'])}\")\n except (ValueError, TypeError):\n # If conversion fails, use the raw value or skip\n if p['page_number']:\n identifier_parts.append(f\"page:{p['page_number']}\")\n if safe_get(\"page_id\"):\n identifier_parts.append(f\"id:{p['page_id']}\")\n elif safe_get(\"questionId\"):\n identifier_parts.append(f\"qid:{p['questionId']}\")\n elif safe_get(\"docId\"):\n identifier_parts.append(f\"docId:{p['docId']}\")\n elif safe_get(\"id\"):\n identifier_parts.append(f\"id:{p['id']}\")\n\n # If no identifier parts found, create one from index\n if identifier_parts:\n identifier = \"|\".join(identifier_parts)\n else:\n # Create identifier from available fields or index\n fallback_parts = []\n # Try common fields that might exist\n for field in [\"ucsf_document_id\", \"docId\", \"questionId\", \"id\"]:\n if safe_get(field):\n fallback_parts.append(f\"{field}:{p[field]}\")\n break\n if fallback_parts:\n identifier = \"|\".join(fallback_parts) + f\"|idx:{i}\"\n else:\n identifier = f\"doc_{i}\"\n\n filepaths.append(identifier)\n\n # Get image - try detected field first, then fallback to other common fields\n img = None\n if image_field in p and p[image_field] is not None:\n img = p[image_field]\n else:\n # Fallback: try other common image field names\n for fallback_field in [\"image\", \"page_image\", \"images\", \"img\"]:\n if fallback_field in p and p[fallback_field] is not None:\n img = p[fallback_field]\n break\n\n if img is None:\n raise RuntimeError(\n f\"No image found for sample {i}. Available fields: {list(p.keys())}. \"\n f\"Expected field: {image_field}\"\n )\n\n # Ensure it's a PIL Image\n if not isinstance(img, Image.Image):\n if hasattr(img, 'convert'):\n img = img.convert('RGB')\n else:\n img = Image.fromarray(img) if hasattr(img, '__array__') else Image.open(img)\n images.append(img)\n else:\n _maybe_convert_pdf_to_images(PDF, PAGES_DIR)\n filepaths, images = _load_images_from_dir(PAGES_DIR)\n if not images:\n raise RuntimeError(\n f\"No images found in {PAGES_DIR}. Provide PDF path in PDF variable or ensure images exist.\"\n )\n print(f\"Loaded {len(images)} images\")\n\n # Memory check before loading model\n try:\n import psutil\n import torch\n process = psutil.Process(os.getpid())\n mem_info = process.memory_info()\n print(f\"Memory usage after loading images: {mem_info.rss / 1024 / 1024 / 1024:.2f} GB\")\n if torch.cuda.is_available():\n print(f\"GPU memory allocated: {torch.cuda.memory_allocated() / 1024**3:.2f} GB\")\n print(f\"GPU memory reserved: {torch.cuda.memory_reserved() / 1024**3:.2f} GB\")\n except ImportError:\n pass\nelse:\n print(\"Skipping dataset loading (using existing index)\")\n filepaths = [] # Not needed when using existing index\n images = [] # Not needed when using existing index\n\n\n# %%\n# Step 3: Load model and processor (only if we need to build index or perform search)\nprint(\"Step 3: Loading model and processor...\")\nprint(f\" Model: {MODEL}\")\ntry:\n import sys\n print(f\" Python version: {sys.version}\")\n print(f\" Python executable: {sys.executable}\")\n\n model_name, model, processor, device_str, device, dtype = _load_colvision(MODEL)\n print(f\"✓ Using model={model_name}, device={device_str}, dtype={dtype}\")\n\n # Memory check after loading model\n try:\n import psutil\n import torch\n process = psutil.Process(os.getpid())\n mem_info = process.memory_info()\n print(f\" Memory usage after loading model: {mem_info.rss / 1024 / 1024 / 1024:.2f} GB\")\n if torch.cuda.is_available():\n print(f\" GPU memory allocated: {torch.cuda.memory_allocated() / 1024**3:.2f} GB\")\n print(f\" GPU memory reserved: {torch.cuda.memory_reserved() / 1024**3:.2f} GB\")\n except ImportError:\n pass\nexcept Exception as e:\n print(f\"✗ Error loading model: {type(e).__name__}: {e}\")\n import traceback\n traceback.print_exc()\n raise\n\n\n# %%\n\n# %%\n# Step 4: Build index if needed\nif need_to_build_index:\n print(\"Step 4: Building index...\")\n print(f\" Number of images: {len(images)}\")\n print(f\" Number of filepaths: {len(filepaths)}\")\n\n try:\n print(\" Embedding images...\")\n doc_vecs = _embed_images(model, processor, images)\n print(f\" Embedded {len(doc_vecs)} documents\")\n print(f\" First doc vec shape: {doc_vecs[0].shape if len(doc_vecs) > 0 else 'N/A'}\")\n except Exception as e:\n print(f\"Error embedding images: {type(e).__name__}: {e}\")\n import traceback\n traceback.print_exc()\n raise\n\n if USE_FAST_PLAID:\n # Build Fast-Plaid index\n print(\" Building Fast-Plaid index...\")\n try:\n fast_plaid_index, build_secs = _build_fast_plaid_index(\n FAST_PLAID_INDEX_PATH, doc_vecs, filepaths, images\n )\n from pathlib import Path\n print(f\"✓ Fast-Plaid index built in {build_secs:.3f}s\")\n print(f\"✓ Index saved to: {FAST_PLAID_INDEX_PATH}\")\n print(f\"✓ Images saved to: {Path(FAST_PLAID_INDEX_PATH) / 'images'}\")\n except Exception as e:\n print(f\"Error building Fast-Plaid index: {type(e).__name__}: {e}\")\n import traceback\n traceback.print_exc()\n raise\n finally:\n # Clear memory\n print(\" Clearing memory...\")\n del images, filepaths, doc_vecs\n else:\n # Build original LEANN index\n try:\n retriever = _build_index(INDEX_PATH, doc_vecs, filepaths, images)\n print(f\"✓ Index built and images saved to: {retriever._images_dir_path()}\")\n except Exception as e:\n print(f\"Error building LEANN index: {type(e).__name__}: {e}\")\n import traceback\n traceback.print_exc()\n raise\n finally:\n # Clear memory\n print(\" Clearing memory...\")\n del images, filepaths, doc_vecs\n\n# Note: Images are now stored separately, retriever/fast_plaid_index will reference them\n\n\n# %%\n# Step 5: Embed query and search\n_t0 = time.perf_counter()\nq_vec = _embed_queries(model, processor, [QUERY])[0]\nquery_embed_secs = time.perf_counter() - _t0\n\nprint(f\"[Search] Method: {SEARCH_METHOD}\")\nprint(f\"[Timing] Query embedding: {query_embed_secs:.3f}s\")\n\n# Run the selected search method and time it\nif USE_FAST_PLAID:\n # Fast-Plaid search\n if fast_plaid_index is None:\n fast_plaid_index = _load_fast_plaid_index_if_exists(FAST_PLAID_INDEX_PATH)\n if fast_plaid_index is None:\n raise RuntimeError(f\"Fast-Plaid index not found at {FAST_PLAID_INDEX_PATH}\")\n\n results, search_secs = _search_fast_plaid(fast_plaid_index, q_vec, TOPK)\n print(f\"[Timing] Fast-Plaid Search: {search_secs:.3f}s\")\nelse:\n # Original LEANN search\n query_np = q_vec.float().numpy()\n\n if retriever is None:\n raise RuntimeError(\"Retriever not initialized\")\n retriever_any = cast(Any, retriever)\n\n if SEARCH_METHOD == \"ann\":\n results = retriever_any.search(query_np, topk=TOPK, first_stage_k=FIRST_STAGE_K)\n search_secs = time.perf_counter() - _t0\n print(f\"[Timing] Search (ANN): {search_secs:.3f}s (first_stage_k={FIRST_STAGE_K})\")\n elif SEARCH_METHOD == \"exact\":\n results = retriever_any.search_exact(query_np, topk=TOPK, first_stage_k=FIRST_STAGE_K)\n search_secs = time.perf_counter() - _t0\n print(f\"[Timing] Search (Exact rerank): {search_secs:.3f}s (first_stage_k={FIRST_STAGE_K})\")\n elif SEARCH_METHOD == \"exact-all\":\n results = retriever_any.search_exact_all(query_np, topk=TOPK)\n search_secs = time.perf_counter() - _t0\n print(f\"[Timing] Search (Exact all): {search_secs:.3f}s\")\n else:\n results = []\nif not results:\n print(\"No results found.\")\nelse:\n print(f'Top {len(results)} results for query: \"{QUERY}\"')\n print(\"\\n[DEBUG] Retrieval details:\")\n top_images: list[Image.Image] = []\n image_hashes = {} # Track image hashes to detect duplicates\n\n for rank, (score, doc_id) in enumerate(results, start=1):\n # Retrieve image and metadata based on index type\n if USE_FAST_PLAID:\n # Fast-Plaid: load image and get metadata\n image = _get_fast_plaid_image(FAST_PLAID_INDEX_PATH, doc_id)\n if image is None:\n print(f\"Warning: Could not find image for doc_id {doc_id}\")\n continue\n\n metadata = _get_fast_plaid_metadata(FAST_PLAID_INDEX_PATH, doc_id)\n path = metadata.get(\"filepath\", f\"doc_{doc_id}\") if metadata else f\"doc_{doc_id}\"\n top_images.append(image)\n else:\n # Original LEANN: retrieve from retriever\n if retriever is None:\n raise RuntimeError(\"Retriever not initialized\")\n retriever_any = cast(Any, retriever)\n image = retriever_any.get_image(doc_id)\n if image is None:\n print(f\"Warning: Could not retrieve image for doc_id {doc_id}\")\n continue\n\n metadata = retriever_any.get_metadata(doc_id)\n path = metadata.get(\"filepath\", \"unknown\") if metadata else \"unknown\"\n top_images.append(image)\n\n # Calculate image hash to detect duplicates\n import hashlib\n import io\n # Convert image to bytes for hashing\n img_bytes = io.BytesIO()\n image.save(img_bytes, format='PNG')\n image_bytes = img_bytes.getvalue()\n image_hash = hashlib.md5(image_bytes).hexdigest()[:8]\n\n # Check if this image was already seen\n duplicate_info = \"\"\n if image_hash in image_hashes:\n duplicate_info = f\" [DUPLICATE of rank {image_hashes[image_hash]}]\"\n else:\n image_hashes[image_hash] = rank\n\n # Print detailed information\n print(f\"{rank}) doc_id={doc_id}, MaxSim={score:.4f}, Page={path}, ImageHash={image_hash}{duplicate_info}\")\n if metadata:\n print(f\" Metadata: {metadata}\")\n\n if SAVE_TOP_IMAGE:\n from pathlib import Path as _Path\n\n base = _Path(SAVE_TOP_IMAGE)\n base.parent.mkdir(parents=True, exist_ok=True)\n for rank, img in enumerate(top_images[:TOPK], start=1):\n if base.suffix:\n out_path = base.parent / f\"{base.stem}_rank{rank}{base.suffix}\"\n else:\n out_path = base / f\"retrieved_page_rank{rank}.png\"\n img.save(str(out_path))\n # Print the retrieval score (document-level MaxSim) alongside the saved path\n try:\n score, _doc_id = results[rank - 1]\n print(f\"Saved retrieved page (rank {rank}) [MaxSim={score:.4f}] to: {out_path}\")\n except Exception:\n print(f\"Saved retrieved page (rank {rank}) to: {out_path}\")\n\n\n# %%\n# Step 6: Similarity maps for top-K results\nif results and SIMILARITY_MAP:\n token_idx = None if SIM_TOKEN_IDX < 0 else int(SIM_TOKEN_IDX)\n from pathlib import Path as _Path\n\n output_base = _Path(SIM_OUTPUT) if SIM_OUTPUT else None\n for rank, img in enumerate(top_images[:TOPK], start=1):\n if output_base:\n if output_base.suffix:\n out_dir = output_base.parent\n out_name = f\"{output_base.stem}_rank{rank}{output_base.suffix}\"\n out_path = str(out_dir / out_name)\n else:\n out_dir = output_base\n out_dir.mkdir(parents=True, exist_ok=True)\n out_path = str(out_dir / f\"similarity_map_rank{rank}.png\")\n else:\n out_path = None\n chosen_idx, max_sim = _generate_similarity_map(\n model=model,\n processor=processor,\n image=img,\n query=QUERY,\n token_idx=token_idx,\n output_path=out_path,\n )\n if out_path:\n print(\n f\"Saved similarity map for rank {rank}, token #{chosen_idx} (max={max_sim:.2f}) to: {out_path}\"\n )\n else:\n print(\n f\"Computed similarity map for rank {rank}, token #{chosen_idx} (max={max_sim:.2f})\"\n )\n\n\n# %%\n# Step 7: Optional answer generation\nif results and ANSWER:\n qwen = QwenVL(device=device_str)\n _t0 = time.perf_counter()\n response = qwen.answer(QUERY, top_images[:TOPK], max_new_tokens=MAX_NEW_TOKENS)\n gen_secs = time.perf_counter() - _t0\n print(f\"[Timing] Generation: {gen_secs:.3f}s\")\n print(\"\\nAnswer:\")\n print(response)\n" }, { "path": "apps/multimodal/vision-based-pdf-multi-vector/vidore_v1_benchmark.py", "content": "#!/usr/bin/env python3\n\"\"\"\nModular script to reproduce NDCG results for ViDoRe v1 benchmark.\n\nThis script uses the interface from leann_multi_vector.py to:\n1. Download ViDoRe v1 datasets\n2. Build indexes (LEANN or Fast-Plaid)\n3. Perform retrieval\n4. Evaluate using NDCG metrics\n\nUsage:\n # Evaluate all ViDoRe v1 tasks\n python vidore_v1_benchmark.py --model colqwen2 --tasks all\n\n # Evaluate specific task\n python vidore_v1_benchmark.py --model colqwen2 --task VidoreArxivQARetrieval\n\n # Use Fast-Plaid index\n python vidore_v1_benchmark.py --model colqwen2 --use-fast-plaid --fast-plaid-index-path ./indexes/vidore_fastplaid\n\n # Rebuild index\n python vidore_v1_benchmark.py --model colqwen2 --rebuild-index\n\"\"\"\n\nimport argparse\nimport json\nimport os\nfrom typing import Any, Optional, cast\n\nfrom datasets import Dataset, load_dataset\nfrom leann_multi_vector import (\n ViDoReBenchmarkEvaluator,\n _ensure_repo_paths_importable,\n)\n\n_ensure_repo_paths_importable(__file__)\n\n# ViDoRe v1 task configurations\n# Prompts match MTEB task metadata prompts\nVIDORE_V1_TASKS = {\n \"VidoreArxivQARetrieval\": {\n \"dataset_path\": \"vidore/arxivqa_test_subsampled_beir\",\n \"revision\": \"7d94d570960eac2408d3baa7a33f9de4822ae3e4\",\n \"prompt\": {\"query\": \"Find a screenshot that relevant to the user's question.\"},\n },\n \"VidoreDocVQARetrieval\": {\n \"dataset_path\": \"vidore/docvqa_test_subsampled_beir\",\n \"revision\": \"162ba2fc1a8437eda8b6c37b240bc1c0f0deb092\",\n \"prompt\": {\"query\": \"Find a screenshot that relevant to the user's question.\"},\n },\n \"VidoreInfoVQARetrieval\": {\n \"dataset_path\": \"vidore/infovqa_test_subsampled_beir\",\n \"revision\": \"b802cc5fd6c605df2d673a963667d74881d2c9a4\",\n \"prompt\": {\"query\": \"Find a screenshot that relevant to the user's question.\"},\n },\n \"VidoreTabfquadRetrieval\": {\n \"dataset_path\": \"vidore/tabfquad_test_subsampled_beir\",\n \"revision\": \"61a2224bcd29b7b261a4892ff4c8bea353527a31\",\n \"prompt\": {\"query\": \"Find a screenshot that relevant to the user's question.\"},\n },\n \"VidoreTatdqaRetrieval\": {\n \"dataset_path\": \"vidore/tatdqa_test_beir\",\n \"revision\": \"5feb5630fdff4d8d189ffedb2dba56862fdd45c0\",\n \"prompt\": {\"query\": \"Find a screenshot that relevant to the user's question.\"},\n },\n \"VidoreShiftProjectRetrieval\": {\n \"dataset_path\": \"vidore/shiftproject_test_beir\",\n \"revision\": \"84a382e05c4473fed9cff2bbae95fe2379416117\",\n \"prompt\": {\"query\": \"Find a screenshot that relevant to the user's question.\"},\n },\n \"VidoreSyntheticDocQAAIRetrieval\": {\n \"dataset_path\": \"vidore/syntheticDocQA_artificial_intelligence_test_beir\",\n \"revision\": \"2d9ebea5a1c6e9ef4a3b902a612f605dca11261c\",\n \"prompt\": {\"query\": \"Find a screenshot that relevant to the user's question.\"},\n },\n \"VidoreSyntheticDocQAEnergyRetrieval\": {\n \"dataset_path\": \"vidore/syntheticDocQA_energy_test_beir\",\n \"revision\": \"9935aadbad5c8deec30910489db1b2c7133ae7a7\",\n \"prompt\": {\"query\": \"Find a screenshot that relevant to the user's question.\"},\n },\n \"VidoreSyntheticDocQAGovernmentReportsRetrieval\": {\n \"dataset_path\": \"vidore/syntheticDocQA_government_reports_test_beir\",\n \"revision\": \"b4909afa930f81282fd20601e860668073ad02aa\",\n \"prompt\": {\"query\": \"Find a screenshot that relevant to the user's question.\"},\n },\n \"VidoreSyntheticDocQAHealthcareIndustryRetrieval\": {\n \"dataset_path\": \"vidore/syntheticDocQA_healthcare_industry_test_beir\",\n \"revision\": \"f9e25d5b6e13e1ad9f5c3cce202565031b3ab164\",\n \"prompt\": {\"query\": \"Find a screenshot that relevant to the user's question.\"},\n },\n}\n\n# Task name aliases (short names -> full names)\nTASK_ALIASES = {\n \"arxivqa\": \"VidoreArxivQARetrieval\",\n \"docvqa\": \"VidoreDocVQARetrieval\",\n \"infovqa\": \"VidoreInfoVQARetrieval\",\n \"tabfquad\": \"VidoreTabfquadRetrieval\",\n \"tatdqa\": \"VidoreTatdqaRetrieval\",\n \"shiftproject\": \"VidoreShiftProjectRetrieval\",\n \"syntheticdocqa_ai\": \"VidoreSyntheticDocQAAIRetrieval\",\n \"syntheticdocqa_energy\": \"VidoreSyntheticDocQAEnergyRetrieval\",\n \"syntheticdocqa_government\": \"VidoreSyntheticDocQAGovernmentReportsRetrieval\",\n \"syntheticdocqa_healthcare\": \"VidoreSyntheticDocQAHealthcareIndustryRetrieval\",\n}\n\n\ndef normalize_task_name(task_name: str) -> str:\n \"\"\"Normalize task name (handle aliases).\"\"\"\n task_name_lower = task_name.lower()\n if task_name in VIDORE_V1_TASKS:\n return task_name\n if task_name_lower in TASK_ALIASES:\n return TASK_ALIASES[task_name_lower]\n # Try partial match\n for alias, full_name in TASK_ALIASES.items():\n if alias in task_name_lower or task_name_lower in alias:\n return full_name\n return task_name\n\n\ndef get_safe_model_name(model_name: str) -> str:\n \"\"\"Get a safe model name for use in file paths.\"\"\"\n import hashlib\n import os\n\n # If it's a path, use basename or hash\n if os.path.exists(model_name) and os.path.isdir(model_name):\n # Use basename if it's reasonable, otherwise use hash\n basename = os.path.basename(model_name.rstrip(\"/\"))\n if basename and len(basename) < 100 and not basename.startswith(\".\"):\n return basename\n # Use hash for very long or problematic paths\n return hashlib.md5(model_name.encode()).hexdigest()[:16]\n # For HuggingFace model names, replace / with _\n return model_name.replace(\"/\", \"_\").replace(\":\", \"_\")\n\n\ndef load_vidore_v1_data(\n dataset_path: str,\n revision: Optional[str] = None,\n split: str = \"test\",\n):\n \"\"\"\n Load ViDoRe v1 dataset.\n\n Returns:\n corpus: dict mapping corpus_id to PIL Image\n queries: dict mapping query_id to query text\n qrels: dict mapping query_id to dict of {corpus_id: relevance_score}\n \"\"\"\n print(f\"Loading dataset: {dataset_path} (split={split})\")\n\n # Load queries - cast to Dataset since we know split returns Dataset not DatasetDict\n query_ds = cast(Dataset, load_dataset(dataset_path, \"queries\", split=split, revision=revision))\n\n queries: dict[str, str] = {}\n for row in query_ds:\n row_dict = cast(dict[str, Any], row)\n query_id = f\"query-{split}-{row_dict['query-id']}\"\n queries[query_id] = row_dict[\"query\"]\n\n # Load corpus (images) - cast to Dataset\n corpus_ds = cast(Dataset, load_dataset(dataset_path, \"corpus\", split=split, revision=revision))\n\n corpus: dict[str, Any] = {}\n for row in corpus_ds:\n row_dict = cast(dict[str, Any], row)\n corpus_id = f\"corpus-{split}-{row_dict['corpus-id']}\"\n # Extract image from the dataset row\n if \"image\" in row_dict:\n corpus[corpus_id] = row_dict[\"image\"]\n elif \"page_image\" in row_dict:\n corpus[corpus_id] = row_dict[\"page_image\"]\n else:\n raise ValueError(\n f\"No image field found in corpus. Available fields: {list(row_dict.keys())}\"\n )\n\n # Load qrels (relevance judgments) - cast to Dataset\n qrels_ds = cast(Dataset, load_dataset(dataset_path, \"qrels\", split=split, revision=revision))\n\n qrels: dict[str, dict[str, int]] = {}\n for row in qrels_ds:\n row_dict = cast(dict[str, Any], row)\n query_id = f\"query-{split}-{row_dict['query-id']}\"\n corpus_id = f\"corpus-{split}-{row_dict['corpus-id']}\"\n if query_id not in qrels:\n qrels[query_id] = {}\n qrels[query_id][corpus_id] = int(row_dict[\"score\"])\n\n print(\n f\"Loaded {len(queries)} queries, {len(corpus)} corpus items, {len(qrels)} query-relevance mappings\"\n )\n\n # Filter qrels to only include queries that exist\n qrels = {qid: rel_docs for qid, rel_docs in qrels.items() if qid in queries}\n\n # Filter out queries without any relevant documents (matching MTEB behavior)\n # This is important for correct NDCG calculation\n qrels_filtered = {qid: rel_docs for qid, rel_docs in qrels.items() if len(rel_docs) > 0}\n queries_filtered = {\n qid: query_text for qid, query_text in queries.items() if qid in qrels_filtered\n }\n\n print(\n f\"After filtering queries without positives: {len(queries_filtered)} queries, {len(qrels_filtered)} query-relevance mappings\"\n )\n\n return corpus, queries_filtered, qrels_filtered\n\n\ndef evaluate_task(\n task_name: str,\n model_name: str,\n index_path: str,\n use_fast_plaid: bool = False,\n fast_plaid_index_path: Optional[str] = None,\n rebuild_index: bool = False,\n top_k: int = 1000,\n first_stage_k: int = 500,\n k_values: Optional[list[int]] = None,\n output_dir: Optional[str] = None,\n):\n \"\"\"\n Evaluate a single ViDoRe v1 task.\n \"\"\"\n print(f\"\\n{'=' * 80}\")\n print(f\"Evaluating task: {task_name}\")\n print(f\"{'=' * 80}\")\n\n # Normalize task name (handle aliases)\n task_name = normalize_task_name(task_name)\n\n # Get task config\n if task_name not in VIDORE_V1_TASKS:\n raise ValueError(f\"Unknown task: {task_name}. Available: {list(VIDORE_V1_TASKS.keys())}\")\n\n task_config = VIDORE_V1_TASKS[task_name]\n dataset_path = str(task_config[\"dataset_path\"])\n revision = str(task_config[\"revision\"])\n\n # Load data\n corpus, queries, qrels = load_vidore_v1_data(\n dataset_path=dataset_path,\n revision=revision,\n split=\"test\",\n )\n\n # Initialize k_values if not provided\n if k_values is None:\n k_values = [1, 3, 5, 10, 20, 100, 1000]\n\n # Check if we have any queries\n if len(queries) == 0:\n print(f\"\\nWarning: No queries found for task {task_name}. Skipping evaluation.\")\n # Return zero scores\n scores = {}\n for k in k_values:\n scores[f\"ndcg_at_{k}\"] = 0.0\n scores[f\"map_at_{k}\"] = 0.0\n scores[f\"recall_at_{k}\"] = 0.0\n scores[f\"precision_at_{k}\"] = 0.0\n scores[f\"mrr_at_{k}\"] = 0.0\n return scores\n\n # Initialize evaluator\n evaluator = ViDoReBenchmarkEvaluator(\n model_name=model_name,\n use_fast_plaid=use_fast_plaid,\n top_k=top_k,\n first_stage_k=first_stage_k,\n k_values=k_values,\n )\n\n # Build or load index\n # Use safe model name for index path (different models need different indexes)\n safe_model_name = get_safe_model_name(model_name)\n index_path_full = index_path if not use_fast_plaid else fast_plaid_index_path\n if index_path_full is None:\n index_path_full = f\"./indexes/{task_name}_{safe_model_name}\"\n if use_fast_plaid:\n index_path_full = f\"./indexes/{task_name}_{safe_model_name}_fastplaid\"\n\n index_or_retriever, corpus_ids_ordered = evaluator.build_index_from_corpus(\n corpus=corpus,\n index_path=index_path_full,\n rebuild=rebuild_index,\n )\n\n # Search queries\n task_prompt = cast(Optional[dict[str, str]], task_config.get(\"prompt\"))\n results = evaluator.search_queries(\n queries=queries,\n corpus_ids=corpus_ids_ordered,\n index_or_retriever=index_or_retriever,\n fast_plaid_index_path=fast_plaid_index_path,\n task_prompt=task_prompt,\n )\n\n # Evaluate\n scores = evaluator.evaluate_results(results, qrels, k_values=k_values)\n\n # Print results\n print(f\"\\n{'=' * 80}\")\n print(f\"Results for {task_name}:\")\n print(f\"{'=' * 80}\")\n for metric, value in scores.items():\n if isinstance(value, (int, float)):\n print(f\" {metric}: {value:.5f}\")\n\n # Save results\n if output_dir:\n os.makedirs(output_dir, exist_ok=True)\n results_file = os.path.join(output_dir, f\"{task_name}_results.json\")\n scores_file = os.path.join(output_dir, f\"{task_name}_scores.json\")\n\n with open(results_file, \"w\") as f:\n json.dump(results, f, indent=2)\n print(f\"\\nSaved results to: {results_file}\")\n\n with open(scores_file, \"w\") as f:\n json.dump(scores, f, indent=2)\n print(f\"Saved scores to: {scores_file}\")\n\n return scores\n\n\ndef main():\n parser = argparse.ArgumentParser(\n description=\"Evaluate ViDoRe v1 benchmark using LEANN/Fast-Plaid indexing\"\n )\n parser.add_argument(\n \"--model\",\n type=str,\n default=\"colqwen2\",\n help=\"Model to use: 'colqwen2', 'colpali', or path to a model directory (supports LoRA adapters)\",\n )\n parser.add_argument(\n \"--task\",\n type=str,\n default=None,\n help=\"Specific task to evaluate (or 'all' for all tasks)\",\n )\n parser.add_argument(\n \"--tasks\",\n type=str,\n default=\"all\",\n help=\"Tasks to evaluate: 'all' or comma-separated list\",\n )\n parser.add_argument(\n \"--index-path\",\n type=str,\n default=None,\n help=\"Path to LEANN index (auto-generated if not provided)\",\n )\n parser.add_argument(\n \"--use-fast-plaid\",\n action=\"store_true\",\n help=\"Use Fast-Plaid instead of LEANN\",\n )\n parser.add_argument(\n \"--fast-plaid-index-path\",\n type=str,\n default=None,\n help=\"Path to Fast-Plaid index (auto-generated if not provided)\",\n )\n parser.add_argument(\n \"--rebuild-index\",\n action=\"store_true\",\n help=\"Rebuild index even if it exists\",\n )\n parser.add_argument(\n \"--top-k\",\n type=int,\n default=1000,\n help=\"Top-k results to retrieve (MTEB default is max(k_values)=1000)\",\n )\n parser.add_argument(\n \"--first-stage-k\",\n type=int,\n default=500,\n help=\"First stage k for LEANN search\",\n )\n parser.add_argument(\n \"--k-values\",\n type=str,\n default=\"1,3,5,10,20,100,1000\",\n help=\"Comma-separated k values for evaluation (e.g., '1,3,5,10,100')\",\n )\n parser.add_argument(\n \"--output-dir\",\n type=str,\n default=\"./vidore_v1_results\",\n help=\"Output directory for results\",\n )\n\n args = parser.parse_args()\n\n # Parse k_values\n k_values = [int(k.strip()) for k in args.k_values.split(\",\")]\n\n # Determine tasks to evaluate\n if args.task:\n tasks_to_eval = [normalize_task_name(args.task)]\n elif args.tasks.lower() == \"all\":\n tasks_to_eval = list(VIDORE_V1_TASKS.keys())\n else:\n tasks_to_eval = [normalize_task_name(t.strip()) for t in args.tasks.split(\",\")]\n\n print(f\"Tasks to evaluate: {tasks_to_eval}\")\n\n # Evaluate each task\n all_scores = {}\n for task_name in tasks_to_eval:\n try:\n scores = evaluate_task(\n task_name=task_name,\n model_name=args.model,\n index_path=args.index_path,\n use_fast_plaid=args.use_fast_plaid,\n fast_plaid_index_path=args.fast_plaid_index_path,\n rebuild_index=args.rebuild_index,\n top_k=args.top_k,\n first_stage_k=args.first_stage_k,\n k_values=k_values,\n output_dir=args.output_dir,\n )\n all_scores[task_name] = scores\n except Exception as e:\n print(f\"\\nError evaluating {task_name}: {e}\")\n import traceback\n\n traceback.print_exc()\n continue\n\n # Print summary\n if all_scores:\n print(f\"\\n{'=' * 80}\")\n print(\"SUMMARY\")\n print(f\"{'=' * 80}\")\n for task_name, scores in all_scores.items():\n print(f\"\\n{task_name}:\")\n # Print main metrics\n for metric in [\"ndcg_at_5\", \"ndcg_at_10\", \"ndcg_at_100\", \"map_at_10\", \"recall_at_10\"]:\n if metric in scores:\n print(f\" {metric}: {scores[metric]:.5f}\")\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "apps/multimodal/vision-based-pdf-multi-vector/vidore_v2_benchmark.py", "content": "#!/usr/bin/env python3\n\"\"\"\nModular script to reproduce NDCG results for ViDoRe v2 benchmark.\n\nThis script uses the interface from leann_multi_vector.py to:\n1. Download ViDoRe v2 datasets\n2. Build indexes (LEANN or Fast-Plaid)\n3. Perform retrieval\n4. Evaluate using NDCG metrics\n\nUsage:\n # Evaluate all ViDoRe v2 tasks\n python vidore_v2_benchmark.py --model colqwen2 --tasks all\n\n # Evaluate specific task\n python vidore_v2_benchmark.py --model colqwen2 --task Vidore2ESGReportsRetrieval\n\n # Use Fast-Plaid index\n python vidore_v2_benchmark.py --model colqwen2 --use-fast-plaid --fast-plaid-index-path ./indexes/vidore_fastplaid\n\n # Rebuild index\n python vidore_v2_benchmark.py --model colqwen2 --rebuild-index\n\"\"\"\n\nimport argparse\nimport json\nimport os\nfrom typing import Any, Optional, cast\n\nfrom datasets import Dataset, load_dataset\nfrom leann_multi_vector import (\n ViDoReBenchmarkEvaluator,\n _ensure_repo_paths_importable,\n)\n\n_ensure_repo_paths_importable(__file__)\n\n# Language name to dataset language field value mapping\n# Dataset uses ISO 639-3 + ISO 15924 format (e.g., \"eng-Latn\")\nLANGUAGE_MAPPING = {\n \"english\": \"eng-Latn\",\n \"french\": \"fra-Latn\",\n \"spanish\": \"spa-Latn\",\n \"german\": \"deu-Latn\",\n}\n\n# ViDoRe v2 task configurations\n# Prompts match MTEB task metadata prompts\nVIDORE_V2_TASKS = {\n \"Vidore2ESGReportsRetrieval\": {\n \"dataset_path\": \"vidore/esg_reports_v2\",\n \"revision\": \"0542c0d03da0ec1c8cbc517c8d78e7e95c75d3d3\",\n \"languages\": [\"french\", \"spanish\", \"english\", \"german\"],\n \"prompt\": {\"query\": \"Find a screenshot that relevant to the user's question.\"},\n },\n \"Vidore2EconomicsReportsRetrieval\": {\n \"dataset_path\": \"vidore/economics_reports_v2\",\n \"revision\": \"b3e3a04b07fbbaffe79be49dabf92f691fbca252\",\n \"languages\": [\"french\", \"spanish\", \"english\", \"german\"],\n \"prompt\": {\"query\": \"Find a screenshot that relevant to the user's question.\"},\n },\n \"Vidore2BioMedicalLecturesRetrieval\": {\n \"dataset_path\": \"vidore/biomedical_lectures_v2\",\n \"revision\": \"a29202f0da409034d651614d87cd8938d254e2ea\",\n \"languages\": [\"french\", \"spanish\", \"english\", \"german\"],\n \"prompt\": {\"query\": \"Find a screenshot that relevant to the user's question.\"},\n },\n \"Vidore2ESGReportsHLRetrieval\": {\n \"dataset_path\": \"vidore/esg_reports_human_labeled_v2\",\n \"revision\": \"6d467dedb09a75144ede1421747e47cf036857dd\",\n # Note: This dataset doesn't have language filtering - all queries are English\n \"languages\": None, # No language filtering needed\n \"prompt\": {\"query\": \"Find a screenshot that relevant to the user's question.\"},\n },\n}\n\n\ndef load_vidore_v2_data(\n dataset_path: str,\n revision: Optional[str] = None,\n split: str = \"test\",\n language: Optional[str] = None,\n):\n \"\"\"\n Load ViDoRe v2 dataset.\n\n Returns:\n corpus: dict mapping corpus_id to PIL Image\n queries: dict mapping query_id to query text\n qrels: dict mapping query_id to dict of {corpus_id: relevance_score}\n \"\"\"\n print(f\"Loading dataset: {dataset_path} (split={split}, language={language})\")\n\n # Load queries - cast to Dataset since we know split returns Dataset not DatasetDict\n query_ds = cast(Dataset, load_dataset(dataset_path, \"queries\", split=split, revision=revision))\n\n # Check if dataset has language field before filtering\n has_language_field = len(query_ds) > 0 and \"language\" in query_ds.column_names\n\n if language and has_language_field:\n # Map language name to dataset language field value (e.g., \"english\" -> \"eng-Latn\")\n dataset_language = LANGUAGE_MAPPING.get(language, language)\n query_ds_filtered = query_ds.filter(lambda x: x.get(\"language\") == dataset_language)\n # Check if filtering resulted in empty dataset\n if len(query_ds_filtered) == 0:\n print(\n f\"Warning: No queries found after filtering by language '{language}' (mapped to '{dataset_language}').\"\n )\n # Try with original language value (dataset might use simple names like 'english')\n print(f\"Trying with original language value '{language}'...\")\n query_ds_filtered = query_ds.filter(lambda x: x.get(\"language\") == language)\n if len(query_ds_filtered) == 0:\n # Try to get a sample to see actual language values\n try:\n sample_ds = cast(\n Dataset,\n load_dataset(dataset_path, \"queries\", split=split, revision=revision),\n )\n if len(sample_ds) > 0 and \"language\" in sample_ds.column_names:\n sample_langs = set(sample_ds[\"language\"])\n print(f\"Available language values in dataset: {sample_langs}\")\n except Exception:\n pass\n else:\n print(\n f\"Found {len(query_ds_filtered)} queries using original language value '{language}'\"\n )\n query_ds = query_ds_filtered\n\n queries: dict[str, str] = {}\n for row in query_ds:\n row_dict = cast(dict[str, Any], row)\n query_id = f\"query-{split}-{row_dict['query-id']}\"\n queries[query_id] = row_dict[\"query\"]\n\n # Load corpus (images) - cast to Dataset\n corpus_ds = cast(Dataset, load_dataset(dataset_path, \"corpus\", split=split, revision=revision))\n\n corpus: dict[str, Any] = {}\n for row in corpus_ds:\n row_dict = cast(dict[str, Any], row)\n corpus_id = f\"corpus-{split}-{row_dict['corpus-id']}\"\n # Extract image from the dataset row\n if \"image\" in row_dict:\n corpus[corpus_id] = row_dict[\"image\"]\n elif \"page_image\" in row_dict:\n corpus[corpus_id] = row_dict[\"page_image\"]\n else:\n raise ValueError(\n f\"No image field found in corpus. Available fields: {list(row_dict.keys())}\"\n )\n\n # Load qrels (relevance judgments) - cast to Dataset\n qrels_ds = cast(Dataset, load_dataset(dataset_path, \"qrels\", split=split, revision=revision))\n\n qrels: dict[str, dict[str, int]] = {}\n for row in qrels_ds:\n row_dict = cast(dict[str, Any], row)\n query_id = f\"query-{split}-{row_dict['query-id']}\"\n corpus_id = f\"corpus-{split}-{row_dict['corpus-id']}\"\n if query_id not in qrels:\n qrels[query_id] = {}\n qrels[query_id][corpus_id] = int(row_dict[\"score\"])\n\n print(\n f\"Loaded {len(queries)} queries, {len(corpus)} corpus items, {len(qrels)} query-relevance mappings\"\n )\n\n # Filter qrels to only include queries that exist\n qrels = {qid: rel_docs for qid, rel_docs in qrels.items() if qid in queries}\n\n # Filter out queries without any relevant documents (matching MTEB behavior)\n # This is important for correct NDCG calculation\n qrels_filtered = {qid: rel_docs for qid, rel_docs in qrels.items() if len(rel_docs) > 0}\n queries_filtered = {\n qid: query_text for qid, query_text in queries.items() if qid in qrels_filtered\n }\n\n print(\n f\"After filtering queries without positives: {len(queries_filtered)} queries, {len(qrels_filtered)} query-relevance mappings\"\n )\n\n return corpus, queries_filtered, qrels_filtered\n\n\ndef evaluate_task(\n task_name: str,\n model_name: str,\n index_path: str,\n use_fast_plaid: bool = False,\n fast_plaid_index_path: Optional[str] = None,\n language: Optional[str] = None,\n rebuild_index: bool = False,\n top_k: int = 100,\n first_stage_k: int = 500,\n k_values: Optional[list[int]] = None,\n output_dir: Optional[str] = None,\n):\n \"\"\"\n Evaluate a single ViDoRe v2 task.\n \"\"\"\n print(f\"\\n{'=' * 80}\")\n print(f\"Evaluating task: {task_name}\")\n print(f\"{'=' * 80}\")\n\n # Get task config\n if task_name not in VIDORE_V2_TASKS:\n raise ValueError(f\"Unknown task: {task_name}. Available: {list(VIDORE_V2_TASKS.keys())}\")\n\n task_config = VIDORE_V2_TASKS[task_name]\n dataset_path = str(task_config[\"dataset_path\"])\n revision = str(task_config[\"revision\"])\n\n # Determine language\n if language is None:\n # Use first language if multiple available\n languages = cast(Optional[list[str]], task_config.get(\"languages\"))\n if languages is None:\n # Task doesn't support language filtering (e.g., Vidore2ESGReportsHLRetrieval)\n language = None\n elif len(languages) == 1:\n language = languages[0]\n else:\n language = None\n\n # Initialize k_values if not provided\n if k_values is None:\n k_values = [1, 3, 5, 10, 100]\n\n # Load data\n corpus, queries, qrels = load_vidore_v2_data(\n dataset_path=dataset_path,\n revision=revision,\n split=\"test\",\n language=language,\n )\n\n # Check if we have any queries\n if len(queries) == 0:\n print(\n f\"\\nWarning: No queries found for task {task_name} with language {language}. Skipping evaluation.\"\n )\n # Return zero scores\n scores = {}\n for k in k_values:\n scores[f\"ndcg_at_{k}\"] = 0.0\n scores[f\"map_at_{k}\"] = 0.0\n scores[f\"recall_at_{k}\"] = 0.0\n scores[f\"precision_at_{k}\"] = 0.0\n scores[f\"mrr_at_{k}\"] = 0.0\n return scores\n\n # Initialize evaluator\n evaluator = ViDoReBenchmarkEvaluator(\n model_name=model_name,\n use_fast_plaid=use_fast_plaid,\n top_k=top_k,\n first_stage_k=first_stage_k,\n k_values=k_values,\n )\n\n # Build or load index\n index_path_full = index_path if not use_fast_plaid else fast_plaid_index_path\n if index_path_full is None:\n index_path_full = f\"./indexes/{task_name}_{model_name}\"\n if use_fast_plaid:\n index_path_full = f\"./indexes/{task_name}_{model_name}_fastplaid\"\n\n index_or_retriever, corpus_ids_ordered = evaluator.build_index_from_corpus(\n corpus=corpus,\n index_path=index_path_full,\n rebuild=rebuild_index,\n )\n\n # Search queries\n task_prompt = cast(Optional[dict[str, str]], task_config.get(\"prompt\"))\n results = evaluator.search_queries(\n queries=queries,\n corpus_ids=corpus_ids_ordered,\n index_or_retriever=index_or_retriever,\n fast_plaid_index_path=fast_plaid_index_path,\n task_prompt=task_prompt,\n )\n\n # Evaluate\n scores = evaluator.evaluate_results(results, qrels, k_values=k_values)\n\n # Print results\n print(f\"\\n{'=' * 80}\")\n print(f\"Results for {task_name}:\")\n print(f\"{'=' * 80}\")\n for metric, value in scores.items():\n if isinstance(value, (int, float)):\n print(f\" {metric}: {value:.5f}\")\n\n # Save results\n if output_dir:\n os.makedirs(output_dir, exist_ok=True)\n results_file = os.path.join(output_dir, f\"{task_name}_results.json\")\n scores_file = os.path.join(output_dir, f\"{task_name}_scores.json\")\n\n with open(results_file, \"w\") as f:\n json.dump(results, f, indent=2)\n print(f\"\\nSaved results to: {results_file}\")\n\n with open(scores_file, \"w\") as f:\n json.dump(scores, f, indent=2)\n print(f\"Saved scores to: {scores_file}\")\n\n return scores\n\n\ndef main():\n parser = argparse.ArgumentParser(\n description=\"Evaluate ViDoRe v2 benchmark using LEANN/Fast-Plaid indexing\"\n )\n parser.add_argument(\n \"--model\",\n type=str,\n default=\"colqwen2\",\n choices=[\"colqwen2\", \"colpali\"],\n help=\"Model to use\",\n )\n parser.add_argument(\n \"--task\",\n type=str,\n default=None,\n help=\"Specific task to evaluate (or 'all' for all tasks)\",\n )\n parser.add_argument(\n \"--tasks\",\n type=str,\n default=\"all\",\n help=\"Tasks to evaluate: 'all' or comma-separated list\",\n )\n parser.add_argument(\n \"--index-path\",\n type=str,\n default=None,\n help=\"Path to LEANN index (auto-generated if not provided)\",\n )\n parser.add_argument(\n \"--use-fast-plaid\",\n action=\"store_true\",\n help=\"Use Fast-Plaid instead of LEANN\",\n )\n parser.add_argument(\n \"--fast-plaid-index-path\",\n type=str,\n default=None,\n help=\"Path to Fast-Plaid index (auto-generated if not provided)\",\n )\n parser.add_argument(\n \"--rebuild-index\",\n action=\"store_true\",\n help=\"Rebuild index even if it exists\",\n )\n parser.add_argument(\n \"--language\",\n type=str,\n default=None,\n help=\"Language to evaluate (default: first available)\",\n )\n parser.add_argument(\n \"--top-k\",\n type=int,\n default=100,\n help=\"Top-k results to retrieve\",\n )\n parser.add_argument(\n \"--first-stage-k\",\n type=int,\n default=500,\n help=\"First stage k for LEANN search\",\n )\n parser.add_argument(\n \"--k-values\",\n type=str,\n default=\"1,3,5,10,100\",\n help=\"Comma-separated k values for evaluation (e.g., '1,3,5,10,100')\",\n )\n parser.add_argument(\n \"--output-dir\",\n type=str,\n default=\"./vidore_v2_results\",\n help=\"Output directory for results\",\n )\n\n args = parser.parse_args()\n\n # Parse k_values\n k_values = [int(k.strip()) for k in args.k_values.split(\",\")]\n\n # Determine tasks to evaluate\n if args.task:\n tasks_to_eval = [args.task]\n elif args.tasks.lower() == \"all\":\n tasks_to_eval = list(VIDORE_V2_TASKS.keys())\n else:\n tasks_to_eval = [t.strip() for t in args.tasks.split(\",\")]\n\n print(f\"Tasks to evaluate: {tasks_to_eval}\")\n\n # Evaluate each task\n all_scores = {}\n for task_name in tasks_to_eval:\n try:\n scores = evaluate_task(\n task_name=task_name,\n model_name=args.model,\n index_path=args.index_path,\n use_fast_plaid=args.use_fast_plaid,\n fast_plaid_index_path=args.fast_plaid_index_path,\n language=args.language,\n rebuild_index=args.rebuild_index,\n top_k=args.top_k,\n first_stage_k=args.first_stage_k,\n k_values=k_values,\n output_dir=args.output_dir,\n )\n all_scores[task_name] = scores\n except Exception as e:\n print(f\"\\nError evaluating {task_name}: {e}\")\n import traceback\n\n traceback.print_exc()\n continue\n\n # Print summary\n if all_scores:\n print(f\"\\n{'=' * 80}\")\n print(\"SUMMARY\")\n print(f\"{'=' * 80}\")\n for task_name, scores in all_scores.items():\n print(f\"\\n{task_name}:\")\n # Print main metrics\n for metric in [\"ndcg_at_5\", \"ndcg_at_10\", \"ndcg_at_100\", \"map_at_10\", \"recall_at_10\"]:\n if metric in scores:\n print(f\" {metric}: {scores[metric]:.5f}\")\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "apps/qwen_data/__init__.py", "content": "" }, { "path": "apps/qwen_data/qwen_reader.py", "content": "import json\nimport logging\nfrom pathlib import Path\nfrom typing import Any\n\nlogger = logging.getLogger(__name__)\n\n\nclass QwenReader:\n \"\"\"Reader for Qwen Code CLI history files.\"\"\"\n\n def __init__(self):\n pass\n\n def load_data(self, history_dir: str, max_count: int = -1) -> list[dict[str, Any]]:\n \"\"\"\n Load data from Qwen Code history directory.\n\n Args:\n history_dir: Path to .qwen-code directory\n max_count: Max number of conversations to load\n\n Returns:\n List of dictionaries with 'text' and 'metadata' keys\n \"\"\"\n history_path = Path(history_dir).expanduser()\n if not history_path.exists():\n print(f\"Qwen history directory not found: {history_path}\")\n return []\n\n documents = []\n\n # 1. Load Memory (QWEN.md or MEMORY.md)\n for memory_filename in [\"QWEN.md\", \"MEMORY.md\"]:\n memory_file = history_path / memory_filename\n if memory_file.exists():\n try:\n text = memory_file.read_text(encoding=\"utf-8\")\n if text.strip():\n documents.append(\n {\n \"text\": f\"Qwen Code Memory:\\n{text}\",\n \"metadata\": {\"source\": str(memory_file), \"type\": \"memory\"},\n }\n )\n except Exception as e:\n print(f\"Error reading memory file {memory_filename}: {e}\")\n\n # 2. Find Session Files\n # Legacy JSON sessions\n session_files = list(history_path.glob(\"session-*.json\"))\n # New JSONL sessions\n session_files.extend(list(history_path.glob(\"session-*.jsonl\")))\n # Checkpoints\n session_files.extend(list(history_path.glob(\"checkpoint-*.json\")))\n\n # Sort by modification time (newest first)\n session_files.sort(key=lambda x: x.stat().st_mtime, reverse=True)\n\n print(f\"Found {len(session_files)} session files.\")\n\n count = 0\n for file_path in session_files:\n if max_count > 0 and count >= max_count:\n break\n\n try:\n content = \"\"\n if file_path.suffix == \".jsonl\":\n content = self._parse_jsonl_session(file_path)\n elif file_path.suffix == \".json\":\n content = self._parse_json_session(file_path)\n\n if content:\n documents.append(\n {\n \"text\": content,\n \"metadata\": {\n \"source\": str(file_path),\n \"type\": \"session\",\n \"filename\": file_path.name,\n },\n }\n )\n count += 1\n except Exception as e:\n print(f\"Error reading {file_path.name}: {e}\")\n\n print(f\"Successfully loaded {len(documents)} items from Qwen history.\")\n return documents\n\n def _parse_json_session(self, file_path: Path) -> str:\n \"\"\"Parse legacy JSON session file.\"\"\"\n data = json.loads(file_path.read_text(encoding=\"utf-8\"))\n\n # Handle dict format (standard session)\n messages = []\n if isinstance(data, dict):\n # Check for 'messages' key (standard format)\n if \"messages\" in data:\n for msg in data[\"messages\"]:\n role = msg.get(\"role\", \"unknown\")\n content = msg.get(\"content\", \"\")\n if content:\n messages.append(f\"{role.upper()}: {content}\")\n # Check for 'parts' key (checkpoint format sometimes)\n elif \"parts\" in data:\n messages.append(f\"Saved Session Content: {data['parts']}\")\n\n # Handle list format (some older array-based sessions)\n elif isinstance(data, list):\n for item in data:\n if isinstance(item, dict):\n role = item.get(\"role\", \"unknown\")\n content = item.get(\"content\", \"\") or item.get(\"parts\", \"\")\n if content:\n messages.append(f\"{role.upper()}: {content}\")\n\n if not messages:\n return \"\"\n\n return f\"File: {file_path.name}\\n\\n\" + \"\\n\\n\".join(messages)\n\n def _parse_jsonl_session(self, file_path: Path) -> str:\n \"\"\"Parse JSONL session file.\"\"\"\n messages = []\n try:\n with open(file_path, encoding=\"utf-8\") as f:\n for line in f:\n line = line.strip()\n if not line:\n continue\n try:\n data = json.loads(line)\n # Skip metadata lines if they don't have content\n if \"role\" in data and \"content\" in data:\n messages.append(f\"{data['role'].upper()}: {data['content']}\")\n elif \"parts\" in data: # sometimes parts is used\n messages.append(\n f\"{data.get('role', 'unknown').upper()}: {data['parts']}\"\n )\n except json.JSONDecodeError:\n continue\n except Exception:\n return \"\"\n\n if not messages:\n return \"\"\n\n return f\"File: {file_path.name}\\n\\n\" + \"\\n\\n\".join(messages)\n" }, { "path": "apps/qwen_rag.py", "content": "\"\"\"\nQwen Code RAG example.\nIndexes and searches Qwen Code CLI history (~/.qwen-code).\n\"\"\"\n\nimport sys\nfrom pathlib import Path\nfrom typing import Any\n\n# Add parent directory to path for imports\nsys.path.insert(0, str(Path(__file__).parent))\n\nfrom base_rag_example import BaseRAGExample\nfrom chunking import create_text_chunks\n\nfrom .qwen_data.qwen_reader import QwenReader\n\n\nclass QwenRAG(BaseRAGExample):\n \"\"\"RAG example for Qwen Code CLI history.\"\"\"\n\n def __init__(self):\n super().__init__(\n name=\"Qwen Code\",\n description=\"Process and query Qwen Code CLI history with LEANN\",\n default_index_name=\"qwen_index\",\n )\n\n def _add_specific_arguments(self, parser):\n \"\"\"Add Qwen-specific arguments.\"\"\"\n group = parser.add_argument_group(\"Qwen Parameters\")\n group.add_argument(\n \"--qwen-path\",\n type=str,\n default=\"~/.qwen-code\",\n help=\"Path to .qwen-code directory (default: ~/.qwen-code)\",\n )\n\n async def load_data(self, args) -> list[dict[str, Any]]:\n \"\"\"Load Qwen history and convert to text chunks.\"\"\"\n print(f\"Loading Qwen history from: {args.qwen_path}\")\n\n reader = QwenReader()\n documents = reader.load_data(history_dir=args.qwen_path, max_count=args.max_items)\n\n if not documents:\n print(\"No documents found! Check if ~/.qwen-code exists and has history.\")\n return []\n\n # Convert dicts to Document objects for chunking\n from llama_index.core import Document\n\n docs = [Document(text=d[\"text\"], metadata=d[\"metadata\"]) for d in documents]\n\n # Convert to text chunks\n print(f\"splitting {len(documents)} documents into chunks...\")\n chunks = create_text_chunks(docs)\n\n return chunks\n\n\nif __name__ == \"__main__\":\n import asyncio\n\n print(\"\\n✨ Qwen Code RAG\")\n print(\"=\" * 50)\n\n rag = QwenRAG()\n asyncio.run(rag.run())\n" }, { "path": "apps/semantic_file_search/leann-plus-temporal-search.py", "content": "#!/usr/bin/env python3\nimport re\nimport sys\nfrom datetime import datetime, timedelta\nfrom pathlib import Path\n\nfrom leann import LeannSearcher\n\nINDEX_PATH = str(Path(\"./\").resolve() / \"demo.leann\")\n\n\nclass TimeParser:\n def __init__(self):\n # Main pattern: captures optional fuzzy modifier, number, unit, and optional \"ago\"\n self.pattern = r\"(?:(around|about|roughly|approximately)\\s+)?(\\d+)\\s+(hour|day|week|month|year)s?(?:\\s+ago)?\"\n\n # Compile for performance\n self.regex = re.compile(self.pattern, re.IGNORECASE)\n\n # Stop words to remove before regex parsing\n self.stop_words = {\n \"in\",\n \"at\",\n \"of\",\n \"by\",\n \"as\",\n \"me\",\n \"the\",\n \"a\",\n \"an\",\n \"and\",\n \"any\",\n \"find\",\n \"search\",\n \"list\",\n \"ago\",\n \"back\",\n \"past\",\n \"earlier\",\n }\n\n def clean_text(self, text):\n \"\"\"Remove stop words from text\"\"\"\n words = text.split()\n cleaned = \" \".join(word for word in words if word.lower() not in self.stop_words)\n return cleaned\n\n def parse(self, text):\n \"\"\"Extract all time expressions from text\"\"\"\n # Clean text first\n cleaned_text = self.clean_text(text)\n\n matches = []\n for match in self.regex.finditer(cleaned_text):\n fuzzy = match.group(1) # \"around\", \"about\", etc.\n number = int(match.group(2))\n unit = match.group(3).lower()\n\n matches.append(\n {\n \"full_match\": match.group(0),\n \"fuzzy\": bool(fuzzy),\n \"number\": number,\n \"unit\": unit,\n \"range\": self.calculate_range(number, unit, bool(fuzzy)),\n }\n )\n\n return matches\n\n def calculate_range(self, number, unit, is_fuzzy):\n \"\"\"Convert to actual datetime range and return ISO format strings\"\"\"\n units = {\n \"hour\": timedelta(hours=number),\n \"day\": timedelta(days=number),\n \"week\": timedelta(weeks=number),\n \"month\": timedelta(days=number * 30),\n \"year\": timedelta(days=number * 365),\n }\n\n delta = units[unit]\n now = datetime.now()\n target = now - delta\n\n if is_fuzzy:\n buffer = delta * 0.2 # 20% buffer for fuzzy\n start = (target - buffer).isoformat()\n end = (target + buffer).isoformat()\n else:\n start = target.isoformat()\n end = now.isoformat()\n\n return (start, end)\n\n\ndef search_files(query, top_k=15):\n \"\"\"Search the index and return results\"\"\"\n # Parse time expressions\n parser = TimeParser()\n time_matches = parser.parse(query)\n\n # Remove time expressions from query for semantic search\n clean_query = query\n if time_matches:\n for match in time_matches:\n clean_query = clean_query.replace(match[\"full_match\"], \"\").strip()\n\n # Check if clean_query is less than 4 characters\n if len(clean_query) < 4:\n print(\"Error: add more input for accurate results.\")\n return\n\n # Single query to vector DB\n searcher = LeannSearcher(INDEX_PATH)\n results = searcher.search(\n clean_query if clean_query else query, top_k=top_k, recompute_embeddings=False\n )\n\n # Filter by time if time expression found\n if time_matches:\n time_range = time_matches[0][\"range\"] # Use first time expression\n start_time, end_time = time_range\n\n filtered_results = []\n for result in results:\n # Access metadata attribute directly (not .get())\n metadata = result.metadata if hasattr(result, \"metadata\") else {}\n\n if metadata:\n # Check modification date first, fall back to creation date\n date_str = metadata.get(\"modification_date\") or metadata.get(\"creation_date\")\n\n if date_str:\n # Convert strings to datetime objects for proper comparison\n try:\n file_date = datetime.fromisoformat(date_str)\n start_dt = datetime.fromisoformat(start_time)\n end_dt = datetime.fromisoformat(end_time)\n\n # Compare dates properly\n if start_dt <= file_date <= end_dt:\n filtered_results.append(result)\n except (ValueError, TypeError):\n # Handle invalid date formats\n print(f\"Warning: Invalid date format in metadata: {date_str}\")\n continue\n\n results = filtered_results\n\n # Print results\n print(f\"\\nSearch results for: '{query}'\")\n if time_matches:\n print(\n f\"Time filter: {time_matches[0]['number']} {time_matches[0]['unit']}(s) {'(fuzzy)' if time_matches[0]['fuzzy'] else ''}\"\n )\n print(\n f\"Date range: {time_matches[0]['range'][0][:10]} to {time_matches[0]['range'][1][:10]}\"\n )\n print(\"-\" * 80)\n\n for i, result in enumerate(results, 1):\n print(f\"\\n[{i}] Score: {result.score:.4f}\")\n print(f\"Content: {result.text}\")\n\n # Show metadata if present\n metadata = result.metadata if hasattr(result, \"metadata\") else None\n if metadata:\n if \"creation_date\" in metadata:\n print(f\"Created: {metadata['creation_date']}\")\n if \"modification_date\" in metadata:\n print(f\"Modified: {metadata['modification_date']}\")\n print(\"-\" * 80)\n\n\nif __name__ == \"__main__\":\n if len(sys.argv) < 2:\n print('Usage: python search_index.py \"\" [top_k]')\n sys.exit(1)\n\n query = sys.argv[1]\n top_k = int(sys.argv[2]) if len(sys.argv) > 2 else 15\n\n search_files(query, top_k)\n" }, { "path": "apps/semantic_file_search/leann_index_builder.py", "content": "#!/usr/bin/env python3\nimport json\nimport sys\nfrom pathlib import Path\n\nfrom leann import LeannBuilder\n\n\ndef process_json_items(json_file_path):\n \"\"\"Load and process JSON file with metadata items\"\"\"\n\n with open(json_file_path, encoding=\"utf-8\") as f:\n items = json.load(f)\n\n # Guard against empty JSON\n if not items:\n print(\"⚠️ No items found in the JSON file. Exiting gracefully.\")\n return\n\n INDEX_PATH = str(Path(\"./\").resolve() / \"demo.leann\")\n builder = LeannBuilder(backend_name=\"hnsw\", is_recompute=False)\n\n total_items = len(items)\n items_added = 0\n print(f\"Processing {total_items} items...\")\n\n for idx, item in enumerate(items):\n try:\n # Create embedding text sentence\n embedding_text = f\"{item.get('Name', 'unknown')} located at {item.get('Path', 'unknown')} and size {item.get('Size', 'unknown')} bytes with content type {item.get('ContentType', 'unknown')} and kind {item.get('Kind', 'unknown')}\"\n\n # Prepare metadata with dates\n metadata = {}\n if \"CreationDate\" in item:\n metadata[\"creation_date\"] = item[\"CreationDate\"]\n if \"ContentChangeDate\" in item:\n metadata[\"modification_date\"] = item[\"ContentChangeDate\"]\n\n # Add to builder\n builder.add_text(embedding_text, metadata=metadata)\n items_added += 1\n\n except Exception as e:\n print(f\"\\n⚠️ Warning: Failed to process item {idx}: {e}\")\n continue\n\n # Show progress\n progress = (idx + 1) / total_items * 100\n sys.stdout.write(f\"\\rProgress: {idx + 1}/{total_items} ({progress:.1f}%)\")\n sys.stdout.flush()\n\n print() # New line after progress\n\n # Guard against no successfully added items\n if items_added == 0:\n print(\"⚠️ No items were successfully added to the index. Exiting gracefully.\")\n return\n\n print(f\"\\n✅ Successfully processed {items_added}/{total_items} items\")\n print(\"Building index...\")\n\n try:\n builder.build_index(INDEX_PATH)\n print(f\"✓ Index saved to {INDEX_PATH}\")\n except ValueError as e:\n if \"No chunks added\" in str(e):\n print(\"⚠️ No chunks were added to the builder. Index not created.\")\n else:\n raise\n\n\nif __name__ == \"__main__\":\n if len(sys.argv) != 2:\n print(\"Usage: python build_index.py \")\n sys.exit(1)\n\n json_file = sys.argv[1]\n if not Path(json_file).exists():\n print(f\"Error: File {json_file} not found\")\n sys.exit(1)\n\n process_json_items(json_file)\n" }, { "path": "apps/semantic_file_search/spotlight_index_dump.py", "content": "#!/usr/bin/env python3\n\"\"\"\nSpotlight Metadata Dumper for Vector DB\nExtracts only essential metadata for semantic search embeddings\nOutput is optimized for vector database storage with minimal fields\n\"\"\"\n\nimport json\nimport sys\nfrom datetime import datetime\n\n# Check platform before importing macOS-specific modules\nif sys.platform != \"darwin\":\n print(\"This script requires macOS (uses Spotlight)\")\n sys.exit(1)\n\nfrom Foundation import NSDate, NSMetadataQuery, NSPredicate, NSRunLoop\n\n# EDIT THIS LIST: Add or remove folders to search\n# Can be either:\n# - Folder names relative to home directory (e.g., \"Desktop\", \"Downloads\")\n# - Absolute paths (e.g., \"/Applications\", \"/System/Library\")\nSEARCH_FOLDERS = [\n \"Desktop\",\n \"Downloads\",\n \"Documents\",\n \"Music\",\n \"Pictures\",\n \"Movies\",\n # \"Library\", # Uncomment to include\n # \"/Applications\", # Absolute path example\n # \"Code/Projects\", # Subfolder example\n # Add any other folders here\n]\n\n\ndef convert_to_serializable(obj):\n \"\"\"Convert NS objects to Python serializable types\"\"\"\n if obj is None:\n return None\n\n # Handle NSDate\n if hasattr(obj, \"timeIntervalSince1970\"):\n return datetime.fromtimestamp(obj.timeIntervalSince1970()).isoformat()\n\n # Handle NSArray\n if hasattr(obj, \"count\") and hasattr(obj, \"objectAtIndex_\"):\n return [convert_to_serializable(obj.objectAtIndex_(i)) for i in range(obj.count())]\n\n # Convert to string\n try:\n return str(obj)\n except Exception:\n return repr(obj)\n\n\ndef dump_spotlight_data(max_items=10, output_file=\"spotlight_dump.json\"):\n \"\"\"\n Dump Spotlight data using public.item predicate\n \"\"\"\n # Build full paths from SEARCH_FOLDERS\n import os\n\n home_dir = os.path.expanduser(\"~\")\n search_paths = []\n\n print(\"Search locations:\")\n for folder in SEARCH_FOLDERS:\n # Check if it's an absolute path or relative\n if folder.startswith(\"/\"):\n full_path = folder\n else:\n full_path = os.path.join(home_dir, folder)\n\n if os.path.exists(full_path):\n search_paths.append(full_path)\n print(f\" ✓ {full_path}\")\n else:\n print(f\" ✗ {full_path} (not found)\")\n\n if not search_paths:\n print(\"No valid search paths found!\")\n return []\n\n print(f\"\\nDumping {max_items} items from Spotlight (public.item)...\")\n\n # Create query with public.item predicate\n query = NSMetadataQuery.alloc().init()\n predicate = NSPredicate.predicateWithFormat_(\"kMDItemContentTypeTree CONTAINS 'public.item'\")\n query.setPredicate_(predicate)\n\n # Set search scopes to our specific folders\n query.setSearchScopes_(search_paths)\n\n print(\"Starting query...\")\n query.startQuery()\n\n # Wait for gathering to complete\n run_loop = NSRunLoop.currentRunLoop()\n print(\"Gathering results...\")\n\n # Let it gather for a few seconds\n for i in range(50): # 5 seconds max\n run_loop.runMode_beforeDate_(\n \"NSDefaultRunLoopMode\", NSDate.dateWithTimeIntervalSinceNow_(0.1)\n )\n # Check gathering status periodically\n if i % 10 == 0:\n current_count = query.resultCount()\n if current_count > 0:\n print(f\" Found {current_count} items so far...\")\n\n # Continue while still gathering (up to 2 more seconds)\n timeout = NSDate.dateWithTimeIntervalSinceNow_(2.0)\n while query.isGathering() and timeout.timeIntervalSinceNow() > 0:\n run_loop.runMode_beforeDate_(\n \"NSDefaultRunLoopMode\", NSDate.dateWithTimeIntervalSinceNow_(0.1)\n )\n\n query.stopQuery()\n\n total_results = query.resultCount()\n print(f\"Found {total_results} total items\")\n\n if total_results == 0:\n print(\"No results found\")\n return []\n\n # Process items\n items_to_process = min(total_results, max_items)\n results = []\n\n # ONLY relevant attributes for vector embeddings\n # These provide essential context for semantic search without bloat\n attributes = [\n \"kMDItemPath\", # Full path for file retrieval\n \"kMDItemFSName\", # Filename for display & embedding\n \"kMDItemFSSize\", # Size for filtering/ranking\n \"kMDItemContentType\", # File type for categorization\n \"kMDItemKind\", # Human-readable type for embedding\n \"kMDItemFSCreationDate\", # Temporal context\n \"kMDItemFSContentChangeDate\", # Recency for ranking\n ]\n\n print(f\"Processing {items_to_process} items...\")\n\n for i in range(items_to_process):\n try:\n item = query.resultAtIndex_(i)\n metadata = {}\n\n # Extract ONLY the relevant attributes\n for attr in attributes:\n try:\n value = item.valueForAttribute_(attr)\n if value is not None:\n # Keep the attribute name clean (remove kMDItem prefix for cleaner JSON)\n clean_key = attr.replace(\"kMDItem\", \"\").replace(\"FS\", \"\")\n metadata[clean_key] = convert_to_serializable(value)\n except (AttributeError, ValueError, TypeError):\n continue\n\n # Only add if we have at least a path\n if metadata.get(\"Path\"):\n results.append(metadata)\n\n except Exception as e:\n print(f\"Error processing item {i}: {e}\")\n continue\n\n # Save to JSON\n with open(output_file, \"w\", encoding=\"utf-8\") as f:\n json.dump(results, f, indent=2, ensure_ascii=False)\n\n print(f\"\\n✓ Saved {len(results)} items to {output_file}\")\n\n # Show summary\n print(\"\\nSample items:\")\n import os\n\n home_dir = os.path.expanduser(\"~\")\n\n for i, item in enumerate(results[:3]):\n print(f\"\\n[Item {i + 1}]\")\n print(f\" Path: {item.get('Path', 'N/A')}\")\n print(f\" Name: {item.get('Name', 'N/A')}\")\n print(f\" Type: {item.get('ContentType', 'N/A')}\")\n print(f\" Kind: {item.get('Kind', 'N/A')}\")\n\n # Handle size properly\n size = item.get(\"Size\")\n if size:\n try:\n size_int = int(size)\n if size_int > 1024 * 1024:\n print(f\" Size: {size_int / (1024 * 1024):.2f} MB\")\n elif size_int > 1024:\n print(f\" Size: {size_int / 1024:.2f} KB\")\n else:\n print(f\" Size: {size_int} bytes\")\n except (ValueError, TypeError):\n print(f\" Size: {size}\")\n\n # Show dates\n if \"CreationDate\" in item:\n print(f\" Created: {item['CreationDate']}\")\n if \"ContentChangeDate\" in item:\n print(f\" Modified: {item['ContentChangeDate']}\")\n\n # Count by type\n type_counts = {}\n for item in results:\n content_type = item.get(\"ContentType\", \"unknown\")\n type_counts[content_type] = type_counts.get(content_type, 0) + 1\n\n print(f\"\\nTotal items saved: {len(results)}\")\n\n if type_counts:\n print(\"\\nTop content types:\")\n for ct, count in sorted(type_counts.items(), key=lambda x: x[1], reverse=True)[:5]:\n print(f\" {ct}: {count} items\")\n\n # Count by folder\n folder_counts = {}\n for item in results:\n path = item.get(\"Path\", \"\")\n for folder in SEARCH_FOLDERS:\n # Build the full folder path\n if folder.startswith(\"/\"):\n folder_path = folder\n else:\n folder_path = os.path.join(home_dir, folder)\n\n if path.startswith(folder_path):\n folder_counts[folder] = folder_counts.get(folder, 0) + 1\n break\n\n if folder_counts:\n print(\"\\nItems by location:\")\n for folder, count in sorted(folder_counts.items(), key=lambda x: x[1], reverse=True):\n print(f\" {folder}: {count} items\")\n\n return results\n\n\ndef main():\n # Parse arguments\n if len(sys.argv) > 1:\n try:\n max_items = int(sys.argv[1])\n except ValueError:\n print(\"Usage: python spot.py [number_of_items]\")\n print(\"Default: 10 items\")\n sys.exit(1)\n else:\n max_items = 10\n\n output_file = sys.argv[2] if len(sys.argv) > 2 else \"spotlight_dump.json\"\n\n # Run dump\n dump_spotlight_data(max_items=max_items, output_file=output_file)\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "apps/slack_data/__init__.py", "content": "# Slack MCP data integration for LEANN\n" }, { "path": "apps/slack_data/slack_mcp_reader.py", "content": "#!/usr/bin/env python3\n\"\"\"\nSlack MCP Reader for LEANN\n\nThis module provides functionality to connect to Slack MCP servers and fetch message data\nfor indexing in LEANN. It supports various Slack MCP server implementations and provides\nflexible message processing options.\n\"\"\"\n\nimport ast\nimport asyncio\nimport json\nimport logging\nfrom typing import Any, Optional\n\nlogger = logging.getLogger(__name__)\n\n\nclass SlackMCPReader:\n \"\"\"\n Reader for Slack data via MCP (Model Context Protocol) servers.\n\n This class connects to Slack MCP servers to fetch message data and convert it\n into a format suitable for LEANN indexing.\n \"\"\"\n\n def __init__(\n self,\n mcp_server_command: str,\n workspace_name: Optional[str] = None,\n concatenate_conversations: bool = True,\n max_messages_per_conversation: int = 100,\n max_retries: int = 5,\n retry_delay: float = 2.0,\n ):\n \"\"\"\n Initialize the Slack MCP Reader.\n\n Args:\n mcp_server_command: Command to start the MCP server (e.g., 'slack-mcp-server')\n workspace_name: Optional workspace name to filter messages\n concatenate_conversations: Whether to group messages by channel/thread\n max_messages_per_conversation: Maximum messages to include per conversation\n max_retries: Maximum number of retries for failed operations\n retry_delay: Initial delay between retries in seconds\n \"\"\"\n self.mcp_server_command = mcp_server_command\n self.workspace_name = workspace_name\n self.concatenate_conversations = concatenate_conversations\n self.max_messages_per_conversation = max_messages_per_conversation\n self.max_retries = max_retries\n self.retry_delay = retry_delay\n self.mcp_process: asyncio.subprocess.Process | None = None\n\n async def start_mcp_server(self):\n \"\"\"Start the MCP server process.\"\"\"\n try:\n self.mcp_process = await asyncio.create_subprocess_exec(\n *self.mcp_server_command.split(),\n stdin=asyncio.subprocess.PIPE,\n stdout=asyncio.subprocess.PIPE,\n stderr=asyncio.subprocess.PIPE,\n )\n logger.info(f\"Started MCP server: {self.mcp_server_command}\")\n except Exception as e:\n logger.error(f\"Failed to start MCP server: {e}\")\n raise\n\n async def stop_mcp_server(self):\n \"\"\"Stop the MCP server process.\"\"\"\n if self.mcp_process:\n self.mcp_process.terminate()\n await self.mcp_process.wait()\n logger.info(\"Stopped MCP server\")\n\n async def send_mcp_request(self, request: dict[str, Any]) -> dict[str, Any]:\n \"\"\"Send a request to the MCP server and get response.\"\"\"\n proc = self.mcp_process\n if proc is None:\n raise RuntimeError(\"MCP server not started\")\n if proc.stdin is None or proc.stdout is None:\n raise RuntimeError(\"MCP server stdio not available\")\n\n request_json = json.dumps(request) + \"\\n\"\n proc.stdin.write(request_json.encode())\n await proc.stdin.drain()\n\n response_line = await proc.stdout.readline()\n if not response_line:\n raise RuntimeError(\"No response from MCP server\")\n\n return json.loads(response_line.decode().strip())\n\n async def initialize_mcp_connection(self):\n \"\"\"Initialize the MCP connection.\"\"\"\n init_request = {\n \"jsonrpc\": \"2.0\",\n \"id\": 1,\n \"method\": \"initialize\",\n \"params\": {\n \"protocolVersion\": \"2024-11-05\",\n \"capabilities\": {},\n \"clientInfo\": {\"name\": \"leann-slack-reader\", \"version\": \"1.0.0\"},\n },\n }\n\n response = await self.send_mcp_request(init_request)\n if \"error\" in response:\n raise RuntimeError(f\"MCP initialization failed: {response['error']}\")\n\n logger.info(\"MCP connection initialized successfully\")\n\n async def list_available_tools(self) -> list[dict[str, Any]]:\n \"\"\"List available tools from the MCP server.\"\"\"\n list_request = {\"jsonrpc\": \"2.0\", \"id\": 2, \"method\": \"tools/list\", \"params\": {}}\n\n response = await self.send_mcp_request(list_request)\n if \"error\" in response:\n raise RuntimeError(f\"Failed to list tools: {response['error']}\")\n\n return response.get(\"result\", {}).get(\"tools\", [])\n\n def _is_cache_sync_error(self, error: dict) -> bool:\n \"\"\"Check if the error is related to users cache not being ready.\"\"\"\n if isinstance(error, dict):\n message = error.get(\"message\", \"\").lower()\n return (\n \"users cache is not ready\" in message or \"sync process is still running\" in message\n )\n return False\n\n async def _retry_with_backoff(self, func, *args, **kwargs):\n \"\"\"Retry a function with exponential backoff, especially for cache sync issues.\"\"\"\n last_exception = None\n\n for attempt in range(self.max_retries + 1):\n try:\n return await func(*args, **kwargs)\n except Exception as e:\n last_exception = e\n\n # Check if this is a cache sync error\n error_dict = {}\n if hasattr(e, \"args\") and e.args and isinstance(e.args[0], dict):\n error_dict = e.args[0]\n elif \"Failed to fetch messages\" in str(e):\n # Try to extract error from the exception message\n import re\n\n match = re.search(r\"'error':\\s*(\\{[^}]+\\})\", str(e))\n if match:\n try:\n error_dict = ast.literal_eval(match.group(1))\n except (ValueError, SyntaxError):\n pass\n else:\n # Try alternative format\n match = re.search(r\"Failed to fetch messages:\\s*(\\{[^}]+\\})\", str(e))\n if match:\n try:\n error_dict = ast.literal_eval(match.group(1))\n except (ValueError, SyntaxError):\n pass\n\n if self._is_cache_sync_error(error_dict):\n if attempt < self.max_retries:\n delay = self.retry_delay * (2**attempt) # Exponential backoff\n logger.info(\n f\"Cache sync not ready, waiting {delay:.1f}s before retry {attempt + 1}/{self.max_retries}\"\n )\n await asyncio.sleep(delay)\n continue\n else:\n logger.warning(\n f\"Cache sync still not ready after {self.max_retries} retries, giving up\"\n )\n break\n else:\n # Not a cache sync error, don't retry\n break\n\n # If we get here, all retries failed or it's not a retryable error\n if last_exception is not None:\n raise last_exception\n raise RuntimeError(\"Unexpected error: no exception captured during retry loop\")\n\n async def fetch_slack_messages(\n self, channel: Optional[str] = None, limit: int = 100\n ) -> list[dict[str, Any]]:\n \"\"\"\n Fetch Slack messages using MCP tools with retry logic for cache sync issues.\n\n Args:\n channel: Optional channel name to filter messages\n limit: Maximum number of messages to fetch\n\n Returns:\n List of message dictionaries\n \"\"\"\n return await self._retry_with_backoff(self._fetch_slack_messages_impl, channel, limit)\n\n async def _fetch_slack_messages_impl(\n self, channel: Optional[str] = None, limit: int = 100\n ) -> list[dict[str, Any]]:\n \"\"\"\n Internal implementation of fetch_slack_messages without retry logic.\n \"\"\"\n # This is a generic implementation - specific MCP servers may have different tool names\n # Common tool names might be: 'get_messages', 'list_messages', 'fetch_channel_history'\n\n tools = await self.list_available_tools()\n logger.info(f\"Available tools: {[tool.get('name') for tool in tools]}\")\n message_tool = None\n\n # Look for a tool that can fetch messages - prioritize conversations_history\n message_tool = None\n\n # First, try to find conversations_history specifically\n for tool in tools:\n tool_name = tool.get(\"name\", \"\").lower()\n if \"conversations_history\" in tool_name:\n message_tool = tool\n logger.info(f\"Found conversations_history tool: {tool}\")\n break\n\n # If not found, look for other message-fetching tools\n if not message_tool:\n for tool in tools:\n tool_name = tool.get(\"name\", \"\").lower()\n if any(\n keyword in tool_name\n for keyword in [\"conversations_search\", \"message\", \"history\"]\n ):\n message_tool = tool\n break\n\n if not message_tool:\n raise RuntimeError(\"No message fetching tool found in MCP server\")\n\n # Prepare tool call parameters\n tool_params = {\"limit\": \"180d\"} # Use 180 days to get older messages\n if channel:\n # For conversations_history, use channel_id parameter\n if message_tool[\"name\"] == \"conversations_history\":\n tool_params[\"channel_id\"] = channel\n else:\n # Try common parameter names for channel specification\n for param_name in [\"channel\", \"channel_id\", \"channel_name\"]:\n tool_params[param_name] = channel\n break\n\n logger.info(f\"Tool parameters: {tool_params}\")\n\n fetch_request = {\n \"jsonrpc\": \"2.0\",\n \"id\": 3,\n \"method\": \"tools/call\",\n \"params\": {\"name\": message_tool[\"name\"], \"arguments\": tool_params},\n }\n\n response = await self.send_mcp_request(fetch_request)\n if \"error\" in response:\n raise RuntimeError(f\"Failed to fetch messages: {response['error']}\")\n\n # Extract messages from response - format may vary by MCP server\n result = response.get(\"result\", {})\n if \"content\" in result and isinstance(result[\"content\"], list):\n # Some MCP servers return content as a list\n content = result[\"content\"][0] if result[\"content\"] else {}\n if \"text\" in content:\n try:\n messages = json.loads(content[\"text\"])\n except json.JSONDecodeError:\n # If not JSON, try to parse as CSV format (Slack MCP server format)\n text_content = content.get(\"text\", \"\")\n messages = self._parse_csv_messages(\n text_content if text_content else \"\", channel or \"unknown\"\n )\n else:\n messages = result[\"content\"]\n else:\n # Direct message format\n messages = result.get(\"messages\", [result])\n\n return messages if isinstance(messages, list) else [messages]\n\n def _parse_csv_messages(self, csv_text: str, channel: str) -> list[dict[str, Any]]:\n \"\"\"Parse CSV format messages from Slack MCP server.\"\"\"\n import csv\n import io\n\n messages = []\n try:\n # Split by lines and process each line as a CSV row\n lines = csv_text.strip().split(\"\\n\")\n if not lines:\n return messages\n\n # Skip header line if it exists\n start_idx = 0\n if lines[0].startswith(\"MsgID,UserID,UserName\"):\n start_idx = 1\n\n for line in lines[start_idx:]:\n if not line.strip():\n continue\n\n # Parse CSV line\n reader = csv.reader(io.StringIO(line))\n try:\n row = next(reader)\n if len(row) >= 7: # Ensure we have enough columns\n message = {\n \"ts\": row[0],\n \"user\": row[1],\n \"username\": row[2],\n \"real_name\": row[3],\n \"channel\": row[4],\n \"thread_ts\": row[5],\n \"text\": row[6],\n \"time\": row[7] if len(row) > 7 else \"\",\n \"reactions\": row[8] if len(row) > 8 else \"\",\n \"cursor\": row[9] if len(row) > 9 else \"\",\n }\n messages.append(message)\n except Exception as e:\n logger.warning(f\"Failed to parse CSV line: {line[:100]}... Error: {e}\")\n continue\n\n except Exception as e:\n logger.warning(f\"Failed to parse CSV messages: {e}\")\n # Fallback: treat entire text as one message\n messages = [{\"text\": csv_text, \"channel\": channel or \"unknown\"}]\n\n return messages\n\n def _format_message(self, message: dict[str, Any]) -> str:\n \"\"\"Format a single message for indexing.\"\"\"\n text = message.get(\"text\", \"\")\n user = message.get(\"user\", message.get(\"username\", \"Unknown\"))\n channel = message.get(\"channel\", message.get(\"channel_name\", \"Unknown\"))\n timestamp = message.get(\"ts\", message.get(\"timestamp\", \"\"))\n\n # Format timestamp if available\n formatted_time = \"\"\n if timestamp:\n try:\n import datetime\n\n if isinstance(timestamp, str) and \".\" in timestamp:\n dt = datetime.datetime.fromtimestamp(float(timestamp))\n formatted_time = dt.strftime(\"%Y-%m-%d %H:%M:%S\")\n elif isinstance(timestamp, (int, float)):\n dt = datetime.datetime.fromtimestamp(timestamp)\n formatted_time = dt.strftime(\"%Y-%m-%d %H:%M:%S\")\n else:\n formatted_time = str(timestamp)\n except (ValueError, TypeError):\n formatted_time = str(timestamp)\n\n # Build formatted message\n parts = []\n if channel:\n parts.append(f\"Channel: #{channel}\")\n if user:\n parts.append(f\"User: {user}\")\n if formatted_time:\n parts.append(f\"Time: {formatted_time}\")\n if text:\n parts.append(f\"Message: {text}\")\n\n return \"\\n\".join(parts)\n\n def _create_concatenated_content(self, messages: list[dict[str, Any]], channel: str) -> str:\n \"\"\"Create concatenated content from multiple messages in a channel.\"\"\"\n if not messages:\n return \"\"\n\n # Sort messages by timestamp if available\n try:\n messages.sort(key=lambda x: float(x.get(\"ts\", x.get(\"timestamp\", 0))))\n except (ValueError, TypeError):\n pass # Keep original order if timestamps aren't numeric\n\n # Limit messages per conversation\n if len(messages) > self.max_messages_per_conversation:\n messages = messages[-self.max_messages_per_conversation :]\n\n # Create header\n content_parts = [\n f\"Slack Channel: #{channel}\",\n f\"Message Count: {len(messages)}\",\n f\"Workspace: {self.workspace_name or 'Unknown'}\",\n \"=\" * 50,\n \"\",\n ]\n\n # Add messages\n for message in messages:\n formatted_msg = self._format_message(message)\n if formatted_msg.strip():\n content_parts.append(formatted_msg)\n content_parts.append(\"-\" * 30)\n content_parts.append(\"\")\n\n return \"\\n\".join(content_parts)\n\n async def get_all_channels(self) -> list[str]:\n \"\"\"Get list of all available channels.\"\"\"\n try:\n channels_list_request = {\n \"jsonrpc\": \"2.0\",\n \"id\": 4,\n \"method\": \"tools/call\",\n \"params\": {\"name\": \"channels_list\", \"arguments\": {}},\n }\n channels_response = await self.send_mcp_request(channels_list_request)\n if \"result\" in channels_response:\n result = channels_response[\"result\"]\n if \"content\" in result and isinstance(result[\"content\"], list):\n content = result[\"content\"][0] if result[\"content\"] else {}\n if \"text\" in content:\n # Parse the channels from the response\n channels = []\n lines = content[\"text\"].split(\"\\n\")\n for line in lines:\n if line.strip() and (\"#\" in line or \"C\" in line[:10]):\n # Extract channel ID or name\n parts = line.split()\n for part in parts:\n if part.startswith(\"C\") and len(part) > 5:\n channels.append(part)\n elif part.startswith(\"#\"):\n channels.append(part[1:]) # Remove #\n logger.info(f\"Found {len(channels)} channels: {channels}\")\n return channels\n return []\n except Exception as e:\n logger.warning(f\"Failed to get channels list: {e}\")\n return []\n\n async def read_slack_data(self, channels: Optional[list[str]] = None) -> list[str]:\n \"\"\"\n Read Slack data and return formatted text chunks.\n\n Args:\n channels: Optional list of channel names to fetch. If None, fetches from all available channels.\n\n Returns:\n List of formatted text chunks ready for LEANN indexing\n \"\"\"\n try:\n await self.start_mcp_server()\n await self.initialize_mcp_connection()\n\n all_texts = []\n\n if channels:\n # Fetch specific channels\n for channel in channels:\n try:\n messages = await self.fetch_slack_messages(channel=channel, limit=1000)\n if messages:\n if self.concatenate_conversations:\n text_content = self._create_concatenated_content(messages, channel)\n if text_content.strip():\n all_texts.append(text_content)\n else:\n # Process individual messages\n for message in messages:\n formatted_msg = self._format_message(message)\n if formatted_msg.strip():\n all_texts.append(formatted_msg)\n except Exception as e:\n logger.warning(f\"Failed to fetch messages from channel {channel}: {e}\")\n continue\n else:\n # Fetch from all available channels\n logger.info(\"Fetching from all available channels...\")\n all_channels = await self.get_all_channels()\n\n if not all_channels:\n # Fallback to common channel names if we can't get the list\n all_channels = [\"general\", \"random\", \"announcements\", \"C0GN5BX0F\"]\n logger.info(f\"Using fallback channels: {all_channels}\")\n\n for channel in all_channels:\n try:\n logger.info(f\"Searching channel: {channel}\")\n messages = await self.fetch_slack_messages(channel=channel, limit=1000)\n if messages:\n if self.concatenate_conversations:\n text_content = self._create_concatenated_content(messages, channel)\n if text_content.strip():\n all_texts.append(text_content)\n else:\n # Process individual messages\n for message in messages:\n formatted_msg = self._format_message(message)\n if formatted_msg.strip():\n all_texts.append(formatted_msg)\n except Exception as e:\n logger.warning(f\"Failed to fetch messages from channel {channel}: {e}\")\n continue\n\n return all_texts\n\n finally:\n await self.stop_mcp_server()\n\n async def __aenter__(self):\n \"\"\"Async context manager entry.\"\"\"\n await self.start_mcp_server()\n await self.initialize_mcp_connection()\n return self\n\n async def __aexit__(self, exc_type, exc_val, exc_tb):\n \"\"\"Async context manager exit.\"\"\"\n await self.stop_mcp_server()\n" }, { "path": "apps/slack_rag.py", "content": "#!/usr/bin/env python3\n\"\"\"\nSlack RAG Application with MCP Support\n\nThis application enables RAG (Retrieval-Augmented Generation) on Slack messages\nby connecting to Slack MCP servers to fetch live data and index it in LEANN.\n\nUsage:\n python -m apps.slack_rag --mcp-server \"slack-mcp-server\" --query \"What did the team discuss about the project?\"\n\"\"\"\n\nimport argparse\nimport asyncio\nfrom typing import Any\n\nfrom apps.base_rag_example import BaseRAGExample\nfrom apps.slack_data.slack_mcp_reader import SlackMCPReader\n\n\nclass SlackMCPRAG(BaseRAGExample):\n \"\"\"\n RAG application for Slack messages via MCP servers.\n\n This class provides a complete RAG pipeline for Slack data, including\n MCP server connection, data fetching, indexing, and interactive chat.\n \"\"\"\n\n def __init__(self):\n super().__init__(\n name=\"Slack MCP RAG\",\n description=\"RAG application for Slack messages via MCP servers\",\n default_index_name=\"slack_messages\",\n )\n\n def _add_specific_arguments(self, parser: argparse.ArgumentParser):\n \"\"\"Add Slack MCP-specific arguments.\"\"\"\n parser.add_argument(\n \"--mcp-server\",\n type=str,\n required=True,\n help=\"Command to start the Slack MCP server (e.g., 'slack-mcp-server' or 'npx slack-mcp-server')\",\n )\n\n parser.add_argument(\n \"--workspace-name\",\n type=str,\n help=\"Slack workspace name for better organization and filtering\",\n )\n\n parser.add_argument(\n \"--channels\",\n nargs=\"+\",\n help=\"Specific Slack channels to index (e.g., general random). If not specified, fetches from all available channels\",\n )\n\n parser.add_argument(\n \"--concatenate-conversations\",\n action=\"store_true\",\n default=True,\n help=\"Group messages by channel/thread for better context (default: True)\",\n )\n\n parser.add_argument(\n \"--no-concatenate-conversations\",\n action=\"store_true\",\n help=\"Process individual messages instead of grouping by channel\",\n )\n\n parser.add_argument(\n \"--max-messages-per-channel\",\n type=int,\n default=100,\n help=\"Maximum number of messages to include per channel (default: 100)\",\n )\n\n parser.add_argument(\n \"--test-connection\",\n action=\"store_true\",\n help=\"Test MCP server connection and list available tools without indexing\",\n )\n\n parser.add_argument(\n \"--max-retries\",\n type=int,\n default=5,\n help=\"Maximum number of retries for failed operations (default: 5)\",\n )\n\n parser.add_argument(\n \"--retry-delay\",\n type=float,\n default=2.0,\n help=\"Initial delay between retries in seconds (default: 2.0)\",\n )\n\n async def test_mcp_connection(self, args) -> bool:\n \"\"\"Test the MCP server connection and display available tools.\"\"\"\n print(f\"Testing connection to MCP server: {args.mcp_server}\")\n\n try:\n reader = SlackMCPReader(\n mcp_server_command=args.mcp_server,\n workspace_name=args.workspace_name,\n concatenate_conversations=not args.no_concatenate_conversations,\n max_messages_per_conversation=args.max_messages_per_channel,\n max_retries=args.max_retries,\n retry_delay=args.retry_delay,\n )\n\n async with reader:\n tools = await reader.list_available_tools()\n\n print(\"Successfully connected to MCP server!\")\n print(f\"Available tools ({len(tools)}):\")\n\n for i, tool in enumerate(tools, 1):\n name = tool.get(\"name\", \"Unknown\")\n description = tool.get(\"description\", \"No description available\")\n print(f\"\\n{i}. {name}\")\n print(\n f\" Description: {description[:100]}{'...' if len(description) > 100 else ''}\"\n )\n\n # Show input schema if available\n schema = tool.get(\"inputSchema\", {})\n if schema.get(\"properties\"):\n props = list(schema[\"properties\"].keys())[:3] # Show first 3 properties\n print(\n f\" Parameters: {', '.join(props)}{'...' if len(schema['properties']) > 3 else ''}\"\n )\n\n return True\n\n except Exception as e:\n print(f\"Failed to connect to MCP server: {e}\")\n print(\"\\nTroubleshooting tips:\")\n print(\"1. Make sure the MCP server is installed and accessible\")\n print(\"2. Check if the server command is correct\")\n print(\"3. Ensure you have proper authentication/credentials configured\")\n print(\"4. Try running the MCP server command directly to test it\")\n return False\n\n async def load_data(self, args) -> list[dict[str, Any]]:\n \"\"\"Load Slack messages via MCP server.\"\"\"\n print(f\"Connecting to Slack MCP server: {args.mcp_server}\")\n\n if args.workspace_name:\n print(f\"Workspace: {args.workspace_name}\")\n\n # Filter out empty strings from channels\n channels = [ch for ch in args.channels if ch.strip()] if args.channels else None\n\n if channels:\n print(f\"Channels: {', '.join(channels)}\")\n else:\n print(\"Fetching from all available channels\")\n\n concatenate = not args.no_concatenate_conversations\n print(\n f\"Processing mode: {'Concatenated conversations' if concatenate else 'Individual messages'}\"\n )\n\n try:\n reader = SlackMCPReader(\n mcp_server_command=args.mcp_server,\n workspace_name=args.workspace_name,\n concatenate_conversations=concatenate,\n max_messages_per_conversation=args.max_messages_per_channel,\n max_retries=args.max_retries,\n retry_delay=args.retry_delay,\n )\n\n texts = await reader.read_slack_data(channels=channels)\n\n if not texts:\n print(\"No messages found! This could mean:\")\n print(\"- The MCP server couldn't fetch messages\")\n print(\"- The specified channels don't exist or are empty\")\n print(\"- Authentication issues with the Slack workspace\")\n return []\n\n print(f\"Successfully loaded {len(texts)} text chunks from Slack\")\n\n # Show sample of what was loaded\n if texts:\n sample_text = texts[0][:200] + \"...\" if len(texts[0]) > 200 else texts[0]\n print(\"\\nSample content:\")\n print(\"-\" * 40)\n print(sample_text)\n print(\"-\" * 40)\n\n # Convert strings to dict format expected by base class\n return [{\"text\": text, \"metadata\": {\"source\": \"slack\"}} for text in texts]\n\n except Exception as e:\n print(f\"Error loading Slack data: {e}\")\n print(\"\\nThis might be due to:\")\n print(\"- MCP server connection issues\")\n print(\"- Authentication problems\")\n print(\"- Network connectivity issues\")\n print(\"- Incorrect channel names\")\n raise\n\n async def run(self):\n \"\"\"Main entry point with MCP connection testing.\"\"\"\n args = self.parser.parse_args()\n\n # Test connection if requested\n if args.test_connection:\n success = await self.test_mcp_connection(args)\n if not success:\n return\n print(\n \"MCP server is working! You can now run without --test-connection to start indexing.\"\n )\n return\n\n # Run the standard RAG pipeline\n await super().run()\n\n\nasync def main():\n \"\"\"Main entry point for the Slack MCP RAG application.\"\"\"\n app = SlackMCPRAG()\n await app.run()\n\n\nif __name__ == \"__main__\":\n asyncio.run(main())\n" }, { "path": "apps/twitter_data/__init__.py", "content": "# Twitter MCP data integration for LEANN\n" }, { "path": "apps/twitter_data/twitter_mcp_reader.py", "content": "#!/usr/bin/env python3\n\"\"\"\nTwitter MCP Reader for LEANN\n\nThis module provides functionality to connect to Twitter MCP servers and fetch bookmark data\nfor indexing in LEANN. It supports various Twitter MCP server implementations and provides\nflexible bookmark processing options.\n\"\"\"\n\nimport asyncio\nimport json\nimport logging\nfrom typing import Any, Optional\n\nlogger = logging.getLogger(__name__)\n\n\nclass TwitterMCPReader:\n \"\"\"\n Reader for Twitter bookmark data via MCP (Model Context Protocol) servers.\n\n This class connects to Twitter MCP servers to fetch bookmark data and convert it\n into a format suitable for LEANN indexing.\n \"\"\"\n\n def __init__(\n self,\n mcp_server_command: str,\n username: Optional[str] = None,\n include_tweet_content: bool = True,\n include_metadata: bool = True,\n max_bookmarks: int = 1000,\n ):\n \"\"\"\n Initialize the Twitter MCP Reader.\n\n Args:\n mcp_server_command: Command to start the MCP server (e.g., 'twitter-mcp-server')\n username: Optional Twitter username to filter bookmarks\n include_tweet_content: Whether to include full tweet content\n include_metadata: Whether to include tweet metadata (likes, retweets, etc.)\n max_bookmarks: Maximum number of bookmarks to fetch\n \"\"\"\n self.mcp_server_command = mcp_server_command\n self.username = username\n self.include_tweet_content = include_tweet_content\n self.include_metadata = include_metadata\n self.max_bookmarks = max_bookmarks\n self.mcp_process: asyncio.subprocess.Process | None = None\n\n async def start_mcp_server(self):\n \"\"\"Start the MCP server process.\"\"\"\n try:\n self.mcp_process = await asyncio.create_subprocess_exec(\n *self.mcp_server_command.split(),\n stdin=asyncio.subprocess.PIPE,\n stdout=asyncio.subprocess.PIPE,\n stderr=asyncio.subprocess.PIPE,\n )\n logger.info(f\"Started MCP server: {self.mcp_server_command}\")\n except Exception as e:\n logger.error(f\"Failed to start MCP server: {e}\")\n raise\n\n async def stop_mcp_server(self):\n \"\"\"Stop the MCP server process.\"\"\"\n if self.mcp_process:\n self.mcp_process.terminate()\n await self.mcp_process.wait()\n logger.info(\"Stopped MCP server\")\n\n async def send_mcp_request(self, request: dict[str, Any]) -> dict[str, Any]:\n \"\"\"Send a request to the MCP server and get response.\"\"\"\n proc = self.mcp_process\n if proc is None:\n raise RuntimeError(\"MCP server not started\")\n if proc.stdin is None or proc.stdout is None:\n raise RuntimeError(\"MCP server stdio not available\")\n\n request_json = json.dumps(request) + \"\\n\"\n proc.stdin.write(request_json.encode())\n await proc.stdin.drain()\n\n response_line = await proc.stdout.readline()\n if not response_line:\n raise RuntimeError(\"No response from MCP server\")\n\n return json.loads(response_line.decode().strip())\n\n async def initialize_mcp_connection(self):\n \"\"\"Initialize the MCP connection.\"\"\"\n init_request = {\n \"jsonrpc\": \"2.0\",\n \"id\": 1,\n \"method\": \"initialize\",\n \"params\": {\n \"protocolVersion\": \"2024-11-05\",\n \"capabilities\": {},\n \"clientInfo\": {\"name\": \"leann-twitter-reader\", \"version\": \"1.0.0\"},\n },\n }\n\n response = await self.send_mcp_request(init_request)\n if \"error\" in response:\n raise RuntimeError(f\"MCP initialization failed: {response['error']}\")\n\n logger.info(\"MCP connection initialized successfully\")\n\n async def list_available_tools(self) -> list[dict[str, Any]]:\n \"\"\"List available tools from the MCP server.\"\"\"\n list_request = {\"jsonrpc\": \"2.0\", \"id\": 2, \"method\": \"tools/list\", \"params\": {}}\n\n response = await self.send_mcp_request(list_request)\n if \"error\" in response:\n raise RuntimeError(f\"Failed to list tools: {response['error']}\")\n\n return response.get(\"result\", {}).get(\"tools\", [])\n\n async def fetch_twitter_bookmarks(self, limit: Optional[int] = None) -> list[dict[str, Any]]:\n \"\"\"\n Fetch Twitter bookmarks using MCP tools.\n\n Args:\n limit: Maximum number of bookmarks to fetch\n\n Returns:\n List of bookmark dictionaries\n \"\"\"\n tools = await self.list_available_tools()\n bookmark_tool = None\n\n # Look for a tool that can fetch bookmarks\n for tool in tools:\n tool_name = tool.get(\"name\", \"\").lower()\n if any(keyword in tool_name for keyword in [\"bookmark\", \"saved\", \"favorite\"]):\n bookmark_tool = tool\n break\n\n if not bookmark_tool:\n raise RuntimeError(\"No bookmark fetching tool found in MCP server\")\n\n # Prepare tool call parameters\n tool_params = {}\n if limit or self.max_bookmarks:\n tool_params[\"limit\"] = limit or self.max_bookmarks\n if self.username:\n tool_params[\"username\"] = self.username\n\n fetch_request = {\n \"jsonrpc\": \"2.0\",\n \"id\": 3,\n \"method\": \"tools/call\",\n \"params\": {\"name\": bookmark_tool[\"name\"], \"arguments\": tool_params},\n }\n\n response = await self.send_mcp_request(fetch_request)\n if \"error\" in response:\n raise RuntimeError(f\"Failed to fetch bookmarks: {response['error']}\")\n\n # Extract bookmarks from response\n result = response.get(\"result\", {})\n if \"content\" in result and isinstance(result[\"content\"], list):\n content = result[\"content\"][0] if result[\"content\"] else {}\n if \"text\" in content:\n try:\n bookmarks = json.loads(content[\"text\"])\n except json.JSONDecodeError:\n # If not JSON, treat as plain text\n bookmarks = [{\"text\": content[\"text\"], \"source\": \"twitter\"}]\n else:\n bookmarks = result[\"content\"]\n else:\n bookmarks = result.get(\"bookmarks\", result.get(\"tweets\", [result]))\n\n return bookmarks if isinstance(bookmarks, list) else [bookmarks]\n\n def _format_bookmark(self, bookmark: dict[str, Any]) -> str:\n \"\"\"Format a single bookmark for indexing.\"\"\"\n # Extract tweet information\n text = bookmark.get(\"text\", bookmark.get(\"content\", \"\"))\n author = bookmark.get(\n \"author\", bookmark.get(\"username\", bookmark.get(\"user\", {}).get(\"username\", \"Unknown\"))\n )\n timestamp = bookmark.get(\"created_at\", bookmark.get(\"timestamp\", \"\"))\n url = bookmark.get(\"url\", bookmark.get(\"tweet_url\", \"\"))\n\n # Extract metadata if available\n likes = bookmark.get(\"likes\", bookmark.get(\"favorite_count\", 0))\n retweets = bookmark.get(\"retweets\", bookmark.get(\"retweet_count\", 0))\n replies = bookmark.get(\"replies\", bookmark.get(\"reply_count\", 0))\n\n # Build formatted bookmark\n parts = []\n\n # Header\n parts.append(\"=== Twitter Bookmark ===\")\n\n if author:\n parts.append(f\"Author: @{author}\")\n\n if timestamp:\n # Format timestamp if it's a standard format\n try:\n import datetime\n\n if \"T\" in str(timestamp): # ISO format\n dt = datetime.datetime.fromisoformat(timestamp.replace(\"Z\", \"+00:00\"))\n formatted_time = dt.strftime(\"%Y-%m-%d %H:%M:%S\")\n else:\n formatted_time = str(timestamp)\n parts.append(f\"Date: {formatted_time}\")\n except (ValueError, TypeError):\n parts.append(f\"Date: {timestamp}\")\n\n if url:\n parts.append(f\"URL: {url}\")\n\n # Tweet content\n if text and self.include_tweet_content:\n parts.append(\"\")\n parts.append(\"Content:\")\n parts.append(text)\n\n # Metadata\n if self.include_metadata and any([likes, retweets, replies]):\n parts.append(\"\")\n parts.append(\"Engagement:\")\n if likes:\n parts.append(f\" Likes: {likes}\")\n if retweets:\n parts.append(f\" Retweets: {retweets}\")\n if replies:\n parts.append(f\" Replies: {replies}\")\n\n # Extract hashtags and mentions if available\n hashtags = bookmark.get(\"hashtags\", [])\n mentions = bookmark.get(\"mentions\", [])\n\n if hashtags or mentions:\n parts.append(\"\")\n if hashtags:\n parts.append(f\"Hashtags: {', '.join(hashtags)}\")\n if mentions:\n parts.append(f\"Mentions: {', '.join(mentions)}\")\n\n return \"\\n\".join(parts)\n\n async def read_twitter_bookmarks(self) -> list[str]:\n \"\"\"\n Read Twitter bookmark data and return formatted text chunks.\n\n Returns:\n List of formatted text chunks ready for LEANN indexing\n \"\"\"\n try:\n await self.start_mcp_server()\n await self.initialize_mcp_connection()\n\n print(f\"Fetching up to {self.max_bookmarks} bookmarks...\")\n if self.username:\n print(f\"Filtering for user: @{self.username}\")\n\n bookmarks = await self.fetch_twitter_bookmarks()\n\n if not bookmarks:\n print(\"No bookmarks found\")\n return []\n\n print(f\"Processing {len(bookmarks)} bookmarks...\")\n\n all_texts = []\n processed_count = 0\n\n for bookmark in bookmarks:\n try:\n formatted_bookmark = self._format_bookmark(bookmark)\n if formatted_bookmark.strip():\n all_texts.append(formatted_bookmark)\n processed_count += 1\n except Exception as e:\n logger.warning(f\"Failed to format bookmark: {e}\")\n continue\n\n print(f\"Successfully processed {processed_count} bookmarks\")\n return all_texts\n\n finally:\n await self.stop_mcp_server()\n\n async def __aenter__(self):\n \"\"\"Async context manager entry.\"\"\"\n await self.start_mcp_server()\n await self.initialize_mcp_connection()\n return self\n\n async def __aexit__(self, exc_type, exc_val, exc_tb):\n \"\"\"Async context manager exit.\"\"\"\n await self.stop_mcp_server()\n" }, { "path": "apps/twitter_rag.py", "content": "#!/usr/bin/env python3\n\"\"\"\nTwitter RAG Application with MCP Support\n\nThis application enables RAG (Retrieval-Augmented Generation) on Twitter bookmarks\nby connecting to Twitter MCP servers to fetch live data and index it in LEANN.\n\nUsage:\n python -m apps.twitter_rag --mcp-server \"twitter-mcp-server\" --query \"What articles did I bookmark about AI?\"\n\"\"\"\n\nimport argparse\nimport asyncio\nfrom typing import Any\n\nfrom apps.base_rag_example import BaseRAGExample\nfrom apps.twitter_data.twitter_mcp_reader import TwitterMCPReader\n\n\nclass TwitterMCPRAG(BaseRAGExample):\n \"\"\"\n RAG application for Twitter bookmarks via MCP servers.\n\n This class provides a complete RAG pipeline for Twitter bookmark data, including\n MCP server connection, data fetching, indexing, and interactive chat.\n \"\"\"\n\n def __init__(self):\n super().__init__(\n name=\"Twitter MCP RAG\",\n description=\"RAG application for Twitter bookmarks via MCP servers\",\n default_index_name=\"twitter_bookmarks\",\n )\n\n def _add_specific_arguments(self, parser: argparse.ArgumentParser):\n \"\"\"Add Twitter MCP-specific arguments.\"\"\"\n parser.add_argument(\n \"--mcp-server\",\n type=str,\n required=True,\n help=\"Command to start the Twitter MCP server (e.g., 'twitter-mcp-server' or 'npx twitter-mcp-server')\",\n )\n\n parser.add_argument(\n \"--username\", type=str, help=\"Twitter username to filter bookmarks (without @)\"\n )\n\n parser.add_argument(\n \"--max-bookmarks\",\n type=int,\n default=1000,\n help=\"Maximum number of bookmarks to fetch (default: 1000)\",\n )\n\n parser.add_argument(\n \"--no-tweet-content\",\n action=\"store_true\",\n help=\"Exclude tweet content, only include metadata\",\n )\n\n parser.add_argument(\n \"--no-metadata\",\n action=\"store_true\",\n help=\"Exclude engagement metadata (likes, retweets, etc.)\",\n )\n\n parser.add_argument(\n \"--test-connection\",\n action=\"store_true\",\n help=\"Test MCP server connection and list available tools without indexing\",\n )\n\n async def test_mcp_connection(self, args) -> bool:\n \"\"\"Test the MCP server connection and display available tools.\"\"\"\n print(f\"Testing connection to MCP server: {args.mcp_server}\")\n\n try:\n reader = TwitterMCPReader(\n mcp_server_command=args.mcp_server,\n username=args.username,\n include_tweet_content=not args.no_tweet_content,\n include_metadata=not args.no_metadata,\n max_bookmarks=args.max_bookmarks,\n )\n\n async with reader:\n tools = await reader.list_available_tools()\n\n print(\"\\n✅ Successfully connected to MCP server!\")\n print(f\"Available tools ({len(tools)}):\")\n\n for i, tool in enumerate(tools, 1):\n name = tool.get(\"name\", \"Unknown\")\n description = tool.get(\"description\", \"No description available\")\n print(f\"\\n{i}. {name}\")\n print(\n f\" Description: {description[:100]}{'...' if len(description) > 100 else ''}\"\n )\n\n # Show input schema if available\n schema = tool.get(\"inputSchema\", {})\n if schema.get(\"properties\"):\n props = list(schema[\"properties\"].keys())[:3] # Show first 3 properties\n print(\n f\" Parameters: {', '.join(props)}{'...' if len(schema['properties']) > 3 else ''}\"\n )\n\n return True\n\n except Exception as e:\n print(f\"\\n❌ Failed to connect to MCP server: {e}\")\n print(\"\\nTroubleshooting tips:\")\n print(\"1. Make sure the Twitter MCP server is installed and accessible\")\n print(\"2. Check if the server command is correct\")\n print(\"3. Ensure you have proper Twitter API credentials configured\")\n print(\"4. Verify your Twitter account has bookmarks to fetch\")\n print(\"5. Try running the MCP server command directly to test it\")\n return False\n\n async def load_data(self, args) -> list[dict[str, Any]]:\n \"\"\"Load Twitter bookmarks via MCP server.\"\"\"\n print(f\"Connecting to Twitter MCP server: {args.mcp_server}\")\n\n if args.username:\n print(f\"Username filter: @{args.username}\")\n\n print(f\"Max bookmarks: {args.max_bookmarks}\")\n print(f\"Include tweet content: {not args.no_tweet_content}\")\n print(f\"Include metadata: {not args.no_metadata}\")\n\n try:\n reader = TwitterMCPReader(\n mcp_server_command=args.mcp_server,\n username=args.username,\n include_tweet_content=not args.no_tweet_content,\n include_metadata=not args.no_metadata,\n max_bookmarks=args.max_bookmarks,\n )\n\n texts = await reader.read_twitter_bookmarks()\n\n if not texts:\n print(\"❌ No bookmarks found! This could mean:\")\n print(\"- You don't have any bookmarks on Twitter\")\n print(\"- The MCP server couldn't access your bookmarks\")\n print(\"- Authentication issues with Twitter API\")\n print(\"- The username filter didn't match any bookmarks\")\n return []\n\n print(f\"✅ Successfully loaded {len(texts)} bookmarks from Twitter\")\n\n # Show sample of what was loaded\n if texts:\n sample_text = texts[0][:300] + \"...\" if len(texts[0]) > 300 else texts[0]\n print(\"\\nSample bookmark:\")\n print(\"-\" * 50)\n print(sample_text)\n print(\"-\" * 50)\n\n # Convert strings to dict format expected by base class\n return [{\"text\": text, \"metadata\": {\"source\": \"twitter\"}} for text in texts]\n\n except Exception as e:\n print(f\"❌ Error loading Twitter bookmarks: {e}\")\n print(\"\\nThis might be due to:\")\n print(\"- MCP server connection issues\")\n print(\"- Twitter API authentication problems\")\n print(\"- Network connectivity issues\")\n print(\"- Rate limiting from Twitter API\")\n raise\n\n async def run(self):\n \"\"\"Main entry point with MCP connection testing.\"\"\"\n args = self.parser.parse_args()\n\n # Test connection if requested\n if args.test_connection:\n success = await self.test_mcp_connection(args)\n if not success:\n return\n print(\n \"\\n🎉 MCP server is working! You can now run without --test-connection to start indexing.\"\n )\n return\n\n # Run the standard RAG pipeline\n await super().run()\n\n\nasync def main():\n \"\"\"Main entry point for the Twitter MCP RAG application.\"\"\"\n app = TwitterMCPRAG()\n await app.run()\n\n\nif __name__ == \"__main__\":\n asyncio.run(main())\n" }, { "path": "apps/wechat_rag.py", "content": "\"\"\"\nWeChat History RAG example using the unified interface.\nSupports WeChat chat history export and search.\n\"\"\"\n\nimport subprocess\nimport sys\nfrom pathlib import Path\nfrom typing import Any\n\n# Add parent directory to path for imports\nsys.path.insert(0, str(Path(__file__).parent))\n\nfrom base_rag_example import BaseRAGExample\n\nfrom .history_data.wechat_history import WeChatHistoryReader\n\n\nclass WeChatRAG(BaseRAGExample):\n \"\"\"RAG example for WeChat chat history.\"\"\"\n\n def __init__(self):\n # Set default values BEFORE calling super().__init__\n self.max_items_default = -1 # Match original default\n self.embedding_model_default = (\n \"sentence-transformers/all-MiniLM-L6-v2\" # Fast 384-dim model\n )\n\n super().__init__(\n name=\"WeChat History\",\n description=\"Process and query WeChat chat history with LEANN\",\n default_index_name=\"wechat_history_magic_test_11Debug_new\",\n )\n\n def _add_specific_arguments(self, parser):\n \"\"\"Add WeChat-specific arguments.\"\"\"\n wechat_group = parser.add_argument_group(\"WeChat Parameters\")\n wechat_group.add_argument(\n \"--export-dir\",\n type=str,\n default=\"./wechat_export\",\n help=\"Directory to store WeChat exports (default: ./wechat_export)\",\n )\n wechat_group.add_argument(\n \"--force-export\",\n action=\"store_true\",\n help=\"Force re-export of WeChat data even if exports exist\",\n )\n wechat_group.add_argument(\n \"--chunk-size\", type=int, default=192, help=\"Text chunk size (default: 192)\"\n )\n wechat_group.add_argument(\n \"--chunk-overlap\", type=int, default=64, help=\"Text chunk overlap (default: 64)\"\n )\n\n def _export_wechat_data(self, export_dir: Path) -> bool:\n \"\"\"Export WeChat data using wechattweak-cli.\"\"\"\n print(\"Exporting WeChat data...\")\n\n # Check if WeChat is running\n try:\n result = subprocess.run([\"pgrep\", \"WeChat\"], capture_output=True, text=True)\n if result.returncode != 0:\n print(\"WeChat is not running. Please start WeChat first.\")\n return False\n except Exception:\n pass # pgrep might not be available on all systems\n\n # Create export directory\n export_dir.mkdir(parents=True, exist_ok=True)\n\n # Run export command\n cmd = [\"packages/wechat-exporter/wechattweak-cli\", \"export\", str(export_dir)]\n\n try:\n print(f\"Running: {' '.join(cmd)}\")\n result = subprocess.run(cmd, capture_output=True, text=True)\n\n if result.returncode == 0:\n print(\"WeChat data exported successfully!\")\n return True\n else:\n print(f\"Export failed: {result.stderr}\")\n return False\n\n except FileNotFoundError:\n print(\"\\nError: wechattweak-cli not found!\")\n print(\"Please install it first:\")\n print(\" sudo packages/wechat-exporter/wechattweak-cli install\")\n return False\n except Exception as e:\n print(f\"Export error: {e}\")\n return False\n\n async def load_data(self, args) -> list[dict[str, Any]]:\n \"\"\"Load WeChat history and convert to text chunks.\"\"\"\n # Initialize WeChat reader with export capabilities\n reader = WeChatHistoryReader()\n\n # Find existing exports or create new ones using the centralized method\n export_dirs = reader.find_or_export_wechat_data(args.export_dir)\n if not export_dirs:\n print(\"Failed to find or export WeChat data. Trying to find any existing exports...\")\n # Try to find any existing exports in common locations\n export_dirs = reader.find_wechat_export_dirs()\n if not export_dirs:\n print(\"No WeChat data found. Please ensure WeChat exports exist.\")\n return []\n\n # Load documents from all found export directories\n all_documents = []\n total_processed = 0\n\n for i, export_dir in enumerate(export_dirs):\n print(f\"\\nProcessing WeChat export {i + 1}/{len(export_dirs)}: {export_dir}\")\n\n try:\n # Apply max_items limit per export\n max_per_export = -1\n if args.max_items > 0:\n remaining = args.max_items - total_processed\n if remaining <= 0:\n break\n max_per_export = remaining\n\n documents = reader.load_data(\n wechat_export_dir=str(export_dir),\n max_count=max_per_export,\n concatenate_messages=True, # Enable message concatenation for better context\n )\n\n if documents:\n print(f\"Loaded {len(documents)} chat documents from {export_dir}\")\n all_documents.extend(documents)\n total_processed += len(documents)\n else:\n print(f\"No documents loaded from {export_dir}\")\n\n except Exception as e:\n print(f\"Error processing {export_dir}: {e}\")\n continue\n\n if not all_documents:\n print(\"No documents loaded from any source. Exiting.\")\n return []\n\n print(f\"\\nTotal loaded {len(all_documents)} chat documents from {len(export_dirs)} exports\")\n print(\"now starting to split into text chunks ... take some time\")\n\n # Convert to text chunks with contact information\n all_texts = []\n for doc in all_documents:\n # Split the document into chunks\n from llama_index.core.node_parser import SentenceSplitter\n\n text_splitter = SentenceSplitter(\n chunk_size=args.chunk_size, chunk_overlap=args.chunk_overlap\n )\n nodes = text_splitter.get_nodes_from_documents([doc])\n\n for node in nodes:\n # Add contact information to each chunk\n contact_name = doc.metadata.get(\"contact_name\", \"Unknown\")\n text = f\"[Contact] means the message is from: {contact_name}\\n\" + node.get_content()\n all_texts.append(text)\n\n print(f\"Created {len(all_texts)} text chunks from {len(all_documents)} documents\")\n return all_texts\n\n\nif __name__ == \"__main__\":\n import asyncio\n\n # Check platform\n if sys.platform != \"darwin\":\n print(\"\\n⚠️ Warning: WeChat export is only supported on macOS\")\n print(\" You can still query existing exports on other platforms\\n\")\n\n # Example queries for WeChat RAG\n print(\"\\n💬 WeChat History RAG Example\")\n print(\"=\" * 50)\n print(\"\\nExample queries you can try:\")\n print(\"- 'Show me conversations about travel plans'\")\n print(\"- 'Find group chats about weekend activities'\")\n print(\"- '我想买魔术师约翰逊的球衣,给我一些对应聊天记录?'\")\n print(\"- 'What did we discuss about the project last month?'\")\n print(\"\\nNote: WeChat must be running for export to work\\n\")\n\n rag = WeChatRAG()\n asyncio.run(rag.run())\n" }, { "path": "benchmarks/README.md", "content": "# 🧪 LEANN Benchmarks & Testing\n\nThis directory contains performance benchmarks and comprehensive tests for the LEANN system, including backend comparisons and sanity checks across different configurations.\n\n## 📁 Test Files\n\n### `diskann_vs_hnsw_speed_comparison.py`\nPerformance comparison between DiskANN and HNSW backends:\n- ✅ **Search latency** comparison with both backends using recompute\n- ✅ **Index size** and **build time** measurements\n- ✅ **Score validity** testing (ensures no -inf scores)\n- ✅ **Configurable dataset sizes** for different scales\n\n```bash\n# Quick comparison with 500 docs, 10 queries\npython benchmarks/diskann_vs_hnsw_speed_comparison.py\n\n# Large-scale comparison with 2000 docs, 20 queries\npython benchmarks/diskann_vs_hnsw_speed_comparison.py 2000 20\n```\n\n### `test_distance_functions.py`\nTests all supported distance functions across DiskANN backend:\n- ✅ **MIPS** (Maximum Inner Product Search)\n- ✅ **L2** (Euclidean Distance)\n- ✅ **Cosine** (Cosine Similarity)\n\n```bash\nuv run python tests/sanity_checks/test_distance_functions.py\n```\n\n### `test_l2_verification.py`\nSpecifically verifies that L2 distance is correctly implemented by:\n- Building indices with L2 vs Cosine metrics\n- Comparing search results and score ranges\n- Validating that different metrics produce expected score patterns\n\n```bash\nuv run python tests/sanity_checks/test_l2_verification.py\n```\n\n### `test_sanity_check.py`\nComprehensive end-to-end verification including:\n- Distance function testing\n- Embedding model compatibility\n- Search result correctness validation\n- Backend integration testing\n\n```bash\nuv run python tests/sanity_checks/test_sanity_check.py\n```\n\n## 🎯 What These Tests Verify\n\n### ✅ Distance Function Support\n- All three distance metrics (MIPS, L2, Cosine) work correctly\n- Score ranges are appropriate for each metric type\n- Different metrics can produce different rankings (as expected)\n\n### ✅ Backend Integration\n- DiskANN backend properly initializes and builds indices\n- Graph construction completes without errors\n- Search operations return valid results\n\n### ✅ Embedding Pipeline\n- Real-time embedding computation works\n- Multiple embedding models are supported\n- ZMQ server communication functions correctly\n\n### ✅ End-to-End Functionality\n- Index building → searching → result retrieval pipeline\n- Metadata preservation through the entire flow\n- Error handling and graceful degradation\n\n## 🔍 Expected Output\n\nWhen all tests pass, you should see:\n\n```\n📊 测试结果总结:\n mips : ✅ 通过\n l2 : ✅ 通过\n cosine : ✅ 通过\n\n🎉 测试完成!\n```\n\n## 🐛 Troubleshooting\n\n### Common Issues\n\n**Import Errors**: Ensure you're running from the project root:\n```bash\ncd /path/to/leann\nuv run python tests/sanity_checks/test_distance_functions.py\n```\n\n**Memory Issues**: Reduce graph complexity for resource-constrained systems:\n```python\nbuilder = LeannBuilder(\n backend_name=\"diskann\",\n graph_degree=8, # Reduced from 16\n complexity=16 # Reduced from 32\n)\n```\n\n**ZMQ Port Conflicts**: The tests use different ports to avoid conflicts, but you may need to kill existing processes:\n```bash\npkill -f \"embedding_server\"\n```\n\n## 📊 Performance Expectations\n\n### Typical Timing (3 documents, consumer hardware):\n- **Index Building**: 2-5 seconds per distance function\n- **Search Query**: 50-200ms\n- **Recompute Mode**: 5-15 seconds (higher accuracy)\n\n### Memory Usage:\n- **Index Storage**: ~1-2 MB per distance function\n- **Runtime Memory**: ~500MB (including model loading)\n\n## 🔗 Integration with CI/CD\n\nThese tests are designed to be run in automated environments:\n\n```yaml\n# GitHub Actions example\n- name: Run Sanity Checks\n run: |\n uv run python tests/sanity_checks/test_distance_functions.py\n uv run python tests/sanity_checks/test_l2_verification.py\n```\n\nThe tests are deterministic and should produce consistent results across different platforms.\n" }, { "path": "benchmarks/__init__.py", "content": "" }, { "path": "benchmarks/benchmark_embeddings.py", "content": "import time\n\nimport matplotlib.pyplot as plt\nimport mlx.core as mx\nimport numpy as np\nimport torch\nfrom mlx_lm import load\nfrom sentence_transformers import SentenceTransformer\n\n# --- Configuration ---\nMODEL_NAME_TORCH = \"Qwen/Qwen3-Embedding-0.6B\"\nMODEL_NAME_MLX = \"mlx-community/Qwen3-Embedding-0.6B-4bit-DWQ\"\nBATCH_SIZES = [1, 8, 16, 32, 64, 128]\nNUM_RUNS = 10 # Number of runs to average for each batch size\nWARMUP_RUNS = 2 # Number of warm-up runs\n\n# --- Generate Dummy Data ---\nDUMMY_SENTENCES = [\"This is a test sentence for benchmarking.\" * 5] * max(BATCH_SIZES)\n\n# --- Benchmark Functions ---b\n\n\ndef benchmark_torch(model, sentences):\n start_time = time.time()\n model.encode(sentences, convert_to_numpy=True)\n end_time = time.time()\n return (end_time - start_time) * 1000 # Return time in ms\n\n\ndef benchmark_mlx(model, tokenizer, sentences):\n start_time = time.time()\n\n # Tokenize sentences using MLX tokenizer\n tokens = []\n for sentence in sentences:\n token_ids = tokenizer.encode(sentence)\n tokens.append(token_ids)\n\n # Pad sequences to the same length\n max_len = max(len(t) for t in tokens)\n input_ids = []\n attention_mask = []\n\n for token_seq in tokens:\n # Pad sequence\n padded = token_seq + [tokenizer.eos_token_id] * (max_len - len(token_seq))\n input_ids.append(padded)\n # Create attention mask (1 for real tokens, 0 for padding)\n mask = [1] * len(token_seq) + [0] * (max_len - len(token_seq))\n attention_mask.append(mask)\n\n # Convert to MLX arrays\n input_ids = mx.array(input_ids)\n attention_mask = mx.array(attention_mask)\n\n # Get embeddings\n embeddings = model(input_ids)\n\n # Mean pooling\n mask = mx.expand_dims(attention_mask, -1)\n sum_embeddings = (embeddings * mask).sum(axis=1)\n sum_mask = mask.sum(axis=1)\n _ = sum_embeddings / sum_mask\n\n mx.eval() # Ensure computation is finished\n end_time = time.time()\n return (end_time - start_time) * 1000 # Return time in ms\n\n\n# --- Main Execution ---\ndef main():\n print(\"--- Initializing Models ---\")\n # Load PyTorch model\n print(f\"Loading PyTorch model: {MODEL_NAME_TORCH}\")\n device = \"mps\" if torch.backends.mps.is_available() else \"cpu\"\n model_torch = SentenceTransformer(MODEL_NAME_TORCH, device=device)\n print(f\"PyTorch model loaded on: {device}\")\n\n # Load MLX model\n print(f\"Loading MLX model: {MODEL_NAME_MLX}\")\n model_mlx, tokenizer_mlx = load(MODEL_NAME_MLX)\n print(\"MLX model loaded.\")\n\n # --- Warm-up ---\n print(\"\\n--- Performing Warm-up Runs ---\")\n for _ in range(WARMUP_RUNS):\n benchmark_torch(model_torch, DUMMY_SENTENCES[:1])\n benchmark_mlx(model_mlx, tokenizer_mlx, DUMMY_SENTENCES[:1])\n print(\"Warm-up complete.\")\n\n # --- Benchmarking ---\n print(\"\\n--- Starting Benchmark ---\")\n results_torch = []\n results_mlx = []\n\n for batch_size in BATCH_SIZES:\n print(f\"Benchmarking batch size: {batch_size}\")\n sentences_batch = DUMMY_SENTENCES[:batch_size]\n\n # Benchmark PyTorch\n torch_times = [benchmark_torch(model_torch, sentences_batch) for _ in range(NUM_RUNS)]\n results_torch.append(np.mean(torch_times))\n\n # Benchmark MLX\n mlx_times = [\n benchmark_mlx(model_mlx, tokenizer_mlx, sentences_batch) for _ in range(NUM_RUNS)\n ]\n results_mlx.append(np.mean(mlx_times))\n\n print(\"\\n--- Benchmark Results (Average time per batch in ms) ---\")\n print(f\"Batch Sizes: {BATCH_SIZES}\")\n print(f\"PyTorch (mps): {[f'{t:.2f}' for t in results_torch]}\")\n print(f\"MLX: {[f'{t:.2f}' for t in results_mlx]}\")\n\n # --- Plotting ---\n print(\"\\n--- Generating Plot ---\")\n plt.figure(figsize=(10, 6))\n plt.plot(\n BATCH_SIZES,\n results_torch,\n marker=\"o\",\n linestyle=\"-\",\n label=f\"PyTorch ({device})\",\n )\n plt.plot(BATCH_SIZES, results_mlx, marker=\"s\", linestyle=\"-\", label=\"MLX\")\n\n plt.title(f\"Embedding Performance: MLX vs PyTorch\\nModel: {MODEL_NAME_TORCH}\")\n plt.xlabel(\"Batch Size\")\n plt.ylabel(\"Average Time per Batch (ms)\")\n plt.xticks(BATCH_SIZES)\n plt.grid(True)\n plt.legend()\n\n # Save the plot\n output_filename = \"embedding_benchmark.png\"\n plt.savefig(output_filename)\n print(f\"Plot saved to {output_filename}\")\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "benchmarks/benchmark_no_recompute.py", "content": "import argparse\nimport os\nimport time\nfrom pathlib import Path\n\nfrom leann import LeannBuilder, LeannSearcher\n\n\ndef _meta_exists(index_path: str) -> bool:\n p = Path(index_path)\n return (p.parent / f\"{p.stem}.meta.json\").exists()\n\n\ndef ensure_index(index_path: str, backend_name: str, num_docs: int, is_recompute: bool) -> None:\n # if _meta_exists(index_path):\n # return\n kwargs = {}\n if backend_name == \"hnsw\":\n kwargs[\"is_compact\"] = is_recompute\n builder = LeannBuilder(\n backend_name=backend_name,\n embedding_model=os.getenv(\"LEANN_EMBED_MODEL\", \"facebook/contriever\"),\n embedding_mode=os.getenv(\"LEANN_EMBED_MODE\", \"sentence-transformers\"),\n graph_degree=32,\n complexity=64,\n is_recompute=is_recompute,\n num_threads=4,\n **kwargs,\n )\n for i in range(num_docs):\n builder.add_text(\n f\"This is a test document number {i}. It contains some repeated text for benchmarking.\"\n )\n builder.build_index(index_path)\n\n\ndef _bench_group(\n index_path: str,\n recompute: bool,\n query: str,\n repeats: int,\n complexity: int = 32,\n top_k: int = 10,\n) -> float:\n # Independent searcher per group; fixed port when recompute\n searcher = LeannSearcher(index_path=index_path)\n\n # Warm-up once\n _ = searcher.search(\n query,\n top_k=top_k,\n complexity=complexity,\n recompute_embeddings=recompute,\n )\n\n def _once() -> float:\n t0 = time.time()\n _ = searcher.search(\n query,\n top_k=top_k,\n complexity=complexity,\n recompute_embeddings=recompute,\n )\n return time.time() - t0\n\n if repeats <= 1:\n t = _once()\n else:\n vals = [_once() for _ in range(repeats)]\n vals.sort()\n t = vals[len(vals) // 2]\n\n searcher.cleanup()\n return t\n\n\ndef main():\n parser = argparse.ArgumentParser()\n parser.add_argument(\"--num-docs\", type=int, default=5000)\n parser.add_argument(\"--repeats\", type=int, default=3)\n parser.add_argument(\"--complexity\", type=int, default=32)\n args = parser.parse_args()\n\n base = Path.cwd() / \".leann\" / \"indexes\" / f\"bench_n{args.num_docs}\"\n base.parent.mkdir(parents=True, exist_ok=True)\n # ---------- Build HNSW variants ----------\n hnsw_r = str(base / f\"hnsw_recompute_n{args.num_docs}.leann\")\n hnsw_nr = str(base / f\"hnsw_norecompute_n{args.num_docs}.leann\")\n ensure_index(hnsw_r, \"hnsw\", args.num_docs, True)\n ensure_index(hnsw_nr, \"hnsw\", args.num_docs, False)\n\n # ---------- Build DiskANN variants ----------\n diskann_r = str(base / \"diskann_r.leann\")\n diskann_nr = str(base / \"diskann_nr.leann\")\n ensure_index(diskann_r, \"diskann\", args.num_docs, True)\n ensure_index(diskann_nr, \"diskann\", args.num_docs, False)\n\n # ---------- Helpers ----------\n def _size_for(prefix: str) -> int:\n p = Path(prefix)\n base_dir = p.parent\n stem = p.stem\n total = 0\n for f in base_dir.iterdir():\n if f.is_file() and f.name.startswith(stem):\n total += f.stat().st_size\n return total\n\n # ---------- HNSW benchmark ----------\n t_hnsw_r = _bench_group(\n hnsw_r, True, \"test document number 42\", repeats=args.repeats, complexity=args.complexity\n )\n t_hnsw_nr = _bench_group(\n hnsw_nr, False, \"test document number 42\", repeats=args.repeats, complexity=args.complexity\n )\n size_hnsw_r = _size_for(hnsw_r)\n size_hnsw_nr = _size_for(hnsw_nr)\n\n print(\"Benchmark results (HNSW):\")\n print(f\" recompute=True: search_time={t_hnsw_r:.3f}s, size={size_hnsw_r / 1024 / 1024:.1f}MB\")\n print(\n f\" recompute=False: search_time={t_hnsw_nr:.3f}s, size={size_hnsw_nr / 1024 / 1024:.1f}MB\"\n )\n print(\" Expectation: no-recompute should be faster but larger on disk.\")\n\n # ---------- DiskANN benchmark ----------\n t_diskann_r = _bench_group(\n diskann_r, True, \"DiskANN R test doc 123\", repeats=args.repeats, complexity=args.complexity\n )\n t_diskann_nr = _bench_group(\n diskann_nr,\n False,\n \"DiskANN NR test doc 123\",\n repeats=args.repeats,\n complexity=args.complexity,\n )\n size_diskann_r = _size_for(diskann_r)\n size_diskann_nr = _size_for(diskann_nr)\n\n print(\"\\nBenchmark results (DiskANN):\")\n print(f\" build(recompute=True, partition): size={size_diskann_r / 1024 / 1024:.1f}MB\")\n print(f\" build(recompute=False): size={size_diskann_nr / 1024 / 1024:.1f}MB\")\n print(f\" search recompute=True (final rerank): {t_diskann_r:.3f}s\")\n print(f\" search recompute=False (PQ only): {t_diskann_nr:.3f}s\")\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "benchmarks/bm25_diskann_baselines/README.md", "content": "BM25 vs DiskANN Baselines\n\n```bash\naws s3 sync s3://powerrag-diskann-rpj-wiki-20250824-224037-194d640c/bm25_rpj_wiki/index_en_only/ benchmarks/data/indices/bm25_index/\naws s3 sync s3://powerrag-diskann-rpj-wiki-20250824-224037-194d640c/diskann_rpj_wiki/ benchmarks/data/indices/diskann_rpj_wiki/\n```\n\n- Dataset: `benchmarks/data/queries/nq_open.jsonl` (Natural Questions)\n- Machine-specific; results measured locally with the current repo.\n\nDiskANN (NQ queries, search-only)\n- Command: `uv run --script benchmarks/bm25_diskann_baselines/run_diskann.py`\n- Settings: `recompute_embeddings=False`, embeddings precomputed (excluded from timing), batching off, caching off (`cache_mechanism=2`, `num_nodes_to_cache=0`)\n- Result: avg 0.011093 s/query, QPS 90.15 (p50 0.010731 s, p95 0.015000 s)\n\nBM25\n- Command: `uv run --script benchmarks/bm25_diskann_baselines/run_bm25.py`\n- Settings: `k=10`, `k1=0.9`, `b=0.4`, queries=100\n- Result: avg 0.028589 s/query, QPS 34.97 (p50 0.026060 s, p90 0.043695 s, p95 0.053260 s, p99 0.055257 s)\n\nNotes\n- DiskANN measures search-only latency on real NQ queries (embeddings computed beforehand and excluded from timing).\n- Use `benchmarks/bm25_diskann_baselines/run_diskann.py` for DiskANN; `benchmarks/bm25_diskann_baselines/run_bm25.py` for BM25.\n" }, { "path": "benchmarks/bm25_diskann_baselines/run_bm25.py", "content": "# /// script\n# dependencies = [\n# \"pyserini\"\n# ]\n# ///\n# sudo pacman -S jdk21-openjdk\n# export JAVA_HOME=/usr/lib/jvm/java-21-openjdk\n# sudo archlinux-java status\n# sudo archlinux-java set java-21-openjdk\n# set -Ux JAVA_HOME /usr/lib/jvm/java-21-openjdk\n# fish_add_path --global $JAVA_HOME/bin\n# set -Ux LD_LIBRARY_PATH $JAVA_HOME/lib/server $LD_LIBRARY_PATH\n# which javac # Should be /usr/lib/jvm/java-21-openjdk/bin/javac\n\nimport argparse\nimport json\nimport os\nimport sys\nimport time\nfrom statistics import mean\n\n\ndef load_queries(path: str, limit: int | None) -> list[str]:\n queries: list[str] = []\n # Try JSONL with a 'query' or 'text' field; fallback to plain text (one query per line)\n _, ext = os.path.splitext(path)\n if ext.lower() in {\".jsonl\", \".json\"}:\n with open(path, encoding=\"utf-8\") as f:\n for line in f:\n line = line.strip()\n if not line:\n continue\n try:\n obj = json.loads(line)\n except json.JSONDecodeError:\n # Not strict JSONL? treat the whole line as the query\n queries.append(line)\n continue\n q = obj.get(\"query\") or obj.get(\"text\") or obj.get(\"question\")\n if q:\n queries.append(str(q))\n else:\n with open(path, encoding=\"utf-8\") as f:\n for line in f:\n s = line.strip()\n if s:\n queries.append(s)\n\n if limit is not None and limit > 0:\n queries = queries[:limit]\n return queries\n\n\ndef percentile(values: list[float], p: float) -> float:\n if not values:\n return 0.0\n s = sorted(values)\n k = (len(s) - 1) * (p / 100.0)\n f = int(k)\n c = min(f + 1, len(s) - 1)\n if f == c:\n return s[f]\n return s[f] + (s[c] - s[f]) * (k - f)\n\n\ndef main():\n ap = argparse.ArgumentParser(description=\"Standalone BM25 latency benchmark (Pyserini)\")\n ap.add_argument(\n \"--bm25-index\",\n default=\"benchmarks/data/indices/bm25_index\",\n help=\"Path to Pyserini Lucene index directory\",\n )\n ap.add_argument(\n \"--queries\",\n default=\"benchmarks/data/queries/nq_open.jsonl\",\n help=\"Path to queries file (JSONL with 'query'/'text' or plain txt one-per-line)\",\n )\n ap.add_argument(\"--k\", type=int, default=10, help=\"Top-k to retrieve (default: 10)\")\n ap.add_argument(\"--k1\", type=float, default=0.9, help=\"BM25 k1 (default: 0.9)\")\n ap.add_argument(\"--b\", type=float, default=0.4, help=\"BM25 b (default: 0.4)\")\n ap.add_argument(\"--limit\", type=int, default=100, help=\"Max queries to run (default: 100)\")\n ap.add_argument(\n \"--warmup\", type=int, default=5, help=\"Warmup queries not counted in latency (default: 5)\"\n )\n ap.add_argument(\n \"--fetch-docs\", action=\"store_true\", help=\"Also fetch doc contents (slower; default: off)\"\n )\n ap.add_argument(\"--report\", type=str, default=None, help=\"Optional JSON report path\")\n args = ap.parse_args()\n\n try:\n from pyserini.search.lucene import LuceneSearcher\n except Exception:\n print(\"Pyserini not found. Install with: pip install pyserini\", file=sys.stderr)\n raise\n\n if not os.path.isdir(args.bm25_index):\n print(f\"Index directory not found: {args.bm25_index}\", file=sys.stderr)\n sys.exit(1)\n\n queries = load_queries(args.queries, args.limit)\n if not queries:\n print(\"No queries loaded.\", file=sys.stderr)\n sys.exit(1)\n\n print(f\"Loaded {len(queries)} queries from {args.queries}\")\n print(f\"Opening BM25 index: {args.bm25_index}\")\n searcher = LuceneSearcher(args.bm25_index)\n # Some builds of pyserini require explicit set_bm25; others ignore\n try:\n searcher.set_bm25(k1=args.k1, b=args.b)\n except Exception:\n pass\n\n latencies: list[float] = []\n total_searches = 0\n\n # Warmup\n for i in range(min(args.warmup, len(queries))):\n _ = searcher.search(queries[i], k=args.k)\n\n t0 = time.time()\n for i, q in enumerate(queries):\n t1 = time.time()\n hits = searcher.search(q, k=args.k)\n t2 = time.time()\n latencies.append(t2 - t1)\n total_searches += 1\n\n if args.fetch_docs:\n # Optional doc fetch to include I/O time\n for h in hits:\n try:\n _ = searcher.doc(h.docid)\n except Exception:\n pass\n\n if (i + 1) % 50 == 0:\n print(f\"Processed {i + 1}/{len(queries)} queries\")\n\n t1 = time.time()\n total_time = t1 - t0\n\n if latencies:\n avg = mean(latencies)\n p50 = percentile(latencies, 50)\n p90 = percentile(latencies, 90)\n p95 = percentile(latencies, 95)\n p99 = percentile(latencies, 99)\n qps = total_searches / total_time if total_time > 0 else 0.0\n else:\n avg = p50 = p90 = p95 = p99 = qps = 0.0\n\n print(\"BM25 Latency Report\")\n print(f\" queries: {total_searches}\")\n print(f\" k: {args.k}, k1: {args.k1}, b: {args.b}\")\n print(f\" avg per query: {avg:.6f} s\")\n print(f\" p50/p90/p95/p99: {p50:.6f}/{p90:.6f}/{p95:.6f}/{p99:.6f} s\")\n print(f\" total time: {total_time:.3f} s, qps: {qps:.2f}\")\n\n if args.report:\n payload = {\n \"queries\": total_searches,\n \"k\": args.k,\n \"k1\": args.k1,\n \"b\": args.b,\n \"avg_s\": avg,\n \"p50_s\": p50,\n \"p90_s\": p90,\n \"p95_s\": p95,\n \"p99_s\": p99,\n \"total_time_s\": total_time,\n \"qps\": qps,\n \"index_dir\": os.path.abspath(args.bm25_index),\n \"fetch_docs\": bool(args.fetch_docs),\n }\n with open(args.report, \"w\", encoding=\"utf-8\") as f:\n json.dump(payload, f, indent=2)\n print(f\"Saved report to {args.report}\")\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "benchmarks/bm25_diskann_baselines/run_diskann.py", "content": "# /// script\n# dependencies = [\n# \"leann-backend-diskann\"\n# ]\n# ///\n\nimport argparse\nimport json\nimport time\nfrom pathlib import Path\n\nimport numpy as np\n\n\ndef load_queries(path: Path, limit: int | None) -> list[str]:\n out: list[str] = []\n with open(path, encoding=\"utf-8\") as f:\n for line in f:\n obj = json.loads(line)\n out.append(obj[\"query\"])\n if limit and len(out) >= limit:\n break\n return out\n\n\ndef main() -> None:\n ap = argparse.ArgumentParser(\n description=\"DiskANN baseline on real NQ queries (search-only timing)\"\n )\n ap.add_argument(\n \"--index-dir\",\n default=\"benchmarks/data/indices/diskann_rpj_wiki\",\n help=\"Directory containing DiskANN files\",\n )\n ap.add_argument(\"--index-prefix\", default=\"ann\")\n ap.add_argument(\"--queries-file\", default=\"benchmarks/data/queries/nq_open.jsonl\")\n ap.add_argument(\"--num-queries\", type=int, default=200)\n ap.add_argument(\"--top-k\", type=int, default=10)\n ap.add_argument(\"--complexity\", type=int, default=62)\n ap.add_argument(\"--threads\", type=int, default=1)\n ap.add_argument(\"--beam-width\", type=int, default=1)\n ap.add_argument(\"--cache-mechanism\", type=int, default=2)\n ap.add_argument(\"--num-nodes-to-cache\", type=int, default=0)\n args = ap.parse_args()\n\n index_dir = Path(args.index_dir).resolve()\n if not index_dir.is_dir():\n raise SystemExit(f\"Index dir not found: {index_dir}\")\n\n qpath = Path(args.queries_file).resolve()\n if not qpath.exists():\n raise SystemExit(f\"Queries file not found: {qpath}\")\n\n queries = load_queries(qpath, args.num_queries)\n print(f\"Loaded {len(queries)} queries from {qpath}\")\n\n # Compute embeddings once (exclude from timing)\n from leann.api import compute_embeddings as _compute\n\n embs = _compute(\n queries,\n model_name=\"facebook/contriever-msmarco\",\n mode=\"sentence-transformers\",\n use_server=False,\n ).astype(np.float32)\n if embs.ndim != 2:\n raise SystemExit(\"Embedding compute failed or returned wrong shape\")\n\n # Build searcher\n from leann_backend_diskann.diskann_backend import DiskannSearcher as _DiskannSearcher\n\n index_prefix_path = str(index_dir / args.index_prefix)\n searcher = _DiskannSearcher(\n index_prefix_path,\n num_threads=int(args.threads),\n cache_mechanism=int(args.cache_mechanism),\n num_nodes_to_cache=int(args.num_nodes_to_cache),\n )\n\n # Warmup (not timed)\n _ = searcher.search(\n embs[0:1],\n top_k=args.top_k,\n complexity=args.complexity,\n beam_width=args.beam_width,\n prune_ratio=0.0,\n recompute_embeddings=False,\n batch_recompute=False,\n dedup_node_dis=False,\n )\n\n # Timed loop\n times: list[float] = []\n for i in range(embs.shape[0]):\n t0 = time.time()\n _ = searcher.search(\n embs[i : i + 1],\n top_k=args.top_k,\n complexity=args.complexity,\n beam_width=args.beam_width,\n prune_ratio=0.0,\n recompute_embeddings=False,\n batch_recompute=False,\n dedup_node_dis=False,\n )\n times.append(time.time() - t0)\n\n times_sorted = sorted(times)\n avg = float(sum(times) / len(times))\n p50 = times_sorted[len(times) // 2]\n p95 = times_sorted[max(0, int(len(times) * 0.95) - 1)]\n\n print(\"\\nDiskANN (NQ, search-only) Report\")\n print(f\" queries: {len(times)}\")\n print(\n f\" k: {args.top_k}, complexity: {args.complexity}, beam_width: {args.beam_width}, threads: {args.threads}\"\n )\n print(f\" avg per query: {avg:.6f} s\")\n print(f\" p50/p95: {p50:.6f}/{p95:.6f} s\")\n print(f\" QPS: {1.0 / avg:.2f}\")\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "benchmarks/compare_faiss_vs_leann.py", "content": "#!/usr/bin/env python3\n\"\"\"\nMemory comparison between Faiss HNSW and LEANN HNSW backend\n\"\"\"\n\nimport gc\nimport logging\nimport os\nimport subprocess\nimport sys\nimport time\nfrom pathlib import Path\n\nimport psutil\nfrom llama_index.core.node_parser import SentenceSplitter\n\n# Setup logging\nlogging.basicConfig(stream=sys.stdout, level=logging.INFO)\nlogger = logging.getLogger(__name__)\n\n\ndef get_memory_usage():\n \"\"\"Get current memory usage in MB\"\"\"\n process = psutil.Process()\n return process.memory_info().rss / 1024 / 1024\n\n\ndef print_memory_stats(stage: str, start_mem: float):\n \"\"\"Print memory statistics\"\"\"\n current_mem = get_memory_usage()\n diff = current_mem - start_mem\n print(f\"[{stage}] Memory: {current_mem:.1f} MB (+{diff:.1f} MB)\")\n return current_mem\n\n\nclass MemoryTracker:\n def __init__(self, name: str):\n self.name = name\n self.start_mem = get_memory_usage()\n self.stages = []\n\n def checkpoint(self, stage: str):\n current_mem = print_memory_stats(f\"{self.name} - {stage}\", self.start_mem)\n self.stages.append((stage, current_mem))\n return current_mem\n\n def summary(self):\n print(f\"\\n=== {self.name} Memory Summary ===\")\n for stage, mem in self.stages:\n print(f\"{stage}: {mem:.1f} MB\")\n peak_mem = max(mem for _, mem in self.stages)\n print(f\"Peak Memory: {peak_mem:.1f} MB\")\n print(f\"Total Memory Increase: {peak_mem - self.start_mem:.1f} MB\")\n return peak_mem\n\n\ndef test_faiss_hnsw():\n \"\"\"Test Faiss HNSW Vector Store in subprocess\"\"\"\n print(\"\\n\" + \"=\" * 50)\n print(\"TESTING FAISS HNSW VECTOR STORE\")\n print(\"=\" * 50)\n\n try:\n result = subprocess.run(\n [sys.executable, \"benchmarks/faiss_only.py\"],\n capture_output=True,\n text=True,\n timeout=300,\n )\n\n print(result.stdout)\n if result.stderr:\n print(\"Stderr:\", result.stderr)\n\n if result.returncode != 0:\n return {\n \"peak_memory\": float(\"inf\"),\n \"error\": f\"Process failed with code {result.returncode}\",\n }\n\n # Parse peak memory from output\n lines = result.stdout.split(\"\\n\")\n peak_memory = 0.0\n\n for line in lines:\n if \"Peak Memory:\" in line:\n peak_memory = float(line.split(\"Peak Memory:\")[1].split(\"MB\")[0].strip())\n\n return {\"peak_memory\": peak_memory}\n\n except Exception as e:\n return {\n \"peak_memory\": float(\"inf\"),\n \"error\": str(e),\n }\n\n\ndef test_leann_hnsw():\n \"\"\"Test LEANN HNSW Search Memory (load existing index)\"\"\"\n print(\"\\n\" + \"=\" * 50)\n print(\"TESTING LEANN HNSW SEARCH MEMORY\")\n print(\"=\" * 50)\n\n tracker = MemoryTracker(\"LEANN HNSW Search\")\n\n # Import and setup\n tracker.checkpoint(\"Initial\")\n\n from leann.api import LeannSearcher\n\n tracker.checkpoint(\"After imports\")\n\n from leann.api import LeannBuilder\n from llama_index.core import SimpleDirectoryReader\n\n # Load and parse documents\n documents = SimpleDirectoryReader(\n \"data\",\n recursive=True,\n encoding=\"utf-8\",\n required_exts=[\".pdf\", \".txt\", \".md\"],\n ).load_data()\n\n tracker.checkpoint(\"After document loading\")\n\n # Parse into chunks\n node_parser = SentenceSplitter(\n chunk_size=256, chunk_overlap=20, separator=\" \", paragraph_separator=\"\\n\\n\"\n )\n\n all_texts = []\n for doc in documents:\n nodes = node_parser.get_nodes_from_documents([doc])\n for node in nodes:\n all_texts.append(node.get_content())\n print(f\"Total number of chunks: {len(all_texts)}\")\n\n tracker.checkpoint(\"After text chunking\")\n\n # Build LEANN index\n INDEX_DIR = Path(\"./test_leann_comparison\")\n INDEX_PATH = str(INDEX_DIR / \"comparison.leann\")\n\n # Check if index already exists\n if os.path.exists(INDEX_PATH + \".meta.json\"):\n print(\"Loading existing LEANN HNSW index...\")\n tracker.checkpoint(\"After loading existing index\")\n else:\n print(\"Building new LEANN HNSW index...\")\n # Clean up previous index\n import shutil\n\n if INDEX_DIR.exists():\n shutil.rmtree(INDEX_DIR)\n\n builder = LeannBuilder(\n backend_name=\"hnsw\",\n embedding_model=\"facebook/contriever\",\n graph_degree=32,\n complexity=64,\n is_compact=True,\n is_recompute=True,\n num_threads=1,\n )\n\n tracker.checkpoint(\"After builder setup\")\n\n print(\"Building LEANN HNSW index...\")\n\n for chunk_text in all_texts:\n builder.add_text(chunk_text)\n\n builder.build_index(INDEX_PATH)\n del builder\n gc.collect()\n\n tracker.checkpoint(\"After index building\")\n\n # Find existing LEANN index\n index_paths = [\n \"./test_leann_comparison/comparison.leann\",\n ]\n index_path = None\n for path in index_paths:\n if os.path.exists(path + \".meta.json\"):\n index_path = path\n break\n\n if not index_path:\n print(\"❌ LEANN index not found. Please build it first\")\n return {\"peak_memory\": float(\"inf\"), \"error\": \"Index not found\"}\n\n # Measure runtime memory overhead\n print(\"\\nMeasuring runtime memory overhead...\")\n runtime_start_mem = get_memory_usage()\n print(f\"Before load memory: {runtime_start_mem:.1f} MB\")\n tracker.checkpoint(\"Before load memory\")\n\n # Load searcher\n searcher = LeannSearcher(index_path)\n tracker.checkpoint(\"After searcher loading\")\n\n print(\"Running search queries...\")\n queries = [\n \"什么是盘古大模型以及盘古开发过程中遇到了什么阴暗面,任务令一般在什么城市颁发\",\n \"What is LEANN and how does it work?\",\n \"华为诺亚方舟实验室的主要研究内容\",\n ]\n\n for i, query in enumerate(queries):\n start_time = time.time()\n # Use same parameters as Faiss: top_k=20, ef=120 (complexity parameter)\n _ = searcher.search(query, top_k=20, ef=120)\n query_time = time.time() - start_time\n print(f\"Query {i + 1} time: {query_time:.3f}s\")\n tracker.checkpoint(f\"After query {i + 1}\")\n\n runtime_end_mem = get_memory_usage()\n runtime_overhead = runtime_end_mem - runtime_start_mem\n\n peak_memory = tracker.summary()\n print(f\"Runtime Memory Overhead: {runtime_overhead:.1f} MB\")\n\n # Get storage size before cleanup\n storage_size = 0\n INDEX_DIR = Path(index_path).parent\n if INDEX_DIR.exists():\n total_size = 0\n for dirpath, _, filenames in os.walk(str(INDEX_DIR)):\n for filename in filenames:\n # Only count actual index files, skip text data and backups\n if filename.endswith((\".old\", \".tmp\", \".bak\", \".jsonl\", \".json\")):\n continue\n # Count .index, .idx, .map files (actual index structures)\n if filename.endswith((\".index\", \".idx\", \".map\")):\n filepath = os.path.join(dirpath, filename)\n total_size += os.path.getsize(filepath)\n storage_size = total_size / (1024 * 1024) # Convert to MB\n\n # Clean up\n del searcher\n gc.collect()\n\n return {\n \"peak_memory\": peak_memory,\n \"storage_size\": storage_size,\n }\n\n\ndef main():\n \"\"\"Run comparison tests\"\"\"\n print(\"Storage + Search Memory Comparison: Faiss HNSW vs LEANN HNSW\")\n print(\"=\" * 60)\n\n # Test Faiss HNSW\n faiss_results = test_faiss_hnsw()\n\n # Force garbage collection\n gc.collect()\n time.sleep(2)\n\n # Test LEANN HNSW\n leann_results = test_leann_hnsw()\n\n # Final comparison\n print(\"\\n\" + \"=\" * 60)\n print(\"STORAGE + SEARCH MEMORY COMPARISON\")\n print(\"=\" * 60)\n\n # Get storage sizes\n faiss_storage_size = 0\n leann_storage_size = leann_results.get(\"storage_size\", 0)\n\n # Get Faiss storage size using Python\n if os.path.exists(\"./storage_faiss\"):\n total_size = 0\n for dirpath, _, filenames in os.walk(\"./storage_faiss\"):\n for filename in filenames:\n filepath = os.path.join(dirpath, filename)\n total_size += os.path.getsize(filepath)\n faiss_storage_size = total_size / (1024 * 1024) # Convert to MB\n\n print(\"Faiss HNSW:\")\n if \"error\" in faiss_results:\n print(f\" ❌ Failed: {faiss_results['error']}\")\n else:\n print(f\" Search Memory: {faiss_results['peak_memory']:.1f} MB\")\n print(f\" Storage Size: {faiss_storage_size:.1f} MB\")\n\n print(\"\\nLEANN HNSW:\")\n if \"error\" in leann_results:\n print(f\" ❌ Failed: {leann_results['error']}\")\n else:\n print(f\" Search Memory: {leann_results['peak_memory']:.1f} MB\")\n print(f\" Storage Size: {leann_storage_size:.1f} MB\")\n\n # Calculate improvements only if both tests succeeded\n if \"error\" not in faiss_results and \"error\" not in leann_results:\n memory_ratio = faiss_results[\"peak_memory\"] / leann_results[\"peak_memory\"]\n\n print(\"\\nLEANN vs Faiss Performance:\")\n memory_saving = faiss_results[\"peak_memory\"] - leann_results[\"peak_memory\"]\n print(f\" Search Memory: {memory_ratio:.1f}x less ({memory_saving:.1f} MB saved)\")\n\n # Storage comparison\n if leann_storage_size > faiss_storage_size:\n storage_ratio = leann_storage_size / faiss_storage_size\n print(f\" Storage Size: {storage_ratio:.1f}x larger (LEANN uses more storage)\")\n elif faiss_storage_size > leann_storage_size:\n storage_ratio = faiss_storage_size / leann_storage_size\n print(f\" Storage Size: {storage_ratio:.1f}x smaller (LEANN uses less storage)\")\n else:\n print(\" Storage Size: similar\")\n else:\n if \"error\" not in leann_results:\n print(\"\\n✅ LEANN HNSW completed successfully!\")\n print(f\"📊 Search Memory: {leann_results['peak_memory']:.1f} MB\")\n print(f\"📊 Storage Size: {leann_storage_size:.1f} MB\")\n if \"error\" not in faiss_results:\n print(\"\\n✅ Faiss HNSW completed successfully!\")\n print(f\"📊 Search Memory: {faiss_results['peak_memory']:.1f} MB\")\n print(f\"📊 Storage Size: {faiss_storage_size:.1f} MB\")\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "benchmarks/diskann_vs_hnsw_speed_comparison.py", "content": "#!/usr/bin/env python3\n\"\"\"\nDiskANN vs HNSW Search Performance Comparison\n\nThis benchmark compares search performance between DiskANN and HNSW backends:\n- DiskANN: With graph partitioning enabled (is_recompute=True)\n- HNSW: With recompute enabled (is_recompute=True)\n- Tests performance across different dataset sizes\n- Measures search latency, recall, and index size\n\"\"\"\n\nimport gc\nimport multiprocessing as mp\nimport tempfile\nimport time\nfrom pathlib import Path\nfrom typing import Any\n\nimport numpy as np\n\n# Prefer 'fork' start method to avoid POSIX semaphore leaks on macOS\ntry:\n mp.set_start_method(\"fork\", force=True)\nexcept Exception:\n pass\n\n\ndef create_test_texts(n_docs: int) -> list[str]:\n \"\"\"Create synthetic test documents for benchmarking.\"\"\"\n np.random.seed(42)\n topics = [\n \"machine learning and artificial intelligence\",\n \"natural language processing and text analysis\",\n \"computer vision and image recognition\",\n \"data science and statistical analysis\",\n \"deep learning and neural networks\",\n \"information retrieval and search engines\",\n \"database systems and data management\",\n \"software engineering and programming\",\n \"cybersecurity and network protection\",\n \"cloud computing and distributed systems\",\n ]\n\n texts = []\n for i in range(n_docs):\n topic = topics[i % len(topics)]\n variation = np.random.randint(1, 100)\n text = (\n f\"This is document {i} about {topic}. Content variation {variation}. \"\n f\"Additional information about {topic} with details and examples. \"\n f\"Technical discussion of {topic} including implementation aspects.\"\n )\n texts.append(text)\n\n return texts\n\n\ndef benchmark_backend(\n backend_name: str, texts: list[str], test_queries: list[str], backend_kwargs: dict[str, Any]\n) -> dict[str, float]:\n \"\"\"Benchmark a specific backend with the given configuration.\"\"\"\n from leann.api import LeannBuilder, LeannSearcher\n\n print(f\"\\n🔧 Testing {backend_name.upper()} backend...\")\n\n with tempfile.TemporaryDirectory() as temp_dir:\n index_path = str(Path(temp_dir) / f\"benchmark_{backend_name}.leann\")\n\n # Build index\n print(f\"📦 Building {backend_name} index with {len(texts)} documents...\")\n start_time = time.time()\n\n builder = LeannBuilder(\n backend_name=backend_name,\n embedding_model=\"facebook/contriever\",\n embedding_mode=\"sentence-transformers\",\n **backend_kwargs,\n )\n\n for text in texts:\n builder.add_text(text)\n\n builder.build_index(index_path)\n build_time = time.time() - start_time\n\n # Measure index size\n index_dir = Path(index_path).parent\n index_files = list(index_dir.glob(f\"{Path(index_path).stem}.*\"))\n total_size = sum(f.stat().st_size for f in index_files if f.is_file())\n size_mb = total_size / (1024 * 1024)\n\n print(f\" ✅ Build completed in {build_time:.2f}s, index size: {size_mb:.1f}MB\")\n\n # Search benchmark\n print(\"🔍 Running search benchmark...\")\n searcher = LeannSearcher(index_path)\n\n search_times = []\n all_results = []\n\n for query in test_queries:\n start_time = time.time()\n results = searcher.search(query, top_k=5)\n search_time = time.time() - start_time\n search_times.append(search_time)\n all_results.append(results)\n\n avg_search_time = np.mean(search_times) * 1000 # Convert to ms\n print(f\" ✅ Average search time: {avg_search_time:.1f}ms\")\n\n # Check for valid scores (detect -inf issues)\n all_scores = [\n result.score\n for results in all_results\n for result in results\n if result.score is not None\n ]\n valid_scores = [\n score for score in all_scores if score != float(\"-inf\") and score != float(\"inf\")\n ]\n score_validity_rate = len(valid_scores) / len(all_scores) if all_scores else 0\n\n # Clean up (ensure embedding server shutdown and object GC)\n try:\n if hasattr(searcher, \"cleanup\"):\n searcher.cleanup()\n del searcher\n del builder\n gc.collect()\n except Exception as e:\n print(f\"⚠️ Warning: Resource cleanup error: {e}\")\n\n return {\n \"build_time\": build_time,\n \"avg_search_time_ms\": avg_search_time,\n \"index_size_mb\": size_mb,\n \"score_validity_rate\": score_validity_rate,\n }\n\n\ndef run_comparison(n_docs: int = 500, n_queries: int = 10):\n \"\"\"Run performance comparison between DiskANN and HNSW.\"\"\"\n print(\"🚀 Starting DiskANN vs HNSW Performance Comparison\")\n print(f\"📊 Dataset: {n_docs} documents, {n_queries} test queries\")\n\n # Create test data\n texts = create_test_texts(n_docs)\n test_queries = [\n \"machine learning algorithms\",\n \"natural language processing\",\n \"computer vision techniques\",\n \"data analysis methods\",\n \"neural network architectures\",\n \"database query optimization\",\n \"software development practices\",\n \"security vulnerabilities\",\n \"cloud infrastructure\",\n \"distributed computing\",\n ][:n_queries]\n\n # HNSW benchmark\n hnsw_results = benchmark_backend(\n backend_name=\"hnsw\",\n texts=texts,\n test_queries=test_queries,\n backend_kwargs={\n \"is_recompute\": True, # Enable recompute for fair comparison\n \"M\": 16,\n \"efConstruction\": 200,\n },\n )\n\n # DiskANN benchmark\n diskann_results = benchmark_backend(\n backend_name=\"diskann\",\n texts=texts,\n test_queries=test_queries,\n backend_kwargs={\n \"is_recompute\": True, # Enable graph partitioning\n \"num_neighbors\": 32,\n \"search_list_size\": 50,\n },\n )\n\n # Performance comparison\n print(\"\\n📈 Performance Comparison Results\")\n print(f\"{'=' * 60}\")\n print(f\"{'Metric':<25} {'HNSW':<15} {'DiskANN':<15} {'Speedup':<10}\")\n print(f\"{'-' * 60}\")\n\n # Build time comparison\n build_speedup = hnsw_results[\"build_time\"] / diskann_results[\"build_time\"]\n print(\n f\"{'Build Time (s)':<25} {hnsw_results['build_time']:<15.2f} {diskann_results['build_time']:<15.2f} {build_speedup:<10.2f}x\"\n )\n\n # Search time comparison\n search_speedup = hnsw_results[\"avg_search_time_ms\"] / diskann_results[\"avg_search_time_ms\"]\n print(\n f\"{'Search Time (ms)':<25} {hnsw_results['avg_search_time_ms']:<15.1f} {diskann_results['avg_search_time_ms']:<15.1f} {search_speedup:<10.2f}x\"\n )\n\n # Index size comparison\n size_ratio = diskann_results[\"index_size_mb\"] / hnsw_results[\"index_size_mb\"]\n print(\n f\"{'Index Size (MB)':<25} {hnsw_results['index_size_mb']:<15.1f} {diskann_results['index_size_mb']:<15.1f} {size_ratio:<10.2f}x\"\n )\n\n # Score validity\n print(\n f\"{'Score Validity (%)':<25} {hnsw_results['score_validity_rate'] * 100:<15.1f} {diskann_results['score_validity_rate'] * 100:<15.1f}\"\n )\n\n print(f\"{'=' * 60}\")\n print(\"\\n🎯 Summary:\")\n if search_speedup > 1:\n print(f\" DiskANN is {search_speedup:.2f}x faster than HNSW for search\")\n else:\n print(f\" HNSW is {1 / search_speedup:.2f}x faster than DiskANN for search\")\n\n if size_ratio > 1:\n print(f\" DiskANN uses {size_ratio:.2f}x more storage than HNSW\")\n else:\n print(f\" DiskANN uses {1 / size_ratio:.2f}x less storage than HNSW\")\n\n print(\n f\" Both backends achieved {min(hnsw_results['score_validity_rate'], diskann_results['score_validity_rate']) * 100:.1f}% score validity\"\n )\n\n\nif __name__ == \"__main__\":\n import sys\n\n try:\n # Handle help request\n if len(sys.argv) > 1 and sys.argv[1] in [\"-h\", \"--help\", \"help\"]:\n print(\"DiskANN vs HNSW Performance Comparison\")\n print(\"=\" * 50)\n print(f\"Usage: python {sys.argv[0]} [n_docs] [n_queries]\")\n print()\n print(\"Arguments:\")\n print(\" n_docs Number of documents to index (default: 500)\")\n print(\" n_queries Number of test queries to run (default: 10)\")\n print()\n print(\"Examples:\")\n print(\" python benchmarks/diskann_vs_hnsw_speed_comparison.py\")\n print(\" python benchmarks/diskann_vs_hnsw_speed_comparison.py 1000\")\n print(\" python benchmarks/diskann_vs_hnsw_speed_comparison.py 2000 20\")\n sys.exit(0)\n\n # Parse command line arguments\n n_docs = int(sys.argv[1]) if len(sys.argv) > 1 else 500\n n_queries = int(sys.argv[2]) if len(sys.argv) > 2 else 10\n\n print(\"DiskANN vs HNSW Performance Comparison\")\n print(\"=\" * 50)\n print(f\"Dataset: {n_docs} documents, {n_queries} queries\")\n print()\n\n run_comparison(n_docs=n_docs, n_queries=n_queries)\n\n except KeyboardInterrupt:\n print(\"\\n⚠️ Benchmark interrupted by user\")\n sys.exit(130)\n except Exception as e:\n print(f\"\\n❌ Benchmark failed: {e}\")\n sys.exit(1)\n finally:\n # Ensure clean exit (forceful to prevent rare hangs from atexit/threads)\n try:\n gc.collect()\n print(\"\\n🧹 Cleanup completed\")\n # Flush stdio to ensure message is visible before hard-exit\n try:\n import sys as _sys\n\n _sys.stdout.flush()\n _sys.stderr.flush()\n except Exception:\n pass\n except Exception:\n pass\n # Use os._exit to bypass atexit handlers that may hang in rare cases\n import os as _os\n\n _os._exit(0)\n" }, { "path": "benchmarks/enron_emails/README.md", "content": "# Enron Emails Benchmark\n\nA comprehensive RAG benchmark for evaluating LEANN search and generation on the Enron email corpus. It mirrors the structure and CLI of the existing FinanceBench and LAION benches, using stage-based evaluation with Recall@3 and generation timing.\n\n- Dataset: Enron email CSV (e.g., Kaggle wcukierski/enron-email-dataset) for passages\n- Queries: corbt/enron_emails_sample_questions (filtered for realistic questions)\n- Metrics: Recall@3 vs FAISS Flat baseline + Generation evaluation with Qwen3-8B\n\n## Layout\n\nbenchmarks/enron_emails/\n- setup_enron_emails.py: Prepare passages, build LEANN index, build FAISS baseline\n- evaluate_enron_emails.py: Evaluate retrieval recall (Stages 2-5) + generation with Qwen3-8B\n- data/: Generated passages, queries, embeddings-related files\n- baseline/: FAISS Flat baseline files\n- llm_utils.py: LLM utilities for Qwen3-8B generation (in parent directory)\n\n## Quickstart\n\n1) Prepare the data and index\n\ncd benchmarks/enron_emails\npython setup_enron_emails.py --data-dir data\n\nNotes:\n- If `--emails-csv` is omitted, the script attempts to download from Kaggle dataset `wcukierski/enron-email-dataset` using Kaggle API (requires `KAGGLE_USERNAME` and `KAGGLE_KEY`).\n Alternatively, pass a local path to `--emails-csv`.\n\nNotes:\n- The script parses emails, chunks header/body into passages, builds a compact LEANN index, and then builds a FAISS Flat baseline from the same passages and embedding model.\n- Optionally, it will also create evaluation queries from HuggingFace dataset `corbt/enron_emails_sample_questions`.\n\n2) Run recall evaluation (Stage 2)\n\npython evaluate_enron_emails.py --index data/enron_index_hnsw.leann --stage 2\n\n3) Complexity sweep (Stage 3)\n\npython evaluate_enron_emails.py --index data/enron_index_hnsw.leann --stage 3 --target-recall 0.90 --max-queries 200\n\nStage 3 uses binary search over complexity to find the minimal value achieving the target Recall@3 (assumes recall is non-decreasing with complexity). The search expands the upper bound as needed and snaps complexity to multiples of 8.\n\n4) Index comparison (Stage 4)\n\npython evaluate_enron_emails.py --index data/enron_index_hnsw.leann --stage 4 --complexity 88 --max-queries 100 --output results.json\n\n5) Generation evaluation (Stage 5)\n\npython evaluate_enron_emails.py --index data/enron_index_hnsw.leann --stage 5 --complexity 88 --llm-backend hf --model-name Qwen/Qwen3-8B\n\n6) Combined index + generation evaluation (Stages 4+5, recommended)\n\npython evaluate_enron_emails.py --index data/enron_index_hnsw.leann --stage 45 --complexity 88 --llm-backend hf\n\nNotes:\n- Minimal CLI: you can run from repo root with only `--index`, defaults match financebench/laion patterns:\n - `--stage` defaults to `all` (runs 2, 3, 4, 5)\n - `--baseline-dir` defaults to `baseline`\n - `--queries` defaults to `data/evaluation_queries.jsonl` (or falls back to the index directory)\n - `--llm-backend` defaults to `hf` (HuggingFace), can use `vllm`\n - `--model-name` defaults to `Qwen/Qwen3-8B`\n- Fail-fast behavior: no silent fallbacks. If compact index cannot run with recompute, it errors out.\n- Stage 5 requires Stage 4 retrieval results. Use `--stage 45` to run both efficiently.\n\nOptional flags:\n- --queries data/evaluation_queries.jsonl (custom queries file)\n- --baseline-dir baseline (where FAISS baseline lives)\n- --complexity 88 (LEANN complexity parameter, optimal for 90% recall)\n- --llm-backend hf|vllm (LLM backend for generation)\n- --model-name Qwen/Qwen3-8B (LLM model for generation)\n- --max-queries 1000 (limit number of queries for evaluation)\n\n## Files Produced\n- data/enron_passages_preview.jsonl: Small preview of passages used (for inspection)\n- data/enron_index_hnsw.leann.*: LEANN index files\n- baseline/faiss_flat.index + baseline/metadata.pkl: FAISS baseline with passage IDs\n- data/evaluation_queries.jsonl: Query file (id + query; includes GT IDs for reference)\n\n## Notes\n- Evaluates both retrieval Recall@3 and generation timing with Qwen3-8B thinking model.\n- The emails CSV must contain a column named \"message\" (raw RFC822 email) and a column named \"file\" for source identifier. Message-ID headers are parsed as canonical message IDs when present.\n- Qwen3-8B requires special handling for thinking models with chat templates and tag processing.\n\n## Stages Summary\n\n- Stage 2 (Recall@3):\n - Compares LEANN vs FAISS Flat baseline on Recall@3.\n - Compact index runs with `recompute_embeddings=True`.\n\n- Stage 3 (Binary Search for Complexity):\n - Builds a non-compact index (`_noncompact.leann`) and runs binary search with `recompute_embeddings=False` to find the minimal complexity achieving target Recall@3 (default 90%).\n\n- Stage 4 (Index Comparison):\n - Reports .index-only sizes for compact vs non-compact.\n - Measures timings on queries by default: non-compact (no recompute) vs compact (with recompute).\n - Stores retrieval results for Stage 5 generation evaluation.\n - Fails fast if compact recompute cannot run.\n - If `--complexity` is not provided, the script tries to use the best complexity from Stage 3:\n - First from the current run (when running `--stage all`), otherwise\n - From `enron_stage3_results.json` saved next to the index during the last Stage 3 run.\n - If neither exists, Stage 4 will error and ask you to run Stage 3 or pass `--complexity`.\n\n- Stage 5 (Generation Evaluation):\n - Uses Qwen3-8B thinking model for RAG generation on retrieved documents from Stage 4.\n - Supports HuggingFace (`hf`) and vLLM (`vllm`) backends.\n - Measures generation timing separately from search timing.\n - Requires Stage 4 results (no additional searching performed).\n\n## Example Results\n\nThese are sample results obtained on Enron data using all-mpnet-base-v2 and Qwen3-8B.\n\n- Stage 3 (Binary Search):\n - Minimal complexity achieving 90% Recall@3: 88\n - Sampled points:\n - C=8 → 59.9% Recall@3\n - C=72 → 89.4% Recall@3\n - C=88 → 90.2% Recall@3\n - C=96 → 90.7% Recall@3\n - C=112 → 91.1% Recall@3\n - C=136 → 91.3% Recall@3\n - C=256 → 92.0% Recall@3\n\n- Stage 4 (Index Sizes, .index only):\n - Compact: ~2.2 MB\n - Non-compact: ~82.0 MB\n - Storage saving by compact: ~97.3%\n\n- Stage 4 (Search Timing, 988 queries, complexity=88):\n - Non-compact (no recompute): ~0.0075 s avg per query\n - Compact (with recompute): ~1.981 s avg per query\n - Speed ratio (non-compact/compact): ~0.0038x\n\n- Stage 5 (RAG Generation, 988 queries, Qwen3-8B):\n - Average generation time: ~22.302 s per query\n - Total queries processed: 988\n - LLM backend: HuggingFace transformers\n - Model: Qwen/Qwen3-8B (thinking model with processing)\n\nFull JSON output is saved by the script (see `--output`), e.g.:\n`benchmarks/enron_emails/results_enron_stage45.json`.\n" }, { "path": "benchmarks/enron_emails/data/.gitignore", "content": "downloads/\n" }, { "path": "benchmarks/enron_emails/evaluate_enron_emails.py", "content": "\"\"\"\nEnron Emails Benchmark Evaluation - Retrieval Recall@3 (Stages 2/3/4)\nFollows the style of FinanceBench/LAION: Stage 2 recall vs FAISS baseline,\nStage 3 complexity sweep to target recall, Stage 4 index comparison.\nOn errors, fail fast without fallbacks.\n\"\"\"\n\nimport argparse\nimport json\nimport logging\nimport os\nimport pickle\nfrom pathlib import Path\n\nimport numpy as np\nfrom leann import LeannBuilder, LeannSearcher\nfrom leann_backend_hnsw import faiss\n\nfrom ..llm_utils import generate_hf, generate_vllm, load_hf_model, load_vllm_model\n\n# Setup logging to reduce verbose output\nlogging.basicConfig(level=logging.WARNING)\nlogging.getLogger(\"leann.api\").setLevel(logging.WARNING)\nlogging.getLogger(\"leann_backend_hnsw\").setLevel(logging.WARNING)\n\n\nclass RecallEvaluator:\n \"\"\"Stage 2: Evaluate Recall@3 (LEANN vs FAISS)\"\"\"\n\n def __init__(self, index_path: str, baseline_dir: str):\n self.index_path = index_path\n self.baseline_dir = baseline_dir\n self.searcher = LeannSearcher(index_path)\n\n baseline_index_path = os.path.join(baseline_dir, \"faiss_flat.index\")\n metadata_path = os.path.join(baseline_dir, \"metadata.pkl\")\n\n self.faiss_index = faiss.read_index(baseline_index_path)\n with open(metadata_path, \"rb\") as f:\n self.passage_ids = pickle.load(f)\n\n print(f\"📚 Loaded FAISS flat baseline with {self.faiss_index.ntotal} vectors\")\n\n # No fallbacks here; if embedding server is needed but fails, the caller will see the error.\n\n def evaluate_recall_at_3(\n self, queries: list[str], complexity: int = 64, recompute_embeddings: bool = True\n ) -> float:\n \"\"\"Evaluate recall@3 using FAISS Flat as ground truth\"\"\"\n from leann.api import compute_embeddings\n\n recompute_str = \"with recompute\" if recompute_embeddings else \"no recompute\"\n print(f\"🔍 Evaluating recall@3 with complexity={complexity} ({recompute_str})...\")\n\n total_recall = 0.0\n for i, query in enumerate(queries):\n # Compute query embedding with the same model/mode as the index\n q_emb = compute_embeddings(\n [query],\n self.searcher.embedding_model,\n mode=self.searcher.embedding_mode,\n use_server=False,\n ).astype(np.float32)\n\n # Search FAISS Flat ground truth\n n = q_emb.shape[0]\n k = 3\n distances = np.zeros((n, k), dtype=np.float32)\n labels = np.zeros((n, k), dtype=np.int64)\n self.faiss_index.search(\n n,\n faiss.swig_ptr(q_emb),\n k,\n faiss.swig_ptr(distances),\n faiss.swig_ptr(labels),\n )\n\n baseline_ids = {self.passage_ids[idx] for idx in labels[0]}\n\n # Search with LEANN (may require embedding server depending on index configuration)\n results = self.searcher.search(\n query,\n top_k=3,\n complexity=complexity,\n recompute_embeddings=recompute_embeddings,\n )\n test_ids = {r.id for r in results}\n\n intersection = test_ids.intersection(baseline_ids)\n recall = len(intersection) / 3.0\n total_recall += recall\n\n if i < 3:\n print(f\" Q{i + 1}: '{query[:60]}...' -> Recall@3: {recall:.3f}\")\n print(f\" FAISS: {list(baseline_ids)}\")\n print(f\" LEANN: {list(test_ids)}\")\n print(f\" ∩: {list(intersection)}\")\n\n avg = total_recall / max(1, len(queries))\n print(f\"📊 Average Recall@3: {avg:.3f} ({avg * 100:.1f}%)\")\n return avg\n\n def cleanup(self):\n if hasattr(self, \"searcher\"):\n self.searcher.cleanup()\n\n\nclass EnronEvaluator:\n def __init__(self, index_path: str):\n self.index_path = index_path\n self.searcher = LeannSearcher(index_path)\n\n def load_queries(self, queries_file: str) -> list[str]:\n queries: list[str] = []\n with open(queries_file, encoding=\"utf-8\") as f:\n for line in f:\n if not line.strip():\n continue\n data = json.loads(line)\n if \"query\" in data:\n queries.append(data[\"query\"])\n print(f\"📊 Loaded {len(queries)} queries from {queries_file}\")\n return queries\n\n def cleanup(self):\n if self.searcher:\n self.searcher.cleanup()\n\n def analyze_index_sizes(self) -> dict:\n \"\"\"Analyze index sizes (.index only), similar to LAION bench.\"\"\"\n\n print(\"📏 Analyzing index sizes (.index only)...\")\n index_path = Path(self.index_path)\n index_dir = index_path.parent\n index_name = index_path.stem\n\n sizes: dict[str, float] = {}\n index_file = index_dir / f\"{index_name}.index\"\n meta_file = index_dir / f\"{index_path.name}.meta.json\"\n passages_file = index_dir / f\"{index_path.name}.passages.jsonl\"\n passages_idx_file = index_dir / f\"{index_path.name}.passages.idx\"\n\n sizes[\"index_only_mb\"] = (\n index_file.stat().st_size / (1024 * 1024) if index_file.exists() else 0.0\n )\n sizes[\"metadata_mb\"] = (\n meta_file.stat().st_size / (1024 * 1024) if meta_file.exists() else 0.0\n )\n sizes[\"passages_text_mb\"] = (\n passages_file.stat().st_size / (1024 * 1024) if passages_file.exists() else 0.0\n )\n sizes[\"passages_index_mb\"] = (\n passages_idx_file.stat().st_size / (1024 * 1024) if passages_idx_file.exists() else 0.0\n )\n\n print(f\" 📁 .index size: {sizes['index_only_mb']:.1f} MB\")\n return sizes\n\n def create_non_compact_index_for_comparison(self, non_compact_index_path: str) -> dict:\n \"\"\"Create a non-compact index for comparison using current passages and embeddings.\"\"\"\n\n current_index_path = Path(self.index_path)\n current_index_dir = current_index_path.parent\n current_index_name = current_index_path.name\n\n # Read metadata to get passage source and embedding model\n meta_path = current_index_dir / f\"{current_index_name}.meta.json\"\n with open(meta_path, encoding=\"utf-8\") as f:\n meta = json.load(f)\n\n passage_source = meta[\"passage_sources\"][0]\n passage_file = passage_source[\"path\"]\n\n # Convert relative path to absolute\n if not Path(passage_file).is_absolute():\n passage_file = current_index_dir / Path(passage_file).name\n\n # Load all passages and ids\n ids: list[str] = []\n texts: list[str] = []\n with open(passage_file, encoding=\"utf-8\") as f:\n for line in f:\n if line.strip():\n data = json.loads(line)\n ids.append(str(data[\"id\"]))\n texts.append(data[\"text\"])\n\n # Compute embeddings using the same method as LEANN\n from leann.api import compute_embeddings\n\n embeddings = compute_embeddings(\n texts,\n meta[\"embedding_model\"],\n mode=meta.get(\"embedding_mode\", \"sentence-transformers\"),\n use_server=False,\n ).astype(np.float32)\n\n # Build non-compact index with same passages and embeddings\n builder = LeannBuilder(\n backend_name=\"hnsw\",\n embedding_model=meta[\"embedding_model\"],\n embedding_mode=meta.get(\"embedding_mode\", \"sentence-transformers\"),\n is_recompute=False,\n is_compact=False,\n **{\n k: v\n for k, v in meta.get(\"backend_kwargs\", {}).items()\n if k not in [\"is_recompute\", \"is_compact\"]\n },\n )\n\n # Persist a pickle for build_index_from_embeddings\n pkl_path = current_index_dir / f\"{Path(non_compact_index_path).stem}_embeddings.pkl\"\n with open(pkl_path, \"wb\") as pf:\n pickle.dump((ids, embeddings), pf)\n\n print(\n f\"🔨 Building non-compact index at {non_compact_index_path} from precomputed embeddings...\"\n )\n builder.build_index_from_embeddings(non_compact_index_path, str(pkl_path))\n\n # Analyze the non-compact index size\n temp_evaluator = EnronEvaluator(non_compact_index_path)\n non_compact_sizes = temp_evaluator.analyze_index_sizes()\n non_compact_sizes[\"index_type\"] = \"non_compact\"\n\n return non_compact_sizes\n\n def compare_index_performance(\n self, non_compact_path: str, compact_path: str, test_queries: list[str], complexity: int\n ) -> dict:\n \"\"\"Compare search speed for non-compact vs compact indexes.\"\"\"\n import time\n\n results: dict = {\n \"non_compact\": {\"search_times\": []},\n \"compact\": {\"search_times\": []},\n \"avg_search_times\": {},\n \"speed_ratio\": 0.0,\n \"retrieval_results\": [], # Store retrieval results for Stage 5\n }\n\n print(\"⚡ Comparing search performance between indexes...\")\n # Non-compact (no recompute)\n print(\" 🔍 Testing non-compact index (no recompute)...\")\n non_compact_searcher = LeannSearcher(non_compact_path)\n for q in test_queries:\n t0 = time.time()\n _ = non_compact_searcher.search(\n q, top_k=3, complexity=complexity, recompute_embeddings=False\n )\n results[\"non_compact\"][\"search_times\"].append(time.time() - t0)\n\n # Compact (with recompute). Fail fast if it cannot run.\n print(\" 🔍 Testing compact index (with recompute)...\")\n compact_searcher = LeannSearcher(compact_path)\n for q in test_queries:\n t0 = time.time()\n docs = compact_searcher.search(\n q, top_k=3, complexity=complexity, recompute_embeddings=True\n )\n results[\"compact\"][\"search_times\"].append(time.time() - t0)\n\n # Store retrieval results for Stage 5\n results[\"retrieval_results\"].append(\n {\"query\": q, \"retrieved_docs\": [{\"id\": doc.id, \"text\": doc.text} for doc in docs]}\n )\n compact_searcher.cleanup()\n\n if results[\"non_compact\"][\"search_times\"]:\n results[\"avg_search_times\"][\"non_compact\"] = sum(\n results[\"non_compact\"][\"search_times\"]\n ) / len(results[\"non_compact\"][\"search_times\"])\n if results[\"compact\"][\"search_times\"]:\n results[\"avg_search_times\"][\"compact\"] = sum(results[\"compact\"][\"search_times\"]) / len(\n results[\"compact\"][\"search_times\"]\n )\n if results[\"avg_search_times\"].get(\"compact\", 0) > 0:\n results[\"speed_ratio\"] = (\n results[\"avg_search_times\"][\"non_compact\"] / results[\"avg_search_times\"][\"compact\"]\n )\n else:\n results[\"speed_ratio\"] = 0.0\n\n non_compact_searcher.cleanup()\n return results\n\n def evaluate_complexity(\n self,\n recall_eval: \"RecallEvaluator\",\n queries: list[str],\n target: float = 0.90,\n c_min: int = 8,\n c_max: int = 256,\n max_iters: int = 10,\n recompute: bool = False,\n ) -> dict:\n \"\"\"Binary search minimal complexity achieving target recall (monotonic assumption).\"\"\"\n\n def round_c(x: int) -> int:\n # snap to multiple of 8 like other benches typically do\n return max(1, int((x + 7) // 8) * 8)\n\n metrics: list[dict] = []\n\n lo = round_c(c_min)\n hi = round_c(c_max)\n\n print(\n f\"🧪 Binary search complexity in [{lo}, {hi}] for target Recall@3>={int(target * 100)}%...\"\n )\n\n # Ensure upper bound can reach target; expand if needed (up to a cap)\n r_lo = recall_eval.evaluate_recall_at_3(\n queries, complexity=lo, recompute_embeddings=recompute\n )\n metrics.append({\"complexity\": lo, \"recall_at_3\": r_lo})\n r_hi = recall_eval.evaluate_recall_at_3(\n queries, complexity=hi, recompute_embeddings=recompute\n )\n metrics.append({\"complexity\": hi, \"recall_at_3\": r_hi})\n\n cap = 1024\n while r_hi < target and hi < cap:\n lo = hi\n r_lo = r_hi\n hi = round_c(hi * 2)\n r_hi = recall_eval.evaluate_recall_at_3(\n queries, complexity=hi, recompute_embeddings=recompute\n )\n metrics.append({\"complexity\": hi, \"recall_at_3\": r_hi})\n\n if r_hi < target:\n print(f\"⚠️ Max complexity {hi} did not reach target recall {target:.2f}.\")\n print(\"📈 Observations:\")\n for m in metrics:\n print(f\" C={m['complexity']:>4} -> Recall@3={m['recall_at_3'] * 100:.1f}%\")\n return {\"metrics\": metrics, \"best_complexity\": None, \"target_recall\": target}\n\n # Binary search within [lo, hi]\n best = hi\n iters = 0\n while lo < hi and iters < max_iters:\n mid = round_c((lo + hi) // 2)\n r_mid = recall_eval.evaluate_recall_at_3(\n queries, complexity=mid, recompute_embeddings=recompute\n )\n metrics.append({\"complexity\": mid, \"recall_at_3\": r_mid})\n if r_mid >= target:\n best = mid\n hi = mid\n else:\n lo = mid + 8 # move past mid, respecting multiple-of-8 step\n iters += 1\n\n print(\"📈 Binary search results (sampled points):\")\n # Print unique complexity entries ordered by complexity\n for m in sorted(\n {m[\"complexity\"]: m for m in metrics}.values(), key=lambda x: x[\"complexity\"]\n ):\n print(f\" C={m['complexity']:>4} -> Recall@3={m['recall_at_3'] * 100:.1f}%\")\n print(f\"✅ Minimal complexity achieving {int(target * 100)}% recall: {best}\")\n return {\"metrics\": metrics, \"best_complexity\": best, \"target_recall\": target}\n\n\ndef main():\n parser = argparse.ArgumentParser(description=\"Enron Emails Benchmark Evaluation\")\n parser.add_argument(\"--index\", required=True, help=\"Path to LEANN index\")\n parser.add_argument(\n \"--queries\", default=\"data/evaluation_queries.jsonl\", help=\"Path to evaluation queries\"\n )\n parser.add_argument(\n \"--stage\",\n choices=[\"2\", \"3\", \"4\", \"5\", \"all\", \"45\"],\n default=\"all\",\n help=\"Which stage to run (2=recall, 3=complexity, 4=index comparison, 5=generation)\",\n )\n parser.add_argument(\"--complexity\", type=int, default=None, help=\"LEANN search complexity\")\n parser.add_argument(\"--baseline-dir\", default=\"baseline\", help=\"Baseline output directory\")\n parser.add_argument(\n \"--max-queries\", type=int, help=\"Limit number of queries to evaluate\", default=1000\n )\n parser.add_argument(\n \"--target-recall\", type=float, default=0.90, help=\"Target Recall@3 for Stage 3\"\n )\n parser.add_argument(\"--output\", help=\"Save results to JSON file\")\n parser.add_argument(\"--llm-backend\", choices=[\"hf\", \"vllm\"], default=\"hf\", help=\"LLM backend\")\n parser.add_argument(\"--model-name\", default=\"Qwen/Qwen3-8B\", help=\"Model name\")\n\n args = parser.parse_args()\n\n # Resolve queries file: if default path not found, fall back to index's directory\n if not os.path.exists(args.queries):\n from pathlib import Path\n\n idx_dir = Path(args.index).parent\n fallback_q = idx_dir / \"evaluation_queries.jsonl\"\n if fallback_q.exists():\n args.queries = str(fallback_q)\n\n baseline_index_path = os.path.join(args.baseline_dir, \"faiss_flat.index\")\n if not os.path.exists(baseline_index_path):\n print(f\"❌ FAISS baseline not found at {baseline_index_path}\")\n print(\"💡 Please run setup_enron_emails.py first to build the baseline\")\n raise SystemExit(1)\n\n results_out: dict = {}\n\n if args.stage in (\"2\", \"all\"):\n print(\"🚀 Starting Stage 2: Recall@3 evaluation\")\n evaluator = RecallEvaluator(args.index, args.baseline_dir)\n\n enron_eval = EnronEvaluator(args.index)\n queries = enron_eval.load_queries(args.queries)\n queries = queries[:10]\n print(f\"🧪 Using first {len(queries)} queries\")\n\n complexity = args.complexity or 64\n r = evaluator.evaluate_recall_at_3(queries, complexity)\n results_out[\"stage2\"] = {\"complexity\": complexity, \"recall_at_3\": r}\n evaluator.cleanup()\n enron_eval.cleanup()\n print(\"✅ Stage 2 completed!\\n\")\n\n if args.stage in (\"3\", \"all\"):\n print(\"🚀 Starting Stage 3: Binary search for target recall (no recompute)\")\n enron_eval = EnronEvaluator(args.index)\n queries = enron_eval.load_queries(args.queries)\n queries = queries[: args.max_queries]\n print(f\"🧪 Using first {len(queries)} queries\")\n\n # Build non-compact index for fast binary search (recompute_embeddings=False)\n from pathlib import Path\n\n index_path = Path(args.index)\n non_compact_index_path = str(index_path.parent / f\"{index_path.stem}_noncompact.leann\")\n enron_eval.create_non_compact_index_for_comparison(non_compact_index_path)\n\n # Use non-compact evaluator for binary search with recompute=False\n evaluator_nc = RecallEvaluator(non_compact_index_path, args.baseline_dir)\n sweep = enron_eval.evaluate_complexity(\n evaluator_nc, queries, target=args.target_recall, recompute=False\n )\n results_out[\"stage3\"] = sweep\n # Persist default stage 3 results near the index for Stage 4 auto-pickup\n from pathlib import Path\n\n default_stage3_path = Path(args.index).parent / \"enron_stage3_results.json\"\n with open(default_stage3_path, \"w\", encoding=\"utf-8\") as f:\n json.dump({\"stage3\": sweep}, f, indent=2)\n print(f\"📝 Saved Stage 3 summary to {default_stage3_path}\")\n evaluator_nc.cleanup()\n enron_eval.cleanup()\n print(\"✅ Stage 3 completed!\\n\")\n\n if args.stage in (\"4\", \"all\", \"45\"):\n print(\"🚀 Starting Stage 4: Index size + performance comparison\")\n evaluator = RecallEvaluator(args.index, args.baseline_dir)\n enron_eval = EnronEvaluator(args.index)\n queries = enron_eval.load_queries(args.queries)\n test_q = queries[: min(args.max_queries, len(queries))]\n\n current_sizes = enron_eval.analyze_index_sizes()\n # Build non-compact index for comparison (no fallback)\n from pathlib import Path\n\n index_path = Path(args.index)\n non_compact_path = str(index_path.parent / f\"{index_path.stem}_noncompact.leann\")\n non_compact_sizes = enron_eval.create_non_compact_index_for_comparison(non_compact_path)\n nc_eval = EnronEvaluator(non_compact_path)\n\n if (\n current_sizes.get(\"index_only_mb\", 0) > 0\n and non_compact_sizes.get(\"index_only_mb\", 0) > 0\n ):\n storage_saving_percent = max(\n 0.0,\n 100.0 * (1.0 - current_sizes[\"index_only_mb\"] / non_compact_sizes[\"index_only_mb\"]),\n )\n else:\n storage_saving_percent = 0.0\n\n if args.complexity is None:\n # Prefer in-session Stage 3 result\n if \"stage3\" in results_out and results_out[\"stage3\"].get(\"best_complexity\") is not None:\n complexity = results_out[\"stage3\"][\"best_complexity\"]\n print(f\"📥 Using best complexity from Stage 3 in-session: {complexity}\")\n else:\n # Try to load last saved Stage 3 result near index\n default_stage3_path = Path(args.index).parent / \"enron_stage3_results.json\"\n if default_stage3_path.exists():\n with open(default_stage3_path, encoding=\"utf-8\") as f:\n prev = json.load(f)\n complexity = prev.get(\"stage3\", {}).get(\"best_complexity\")\n if complexity is None:\n raise SystemExit(\n \"❌ Stage 4: No --complexity and no best_complexity found in saved Stage 3 results\"\n )\n print(f\"📥 Using best complexity from saved Stage 3: {complexity}\")\n else:\n raise SystemExit(\n \"❌ Stage 4 requires --complexity if Stage 3 hasn't been run. Run stage 3 first or pass --complexity.\"\n )\n else:\n complexity = args.complexity\n\n comp = enron_eval.compare_index_performance(\n non_compact_path, args.index, test_q, complexity=complexity\n )\n results_out[\"stage4\"] = {\n \"current_index\": current_sizes,\n \"non_compact_index\": non_compact_sizes,\n \"storage_saving_percent\": storage_saving_percent,\n \"performance_comparison\": comp,\n }\n nc_eval.cleanup()\n evaluator.cleanup()\n enron_eval.cleanup()\n print(\"✅ Stage 4 completed!\\n\")\n\n if args.stage in (\"5\", \"all\"):\n print(\"🚀 Starting Stage 5: Generation evaluation with Qwen3-8B\")\n\n # Check if Stage 4 results exist\n if \"stage4\" not in results_out or \"performance_comparison\" not in results_out[\"stage4\"]:\n print(\"❌ Stage 5 requires Stage 4 retrieval results\")\n print(\"💡 Run Stage 4 first or use --stage all\")\n raise SystemExit(1)\n\n retrieval_results = results_out[\"stage4\"][\"performance_comparison\"][\"retrieval_results\"]\n if not retrieval_results:\n print(\"❌ No retrieval results found from Stage 4\")\n raise SystemExit(1)\n\n print(f\"📁 Using {len(retrieval_results)} retrieval results from Stage 4\")\n\n # Load LLM\n try:\n if args.llm_backend == \"hf\":\n tokenizer, model = load_hf_model(args.model_name)\n\n def llm_func(prompt):\n return generate_hf(tokenizer, model, prompt)\n else: # vllm\n llm, sampling_params = load_vllm_model(args.model_name)\n\n def llm_func(prompt):\n return generate_vllm(llm, sampling_params, prompt)\n\n # Run generation using stored retrieval results\n import time\n\n from llm_utils import create_prompt\n\n generation_times = []\n responses = []\n\n print(\"🤖 Running generation on pre-retrieved results...\")\n for i, item in enumerate(retrieval_results):\n query = item[\"query\"]\n retrieved_docs = item[\"retrieved_docs\"]\n\n # Prepare context from retrieved docs\n context = \"\\n\\n\".join([doc[\"text\"] for doc in retrieved_docs])\n prompt = create_prompt(context, query, \"emails\")\n\n # Time generation only\n gen_start = time.time()\n response = llm_func(prompt)\n gen_time = time.time() - gen_start\n\n generation_times.append(gen_time)\n responses.append(response)\n\n if i < 3:\n print(f\" Q{i + 1}: Gen={gen_time:.3f}s\")\n\n avg_gen_time = sum(generation_times) / len(generation_times)\n\n print(\"\\n📊 Generation Results:\")\n print(f\" Total Queries: {len(retrieval_results)}\")\n print(f\" Avg Generation Time: {avg_gen_time:.3f}s\")\n print(\" (Search time from Stage 4)\")\n\n results_out[\"stage5\"] = {\n \"total_queries\": len(retrieval_results),\n \"avg_generation_time\": avg_gen_time,\n \"generation_times\": generation_times,\n \"responses\": responses,\n }\n\n # Show sample results\n print(\"\\n📝 Sample Results:\")\n for i in range(min(3, len(retrieval_results))):\n query = retrieval_results[i][\"query\"]\n response = responses[i]\n print(f\" Q{i + 1}: {query[:60]}...\")\n print(f\" A{i + 1}: {response[:100]}...\")\n print()\n\n except Exception as e:\n print(f\"❌ Generation evaluation failed: {e}\")\n print(\"💡 Make sure transformers/vllm is installed and model is available\")\n\n print(\"✅ Stage 5 completed!\\n\")\n\n if args.output and results_out:\n with open(args.output, \"w\", encoding=\"utf-8\") as f:\n json.dump(results_out, f, indent=2)\n print(f\"📝 Saved results to {args.output}\")\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "benchmarks/enron_emails/setup_enron_emails.py", "content": "\"\"\"\nEnron Emails Benchmark Setup Script\nPrepares passages from emails.csv, builds LEANN index, and FAISS Flat baseline\n\"\"\"\n\nimport argparse\nimport csv\nimport json\nimport os\nimport re\nfrom collections.abc import Iterable\nfrom email import message_from_string\nfrom email.policy import default\nfrom pathlib import Path\nfrom typing import Optional\n\nfrom leann import LeannBuilder\n\n\nclass EnronSetup:\n def __init__(self, data_dir: str = \"data\"):\n self.data_dir = Path(data_dir)\n self.data_dir.mkdir(parents=True, exist_ok=True)\n\n self.passages_preview = self.data_dir / \"enron_passages_preview.jsonl\"\n self.index_path = self.data_dir / \"enron_index_hnsw.leann\"\n self.queries_file = self.data_dir / \"evaluation_queries.jsonl\"\n self.downloads_dir = self.data_dir / \"downloads\"\n self.downloads_dir.mkdir(parents=True, exist_ok=True)\n\n # ----------------------------\n # Dataset acquisition\n # ----------------------------\n def ensure_emails_csv(self, emails_csv: Optional[str]) -> str:\n \"\"\"Return a path to emails.csv, downloading from Kaggle if needed.\"\"\"\n if emails_csv:\n p = Path(emails_csv)\n if not p.exists():\n raise FileNotFoundError(f\"emails.csv not found: {emails_csv}\")\n return str(p)\n\n print(\n \"📥 Trying to download Enron emails.csv from Kaggle (wcukierski/enron-email-dataset)...\"\n )\n try:\n from kaggle.api.kaggle_api_extended import KaggleApi\n\n api = KaggleApi()\n api.authenticate()\n api.dataset_download_files(\n \"wcukierski/enron-email-dataset\", path=str(self.downloads_dir), unzip=True\n )\n candidate = self.downloads_dir / \"emails.csv\"\n if candidate.exists():\n print(f\"✅ Downloaded emails.csv: {candidate}\")\n return str(candidate)\n else:\n raise FileNotFoundError(\n f\"emails.csv was not found in {self.downloads_dir} after Kaggle download\"\n )\n except Exception as e:\n print(\n \"❌ Could not download via Kaggle automatically. Provide --emails-csv or configure Kaggle API.\"\n )\n print(\n \" Set KAGGLE_USERNAME and KAGGLE_KEY env vars, or place emails.csv locally and pass --emails-csv.\"\n )\n raise e\n\n # ----------------------------\n # Data preparation\n # ----------------------------\n @staticmethod\n def _extract_message_id(raw_email: str) -> str:\n msg = message_from_string(raw_email, policy=default)\n val = msg.get(\"Message-ID\", \"\")\n if val.startswith(\"<\") and val.endswith(\">\"):\n val = val[1:-1]\n return val or \"\"\n\n @staticmethod\n def _split_header_body(raw_email: str) -> tuple[str, str]:\n parts = raw_email.split(\"\\n\\n\", 1)\n if len(parts) == 2:\n return parts[0].strip(), parts[1].strip()\n # Heuristic fallback\n first_lines = raw_email.splitlines()\n if first_lines and \":\" in first_lines[0]:\n return raw_email.strip(), \"\"\n return \"\", raw_email.strip()\n\n @staticmethod\n def _split_fixed_words(text: str, chunk_words: int, keep_last: bool) -> list[str]:\n text = (text or \"\").strip()\n if not text:\n return []\n if chunk_words <= 0:\n return [text]\n words = text.split()\n if not words:\n return []\n limit = len(words)\n if not keep_last:\n limit = (len(words) // chunk_words) * chunk_words\n if limit == 0:\n return []\n chunks = [\" \".join(words[i : i + chunk_words]) for i in range(0, limit, chunk_words)]\n return [c for c in (s.strip() for s in chunks) if c]\n\n def _iter_passages_from_csv(\n self,\n emails_csv: Path,\n chunk_words: int = 256,\n keep_last_header: bool = True,\n keep_last_body: bool = True,\n max_emails: int | None = None,\n ) -> Iterable[dict]:\n with open(emails_csv, encoding=\"utf-8\") as f:\n reader = csv.DictReader(f)\n count = 0\n for i, row in enumerate(reader):\n if max_emails is not None and count >= max_emails:\n break\n\n raw_message = row.get(\"message\", \"\")\n email_file_id = row.get(\"file\", \"\")\n\n if not raw_message.strip():\n continue\n\n message_id = self._extract_message_id(raw_message)\n if not message_id:\n # Fallback ID based on CSV position and file path\n safe_file = re.sub(r\"[^A-Za-z0-9_.-]\", \"_\", email_file_id)\n message_id = f\"enron_{i}_{safe_file}\"\n\n header, body = self._split_header_body(raw_message)\n\n # Header chunks\n for chunk in self._split_fixed_words(header, chunk_words, keep_last_header):\n yield {\n \"text\": chunk,\n \"metadata\": {\n \"message_id\": message_id,\n \"is_header\": True,\n \"email_file_id\": email_file_id,\n },\n }\n\n # Body chunks\n for chunk in self._split_fixed_words(body, chunk_words, keep_last_body):\n yield {\n \"text\": chunk,\n \"metadata\": {\n \"message_id\": message_id,\n \"is_header\": False,\n \"email_file_id\": email_file_id,\n },\n }\n\n count += 1\n\n # ----------------------------\n # Build LEANN index and FAISS baseline\n # ----------------------------\n def build_leann_index(\n self,\n emails_csv: Optional[str],\n backend: str = \"hnsw\",\n embedding_model: str = \"sentence-transformers/all-mpnet-base-v2\",\n chunk_words: int = 256,\n max_emails: int | None = None,\n ) -> str:\n emails_csv_path = self.ensure_emails_csv(emails_csv)\n print(f\"🏗️ Building LEANN index from {emails_csv_path}...\")\n\n builder = LeannBuilder(\n backend_name=backend,\n embedding_model=embedding_model,\n embedding_mode=\"sentence-transformers\",\n graph_degree=32,\n complexity=64,\n is_recompute=True,\n is_compact=True,\n num_threads=4,\n )\n\n # Stream passages and add to builder\n preview_written = 0\n with open(self.passages_preview, \"w\", encoding=\"utf-8\") as preview_out:\n for p in self._iter_passages_from_csv(\n Path(emails_csv_path), chunk_words=chunk_words, max_emails=max_emails\n ):\n builder.add_text(p[\"text\"], metadata=p[\"metadata\"])\n if preview_written < 200:\n preview_out.write(json.dumps({\"text\": p[\"text\"][:200], **p[\"metadata\"]}) + \"\\n\")\n preview_written += 1\n\n print(f\"🔨 Building index at {self.index_path}...\")\n builder.build_index(str(self.index_path))\n print(\"✅ LEANN index built!\")\n return str(self.index_path)\n\n def build_faiss_flat_baseline(self, index_path: str, output_dir: str = \"baseline\") -> str:\n print(\"🔨 Building FAISS Flat baseline from LEANN passages...\")\n\n import pickle\n\n import numpy as np\n from leann.api import compute_embeddings\n from leann_backend_hnsw import faiss\n\n os.makedirs(output_dir, exist_ok=True)\n baseline_path = os.path.join(output_dir, \"faiss_flat.index\")\n metadata_path = os.path.join(output_dir, \"metadata.pkl\")\n\n if os.path.exists(baseline_path) and os.path.exists(metadata_path):\n print(f\"✅ Baseline already exists at {baseline_path}\")\n return baseline_path\n\n # Read meta for passage source and embedding model\n meta_path = f\"{index_path}.meta.json\"\n with open(meta_path, encoding=\"utf-8\") as f:\n meta = json.load(f)\n\n embedding_model = meta[\"embedding_model\"]\n passage_source = meta[\"passage_sources\"][0]\n passage_file = passage_source[\"path\"]\n\n if not os.path.isabs(passage_file):\n index_dir = os.path.dirname(index_path)\n passage_file = os.path.join(index_dir, os.path.basename(passage_file))\n\n # Load passages from builder output so IDs match LEANN\n passages: list[str] = []\n passage_ids: list[str] = []\n with open(passage_file, encoding=\"utf-8\") as f:\n for line in f:\n if not line.strip():\n continue\n data = json.loads(line)\n passages.append(data[\"text\"])\n passage_ids.append(data[\"id\"]) # builder-assigned ID\n\n print(f\"📄 Loaded {len(passages)} passages for baseline\")\n print(f\"🤖 Embedding model: {embedding_model}\")\n\n embeddings = compute_embeddings(\n passages,\n embedding_model,\n mode=\"sentence-transformers\",\n use_server=False,\n )\n\n # Build FAISS IndexFlatIP\n dim = embeddings.shape[1]\n index = faiss.IndexFlatIP(dim)\n emb_f32 = embeddings.astype(np.float32)\n index.add(emb_f32.shape[0], faiss.swig_ptr(emb_f32))\n\n faiss.write_index(index, baseline_path)\n with open(metadata_path, \"wb\") as pf:\n pickle.dump(passage_ids, pf)\n\n print(f\"✅ FAISS baseline saved: {baseline_path}\")\n print(f\"✅ Metadata saved: {metadata_path}\")\n print(f\"📊 Total vectors: {index.ntotal}\")\n return baseline_path\n\n # ----------------------------\n # Queries (optional): prepare evaluation queries file\n # ----------------------------\n def prepare_queries(self, min_realism: float = 0.85) -> Path:\n print(\n \"📝 Preparing evaluation queries from HuggingFace dataset corbt/enron_emails_sample_questions ...\"\n )\n try:\n from datasets import load_dataset\n\n ds = load_dataset(\"corbt/enron_emails_sample_questions\", split=\"train\")\n except Exception as e:\n print(f\"⚠️ Failed to load dataset: {e}\")\n return self.queries_file\n\n kept = 0\n with open(self.queries_file, \"w\", encoding=\"utf-8\") as out:\n for i, item in enumerate(ds):\n how_realistic = float(item.get(\"how_realistic\", 0.0))\n if how_realistic < min_realism:\n continue\n qid = str(item.get(\"id\", f\"enron_q_{i}\"))\n query = item.get(\"question\", \"\")\n if not query:\n continue\n record = {\n \"id\": qid,\n \"query\": query,\n # For reference only, not used in recall metric below\n \"gt_message_ids\": item.get(\"message_ids\", []),\n }\n out.write(json.dumps(record) + \"\\n\")\n kept += 1\n print(f\"✅ Wrote {kept} queries to {self.queries_file}\")\n return self.queries_file\n\n\ndef main():\n parser = argparse.ArgumentParser(description=\"Setup Enron Emails Benchmark\")\n parser.add_argument(\n \"--emails-csv\",\n help=\"Path to emails.csv (Enron dataset). If omitted, attempt Kaggle download.\",\n )\n parser.add_argument(\"--data-dir\", default=\"data\", help=\"Data directory\")\n parser.add_argument(\"--backend\", choices=[\"hnsw\", \"diskann\"], default=\"hnsw\")\n parser.add_argument(\n \"--embedding-model\",\n default=\"sentence-transformers/all-mpnet-base-v2\",\n help=\"Embedding model for LEANN\",\n )\n parser.add_argument(\"--chunk-words\", type=int, default=256, help=\"Fixed word chunk size\")\n parser.add_argument(\"--max-emails\", type=int, help=\"Limit number of emails to process\")\n parser.add_argument(\"--skip-queries\", action=\"store_true\", help=\"Skip creating queries file\")\n parser.add_argument(\"--skip-build\", action=\"store_true\", help=\"Skip building LEANN index\")\n\n args = parser.parse_args()\n\n setup = EnronSetup(args.data_dir)\n\n # Build index\n if not args.skip_build:\n index_path = setup.build_leann_index(\n emails_csv=args.emails_csv,\n backend=args.backend,\n embedding_model=args.embedding_model,\n chunk_words=args.chunk_words,\n max_emails=args.max_emails,\n )\n\n # Build FAISS baseline from the same passages & embeddings\n setup.build_faiss_flat_baseline(index_path)\n else:\n print(\"⏭️ Skipping LEANN index build and baseline\")\n\n # Queries file (optional)\n if not args.skip_queries:\n setup.prepare_queries()\n else:\n print(\"⏭️ Skipping query preparation\")\n\n print(\"\\n🎉 Enron Emails setup completed!\")\n print(f\"📁 Data directory: {setup.data_dir.absolute()}\")\n print(\"Next steps:\")\n print(\n \"1) Evaluate recall: python evaluate_enron_emails.py --index data/enron_index_hnsw.leann --stage 2\"\n )\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "benchmarks/faiss_only.py", "content": "#!/usr/bin/env python3\n\"\"\"Test only Faiss HNSW\"\"\"\n\nimport os\nimport sys\nimport time\n\nimport psutil\n\n\ndef get_memory_usage():\n process = psutil.Process()\n return process.memory_info().rss / 1024 / 1024\n\n\nclass MemoryTracker:\n def __init__(self, name: str):\n self.name = name\n self.start_mem = get_memory_usage()\n self.stages = []\n\n def checkpoint(self, stage: str):\n current_mem = get_memory_usage()\n diff = current_mem - self.start_mem\n print(f\"[{self.name} - {stage}] Memory: {current_mem:.1f} MB (+{diff:.1f} MB)\")\n self.stages.append((stage, current_mem))\n return current_mem\n\n def summary(self):\n peak_mem = max(mem for _, mem in self.stages)\n print(f\"Peak Memory: {peak_mem:.1f} MB\")\n return peak_mem\n\n\ndef main():\n try:\n import faiss\n except ImportError:\n print(\"Faiss is not installed.\")\n print(\n \"Please install it with `uv pip install faiss-cpu` and you can then run this script again\"\n )\n sys.exit(1)\n\n from llama_index.core import (\n Settings,\n SimpleDirectoryReader,\n StorageContext,\n VectorStoreIndex,\n )\n from llama_index.core.node_parser import SentenceSplitter\n from llama_index.embeddings.huggingface import HuggingFaceEmbedding\n from llama_index.vector_stores.faiss import FaissVectorStore\n\n tracker = MemoryTracker(\"Faiss HNSW\")\n tracker.checkpoint(\"Initial\")\n\n embed_model = HuggingFaceEmbedding(model_name=\"facebook/contriever\")\n Settings.embed_model = embed_model\n tracker.checkpoint(\"After embedding model setup\")\n\n d = 768\n faiss_index = faiss.IndexHNSWFlat(d, 32)\n faiss_index.hnsw.efConstruction = 64\n tracker.checkpoint(\"After Faiss index creation\")\n\n documents = SimpleDirectoryReader(\n \"data\",\n recursive=True,\n encoding=\"utf-8\",\n required_exts=[\".pdf\", \".txt\", \".md\"],\n ).load_data()\n tracker.checkpoint(\"After document loading\")\n\n # Parse into chunks using the same splitter as LEANN\n node_parser = SentenceSplitter(\n chunk_size=256, chunk_overlap=20, separator=\" \", paragraph_separator=\"\\n\\n\"\n )\n\n tracker.checkpoint(\"After text splitter setup\")\n\n # Check if index already exists and try to load it\n index_loaded = False\n if os.path.exists(\"./storage_faiss\"):\n print(\"Loading existing Faiss HNSW index...\")\n try:\n # Use the correct Faiss loading pattern from the example\n vector_store = FaissVectorStore.from_persist_dir(\"./storage_faiss\")\n storage_context = StorageContext.from_defaults(\n vector_store=vector_store, persist_dir=\"./storage_faiss\"\n )\n from llama_index.core import load_index_from_storage\n\n index = load_index_from_storage(storage_context=storage_context)\n print(\"Index loaded from ./storage_faiss\")\n tracker.checkpoint(\"After loading existing index\")\n index_loaded = True\n except Exception as e:\n print(f\"Failed to load existing index: {e}\")\n print(\"Cleaning up corrupted index and building new one...\")\n # Clean up corrupted index\n import shutil\n\n if os.path.exists(\"./storage_faiss\"):\n shutil.rmtree(\"./storage_faiss\")\n\n if not index_loaded:\n print(\"Building new Faiss HNSW index...\")\n\n # Use the correct Faiss building pattern from the example\n vector_store = FaissVectorStore(faiss_index=faiss_index)\n storage_context = StorageContext.from_defaults(vector_store=vector_store)\n index = VectorStoreIndex.from_documents(\n documents, storage_context=storage_context, transformations=[node_parser]\n )\n tracker.checkpoint(\"After index building\")\n\n # Save index to disk using the correct pattern\n index.storage_context.persist(persist_dir=\"./storage_faiss\")\n tracker.checkpoint(\"After index saving\")\n\n # Measure runtime memory overhead\n print(\"\\nMeasuring runtime memory overhead...\")\n runtime_start_mem = get_memory_usage()\n print(f\"Before load memory: {runtime_start_mem:.1f} MB\")\n tracker.checkpoint(\"Before load memory\")\n\n query_engine = index.as_query_engine(similarity_top_k=20)\n queries = [\n \"什么是盘古大模型以及盘古开发过程中遇到了什么阴暗面,任务令一般在什么城市颁发\",\n \"What is LEANN and how does it work?\",\n \"华为诺亚方舟实验室的主要研究内容\",\n ]\n\n for i, query in enumerate(queries):\n start_time = time.time()\n _ = query_engine.query(query)\n query_time = time.time() - start_time\n print(f\"Query {i + 1} time: {query_time:.3f}s\")\n tracker.checkpoint(f\"After query {i + 1}\")\n\n runtime_end_mem = get_memory_usage()\n runtime_overhead = runtime_end_mem - runtime_start_mem\n\n peak_memory = tracker.summary()\n print(f\"Peak Memory: {peak_memory:.1f} MB\")\n print(f\"Runtime Memory Overhead: {runtime_overhead:.1f} MB\")\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "benchmarks/financebench/README.md", "content": "# FinanceBench Benchmark for LEANN-RAG\n\nFinanceBench is a benchmark for evaluating retrieval-augmented generation (RAG) systems on financial document question-answering tasks.\n\n## Dataset\n\n- **Source**: [PatronusAI/financebench](https://huggingface.co/datasets/PatronusAI/financebench)\n- **Questions**: 150 financial Q&A examples\n- **Documents**: 368 PDF files (10-K, 10-Q, 8-K, earnings reports)\n- **Companies**: Major public companies (3M, Apple, Microsoft, Amazon, etc.)\n- **Paper**: [FinanceBench: A New Benchmark for Financial Question Answering](https://arxiv.org/abs/2311.11944)\n\n## Structure\n\n```\nbenchmarks/financebench/\n├── setup_financebench.py # Downloads PDFs and builds index\n├── evaluate_financebench.py # Intelligent evaluation script\n├── data/\n│ ├── financebench_merged.jsonl # Q&A dataset\n│ ├── pdfs/ # Downloaded financial documents\n│ └── index/ # LEANN indexes\n│ └── financebench_full_hnsw.leann\n└── README.md\n```\n\n## Usage\n\n### 1. Setup (Download & Build Index)\n\n```bash\ncd benchmarks/financebench\npython setup_financebench.py\n```\n\nThis will:\n- Download the 150 Q&A examples\n- Download all 368 PDF documents (parallel processing)\n- Build a LEANN index from 53K+ text chunks\n- Verify setup with test query\n\n### 2. Evaluation\n\n```bash\n# Basic retrieval evaluation\npython evaluate_financebench.py --index data/index/financebench_full_hnsw.leann\n\n\n# RAG generation evaluation with Qwen3-8B\npython evaluate_financebench.py --index data/index/financebench_full_hnsw.leann --stage 4 --complexity 64 --llm-backend hf --model-name Qwen/Qwen3-8B --output results_qwen3.json\n```\n\n## Evaluation Methods\n\n### Retrieval Evaluation\nUses intelligent matching with three strategies:\n1. **Exact text overlap** - Direct substring matches\n2. **Number matching** - Key financial figures ($1,577, 1.2B, etc.)\n3. **Semantic similarity** - Word overlap with 20% threshold\n\n### QA Evaluation\nLLM-based answer evaluation using GPT-4o:\n- Handles numerical rounding and equivalent representations\n- Considers fractions, percentages, and decimal equivalents\n- Evaluates semantic meaning rather than exact text match\n\n## Benchmark Results\n\n### LEANN-RAG Performance (sentence-transformers/all-mpnet-base-v2)\n\n**Retrieval Metrics:**\n- **Question Coverage**: 100.0% (all questions retrieve relevant docs)\n- **Exact Match Rate**: 0.7% (substring overlap with evidence)\n- **Number Match Rate**: 120.7% (key financial figures matched)*\n- **Semantic Match Rate**: 4.7% (word overlap ≥20%)\n- **Average Search Time**: 0.097s\n\n**QA Metrics:**\n- **Accuracy**: 42.7% (LLM-evaluated answer correctness)\n- **Average QA Time**: 4.71s (end-to-end response time)\n\n**System Performance:**\n- **Index Size**: 53,985 chunks from 368 PDFs\n- **Build Time**: ~5-10 minutes with sentence-transformers/all-mpnet-base-v2\n\n*Note: Number match rate >100% indicates multiple retrieved documents contain the same financial figures, which is expected behavior for financial data appearing across multiple document sections.\n\n### LEANN-RAG Generation Performance (Qwen3-8B)\n\n- **Stage 4 (Index Comparison):**\n - Compact Index: 5.0 MB\n - Non-compact Index: 172.2 MB\n - **Storage Saving**: 97.1%\n- **Search Performance**:\n - Non-compact (no recompute): 0.009s avg per query\n - Compact (with recompute): 2.203s avg per query\n - Speed ratio: 0.004x\n\n**Generation Evaluation (20 queries, complexity=64):**\n- **Average Search Time**: 1.638s per query\n- **Average Generation Time**: 45.957s per query\n- **LLM Backend**: HuggingFace transformers\n- **Model**: Qwen/Qwen3-8B (thinking model with processing)\n- **Total Questions Processed**: 20\n\n## Options\n\n```bash\n# Use different backends\npython setup_financebench.py --backend diskann\npython evaluate_financebench.py --index data/index/financebench_full_diskann.leann\n\n# Use different embedding models\npython setup_financebench.py --embedding-model facebook/contriever\n```\n" }, { "path": "benchmarks/financebench/evaluate_financebench.py", "content": "\"\"\"\nFinanceBench Evaluation Script - Modular Recall-based Evaluation\n\"\"\"\n\nimport argparse\nimport json\nimport logging\nimport os\nimport pickle\nimport time\nfrom pathlib import Path\nfrom typing import Optional\n\nimport numpy as np\nimport openai\nfrom leann import LeannChat, LeannSearcher\nfrom leann_backend_hnsw import faiss\n\nfrom ..llm_utils import evaluate_rag, generate_hf, generate_vllm, load_hf_model, load_vllm_model\n\n# Setup logging to reduce verbose output\nlogging.basicConfig(level=logging.WARNING)\nlogging.getLogger(\"leann.api\").setLevel(logging.WARNING)\nlogging.getLogger(\"leann_backend_hnsw\").setLevel(logging.WARNING)\n\n\nclass RecallEvaluator:\n \"\"\"Stage 2: Evaluate Recall@3 (searcher vs baseline)\"\"\"\n\n def __init__(self, index_path: str, baseline_dir: str):\n self.index_path = index_path\n self.baseline_dir = baseline_dir\n self.searcher = LeannSearcher(index_path)\n\n # Load FAISS flat baseline\n baseline_index_path = os.path.join(baseline_dir, \"faiss_flat.index\")\n metadata_path = os.path.join(baseline_dir, \"metadata.pkl\")\n\n self.faiss_index = faiss.read_index(baseline_index_path)\n with open(metadata_path, \"rb\") as f:\n self.passage_ids = pickle.load(f)\n print(f\"📚 Loaded FAISS flat baseline with {self.faiss_index.ntotal} vectors\")\n\n def evaluate_recall_at_3(\n self, queries: list[str], complexity: int = 64, recompute_embeddings: bool = True\n ) -> float:\n \"\"\"Evaluate recall@3 for given queries at specified complexity\"\"\"\n recompute_str = \"with recompute\" if recompute_embeddings else \"no recompute\"\n print(f\"🔍 Evaluating recall@3 with complexity={complexity} ({recompute_str})...\")\n\n total_recall = 0.0\n num_queries = len(queries)\n\n for i, query in enumerate(queries):\n # Get ground truth: search with FAISS flat\n from leann.api import compute_embeddings\n\n query_embedding = compute_embeddings(\n [query],\n self.searcher.embedding_model,\n mode=self.searcher.embedding_mode,\n use_server=False,\n ).astype(np.float32)\n\n # Search FAISS flat for ground truth using LEANN's modified faiss API\n n = query_embedding.shape[0] # Number of queries\n k = 3 # Number of nearest neighbors\n distances = np.zeros((n, k), dtype=np.float32)\n labels = np.zeros((n, k), dtype=np.int64)\n\n self.faiss_index.search(\n n,\n faiss.swig_ptr(query_embedding),\n k,\n faiss.swig_ptr(distances),\n faiss.swig_ptr(labels),\n )\n\n # Extract the results\n baseline_ids = {self.passage_ids[idx] for idx in labels[0]}\n\n # Search with LEANN at specified complexity\n test_results = self.searcher.search(\n query,\n top_k=3,\n complexity=complexity,\n recompute_embeddings=recompute_embeddings,\n )\n test_ids = {result.id for result in test_results}\n\n # Calculate recall@3 = |intersection| / |ground_truth|\n intersection = test_ids.intersection(baseline_ids)\n recall = len(intersection) / 3.0 # Ground truth size is 3\n total_recall += recall\n\n if i < 3: # Show first few examples\n print(f\" Query {i + 1}: '{query[:50]}...' -> Recall@3: {recall:.3f}\")\n print(f\" FAISS ground truth: {list(baseline_ids)}\")\n print(f\" LEANN results (C={complexity}, {recompute_str}): {list(test_ids)}\")\n print(f\" Intersection: {list(intersection)}\")\n\n avg_recall = total_recall / num_queries\n print(f\"📊 Average Recall@3: {avg_recall:.3f} ({avg_recall * 100:.1f}%)\")\n return avg_recall\n\n def cleanup(self):\n \"\"\"Cleanup resources\"\"\"\n if hasattr(self, \"searcher\"):\n self.searcher.cleanup()\n\n\nclass FinanceBenchEvaluator:\n def __init__(self, index_path: str, openai_api_key: Optional[str] = None):\n self.index_path = index_path\n self.openai_client = openai.OpenAI(api_key=openai_api_key) if openai_api_key else None\n\n self.searcher = LeannSearcher(index_path)\n self.chat = LeannChat(index_path) if openai_api_key else None\n\n def load_dataset(self, dataset_path: str = \"data/financebench_merged.jsonl\"):\n \"\"\"Load FinanceBench dataset\"\"\"\n data = []\n with open(dataset_path, encoding=\"utf-8\") as f:\n for line in f:\n if line.strip():\n data.append(json.loads(line))\n\n print(f\"📊 Loaded {len(data)} FinanceBench examples\")\n return data\n\n def analyze_index_sizes(self) -> dict:\n \"\"\"Analyze index sizes with and without embeddings\"\"\"\n\n print(\"📏 Analyzing index sizes...\")\n\n # Get all index-related files\n index_path = Path(self.index_path)\n index_dir = index_path.parent\n index_name = index_path.stem # Remove .leann extension\n\n sizes = {}\n total_with_embeddings = 0\n\n # Core index files\n index_file = index_dir / f\"{index_name}.index\"\n meta_file = index_dir / f\"{index_path.name}.meta.json\" # Keep .leann for meta file\n passages_file = index_dir / f\"{index_path.name}.passages.jsonl\" # Keep .leann for passages\n passages_idx_file = index_dir / f\"{index_path.name}.passages.idx\" # Keep .leann for idx\n\n for file_path, name in [\n (index_file, \"index\"),\n (meta_file, \"metadata\"),\n (passages_file, \"passages_text\"),\n (passages_idx_file, \"passages_index\"),\n ]:\n if file_path.exists():\n size_mb = file_path.stat().st_size / (1024 * 1024)\n sizes[name] = size_mb\n total_with_embeddings += size_mb\n\n else:\n sizes[name] = 0\n\n sizes[\"total_with_embeddings\"] = total_with_embeddings\n sizes[\"index_only_mb\"] = sizes[\"index\"] # Just the .index file for fair comparison\n\n print(f\" 📁 Total index size: {total_with_embeddings:.1f} MB\")\n print(f\" 📁 Index file only: {sizes['index']:.1f} MB\")\n\n return sizes\n\n def create_compact_index_for_comparison(self, compact_index_path: str) -> dict:\n \"\"\"Create a compact index for comparison purposes\"\"\"\n print(\"🏗️ Building compact index from existing passages...\")\n\n # Load existing passages from current index\n\n from leann import LeannBuilder\n\n current_index_path = Path(self.index_path)\n current_index_dir = current_index_path.parent\n current_index_name = current_index_path.name\n\n # Read metadata to get passage source\n meta_path = current_index_dir / f\"{current_index_name}.meta.json\"\n with open(meta_path) as f:\n import json\n\n meta = json.load(f)\n\n passage_source = meta[\"passage_sources\"][0]\n passage_file = passage_source[\"path\"]\n\n # Convert relative path to absolute\n if not Path(passage_file).is_absolute():\n passage_file = current_index_dir / Path(passage_file).name\n\n print(f\"📄 Loading passages from {passage_file}...\")\n\n # Build compact index with same passages\n builder = LeannBuilder(\n backend_name=\"hnsw\",\n embedding_model=meta[\"embedding_model\"],\n embedding_mode=meta.get(\"embedding_mode\", \"sentence-transformers\"),\n is_recompute=True, # Enable recompute (no stored embeddings)\n is_compact=True, # Enable compact storage\n **meta.get(\"backend_kwargs\", {}),\n )\n\n # Load all passages\n with open(passage_file, encoding=\"utf-8\") as f:\n for line in f:\n if line.strip():\n data = json.loads(line)\n builder.add_text(data[\"text\"], metadata=data.get(\"metadata\", {}))\n\n print(f\"🔨 Building compact index at {compact_index_path}...\")\n builder.build_index(compact_index_path)\n\n # Analyze the compact index size\n temp_evaluator = FinanceBenchEvaluator(compact_index_path)\n compact_sizes = temp_evaluator.analyze_index_sizes()\n compact_sizes[\"index_type\"] = \"compact\"\n\n return compact_sizes\n\n def create_non_compact_index_for_comparison(self, non_compact_index_path: str) -> dict:\n \"\"\"Create a non-compact index for comparison purposes\"\"\"\n print(\"🏗️ Building non-compact index from existing passages...\")\n\n # Load existing passages from current index\n\n from leann import LeannBuilder\n\n current_index_path = Path(self.index_path)\n current_index_dir = current_index_path.parent\n current_index_name = current_index_path.name\n\n # Read metadata to get passage source\n meta_path = current_index_dir / f\"{current_index_name}.meta.json\"\n with open(meta_path) as f:\n import json\n\n meta = json.load(f)\n\n passage_source = meta[\"passage_sources\"][0]\n passage_file = passage_source[\"path\"]\n\n # Convert relative path to absolute\n if not Path(passage_file).is_absolute():\n passage_file = current_index_dir / Path(passage_file).name\n\n print(f\"📄 Loading passages from {passage_file}...\")\n\n # Build non-compact index with same passages\n builder = LeannBuilder(\n backend_name=\"hnsw\",\n embedding_model=meta[\"embedding_model\"],\n embedding_mode=meta.get(\"embedding_mode\", \"sentence-transformers\"),\n is_recompute=False, # Disable recompute (store embeddings)\n is_compact=False, # Disable compact storage\n **{\n k: v\n for k, v in meta.get(\"backend_kwargs\", {}).items()\n if k not in [\"is_recompute\", \"is_compact\"]\n },\n )\n\n # Load all passages\n with open(passage_file, encoding=\"utf-8\") as f:\n for line in f:\n if line.strip():\n data = json.loads(line)\n builder.add_text(data[\"text\"], metadata=data.get(\"metadata\", {}))\n\n print(f\"🔨 Building non-compact index at {non_compact_index_path}...\")\n builder.build_index(non_compact_index_path)\n\n # Analyze the non-compact index size\n temp_evaluator = FinanceBenchEvaluator(non_compact_index_path)\n non_compact_sizes = temp_evaluator.analyze_index_sizes()\n non_compact_sizes[\"index_type\"] = \"non_compact\"\n\n return non_compact_sizes\n\n def compare_index_performance(\n self, non_compact_path: str, compact_path: str, test_data: list, complexity: int\n ) -> dict:\n \"\"\"Compare performance between non-compact and compact indexes\"\"\"\n print(\"⚡ Comparing search performance between indexes...\")\n\n import time\n\n from leann import LeannSearcher\n\n # Test queries\n test_queries = [item[\"question\"] for item in test_data[:5]]\n\n results = {\n \"non_compact\": {\"search_times\": []},\n \"compact\": {\"search_times\": []},\n \"avg_search_times\": {},\n \"speed_ratio\": 0.0,\n }\n\n # Test non-compact index (no recompute)\n print(\" 🔍 Testing non-compact index (no recompute)...\")\n non_compact_searcher = LeannSearcher(non_compact_path)\n\n for query in test_queries:\n start_time = time.time()\n _ = non_compact_searcher.search(\n query, top_k=3, complexity=complexity, recompute_embeddings=False\n )\n search_time = time.time() - start_time\n results[\"non_compact\"][\"search_times\"].append(search_time)\n\n # Test compact index (with recompute)\n print(\" 🔍 Testing compact index (with recompute)...\")\n compact_searcher = LeannSearcher(compact_path)\n\n for query in test_queries:\n start_time = time.time()\n _ = compact_searcher.search(\n query, top_k=3, complexity=complexity, recompute_embeddings=True\n )\n search_time = time.time() - start_time\n results[\"compact\"][\"search_times\"].append(search_time)\n\n # Calculate averages\n results[\"avg_search_times\"][\"non_compact\"] = sum(\n results[\"non_compact\"][\"search_times\"]\n ) / len(results[\"non_compact\"][\"search_times\"])\n results[\"avg_search_times\"][\"compact\"] = sum(results[\"compact\"][\"search_times\"]) / len(\n results[\"compact\"][\"search_times\"]\n )\n\n # Performance ratio\n if results[\"avg_search_times\"][\"compact\"] > 0:\n results[\"speed_ratio\"] = (\n results[\"avg_search_times\"][\"non_compact\"] / results[\"avg_search_times\"][\"compact\"]\n )\n else:\n results[\"speed_ratio\"] = float(\"inf\")\n\n print(\n f\" Non-compact (no recompute): {results['avg_search_times']['non_compact']:.3f}s avg\"\n )\n print(f\" Compact (with recompute): {results['avg_search_times']['compact']:.3f}s avg\")\n print(f\" Speed ratio: {results['speed_ratio']:.2f}x\")\n\n # Cleanup\n non_compact_searcher.cleanup()\n compact_searcher.cleanup()\n\n return results\n\n def evaluate_timing_breakdown(\n self, data: list[dict], max_samples: Optional[int] = None\n ) -> dict:\n \"\"\"Evaluate timing breakdown and accuracy by hacking LeannChat.ask() for separated timing\"\"\"\n if not self.chat or not self.openai_client:\n print(\"⚠️ Skipping timing evaluation (no OpenAI API key provided)\")\n return {\n \"total_questions\": 0,\n \"avg_search_time\": 0.0,\n \"avg_generation_time\": 0.0,\n \"avg_total_time\": 0.0,\n \"accuracy\": 0.0,\n }\n\n print(\"🔍🤖 Evaluating timing breakdown and accuracy (search + generation)...\")\n\n if max_samples:\n data = data[:max_samples]\n print(f\"📝 Using first {max_samples} samples for timing evaluation\")\n\n search_times = []\n generation_times = []\n total_times = []\n correct_answers = 0\n\n for i, item in enumerate(data):\n question = item[\"question\"]\n ground_truth = item[\"answer\"]\n\n try:\n # Hack: Monkey-patch the ask method to capture internal timing\n original_ask = self.chat.ask\n captured_search_time = None\n captured_generation_time = None\n\n def patched_ask(*args, **kwargs):\n nonlocal captured_search_time, captured_generation_time\n\n # Time the search part\n search_start = time.time()\n results = self.chat.searcher.search(args[0], top_k=3, complexity=64)\n captured_search_time = time.time() - search_start\n\n # Time the generation part\n context = \"\\n\\n\".join([r.text for r in results])\n prompt = (\n \"Here is some retrieved context that might help answer your question:\\n\\n\"\n f\"{context}\\n\\n\"\n f\"Question: {args[0]}\\n\\n\"\n \"Please provide the best answer you can based on this context and your knowledge.\"\n )\n\n generation_start = time.time()\n answer = self.chat.llm.ask(prompt)\n captured_generation_time = time.time() - generation_start\n\n return answer\n\n # Apply the patch\n self.chat.ask = patched_ask\n\n # Time the total QA\n total_start = time.time()\n generated_answer = self.chat.ask(question)\n total_time = time.time() - total_start\n\n # Restore original method\n self.chat.ask = original_ask\n\n # Store the timings\n search_times.append(captured_search_time)\n generation_times.append(captured_generation_time)\n total_times.append(total_time)\n\n # Check accuracy using LLM as judge\n is_correct = self._check_answer_accuracy(generated_answer, ground_truth, question)\n if is_correct:\n correct_answers += 1\n\n status = \"✅\" if is_correct else \"❌\"\n print(\n f\"Question {i + 1}/{len(data)}: {status} Search={captured_search_time:.3f}s, Gen={captured_generation_time:.3f}s, Total={total_time:.3f}s\"\n )\n print(f\" GT: {ground_truth}\")\n print(f\" Gen: {generated_answer[:100]}...\")\n\n except Exception as e:\n print(f\" ❌ Error: {e}\")\n search_times.append(0.0)\n generation_times.append(0.0)\n total_times.append(0.0)\n\n accuracy = correct_answers / len(data) if data else 0.0\n\n metrics = {\n \"total_questions\": len(data),\n \"avg_search_time\": sum(search_times) / len(search_times) if search_times else 0.0,\n \"avg_generation_time\": sum(generation_times) / len(generation_times)\n if generation_times\n else 0.0,\n \"avg_total_time\": sum(total_times) / len(total_times) if total_times else 0.0,\n \"accuracy\": accuracy,\n \"correct_answers\": correct_answers,\n \"search_times\": search_times,\n \"generation_times\": generation_times,\n \"total_times\": total_times,\n }\n\n return metrics\n\n def _check_answer_accuracy(\n self, generated_answer: str, ground_truth: str, question: str\n ) -> bool:\n \"\"\"Check if generated answer matches ground truth using LLM as judge\"\"\"\n judge_prompt = f\"\"\"You are an expert judge evaluating financial question answering.\n\nQuestion: {question}\n\nGround Truth Answer: {ground_truth}\n\nGenerated Answer: {generated_answer}\n\nTask: Determine if the generated answer is factually correct compared to the ground truth. Focus on:\n1. Numerical accuracy (exact values, units, currency)\n2. Key financial concepts and terminology\n3. Overall factual correctness\n\nFor financial data, small formatting differences are OK (e.g., \"$1,577\" vs \"1577 million\" vs \"$1.577 billion\"), but the core numerical value must match.\n\nRespond with exactly one word: \"CORRECT\" if the generated answer is factually accurate, or \"INCORRECT\" if it's wrong or significantly different.\"\"\"\n\n try:\n judge_response = self.openai_client.chat.completions.create(\n model=\"gpt-4o-mini\",\n messages=[{\"role\": \"user\", \"content\": judge_prompt}],\n max_tokens=10,\n temperature=0,\n )\n judgment = judge_response.choices[0].message.content.strip().upper()\n return judgment == \"CORRECT\"\n except Exception as e:\n print(f\" ⚠️ Judge error: {e}, falling back to string matching\")\n # Fallback to simple string matching\n gen_clean = generated_answer.strip().lower().replace(\"$\", \"\").replace(\",\", \"\")\n gt_clean = ground_truth.strip().lower().replace(\"$\", \"\").replace(\",\", \"\")\n return gt_clean in gen_clean\n\n def _print_results(self, timing_metrics: dict):\n \"\"\"Print evaluation results\"\"\"\n print(\"\\n🎯 EVALUATION RESULTS\")\n print(\"=\" * 50)\n\n # Index comparison analysis\n if \"current_index\" in timing_metrics and \"non_compact_index\" in timing_metrics:\n print(\"\\n📏 Index Comparison Analysis:\")\n current = timing_metrics[\"current_index\"]\n non_compact = timing_metrics[\"non_compact_index\"]\n\n print(f\" Compact index (current): {current.get('total_with_embeddings', 0):.1f} MB\")\n print(\n f\" Non-compact index (with embeddings): {non_compact.get('total_with_embeddings', 0):.1f} MB\"\n )\n print(\n f\" Storage saving by compact: {timing_metrics.get('storage_saving_percent', 0):.1f}%\"\n )\n\n print(\" Component breakdown (non-compact):\")\n print(f\" - Main index: {non_compact.get('index', 0):.1f} MB\")\n print(f\" - Passages text: {non_compact.get('passages_text', 0):.1f} MB\")\n print(f\" - Passages index: {non_compact.get('passages_index', 0):.1f} MB\")\n print(f\" - Metadata: {non_compact.get('metadata', 0):.1f} MB\")\n\n # Performance comparison\n if \"performance_comparison\" in timing_metrics:\n perf = timing_metrics[\"performance_comparison\"]\n print(\"\\n⚡ Performance Comparison:\")\n print(\n f\" Non-compact (no recompute): {perf.get('avg_search_times', {}).get('non_compact', 0):.3f}s avg\"\n )\n print(\n f\" Compact (with recompute): {perf.get('avg_search_times', {}).get('compact', 0):.3f}s avg\"\n )\n print(f\" Speed ratio: {perf.get('speed_ratio', 0):.2f}x\")\n\n # Legacy single index analysis (fallback)\n if \"total_with_embeddings\" in timing_metrics and \"current_index\" not in timing_metrics:\n print(\"\\n📏 Index Size Analysis:\")\n print(f\" Total index size: {timing_metrics.get('total_with_embeddings', 0):.1f} MB\")\n\n print(\"\\n📊 Accuracy:\")\n print(f\" Accuracy: {timing_metrics.get('accuracy', 0) * 100:.1f}%\")\n print(\n f\" Correct Answers: {timing_metrics.get('correct_answers', 0)}/{timing_metrics.get('total_questions', 0)}\"\n )\n\n print(\"\\n📊 Timing Breakdown:\")\n print(f\" Total Questions: {timing_metrics.get('total_questions', 0)}\")\n print(f\" Avg Search Time: {timing_metrics.get('avg_search_time', 0):.3f}s\")\n print(f\" Avg Generation Time: {timing_metrics.get('avg_generation_time', 0):.3f}s\")\n print(f\" Avg Total Time: {timing_metrics.get('avg_total_time', 0):.3f}s\")\n\n if timing_metrics.get(\"avg_total_time\", 0) > 0:\n search_pct = (\n timing_metrics.get(\"avg_search_time\", 0)\n / timing_metrics.get(\"avg_total_time\", 1)\n * 100\n )\n gen_pct = (\n timing_metrics.get(\"avg_generation_time\", 0)\n / timing_metrics.get(\"avg_total_time\", 1)\n * 100\n )\n print(\"\\n📈 Time Distribution:\")\n print(f\" Search: {search_pct:.1f}%\")\n print(f\" Generation: {gen_pct:.1f}%\")\n\n def cleanup(self):\n \"\"\"Cleanup resources\"\"\"\n if self.searcher:\n self.searcher.cleanup()\n\n\ndef main():\n parser = argparse.ArgumentParser(description=\"Modular FinanceBench Evaluation\")\n parser.add_argument(\"--index\", required=True, help=\"Path to LEANN index\")\n parser.add_argument(\"--dataset\", default=\"data/financebench_merged.jsonl\", help=\"Dataset path\")\n parser.add_argument(\n \"--stage\",\n choices=[\"2\", \"3\", \"4\", \"all\"],\n default=\"all\",\n help=\"Which stage to run (2=recall, 3=complexity, 4=generation)\",\n )\n parser.add_argument(\"--complexity\", type=int, default=None, help=\"Complexity for search\")\n parser.add_argument(\"--baseline-dir\", default=\"baseline\", help=\"Baseline output directory\")\n parser.add_argument(\"--openai-api-key\", help=\"OpenAI API key for generation evaluation\")\n parser.add_argument(\"--output\", help=\"Save results to JSON file\")\n parser.add_argument(\n \"--llm-backend\", choices=[\"openai\", \"hf\", \"vllm\"], default=\"openai\", help=\"LLM backend\"\n )\n parser.add_argument(\"--model-name\", default=\"Qwen3-8B\", help=\"Model name for HF/vLLM\")\n\n args = parser.parse_args()\n\n try:\n # Check if baseline exists\n baseline_index_path = os.path.join(args.baseline_dir, \"faiss_flat.index\")\n if not os.path.exists(baseline_index_path):\n print(f\"❌ FAISS baseline not found at {baseline_index_path}\")\n print(\"💡 Please run setup_financebench.py first to build the baseline\")\n exit(1)\n\n if args.stage == \"2\" or args.stage == \"all\":\n # Stage 2: Recall@3 evaluation\n print(\"🚀 Starting Stage 2: Recall@3 evaluation\")\n\n evaluator = RecallEvaluator(args.index, args.baseline_dir)\n\n # Load FinanceBench queries for testing\n print(\"📖 Loading FinanceBench dataset...\")\n queries = []\n with open(args.dataset, encoding=\"utf-8\") as f:\n for line in f:\n if line.strip():\n data = json.loads(line)\n queries.append(data[\"question\"])\n\n # Test with more queries for robust measurement\n test_queries = queries[:2000]\n print(f\"🧪 Testing with {len(test_queries)} queries\")\n\n # Test with complexity 64\n complexity = 64\n recall = evaluator.evaluate_recall_at_3(test_queries, complexity)\n print(f\"📈 Recall@3 at complexity {complexity}: {recall * 100:.1f}%\")\n\n evaluator.cleanup()\n print(\"✅ Stage 2 completed!\\n\")\n\n # Shared non-compact index path for Stage 3 and 4\n non_compact_index_path = args.index.replace(\".leann\", \"_noncompact.leann\")\n complexity = args.complexity\n\n if args.stage == \"3\" or args.stage == \"all\":\n # Stage 3: Binary search for 90% recall complexity (using non-compact index for speed)\n print(\"🚀 Starting Stage 3: Binary search for 90% recall complexity\")\n print(\n \"💡 Creating non-compact index for fast binary search with recompute_embeddings=False\"\n )\n\n # Create non-compact index for binary search (will be reused in Stage 4)\n print(\"🏗️ Creating non-compact index for binary search...\")\n evaluator = FinanceBenchEvaluator(args.index)\n evaluator.create_non_compact_index_for_comparison(non_compact_index_path)\n\n # Use non-compact index for binary search\n binary_search_evaluator = RecallEvaluator(non_compact_index_path, args.baseline_dir)\n\n # Load queries for testing\n print(\"📖 Loading FinanceBench dataset...\")\n queries = []\n with open(args.dataset, encoding=\"utf-8\") as f:\n for line in f:\n if line.strip():\n data = json.loads(line)\n queries.append(data[\"question\"])\n\n # Use more queries for robust measurement\n test_queries = queries[:200]\n print(f\"🧪 Testing with {len(test_queries)} queries\")\n\n # Binary search for 90% recall complexity (without recompute for speed)\n target_recall = 0.9\n min_complexity, max_complexity = 1, 32\n\n print(f\"🔍 Binary search for {target_recall * 100}% recall complexity...\")\n print(f\"Search range: {min_complexity} to {max_complexity}\")\n\n best_complexity = None\n best_recall = 0.0\n\n while min_complexity <= max_complexity:\n mid_complexity = (min_complexity + max_complexity) // 2\n\n print(\n f\"\\n🧪 Testing complexity {mid_complexity} (no recompute, non-compact index)...\"\n )\n # Use recompute_embeddings=False on non-compact index for fast binary search\n recall = binary_search_evaluator.evaluate_recall_at_3(\n test_queries, mid_complexity, recompute_embeddings=False\n )\n\n print(\n f\" Complexity {mid_complexity}: Recall@3 = {recall:.3f} ({recall * 100:.1f}%)\"\n )\n\n if recall >= target_recall:\n best_complexity = mid_complexity\n best_recall = recall\n max_complexity = mid_complexity - 1\n print(\" ✅ Target reached! Searching for lower complexity...\")\n else:\n min_complexity = mid_complexity + 1\n print(\" ❌ Below target. Searching for higher complexity...\")\n\n if best_complexity is not None:\n print(\"\\n🎯 Optimal complexity found!\")\n print(f\" Complexity: {best_complexity}\")\n print(f\" Recall@3: {best_recall:.3f} ({best_recall * 100:.1f}%)\")\n\n # Test a few complexities around the optimal one for verification\n print(\"\\n🔬 Verification test around optimal complexity:\")\n verification_complexities = [\n max(1, best_complexity - 2),\n max(1, best_complexity - 1),\n best_complexity,\n best_complexity + 1,\n best_complexity + 2,\n ]\n\n for complexity in verification_complexities:\n if complexity <= 512: # reasonable upper bound\n recall = binary_search_evaluator.evaluate_recall_at_3(\n test_queries, complexity, recompute_embeddings=False\n )\n status = \"✅\" if recall >= target_recall else \"❌\"\n print(f\" {status} Complexity {complexity:3d}: {recall * 100:5.1f}%\")\n\n # Now test the optimal complexity with compact index and recompute for comparison\n print(\n f\"\\n🔄 Testing optimal complexity {best_complexity} on compact index WITH recompute...\"\n )\n compact_evaluator = RecallEvaluator(args.index, args.baseline_dir)\n recall_with_recompute = compact_evaluator.evaluate_recall_at_3(\n test_queries[:10], best_complexity, recompute_embeddings=True\n )\n print(\n f\" ✅ Complexity {best_complexity} (compact index with recompute): {recall_with_recompute * 100:.1f}%\"\n )\n complexity = best_complexity\n print(\n f\" 📊 Recall difference: {abs(best_recall - recall_with_recompute) * 100:.2f}%\"\n )\n compact_evaluator.cleanup()\n else:\n print(f\"\\n❌ Could not find complexity achieving {target_recall * 100}% recall\")\n print(\"All tested complexities were below target.\")\n\n # Cleanup evaluators (keep non-compact index for Stage 4)\n binary_search_evaluator.cleanup()\n evaluator.cleanup()\n\n print(\"✅ Stage 3 completed! Non-compact index saved for Stage 4.\\n\")\n\n if args.stage == \"4\" or args.stage == \"all\":\n # Stage 4: Comprehensive evaluation with dual index comparison\n print(\"🚀 Starting Stage 4: Comprehensive evaluation with dual index comparison\")\n\n # Use FinanceBench evaluator for QA evaluation\n evaluator = FinanceBenchEvaluator(\n args.index, args.openai_api_key if args.llm_backend == \"openai\" else None\n )\n\n print(\"📖 Loading FinanceBench dataset...\")\n data = evaluator.load_dataset(args.dataset)\n\n # Step 1: Analyze current (compact) index\n print(\"\\n📏 Analyzing current index (compact, pruned)...\")\n compact_size_metrics = evaluator.analyze_index_sizes()\n compact_size_metrics[\"index_type\"] = \"compact\"\n\n # Step 2: Use existing non-compact index or create if needed\n from pathlib import Path\n\n if Path(non_compact_index_path).exists():\n print(\n f\"\\n📁 Using existing non-compact index from Stage 3: {non_compact_index_path}\"\n )\n temp_evaluator = FinanceBenchEvaluator(non_compact_index_path)\n non_compact_size_metrics = temp_evaluator.analyze_index_sizes()\n non_compact_size_metrics[\"index_type\"] = \"non_compact\"\n else:\n print(\"\\n🏗️ Creating non-compact index (with embeddings) for comparison...\")\n non_compact_size_metrics = evaluator.create_non_compact_index_for_comparison(\n non_compact_index_path\n )\n\n # Step 3: Compare index sizes\n print(\"\\n📊 Index size comparison:\")\n print(\n f\" Compact index (current): {compact_size_metrics['total_with_embeddings']:.1f} MB\"\n )\n print(\n f\" Non-compact index: {non_compact_size_metrics['total_with_embeddings']:.1f} MB\"\n )\n print(\"\\n📊 Index-only size comparison (.index file only):\")\n print(f\" Compact index: {compact_size_metrics['index_only_mb']:.1f} MB\")\n print(f\" Non-compact index: {non_compact_size_metrics['index_only_mb']:.1f} MB\")\n # Use index-only size for fair comparison (same as Enron emails)\n storage_saving = (\n (non_compact_size_metrics[\"index_only_mb\"] - compact_size_metrics[\"index_only_mb\"])\n / non_compact_size_metrics[\"index_only_mb\"]\n * 100\n )\n print(f\" Storage saving by compact: {storage_saving:.1f}%\")\n\n # Step 4: Performance comparison between the two indexes\n if complexity is None:\n raise ValueError(\"Complexity is required for performance comparison\")\n\n print(\"\\n⚡ Performance comparison between indexes...\")\n performance_metrics = evaluator.compare_index_performance(\n non_compact_index_path, args.index, data[:10], complexity=complexity\n )\n\n # Step 5: Generation evaluation\n test_samples = 20\n print(f\"\\n🧪 Testing with first {test_samples} samples for generation analysis\")\n\n if args.llm_backend == \"openai\" and args.openai_api_key:\n print(\"🔍🤖 Running OpenAI-based generation evaluation...\")\n evaluation_start = time.time()\n timing_metrics = evaluator.evaluate_timing_breakdown(data[:test_samples])\n evaluation_time = time.time() - evaluation_start\n else:\n print(\n f\"🔍🤖 Running {args.llm_backend} generation evaluation with {args.model_name}...\"\n )\n try:\n # Load LLM\n if args.llm_backend == \"hf\":\n tokenizer, model = load_hf_model(args.model_name)\n\n def llm_func(prompt):\n return generate_hf(tokenizer, model, prompt)\n else: # vllm\n llm, sampling_params = load_vllm_model(args.model_name)\n\n def llm_func(prompt):\n return generate_vllm(llm, sampling_params, prompt)\n\n # Simple generation evaluation\n queries = [item[\"question\"] for item in data[:test_samples]]\n gen_results = evaluate_rag(\n evaluator.searcher,\n llm_func,\n queries,\n domain=\"finance\",\n complexity=complexity,\n )\n\n timing_metrics = {\n \"total_questions\": len(queries),\n \"avg_search_time\": gen_results[\"avg_search_time\"],\n \"avg_generation_time\": gen_results[\"avg_generation_time\"],\n \"results\": gen_results[\"results\"],\n }\n evaluation_time = time.time()\n\n except Exception as e:\n print(f\"❌ Generation evaluation failed: {e}\")\n timing_metrics = {\n \"total_questions\": 0,\n \"avg_search_time\": 0,\n \"avg_generation_time\": 0,\n }\n evaluation_time = 0\n\n # Combine all metrics\n combined_metrics = {\n **timing_metrics,\n \"total_evaluation_time\": evaluation_time,\n \"current_index\": compact_size_metrics,\n \"non_compact_index\": non_compact_size_metrics,\n \"performance_comparison\": performance_metrics,\n \"storage_saving_percent\": storage_saving,\n }\n\n # Print results\n print(\"\\n📊 Generation Results:\")\n print(f\" Total Questions: {timing_metrics.get('total_questions', 0)}\")\n print(f\" Avg Search Time: {timing_metrics.get('avg_search_time', 0):.3f}s\")\n print(f\" Avg Generation Time: {timing_metrics.get('avg_generation_time', 0):.3f}s\")\n\n # Save results if requested\n if args.output:\n print(f\"\\n💾 Saving results to {args.output}...\")\n with open(args.output, \"w\") as f:\n json.dump(combined_metrics, f, indent=2, default=str)\n print(f\"✅ Results saved to {args.output}\")\n\n evaluator.cleanup()\n print(\"✅ Stage 4 completed!\\n\")\n\n if args.stage == \"all\":\n print(\"🎉 All evaluation stages completed successfully!\")\n print(\"\\n📋 Summary:\")\n print(\" Stage 2: ✅ Recall@3 evaluation completed\")\n print(\" Stage 3: ✅ Optimal complexity found\")\n print(\" Stage 4: ✅ Generation accuracy & timing evaluation completed\")\n print(\"\\n🔧 Recommended next steps:\")\n print(\" - Use optimal complexity for best speed/accuracy balance\")\n print(\" - Review accuracy and timing breakdown for performance optimization\")\n print(\" - Run full evaluation on complete dataset if needed\")\n\n # Clean up non-compact index after all stages complete\n print(\"\\n🧹 Cleaning up temporary non-compact index...\")\n from pathlib import Path\n\n if Path(non_compact_index_path).exists():\n temp_index_dir = Path(non_compact_index_path).parent\n temp_index_name = Path(non_compact_index_path).name\n for temp_file in temp_index_dir.glob(f\"{temp_index_name}*\"):\n temp_file.unlink()\n print(f\"✅ Cleaned up {non_compact_index_path}\")\n else:\n print(\"📝 No temporary index to clean up\")\n except KeyboardInterrupt:\n print(\"\\n⚠️ Evaluation interrupted by user\")\n exit(1)\n except Exception as e:\n print(f\"\\n❌ Stage {args.stage} failed: {e}\")\n exit(1)\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "benchmarks/financebench/setup_financebench.py", "content": "#!/usr/bin/env python3\n\"\"\"\nFinanceBench Complete Setup Script\nDownloads all PDFs and builds full LEANN datastore\n\"\"\"\n\nimport argparse\nimport os\nimport re\nimport time\nfrom concurrent.futures import ThreadPoolExecutor, as_completed\nfrom pathlib import Path\nfrom threading import Lock\n\nimport pymupdf\nimport requests\nfrom leann import LeannBuilder, LeannSearcher\nfrom tqdm import tqdm\n\n\nclass FinanceBenchSetup:\n def __init__(self, data_dir: str = \"data\"):\n self.base_dir = Path(__file__).parent # benchmarks/financebench/\n self.data_dir = self.base_dir / data_dir\n self.pdf_dir = self.data_dir / \"pdfs\"\n self.dataset_file = self.data_dir / \"financebench_merged.jsonl\"\n self.index_dir = self.data_dir / \"index\"\n self.download_lock = Lock()\n\n def download_dataset(self):\n \"\"\"Download the main FinanceBench dataset\"\"\"\n print(\"📊 Downloading FinanceBench dataset...\")\n self.data_dir.mkdir(parents=True, exist_ok=True)\n\n if self.dataset_file.exists():\n print(f\"✅ Dataset already exists: {self.dataset_file}\")\n return\n\n url = \"https://huggingface.co/datasets/PatronusAI/financebench/raw/main/financebench_merged.jsonl\"\n response = requests.get(url, stream=True)\n response.raise_for_status()\n\n with open(self.dataset_file, \"wb\") as f:\n for chunk in response.iter_content(chunk_size=8192):\n f.write(chunk)\n\n print(f\"✅ Dataset downloaded: {self.dataset_file}\")\n\n def get_pdf_list(self):\n \"\"\"Get list of all PDF files from GitHub\"\"\"\n print(\"📋 Fetching PDF list from GitHub...\")\n\n response = requests.get(\n \"https://api.github.com/repos/patronus-ai/financebench/contents/pdfs\"\n )\n response.raise_for_status()\n pdf_files = response.json()\n\n print(f\"Found {len(pdf_files)} PDF files\")\n return pdf_files\n\n def download_single_pdf(self, pdf_info, position):\n \"\"\"Download a single PDF file\"\"\"\n pdf_name = pdf_info[\"name\"]\n pdf_path = self.pdf_dir / pdf_name\n\n # Skip if already downloaded\n if pdf_path.exists() and pdf_path.stat().st_size > 0:\n return f\"✅ {pdf_name} (cached)\"\n\n try:\n # Download PDF\n response = requests.get(pdf_info[\"download_url\"], timeout=60)\n response.raise_for_status()\n\n # Write to file\n with self.download_lock:\n with open(pdf_path, \"wb\") as f:\n f.write(response.content)\n\n return f\"✅ {pdf_name} ({len(response.content) // 1024}KB)\"\n\n except Exception as e:\n return f\"❌ {pdf_name}: {e!s}\"\n\n def download_all_pdfs(self, max_workers: int = 5):\n \"\"\"Download all PDF files with parallel processing\"\"\"\n self.pdf_dir.mkdir(parents=True, exist_ok=True)\n\n pdf_files = self.get_pdf_list()\n\n print(f\"📥 Downloading {len(pdf_files)} PDFs with {max_workers} workers...\")\n\n with ThreadPoolExecutor(max_workers=max_workers) as executor:\n # Submit all download tasks\n future_to_pdf = {\n executor.submit(self.download_single_pdf, pdf_info, i): pdf_info[\"name\"]\n for i, pdf_info in enumerate(pdf_files)\n }\n\n # Process completed downloads with progress bar\n with tqdm(total=len(pdf_files), desc=\"Downloading PDFs\") as pbar:\n for future in as_completed(future_to_pdf):\n result = future.result()\n pbar.set_postfix_str(result.split()[-1] if \"✅\" in result else \"Error\")\n pbar.update(1)\n\n # Verify downloads\n downloaded_pdfs = list(self.pdf_dir.glob(\"*.pdf\"))\n print(f\"✅ Successfully downloaded {len(downloaded_pdfs)}/{len(pdf_files)} PDFs\")\n\n # Show any failures\n missing_pdfs = []\n for pdf_info in pdf_files:\n pdf_path = self.pdf_dir / pdf_info[\"name\"]\n if not pdf_path.exists() or pdf_path.stat().st_size == 0:\n missing_pdfs.append(pdf_info[\"name\"])\n\n if missing_pdfs:\n print(f\"⚠️ Failed to download {len(missing_pdfs)} PDFs:\")\n for pdf in missing_pdfs[:5]: # Show first 5\n print(f\" - {pdf}\")\n if len(missing_pdfs) > 5:\n print(f\" ... and {len(missing_pdfs) - 5} more\")\n\n def build_leann_index(\n self,\n backend: str = \"hnsw\",\n embedding_model: str = \"sentence-transformers/all-mpnet-base-v2\",\n ):\n \"\"\"Build LEANN index from all PDFs\"\"\"\n print(f\"🏗️ Building LEANN index with {backend} backend...\")\n\n # Check if we have PDFs\n pdf_files = list(self.pdf_dir.glob(\"*.pdf\"))\n if not pdf_files:\n raise RuntimeError(\"No PDF files found! Run download first.\")\n\n print(f\"Found {len(pdf_files)} PDF files to process\")\n\n start_time = time.time()\n\n # Initialize builder with standard compact configuration\n builder = LeannBuilder(\n backend_name=backend,\n embedding_model=embedding_model,\n embedding_mode=\"sentence-transformers\",\n graph_degree=32,\n complexity=64,\n is_recompute=True, # Enable recompute (no stored embeddings)\n is_compact=True, # Enable compact storage (pruned)\n num_threads=4,\n )\n\n # Process PDFs and extract text\n total_chunks = 0\n failed_pdfs = []\n\n for pdf_path in tqdm(pdf_files, desc=\"Processing PDFs\"):\n try:\n chunks = self.extract_pdf_text(pdf_path)\n for chunk in chunks:\n builder.add_text(chunk[\"text\"], metadata=chunk[\"metadata\"])\n total_chunks += 1\n\n except Exception as e:\n print(f\"❌ Failed to process {pdf_path.name}: {e}\")\n failed_pdfs.append(pdf_path.name)\n continue\n\n # Build index in index directory\n self.index_dir.mkdir(parents=True, exist_ok=True)\n index_path = self.index_dir / f\"financebench_full_{backend}.leann\"\n print(f\"🔨 Building index: {index_path}\")\n builder.build_index(str(index_path))\n\n build_time = time.time() - start_time\n\n print(\"✅ Index built successfully!\")\n print(f\" 📁 Index path: {index_path}\")\n print(f\" 📊 Total chunks: {total_chunks:,}\")\n print(f\" 📄 Processed PDFs: {len(pdf_files) - len(failed_pdfs)}/{len(pdf_files)}\")\n print(f\" ⏱️ Build time: {build_time:.1f}s\")\n\n if failed_pdfs:\n print(f\" ⚠️ Failed PDFs: {failed_pdfs}\")\n\n return str(index_path)\n\n def build_faiss_flat_baseline(self, index_path: str, output_dir: str = \"baseline\"):\n \"\"\"Build FAISS flat baseline using the same embeddings as LEANN index\"\"\"\n print(\"🔨 Building FAISS Flat baseline...\")\n\n import os\n import pickle\n\n import numpy as np\n from leann.api import compute_embeddings\n from leann_backend_hnsw import faiss\n\n os.makedirs(output_dir, exist_ok=True)\n baseline_path = os.path.join(output_dir, \"faiss_flat.index\")\n metadata_path = os.path.join(output_dir, \"metadata.pkl\")\n\n if os.path.exists(baseline_path) and os.path.exists(metadata_path):\n print(f\"✅ Baseline already exists at {baseline_path}\")\n return baseline_path\n\n # Read metadata from the built index\n meta_path = f\"{index_path}.meta.json\"\n with open(meta_path) as f:\n import json\n\n meta = json.loads(f.read())\n\n embedding_model = meta[\"embedding_model\"]\n passage_source = meta[\"passage_sources\"][0]\n passage_file = passage_source[\"path\"]\n\n # Convert relative path to absolute\n if not os.path.isabs(passage_file):\n index_dir = os.path.dirname(index_path)\n passage_file = os.path.join(index_dir, os.path.basename(passage_file))\n\n print(f\"📊 Loading passages from {passage_file}...\")\n print(f\"🤖 Using embedding model: {embedding_model}\")\n\n # Load all passages for baseline\n passages = []\n passage_ids = []\n with open(passage_file, encoding=\"utf-8\") as f:\n for line in f:\n if line.strip():\n data = json.loads(line)\n passages.append(data[\"text\"])\n passage_ids.append(data[\"id\"])\n\n print(f\"📄 Loaded {len(passages)} passages\")\n\n # Compute embeddings using the same method as LEANN\n print(\"🧮 Computing embeddings...\")\n embeddings = compute_embeddings(\n passages,\n embedding_model,\n mode=\"sentence-transformers\",\n use_server=False,\n )\n\n print(f\"📐 Embedding shape: {embeddings.shape}\")\n\n # Build FAISS flat index\n print(\"🏗️ Building FAISS IndexFlatIP...\")\n dimension = embeddings.shape[1]\n index = faiss.IndexFlatIP(dimension)\n\n # Add embeddings to flat index\n embeddings_f32 = embeddings.astype(np.float32)\n index.add(embeddings_f32.shape[0], faiss.swig_ptr(embeddings_f32))\n\n # Save index and metadata\n faiss.write_index(index, baseline_path)\n with open(metadata_path, \"wb\") as f:\n pickle.dump(passage_ids, f)\n\n print(f\"✅ FAISS baseline saved to {baseline_path}\")\n print(f\"✅ Metadata saved to {metadata_path}\")\n print(f\"📊 Total vectors: {index.ntotal}\")\n\n return baseline_path\n\n def extract_pdf_text(self, pdf_path: Path) -> list[dict]:\n \"\"\"Extract and chunk text from a PDF file\"\"\"\n chunks = []\n doc = pymupdf.open(pdf_path)\n\n for page_num in range(len(doc)):\n page = doc[page_num]\n text = page.get_text() # type: ignore\n\n if not text.strip():\n continue\n\n # Create metadata\n metadata = {\n \"source_file\": pdf_path.name,\n \"page_number\": page_num + 1,\n \"document_type\": \"10K\" if \"10K\" in pdf_path.name else \"10Q\",\n \"company\": pdf_path.name.split(\"_\")[0],\n \"doc_period\": self.extract_year_from_filename(pdf_path.name),\n }\n\n # Use recursive character splitting like LangChain\n if len(text.split()) > 500:\n # Split by double newlines (paragraphs)\n paragraphs = [p.strip() for p in text.split(\"\\n\\n\") if p.strip()]\n\n current_chunk = \"\"\n for para in paragraphs:\n # If adding this paragraph would make chunk too long, save current chunk\n if current_chunk and len((current_chunk + \" \" + para).split()) > 300:\n if current_chunk.strip():\n chunks.append(\n {\n \"text\": current_chunk.strip(),\n \"metadata\": {\n **metadata,\n \"chunk_id\": f\"page_{page_num + 1}_chunk_{len(chunks)}\",\n },\n }\n )\n current_chunk = para\n else:\n current_chunk = (current_chunk + \" \" + para).strip()\n\n # Add the last chunk\n if current_chunk.strip():\n chunks.append(\n {\n \"text\": current_chunk.strip(),\n \"metadata\": {\n **metadata,\n \"chunk_id\": f\"page_{page_num + 1}_chunk_{len(chunks)}\",\n },\n }\n )\n else:\n # Page is short enough, use as single chunk\n chunks.append(\n {\n \"text\": text.strip(),\n \"metadata\": {**metadata, \"chunk_id\": f\"page_{page_num + 1}\"},\n }\n )\n\n doc.close()\n return chunks\n\n def extract_year_from_filename(self, filename: str) -> str:\n \"\"\"Extract year from PDF filename\"\"\"\n # Try to find 4-digit year in filename\n\n match = re.search(r\"(\\d{4})\", filename)\n return match.group(1) if match else \"unknown\"\n\n def verify_setup(self, index_path: str):\n \"\"\"Verify the setup by testing a simple query\"\"\"\n print(\"🧪 Verifying setup with test query...\")\n\n try:\n searcher = LeannSearcher(index_path)\n\n # Test query\n test_query = \"What is the capital expenditure for 3M in 2018?\"\n results = searcher.search(test_query, top_k=3)\n\n print(f\"✅ Test query successful! Found {len(results)} results:\")\n for i, result in enumerate(results, 1):\n company = result.metadata.get(\"company\", \"Unknown\")\n year = result.metadata.get(\"doc_period\", \"Unknown\")\n page = result.metadata.get(\"page_number\", \"Unknown\")\n print(f\" {i}. {company} {year} (page {page}) - Score: {result.score:.3f}\")\n print(f\" {result.text[:100]}...\")\n\n searcher.cleanup()\n print(\"✅ Setup verification completed successfully!\")\n\n except Exception as e:\n print(f\"❌ Setup verification failed: {e}\")\n raise\n\n\ndef main():\n parser = argparse.ArgumentParser(description=\"Setup FinanceBench with full PDF datastore\")\n parser.add_argument(\"--data-dir\", default=\"data\", help=\"Data directory\")\n parser.add_argument(\n \"--backend\", choices=[\"hnsw\", \"diskann\"], default=\"hnsw\", help=\"LEANN backend\"\n )\n parser.add_argument(\n \"--embedding-model\",\n default=\"sentence-transformers/all-mpnet-base-v2\",\n help=\"Embedding model\",\n )\n parser.add_argument(\"--max-workers\", type=int, default=5, help=\"Parallel download workers\")\n parser.add_argument(\"--skip-download\", action=\"store_true\", help=\"Skip PDF download\")\n parser.add_argument(\"--skip-build\", action=\"store_true\", help=\"Skip index building\")\n parser.add_argument(\n \"--build-baseline-only\",\n action=\"store_true\",\n help=\"Only build FAISS baseline from existing index\",\n )\n\n args = parser.parse_args()\n\n print(\"🏦 FinanceBench Complete Setup\")\n print(\"=\" * 50)\n\n setup = FinanceBenchSetup(args.data_dir)\n\n try:\n if args.build_baseline_only:\n # Only build baseline from existing index\n index_path = setup.index_dir / f\"financebench_full_{args.backend}\"\n index_file = f\"{index_path}.index\"\n meta_file = f\"{index_path}.leann.meta.json\"\n\n if not os.path.exists(index_file) or not os.path.exists(meta_file):\n print(\"❌ Index files not found:\")\n print(f\" Index: {index_file}\")\n print(f\" Meta: {meta_file}\")\n print(\"💡 Run without --build-baseline-only to build the index first\")\n exit(1)\n\n print(f\"🔨 Building baseline from existing index: {index_path}\")\n baseline_path = setup.build_faiss_flat_baseline(str(index_path))\n print(f\"✅ Baseline built at {baseline_path}\")\n return\n\n # Step 1: Download dataset\n setup.download_dataset()\n\n # Step 2: Download PDFs\n if not args.skip_download:\n setup.download_all_pdfs(max_workers=args.max_workers)\n else:\n print(\"⏭️ Skipping PDF download\")\n\n # Step 3: Build LEANN index\n if not args.skip_build:\n index_path = setup.build_leann_index(\n backend=args.backend, embedding_model=args.embedding_model\n )\n\n # Step 4: Build FAISS flat baseline\n print(\"\\n🔨 Building FAISS flat baseline...\")\n baseline_path = setup.build_faiss_flat_baseline(index_path)\n print(f\"✅ Baseline built at {baseline_path}\")\n\n # Step 5: Verify setup\n setup.verify_setup(index_path)\n else:\n print(\"⏭️ Skipping index building\")\n\n print(\"\\n🎉 FinanceBench setup completed!\")\n print(f\"📁 Data directory: {setup.data_dir.absolute()}\")\n print(\"\\nNext steps:\")\n print(\n \"1. Run evaluation: python evaluate_financebench.py --index data/index/financebench_full_hnsw.leann\"\n )\n print(\n \"2. Or test manually: python -c \\\"from leann import LeannSearcher; s = LeannSearcher('data/index/financebench_full_hnsw.leann'); print(s.search('3M capital expenditure 2018'))\\\"\"\n )\n\n except KeyboardInterrupt:\n print(\"\\n⚠️ Setup interrupted by user\")\n exit(1)\n except Exception as e:\n print(f\"\\n❌ Setup failed: {e}\")\n exit(1)\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "benchmarks/financebench/verify_recall.py", "content": "#!/usr/bin/env python3\n# /// script\n# requires-python = \">=3.9\"\n# dependencies = [\n# \"faiss-cpu\",\n# \"numpy\",\n# \"sentence-transformers\",\n# \"torch\",\n# \"tqdm\",\n# ]\n# ///\n\n\"\"\"\nIndependent recall verification script using standard FAISS.\nCreates two indexes (HNSW and Flat) and compares recall@3 at different complexities.\n\"\"\"\n\nimport json\nimport time\nfrom pathlib import Path\n\nimport faiss\nimport numpy as np\nfrom sentence_transformers import SentenceTransformer\nfrom tqdm import tqdm\n\n\ndef compute_embeddings_direct(chunks: list[str], model_name: str) -> np.ndarray:\n \"\"\"\n Direct embedding computation using sentence-transformers.\n Copied logic to avoid dependency issues.\n \"\"\"\n print(f\"Loading model: {model_name}\")\n model = SentenceTransformer(model_name)\n\n print(f\"Computing embeddings for {len(chunks)} chunks...\")\n embeddings = model.encode(\n chunks,\n show_progress_bar=True,\n batch_size=32,\n convert_to_numpy=True,\n normalize_embeddings=False,\n )\n\n return embeddings.astype(np.float32)\n\n\ndef load_financebench_queries(dataset_path: str, max_queries: int = 200) -> list[str]:\n \"\"\"Load FinanceBench queries from dataset\"\"\"\n queries = []\n with open(dataset_path, encoding=\"utf-8\") as f:\n for line in f:\n if line.strip():\n data = json.loads(line)\n queries.append(data[\"question\"])\n if len(queries) >= max_queries:\n break\n return queries\n\n\ndef load_passages_from_leann_index(index_path: str) -> tuple[list[str], list[str]]:\n \"\"\"Load passages from LEANN index structure\"\"\"\n meta_path = f\"{index_path}.meta.json\"\n with open(meta_path) as f:\n meta = json.load(f)\n\n passage_source = meta[\"passage_sources\"][0]\n passage_file = passage_source[\"path\"]\n\n # Convert relative path to absolute\n if not Path(passage_file).is_absolute():\n index_dir = Path(index_path).parent\n passage_file = index_dir / Path(passage_file).name\n\n print(f\"Loading passages from {passage_file}\")\n\n passages = []\n passage_ids = []\n with open(passage_file, encoding=\"utf-8\") as f:\n for line in tqdm(f, desc=\"Loading passages\"):\n if line.strip():\n data = json.loads(line)\n passages.append(data[\"text\"])\n passage_ids.append(data[\"id\"])\n\n print(f\"Loaded {len(passages)} passages\")\n return passages, passage_ids\n\n\ndef build_faiss_indexes(embeddings: np.ndarray) -> tuple[faiss.Index, faiss.Index]:\n \"\"\"Build FAISS indexes: Flat (ground truth) and HNSW\"\"\"\n dimension = embeddings.shape[1]\n\n # Build Flat index (ground truth)\n print(\"Building FAISS IndexFlatIP (ground truth)...\")\n flat_index = faiss.IndexFlatIP(dimension)\n flat_index.add(embeddings)\n\n # Build HNSW index\n print(\"Building FAISS IndexHNSWFlat...\")\n M = 32 # Same as LEANN default\n hnsw_index = faiss.IndexHNSWFlat(dimension, M, faiss.METRIC_INNER_PRODUCT)\n hnsw_index.hnsw.efConstruction = 200 # Same as LEANN default\n hnsw_index.add(embeddings)\n\n print(f\"Built indexes with {flat_index.ntotal} vectors, dimension {dimension}\")\n return flat_index, hnsw_index\n\n\ndef evaluate_recall_at_k(\n query_embeddings: np.ndarray,\n flat_index: faiss.Index,\n hnsw_index: faiss.Index,\n passage_ids: list[str],\n k: int = 3,\n ef_search: int = 64,\n) -> float:\n \"\"\"Evaluate recall@k comparing HNSW vs Flat\"\"\"\n\n # Set search parameters for HNSW\n hnsw_index.hnsw.efSearch = ef_search\n\n total_recall = 0.0\n num_queries = query_embeddings.shape[0]\n\n for i in range(num_queries):\n query = query_embeddings[i : i + 1] # Keep 2D shape\n\n # Get ground truth from Flat index (standard FAISS API)\n flat_distances, flat_indices = flat_index.search(query, k)\n ground_truth_ids = {passage_ids[idx] for idx in flat_indices[0]}\n\n # Get results from HNSW index (standard FAISS API)\n hnsw_distances, hnsw_indices = hnsw_index.search(query, k)\n hnsw_ids = {passage_ids[idx] for idx in hnsw_indices[0]}\n\n # Calculate recall\n intersection = ground_truth_ids.intersection(hnsw_ids)\n recall = len(intersection) / k\n total_recall += recall\n\n if i < 3: # Show first few examples\n print(f\" Query {i + 1}: Recall@{k} = {recall:.3f}\")\n print(f\" Flat: {list(ground_truth_ids)}\")\n print(f\" HNSW: {list(hnsw_ids)}\")\n print(f\" Intersection: {list(intersection)}\")\n\n avg_recall = total_recall / num_queries\n return avg_recall\n\n\ndef main():\n # Configuration\n dataset_path = \"data/financebench_merged.jsonl\"\n index_path = \"data/index/financebench_full_hnsw.leann\"\n embedding_model = \"sentence-transformers/all-mpnet-base-v2\"\n\n print(\"🔍 FAISS Recall Verification\")\n print(\"=\" * 50)\n\n # Check if files exist\n if not Path(dataset_path).exists():\n print(f\"❌ Dataset not found: {dataset_path}\")\n return\n if not Path(f\"{index_path}.meta.json\").exists():\n print(f\"❌ Index metadata not found: {index_path}.meta.json\")\n return\n\n # Load data\n print(\"📖 Loading FinanceBench queries...\")\n queries = load_financebench_queries(dataset_path, max_queries=50)\n print(f\"Loaded {len(queries)} queries\")\n\n print(\"📄 Loading passages from LEANN index...\")\n passages, passage_ids = load_passages_from_leann_index(index_path)\n\n # Compute embeddings\n print(\"🧮 Computing passage embeddings...\")\n passage_embeddings = compute_embeddings_direct(passages, embedding_model)\n\n print(\"🧮 Computing query embeddings...\")\n query_embeddings = compute_embeddings_direct(queries, embedding_model)\n\n # Build FAISS indexes\n print(\"🏗️ Building FAISS indexes...\")\n flat_index, hnsw_index = build_faiss_indexes(passage_embeddings)\n\n # Test different efSearch values (equivalent to LEANN complexity)\n print(\"\\n📊 Evaluating Recall@3 at different efSearch values...\")\n ef_search_values = [16, 32, 64, 128, 256]\n\n for ef_search in ef_search_values:\n print(f\"\\n🧪 Testing efSearch = {ef_search}\")\n start_time = time.time()\n\n recall = evaluate_recall_at_k(\n query_embeddings, flat_index, hnsw_index, passage_ids, k=3, ef_search=ef_search\n )\n\n elapsed = time.time() - start_time\n print(\n f\"📈 efSearch {ef_search}: Recall@3 = {recall:.3f} ({recall * 100:.1f}%) in {elapsed:.2f}s\"\n )\n\n print(\"\\n✅ Verification completed!\")\n print(\"\\n📋 Summary:\")\n print(\" - Built independent FAISS Flat and HNSW indexes\")\n print(\" - Compared recall@3 at different efSearch values\")\n print(\" - Used same embedding model as LEANN\")\n print(\" - This validates LEANN's recall measurements\")\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "benchmarks/issue_159.py", "content": "#!/usr/bin/env python3\n\"\"\"\nTest script to reproduce issue #159: Slow search performance\nConfiguration:\n- GPU: A10\n- embedding_model: BAAI/bge-large-zh-v1.5\n- data size: 180M text (~90K chunks)\n- backend: hnsw\n\"\"\"\n\nimport os\nimport time\nfrom pathlib import Path\n\nfrom leann.api import LeannBuilder, LeannSearcher\n\nos.environ[\"LEANN_LOG_LEVEL\"] = \"DEBUG\"\n\n# Configuration matching the issue\nINDEX_PATH = \"./test_issue_159.leann\"\nEMBEDDING_MODEL = \"BAAI/bge-large-zh-v1.5\"\nBACKEND_NAME = \"hnsw\"\n\n\ndef generate_test_data(num_chunks=90000, chunk_size=2000):\n \"\"\"Generate test data similar to 180MB text (~90K chunks)\"\"\"\n # Each chunk is approximately 2000 characters\n # 90K chunks * 2000 chars ≈ 180MB\n chunks = []\n base_text = (\n \"这是一个测试文档。LEANN是一个创新的向量数据库, 通过图基选择性重计算实现97%的存储节省。\"\n )\n\n for i in range(num_chunks):\n chunk = f\"{base_text} 文档编号: {i}. \" * (chunk_size // len(base_text) + 1)\n chunks.append(chunk[:chunk_size])\n\n return chunks\n\n\ndef test_search_performance():\n \"\"\"Test search performance with different configurations\"\"\"\n print(\"=\" * 80)\n print(\"Testing LEANN Search Performance (Issue #159)\")\n print(\"=\" * 80)\n\n meta_path = Path(f\"{INDEX_PATH}.meta.json\")\n if meta_path.exists():\n print(f\"\\n✓ Index already exists at {INDEX_PATH}\")\n print(\" Skipping build phase. Delete the index to rebuild.\")\n else:\n print(\"\\n📦 Building index...\")\n print(f\" Backend: {BACKEND_NAME}\")\n print(f\" Embedding Model: {EMBEDDING_MODEL}\")\n print(\" Generating test data (~90K chunks, ~180MB)...\")\n\n chunks = generate_test_data(num_chunks=90000)\n print(f\" Generated {len(chunks)} chunks\")\n print(f\" Total text size: {sum(len(c) for c in chunks) / (1024 * 1024):.2f} MB\")\n\n builder = LeannBuilder(\n backend_name=BACKEND_NAME,\n embedding_model=EMBEDDING_MODEL,\n )\n\n print(\" Adding chunks to builder...\")\n start_time = time.time()\n for i, chunk in enumerate(chunks):\n builder.add_text(chunk)\n if (i + 1) % 10000 == 0:\n print(f\" Added {i + 1}/{len(chunks)} chunks...\")\n\n print(\" Building index...\")\n build_start = time.time()\n builder.build_index(INDEX_PATH)\n build_time = time.time() - build_start\n print(f\" ✓ Index built in {build_time:.2f} seconds\")\n\n # Test search with different complexity values\n print(\"\\n🔍 Testing search performance...\")\n searcher = LeannSearcher(INDEX_PATH)\n\n test_query = \"LEANN向量数据库存储优化\"\n\n # Test with minimal complexity (8)\n print(\"\\n Test 4: Minimal complexity (8)\")\n print(f\" Query: '{test_query}'\")\n start_time = time.time()\n results = searcher.search(test_query, top_k=10, complexity=8)\n search_time = time.time() - start_time\n print(f\" ✓ Search completed in {search_time:.2f} seconds\")\n print(f\" Results: {len(results)} items\")\n\n print(\"\\n\" + \"=\" * 80)\n\n\nif __name__ == \"__main__\":\n test_search_performance()\n" }, { "path": "benchmarks/laion/.gitignore", "content": "data/\n" }, { "path": "benchmarks/laion/README.md", "content": "# LAION Multimodal Benchmark\n\nA multimodal benchmark for evaluating image retrieval and generation performance using LEANN with CLIP embeddings and Qwen2.5-VL for multimodal generation on LAION dataset subset.\n\n## Overview\n\nThis benchmark evaluates:\n- **Image retrieval timing** using caption-based queries\n- **Recall@K performance** for image search\n- **Complexity analysis** across different search parameters\n- **Index size and storage efficiency**\n- **Multimodal generation** with Qwen2.5-VL for image understanding and description\n\n## Dataset Configuration\n\n- **Dataset**: LAION-400M subset (10,000 images)\n- **Embeddings**: Pre-computed CLIP ViT-B/32 (512 dimensions)\n- **Queries**: 200 random captions from the dataset\n- **Ground Truth**: Self-recall (query caption → original image)\n\n## Quick Start\n\n### 1. Setup the benchmark\n\n```bash\ncd benchmarks/laion\npython setup_laion.py --num-samples 10000 --num-queries 200\n```\n\nThis will:\n- Create dummy LAION data (10K samples)\n- Generate CLIP embeddings (512-dim)\n- Build LEANN index with HNSW backend\n- Create 200 evaluation queries\n\n### 2. Run evaluation\n\n```bash\n# Run all evaluation stages\npython evaluate_laion.py --index data/laion_index.leann\n\n# Run specific stages\npython evaluate_laion.py --index data/laion_index.leann --stage 2 # Recall evaluation\npython evaluate_laion.py --index data/laion_index.leann --stage 3 # Complexity analysis\npython evaluate_laion.py --index data/laion_index.leann --stage 4 # Index comparison\npython evaluate_laion.py --index data/laion_index.leann --stage 5 # Multimodal generation\n\n# Multimodal generation with Qwen2.5-VL\npython evaluate_laion.py --index data/laion_index.leann --stage 5 --model-name Qwen/Qwen2.5-VL-7B-Instruct\n```\n\n### 3. Save results\n\n```bash\npython evaluate_laion.py --index data/laion_index.leann --output results.json\n```\n\n## Configuration Options\n\n### Setup Options\n```bash\npython setup_laion.py \\\n --num-samples 10000 \\\n --num-queries 200 \\\n --index-path data/laion_index.leann \\\n --backend hnsw\n```\n\n### Evaluation Options\n```bash\npython evaluate_laion.py \\\n --index data/laion_index.leann \\\n --queries data/evaluation_queries.jsonl \\\n --complexity 64 \\\n --top-k 3 \\\n --num-samples 100 \\\n --stage all\n```\n\n## Evaluation Stages\n\n### Stage 2: Recall Evaluation\n- Evaluates Recall@3 for multimodal retrieval\n- Compares LEANN vs FAISS baseline performance\n- Self-recall: query caption should retrieve original image\n\n### Stage 3: Complexity Analysis\n- Binary search for optimal complexity (90% recall target)\n- Tests performance across different complexity levels\n- Analyzes speed vs. accuracy tradeoffs\n\n### Stage 4: Index Comparison\n- Compares compact vs non-compact index sizes\n- Measures search performance differences\n- Reports storage efficiency and speed ratios\n\n### Stage 5: Multimodal Generation\n- Uses Qwen2.5-VL for image understanding and description\n- Retrieval-Augmented Generation (RAG) with multimodal context\n- Measures both search and generation timing\n\n## Output Metrics\n\n### Timing Metrics\n- Average/median/min/max search time\n- Standard deviation\n- Searches per second\n- Latency in milliseconds\n\n### Recall Metrics\n- Recall@3 percentage for image retrieval\n- Number of queries with ground truth\n\n### Index Metrics\n- Total index size (MB)\n- Component breakdown (index, passages, metadata)\n- Storage savings (compact vs non-compact)\n- Backend and embedding model info\n\n### Generation Metrics (Stage 5)\n- Average search time per query\n- Average generation time per query\n- Time distribution (search vs generation)\n- Sample multimodal responses\n- Model: Qwen2.5-VL performance\n\n## Benchmark Results\n\n### LEANN-RAG Performance (CLIP ViT-L/14 + Qwen2.5-VL)\n\n**Stage 3: Optimal Complexity Analysis**\n- **Optimal Complexity**: 85 (achieving 90% Recall@3)\n- **Binary Search Range**: 1-128\n- **Target Recall**: 90%\n- **Index Type**: Non-compact (for fast binary search)\n\n**Stage 5: Multimodal Generation Performance (Qwen2.5-VL)**\n- **Total Queries**: 20\n- **Average Search Time**: 1.200s per query\n- **Average Generation Time**: 6.558s per query\n- **Time Distribution**: Search 15.5%, Generation 84.5%\n- **LLM Backend**: HuggingFace transformers\n- **Model**: Qwen/Qwen2.5-VL-7B-Instruct\n- **Optimal Complexity**: 85\n\n**System Performance:**\n- **Index Size**: ~10,000 image embeddings from LAION subset\n- **Embedding Model**: CLIP ViT-L/14 (768 dimensions)\n- **Backend**: HNSW with cosine distance\n\n### Example Results\n\n```\n🎯 LAION MULTIMODAL BENCHMARK RESULTS\n============================================================\n\n📊 Multimodal Generation Results:\n Total Queries: 20\n Avg Search Time: 1.200s\n Avg Generation Time: 6.558s\n Time Distribution: Search 15.5%, Generation 84.5%\n LLM Backend: HuggingFace transformers\n Model: Qwen/Qwen2.5-VL-7B-Instruct\n\n⚙️ Optimal Complexity Analysis:\n Target Recall: 90%\n Optimal Complexity: 85\n Binary Search Range: 1-128\n Non-compact Index (fast search, no recompute)\n\n🚀 Performance Summary:\n Multimodal RAG: 7.758s total per query\n Search: 15.5% of total time\n Generation: 84.5% of total time\n```\n\n## Directory Structure\n\n```\nbenchmarks/laion/\n├── setup_laion.py # Setup script\n├── evaluate_laion.py # Evaluation script\n├── README.md # This file\n└── data/ # Generated data\n ├── laion_images/ # Image files (placeholder)\n ├── laion_metadata.jsonl # Image metadata\n ├── laion_passages.jsonl # LEANN passages\n ├── laion_embeddings.npy # CLIP embeddings\n ├── evaluation_queries.jsonl # Evaluation queries\n └── laion_index.leann/ # LEANN index files\n```\n\n## Notes\n\n- Current implementation uses dummy data for demonstration\n- For real LAION data, implement actual download logic in `setup_laion.py`\n- CLIP embeddings are randomly generated - replace with real CLIP model for production\n- Adjust `num_samples` and `num_queries` based on available resources\n- Consider using `--num-samples` during evaluation for faster testing\n" }, { "path": "benchmarks/laion/evaluate_laion.py", "content": "\"\"\"\nLAION Multimodal Benchmark Evaluation Script - Modular Recall-based Evaluation\n\"\"\"\n\nimport argparse\nimport json\nimport logging\nimport os\nimport pickle\nimport time\nfrom pathlib import Path\n\nimport numpy as np\nfrom leann import LeannSearcher\nfrom leann_backend_hnsw import faiss\nfrom sentence_transformers import SentenceTransformer\n\nfrom ..llm_utils import evaluate_multimodal_rag, load_qwen_vl_model\n\n# Setup logging to reduce verbose output\nlogging.basicConfig(level=logging.WARNING)\nlogging.getLogger(\"leann.api\").setLevel(logging.WARNING)\nlogging.getLogger(\"leann_backend_hnsw\").setLevel(logging.WARNING)\n\n\nclass RecallEvaluator:\n \"\"\"Stage 2: Evaluate Recall@3 (LEANN vs FAISS baseline for multimodal retrieval)\"\"\"\n\n def __init__(self, index_path: str, baseline_dir: str):\n self.index_path = index_path\n self.baseline_dir = baseline_dir\n self.searcher = LeannSearcher(index_path)\n\n # Load FAISS flat baseline (image embeddings)\n baseline_index_path = os.path.join(baseline_dir, \"faiss_flat.index\")\n metadata_path = os.path.join(baseline_dir, \"metadata.pkl\")\n\n self.faiss_index = faiss.read_index(baseline_index_path)\n with open(metadata_path, \"rb\") as f:\n self.image_ids = pickle.load(f)\n print(f\"📚 Loaded FAISS flat baseline with {self.faiss_index.ntotal} image vectors\")\n\n # Load sentence-transformers CLIP for text embedding (ViT-L/14)\n self.st_clip = SentenceTransformer(\"clip-ViT-L-14\")\n\n def evaluate_recall_at_3(\n self, captions: list[str], complexity: int = 64, recompute_embeddings: bool = True\n ) -> float:\n \"\"\"Evaluate recall@3 for multimodal retrieval: caption queries -> image results\"\"\"\n recompute_str = \"with recompute\" if recompute_embeddings else \"no recompute\"\n print(f\"🔍 Evaluating recall@3 with complexity={complexity} ({recompute_str})...\")\n\n total_recall = 0.0\n num_queries = len(captions)\n\n for i, caption in enumerate(captions):\n # Get ground truth: search with FAISS flat using caption text embedding\n # Generate CLIP text embedding for caption via sentence-transformers (normalized)\n query_embedding = self.st_clip.encode(\n [caption], convert_to_numpy=True, normalize_embeddings=True, show_progress_bar=False\n ).astype(np.float32)\n\n # Search FAISS flat for ground truth using LEANN's modified faiss API\n n = query_embedding.shape[0] # Number of queries\n k = 3 # Number of nearest neighbors\n distances = np.zeros((n, k), dtype=np.float32)\n labels = np.zeros((n, k), dtype=np.int64)\n\n self.faiss_index.search(\n n,\n faiss.swig_ptr(query_embedding),\n k,\n faiss.swig_ptr(distances),\n faiss.swig_ptr(labels),\n )\n\n # Extract the results (image IDs from FAISS)\n baseline_ids = {self.image_ids[idx] for idx in labels[0]}\n\n # Search with LEANN at specified complexity (using caption as text query)\n test_results = self.searcher.search(\n caption,\n top_k=3,\n complexity=complexity,\n recompute_embeddings=recompute_embeddings,\n )\n test_ids = {result.id for result in test_results}\n\n # Calculate recall@3 = |intersection| / |ground_truth|\n intersection = test_ids.intersection(baseline_ids)\n recall = len(intersection) / 3.0 # Ground truth size is 3\n total_recall += recall\n\n if i < 3: # Show first few examples\n print(f\" Query {i + 1}: '{caption[:50]}...' -> Recall@3: {recall:.3f}\")\n print(f\" FAISS ground truth: {list(baseline_ids)}\")\n print(f\" LEANN results (C={complexity}, {recompute_str}): {list(test_ids)}\")\n print(f\" Intersection: {list(intersection)}\")\n\n avg_recall = total_recall / num_queries\n print(f\"📊 Average Recall@3: {avg_recall:.3f} ({avg_recall * 100:.1f}%)\")\n return avg_recall\n\n def cleanup(self):\n \"\"\"Cleanup resources\"\"\"\n if hasattr(self, \"searcher\"):\n self.searcher.cleanup()\n\n\nclass LAIONEvaluator:\n def __init__(self, index_path: str):\n self.index_path = index_path\n self.searcher = LeannSearcher(index_path)\n\n def load_queries(self, queries_file: str) -> list[str]:\n \"\"\"Load caption queries from evaluation file\"\"\"\n captions = []\n with open(queries_file, encoding=\"utf-8\") as f:\n for line in f:\n if line.strip():\n query_data = json.loads(line)\n captions.append(query_data[\"query\"])\n\n print(f\"📊 Loaded {len(captions)} caption queries\")\n return captions\n\n def analyze_index_sizes(self) -> dict:\n \"\"\"Analyze index sizes, emphasizing .index only (exclude passages).\"\"\"\n print(\"📏 Analyzing index sizes (.index only)...\")\n\n # Get all index-related files\n index_path = Path(self.index_path)\n index_dir = index_path.parent\n index_name = index_path.stem # Remove .leann extension\n\n sizes: dict[str, float] = {}\n\n # Core index files\n index_file = index_dir / f\"{index_name}.index\"\n meta_file = index_dir / f\"{index_path.name}.meta.json\" # Keep .leann for meta file\n passages_file = index_dir / f\"{index_path.name}.passages.jsonl\" # Keep .leann for passages\n passages_idx_file = index_dir / f\"{index_path.name}.passages.idx\" # Keep .leann for idx\n\n # Core index size (.index only)\n index_mb = index_file.stat().st_size / (1024 * 1024) if index_file.exists() else 0.0\n sizes[\"index_only_mb\"] = index_mb\n\n # Other files for reference (not counted in index_only_mb)\n sizes[\"metadata_mb\"] = (\n meta_file.stat().st_size / (1024 * 1024) if meta_file.exists() else 0.0\n )\n sizes[\"passages_text_mb\"] = (\n passages_file.stat().st_size / (1024 * 1024) if passages_file.exists() else 0.0\n )\n sizes[\"passages_index_mb\"] = (\n passages_idx_file.stat().st_size / (1024 * 1024) if passages_idx_file.exists() else 0.0\n )\n\n print(f\" 📁 .index size: {index_mb:.1f} MB\")\n if sizes[\"metadata_mb\"]:\n print(f\" 🧾 metadata: {sizes['metadata_mb']:.3f} MB\")\n if sizes[\"passages_text_mb\"] or sizes[\"passages_index_mb\"]:\n print(\n f\" (passages excluded) text: {sizes['passages_text_mb']:.1f} MB, idx: {sizes['passages_index_mb']:.1f} MB\"\n )\n\n return sizes\n\n def create_non_compact_index_for_comparison(self, non_compact_index_path: str) -> dict:\n \"\"\"Create a non-compact index for comparison purposes\"\"\"\n print(\"🏗️ Building non-compact index from existing passages...\")\n\n # Load existing passages from current index\n from leann import LeannBuilder\n\n current_index_path = Path(self.index_path)\n current_index_dir = current_index_path.parent\n current_index_name = current_index_path.name\n\n # Read metadata to get passage source\n meta_path = current_index_dir / f\"{current_index_name}.meta.json\"\n with open(meta_path) as f:\n meta = json.load(f)\n\n passage_source = meta[\"passage_sources\"][0]\n passage_file = passage_source[\"path\"]\n\n # Convert relative path to absolute\n if not Path(passage_file).is_absolute():\n passage_file = current_index_dir / Path(passage_file).name\n\n print(f\"📄 Loading passages from {passage_file}...\")\n\n # Load CLIP embeddings\n embeddings_file = current_index_dir / \"clip_image_embeddings.npy\"\n embeddings = np.load(embeddings_file)\n print(f\"📐 Loaded embeddings shape: {embeddings.shape}\")\n\n # Build non-compact index with same passages and embeddings\n builder = LeannBuilder(\n backend_name=\"hnsw\",\n # Use CLIP text encoder (ViT-L/14) to match image embeddings (768-dim)\n embedding_model=\"clip-ViT-L-14\",\n embedding_mode=\"sentence-transformers\",\n is_recompute=False, # Disable recompute (store embeddings)\n is_compact=False, # Disable compact storage\n distance_metric=\"cosine\",\n **{\n k: v\n for k, v in meta.get(\"backend_kwargs\", {}).items()\n if k not in [\"is_recompute\", \"is_compact\", \"distance_metric\"]\n },\n )\n\n # Prepare ids and add passages\n ids: list[str] = []\n with open(passage_file, encoding=\"utf-8\") as f:\n for line in f:\n if line.strip():\n data = json.loads(line)\n ids.append(str(data[\"id\"]))\n # Ensure metadata contains the id used by the vector index\n metadata = {**data.get(\"metadata\", {}), \"id\": data[\"id\"]}\n builder.add_text(text=data[\"text\"], metadata=metadata)\n\n if len(ids) != embeddings.shape[0]:\n raise ValueError(\n f\"IDs count ({len(ids)}) does not match embeddings ({embeddings.shape[0]}).\"\n )\n\n # Persist a pickle for build_index_from_embeddings\n pkl_path = current_index_dir / \"clip_image_embeddings.pkl\"\n with open(pkl_path, \"wb\") as pf:\n pickle.dump((ids, embeddings.astype(np.float32)), pf)\n\n print(\n f\"🔨 Building non-compact index at {non_compact_index_path} from precomputed embeddings...\"\n )\n builder.build_index_from_embeddings(non_compact_index_path, str(pkl_path))\n\n # Analyze the non-compact index size\n temp_evaluator = LAIONEvaluator(non_compact_index_path)\n non_compact_sizes = temp_evaluator.analyze_index_sizes()\n non_compact_sizes[\"index_type\"] = \"non_compact\"\n\n return non_compact_sizes\n\n def compare_index_performance(\n self, non_compact_path: str, compact_path: str, test_captions: list, complexity: int\n ) -> dict:\n \"\"\"Compare performance between non-compact and compact indexes\"\"\"\n print(\"⚡ Comparing search performance between indexes...\")\n\n # Test queries\n test_queries = test_captions[:5]\n\n results = {\n \"non_compact\": {\"search_times\": []},\n \"compact\": {\"search_times\": []},\n \"avg_search_times\": {},\n \"speed_ratio\": 0.0,\n }\n\n # Test non-compact index (no recompute)\n print(\" 🔍 Testing non-compact index (no recompute)...\")\n non_compact_searcher = LeannSearcher(non_compact_path)\n\n for caption in test_queries:\n start_time = time.time()\n _ = non_compact_searcher.search(\n caption, top_k=3, complexity=complexity, recompute_embeddings=False\n )\n search_time = time.time() - start_time\n results[\"non_compact\"][\"search_times\"].append(search_time)\n\n # Test compact index (with recompute)\n print(\" 🔍 Testing compact index (with recompute)...\")\n compact_searcher = LeannSearcher(compact_path)\n\n for caption in test_queries:\n start_time = time.time()\n _ = compact_searcher.search(\n caption, top_k=3, complexity=complexity, recompute_embeddings=True\n )\n search_time = time.time() - start_time\n results[\"compact\"][\"search_times\"].append(search_time)\n\n # Calculate averages\n results[\"avg_search_times\"][\"non_compact\"] = sum(\n results[\"non_compact\"][\"search_times\"]\n ) / len(results[\"non_compact\"][\"search_times\"])\n results[\"avg_search_times\"][\"compact\"] = sum(results[\"compact\"][\"search_times\"]) / len(\n results[\"compact\"][\"search_times\"]\n )\n\n # Performance ratio\n if results[\"avg_search_times\"][\"compact\"] > 0:\n results[\"speed_ratio\"] = (\n results[\"avg_search_times\"][\"non_compact\"] / results[\"avg_search_times\"][\"compact\"]\n )\n else:\n results[\"speed_ratio\"] = float(\"inf\")\n\n print(\n f\" Non-compact (no recompute): {results['avg_search_times']['non_compact']:.3f}s avg\"\n )\n print(f\" Compact (with recompute): {results['avg_search_times']['compact']:.3f}s avg\")\n print(f\" Speed ratio: {results['speed_ratio']:.2f}x\")\n\n # Cleanup\n non_compact_searcher.cleanup()\n compact_searcher.cleanup()\n\n return results\n\n def _print_results(self, timing_metrics: dict):\n \"\"\"Print evaluation results\"\"\"\n print(\"\\n🎯 LAION MULTIMODAL BENCHMARK RESULTS\")\n print(\"=\" * 60)\n\n # Index comparison analysis (prefer .index-only view if present)\n if \"current_index\" in timing_metrics and \"non_compact_index\" in timing_metrics:\n current = timing_metrics[\"current_index\"]\n non_compact = timing_metrics[\"non_compact_index\"]\n\n if \"index_only_mb\" in current and \"index_only_mb\" in non_compact:\n print(\"\\n📏 Index Comparison Analysis (.index only):\")\n print(f\" Compact index (current): {current.get('index_only_mb', 0):.1f} MB\")\n print(f\" Non-compact index: {non_compact.get('index_only_mb', 0):.1f} MB\")\n print(\n f\" Storage saving by compact: {timing_metrics.get('storage_saving_percent', 0):.1f}%\"\n )\n # Show excluded components for reference if available\n if any(\n k in non_compact\n for k in (\"passages_text_mb\", \"passages_index_mb\", \"metadata_mb\")\n ):\n print(\" (passages excluded in totals, shown for reference):\")\n print(\n f\" - Passages text: {non_compact.get('passages_text_mb', 0):.1f} MB, \"\n f\"Passages index: {non_compact.get('passages_index_mb', 0):.1f} MB, \"\n f\"Metadata: {non_compact.get('metadata_mb', 0):.3f} MB\"\n )\n else:\n # Fallback to legacy totals if running with older metrics\n print(\"\\n📏 Index Comparison Analysis:\")\n print(\n f\" Compact index (current): {current.get('total_with_embeddings', 0):.1f} MB\"\n )\n print(\n f\" Non-compact index (with embeddings): {non_compact.get('total_with_embeddings', 0):.1f} MB\"\n )\n print(\n f\" Storage saving by compact: {timing_metrics.get('storage_saving_percent', 0):.1f}%\"\n )\n print(\" Component breakdown (non-compact):\")\n print(f\" - Main index: {non_compact.get('index', 0):.1f} MB\")\n print(f\" - Passages text: {non_compact.get('passages_text', 0):.1f} MB\")\n print(f\" - Passages index: {non_compact.get('passages_index', 0):.1f} MB\")\n print(f\" - Metadata: {non_compact.get('metadata', 0):.1f} MB\")\n\n # Performance comparison\n if \"performance_comparison\" in timing_metrics:\n perf = timing_metrics[\"performance_comparison\"]\n print(\"\\n⚡ Performance Comparison:\")\n print(\n f\" Non-compact (no recompute): {perf.get('avg_search_times', {}).get('non_compact', 0):.3f}s avg\"\n )\n print(\n f\" Compact (with recompute): {perf.get('avg_search_times', {}).get('compact', 0):.3f}s avg\"\n )\n print(f\" Speed ratio: {perf.get('speed_ratio', 0):.2f}x\")\n\n # Legacy single index analysis (fallback)\n if \"total_with_embeddings\" in timing_metrics and \"current_index\" not in timing_metrics:\n print(\"\\n📏 Index Size Analysis:\")\n print(\n f\" Index with embeddings: {timing_metrics.get('total_with_embeddings', 0):.1f} MB\"\n )\n print(\n f\" Estimated pruned index: {timing_metrics.get('total_without_embeddings', 0):.1f} MB\"\n )\n print(f\" Compression ratio: {timing_metrics.get('compression_ratio', 0):.2f}x\")\n\n def cleanup(self):\n \"\"\"Cleanup resources\"\"\"\n if self.searcher:\n self.searcher.cleanup()\n\n\ndef main():\n parser = argparse.ArgumentParser(description=\"LAION Multimodal Benchmark Evaluation\")\n parser.add_argument(\"--index\", required=True, help=\"Path to LEANN index\")\n parser.add_argument(\n \"--queries\", default=\"data/evaluation_queries.jsonl\", help=\"Path to evaluation queries\"\n )\n parser.add_argument(\n \"--stage\",\n choices=[\"2\", \"3\", \"4\", \"5\", \"all\"],\n default=\"all\",\n help=\"Which stage to run (2=recall, 3=complexity, 4=index comparison, 5=generation)\",\n )\n parser.add_argument(\"--complexity\", type=int, default=None, help=\"Complexity for search\")\n parser.add_argument(\"--baseline-dir\", default=\"baseline\", help=\"Baseline output directory\")\n parser.add_argument(\"--output\", help=\"Save results to JSON file\")\n parser.add_argument(\n \"--llm-backend\",\n choices=[\"hf\"],\n default=\"hf\",\n help=\"LLM backend (Qwen2.5-VL only supports HF)\",\n )\n parser.add_argument(\n \"--model-name\", default=\"Qwen/Qwen2.5-VL-7B-Instruct\", help=\"Multimodal model name\"\n )\n\n args = parser.parse_args()\n\n try:\n # Check if baseline exists\n baseline_index_path = os.path.join(args.baseline_dir, \"faiss_flat.index\")\n if not os.path.exists(baseline_index_path):\n print(f\"❌ FAISS baseline not found at {baseline_index_path}\")\n print(\"💡 Please run setup_laion.py first to build the baseline\")\n exit(1)\n\n if args.stage == \"2\" or args.stage == \"all\":\n # Stage 2: Recall@3 evaluation\n print(\"🚀 Starting Stage 2: Recall@3 evaluation for multimodal retrieval\")\n\n evaluator = RecallEvaluator(args.index, args.baseline_dir)\n\n # Load caption queries for testing\n laion_evaluator = LAIONEvaluator(args.index)\n captions = laion_evaluator.load_queries(args.queries)\n\n # Test with queries for robust measurement\n test_captions = captions[:100] # Use subset for speed\n print(f\"🧪 Testing with {len(test_captions)} caption queries\")\n\n # Test with complexity 64\n complexity = 64\n recall = evaluator.evaluate_recall_at_3(test_captions, complexity)\n print(f\"📈 Recall@3 at complexity {complexity}: {recall * 100:.1f}%\")\n\n evaluator.cleanup()\n print(\"✅ Stage 2 completed!\\n\")\n\n # Shared non-compact index path for Stage 3 and 4\n non_compact_index_path = args.index.replace(\".leann\", \"_noncompact.leann\")\n complexity = args.complexity\n\n if args.stage == \"3\" or args.stage == \"all\":\n # Stage 3: Binary search for 90% recall complexity\n print(\"🚀 Starting Stage 3: Binary search for 90% recall complexity\")\n print(\n \"💡 Creating non-compact index for fast binary search with recompute_embeddings=False\"\n )\n\n # Create non-compact index for binary search\n print(\"🏗️ Creating non-compact index for binary search...\")\n evaluator = LAIONEvaluator(args.index)\n evaluator.create_non_compact_index_for_comparison(non_compact_index_path)\n\n # Use non-compact index for binary search\n binary_search_evaluator = RecallEvaluator(non_compact_index_path, args.baseline_dir)\n\n # Load caption queries for testing\n captions = evaluator.load_queries(args.queries)\n\n # Use subset for robust measurement\n test_captions = captions[:50] # Smaller subset for binary search speed\n print(f\"🧪 Testing with {len(test_captions)} caption queries\")\n\n # Binary search for 90% recall complexity\n target_recall = 0.9\n min_complexity, max_complexity = 1, 128\n\n print(f\"🔍 Binary search for {target_recall * 100}% recall complexity...\")\n print(f\"Search range: {min_complexity} to {max_complexity}\")\n\n best_complexity = None\n best_recall = 0.0\n\n while min_complexity <= max_complexity:\n mid_complexity = (min_complexity + max_complexity) // 2\n\n print(\n f\"\\n🧪 Testing complexity {mid_complexity} (no recompute, non-compact index)...\"\n )\n # Use recompute_embeddings=False on non-compact index for fast binary search\n recall = binary_search_evaluator.evaluate_recall_at_3(\n test_captions, mid_complexity, recompute_embeddings=False\n )\n\n print(\n f\" Complexity {mid_complexity}: Recall@3 = {recall:.3f} ({recall * 100:.1f}%)\"\n )\n\n if recall >= target_recall:\n best_complexity = mid_complexity\n best_recall = recall\n max_complexity = mid_complexity - 1\n print(\" ✅ Target reached! Searching for lower complexity...\")\n else:\n min_complexity = mid_complexity + 1\n print(\" ❌ Below target. Searching for higher complexity...\")\n\n if best_complexity is not None:\n print(\"\\n🎯 Optimal complexity found!\")\n print(f\" Complexity: {best_complexity}\")\n print(f\" Recall@3: {best_recall:.3f} ({best_recall * 100:.1f}%)\")\n\n # Test a few complexities around the optimal one for verification\n print(\"\\n🔬 Verification test around optimal complexity:\")\n verification_complexities = [\n max(1, best_complexity - 2),\n max(1, best_complexity - 1),\n best_complexity,\n best_complexity + 1,\n best_complexity + 2,\n ]\n\n for complexity in verification_complexities:\n if complexity <= 512: # reasonable upper bound\n recall = binary_search_evaluator.evaluate_recall_at_3(\n test_captions, complexity, recompute_embeddings=False\n )\n status = \"✅\" if recall >= target_recall else \"❌\"\n print(f\" {status} Complexity {complexity:3d}: {recall * 100:5.1f}%\")\n\n # Now test the optimal complexity with compact index and recompute for comparison\n print(\n f\"\\n🔄 Testing optimal complexity {best_complexity} on compact index WITH recompute...\"\n )\n compact_evaluator = RecallEvaluator(args.index, args.baseline_dir)\n recall_with_recompute = compact_evaluator.evaluate_recall_at_3(\n test_captions[:10], best_complexity, recompute_embeddings=True\n )\n print(\n f\" ✅ Complexity {best_complexity} (compact index with recompute): {recall_with_recompute * 100:.1f}%\"\n )\n complexity = best_complexity\n print(\n f\" 📊 Recall difference: {abs(best_recall - recall_with_recompute) * 100:.2f}%\"\n )\n compact_evaluator.cleanup()\n else:\n print(f\"\\n❌ Could not find complexity achieving {target_recall * 100}% recall\")\n print(\"All tested complexities were below target.\")\n\n # Cleanup evaluators (keep non-compact index for Stage 4)\n binary_search_evaluator.cleanup()\n evaluator.cleanup()\n\n print(\"✅ Stage 3 completed! Non-compact index saved for Stage 4.\\n\")\n\n if args.stage == \"4\" or args.stage == \"all\":\n # Stage 4: Index comparison (without LLM generation)\n print(\"🚀 Starting Stage 4: Index comparison analysis\")\n\n # Use LAION evaluator for index comparison\n evaluator = LAIONEvaluator(args.index)\n\n # Load caption queries\n captions = evaluator.load_queries(args.queries)\n\n # Step 1: Analyze current (compact) index\n print(\"\\n📏 Analyzing current index (compact, pruned)...\")\n compact_size_metrics = evaluator.analyze_index_sizes()\n compact_size_metrics[\"index_type\"] = \"compact\"\n\n # Step 2: Use existing non-compact index or create if needed\n if Path(non_compact_index_path).exists():\n print(\n f\"\\n📁 Using existing non-compact index from Stage 3: {non_compact_index_path}\"\n )\n temp_evaluator = LAIONEvaluator(non_compact_index_path)\n non_compact_size_metrics = temp_evaluator.analyze_index_sizes()\n non_compact_size_metrics[\"index_type\"] = \"non_compact\"\n else:\n print(\"\\n🏗️ Creating non-compact index (with embeddings) for comparison...\")\n non_compact_size_metrics = evaluator.create_non_compact_index_for_comparison(\n non_compact_index_path\n )\n\n # Step 3: Compare index sizes (.index only)\n print(\"\\n📊 Index size comparison (.index only):\")\n print(\n f\" Compact index (current): {compact_size_metrics.get('index_only_mb', 0):.1f} MB\"\n )\n print(f\" Non-compact index: {non_compact_size_metrics.get('index_only_mb', 0):.1f} MB\")\n\n storage_saving = 0.0\n if non_compact_size_metrics.get(\"index_only_mb\", 0) > 0:\n storage_saving = (\n (\n non_compact_size_metrics.get(\"index_only_mb\", 0)\n - compact_size_metrics.get(\"index_only_mb\", 0)\n )\n / non_compact_size_metrics.get(\"index_only_mb\", 1)\n * 100\n )\n print(f\" Storage saving by compact: {storage_saving:.1f}%\")\n\n # Step 4: Performance comparison between the two indexes\n if complexity is None:\n raise ValueError(\"Complexity is required for index comparison\")\n\n print(\"\\n⚡ Performance comparison between indexes...\")\n performance_metrics = evaluator.compare_index_performance(\n non_compact_index_path, args.index, captions[:10], complexity=complexity\n )\n\n # Combine all metrics\n combined_metrics = {\n \"current_index\": compact_size_metrics,\n \"non_compact_index\": non_compact_size_metrics,\n \"performance_comparison\": performance_metrics,\n \"storage_saving_percent\": storage_saving,\n }\n\n # Print comprehensive results\n evaluator._print_results(combined_metrics)\n\n # Save results if requested\n if args.output:\n print(f\"\\n💾 Saving results to {args.output}...\")\n with open(args.output, \"w\") as f:\n json.dump(combined_metrics, f, indent=2, default=str)\n print(f\"✅ Results saved to {args.output}\")\n\n evaluator.cleanup()\n print(\"✅ Stage 4 completed!\\n\")\n\n if args.stage in (\"5\", \"all\"):\n print(\"🚀 Starting Stage 5: Multimodal generation with Qwen2.5-VL\")\n evaluator = LAIONEvaluator(args.index)\n captions = evaluator.load_queries(args.queries)\n test_captions = captions[: min(20, len(captions))] # Use subset for generation\n\n print(f\"🧪 Testing multimodal generation with {len(test_captions)} queries\")\n\n # Load Qwen2.5-VL model\n try:\n print(\"Loading Qwen2.5-VL model...\")\n processor, model = load_qwen_vl_model(args.model_name)\n\n # Run multimodal generation evaluation\n complexity = args.complexity or 64\n gen_results = evaluate_multimodal_rag(\n evaluator.searcher,\n test_captions,\n processor=processor,\n model=model,\n complexity=complexity,\n )\n\n print(\"\\n📊 Multimodal Generation Results:\")\n print(f\" Total Queries: {len(test_captions)}\")\n print(f\" Avg Search Time: {gen_results['avg_search_time']:.3f}s\")\n print(f\" Avg Generation Time: {gen_results['avg_generation_time']:.3f}s\")\n total_time = gen_results[\"avg_search_time\"] + gen_results[\"avg_generation_time\"]\n search_pct = (gen_results[\"avg_search_time\"] / total_time) * 100\n gen_pct = (gen_results[\"avg_generation_time\"] / total_time) * 100\n print(f\" Time Distribution: Search {search_pct:.1f}%, Generation {gen_pct:.1f}%\")\n print(\" LLM Backend: HuggingFace transformers\")\n print(f\" Model: {args.model_name}\")\n\n # Show sample results\n print(\"\\n📝 Sample Multimodal Generations:\")\n for i, response in enumerate(gen_results[\"results\"][:3]):\n # Handle both string and dict formats for captions\n if isinstance(test_captions[i], dict):\n caption_text = test_captions[i].get(\"query\", str(test_captions[i]))\n else:\n caption_text = str(test_captions[i])\n print(f\" Query {i + 1}: {caption_text[:60]}...\")\n print(f\" Response {i + 1}: {response[:100]}...\")\n print()\n\n except Exception as e:\n print(f\"❌ Multimodal generation evaluation failed: {e}\")\n print(\"💡 Make sure transformers and Qwen2.5-VL are installed\")\n import traceback\n\n traceback.print_exc()\n\n evaluator.cleanup()\n print(\"✅ Stage 5 completed!\\n\")\n\n if args.stage == \"all\":\n print(\"🎉 All evaluation stages completed successfully!\")\n print(\"\\n📋 Summary:\")\n print(\" Stage 2: ✅ Multimodal Recall@3 evaluation completed\")\n print(\" Stage 3: ✅ Optimal complexity found\")\n print(\" Stage 4: ✅ Index comparison analysis completed\")\n print(\" Stage 5: ✅ Multimodal generation evaluation completed\")\n print(\"\\n🔧 Recommended next steps:\")\n print(\" - Use optimal complexity for best speed/accuracy balance\")\n print(\" - Review index comparison for storage vs performance tradeoffs\")\n\n # Clean up non-compact index after all stages complete\n print(\"\\n🧹 Cleaning up temporary non-compact index...\")\n if Path(non_compact_index_path).exists():\n temp_index_dir = Path(non_compact_index_path).parent\n temp_index_name = Path(non_compact_index_path).name\n for temp_file in temp_index_dir.glob(f\"{temp_index_name}*\"):\n temp_file.unlink()\n print(f\"✅ Cleaned up {non_compact_index_path}\")\n else:\n print(\"📝 No temporary index to clean up\")\n\n except KeyboardInterrupt:\n print(\"\\n⚠️ Evaluation interrupted by user\")\n exit(1)\n except Exception as e:\n print(f\"\\n❌ Stage {args.stage} failed: {e}\")\n import traceback\n\n traceback.print_exc()\n exit(1)\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "benchmarks/laion/setup_laion.py", "content": "\"\"\"\nLAION Multimodal Benchmark Setup Script\nDownloads LAION subset and builds LEANN index with sentence embeddings\n\"\"\"\n\nimport argparse\nimport asyncio\nimport io\nimport json\nimport os\nimport pickle\nimport time\nfrom pathlib import Path\n\nimport aiohttp\nimport numpy as np\nfrom datasets import load_dataset\nfrom leann import LeannBuilder\nfrom PIL import Image\nfrom sentence_transformers import SentenceTransformer\nfrom tqdm import tqdm\n\n\nclass LAIONSetup:\n def __init__(self, data_dir: str = \"data\"):\n self.data_dir = Path(data_dir)\n self.images_dir = self.data_dir / \"laion_images\"\n self.metadata_file = self.data_dir / \"laion_metadata.jsonl\"\n\n # Create directories\n self.data_dir.mkdir(exist_ok=True)\n self.images_dir.mkdir(exist_ok=True)\n\n async def download_single_image(self, session, sample_data, semaphore, progress_bar):\n \"\"\"Download a single image asynchronously\"\"\"\n async with semaphore: # Limit concurrent downloads\n try:\n image_url = sample_data[\"url\"]\n image_path = sample_data[\"image_path\"]\n\n # Skip if already exists\n if os.path.exists(image_path):\n progress_bar.update(1)\n return sample_data\n\n async with session.get(image_url, timeout=10) as response:\n if response.status == 200:\n content = await response.read()\n\n # Verify it's a valid image\n try:\n img = Image.open(io.BytesIO(content))\n img = img.convert(\"RGB\")\n img.save(image_path, \"JPEG\")\n progress_bar.update(1)\n return sample_data\n except Exception:\n progress_bar.update(1)\n return None # Skip invalid images\n else:\n progress_bar.update(1)\n return None\n\n except Exception:\n progress_bar.update(1)\n return None\n\n def download_laion_subset(self, num_samples: int = 1000):\n \"\"\"Download LAION subset from HuggingFace datasets with async parallel downloading\"\"\"\n print(f\"📥 Downloading LAION subset ({num_samples} samples)...\")\n\n # Load LAION-400M subset from HuggingFace\n print(\"🤗 Loading from HuggingFace datasets...\")\n dataset = load_dataset(\"laion/laion400m\", split=\"train\", streaming=True)\n\n # Collect sample metadata first (fast)\n print(\"📋 Collecting sample metadata...\")\n candidates = []\n for sample in dataset:\n if len(candidates) >= num_samples * 3: # Get 3x more candidates in case some fail\n break\n\n image_url = sample.get(\"url\", \"\")\n caption = sample.get(\"caption\", \"\")\n\n if not image_url or not caption:\n continue\n\n image_filename = f\"laion_{len(candidates):06d}.jpg\"\n image_path = self.images_dir / image_filename\n\n candidate = {\n \"id\": f\"laion_{len(candidates):06d}\",\n \"url\": image_url,\n \"caption\": caption,\n \"image_path\": str(image_path),\n \"width\": sample.get(\"original_width\", 512),\n \"height\": sample.get(\"original_height\", 512),\n \"similarity\": sample.get(\"similarity\", 0.0),\n }\n candidates.append(candidate)\n\n print(\n f\"📊 Collected {len(candidates)} candidates, downloading {num_samples} in parallel...\"\n )\n\n # Download images in parallel\n async def download_batch():\n semaphore = asyncio.Semaphore(20) # Limit to 20 concurrent downloads\n connector = aiohttp.TCPConnector(limit=100, limit_per_host=20)\n timeout = aiohttp.ClientTimeout(total=30)\n\n progress_bar = tqdm(total=len(candidates[: num_samples * 2]), desc=\"Downloading images\")\n\n async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session:\n tasks = []\n for candidate in candidates[: num_samples * 2]: # Try 2x more than needed\n task = self.download_single_image(session, candidate, semaphore, progress_bar)\n tasks.append(task)\n\n # Wait for all downloads\n results = await asyncio.gather(*tasks, return_exceptions=True)\n progress_bar.close()\n\n # Filter successful downloads\n successful = [r for r in results if r is not None and not isinstance(r, Exception)]\n return successful[:num_samples]\n\n # Run async download\n loop = asyncio.new_event_loop()\n asyncio.set_event_loop(loop)\n try:\n samples = loop.run_until_complete(download_batch())\n finally:\n loop.close()\n\n # Save metadata\n with open(self.metadata_file, \"w\", encoding=\"utf-8\") as f:\n for sample in samples:\n f.write(json.dumps(sample) + \"\\n\")\n\n print(f\"✅ Downloaded {len(samples)} real LAION samples with async parallel downloading\")\n return samples\n\n def generate_clip_image_embeddings(self, samples: list[dict]):\n \"\"\"Generate CLIP image embeddings for downloaded images\"\"\"\n print(\"🔍 Generating CLIP image embeddings...\")\n\n # Load sentence-transformers CLIP (ViT-L/14, 768-dim) for image embeddings\n # This single model can encode both images and text.\n model = SentenceTransformer(\"clip-ViT-L-14\")\n\n embeddings = []\n valid_samples = []\n\n for sample in tqdm(samples, desc=\"Processing images\"):\n try:\n # Load image\n image_path = sample[\"image_path\"]\n image = Image.open(image_path).convert(\"RGB\")\n\n # Encode image to 768-dim embedding via sentence-transformers (normalized)\n vec = model.encode(\n [image],\n convert_to_numpy=True,\n normalize_embeddings=True,\n batch_size=1,\n show_progress_bar=False,\n )[0]\n embeddings.append(vec.astype(np.float32))\n valid_samples.append(sample)\n\n except Exception as e:\n print(f\" ⚠️ Failed to process {sample['id']}: {e}\")\n # Skip invalid images\n\n embeddings = np.array(embeddings, dtype=np.float32)\n\n # Save embeddings\n embeddings_file = self.data_dir / \"clip_image_embeddings.npy\"\n np.save(embeddings_file, embeddings)\n print(f\"✅ Generated {len(embeddings)} image embeddings, shape: {embeddings.shape}\")\n\n return embeddings, valid_samples\n\n def build_faiss_baseline(\n self, embeddings: np.ndarray, samples: list[dict], output_dir: str = \"baseline\"\n ):\n \"\"\"Build FAISS flat baseline using CLIP image embeddings\"\"\"\n print(\"🔨 Building FAISS Flat baseline...\")\n\n from leann_backend_hnsw import faiss\n\n os.makedirs(output_dir, exist_ok=True)\n baseline_path = os.path.join(output_dir, \"faiss_flat.index\")\n metadata_path = os.path.join(output_dir, \"metadata.pkl\")\n\n if os.path.exists(baseline_path) and os.path.exists(metadata_path):\n print(f\"✅ Baseline already exists at {baseline_path}\")\n return baseline_path\n\n # Extract image IDs (must be present)\n if not samples or \"id\" not in samples[0]:\n raise KeyError(\"samples missing 'id' field for FAISS baseline\")\n image_ids: list[str] = [str(sample[\"id\"]) for sample in samples]\n\n print(f\"📐 Embedding shape: {embeddings.shape}\")\n print(f\"📄 Processing {len(image_ids)} images\")\n\n # Build FAISS flat index\n print(\"🏗️ Building FAISS IndexFlatIP...\")\n dimension = embeddings.shape[1]\n index = faiss.IndexFlatIP(dimension)\n\n # Add embeddings to flat index\n embeddings_f32 = embeddings.astype(np.float32)\n index.add(embeddings_f32.shape[0], faiss.swig_ptr(embeddings_f32))\n\n # Save index and metadata\n faiss.write_index(index, baseline_path)\n with open(metadata_path, \"wb\") as f:\n pickle.dump(image_ids, f)\n\n print(f\"✅ FAISS baseline saved to {baseline_path}\")\n print(f\"✅ Metadata saved to {metadata_path}\")\n print(f\"📊 Total vectors: {index.ntotal}\")\n\n return baseline_path\n\n def create_leann_passages(self, samples: list[dict]):\n \"\"\"Create LEANN-compatible passages from LAION data\"\"\"\n print(\"📝 Creating LEANN passages...\")\n\n passages_file = self.data_dir / \"laion_passages.jsonl\"\n\n with open(passages_file, \"w\", encoding=\"utf-8\") as f:\n for i, sample in enumerate(samples):\n passage = {\n \"id\": sample[\"id\"],\n \"text\": sample[\"caption\"], # Use caption as searchable text\n \"metadata\": {\n \"image_url\": sample[\"url\"],\n \"image_path\": sample.get(\"image_path\", \"\"),\n \"width\": sample[\"width\"],\n \"height\": sample[\"height\"],\n \"similarity\": sample[\"similarity\"],\n \"image_index\": i, # Index for embedding lookup\n },\n }\n f.write(json.dumps(passage) + \"\\n\")\n\n print(f\"✅ Created {len(samples)} passages\")\n return passages_file\n\n def build_compact_index(\n self, passages_file: Path, embeddings: np.ndarray, index_path: str, backend: str = \"hnsw\"\n ):\n \"\"\"Build compact LEANN index with CLIP embeddings (recompute=True, compact=True)\"\"\"\n print(f\"🏗️ Building compact LEANN index with {backend} backend...\")\n\n start_time = time.time()\n\n # Save CLIP embeddings (npy) and also a pickle with (ids, embeddings)\n npy_path = self.data_dir / \"clip_image_embeddings.npy\"\n np.save(npy_path, embeddings)\n print(f\"💾 Saved CLIP embeddings to {npy_path}\")\n\n # Prepare ids in the same order as passages_file (matches embeddings order)\n ids: list[str] = []\n with open(passages_file, encoding=\"utf-8\") as f:\n for line in f:\n if line.strip():\n rec = json.loads(line)\n ids.append(str(rec[\"id\"]))\n\n if len(ids) != embeddings.shape[0]:\n raise ValueError(\n f\"IDs count ({len(ids)}) does not match embeddings ({embeddings.shape[0]}).\"\n )\n\n pkl_path = self.data_dir / \"clip_image_embeddings.pkl\"\n with open(pkl_path, \"wb\") as pf:\n pickle.dump((ids, embeddings.astype(np.float32)), pf)\n print(f\"💾 Saved (ids, embeddings) pickle to {pkl_path}\")\n\n # Initialize builder - compact with recompute\n # Note: For multimodal case, we need to handle embeddings differently\n # Let's try using sentence-transformers mode but with custom embeddings\n builder = LeannBuilder(\n backend_name=backend,\n # Use CLIP text encoder (ViT-L/14) to match image space (768-dim)\n embedding_model=\"clip-ViT-L-14\",\n embedding_mode=\"sentence-transformers\",\n # HNSW params (or forwarded to chosen backend)\n graph_degree=32,\n complexity=64,\n # Compact/pruned with recompute at query time\n is_recompute=True,\n is_compact=True,\n distance_metric=\"cosine\", # CLIP uses normalized vectors; cosine is appropriate\n num_threads=4,\n )\n\n # Add passages (text + metadata)\n print(\"📚 Adding passages...\")\n self._add_passages_with_embeddings(builder, passages_file, embeddings)\n\n print(f\"🔨 Building compact index at {index_path} from precomputed embeddings...\")\n builder.build_index_from_embeddings(index_path, str(pkl_path))\n\n build_time = time.time() - start_time\n print(f\"✅ Compact index built in {build_time:.2f}s\")\n\n # Analyze index size\n self._analyze_index_size(index_path)\n\n return index_path\n\n def build_non_compact_index(\n self, passages_file: Path, embeddings: np.ndarray, index_path: str, backend: str = \"hnsw\"\n ):\n \"\"\"Build non-compact LEANN index with CLIP embeddings (recompute=False, compact=False)\"\"\"\n print(f\"🏗️ Building non-compact LEANN index with {backend} backend...\")\n\n start_time = time.time()\n\n # Ensure embeddings are saved (npy + pickle)\n npy_path = self.data_dir / \"clip_image_embeddings.npy\"\n if not npy_path.exists():\n np.save(npy_path, embeddings)\n print(f\"💾 Saved CLIP embeddings to {npy_path}\")\n # Prepare ids in same order as passages_file\n ids: list[str] = []\n with open(passages_file, encoding=\"utf-8\") as f:\n for line in f:\n if line.strip():\n rec = json.loads(line)\n ids.append(str(rec[\"id\"]))\n if len(ids) != embeddings.shape[0]:\n raise ValueError(\n f\"IDs count ({len(ids)}) does not match embeddings ({embeddings.shape[0]}).\"\n )\n pkl_path = self.data_dir / \"clip_image_embeddings.pkl\"\n if not pkl_path.exists():\n with open(pkl_path, \"wb\") as pf:\n pickle.dump((ids, embeddings.astype(np.float32)), pf)\n print(f\"💾 Saved (ids, embeddings) pickle to {pkl_path}\")\n\n # Initialize builder - non-compact without recompute\n builder = LeannBuilder(\n backend_name=backend,\n embedding_model=\"clip-ViT-L-14\",\n embedding_mode=\"sentence-transformers\",\n graph_degree=32,\n complexity=64,\n is_recompute=False, # Store embeddings (no recompute needed)\n is_compact=False, # Store full index (not pruned)\n distance_metric=\"cosine\",\n num_threads=4,\n )\n\n # Add passages - embeddings will be loaded from file\n print(\"📚 Adding passages...\")\n self._add_passages_with_embeddings(builder, passages_file, embeddings)\n\n print(f\"🔨 Building non-compact index at {index_path} from precomputed embeddings...\")\n builder.build_index_from_embeddings(index_path, str(pkl_path))\n\n build_time = time.time() - start_time\n print(f\"✅ Non-compact index built in {build_time:.2f}s\")\n\n # Analyze index size\n self._analyze_index_size(index_path)\n\n return index_path\n\n def _add_passages_with_embeddings(self, builder, passages_file: Path, embeddings: np.ndarray):\n \"\"\"Helper to add passages with pre-computed CLIP embeddings\"\"\"\n with open(passages_file, encoding=\"utf-8\") as f:\n for line in tqdm(f, desc=\"Adding passages\"):\n if line.strip():\n passage = json.loads(line)\n\n # Add image metadata - LEANN will handle embeddings separately\n # Note: We store image metadata and caption text for searchability\n # Important: ensure passage ID in metadata matches vector ID\n builder.add_text(\n text=passage[\"text\"], # Image caption for searchability\n metadata={**passage[\"metadata\"], \"id\": passage[\"id\"]},\n )\n\n def _analyze_index_size(self, index_path: str):\n \"\"\"Analyze index file sizes\"\"\"\n print(\"📏 Analyzing index sizes...\")\n\n index_path = Path(index_path)\n index_dir = index_path.parent\n index_name = index_path.name # e.g., laion_index.leann\n index_prefix = index_path.stem # e.g., laion_index\n\n files = [\n (f\"{index_prefix}.index\", \".index\", \"core\"),\n (f\"{index_name}.meta.json\", \".meta.json\", \"core\"),\n (f\"{index_name}.ids.txt\", \".ids.txt\", \"core\"),\n (f\"{index_name}.passages.jsonl\", \".passages.jsonl\", \"passages\"),\n (f\"{index_name}.passages.idx\", \".passages.idx\", \"passages\"),\n ]\n\n def _fmt_size(bytes_val: int) -> str:\n if bytes_val < 1024:\n return f\"{bytes_val} B\"\n kb = bytes_val / 1024\n if kb < 1024:\n return f\"{kb:.1f} KB\"\n mb = kb / 1024\n if mb < 1024:\n return f\"{mb:.2f} MB\"\n gb = mb / 1024\n return f\"{gb:.2f} GB\"\n\n total_index_only_mb = 0.0\n total_all_mb = 0.0\n for filename, label, group in files:\n file_path = index_dir / filename\n if file_path.exists():\n size_bytes = file_path.stat().st_size\n print(f\" {label}: {_fmt_size(size_bytes)}\")\n size_mb = size_bytes / (1024 * 1024)\n total_all_mb += size_mb\n if group == \"core\":\n total_index_only_mb += size_mb\n else:\n print(f\" {label}: (missing)\")\n print(f\" Total (index only, exclude passages): {total_index_only_mb:.2f} MB\")\n print(f\" Total (including passages): {total_all_mb:.2f} MB\")\n\n def create_evaluation_queries(self, samples: list[dict], num_queries: int = 200):\n \"\"\"Create evaluation queries from captions\"\"\"\n print(f\"📝 Creating {num_queries} evaluation queries...\")\n\n # Sample random captions as queries\n import random\n\n random.seed(42) # For reproducibility\n\n query_samples = random.sample(samples, min(num_queries, len(samples)))\n\n queries_file = self.data_dir / \"evaluation_queries.jsonl\"\n with open(queries_file, \"w\", encoding=\"utf-8\") as f:\n for sample in query_samples:\n query = {\n \"id\": sample[\"id\"],\n \"query\": sample[\"caption\"],\n \"ground_truth_id\": sample[\"id\"], # For potential recall evaluation\n }\n f.write(json.dumps(query) + \"\\n\")\n\n print(f\"✅ Created {len(query_samples)} evaluation queries\")\n return queries_file\n\n\ndef main():\n parser = argparse.ArgumentParser(description=\"Setup LAION Multimodal Benchmark\")\n parser.add_argument(\"--data-dir\", default=\"data\", help=\"Data directory\")\n parser.add_argument(\"--num-samples\", type=int, default=1000, help=\"Number of LAION samples\")\n parser.add_argument(\"--num-queries\", type=int, default=50, help=\"Number of evaluation queries\")\n parser.add_argument(\"--index-path\", default=\"data/laion_index.leann\", help=\"Output index path\")\n parser.add_argument(\n \"--backend\", default=\"hnsw\", choices=[\"hnsw\", \"diskann\"], help=\"LEANN backend\"\n )\n parser.add_argument(\"--skip-download\", action=\"store_true\", help=\"Skip LAION dataset download\")\n parser.add_argument(\"--skip-build\", action=\"store_true\", help=\"Skip index building\")\n\n args = parser.parse_args()\n\n print(\"🚀 Setting up LAION Multimodal Benchmark\")\n print(\"=\" * 50)\n\n try:\n # Initialize setup\n setup = LAIONSetup(args.data_dir)\n\n # Step 1: Download LAION subset\n if not args.skip_download:\n print(\"\\n📦 Step 1: Download LAION subset\")\n samples = setup.download_laion_subset(args.num_samples)\n\n # Step 2: Generate CLIP image embeddings\n print(\"\\n🔍 Step 2: Generate CLIP image embeddings\")\n embeddings, valid_samples = setup.generate_clip_image_embeddings(samples)\n\n # Step 3: Create LEANN passages (image metadata with embeddings)\n print(\"\\n📝 Step 3: Create LEANN passages\")\n passages_file = setup.create_leann_passages(valid_samples)\n else:\n print(\"⏭️ Skipping LAION dataset download\")\n # Load existing data\n passages_file = setup.data_dir / \"laion_passages.jsonl\"\n embeddings_file = setup.data_dir / \"clip_image_embeddings.npy\"\n\n if not passages_file.exists() or not embeddings_file.exists():\n raise FileNotFoundError(\n \"Passages or embeddings file not found. Run without --skip-download first.\"\n )\n\n embeddings = np.load(embeddings_file)\n print(f\"📊 Loaded {len(embeddings)} embeddings from {embeddings_file}\")\n\n # Step 4: Build LEANN indexes (both compact and non-compact)\n if not args.skip_build:\n print(\"\\n🏗️ Step 4: Build LEANN indexes with CLIP image embeddings\")\n\n # Build compact index (production mode - small, recompute required)\n compact_index_path = args.index_path\n print(f\"Building compact index: {compact_index_path}\")\n setup.build_compact_index(passages_file, embeddings, compact_index_path, args.backend)\n\n # Build non-compact index (comparison mode - large, fast search)\n non_compact_index_path = args.index_path.replace(\".leann\", \"_noncompact.leann\")\n print(f\"Building non-compact index: {non_compact_index_path}\")\n setup.build_non_compact_index(\n passages_file, embeddings, non_compact_index_path, args.backend\n )\n\n # Step 5: Build FAISS flat baseline\n print(\"\\n🔨 Step 5: Build FAISS flat baseline\")\n if not args.skip_download:\n baseline_path = setup.build_faiss_baseline(embeddings, valid_samples)\n else:\n # Load valid_samples from passages file for FAISS baseline\n valid_samples = []\n with open(passages_file, encoding=\"utf-8\") as f:\n for line in f:\n if line.strip():\n passage = json.loads(line)\n valid_samples.append({\"id\": passage[\"id\"], \"caption\": passage[\"text\"]})\n baseline_path = setup.build_faiss_baseline(embeddings, valid_samples)\n\n # Step 6: Create evaluation queries\n print(\"\\n📝 Step 6: Create evaluation queries\")\n queries_file = setup.create_evaluation_queries(valid_samples, args.num_queries)\n else:\n print(\"⏭️ Skipping index building\")\n baseline_path = \"data/baseline/faiss_index.bin\"\n queries_file = setup.data_dir / \"evaluation_queries.jsonl\"\n\n print(\"\\n🎉 Setup completed successfully!\")\n print(\"📊 Summary:\")\n if not args.skip_download:\n print(f\" Downloaded samples: {len(samples)}\")\n print(f\" Valid samples with embeddings: {len(valid_samples)}\")\n else:\n print(f\" Loaded {len(embeddings)} embeddings\")\n\n if not args.skip_build:\n print(f\" Compact index: {compact_index_path}\")\n print(f\" Non-compact index: {non_compact_index_path}\")\n print(f\" FAISS baseline: {baseline_path}\")\n print(f\" Queries: {queries_file}\")\n\n print(\"\\n🔧 Next steps:\")\n print(f\" Run evaluation: python evaluate_laion.py --index {compact_index_path}\")\n print(f\" Or compare with: python evaluate_laion.py --index {non_compact_index_path}\")\n else:\n print(\" Skipped building indexes\")\n\n except KeyboardInterrupt:\n print(\"\\n⚠️ Setup interrupted by user\")\n exit(1)\n except Exception as e:\n print(f\"\\n❌ Setup failed: {e}\")\n exit(1)\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "benchmarks/llm_utils.py", "content": "\"\"\"\nLLM utils for RAG benchmarks with Qwen3-8B and Qwen2.5-VL (multimodal)\n\"\"\"\n\nimport time\n\ntry:\n import torch\n from transformers import AutoModelForCausalLM, AutoTokenizer\n\n HF_AVAILABLE = True\nexcept ImportError:\n HF_AVAILABLE = False\n\ntry:\n from vllm import LLM, SamplingParams\n\n VLLM_AVAILABLE = True\nexcept ImportError:\n VLLM_AVAILABLE = False\n\n\ndef is_qwen3_model(model_name):\n \"\"\"Check if model is Qwen3\"\"\"\n return \"Qwen3\" in model_name or \"qwen3\" in model_name.lower()\n\n\ndef is_qwen_vl_model(model_name):\n \"\"\"Check if model is Qwen2.5-VL\"\"\"\n return \"Qwen2.5-VL\" in model_name or \"qwen2.5-vl\" in model_name.lower()\n\n\ndef apply_qwen3_chat_template(tokenizer, prompt):\n \"\"\"Apply Qwen3 chat template with thinking enabled\"\"\"\n messages = [{\"role\": \"user\", \"content\": prompt}]\n return tokenizer.apply_chat_template(\n messages,\n tokenize=False,\n add_generation_prompt=True,\n enable_thinking=True,\n )\n\n\ndef extract_thinking_answer(response):\n \"\"\"Extract final answer from Qwen3 thinking model response\"\"\"\n if \"\" in response and \"\" in response:\n try:\n think_end = response.index(\"\") + len(\"\")\n final_answer = response[think_end:].strip()\n return final_answer\n except (ValueError, IndexError):\n pass\n\n return response.strip()\n\n\ndef load_hf_model(model_name=\"Qwen/Qwen3-8B\", trust_remote_code=False):\n \"\"\"Load HuggingFace model\n\n Args:\n model_name (str): Name of the model to load\n trust_remote_code (bool): Whether to allow execution of code from the model repository.\n Defaults to False for security. Only enable for trusted models.\n \"\"\"\n if not HF_AVAILABLE:\n raise ImportError(\"transformers not available\")\n\n if trust_remote_code:\n print(\n \"⚠️ WARNING: Loading model with trust_remote_code=True. This can execute arbitrary code.\"\n )\n\n print(f\"Loading HF: {model_name}\")\n tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=trust_remote_code)\n model = AutoModelForCausalLM.from_pretrained(\n model_name,\n torch_dtype=torch.float16 if torch.cuda.is_available() else torch.float32,\n device_map=\"auto\",\n trust_remote_code=trust_remote_code,\n )\n return tokenizer, model\n\n\ndef load_vllm_model(model_name=\"Qwen/Qwen3-8B\", trust_remote_code=False):\n \"\"\"Load vLLM model\n\n Args:\n model_name (str): Name of the model to load\n trust_remote_code (bool): Whether to allow execution of code from the model repository.\n Defaults to False for security. Only enable for trusted models.\n \"\"\"\n if not VLLM_AVAILABLE:\n raise ImportError(\"vllm not available\")\n\n if trust_remote_code:\n print(\n \"⚠️ WARNING: Loading model with trust_remote_code=True. This can execute arbitrary code.\"\n )\n\n print(f\"Loading vLLM: {model_name}\")\n llm = LLM(model=model_name, trust_remote_code=trust_remote_code)\n\n # Qwen3 specific config\n if is_qwen3_model(model_name):\n stop_tokens = [\"<|im_end|>\", \"<|end_of_text|>\"]\n max_tokens = 2048\n else:\n stop_tokens = None\n max_tokens = 1024\n\n sampling_params = SamplingParams(temperature=0.7, max_tokens=max_tokens, stop=stop_tokens)\n return llm, sampling_params\n\n\ndef generate_hf(tokenizer, model, prompt, max_tokens=None):\n \"\"\"Generate with HF - supports Qwen3 thinking models\"\"\"\n model_name = getattr(model, \"name_or_path\", \"unknown\")\n is_qwen3 = is_qwen3_model(model_name)\n\n # Apply chat template for Qwen3\n if is_qwen3:\n prompt = apply_qwen3_chat_template(tokenizer, prompt)\n max_tokens = max_tokens or 2048\n else:\n max_tokens = max_tokens or 1024\n\n inputs = tokenizer(prompt, return_tensors=\"pt\").to(model.device)\n with torch.no_grad():\n outputs = model.generate(\n **inputs,\n max_new_tokens=max_tokens,\n temperature=0.7,\n do_sample=True,\n pad_token_id=tokenizer.eos_token_id,\n )\n response = tokenizer.decode(outputs[0], skip_special_tokens=True)\n response = response[len(prompt) :].strip()\n\n # Extract final answer for thinking models\n if is_qwen3:\n return extract_thinking_answer(response)\n return response\n\n\ndef generate_vllm(llm, sampling_params, prompt):\n \"\"\"Generate with vLLM - supports Qwen3 thinking models\"\"\"\n outputs = llm.generate([prompt], sampling_params)\n response = outputs[0].outputs[0].text.strip()\n\n # Extract final answer for Qwen3 thinking models\n model_name = str(llm.llm_engine.model_config.model)\n if is_qwen3_model(model_name):\n return extract_thinking_answer(response)\n return response\n\n\ndef create_prompt(context, query, domain=\"default\"):\n \"\"\"Create RAG prompt\"\"\"\n if domain == \"emails\":\n return f\"Email content:\\n{context}\\n\\nQuestion: {query}\\n\\nAnswer:\"\n elif domain == \"finance\":\n return f\"Financial content:\\n{context}\\n\\nQuestion: {query}\\n\\nAnswer:\"\n elif domain == \"multimodal\":\n return f\"Image context:\\n{context}\\n\\nQuestion: {query}\\n\\nAnswer:\"\n else:\n return f\"Context: {context}\\n\\nQuestion: {query}\\n\\nAnswer:\"\n\n\ndef evaluate_rag(searcher, llm_func, queries, domain=\"default\", top_k=3, complexity=64):\n \"\"\"Simple RAG evaluation with timing\"\"\"\n search_times = []\n gen_times = []\n results = []\n\n for i, query in enumerate(queries):\n # Search\n start = time.time()\n docs = searcher.search(query, top_k=top_k, complexity=complexity)\n search_time = time.time() - start\n\n # Generate\n context = \"\\n\\n\".join([doc.text for doc in docs])\n prompt = create_prompt(context, query, domain)\n\n start = time.time()\n response = llm_func(prompt)\n gen_time = time.time() - start\n\n search_times.append(search_time)\n gen_times.append(gen_time)\n results.append(response)\n\n if i < 3:\n print(f\"Q{i + 1}: Search={search_time:.3f}s, Gen={gen_time:.3f}s\")\n\n return {\n \"avg_search_time\": sum(search_times) / len(search_times),\n \"avg_generation_time\": sum(gen_times) / len(gen_times),\n \"results\": results,\n }\n\n\ndef load_qwen_vl_model(model_name=\"Qwen/Qwen2.5-VL-7B-Instruct\", trust_remote_code=False):\n \"\"\"Load Qwen2.5-VL multimodal model\n\n Args:\n model_name (str): Name of the model to load\n trust_remote_code (bool): Whether to allow execution of code from the model repository.\n Defaults to False for security. Only enable for trusted models.\n \"\"\"\n if not HF_AVAILABLE:\n raise ImportError(\"transformers not available\")\n\n if trust_remote_code:\n print(\n \"⚠️ WARNING: Loading model with trust_remote_code=True. This can execute arbitrary code.\"\n )\n\n print(f\"Loading Qwen2.5-VL: {model_name}\")\n\n try:\n from transformers import AutoModelForVision2Seq, AutoProcessor\n\n processor = AutoProcessor.from_pretrained(model_name, trust_remote_code=trust_remote_code)\n model = AutoModelForVision2Seq.from_pretrained(\n model_name,\n torch_dtype=torch.bfloat16,\n device_map=\"auto\",\n trust_remote_code=trust_remote_code,\n )\n\n return processor, model\n\n except Exception as e:\n print(f\"Failed to load with AutoModelForVision2Seq, trying specific class: {e}\")\n\n # Fallback to specific class\n try:\n from transformers import AutoProcessor, Qwen2VLForConditionalGeneration\n\n processor = AutoProcessor.from_pretrained(\n model_name, trust_remote_code=trust_remote_code\n )\n model = Qwen2VLForConditionalGeneration.from_pretrained(\n model_name,\n torch_dtype=torch.bfloat16,\n device_map=\"auto\",\n trust_remote_code=trust_remote_code,\n )\n\n return processor, model\n\n except Exception as e2:\n raise ImportError(f\"Failed to load Qwen2.5-VL model: {e2}\")\n\n\ndef generate_qwen_vl(processor, model, prompt, image_path=None, max_tokens=512):\n \"\"\"Generate with Qwen2.5-VL multimodal model\"\"\"\n from PIL import Image\n\n # Prepare inputs\n if image_path:\n image = Image.open(image_path)\n inputs = processor(text=prompt, images=image, return_tensors=\"pt\").to(model.device)\n else:\n inputs = processor(text=prompt, return_tensors=\"pt\").to(model.device)\n\n # Generate\n with torch.no_grad():\n generated_ids = model.generate(\n **inputs, max_new_tokens=max_tokens, do_sample=False, temperature=0.1\n )\n\n # Decode response\n generated_ids = generated_ids[:, inputs[\"input_ids\"].shape[1] :]\n response = processor.decode(generated_ids[0], skip_special_tokens=True)\n\n return response\n\n\ndef create_multimodal_prompt(context, query, image_descriptions, task_type=\"images\"):\n \"\"\"Create prompt for multimodal RAG\"\"\"\n if task_type == \"images\":\n return f\"\"\"Based on the retrieved images and their descriptions, answer the following question.\n\nRetrieved Image Descriptions:\n{context}\n\nQuestion: {query}\n\nProvide a detailed answer based on the visual content described above.\"\"\"\n\n return f\"Context: {context}\\nQuestion: {query}\\nAnswer:\"\n\n\ndef evaluate_multimodal_rag(searcher, queries, processor=None, model=None, complexity=64):\n \"\"\"Evaluate multimodal RAG with Qwen2.5-VL\"\"\"\n search_times = []\n gen_times = []\n results = []\n\n for i, query_item in enumerate(queries):\n # Handle both string and dict formats for queries\n if isinstance(query_item, dict):\n query = query_item.get(\"query\", \"\")\n image_path = query_item.get(\"image_path\") # Optional reference image\n else:\n query = str(query_item)\n image_path = None\n\n # Search\n start_time = time.time()\n search_results = searcher.search(query, top_k=3, complexity=complexity)\n search_time = time.time() - start_time\n search_times.append(search_time)\n\n # Prepare context from search results\n context_parts = []\n for result in search_results:\n context_parts.append(f\"- {result.text}\")\n context = \"\\n\".join(context_parts)\n\n # Generate with multimodal model\n start_time = time.time()\n if processor and model:\n prompt = create_multimodal_prompt(context, query, context_parts)\n response = generate_qwen_vl(processor, model, prompt, image_path)\n else:\n response = f\"Context: {context}\"\n gen_time = time.time() - start_time\n\n gen_times.append(gen_time)\n results.append(response)\n\n if i < 3:\n print(f\"Q{i + 1}: Search={search_time:.3f}s, Gen={gen_time:.3f}s\")\n\n return {\n \"avg_search_time\": sum(search_times) / len(search_times),\n \"avg_generation_time\": sum(gen_times) / len(gen_times),\n \"results\": results,\n }\n" }, { "path": "benchmarks/micro_tpt.py", "content": "# python embedd_micro.py --use_int8 Fastest\n\nimport argparse\nimport time\nfrom contextlib import contextmanager\nfrom dataclasses import dataclass\n\nimport numpy as np\nimport torch\nfrom torch import nn\nfrom tqdm import tqdm\nfrom transformers import AutoModel, BitsAndBytesConfig\n\n\n@dataclass\nclass BenchmarkConfig:\n model_path: str\n batch_sizes: list[int]\n seq_length: int\n num_runs: int\n use_fp16: bool = True\n use_int4: bool = False\n use_int8: bool = False # Add this parameter\n use_cuda_graphs: bool = False\n use_flash_attention: bool = False\n use_linear8bitlt: bool = False\n\n\nclass GraphContainer:\n \"\"\"Container for managing graphs for different batch sizes (CUDA graphs on NVIDIA, regular on others).\"\"\"\n\n def __init__(self, model: nn.Module, seq_length: int):\n self.model = model\n self.seq_length = seq_length\n self.graphs: dict[int, GraphWrapper] = {}\n\n def get_or_create(self, batch_size: int) -> \"GraphWrapper\":\n if batch_size not in self.graphs:\n self.graphs[batch_size] = GraphWrapper(self.model, batch_size, self.seq_length)\n return self.graphs[batch_size]\n\n\nclass GraphWrapper:\n \"\"\"Wrapper for graph capture and replay (CUDA graphs on NVIDIA, regular on others).\"\"\"\n\n def __init__(self, model: nn.Module, batch_size: int, seq_length: int):\n self.model = model\n self.device = self._get_device()\n self.static_input = self._create_random_batch(batch_size, seq_length)\n self.static_attention_mask = torch.ones_like(self.static_input)\n\n # Warm up\n self._warmup()\n\n # Only use CUDA graphs on NVIDIA GPUs\n if torch.cuda.is_available() and hasattr(torch.cuda, \"CUDAGraph\"):\n # Capture graph\n self.graph = torch.cuda.CUDAGraph()\n with torch.cuda.graph(self.graph):\n self.static_output = self.model(\n input_ids=self.static_input,\n attention_mask=self.static_attention_mask,\n )\n self.use_cuda_graph = True\n else:\n # For MPS or CPU, just store the model\n self.use_cuda_graph = False\n self.static_output = None\n\n def _get_device(self) -> str:\n if torch.cuda.is_available():\n return \"cuda\"\n elif torch.backends.mps.is_available():\n return \"mps\"\n else:\n return \"cpu\"\n\n def _create_random_batch(self, batch_size: int, seq_length: int) -> torch.Tensor:\n return torch.randint(\n 0, 1000, (batch_size, seq_length), device=self.device, dtype=torch.long\n )\n\n def _warmup(self, num_warmup: int = 3):\n with torch.no_grad():\n for _ in range(num_warmup):\n self.model(\n input_ids=self.static_input,\n attention_mask=self.static_attention_mask,\n )\n\n def __call__(self, input_ids: torch.Tensor, attention_mask: torch.Tensor) -> torch.Tensor:\n if self.use_cuda_graph:\n self.static_input.copy_(input_ids)\n self.static_attention_mask.copy_(attention_mask)\n self.graph.replay()\n return self.static_output\n else:\n # For MPS/CPU, just run normally\n return self.model(input_ids=input_ids, attention_mask=attention_mask)\n\n\nclass ModelOptimizer:\n \"\"\"Applies various optimizations to the model.\"\"\"\n\n @staticmethod\n def optimize(model: nn.Module, config: BenchmarkConfig) -> nn.Module:\n print(\"\\nApplying model optimizations:\")\n\n if model is None:\n raise ValueError(\"Cannot optimize None model\")\n\n # Move to GPU\n if torch.cuda.is_available():\n model = model.cuda()\n device = \"cuda\"\n elif torch.backends.mps.is_available():\n model = model.to(\"mps\")\n device = \"mps\"\n else:\n model = model.cpu()\n device = \"cpu\"\n print(f\"- Model moved to {device}\")\n\n # FP16\n if config.use_fp16 and not config.use_int4:\n model = model.half()\n # use torch compile\n model = torch.compile(model)\n print(\"- Using FP16 precision\")\n\n # Check if using SDPA (only on CUDA)\n if (\n torch.cuda.is_available()\n and torch.version.cuda\n and float(torch.version.cuda[:3]) >= 11.6\n ):\n if hasattr(torch.nn.functional, \"scaled_dot_product_attention\"):\n print(\"- Using PyTorch SDPA (scaled_dot_product_attention)\")\n else:\n print(\"- PyTorch SDPA not available\")\n\n # Flash Attention (only on CUDA)\n if config.use_flash_attention and torch.cuda.is_available():\n try:\n from flash_attn.flash_attention import FlashAttention # noqa: F401\n\n print(\"- Flash Attention 2 available\")\n if hasattr(model.config, \"attention_mode\"):\n model.config.attention_mode = \"flash_attention_2\"\n print(\" - Enabled Flash Attention 2 mode\")\n except ImportError:\n print(\"- Flash Attention not available\")\n\n # Memory efficient attention (only on CUDA)\n if torch.cuda.is_available():\n try:\n from xformers.ops import memory_efficient_attention # noqa: F401\n\n if hasattr(model, \"enable_xformers_memory_efficient_attention\"):\n model.enable_xformers_memory_efficient_attention()\n print(\"- Enabled xformers memory efficient attention\")\n else:\n print(\"- Model doesn't support xformers\")\n except (ImportError, AttributeError):\n print(\"- Xformers not available\")\n\n model.eval()\n print(\"- Model set to eval mode\")\n\n return model\n\n\nclass Timer:\n \"\"\"Handles accurate GPU timing using GPU events or CPU timing.\"\"\"\n\n def __init__(self):\n if torch.cuda.is_available():\n self.start_event = torch.cuda.Event(enable_timing=True)\n self.end_event = torch.cuda.Event(enable_timing=True)\n self.use_gpu_timing = True\n elif torch.backends.mps.is_available():\n # MPS doesn't have events, use CPU timing\n self.use_gpu_timing = False\n else:\n # CPU timing\n self.use_gpu_timing = False\n\n @contextmanager\n def timing(self):\n if self.use_gpu_timing:\n self.start_event.record()\n yield\n self.end_event.record()\n self.end_event.synchronize()\n else:\n # Use CPU timing for MPS/CPU\n start_time = time.time()\n yield\n self.cpu_elapsed = time.time() - start_time\n\n def elapsed_time(self) -> float:\n if self.use_gpu_timing:\n return self.start_event.elapsed_time(self.end_event) / 1000 # ms to seconds\n else:\n return self.cpu_elapsed\n\n\nclass Benchmark:\n \"\"\"Main benchmark runner.\"\"\"\n\n def __init__(self, config: BenchmarkConfig):\n self.config = config\n try:\n self.model = self._load_model()\n if self.model is None:\n raise ValueError(\"Model initialization failed - model is None\")\n\n # Only use CUDA graphs on NVIDIA GPUs\n if config.use_cuda_graphs and torch.cuda.is_available():\n self.graphs = GraphContainer(self.model, config.seq_length)\n else:\n self.graphs = None\n self.timer = Timer()\n except Exception as e:\n print(f\"ERROR in benchmark initialization: {e!s}\")\n raise\n\n def _load_model(self) -> nn.Module:\n print(f\"Loading model from {self.config.model_path}...\")\n\n try:\n # Int4 quantization using HuggingFace integration\n if self.config.use_int4:\n import bitsandbytes as bnb\n\n print(f\"- bitsandbytes version: {bnb.__version__}\")\n\n # Check if using custom 8bit quantization\n if hasattr(self.config, \"use_linear8bitlt\") and self.config.use_linear8bitlt:\n print(\"- Using custom Linear8bitLt replacement for all linear layers\")\n\n # Load original model (without quantization config)\n import bitsandbytes as bnb\n import torch\n\n # set default to half\n torch.set_default_dtype(torch.float16)\n compute_dtype = torch.float16 if self.config.use_fp16 else torch.float32\n model = AutoModel.from_pretrained(\n self.config.model_path,\n torch_dtype=compute_dtype,\n )\n\n # Define replacement function\n def replace_linear_with_linear8bitlt(model):\n \"\"\"Recursively replace all nn.Linear layers with Linear8bitLt\"\"\"\n for name, module in list(model.named_children()):\n if isinstance(module, nn.Linear):\n # Get original linear layer parameters\n in_features = module.in_features\n out_features = module.out_features\n bias = module.bias is not None\n\n # Create 8bit linear layer\n # print size\n print(f\"in_features: {in_features}, out_features: {out_features}\")\n new_module = bnb.nn.Linear8bitLt(\n in_features,\n out_features,\n bias=bias,\n has_fp16_weights=False,\n )\n\n # Copy weights and bias\n new_module.weight.data = module.weight.data\n if bias:\n new_module.bias.data = module.bias.data\n\n # Replace module\n setattr(model, name, new_module)\n else:\n # Process child modules recursively\n replace_linear_with_linear8bitlt(module)\n\n return model\n\n # Replace all linear layers\n model = replace_linear_with_linear8bitlt(model)\n # add torch compile\n model = torch.compile(model)\n\n # Move model to GPU (quantization happens here)\n device = (\n \"cuda\"\n if torch.cuda.is_available()\n else \"mps\"\n if torch.backends.mps.is_available()\n else \"cpu\"\n )\n model = model.to(device)\n\n print(\"- All linear layers replaced with Linear8bitLt\")\n\n else:\n # Use original Int4 quantization method\n print(\"- Using bitsandbytes for Int4 quantization\")\n\n # Create quantization config\n\n compute_dtype = torch.float16 if self.config.use_fp16 else torch.float32\n quantization_config = BitsAndBytesConfig(\n load_in_4bit=True,\n bnb_4bit_compute_dtype=compute_dtype,\n bnb_4bit_use_double_quant=True,\n bnb_4bit_quant_type=\"nf4\",\n )\n\n print(\"- Quantization config:\", quantization_config)\n\n # Load model directly with quantization config\n model = AutoModel.from_pretrained(\n self.config.model_path,\n quantization_config=quantization_config,\n torch_dtype=compute_dtype,\n device_map=\"auto\", # Let HF decide on device mapping\n )\n\n # Check if model loaded successfully\n if model is None:\n raise ValueError(\"Model loading returned None\")\n\n print(f\"- Model type: {type(model)}\")\n\n # Apply optimizations directly here\n print(\"\\nApplying model optimizations:\")\n\n if hasattr(self.config, \"use_linear8bitlt\") and self.config.use_linear8bitlt:\n print(\"- Model moved to GPU with Linear8bitLt quantization\")\n else:\n # Skip moving to GPU since device_map=\"auto\" already did that\n print(\"- Model already on GPU due to device_map='auto'\")\n\n # Skip FP16 conversion since we specified compute_dtype\n print(f\"- Using {compute_dtype} for compute dtype\")\n\n # Check CUDA and SDPA\n if (\n torch.cuda.is_available()\n and torch.version.cuda\n and float(torch.version.cuda[:3]) >= 11.6\n ):\n if hasattr(torch.nn.functional, \"scaled_dot_product_attention\"):\n print(\"- Using PyTorch SDPA (scaled_dot_product_attention)\")\n else:\n print(\"- PyTorch SDPA not available\")\n\n # Try xformers if available (only on CUDA)\n if torch.cuda.is_available():\n try:\n if hasattr(model, \"enable_xformers_memory_efficient_attention\"):\n model.enable_xformers_memory_efficient_attention()\n print(\"- Enabled xformers memory efficient attention\")\n else:\n print(\"- Model doesn't support xformers\")\n except (ImportError, AttributeError):\n print(\"- Xformers not available\")\n\n # Set to eval mode\n model.eval()\n print(\"- Model set to eval mode\")\n # Int8 quantization using HuggingFace integration\n elif self.config.use_int8:\n print(\"- Using INT8 quantization\")\n # For now, just use standard loading with INT8 config\n compute_dtype = torch.float16 if self.config.use_fp16 else torch.float32\n quantization_config = BitsAndBytesConfig(\n load_in_8bit=True,\n llm_int8_threshold=6.0,\n llm_int8_has_fp16_weight=False,\n )\n\n model = AutoModel.from_pretrained(\n self.config.model_path,\n quantization_config=quantization_config,\n torch_dtype=compute_dtype,\n device_map=\"auto\",\n )\n\n if model is None:\n raise ValueError(\"Model loading returned None\")\n\n print(f\"- Model type: {type(model)}\")\n model.eval()\n print(\"- Model set to eval mode\")\n\n else:\n # Standard loading for FP16/FP32\n model = AutoModel.from_pretrained(self.config.model_path)\n print(\"- Model loaded in standard precision\")\n print(f\"- Model type: {type(model)}\")\n\n # Apply standard optimizations\n # set default to half\n import torch\n\n torch.set_default_dtype(torch.bfloat16)\n model = ModelOptimizer.optimize(model, self.config)\n model = model.half()\n # add torch compile\n model = torch.compile(model)\n\n # Final check to ensure model is not None\n if model is None:\n raise ValueError(\"Model is None after optimization\")\n\n print(f\"- Final model type: {type(model)}\")\n return model\n\n except Exception as e:\n print(f\"ERROR loading model: {e!s}\")\n import traceback\n\n traceback.print_exc()\n raise\n\n def _create_random_batch(self, batch_size: int) -> torch.Tensor:\n device = (\n \"cuda\"\n if torch.cuda.is_available()\n else \"mps\"\n if torch.backends.mps.is_available()\n else \"cpu\"\n )\n return torch.randint(\n 0,\n 1000,\n (batch_size, self.config.seq_length),\n device=device,\n dtype=torch.long,\n )\n\n def _run_inference(\n self, input_ids: torch.Tensor, graph_wrapper: GraphWrapper | None = None\n ) -> tuple[float, torch.Tensor]:\n attention_mask = torch.ones_like(input_ids)\n\n with torch.no_grad(), self.timer.timing():\n if graph_wrapper is not None:\n output = graph_wrapper(input_ids, attention_mask)\n else:\n output = self.model(input_ids=input_ids, attention_mask=attention_mask)\n\n return self.timer.elapsed_time(), output\n\n def run(self) -> dict[int, dict[str, float]]:\n results = {}\n\n # Reset peak memory stats\n if torch.cuda.is_available():\n torch.cuda.reset_peak_memory_stats()\n elif torch.backends.mps.is_available():\n # MPS doesn't have reset_peak_memory_stats, skip it\n pass\n else:\n print(\"- No GPU memory stats available\")\n\n for batch_size in self.config.batch_sizes:\n print(f\"\\nTesting batch size: {batch_size}\")\n times = []\n\n # Get or create graph for this batch size\n graph_wrapper = (\n self.graphs.get_or_create(batch_size) if self.graphs is not None else None\n )\n\n # Pre-allocate input tensor\n input_ids = self._create_random_batch(batch_size)\n print(f\"Input shape: {input_ids.shape}\")\n\n # Run benchmark\n for i in tqdm(range(self.config.num_runs), desc=f\"Batch size {batch_size}\"):\n try:\n elapsed_time, output = self._run_inference(input_ids, graph_wrapper)\n if i == 0: # Only print on first run\n print(f\"Output shape: {output.last_hidden_state.shape}\")\n times.append(elapsed_time)\n except Exception as e:\n print(f\"Error during inference: {e}\")\n break\n\n if not times:\n print(f\"No successful runs for batch size {batch_size}, skipping\")\n continue\n\n # Calculate statistics\n avg_time = np.mean(times)\n std_time = np.std(times)\n throughput = batch_size / avg_time\n\n results[batch_size] = {\n \"avg_time\": avg_time,\n \"std_time\": std_time,\n \"throughput\": throughput,\n }\n\n print(f\"Avg Time: {avg_time:.4f}s ± {std_time:.4f}s\")\n print(f\"Throughput: {throughput:.2f} sequences/second\")\n\n # Log memory usage\n if torch.cuda.is_available():\n peak_memory_gb = torch.cuda.max_memory_allocated() / (1024**3)\n elif torch.backends.mps.is_available():\n # MPS doesn't have max_memory_allocated, use 0\n peak_memory_gb = 0.0\n else:\n peak_memory_gb = 0.0\n print(\"- No GPU memory usage available\")\n\n if peak_memory_gb > 0:\n print(f\"\\nPeak GPU memory usage: {peak_memory_gb:.2f} GB\")\n else:\n print(\"\\n- GPU memory usage not available\")\n\n # Add memory info to results\n for batch_size in results:\n results[batch_size][\"peak_memory_gb\"] = peak_memory_gb\n\n return results\n\n\ndef main():\n parser = argparse.ArgumentParser(description=\"Model Inference Benchmark\")\n parser.add_argument(\n \"--model_path\",\n type=str,\n default=\"facebook/contriever\",\n help=\"Path to the model\",\n )\n parser.add_argument(\n \"--batch_sizes\",\n type=str,\n default=\"1,2,4,8,16,32\",\n help=\"Comma-separated list of batch sizes\",\n )\n parser.add_argument(\n \"--seq_length\",\n type=int,\n default=256,\n help=\"Sequence length for input\",\n )\n parser.add_argument(\n \"--num_runs\",\n type=int,\n default=5,\n help=\"Number of runs for each batch size\",\n )\n parser.add_argument(\n \"--use_fp16\",\n action=\"store_true\",\n help=\"Enable FP16 inference\",\n )\n parser.add_argument(\n \"--use_int4\",\n action=\"store_true\",\n help=\"Enable INT4 quantization using bitsandbytes\",\n )\n parser.add_argument(\n \"--use_int8\",\n action=\"store_true\",\n help=\"Enable INT8 quantization for both activations and weights using bitsandbytes\",\n )\n parser.add_argument(\n \"--use_cuda_graphs\",\n action=\"store_true\",\n help=\"Enable CUDA Graphs optimization (only on NVIDIA GPUs)\",\n )\n parser.add_argument(\n \"--use_flash_attention\",\n action=\"store_true\",\n help=\"Enable Flash Attention 2 if available (only on NVIDIA GPUs)\",\n )\n parser.add_argument(\n \"--use_linear8bitlt\",\n action=\"store_true\",\n help=\"Enable Linear8bitLt quantization for all linear layers\",\n )\n\n args = parser.parse_args()\n\n # Print arguments for debugging\n print(\"\\nCommand line arguments:\")\n for arg, value in vars(args).items():\n print(f\"- {arg}: {value}\")\n\n config = BenchmarkConfig(\n model_path=args.model_path,\n batch_sizes=[int(bs) for bs in args.batch_sizes.split(\",\")],\n seq_length=args.seq_length,\n num_runs=args.num_runs,\n use_fp16=args.use_fp16,\n use_int4=args.use_int4,\n use_int8=args.use_int8, # Add this line\n use_cuda_graphs=args.use_cuda_graphs,\n use_flash_attention=args.use_flash_attention,\n use_linear8bitlt=args.use_linear8bitlt,\n )\n\n # Print configuration for debugging\n print(\"\\nBenchmark configuration:\")\n for field, value in vars(config).items():\n print(f\"- {field}: {value}\")\n\n try:\n benchmark = Benchmark(config)\n results = benchmark.run()\n\n # Save results to file\n import json\n import os\n\n # Create results directory if it doesn't exist\n os.makedirs(\"results\", exist_ok=True)\n\n # Generate filename based on configuration\n precision_type = (\n \"int4\"\n if config.use_int4\n else \"int8\"\n if config.use_int8\n else \"fp16\"\n if config.use_fp16\n else \"fp32\"\n )\n model_name = os.path.basename(config.model_path)\n output_file = f\"results/benchmark_{model_name}_{precision_type}.json\"\n\n # Save results\n with open(output_file, \"w\") as f:\n json.dump(\n {\n \"config\": {\n k: str(v) if isinstance(v, list) else v for k, v in vars(config).items()\n },\n \"results\": {str(k): v for k, v in results.items()},\n },\n f,\n indent=2,\n )\n print(f\"Results saved to {output_file}\")\n\n except Exception as e:\n print(f\"Benchmark failed: {e}\")\n import traceback\n\n traceback.print_exc()\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "benchmarks/run_evaluation.py", "content": "#!/usr/bin/env python3\n\"\"\"\nThis script runs a recall evaluation on a given LEANN index.\nIt correctly compares results by fetching the text content for both the new search\nresults and the golden standard results, making the comparison robust to ID changes.\n\"\"\"\n\nimport argparse\nimport json\nimport sys\nimport time\nfrom pathlib import Path\n\nimport numpy as np\nfrom leann.api import LeannBuilder, LeannChat, LeannSearcher\n\n\ndef download_data_if_needed(data_root: Path, download_embeddings: bool = False):\n \"\"\"Checks if the data directory exists, and if not, downloads it from HF Hub.\"\"\"\n if not data_root.exists():\n print(f\"Data directory '{data_root}' not found.\")\n print(\"Downloading evaluation data from Hugging Face Hub... (this may take a moment)\")\n try:\n from huggingface_hub import snapshot_download\n\n if download_embeddings:\n # Download everything including embeddings (large files)\n snapshot_download(\n repo_id=\"LEANN-RAG/leann-rag-evaluation-data\",\n repo_type=\"dataset\",\n local_dir=data_root,\n local_dir_use_symlinks=False,\n )\n print(\"Data download complete (including embeddings)!\")\n else:\n # Download only specific folders, excluding embeddings\n allow_patterns = [\n \"ground_truth/**\",\n \"indices/**\",\n \"queries/**\",\n \"*.md\",\n \"*.txt\",\n ]\n snapshot_download(\n repo_id=\"LEANN-RAG/leann-rag-evaluation-data\",\n repo_type=\"dataset\",\n local_dir=data_root,\n local_dir_use_symlinks=False,\n allow_patterns=allow_patterns,\n )\n print(\"Data download complete (excluding embeddings)!\")\n except ImportError:\n print(\n \"Error: huggingface_hub is not installed. Please install it to download the data:\"\n )\n print(\"uv sync --only-group dev\")\n sys.exit(1)\n except Exception as e:\n print(f\"An error occurred during data download: {e}\")\n sys.exit(1)\n\n\ndef download_embeddings_if_needed(data_root: Path, dataset_type: str | None = None):\n \"\"\"Download embeddings files specifically.\"\"\"\n embeddings_dir = data_root / \"embeddings\"\n\n if dataset_type:\n # Check if specific dataset embeddings exist\n target_file = embeddings_dir / dataset_type / \"passages_00.pkl\"\n if target_file.exists():\n print(f\"Embeddings for {dataset_type} already exist\")\n return str(target_file)\n\n print(\"Downloading embeddings from HuggingFace Hub...\")\n try:\n from huggingface_hub import snapshot_download\n\n # Download only embeddings folder\n snapshot_download(\n repo_id=\"LEANN-RAG/leann-rag-evaluation-data\",\n repo_type=\"dataset\",\n local_dir=data_root,\n local_dir_use_symlinks=False,\n allow_patterns=[\"embeddings/**/*.pkl\"],\n )\n print(\"Embeddings download complete!\")\n\n if dataset_type:\n target_file = embeddings_dir / dataset_type / \"passages_00.pkl\"\n if target_file.exists():\n return str(target_file)\n\n return str(embeddings_dir)\n\n except Exception as e:\n print(f\"Error downloading embeddings: {e}\")\n sys.exit(1)\n\n\n# --- Helper Function to get Golden Passages ---\ndef get_golden_texts(searcher: LeannSearcher, golden_ids: list[int]) -> set:\n \"\"\"\n Retrieves the text for golden passage IDs directly from the LeannSearcher's\n passage manager.\n \"\"\"\n golden_texts = set()\n for gid in golden_ids:\n try:\n # PassageManager uses string IDs\n passage_data = searcher.passage_manager.get_passage(str(gid))\n golden_texts.add(passage_data[\"text\"])\n except KeyError:\n print(f\"Warning: Golden passage ID '{gid}' not found in the index's passage data.\")\n return golden_texts\n\n\ndef load_queries(file_path: Path) -> list[str]:\n queries = []\n with open(file_path, encoding=\"utf-8\") as f:\n for line in f:\n data = json.loads(line)\n queries.append(data[\"query\"])\n return queries\n\n\ndef build_index_from_embeddings(embeddings_file: str, output_path: str, backend: str = \"hnsw\"):\n \"\"\"\n Build a LEANN index from pre-computed embeddings.\n\n Args:\n embeddings_file: Path to pickle file with (ids, embeddings) tuple\n output_path: Path where to save the index\n backend: Backend to use (\"hnsw\" or \"diskann\")\n \"\"\"\n print(f\"Building {backend} index from embeddings: {embeddings_file}\")\n\n # Create builder with appropriate parameters\n if backend == \"hnsw\":\n builder_kwargs = {\n \"M\": 32, # Graph degree\n \"efConstruction\": 256, # Construction complexity\n \"is_compact\": True, # Use compact storage\n \"is_recompute\": True, # Enable pruning for better recall\n }\n elif backend == \"diskann\":\n builder_kwargs = {\n \"complexity\": 64,\n \"graph_degree\": 32,\n \"search_memory_maximum\": 8.0, # GB\n \"build_memory_maximum\": 16.0, # GB\n }\n else:\n builder_kwargs = {}\n\n builder = LeannBuilder(\n backend_name=backend,\n embedding_model=\"facebook/contriever-msmarco\", # Model used to create embeddings\n dimensions=768, # Will be auto-detected from embeddings\n **builder_kwargs,\n )\n\n # Build index from precomputed embeddings\n builder.build_index_from_embeddings(output_path, embeddings_file)\n print(f\"Index saved to: {output_path}\")\n return output_path\n\n\ndef main():\n parser = argparse.ArgumentParser(description=\"Run recall evaluation on a LEANN index.\")\n parser.add_argument(\n \"index_path\",\n type=str,\n nargs=\"?\",\n help=\"Path to the LEANN index to evaluate or build (optional).\",\n )\n parser.add_argument(\n \"--mode\",\n choices=[\"evaluate\", \"build\"],\n default=\"evaluate\",\n help=\"Mode: 'evaluate' existing index or 'build' from embeddings\",\n )\n parser.add_argument(\n \"--embeddings-file\",\n type=str,\n help=\"Path to embeddings pickle file (optional for build mode)\",\n )\n parser.add_argument(\n \"--backend\",\n choices=[\"hnsw\", \"diskann\"],\n default=\"hnsw\",\n help=\"Backend to use for building index (default: hnsw)\",\n )\n parser.add_argument(\n \"--num-queries\", type=int, default=10, help=\"Number of queries to evaluate.\"\n )\n parser.add_argument(\"--top-k\", type=int, default=3, help=\"The 'k' value for recall@k.\")\n parser.add_argument(\n \"--ef-search\", type=int, default=120, help=\"The 'efSearch' parameter for HNSW.\"\n )\n parser.add_argument(\n \"--batch-size\",\n type=int,\n default=0,\n help=\"Batch size for HNSW batched search (0 disables batching)\",\n )\n parser.add_argument(\n \"--llm-type\",\n type=str,\n choices=[\"ollama\", \"hf\", \"openai\", \"gemini\", \"simulated\"],\n default=\"ollama\",\n help=\"LLM backend type to optionally query during evaluation (default: ollama)\",\n )\n parser.add_argument(\n \"--llm-model\",\n type=str,\n default=\"qwen3:1.7b\",\n help=\"LLM model identifier for the chosen backend (default: qwen3:1.7b)\",\n )\n args = parser.parse_args()\n\n # --- Path Configuration ---\n # Assumes a project structure where the script is in 'benchmarks/'\n # and evaluation data is in 'benchmarks/data/'.\n script_dir = Path(__file__).resolve().parent\n data_root = script_dir / \"data\"\n\n # Download data based on mode\n if args.mode == \"build\":\n # For building mode, we need embeddings\n download_data_if_needed(data_root, download_embeddings=False) # Basic data first\n\n # Auto-detect dataset type and download embeddings\n if args.embeddings_file:\n embeddings_file = args.embeddings_file\n # Try to detect dataset type from embeddings file path\n if \"rpj_wiki\" in str(embeddings_file):\n dataset_type = \"rpj_wiki\"\n elif \"dpr\" in str(embeddings_file):\n dataset_type = \"dpr\"\n else:\n dataset_type = \"dpr\" # Default\n else:\n # Auto-detect from index path if provided, otherwise default to DPR\n if args.index_path:\n index_path_str = str(args.index_path)\n if \"rpj_wiki\" in index_path_str:\n dataset_type = \"rpj_wiki\"\n elif \"dpr\" in index_path_str:\n dataset_type = \"dpr\"\n else:\n dataset_type = \"dpr\" # Default to DPR\n else:\n dataset_type = \"dpr\" # Default to DPR\n\n embeddings_file = download_embeddings_if_needed(data_root, dataset_type)\n\n # Auto-generate index path if not provided\n if not args.index_path:\n indices_dir = data_root / \"indices\" / dataset_type\n indices_dir.mkdir(parents=True, exist_ok=True)\n args.index_path = str(indices_dir / f\"{dataset_type}_from_embeddings\")\n print(f\"Auto-generated index path: {args.index_path}\")\n\n print(f\"Building index from embeddings: {embeddings_file}\")\n built_index_path = build_index_from_embeddings(\n embeddings_file, args.index_path, args.backend\n )\n print(f\"Index built successfully: {built_index_path}\")\n\n # Ask if user wants to run evaluation\n eval_response = input(\"Run evaluation on the built index? (y/n): \").strip().lower()\n if eval_response != \"y\":\n print(\"Index building complete. Exiting.\")\n return\n else:\n # For evaluation mode, don't need embeddings\n download_data_if_needed(data_root, download_embeddings=False)\n\n # Auto-detect index path if not provided\n if not args.index_path:\n # Default to using downloaded indices\n indices_dir = data_root / \"indices\"\n\n # Try common datasets in order of preference\n for dataset in [\"dpr\", \"rpj_wiki\"]:\n dataset_dir = indices_dir / dataset\n if dataset_dir.exists():\n # Look for index files\n index_files = list(dataset_dir.glob(\"*.index\")) + list(\n dataset_dir.glob(\"*_disk.index\")\n )\n if index_files:\n args.index_path = str(\n index_files[0].with_suffix(\"\")\n ) # Remove .index extension\n print(f\"Using index: {args.index_path}\")\n break\n\n if not args.index_path:\n print(\"No indices found. The data download should have included pre-built indices.\")\n print(\n \"Please check the benchmarks/data/indices/ directory or provide --index-path manually.\"\n )\n sys.exit(1)\n\n # Detect dataset type from index path to select the correct ground truth\n index_path_str = str(args.index_path)\n if \"rpj_wiki\" in index_path_str:\n dataset_type = \"rpj_wiki\"\n elif \"dpr\" in index_path_str:\n dataset_type = \"dpr\"\n else:\n # Fallback: try to infer from the index directory name\n dataset_type = Path(args.index_path).name\n print(f\"WARNING: Could not detect dataset type from path, inferred '{dataset_type}'.\")\n\n queries_file = data_root / \"queries\" / \"nq_open.jsonl\"\n golden_results_file = data_root / \"ground_truth\" / dataset_type / \"flat_results_nq_k3.json\"\n\n print(f\"INFO: Detected dataset type: {dataset_type}\")\n print(f\"INFO: Using queries file: {queries_file}\")\n print(f\"INFO: Using ground truth file: {golden_results_file}\")\n\n try:\n searcher = LeannSearcher(args.index_path)\n queries = load_queries(queries_file)\n\n with open(golden_results_file) as f:\n golden_results_data = json.load(f)\n\n num_eval_queries = min(args.num_queries, len(queries))\n queries = queries[:num_eval_queries]\n\n print(f\"\\nRunning evaluation on {num_eval_queries} queries...\")\n recall_scores = []\n search_times = []\n\n for i in range(num_eval_queries):\n start_time = time.time()\n new_results = searcher.search(\n queries[i],\n top_k=args.top_k,\n complexity=args.ef_search,\n batch_size=args.batch_size,\n )\n search_times.append(time.time() - start_time)\n\n # Optional: also call the LLM with configurable backend/model (does not affect recall)\n llm_config = {\"type\": args.llm_type, \"model\": args.llm_model}\n chat = LeannChat(args.index_path, llm_config=llm_config, searcher=searcher)\n answer = chat.ask(\n queries[i],\n top_k=args.top_k,\n complexity=args.ef_search,\n batch_size=args.batch_size,\n )\n print(f\"Answer: {answer}\")\n # Correct Recall Calculation: Based on TEXT content\n new_texts = {result.text for result in new_results}\n\n # Get golden texts directly from the searcher's passage manager\n golden_ids = golden_results_data[\"indices\"][i][: args.top_k]\n golden_texts = get_golden_texts(searcher, golden_ids)\n\n overlap = len(new_texts & golden_texts)\n recall = overlap / len(golden_texts) if golden_texts else 0\n recall_scores.append(recall)\n\n print(\"\\n--- EVALUATION RESULTS ---\")\n print(f\"Query: {queries[i]}\")\n print(f\"New Results: {new_texts}\")\n print(f\"Golden Results: {golden_texts}\")\n print(f\"Overlap: {overlap}\")\n print(f\"Recall: {recall}\")\n print(f\"Search Time: {search_times[-1]:.4f}s\")\n print(\"--------------------------------\")\n\n avg_recall = np.mean(recall_scores) if recall_scores else 0\n avg_time = np.mean(search_times) if search_times else 0\n\n print(\"\\n🎉 --- Evaluation Complete ---\")\n print(f\"Avg. Recall@{args.top_k} (efSearch={args.ef_search}): {avg_recall:.4f}\")\n print(f\"Avg. Search Time: {avg_time:.4f}s\")\n\n except Exception as e:\n print(f\"\\n❌ An error occurred during evaluation: {e}\")\n import traceback\n\n traceback.print_exc()\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "benchmarks/simple_mac_tpt_test.py", "content": "import time\nfrom dataclasses import dataclass\n\nimport numpy as np\nimport torch\nfrom torch import nn\nfrom tqdm import tqdm\nfrom transformers import AutoModel\n\n# Add MLX imports\ntry:\n import mlx.core as mx\n from mlx_lm.utils import load\n\n MLX_AVAILABLE = True\nexcept ImportError:\n print(\"MLX not available. Install with: uv pip install mlx mlx-lm\")\n MLX_AVAILABLE = False\n\n\n@dataclass\nclass BenchmarkConfig:\n model_path: str = \"facebook/contriever-msmarco\"\n batch_sizes: list[int] = None\n seq_length: int = 256\n num_runs: int = 5\n use_fp16: bool = True\n use_int4: bool = False\n use_int8: bool = False\n use_cuda_graphs: bool = False\n use_flash_attention: bool = False\n use_linear8bitlt: bool = False\n use_mlx: bool = False # New flag for MLX testing\n\n def __post_init__(self):\n if self.batch_sizes is None:\n self.batch_sizes = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024]\n\n\nclass MLXBenchmark:\n \"\"\"MLX-specific benchmark for embedding models\"\"\"\n\n def __init__(self, config: BenchmarkConfig):\n self.config = config\n self.model, self.tokenizer = self._load_model()\n\n def _load_model(self):\n \"\"\"Load MLX model and tokenizer following the API pattern\"\"\"\n print(f\"Loading MLX model from {self.config.model_path}...\")\n try:\n model, tokenizer = load(self.config.model_path)\n print(\"MLX model loaded successfully\")\n return model, tokenizer\n except Exception as e:\n print(f\"Error loading MLX model: {e}\")\n raise\n\n def _create_random_batch(self, batch_size: int):\n \"\"\"Create random input batches for MLX testing - same as PyTorch\"\"\"\n return torch.randint(0, 1000, (batch_size, self.config.seq_length), dtype=torch.long)\n\n def _run_inference(self, input_ids: torch.Tensor) -> float:\n \"\"\"Run MLX inference with same input as PyTorch\"\"\"\n start_time = time.time()\n try:\n # Convert PyTorch tensor to MLX array\n input_ids_mlx = mx.array(input_ids.numpy())\n\n # Get embeddings\n embeddings = self.model(input_ids_mlx)\n\n # Mean pooling (following the API pattern)\n pooled = embeddings.mean(axis=1)\n\n # Convert to numpy (following the API pattern)\n pooled_numpy = np.array(pooled.tolist(), dtype=np.float32)\n\n # Force computation\n _ = pooled_numpy.shape\n\n except Exception as e:\n print(f\"MLX inference error: {e}\")\n return float(\"inf\")\n end_time = time.time()\n\n return end_time - start_time\n\n def run(self) -> dict[int, dict[str, float]]:\n \"\"\"Run the MLX benchmark across all batch sizes\"\"\"\n results = {}\n\n print(f\"Starting MLX benchmark with model: {self.config.model_path}\")\n print(f\"Testing batch sizes: {self.config.batch_sizes}\")\n\n for batch_size in self.config.batch_sizes:\n print(f\"\\n=== Testing MLX batch size: {batch_size} ===\")\n times = []\n\n # Create input batch (same as PyTorch)\n input_ids = self._create_random_batch(batch_size)\n\n # Warm up\n print(\"Warming up...\")\n for _ in range(3):\n try:\n self._run_inference(input_ids[:2]) # Warm up with smaller batch\n except Exception as e:\n print(f\"Warmup error: {e}\")\n break\n\n # Run benchmark\n for _i in tqdm(range(self.config.num_runs), desc=f\"MLX Batch size {batch_size}\"):\n try:\n elapsed_time = self._run_inference(input_ids)\n if elapsed_time != float(\"inf\"):\n times.append(elapsed_time)\n except Exception as e:\n print(f\"Error during MLX inference: {e}\")\n break\n\n if not times:\n print(f\"Skipping batch size {batch_size} due to errors\")\n continue\n\n # Calculate statistics\n avg_time = np.mean(times)\n std_time = np.std(times)\n throughput = batch_size / avg_time\n\n results[batch_size] = {\n \"avg_time\": avg_time,\n \"std_time\": std_time,\n \"throughput\": throughput,\n \"min_time\": np.min(times),\n \"max_time\": np.max(times),\n }\n\n print(f\"MLX Results for batch size {batch_size}:\")\n print(f\" Avg Time: {avg_time:.4f}s ± {std_time:.4f}s\")\n print(f\" Min Time: {np.min(times):.4f}s\")\n print(f\" Max Time: {np.max(times):.4f}s\")\n print(f\" Throughput: {throughput:.2f} sequences/second\")\n\n return results\n\n\nclass Benchmark:\n def __init__(self, config: BenchmarkConfig):\n self.config = config\n self.device = (\n \"cuda\"\n if torch.cuda.is_available()\n else \"mps\"\n if torch.backends.mps.is_available()\n else \"cpu\"\n )\n self.model = self._load_model()\n\n def _load_model(self) -> nn.Module:\n print(f\"Loading model from {self.config.model_path}...\")\n\n model = AutoModel.from_pretrained(self.config.model_path)\n if self.config.use_fp16:\n model = model.half()\n model = torch.compile(model)\n model = model.to(self.device)\n\n model.eval()\n return model\n\n def _create_random_batch(self, batch_size: int) -> torch.Tensor:\n return torch.randint(\n 0,\n 1000,\n (batch_size, self.config.seq_length),\n device=self.device,\n dtype=torch.long,\n )\n\n def _run_inference(self, input_ids: torch.Tensor) -> float:\n attention_mask = torch.ones_like(input_ids)\n # print shape of input_ids and attention_mask\n print(f\"input_ids shape: {input_ids.shape}\")\n print(f\"attention_mask shape: {attention_mask.shape}\")\n start_time = time.time()\n with torch.no_grad():\n self.model(input_ids=input_ids, attention_mask=attention_mask)\n if torch.cuda.is_available():\n torch.cuda.synchronize()\n if torch.backends.mps.is_available():\n torch.mps.synchronize()\n end_time = time.time()\n\n return end_time - start_time\n\n def run(self) -> dict[int, dict[str, float]]:\n results = {}\n\n if torch.cuda.is_available():\n torch.cuda.reset_peak_memory_stats()\n\n for batch_size in self.config.batch_sizes:\n print(f\"\\nTesting batch size: {batch_size}\")\n times = []\n\n input_ids = self._create_random_batch(batch_size)\n\n for _i in tqdm(range(self.config.num_runs), desc=f\"Batch size {batch_size}\"):\n try:\n elapsed_time = self._run_inference(input_ids)\n times.append(elapsed_time)\n except Exception as e:\n print(f\"Error during inference: {e}\")\n break\n\n if not times:\n continue\n\n avg_time = np.mean(times)\n std_time = np.std(times)\n throughput = batch_size / avg_time\n\n results[batch_size] = {\n \"avg_time\": avg_time,\n \"std_time\": std_time,\n \"throughput\": throughput,\n }\n\n print(f\"Avg Time: {avg_time:.4f}s ± {std_time:.4f}s\")\n print(f\"Throughput: {throughput:.2f} sequences/second\")\n\n if torch.cuda.is_available():\n peak_memory_gb = torch.cuda.max_memory_allocated() / (1024**3)\n else:\n peak_memory_gb = 0.0\n\n for batch_size in results:\n results[batch_size][\"peak_memory_gb\"] = peak_memory_gb\n\n return results\n\n\ndef run_benchmark():\n \"\"\"Main function to run the benchmark with optimized parameters.\"\"\"\n config = BenchmarkConfig()\n\n try:\n benchmark = Benchmark(config)\n results = benchmark.run()\n\n max_throughput = max(results[batch_size][\"throughput\"] for batch_size in results)\n avg_throughput = np.mean([results[batch_size][\"throughput\"] for batch_size in results])\n\n return {\n \"max_throughput\": max_throughput,\n \"avg_throughput\": avg_throughput,\n \"results\": results,\n }\n\n except Exception as e:\n print(f\"Benchmark failed: {e}\")\n return {\"max_throughput\": 0.0, \"avg_throughput\": 0.0, \"error\": str(e)}\n\n\ndef run_mlx_benchmark():\n \"\"\"Run MLX-specific benchmark\"\"\"\n if not MLX_AVAILABLE:\n print(\"MLX not available, skipping MLX benchmark\")\n return {\n \"max_throughput\": 0.0,\n \"avg_throughput\": 0.0,\n \"error\": \"MLX not available\",\n }\n\n config = BenchmarkConfig(model_path=\"mlx-community/all-MiniLM-L6-v2-4bit\", use_mlx=True)\n\n try:\n benchmark = MLXBenchmark(config)\n results = benchmark.run()\n\n if not results:\n return {\n \"max_throughput\": 0.0,\n \"avg_throughput\": 0.0,\n \"error\": \"No valid results\",\n }\n\n max_throughput = max(results[batch_size][\"throughput\"] for batch_size in results)\n avg_throughput = np.mean([results[batch_size][\"throughput\"] for batch_size in results])\n\n return {\n \"max_throughput\": max_throughput,\n \"avg_throughput\": avg_throughput,\n \"results\": results,\n }\n\n except Exception as e:\n print(f\"MLX benchmark failed: {e}\")\n return {\"max_throughput\": 0.0, \"avg_throughput\": 0.0, \"error\": str(e)}\n\n\nif __name__ == \"__main__\":\n print(\"=== PyTorch Benchmark ===\")\n pytorch_result = run_benchmark()\n print(f\"PyTorch Max throughput: {pytorch_result['max_throughput']:.2f} sequences/second\")\n print(f\"PyTorch Average throughput: {pytorch_result['avg_throughput']:.2f} sequences/second\")\n\n print(\"\\n=== MLX Benchmark ===\")\n mlx_result = run_mlx_benchmark()\n print(f\"MLX Max throughput: {mlx_result['max_throughput']:.2f} sequences/second\")\n print(f\"MLX Average throughput: {mlx_result['avg_throughput']:.2f} sequences/second\")\n\n # Compare results\n if pytorch_result[\"max_throughput\"] > 0 and mlx_result[\"max_throughput\"] > 0:\n speedup = mlx_result[\"max_throughput\"] / pytorch_result[\"max_throughput\"]\n print(\"\\n=== Comparison ===\")\n print(f\"MLX is {speedup:.2f}x {'faster' if speedup > 1 else 'slower'} than PyTorch\")\n" }, { "path": "benchmarks/update/README.md", "content": "# Update Benchmarks\n\nThis directory hosts two benchmark suites that exercise LEANN’s HNSW “update +\nsearch” pipeline under different assumptions:\n\n1. **RNG recompute latency** – measure how random-neighbour pruning and cache\n settings influence incremental `add()` latency when embeddings are fetched\n over the ZMQ embedding server.\n2. **Update strategy comparison** – compare a fully sequential update pipeline\n against an offline approach that keeps the graph static and fuses results.\n\nBoth suites build a non-compact, `is_recompute=True` index so that new\nembeddings are pulled from the embedding server. Benchmark outputs are written\nunder `.leann/bench/` by default and appended to CSV files for later plotting.\n\n## Benchmarks\n\n### 1. HNSW RNG Recompute Benchmark\n\n`bench_hnsw_rng_recompute.py` evaluates incremental update latency under four\nrandom-neighbour (RNG) configurations. Each scenario uses the same dataset but\nchanges the forward / reverse RNG pruning flags and whether the embedding cache\nis enabled:\n\n| Scenario name | Forward RNG | Reverse RNG | ZMQ embedding cache |\n| ---------------------------------- | ----------- | ----------- | ------------------- |\n| `baseline` | Enabled | Enabled | Enabled |\n| `no_cache_baseline` | Enabled | Enabled | **Disabled** |\n| `disable_forward_rng` | **Disabled**| Enabled | Enabled |\n| `disable_forward_and_reverse_rng` | **Disabled**| **Disabled**| Enabled |\n\nFor each scenario the script:\n1. (Re)builds a `is_recompute=True` index and writes it to `.leann/bench/`.\n2. Starts `leann_backend_hnsw.hnsw_embedding_server` for remote embeddings.\n3. Appends the requested updates using the scenario’s RNG flags.\n4. Records total time, latency per passage, ZMQ fetch counts, and stage-level\n timings before appending a row to the CSV output.\n\n**Run:**\n```bash\nLEANN_HNSW_LOG_PATH=.leann/bench/hnsw_server.log \\\nLEANN_LOG_LEVEL=INFO \\\nuv run -m benchmarks.update.bench_hnsw_rng_recompute \\\n --runs 1 \\\n --index-path .leann/bench/test.leann \\\n --initial-files data/PrideandPrejudice.txt \\\n --update-files data/huawei_pangu.md \\\n --max-initial 300 \\\n --max-updates 1 \\\n --add-timeout 120\n```\n\n**Output:**\n- `benchmarks/update/bench_results.csv` – per-scenario timing statistics\n (including ms/passage) for each run.\n- `.leann/bench/hnsw_server.log` – detailed ZMQ/server logs (path controlled by\n `LEANN_HNSW_LOG_PATH`).\n _The reference CSVs checked into this branch were generated on a workstation with an NVIDIA RTX 4090 GPU; throughput numbers will differ on other hardware._\n\n### 2. Sequential vs. Offline Update Benchmark\n\n`bench_update_vs_offline_search.py` compares two end-to-end strategies on the\nsame dataset:\n\n- **Scenario A – Sequential Update**\n - Start an embedding server.\n - Sequentially call `index.add()`; each call fetches embeddings via ZMQ and\n mutates the HNSW graph.\n - After all inserts, run a search on the updated graph.\n - Metrics recorded: update time (`add_total_s`), post-update search time\n (`search_time_s`), combined total (`total_time_s`), and per-passage\n latency.\n\n- **Scenario B – Offline Embedding + Concurrent Search**\n - Stop Scenario A’s server and start a fresh embedding server.\n - Spawn two threads: one generates embeddings for the new passages offline\n (graph unchanged); the other computes the query embedding and searches the\n existing graph.\n - Merge offline similarities with the graph search results to emulate late\n fusion, then report the merged top‑k preview.\n - Metrics recorded: embedding time (`emb_time_s`), search time\n (`search_time_s`), concurrent makespan (`makespan_s`), and scenario total.\n\n**Run (both scenarios):**\n```bash\nuv run -m benchmarks.update.bench_update_vs_offline_search \\\n --index-path .leann/bench/offline_vs_update.leann \\\n --max-initial 300 \\\n --num-updates 1\n```\n\nYou can pass `--only A` or `--only B` to run a single scenario. The script will\nprint timing summaries to stdout and append the results to CSV.\n\n**Output:**\n- `benchmarks/update/offline_vs_update.csv` – per-scenario timing statistics for\n Scenario A and B.\n- Console output includes Scenario B’s merged top‑k preview for quick sanity\n checks.\n _The sample results committed here come from runs on an RTX 4090-equipped machine; expect variations if you benchmark on different GPUs._\n\n### 3. Visualisation\n\n`plot_bench_results.py` combines the RNG benchmark and the update strategy\nbenchmark into a single two-panel plot.\n\n**Run:**\n```bash\nuv run -m benchmarks.update.plot_bench_results \\\n --csv benchmarks/update/bench_results.csv \\\n --csv-right benchmarks/update/offline_vs_update.csv \\\n --out benchmarks/update/bench_latency_from_csv.png\n```\n\n**Options:**\n- `--broken-y` – Enable a broken Y-axis (default: true when appropriate).\n- `--csv` – RNG benchmark results CSV (left panel).\n- `--csv-right` – Update strategy results CSV (right panel).\n- `--out` – Output image path (PNG/PDF supported).\n\n**Output:**\n- `benchmarks/update/bench_latency_from_csv.png` – visual comparison of the two\n suites.\n- `benchmarks/update/bench_latency_from_csv.pdf` – PDF version, suitable for\n slides/papers.\n\n## Parameters & Environment\n\n### Common CLI Flags\n- `--max-initial` – Number of initial passages used to seed the index.\n- `--max-updates` / `--num-updates` – Number of passages to treat as updates.\n- `--index-path` – Base path (without extension) where the LEANN index is stored.\n- `--runs` – Number of repetitions (RNG benchmark only).\n\n### Environment Variables\n- `LEANN_HNSW_LOG_PATH` – File to receive embedding-server logs (optional).\n- `LEANN_LOG_LEVEL` – Logging verbosity (DEBUG/INFO/WARNING/ERROR).\n- `CUDA_VISIBLE_DEVICES` – Set to empty string if you want to force CPU\n execution of the embedding model.\n\nWith these scripts you can easily replicate LEANN’s update benchmarks, compare\nmultiple RNG strategies, and evaluate whether sequential updates or offline\nfusion better match your latency/accuracy trade-offs.\n" }, { "path": "benchmarks/update/__init__.py", "content": "\"\"\"Benchmarks for LEANN update workflows.\"\"\"\n\n# Expose helper to locate repository root for other modules that need it.\nfrom pathlib import Path\n\n\ndef find_repo_root() -> Path:\n \"\"\"Return the project root containing pyproject.toml.\"\"\"\n current = Path(__file__).resolve()\n for parent in current.parents:\n if (parent / \"pyproject.toml\").exists():\n return parent\n return current.parents[1]\n\n\n__all__ = [\"find_repo_root\"]\n" }, { "path": "benchmarks/update/bench_hnsw_rng_recompute.py", "content": "\"\"\"Benchmark incremental HNSW add() under different RNG pruning modes with real\nembedding recomputation.\n\nThis script clones the structure of ``examples/dynamic_update_no_recompute.py``\nso that we build a non-compact ``is_recompute=True`` index, spin up the\nstandard HNSW embedding server, and measure how long incremental ``add`` takes\nwhen RNG pruning is fully enabled vs. partially/fully disabled.\n\nExample usage (run from the repo root; downloads the model on first run)::\n\n uv run -m benchmarks.update.bench_hnsw_rng_recompute \\\n --index-path .leann/bench/leann-demo.leann \\\n --runs 1\n\nYou can tweak the input documents with ``--initial-files`` / ``--update-files``\nif you want a larger or different workload, and change the embedding model via\n``--model-name``.\n\"\"\"\n\nimport argparse\nimport json\nimport logging\nimport os\nimport pickle\nimport re\nimport sys\nimport time\nfrom pathlib import Path\nfrom typing import Any\n\nimport msgpack\nimport numpy as np\nimport zmq\nfrom leann.api import LeannBuilder\n\nif os.environ.get(\"LEANN_FORCE_CPU\", \"\").lower() in (\"1\", \"true\", \"yes\"):\n os.environ.setdefault(\"CUDA_VISIBLE_DEVICES\", \"\")\n\nfrom leann.embedding_compute import compute_embeddings\nfrom leann.embedding_server_manager import EmbeddingServerManager\nfrom leann.registry import register_project_directory\nfrom leann_backend_hnsw import faiss # type: ignore\nfrom leann_backend_hnsw.convert_to_csr import prune_hnsw_embeddings_inplace\n\nlogger = logging.getLogger(__name__)\nif not logging.getLogger().handlers:\n logging.basicConfig(level=logging.INFO)\n\n\ndef _find_repo_root() -> Path:\n \"\"\"Locate project root by walking up until pyproject.toml is found.\"\"\"\n current = Path(__file__).resolve()\n for parent in current.parents:\n if (parent / \"pyproject.toml\").exists():\n return parent\n # Fallback: assume repo is two levels up (../..)\n return current.parents[2]\n\n\nREPO_ROOT = _find_repo_root()\nif str(REPO_ROOT) not in sys.path:\n sys.path.insert(0, str(REPO_ROOT))\n\nfrom apps.chunking import create_text_chunks # noqa: E402\n\nDEFAULT_INITIAL_FILES = [\n REPO_ROOT / \"data\" / \"2501.14312v1 (1).pdf\",\n REPO_ROOT / \"data\" / \"huawei_pangu.md\",\n]\nDEFAULT_UPDATE_FILES = [REPO_ROOT / \"data\" / \"2506.08276v1.pdf\"]\n\nDEFAULT_HNSW_LOG = Path(\".leann/bench/hnsw_server.log\")\n\n\ndef load_chunks_from_files(paths: list[Path], limit: int | None = None) -> list[str]:\n from llama_index.core import SimpleDirectoryReader\n\n documents = []\n for path in paths:\n p = path.expanduser().resolve()\n if not p.exists():\n raise FileNotFoundError(f\"Input path not found: {p}\")\n if p.is_dir():\n reader = SimpleDirectoryReader(str(p), recursive=False)\n documents.extend(reader.load_data(show_progress=True))\n else:\n reader = SimpleDirectoryReader(input_files=[str(p)])\n documents.extend(reader.load_data(show_progress=True))\n\n if not documents:\n return []\n\n chunks = create_text_chunks(\n documents,\n chunk_size=512,\n chunk_overlap=128,\n use_ast_chunking=False,\n )\n cleaned = [c for c in chunks if isinstance(c, str) and c.strip()]\n if limit is not None:\n cleaned = cleaned[:limit]\n return cleaned\n\n\ndef ensure_index_dir(index_path: Path) -> None:\n index_path.parent.mkdir(parents=True, exist_ok=True)\n\n\ndef cleanup_index_files(index_path: Path) -> None:\n parent = index_path.parent\n if not parent.exists():\n return\n stem = index_path.stem\n for file in parent.glob(f\"{stem}*\"):\n if file.is_file():\n file.unlink()\n\n\ndef build_initial_index(\n index_path: Path,\n paragraphs: list[str],\n model_name: str,\n embedding_mode: str,\n distance_metric: str,\n ef_construction: int,\n) -> None:\n builder = LeannBuilder(\n backend_name=\"hnsw\",\n embedding_model=model_name,\n embedding_mode=embedding_mode,\n is_compact=False,\n is_recompute=True,\n distance_metric=distance_metric,\n backend_kwargs={\n \"distance_metric\": distance_metric,\n \"is_compact\": False,\n \"is_recompute\": True,\n \"efConstruction\": ef_construction,\n },\n )\n for idx, passage in enumerate(paragraphs):\n builder.add_text(passage, metadata={\"id\": str(idx)})\n builder.build_index(str(index_path))\n\n\ndef prepare_new_chunks(paragraphs: list[str]) -> list[dict[str, Any]]:\n return [{\"text\": text, \"metadata\": {}} for text in paragraphs]\n\n\ndef benchmark_update_with_mode(\n index_path: Path,\n new_chunks: list[dict[str, Any]],\n model_name: str,\n embedding_mode: str,\n distance_metric: str,\n disable_forward_rng: bool,\n disable_reverse_rng: bool,\n server_port: int,\n add_timeout: int,\n ef_construction: int,\n) -> tuple[float, float]:\n meta_path = index_path.parent / f\"{index_path.name}.meta.json\"\n passages_file = index_path.parent / f\"{index_path.name}.passages.jsonl\"\n offset_file = index_path.parent / f\"{index_path.name}.passages.idx\"\n index_file = index_path.parent / f\"{index_path.stem}.index\"\n\n with open(meta_path, encoding=\"utf-8\") as f:\n meta = json.load(f)\n\n with open(offset_file, \"rb\") as f:\n offset_map: dict[str, int] = pickle.load(f)\n existing_ids = set(offset_map.keys())\n\n valid_chunks: list[dict[str, Any]] = []\n for chunk in new_chunks:\n text = chunk.get(\"text\", \"\")\n if not isinstance(text, str) or not text.strip():\n continue\n metadata = chunk.setdefault(\"metadata\", {})\n passage_id = chunk.get(\"id\") or metadata.get(\"id\")\n if passage_id and passage_id in existing_ids:\n raise ValueError(f\"Passage ID '{passage_id}' already exists in the index.\")\n valid_chunks.append(chunk)\n\n if not valid_chunks:\n raise ValueError(\"No valid chunks to append.\")\n\n texts_to_embed = [chunk[\"text\"] for chunk in valid_chunks]\n embeddings = compute_embeddings(\n texts_to_embed,\n model_name,\n mode=embedding_mode,\n is_build=False,\n batch_size=16,\n )\n\n embeddings = np.ascontiguousarray(embeddings, dtype=np.float32)\n if distance_metric == \"cosine\":\n norms = np.linalg.norm(embeddings, axis=1, keepdims=True)\n norms[norms == 0] = 1\n embeddings = embeddings / norms\n\n index = faiss.read_index(str(index_file))\n index.is_recompute = True\n if getattr(index, \"storage\", None) is None:\n if index.metric_type == faiss.METRIC_INNER_PRODUCT:\n storage_index = faiss.IndexFlatIP(index.d)\n else:\n storage_index = faiss.IndexFlatL2(index.d)\n index.storage = storage_index\n index.own_fields = True\n try:\n storage_index.ntotal = index.ntotal\n except AttributeError:\n pass\n try:\n index.hnsw.set_disable_rng_during_add(disable_forward_rng)\n index.hnsw.set_disable_reverse_prune(disable_reverse_rng)\n if ef_construction is not None:\n index.hnsw.efConstruction = ef_construction\n except AttributeError:\n pass\n\n applied_forward = getattr(index.hnsw, \"disable_rng_during_add\", None)\n applied_reverse = getattr(index.hnsw, \"disable_reverse_prune\", None)\n logger.info(\n \"HNSW RNG config -> requested forward=%s, reverse=%s | applied forward=%s, reverse=%s\",\n disable_forward_rng,\n disable_reverse_rng,\n applied_forward,\n applied_reverse,\n )\n\n base_id = index.ntotal\n for offset, chunk in enumerate(valid_chunks):\n new_id = str(base_id + offset)\n chunk.setdefault(\"metadata\", {})[\"id\"] = new_id\n chunk[\"id\"] = new_id\n\n rollback_size = passages_file.stat().st_size if passages_file.exists() else 0\n offset_map_backup = offset_map.copy()\n\n try:\n with open(passages_file, \"a\", encoding=\"utf-8\") as f:\n for chunk in valid_chunks:\n offset = f.tell()\n json.dump(\n {\n \"id\": chunk[\"id\"],\n \"text\": chunk[\"text\"],\n \"metadata\": chunk.get(\"metadata\", {}),\n },\n f,\n ensure_ascii=False,\n )\n f.write(\"\\n\")\n offset_map[chunk[\"id\"]] = offset\n\n with open(offset_file, \"wb\") as f:\n pickle.dump(offset_map, f)\n\n server_manager = EmbeddingServerManager(\n backend_module_name=\"leann_backend_hnsw.hnsw_embedding_server\"\n )\n server_started, actual_port = server_manager.start_server(\n port=server_port,\n model_name=model_name,\n embedding_mode=embedding_mode,\n passages_file=str(meta_path),\n distance_metric=distance_metric,\n )\n if not server_started:\n raise RuntimeError(\"Failed to start embedding server.\")\n\n if hasattr(index.hnsw, \"set_zmq_port\"):\n index.hnsw.set_zmq_port(actual_port)\n elif hasattr(index, \"set_zmq_port\"):\n index.set_zmq_port(actual_port)\n\n _warmup_embedding_server(actual_port)\n\n total_start = time.time()\n add_elapsed = 0.0\n\n try:\n import signal\n\n def _timeout_handler(signum, frame):\n raise TimeoutError(\"incremental add timed out\")\n\n if add_timeout > 0:\n signal.signal(signal.SIGALRM, _timeout_handler)\n signal.alarm(add_timeout)\n\n add_start = time.time()\n for i in range(embeddings.shape[0]):\n index.add(1, faiss.swig_ptr(embeddings[i : i + 1]))\n add_elapsed = time.time() - add_start\n if add_timeout > 0:\n signal.alarm(0)\n faiss.write_index(index, str(index_file))\n finally:\n server_manager.stop_server()\n\n except TimeoutError:\n raise\n except Exception:\n if passages_file.exists():\n with open(passages_file, \"rb+\") as f:\n f.truncate(rollback_size)\n with open(offset_file, \"wb\") as f:\n pickle.dump(offset_map_backup, f)\n raise\n\n prune_hnsw_embeddings_inplace(str(index_file))\n\n meta[\"total_passages\"] = len(offset_map)\n with open(meta_path, \"w\", encoding=\"utf-8\") as f:\n json.dump(meta, f, indent=2)\n\n # Reset toggles so the index on disk returns to baseline behaviour.\n try:\n index.hnsw.set_disable_rng_during_add(False)\n index.hnsw.set_disable_reverse_prune(False)\n except AttributeError:\n pass\n faiss.write_index(index, str(index_file))\n\n total_elapsed = time.time() - total_start\n\n return total_elapsed, add_elapsed\n\n\ndef _total_zmq_nodes(log_path: Path) -> int:\n if not log_path.exists():\n return 0\n with log_path.open(\"r\", encoding=\"utf-8\") as log_file:\n text = log_file.read()\n return sum(int(match) for match in re.findall(r\"ZMQ received (\\d+) node IDs\", text))\n\n\ndef _warmup_embedding_server(port: int) -> None:\n \"\"\"Send a dummy REQ so the embedding server loads its model.\"\"\"\n ctx = zmq.Context()\n try:\n sock = ctx.socket(zmq.REQ)\n sock.setsockopt(zmq.LINGER, 0)\n sock.setsockopt(zmq.RCVTIMEO, 5000)\n sock.setsockopt(zmq.SNDTIMEO, 5000)\n sock.connect(f\"tcp://127.0.0.1:{port}\")\n payload = msgpack.packb([\"__WARMUP__\"], use_bin_type=True)\n sock.send(payload)\n try:\n sock.recv()\n except zmq.error.Again:\n pass\n finally:\n sock.close()\n ctx.term()\n\n\ndef main() -> None:\n parser = argparse.ArgumentParser(description=__doc__)\n parser.add_argument(\n \"--index-path\",\n type=Path,\n default=Path(\".leann/bench/leann-demo.leann\"),\n help=\"Output index base path (without extension).\",\n )\n parser.add_argument(\n \"--initial-files\",\n nargs=\"*\",\n type=Path,\n default=DEFAULT_INITIAL_FILES,\n help=\"Files used to build the initial index.\",\n )\n parser.add_argument(\n \"--update-files\",\n nargs=\"*\",\n type=Path,\n default=DEFAULT_UPDATE_FILES,\n help=\"Files appended during the benchmark.\",\n )\n parser.add_argument(\n \"--runs\", type=int, default=1, help=\"How many times to repeat each scenario.\"\n )\n parser.add_argument(\n \"--model-name\",\n default=\"sentence-transformers/all-MiniLM-L6-v2\",\n help=\"Embedding model used for build/update.\",\n )\n parser.add_argument(\n \"--embedding-mode\",\n default=\"sentence-transformers\",\n help=\"Embedding mode passed to LeannBuilder/embedding server.\",\n )\n parser.add_argument(\n \"--distance-metric\",\n default=\"mips\",\n choices=[\"mips\", \"l2\", \"cosine\"],\n help=\"Distance metric for HNSW backend.\",\n )\n parser.add_argument(\n \"--ef-construction\",\n type=int,\n default=200,\n help=\"efConstruction setting for initial build.\",\n )\n parser.add_argument(\n \"--server-port\",\n type=int,\n default=5557,\n help=\"Port for the real embedding server.\",\n )\n parser.add_argument(\n \"--max-initial\",\n type=int,\n default=300,\n help=\"Optional cap on initial passages (after chunking).\",\n )\n parser.add_argument(\n \"--max-updates\",\n type=int,\n default=1,\n help=\"Optional cap on update passages (after chunking).\",\n )\n parser.add_argument(\n \"--add-timeout\",\n type=int,\n default=900,\n help=\"Timeout in seconds for the incremental add loop (0 = no timeout).\",\n )\n parser.add_argument(\n \"--plot-path\",\n type=Path,\n default=Path(\"bench_latency.png\"),\n help=\"Where to save the latency bar plot.\",\n )\n parser.add_argument(\n \"--cap-y\",\n type=float,\n default=None,\n help=\"Cap Y-axis (ms). Bars above are hatched and annotated.\",\n )\n parser.add_argument(\n \"--broken-y\",\n action=\"store_true\",\n help=\"Use broken Y-axis (two stacked axes with gap). Overrides --cap-y unless both provided.\",\n )\n parser.add_argument(\n \"--lower-cap-y\",\n type=float,\n default=None,\n help=\"Lower axes upper bound for broken Y (ms). Default=1.1x second-highest.\",\n )\n parser.add_argument(\n \"--upper-start-y\",\n type=float,\n default=None,\n help=\"Upper axes lower bound for broken Y (ms). Default=1.2x second-highest.\",\n )\n parser.add_argument(\n \"--csv-path\",\n type=Path,\n default=Path(\"benchmarks/update/bench_results.csv\"),\n help=\"Where to append per-scenario results as CSV.\",\n )\n\n args = parser.parse_args()\n\n register_project_directory(REPO_ROOT)\n\n initial_paragraphs = load_chunks_from_files(args.initial_files, args.max_initial)\n update_paragraphs = load_chunks_from_files(args.update_files, args.max_updates)\n if not update_paragraphs:\n raise ValueError(\"No update passages found; please provide --update-files with content.\")\n\n update_chunks = prepare_new_chunks(update_paragraphs)\n ensure_index_dir(args.index_path)\n\n scenarios = [\n (\"baseline\", False, False, True),\n (\"no_cache_baseline\", False, False, False),\n (\"disable_forward_rng\", True, False, True),\n (\"disable_forward_and_reverse_rng\", True, True, True),\n ]\n\n log_path = Path(os.environ.get(\"LEANN_HNSW_LOG_PATH\", DEFAULT_HNSW_LOG))\n log_path.parent.mkdir(parents=True, exist_ok=True)\n os.environ[\"LEANN_HNSW_LOG_PATH\"] = str(log_path.resolve())\n os.environ.setdefault(\"LEANN_LOG_LEVEL\", \"INFO\")\n\n results_total: dict[str, list[float]] = {name: [] for name, *_ in scenarios}\n results_add: dict[str, list[float]] = {name: [] for name, *_ in scenarios}\n results_zmq: dict[str, list[int]] = {name: [] for name, *_ in scenarios}\n results_stageA: dict[str, list[float]] = {name: [] for name, *_ in scenarios}\n results_stageBC: dict[str, list[float]] = {name: [] for name, *_ in scenarios}\n results_ms_per_passage: dict[str, list[float]] = {name: [] for name, *_ in scenarios}\n\n # CSV setup\n import csv\n\n run_id = time.strftime(\"%Y%m%d-%H%M%S\")\n csv_fields = [\n \"run_id\",\n \"scenario\",\n \"cache_enabled\",\n \"ef_construction\",\n \"max_initial\",\n \"max_updates\",\n \"total_time_s\",\n \"add_only_s\",\n \"latency_ms_per_passage\",\n \"zmq_nodes\",\n \"stageA_time_s\",\n \"stageBC_time_s\",\n \"model_name\",\n \"embedding_mode\",\n \"distance_metric\",\n ]\n # Create CSV with header if missing\n if args.csv_path:\n args.csv_path.parent.mkdir(parents=True, exist_ok=True)\n if not args.csv_path.exists() or args.csv_path.stat().st_size == 0:\n with args.csv_path.open(\"w\", newline=\"\", encoding=\"utf-8\") as f:\n writer = csv.DictWriter(f, fieldnames=csv_fields)\n writer.writeheader()\n\n for run in range(args.runs):\n print(f\"\\n=== Benchmark run {run + 1}/{args.runs} ===\")\n for name, disable_forward, disable_reverse, cache_enabled in scenarios:\n print(f\"\\nScenario: {name}\")\n cleanup_index_files(args.index_path)\n if log_path.exists():\n try:\n log_path.unlink()\n except OSError:\n pass\n os.environ[\"LEANN_ZMQ_EMBED_CACHE\"] = \"1\" if cache_enabled else \"0\"\n build_initial_index(\n args.index_path,\n initial_paragraphs,\n args.model_name,\n args.embedding_mode,\n args.distance_metric,\n args.ef_construction,\n )\n\n prev_size = log_path.stat().st_size if log_path.exists() else 0\n\n try:\n total_elapsed, add_elapsed = benchmark_update_with_mode(\n args.index_path,\n update_chunks,\n args.model_name,\n args.embedding_mode,\n args.distance_metric,\n disable_forward,\n disable_reverse,\n args.server_port,\n args.add_timeout,\n args.ef_construction,\n )\n except TimeoutError as exc:\n print(f\"Scenario {name} timed out: {exc}\")\n continue\n\n curr_size = log_path.stat().st_size if log_path.exists() else 0\n if curr_size < prev_size:\n prev_size = 0\n zmq_count = 0\n if log_path.exists():\n with log_path.open(\"r\", encoding=\"utf-8\") as log_file:\n log_file.seek(prev_size)\n new_entries = log_file.read()\n zmq_count = sum(\n int(match) for match in re.findall(r\"ZMQ received (\\d+) node IDs\", new_entries)\n )\n stageA = sum(\n float(x)\n for x in re.findall(r\"Distance calculation E2E time: ([0-9.]+)s\", new_entries)\n )\n stageBC = sum(\n float(x) for x in re.findall(r\"ZMQ E2E time: ([0-9.]+)s\", new_entries)\n )\n else:\n stageA = 0.0\n stageBC = 0.0\n\n per_chunk = add_elapsed / len(update_chunks)\n print(\n f\"Total time: {total_elapsed:.3f} s | add-only: {add_elapsed:.3f} s \"\n f\"for {len(update_chunks)} passages => {per_chunk * 1e3:.3f} ms/passage\"\n )\n print(f\"ZMQ node fetch total: {zmq_count}\")\n results_total[name].append(total_elapsed)\n results_add[name].append(add_elapsed)\n results_zmq[name].append(zmq_count)\n results_ms_per_passage[name].append(per_chunk * 1e3)\n results_stageA[name].append(stageA)\n results_stageBC[name].append(stageBC)\n\n # Append row to CSV\n if args.csv_path:\n row = {\n \"run_id\": run_id,\n \"scenario\": name,\n \"cache_enabled\": 1 if cache_enabled else 0,\n \"ef_construction\": args.ef_construction,\n \"max_initial\": args.max_initial,\n \"max_updates\": args.max_updates,\n \"total_time_s\": round(total_elapsed, 6),\n \"add_only_s\": round(add_elapsed, 6),\n \"latency_ms_per_passage\": round(per_chunk * 1e3, 6),\n \"zmq_nodes\": int(zmq_count),\n \"stageA_time_s\": round(stageA, 6),\n \"stageBC_time_s\": round(stageBC, 6),\n \"model_name\": args.model_name,\n \"embedding_mode\": args.embedding_mode,\n \"distance_metric\": args.distance_metric,\n }\n with args.csv_path.open(\"a\", newline=\"\", encoding=\"utf-8\") as f:\n writer = csv.DictWriter(f, fieldnames=csv_fields)\n writer.writerow(row)\n\n print(\"\\n=== Summary ===\")\n for name in results_add:\n add_values = results_add[name]\n total_values = results_total[name]\n zmq_values = results_zmq[name]\n latency_values = results_ms_per_passage[name]\n if not add_values:\n print(f\"{name}: no successful runs\")\n continue\n avg_add = sum(add_values) / len(add_values)\n avg_total = sum(total_values) / len(total_values)\n avg_zmq = sum(zmq_values) / len(zmq_values) if zmq_values else 0.0\n avg_latency = sum(latency_values) / len(latency_values) if latency_values else 0.0\n runs = len(add_values)\n print(\n f\"{name}: add-only avg {avg_add:.3f} s | total avg {avg_total:.3f} s \"\n f\"| ZMQ avg {avg_zmq:.1f} node fetches | latency {avg_latency:.2f} ms/passage over {runs} run(s)\"\n )\n\n if args.plot_path:\n try:\n import matplotlib.pyplot as plt\n\n labels = [name for name, *_ in scenarios]\n values = [\n sum(results_ms_per_passage[name]) / len(results_ms_per_passage[name])\n if results_ms_per_passage[name]\n else 0.0\n for name in labels\n ]\n\n def _auto_cap(vals: list[float]) -> float | None:\n s = sorted(vals, reverse=True)\n if len(s) < 2:\n return None\n if s[1] > 0 and s[0] >= 2.5 * s[1]:\n return s[1] * 1.1\n return None\n\n def _fmt_ms(v: float) -> str:\n return f\"{v / 1000:.1f}k\" if v >= 1000 else f\"{v:.1f}\"\n\n colors = [\"#4e79a7\", \"#f28e2c\", \"#e15759\", \"#76b7b2\"]\n\n if args.broken_y:\n s = sorted(values, reverse=True)\n second = s[1] if len(s) >= 2 else (s[0] if s else 0.0)\n lower_cap = args.lower_cap_y if args.lower_cap_y is not None else second * 1.1\n upper_start = (\n args.upper_start_y\n if args.upper_start_y is not None\n else max(second * 1.2, lower_cap * 1.02)\n )\n ymax = max(values) * 1.10 if values else 1.0\n fig, (ax_top, ax_bottom) = plt.subplots(\n 2,\n 1,\n sharex=True,\n figsize=(7.4, 5.0),\n gridspec_kw={\"height_ratios\": [1, 3], \"hspace\": 0.05},\n )\n x = list(range(len(labels)))\n ax_bottom.bar(x, values, color=colors[: len(labels)], width=0.8)\n ax_top.bar(x, values, color=colors[: len(labels)], width=0.8)\n ax_bottom.set_ylim(0, lower_cap)\n ax_top.set_ylim(upper_start, ymax)\n for i, v in enumerate(values):\n if v <= lower_cap:\n ax_bottom.text(\n i,\n v + lower_cap * 0.02,\n _fmt_ms(v),\n ha=\"center\",\n va=\"bottom\",\n fontsize=9,\n )\n else:\n ax_top.text(i, v, _fmt_ms(v), ha=\"center\", va=\"bottom\", fontsize=9)\n ax_top.spines[\"bottom\"].set_visible(False)\n ax_bottom.spines[\"top\"].set_visible(False)\n ax_top.tick_params(labeltop=False)\n ax_bottom.xaxis.tick_bottom()\n d = 0.015\n kwargs = {\"transform\": ax_top.transAxes, \"color\": \"k\", \"clip_on\": False}\n ax_top.plot((-d, +d), (-d, +d), **kwargs)\n ax_top.plot((1 - d, 1 + d), (-d, +d), **kwargs)\n kwargs.update({\"transform\": ax_bottom.transAxes})\n ax_bottom.plot((-d, +d), (1 - d, 1 + d), **kwargs)\n ax_bottom.plot((1 - d, 1 + d), (1 - d, 1 + d), **kwargs)\n ax_bottom.set_xticks(range(len(labels)))\n ax_bottom.set_xticklabels(labels)\n ax = ax_bottom\n else:\n cap = args.cap_y or _auto_cap(values)\n plt.figure(figsize=(7.2, 4.2))\n ax = plt.gca()\n if cap is not None:\n show_vals = [min(v, cap) for v in values]\n bars = []\n for i, (v, show) in enumerate(zip(values, show_vals)):\n b = ax.bar(i, show, color=colors[i], width=0.8)\n bars.append(b[0])\n if v > cap:\n bars[-1].set_hatch(\"//\")\n ax.text(i, cap * 1.02, _fmt_ms(v), ha=\"center\", va=\"bottom\", fontsize=9)\n else:\n ax.text(\n i,\n show + max(1.0, 0.01 * (cap or show)),\n _fmt_ms(v),\n ha=\"center\",\n va=\"bottom\",\n fontsize=9,\n )\n ax.set_ylim(0, cap * 1.10)\n ax.plot(\n [0.02 - 0.02, 0.02 + 0.02],\n [0.98 + 0.02, 0.98 - 0.02],\n transform=ax.transAxes,\n color=\"k\",\n lw=1,\n )\n ax.plot(\n [0.98 - 0.02, 0.98 + 0.02],\n [0.98 + 0.02, 0.98 - 0.02],\n transform=ax.transAxes,\n color=\"k\",\n lw=1,\n )\n if any(v > cap for v in values):\n ax.legend(\n [bars[0]], [\"capped\"], fontsize=8, frameon=False, loc=\"upper right\"\n )\n ax.set_xticks(range(len(labels)))\n ax.set_xticklabels(labels)\n else:\n ax.bar(labels, values, color=colors[: len(labels)])\n for idx, val in enumerate(values):\n ax.text(idx, val + 1.0, f\"{val:.1f}\", ha=\"center\", va=\"bottom\")\n\n plt.ylabel(\"Average add latency (ms per passage)\")\n plt.title(f\"Initial passages {args.max_initial}, updates {args.max_updates}\")\n plt.tight_layout()\n plt.savefig(args.plot_path)\n print(f\"Saved latency bar plot to {args.plot_path}\")\n # ZMQ time split (Stage A vs B/C)\n try:\n plt.figure(figsize=(6, 4))\n a_vals = [sum(results_stageA[n]) / max(1, len(results_stageA[n])) for n in labels]\n bc_vals = [\n sum(results_stageBC[n]) / max(1, len(results_stageBC[n])) for n in labels\n ]\n ind = range(len(labels))\n plt.bar(ind, a_vals, color=\"#4e79a7\", label=\"Stage A distance (s)\")\n plt.bar(\n ind, bc_vals, bottom=a_vals, color=\"#e15759\", label=\"Stage B/C embed-by-id (s)\"\n )\n plt.xticks(list(ind), labels, rotation=10)\n plt.ylabel(\"Server ZMQ time (s)\")\n plt.title(\n f\"ZMQ time split (initial {args.max_initial}, updates {args.max_updates})\"\n )\n plt.legend()\n out2 = args.plot_path.with_name(\n args.plot_path.stem + \"_zmq_split\" + args.plot_path.suffix\n )\n plt.tight_layout()\n plt.savefig(out2)\n print(f\"Saved ZMQ time split plot to {out2}\")\n except Exception as e:\n print(\"Failed to plot ZMQ split:\", e)\n except ImportError:\n print(\"matplotlib not available; skipping plot generation\")\n\n # leave the last build on disk for inspection\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "benchmarks/update/bench_update_vs_offline_search.py", "content": "\"\"\"\nCompare two latency models for small incremental updates vs. search:\n\nScenario A (sequential update then search):\n - Build initial HNSW (is_recompute=True)\n - Start embedding server (ZMQ) for recompute\n - Add N passages one-by-one (each triggers recompute over ZMQ)\n - Then run a search query on the updated index\n - Report total time = sum(add_i) + search_time, with breakdowns\n\nScenario B (offline embeds + concurrent search; no graph updates):\n - Do NOT insert the N passages into the graph\n - In parallel: (1) compute embeddings for the N passages; (2) compute query\n embedding and run a search on the existing index\n - After both finish, compute similarity between the query embedding and the N\n new passage embeddings, merge with the index search results by score, and\n report time = max(embed_time, search_time) (i.e., no blocking on updates)\n\nThis script reuses the model/data loading conventions of\nexamples/bench_hnsw_rng_recompute.py but focuses on end-to-end latency\ncomparison for the two execution strategies above.\n\nExample (from the repository root):\n uv run -m benchmarks.update.bench_update_vs_offline_search \\\n --index-path .leann/bench/offline_vs_update.leann \\\n --max-initial 300 --num-updates 5 --k 10\n\"\"\"\n\nimport argparse\nimport csv\nimport json\nimport logging\nimport os\nimport pickle\nimport sys\nimport threading\nimport time\nfrom dataclasses import dataclass\nfrom pathlib import Path\nfrom typing import Any\n\nimport numpy as np\nimport psutil # type: ignore\nfrom leann.api import LeannBuilder\n\nif os.environ.get(\"LEANN_FORCE_CPU\", \"\").lower() in (\"1\", \"true\", \"yes\"):\n os.environ.setdefault(\"CUDA_VISIBLE_DEVICES\", \"\")\n\nfrom leann.embedding_compute import compute_embeddings\nfrom leann.embedding_server_manager import EmbeddingServerManager\nfrom leann.registry import register_project_directory\nfrom leann_backend_hnsw import faiss # type: ignore\n\nlogger = logging.getLogger(__name__)\nif not logging.getLogger().handlers:\n logging.basicConfig(level=logging.INFO)\n\n\ndef _find_repo_root() -> Path:\n \"\"\"Locate project root by walking up until pyproject.toml is found.\"\"\"\n current = Path(__file__).resolve()\n for parent in current.parents:\n if (parent / \"pyproject.toml\").exists():\n return parent\n # Fallback: assume repo is two levels up (../..)\n return current.parents[2]\n\n\nREPO_ROOT = _find_repo_root()\nif str(REPO_ROOT) not in sys.path:\n sys.path.insert(0, str(REPO_ROOT))\n\nfrom apps.chunking import create_text_chunks # noqa: E402\n\nDEFAULT_INITIAL_FILES = [\n REPO_ROOT / \"data\" / \"2501.14312v1 (1).pdf\",\n REPO_ROOT / \"data\" / \"huawei_pangu.md\",\n]\nDEFAULT_UPDATE_FILES = [REPO_ROOT / \"data\" / \"2506.08276v1.pdf\"]\n\n\ndef load_chunks_from_files(paths: list[Path], limit: int | None = None) -> list[str]:\n from llama_index.core import SimpleDirectoryReader\n\n documents = []\n for path in paths:\n p = path.expanduser().resolve()\n if not p.exists():\n raise FileNotFoundError(f\"Input path not found: {p}\")\n if p.is_dir():\n reader = SimpleDirectoryReader(str(p), recursive=False)\n documents.extend(reader.load_data(show_progress=True))\n else:\n reader = SimpleDirectoryReader(input_files=[str(p)])\n documents.extend(reader.load_data(show_progress=True))\n\n if not documents:\n return []\n\n chunks = create_text_chunks(\n documents,\n chunk_size=512,\n chunk_overlap=128,\n use_ast_chunking=False,\n )\n cleaned = [c for c in chunks if isinstance(c, str) and c.strip()]\n if limit is not None:\n cleaned = cleaned[:limit]\n return cleaned\n\n\ndef ensure_index_dir(index_path: Path) -> None:\n index_path.parent.mkdir(parents=True, exist_ok=True)\n\n\ndef cleanup_index_files(index_path: Path) -> None:\n parent = index_path.parent\n if not parent.exists():\n return\n stem = index_path.stem\n for file in parent.glob(f\"{stem}*\"):\n if file.is_file():\n file.unlink()\n\n\ndef build_initial_index(\n index_path: Path,\n paragraphs: list[str],\n model_name: str,\n embedding_mode: str,\n distance_metric: str,\n ef_construction: int,\n) -> None:\n builder = LeannBuilder(\n backend_name=\"hnsw\",\n embedding_model=model_name,\n embedding_mode=embedding_mode,\n is_compact=False,\n is_recompute=True,\n distance_metric=distance_metric,\n backend_kwargs={\n \"distance_metric\": distance_metric,\n \"is_compact\": False,\n \"is_recompute\": True,\n \"efConstruction\": ef_construction,\n },\n )\n for idx, passage in enumerate(paragraphs):\n builder.add_text(passage, metadata={\"id\": str(idx)})\n builder.build_index(str(index_path))\n\n\ndef _maybe_norm_cosine(vecs: np.ndarray, metric: str) -> np.ndarray:\n if metric == \"cosine\":\n vecs = np.ascontiguousarray(vecs, dtype=np.float32)\n norms = np.linalg.norm(vecs, axis=1, keepdims=True)\n norms[norms == 0] = 1\n vecs = vecs / norms\n return vecs\n\n\ndef _read_index_for_search(index_path: Path) -> Any:\n index_file = index_path.parent / f\"{index_path.stem}.index\"\n # Force-disable experimental disk cache when loading the index so that\n # incremental benchmarks don't pick up stale top-degree bitmaps.\n cfg = faiss.HNSWIndexConfig()\n cfg.is_recompute = True\n if hasattr(cfg, \"disk_cache_ratio\"):\n cfg.disk_cache_ratio = 0.0\n if hasattr(cfg, \"external_storage_path\"):\n cfg.external_storage_path = None\n io_flags = getattr(faiss, \"IO_FLAG_MMAP\", 0)\n index = faiss.read_index(str(index_file), io_flags, cfg)\n # ensure recompute mode persists after reload\n try:\n index.is_recompute = True\n except AttributeError:\n pass\n try:\n actual_ntotal = index.hnsw.levels.size()\n except AttributeError:\n actual_ntotal = index.ntotal\n if actual_ntotal != index.ntotal:\n print(\n f\"[bench_update_vs_offline_search] Correcting ntotal from {index.ntotal} to {actual_ntotal}\",\n flush=True,\n )\n index.ntotal = actual_ntotal\n if getattr(index, \"storage\", None) is None:\n if index.metric_type == faiss.METRIC_INNER_PRODUCT:\n storage_index = faiss.IndexFlatIP(index.d)\n else:\n storage_index = faiss.IndexFlatL2(index.d)\n index.storage = storage_index\n index.own_fields = True\n return index\n\n\ndef _append_passages_for_updates(\n meta_path: Path,\n start_id: int,\n texts: list[str],\n) -> list[str]:\n \"\"\"Append update passages so the embedding server can serve recompute fetches.\"\"\"\n\n if not texts:\n return []\n\n index_dir = meta_path.parent\n meta_name = meta_path.name\n if not meta_name.endswith(\".meta.json\"):\n raise ValueError(f\"Unexpected meta filename: {meta_path}\")\n index_base = meta_name[: -len(\".meta.json\")]\n\n passages_file = index_dir / f\"{index_base}.passages.jsonl\"\n offsets_file = index_dir / f\"{index_base}.passages.idx\"\n\n if not passages_file.exists() or not offsets_file.exists():\n raise FileNotFoundError(\n \"Passage store missing; cannot register update passages for recompute mode.\"\n )\n\n with open(offsets_file, \"rb\") as f:\n offset_map: dict[str, int] = pickle.load(f)\n\n assigned_ids: list[str] = []\n with open(passages_file, \"a\", encoding=\"utf-8\") as f:\n for i, text in enumerate(texts):\n passage_id = str(start_id + i)\n offset = f.tell()\n json.dump({\"id\": passage_id, \"text\": text, \"metadata\": {}}, f, ensure_ascii=False)\n f.write(\"\\n\")\n offset_map[passage_id] = offset\n assigned_ids.append(passage_id)\n\n with open(offsets_file, \"wb\") as f:\n pickle.dump(offset_map, f)\n\n try:\n with open(meta_path, encoding=\"utf-8\") as f:\n meta = json.load(f)\n except json.JSONDecodeError:\n meta = {}\n meta[\"total_passages\"] = len(offset_map)\n with open(meta_path, \"w\", encoding=\"utf-8\") as f:\n json.dump(meta, f, indent=2)\n\n return assigned_ids\n\n\ndef _search(index: Any, q: np.ndarray, k: int) -> tuple[np.ndarray, np.ndarray]:\n q = np.ascontiguousarray(q, dtype=np.float32)\n distances = np.zeros((1, k), dtype=np.float32)\n indices = np.zeros((1, k), dtype=np.int64)\n index.search(\n 1,\n faiss.swig_ptr(q),\n k,\n faiss.swig_ptr(distances),\n faiss.swig_ptr(indices),\n )\n return distances[0], indices[0]\n\n\ndef _score_for_metric(dist: float, metric: str) -> float:\n # Convert FAISS distance to a \"higher is better\" score\n if metric in (\"mips\", \"cosine\"):\n return float(dist)\n # l2 distance (smaller better) -> negative distance as score\n return -float(dist)\n\n\ndef _merge_results(\n index_results: tuple[np.ndarray, np.ndarray],\n offline_scores: list[tuple[int, float]],\n k: int,\n metric: str,\n) -> list[tuple[str, float]]:\n distances, indices = index_results\n merged: list[tuple[str, float]] = []\n for distance, idx in zip(distances.tolist(), indices.tolist()):\n merged.append((f\"idx:{idx}\", _score_for_metric(distance, metric)))\n for j, s in offline_scores:\n merged.append((f\"offline:{j}\", s))\n merged.sort(key=lambda x: x[1], reverse=True)\n return merged[:k]\n\n\n@dataclass\nclass ScenarioResult:\n name: str\n update_total_s: float\n search_s: float\n overall_s: float\n\n\ndef main() -> None:\n parser = argparse.ArgumentParser(description=__doc__)\n parser.add_argument(\n \"--index-path\",\n type=Path,\n default=Path(\".leann/bench/offline-vs-update.leann\"),\n )\n parser.add_argument(\n \"--initial-files\",\n nargs=\"*\",\n type=Path,\n default=DEFAULT_INITIAL_FILES,\n )\n parser.add_argument(\n \"--update-files\",\n nargs=\"*\",\n type=Path,\n default=DEFAULT_UPDATE_FILES,\n )\n parser.add_argument(\"--max-initial\", type=int, default=300)\n parser.add_argument(\"--num-updates\", type=int, default=5)\n parser.add_argument(\"--k\", type=int, default=10, help=\"Top-k for search/merge\")\n parser.add_argument(\n \"--query\",\n type=str,\n default=\"neural network\",\n help=\"Query text used for the search benchmark.\",\n )\n parser.add_argument(\"--server-port\", type=int, default=5557)\n parser.add_argument(\"--add-timeout\", type=int, default=600)\n parser.add_argument(\"--model-name\", default=\"sentence-transformers/all-MiniLM-L6-v2\")\n parser.add_argument(\"--embedding-mode\", default=\"sentence-transformers\")\n parser.add_argument(\n \"--distance-metric\",\n default=\"mips\",\n choices=[\"mips\", \"l2\", \"cosine\"],\n )\n parser.add_argument(\"--ef-construction\", type=int, default=200)\n parser.add_argument(\n \"--only\",\n choices=[\"A\", \"B\", \"both\"],\n default=\"both\",\n help=\"Run only Scenario A, Scenario B, or both\",\n )\n parser.add_argument(\n \"--csv-path\",\n type=Path,\n default=Path(\"benchmarks/update/offline_vs_update.csv\"),\n help=\"Where to append results (CSV).\",\n )\n\n args = parser.parse_args()\n\n register_project_directory(REPO_ROOT)\n\n # Load data\n initial_paragraphs = load_chunks_from_files(args.initial_files, args.max_initial)\n update_paragraphs = load_chunks_from_files(args.update_files, None)\n if not update_paragraphs:\n raise ValueError(\"No update passages loaded from --update-files\")\n update_paragraphs = update_paragraphs[: args.num_updates]\n if len(update_paragraphs) < args.num_updates:\n raise ValueError(\n f\"Not enough update passages ({len(update_paragraphs)}) for --num-updates={args.num_updates}\"\n )\n\n ensure_index_dir(args.index_path)\n cleanup_index_files(args.index_path)\n\n # Build initial index\n build_initial_index(\n args.index_path,\n initial_paragraphs,\n args.model_name,\n args.embedding_mode,\n args.distance_metric,\n args.ef_construction,\n )\n\n # Prepare index object and meta\n meta_path = args.index_path.parent / f\"{args.index_path.name}.meta.json\"\n index = _read_index_for_search(args.index_path)\n\n # CSV setup\n run_id = time.strftime(\"%Y%m%d-%H%M%S\")\n if args.csv_path:\n args.csv_path.parent.mkdir(parents=True, exist_ok=True)\n csv_fields = [\n \"run_id\",\n \"scenario\",\n \"max_initial\",\n \"num_updates\",\n \"k\",\n \"total_time_s\",\n \"add_total_s\",\n \"search_time_s\",\n \"emb_time_s\",\n \"makespan_s\",\n \"model_name\",\n \"embedding_mode\",\n \"distance_metric\",\n ]\n if not args.csv_path.exists() or args.csv_path.stat().st_size == 0:\n with args.csv_path.open(\"w\", newline=\"\", encoding=\"utf-8\") as f:\n writer = csv.DictWriter(f, fieldnames=csv_fields)\n writer.writeheader()\n\n # Debug: list existing HNSW server PIDs before starting\n try:\n existing = [\n p\n for p in psutil.process_iter(attrs=[\"pid\", \"cmdline\"])\n if any(\n isinstance(arg, str) and \"leann_backend_hnsw.hnsw_embedding_server\" in arg\n for arg in (p.info.get(\"cmdline\") or [])\n )\n ]\n if existing:\n print(\"[debug] Found existing hnsw_embedding_server processes before run:\")\n for p in existing:\n print(f\"[debug] PID={p.info['pid']} cmd={' '.join(p.info.get('cmdline') or [])}\")\n except Exception as _e:\n pass\n\n add_total = 0.0\n search_after_add = 0.0\n total_seq = 0.0\n port_a = None\n if args.only in (\"A\", \"both\"):\n # Scenario A: sequential update then search\n start_id = index.ntotal\n assigned_ids = _append_passages_for_updates(meta_path, start_id, update_paragraphs)\n if assigned_ids:\n logger.debug(\n \"Registered %d update passages starting at id %s\",\n len(assigned_ids),\n assigned_ids[0],\n )\n server_manager = EmbeddingServerManager(\n backend_module_name=\"leann_backend_hnsw.hnsw_embedding_server\"\n )\n ok, port = server_manager.start_server(\n port=args.server_port,\n model_name=args.model_name,\n embedding_mode=args.embedding_mode,\n passages_file=str(meta_path),\n distance_metric=args.distance_metric,\n )\n if not ok:\n raise RuntimeError(\"Failed to start embedding server\")\n try:\n # Set ZMQ port for recompute mode\n if hasattr(index.hnsw, \"set_zmq_port\"):\n index.hnsw.set_zmq_port(port)\n elif hasattr(index, \"set_zmq_port\"):\n index.set_zmq_port(port)\n\n # Start A overall timer BEFORE computing update embeddings\n t0 = time.time()\n\n # Compute embeddings for updates (counted into A's overall)\n t_emb0 = time.time()\n upd_embs = compute_embeddings(\n update_paragraphs,\n args.model_name,\n mode=args.embedding_mode,\n is_build=False,\n batch_size=16,\n )\n emb_time_updates = time.time() - t_emb0\n upd_embs = np.asarray(upd_embs, dtype=np.float32)\n upd_embs = _maybe_norm_cosine(upd_embs, args.distance_metric)\n\n # Perform sequential adds\n for i in range(upd_embs.shape[0]):\n t_add0 = time.time()\n index.add(1, faiss.swig_ptr(upd_embs[i : i + 1]))\n add_total += time.time() - t_add0\n # Don't persist index after adds to avoid contaminating Scenario B\n # index_file = args.index_path.parent / f\"{args.index_path.stem}.index\"\n # faiss.write_index(index, str(index_file))\n\n # Search after updates\n q_emb = compute_embeddings(\n [args.query], args.model_name, mode=args.embedding_mode, is_build=False\n )\n q_emb = np.asarray(q_emb, dtype=np.float32)\n q_emb = _maybe_norm_cosine(q_emb, args.distance_metric)\n\n # Warm up search with a dummy query first\n print(\"[DEBUG] Warming up search...\")\n _ = _search(index, q_emb, 1)\n\n t_s0 = time.time()\n D_upd, I_upd = _search(index, q_emb, args.k)\n search_after_add = time.time() - t_s0\n total_seq = time.time() - t0\n finally:\n server_manager.stop_server()\n port_a = port\n\n print(\"\\n=== Scenario A: update->search (sequential) ===\")\n # emb_time_updates is defined only when A runs\n try:\n _emb_a = emb_time_updates\n except NameError:\n _emb_a = 0.0\n print(\n f\"Adds: {args.num_updates} passages; embeds={_emb_a:.3f}s; add_total={add_total:.3f}s; \"\n f\"search={search_after_add:.3f}s; overall={total_seq:.3f}s\"\n )\n # CSV row for A\n if args.csv_path:\n row_a = {\n \"run_id\": run_id,\n \"scenario\": \"A\",\n \"max_initial\": args.max_initial,\n \"num_updates\": args.num_updates,\n \"k\": args.k,\n \"total_time_s\": round(total_seq, 6),\n \"add_total_s\": round(add_total, 6),\n \"search_time_s\": round(search_after_add, 6),\n \"emb_time_s\": round(_emb_a, 6),\n \"makespan_s\": 0.0,\n \"model_name\": args.model_name,\n \"embedding_mode\": args.embedding_mode,\n \"distance_metric\": args.distance_metric,\n }\n with args.csv_path.open(\"a\", newline=\"\", encoding=\"utf-8\") as f:\n writer = csv.DictWriter(f, fieldnames=csv_fields)\n writer.writerow(row_a)\n\n # Verify server cleanup\n try:\n # short sleep to allow signal handling to finish\n time.sleep(0.5)\n leftovers = [\n p\n for p in psutil.process_iter(attrs=[\"pid\", \"cmdline\"])\n if any(\n isinstance(arg, str) and \"leann_backend_hnsw.hnsw_embedding_server\" in arg\n for arg in (p.info.get(\"cmdline\") or [])\n )\n ]\n if leftovers:\n print(\"[warn] hnsw_embedding_server process(es) still alive after A-stop:\")\n for p in leftovers:\n print(\n f\"[warn] PID={p.info['pid']} cmd={' '.join(p.info.get('cmdline') or [])}\"\n )\n else:\n print(\"[debug] server cleanup confirmed: no hnsw_embedding_server found\")\n except Exception:\n pass\n\n # Scenario B: offline embeds + concurrent search (no graph updates)\n if args.only in (\"B\", \"both\"):\n # ensure a server is available for recompute search\n server_manager_b = EmbeddingServerManager(\n backend_module_name=\"leann_backend_hnsw.hnsw_embedding_server\"\n )\n requested_port = args.server_port if port_a is None else port_a\n ok_b, port_b = server_manager_b.start_server(\n port=requested_port,\n model_name=args.model_name,\n embedding_mode=args.embedding_mode,\n passages_file=str(meta_path),\n distance_metric=args.distance_metric,\n )\n if not ok_b:\n raise RuntimeError(\"Failed to start embedding server for Scenario B\")\n\n # Wait for server to fully initialize\n print(\"[DEBUG] Waiting 2s for embedding server to fully initialize...\")\n time.sleep(2)\n\n try:\n # Read the index first\n index_no_update = _read_index_for_search(args.index_path) # unchanged index\n\n # Then configure ZMQ port on the correct index object\n if hasattr(index_no_update.hnsw, \"set_zmq_port\"):\n index_no_update.hnsw.set_zmq_port(port_b)\n elif hasattr(index_no_update, \"set_zmq_port\"):\n index_no_update.set_zmq_port(port_b)\n\n # Warmup the embedding model before benchmarking (do this for both --only B and --only both)\n # This ensures fair comparison as Scenario A has warmed up the model during update embeddings\n logger.info(\"Warming up embedding model for Scenario B...\")\n _ = compute_embeddings(\n [\"warmup text\"], args.model_name, mode=args.embedding_mode, is_build=False\n )\n\n # Prepare worker A: compute embeddings for the same N passages\n emb_time = 0.0\n updates_embs_offline: np.ndarray | None = None\n\n def _worker_emb():\n nonlocal emb_time, updates_embs_offline\n t = time.time()\n updates_embs_offline = compute_embeddings(\n update_paragraphs,\n args.model_name,\n mode=args.embedding_mode,\n is_build=False,\n batch_size=16,\n )\n emb_time = time.time() - t\n\n # Pre-compute query embedding and warm up search outside of timed section.\n q_vec = compute_embeddings(\n [args.query], args.model_name, mode=args.embedding_mode, is_build=False\n )\n q_vec = np.asarray(q_vec, dtype=np.float32)\n q_vec = _maybe_norm_cosine(q_vec, args.distance_metric)\n print(\"[DEBUG B] Warming up search...\")\n _ = _search(index_no_update, q_vec, 1)\n\n # Worker B: timed search on the warmed index\n search_time = 0.0\n offline_elapsed = 0.0\n index_results: tuple[np.ndarray, np.ndarray] | None = None\n\n def _worker_search():\n nonlocal search_time, index_results\n t = time.time()\n distances, indices = _search(index_no_update, q_vec, args.k)\n search_time = time.time() - t\n index_results = (distances, indices)\n\n # Run two workers concurrently\n t0 = time.time()\n th1 = threading.Thread(target=_worker_emb)\n th2 = threading.Thread(target=_worker_search)\n th1.start()\n th2.start()\n th1.join()\n th2.join()\n offline_elapsed = time.time() - t0\n\n # For mixing: compute query vs. offline update similarities (pure client-side)\n offline_scores: list[tuple[int, float]] = []\n if updates_embs_offline is not None:\n upd2 = np.asarray(updates_embs_offline, dtype=np.float32)\n upd2 = _maybe_norm_cosine(upd2, args.distance_metric)\n # For mips/cosine, score = dot; for l2, score = -||x-y||^2\n for j in range(upd2.shape[0]):\n if args.distance_metric in (\"mips\", \"cosine\"):\n s = float(np.dot(q_vec[0], upd2[j]))\n else:\n diff = q_vec[0] - upd2[j]\n s = -float(np.dot(diff, diff))\n offline_scores.append((j, s))\n\n merged_topk = (\n _merge_results(index_results, offline_scores, args.k, args.distance_metric)\n if index_results\n else []\n )\n\n print(\"\\n=== Scenario B: offline embeds + concurrent search (no add) ===\")\n print(\n f\"embeddings({args.num_updates})={emb_time:.3f}s; search={search_time:.3f}s; makespan≈{offline_elapsed:.3f}s (≈max)\"\n )\n if merged_topk:\n preview = \", \".join([f\"{lab}:{score:.3f}\" for lab, score in merged_topk[:5]])\n print(f\"Merged top-5 preview: {preview}\")\n # CSV row for B\n if args.csv_path:\n row_b = {\n \"run_id\": run_id,\n \"scenario\": \"B\",\n \"max_initial\": args.max_initial,\n \"num_updates\": args.num_updates,\n \"k\": args.k,\n \"total_time_s\": 0.0,\n \"add_total_s\": 0.0,\n \"search_time_s\": round(search_time, 6),\n \"emb_time_s\": round(emb_time, 6),\n \"makespan_s\": round(offline_elapsed, 6),\n \"model_name\": args.model_name,\n \"embedding_mode\": args.embedding_mode,\n \"distance_metric\": args.distance_metric,\n }\n with args.csv_path.open(\"a\", newline=\"\", encoding=\"utf-8\") as f:\n writer = csv.DictWriter(f, fieldnames=csv_fields)\n writer.writerow(row_b)\n\n finally:\n server_manager_b.stop_server()\n\n # Summary\n print(\"\\n=== Summary ===\")\n msg_a = (\n f\"A: seq-add+search overall={total_seq:.3f}s (adds={add_total:.3f}s, search={search_after_add:.3f}s)\"\n if args.only in (\"A\", \"both\")\n else \"A: skipped\"\n )\n msg_b = (\n f\"B: offline+concurrent overall≈{offline_elapsed:.3f}s (emb={emb_time:.3f}s, search={search_time:.3f}s)\"\n if args.only in (\"B\", \"both\")\n else \"B: skipped\"\n )\n print(msg_a + \"\\n\" + msg_b)\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "benchmarks/update/plot_bench_results.py", "content": "#!/usr/bin/env python3\n\"\"\"\nPlot latency bars from the benchmark CSV produced by\nbenchmarks/update/bench_hnsw_rng_recompute.py.\n\nIf you also provide an offline_vs_update.csv via --csv-right\n(from benchmarks/update/bench_update_vs_offline_search.py), this script will\noutput a side-by-side figure:\n- Left: ms/passage bars (four RNG scenarios).\n- Right: seconds bars (Scenario A seq add+search vs Scenario B offline+search).\n\nUsage:\n uv run python benchmarks/update/plot_bench_results.py \\\n --csv benchmarks/update/bench_results.csv \\\n --out benchmarks/update/bench_latency_from_csv.png\n\nThe script selects the latest run_id in the CSV and plots four bars for\nthe default scenarios:\n - baseline\n - no_cache_baseline\n - disable_forward_rng\n - disable_forward_and_reverse_rng\n\nIf multiple rows exist per scenario for that run_id, the script averages\ntheir latency_ms_per_passage values.\n\"\"\"\n\nimport argparse\nimport csv\nfrom collections import defaultdict\nfrom pathlib import Path\n\nDEFAULT_SCENARIOS = [\n \"no_cache_baseline\",\n \"baseline\",\n \"disable_forward_rng\",\n \"disable_forward_and_reverse_rng\",\n]\n\nSCENARIO_LABELS = {\n \"baseline\": \"+ Cache\",\n \"no_cache_baseline\": \"Naive \\n Recompute\",\n \"disable_forward_rng\": \"+ w/o \\n Fwd RNG\",\n \"disable_forward_and_reverse_rng\": \"+ w/o \\n Bwd RNG\",\n}\n\n# Paper-style colors and hatches for scenarios\nSCENARIO_STYLES = {\n \"no_cache_baseline\": {\"edgecolor\": \"dimgrey\", \"hatch\": \"/////\"},\n \"baseline\": {\"edgecolor\": \"#63B8B6\", \"hatch\": \"xxxxx\"},\n \"disable_forward_rng\": {\"edgecolor\": \"green\", \"hatch\": \".....\"},\n \"disable_forward_and_reverse_rng\": {\"edgecolor\": \"tomato\", \"hatch\": \"\\\\\\\\\\\\\\\\\\\\\"},\n}\n\n\ndef load_latest_run(csv_path: Path):\n rows = []\n with csv_path.open(\"r\", encoding=\"utf-8\") as f:\n reader = csv.DictReader(f)\n for row in reader:\n rows.append(row)\n if not rows:\n raise SystemExit(\"CSV is empty: no rows to plot\")\n # Choose latest run_id lexicographically (YYYYMMDD-HHMMSS)\n run_ids = [r.get(\"run_id\", \"\") for r in rows]\n latest = max(run_ids)\n latest_rows = [r for r in rows if r.get(\"run_id\", \"\") == latest]\n if not latest_rows:\n # Fallback: take last 4 rows\n latest_rows = rows[-4:]\n latest = latest_rows[-1].get(\"run_id\", \"unknown\")\n return latest, latest_rows\n\n\ndef aggregate_latency(rows):\n acc = defaultdict(list)\n for r in rows:\n sc = r.get(\"scenario\", \"\")\n try:\n val = float(r.get(\"latency_ms_per_passage\", \"nan\"))\n except ValueError:\n continue\n acc[sc].append(val)\n avg = {k: (sum(v) / len(v) if v else 0.0) for k, v in acc.items()}\n return avg\n\n\ndef _auto_cap(values: list[float]) -> float | None:\n if not values:\n return None\n sorted_vals = sorted(values, reverse=True)\n if len(sorted_vals) < 2:\n return None\n max_v, second = sorted_vals[0], sorted_vals[1]\n if second <= 0:\n return None\n # If the tallest bar dwarfs the second by 2.5x+, cap near the second\n if max_v >= 2.5 * second:\n return second * 1.1\n return None\n\n\ndef _add_break_marker(ax, y, rel_x0=0.02, rel_x1=0.98, size=0.02):\n # Draw small diagonal ticks near left/right to signal cap\n x0, x1 = rel_x0, rel_x1\n ax.plot([x0 - size, x0 + size], [y + size, y - size], transform=ax.transAxes, color=\"k\", lw=1)\n ax.plot([x1 - size, x1 + size], [y + size, y - size], transform=ax.transAxes, color=\"k\", lw=1)\n\n\ndef _fmt_ms(v: float) -> str:\n if v >= 1000:\n return f\"{v / 1000:.1f}k\"\n return f\"{v:.1f}\"\n\n\ndef main():\n # Set LaTeX style for paper figures (matching paper_fig.py)\n import matplotlib.pyplot as plt\n\n plt.rcParams[\"font.family\"] = \"Helvetica\"\n plt.rcParams[\"ytick.direction\"] = \"in\"\n plt.rcParams[\"hatch.linewidth\"] = 1.5\n plt.rcParams[\"font.weight\"] = \"bold\"\n plt.rcParams[\"axes.labelweight\"] = \"bold\"\n plt.rcParams[\"text.usetex\"] = True\n\n ap = argparse.ArgumentParser(description=__doc__)\n ap.add_argument(\n \"--csv\",\n type=Path,\n default=Path(\"benchmarks/update/bench_results.csv\"),\n help=\"Path to results CSV (defaults to bench_results.csv)\",\n )\n ap.add_argument(\n \"--out\",\n type=Path,\n default=Path(\"add_ablation.pdf\"),\n help=\"Output image path\",\n )\n ap.add_argument(\n \"--csv-right\",\n type=Path,\n default=Path(\"benchmarks/update/offline_vs_update.csv\"),\n help=\"Optional: offline_vs_update.csv to render right subplot (A vs B)\",\n )\n ap.add_argument(\n \"--cap-y\",\n type=float,\n default=None,\n help=\"Cap Y-axis at this ms value; bars above are hatched and annotated.\",\n )\n ap.add_argument(\n \"--no-auto-cap\",\n action=\"store_true\",\n help=\"Disable auto-cap heuristic when --cap-y is not provided.\",\n )\n ap.add_argument(\n \"--broken-y\",\n action=\"store_true\",\n default=True,\n help=\"Use a broken Y-axis (two stacked axes with a gap). Overrides --cap-y unless both provided.\",\n )\n ap.add_argument(\n \"--lower-cap-y\",\n type=float,\n default=None,\n help=\"Lower axes upper bound for broken Y (ms). Default = 1.1x second-highest.\",\n )\n ap.add_argument(\n \"--upper-start-y\",\n type=float,\n default=None,\n help=\"Upper axes lower bound for broken Y (ms). Default = 1.2x second-highest.\",\n )\n args = ap.parse_args()\n\n latest_run, latest_rows = load_latest_run(args.csv)\n avg = aggregate_latency(latest_rows)\n\n try:\n import matplotlib.pyplot as plt\n except Exception as e:\n raise SystemExit(f\"matplotlib not available: {e}\")\n\n scenarios = DEFAULT_SCENARIOS\n values = [avg.get(name, 0.0) for name in scenarios]\n labels = [SCENARIO_LABELS.get(name, name) for name in scenarios]\n colors = [\"#4e79a7\", \"#f28e2c\", \"#e15759\", \"#76b7b2\"]\n\n # If right CSV is provided, build side-by-side figure\n if args.csv_right is not None:\n try:\n right_rows_all = []\n with args.csv_right.open(\"r\", encoding=\"utf-8\") as f:\n rreader = csv.DictReader(f)\n right_rows_all = list(rreader)\n if right_rows_all:\n r_latest = max(r.get(\"run_id\", \"\") for r in right_rows_all)\n right_rows = [r for r in right_rows_all if r.get(\"run_id\", \"\") == r_latest]\n else:\n r_latest = None\n right_rows = []\n except Exception:\n r_latest = None\n right_rows = []\n\n a_total = 0.0\n b_makespan = 0.0\n for r in right_rows:\n sc = (r.get(\"scenario\", \"\") or \"\").strip().upper()\n if sc == \"A\":\n try:\n a_total = float(r.get(\"total_time_s\", 0.0))\n except Exception:\n pass\n elif sc == \"B\":\n try:\n b_makespan = float(r.get(\"makespan_s\", 0.0))\n except Exception:\n pass\n\n import matplotlib.pyplot as plt\n from matplotlib import gridspec\n\n # Left subplot (reuse current style, with optional cap)\n cap = args.cap_y\n if cap is None and not args.no_auto_cap:\n cap = _auto_cap(values)\n x = list(range(len(labels)))\n\n if args.broken_y:\n # Use broken axis for left subplot\n # Auto-adjust width ratios: left has 4 bars, right has 2 bars\n fig = plt.figure(figsize=(4.8, 1.8)) # Scaled down to 80%\n gs = gridspec.GridSpec(\n 2, 2, height_ratios=[1, 3], width_ratios=[1.5, 1], hspace=0.08, wspace=0.35\n )\n ax_left_top = fig.add_subplot(gs[0, 0])\n ax_left_bottom = fig.add_subplot(gs[1, 0], sharex=ax_left_top)\n ax_right = fig.add_subplot(gs[:, 1])\n\n # Determine break points\n s = sorted(values, reverse=True)\n second = s[1] if len(s) >= 2 else (s[0] if s else 0.0)\n lower_cap = (\n args.lower_cap_y if args.lower_cap_y is not None else second * 1.4\n ) # Increased to show more range\n upper_start = (\n args.upper_start_y\n if args.upper_start_y is not None\n else max(second * 1.5, lower_cap * 1.02)\n )\n ymax = (\n max(values) * 1.90 if values else 1.0\n ) # Increase headroom to 1.90 for text label and tick range\n\n # Draw bars on both axes\n ax_left_bottom.bar(x, values, color=colors[: len(labels)], width=0.8)\n ax_left_top.bar(x, values, color=colors[: len(labels)], width=0.8)\n\n # Set limits\n ax_left_bottom.set_ylim(0, lower_cap)\n ax_left_top.set_ylim(upper_start, ymax)\n\n # Annotate values (convert ms to s)\n values_s = [v / 1000.0 for v in values]\n lower_cap_s = lower_cap / 1000.0\n upper_start_s = upper_start / 1000.0\n ymax_s = ymax / 1000.0\n\n ax_left_bottom.set_ylim(0, lower_cap_s)\n ax_left_top.set_ylim(upper_start_s, ymax_s)\n\n # Redraw bars with s values (paper style: white fill + colored edge + hatch)\n ax_left_bottom.clear()\n ax_left_top.clear()\n bar_width = 0.50 # Reduced for wider spacing between bars\n for i, (scenario_name, v) in enumerate(zip(scenarios, values_s)):\n style = SCENARIO_STYLES.get(scenario_name, {\"edgecolor\": \"black\", \"hatch\": \"\"})\n # Draw in bottom axis for all bars\n ax_left_bottom.bar(\n i,\n v,\n width=bar_width,\n color=\"white\",\n edgecolor=style[\"edgecolor\"],\n hatch=style[\"hatch\"],\n linewidth=1.2,\n )\n # Only draw in top axis if the bar is tall enough to reach the upper range\n if v > upper_start_s:\n ax_left_top.bar(\n i,\n v,\n width=bar_width,\n color=\"white\",\n edgecolor=style[\"edgecolor\"],\n hatch=style[\"hatch\"],\n linewidth=1.2,\n )\n ax_left_bottom.set_ylim(0, lower_cap_s)\n ax_left_top.set_ylim(upper_start_s, ymax_s)\n\n for i, v in enumerate(values_s):\n if v <= lower_cap_s:\n ax_left_bottom.text(\n i,\n v + lower_cap_s * 0.02,\n f\"{v:.2f}\",\n ha=\"center\",\n va=\"bottom\",\n fontsize=8,\n fontweight=\"bold\",\n )\n else:\n ax_left_top.text(\n i,\n v + (ymax_s - upper_start_s) * 0.02,\n f\"{v:.2f}\",\n ha=\"center\",\n va=\"bottom\",\n fontsize=8,\n fontweight=\"bold\",\n )\n\n # Hide spines between axes\n ax_left_top.spines[\"bottom\"].set_visible(False)\n ax_left_bottom.spines[\"top\"].set_visible(False)\n ax_left_top.tick_params(\n labeltop=False, labelbottom=False, bottom=False\n ) # Hide tick marks\n ax_left_bottom.xaxis.tick_bottom()\n ax_left_bottom.tick_params(top=False) # Hide top tick marks\n\n # Draw break marks (matching paper_fig.py style)\n d = 0.015\n kwargs = {\n \"transform\": ax_left_top.transAxes,\n \"color\": \"k\",\n \"clip_on\": False,\n \"linewidth\": 0.8,\n \"zorder\": 10,\n }\n ax_left_top.plot((-d, +d), (-d, +d), **kwargs)\n ax_left_top.plot((1 - d, 1 + d), (-d, +d), **kwargs)\n kwargs.update({\"transform\": ax_left_bottom.transAxes})\n ax_left_bottom.plot((-d, +d), (1 - d, 1 + d), **kwargs)\n ax_left_bottom.plot((1 - d, 1 + d), (1 - d, 1 + d), **kwargs)\n\n ax_left_bottom.set_xticks(x)\n ax_left_bottom.set_xticklabels(labels, rotation=0, fontsize=7)\n # Don't set ylabel here - will use fig.text for alignment\n ax_left_bottom.tick_params(axis=\"y\", labelsize=10)\n ax_left_top.tick_params(axis=\"y\", labelsize=10)\n # Add subtle grid for better readability\n ax_left_bottom.grid(axis=\"y\", alpha=0.3, linestyle=\"--\", linewidth=0.5)\n ax_left_top.grid(axis=\"y\", alpha=0.3, linestyle=\"--\", linewidth=0.5)\n ax_left_top.set_title(\"Single Add Operation\", fontsize=11, pad=10, fontweight=\"bold\")\n\n # Set x-axis limits to match bar width with right subplot\n ax_left_bottom.set_xlim(-0.6, 3.6)\n ax_left_top.set_xlim(-0.6, 3.6)\n\n ax_left = ax_left_bottom # for compatibility\n else:\n # Regular side-by-side layout\n fig, (ax_left, ax_right) = plt.subplots(1, 2, figsize=(8.4, 3.15))\n\n if cap is not None:\n show_vals = [min(v, cap) for v in values]\n bars = ax_left.bar(x, show_vals, color=colors[: len(labels)], width=0.8)\n for i, (val, show) in enumerate(zip(values, show_vals)):\n if val > cap:\n bars[i].set_hatch(\"//\")\n ax_left.text(\n i, cap * 1.02, _fmt_ms(val), ha=\"center\", va=\"bottom\", fontsize=9\n )\n else:\n ax_left.text(\n i,\n show + max(1.0, 0.01 * (cap or show)),\n _fmt_ms(val),\n ha=\"center\",\n va=\"bottom\",\n fontsize=9,\n )\n ax_left.set_ylim(0, cap * 1.10)\n _add_break_marker(ax_left, y=0.98)\n ax_left.set_xticks(x)\n ax_left.set_xticklabels(labels, rotation=0, fontsize=10)\n else:\n ax_left.bar(x, values, color=colors[: len(labels)], width=0.8)\n for i, v in enumerate(values):\n ax_left.text(i, v + 1.0, _fmt_ms(v), ha=\"center\", va=\"bottom\", fontsize=9)\n ax_left.set_xticks(x)\n ax_left.set_xticklabels(labels, rotation=0, fontsize=10)\n ax_left.set_ylabel(\"Latency (ms per passage)\")\n max_initial = latest_rows[0].get(\"max_initial\", \"?\")\n max_updates = latest_rows[0].get(\"max_updates\", \"?\")\n ax_left.set_title(\n f\"HNSW RNG (run {latest_run}) | init={max_initial}, upd={max_updates}\"\n )\n\n # Right subplot (A vs B, seconds) - paper style\n r_labels = [\"Sequential\", \"Delayed \\n Add+Search\"]\n r_values = [a_total or 0.0, b_makespan or 0.0]\n r_styles = [\n {\"edgecolor\": \"#59a14f\", \"hatch\": \"xxxxx\"},\n {\"edgecolor\": \"#edc948\", \"hatch\": \"/////\"},\n ]\n # 2 bars, centered with proper spacing\n xr = [0, 1]\n bar_width = 0.50 # Reduced for wider spacing between bars\n for i, (v, style) in enumerate(zip(r_values, r_styles)):\n ax_right.bar(\n xr[i],\n v,\n width=bar_width,\n color=\"white\",\n edgecolor=style[\"edgecolor\"],\n hatch=style[\"hatch\"],\n linewidth=1.2,\n )\n for i, v in enumerate(r_values):\n max_v = max(r_values) if r_values else 1.0\n offset = max(0.0002, 0.02 * max_v)\n ax_right.text(\n xr[i],\n v + offset,\n f\"{v:.2f}\",\n ha=\"center\",\n va=\"bottom\",\n fontsize=8,\n fontweight=\"bold\",\n )\n ax_right.set_xticks(xr)\n ax_right.set_xticklabels(r_labels, rotation=0, fontsize=7)\n # Don't set ylabel here - will use fig.text for alignment\n ax_right.tick_params(axis=\"y\", labelsize=10)\n # Add subtle grid for better readability\n ax_right.grid(axis=\"y\", alpha=0.3, linestyle=\"--\", linewidth=0.5)\n ax_right.set_title(\"Batched Add Operation\", fontsize=11, pad=10, fontweight=\"bold\")\n\n # Set x-axis limits to match left subplot's bar width visually\n # Accounting for width_ratios=[1.5, 1]:\n # Left: 4 bars, xlim(-0.6, 3.6), range=4.2, physical_width=1.5*unit\n # bar_width_visual = 0.72 * (1.5*unit / 4.2)\n # Right: 2 bars, need same visual width\n # 0.72 * (1.0*unit / range_right) = 0.72 * (1.5*unit / 4.2)\n # range_right = 4.2 / 1.5 = 2.8\n # For bars at 0, 1: padding = (2.8 - 1) / 2 = 0.9\n ax_right.set_xlim(-0.9, 1.9)\n\n # Set y-axis limit with headroom for text labels\n if r_values:\n max_v = max(r_values)\n ax_right.set_ylim(0, max_v * 1.15)\n\n # Format y-axis to avoid scientific notation\n ax_right.ticklabel_format(style=\"plain\", axis=\"y\")\n\n plt.tight_layout()\n\n # Add aligned ylabels using fig.text (after tight_layout)\n # Get the vertical center of the entire figure\n fig_center_y = 0.5\n # Left ylabel - closer to left plot\n left_x = 0.05\n fig.text(\n left_x,\n fig_center_y,\n \"Latency (s)\",\n va=\"center\",\n rotation=\"vertical\",\n fontsize=11,\n fontweight=\"bold\",\n )\n # Right ylabel - closer to right plot\n right_bbox = ax_right.get_position()\n right_x = right_bbox.x0 - 0.07\n fig.text(\n right_x,\n fig_center_y,\n \"Latency (s)\",\n va=\"center\",\n rotation=\"vertical\",\n fontsize=11,\n fontweight=\"bold\",\n )\n\n plt.savefig(args.out, bbox_inches=\"tight\", pad_inches=0.05)\n # Also save PDF for paper\n pdf_out = args.out.with_suffix(\".pdf\")\n plt.savefig(pdf_out, bbox_inches=\"tight\", pad_inches=0.05)\n print(f\"Saved: {args.out}\")\n print(f\"Saved: {pdf_out}\")\n return\n\n # Broken-Y mode\n if args.broken_y:\n import matplotlib.pyplot as plt\n\n fig, (ax_top, ax_bottom) = plt.subplots(\n 2,\n 1,\n sharex=True,\n figsize=(7.5, 6.75),\n gridspec_kw={\"height_ratios\": [1, 3], \"hspace\": 0.08},\n )\n\n # Determine default breaks from second-highest\n s = sorted(values, reverse=True)\n second = s[1] if len(s) >= 2 else (s[0] if s else 0.0)\n lower_cap = args.lower_cap_y if args.lower_cap_y is not None else second * 1.1\n upper_start = (\n args.upper_start_y\n if args.upper_start_y is not None\n else max(second * 1.2, lower_cap * 1.02)\n )\n ymax = max(values) * 1.10 if values else 1.0\n\n x = list(range(len(labels)))\n ax_bottom.bar(x, values, color=colors[: len(labels)], width=0.8)\n ax_top.bar(x, values, color=colors[: len(labels)], width=0.8)\n\n # Limits\n ax_bottom.set_ylim(0, lower_cap)\n ax_top.set_ylim(upper_start, ymax)\n\n # Annotate values\n for i, v in enumerate(values):\n if v <= lower_cap:\n ax_bottom.text(\n i, v + lower_cap * 0.02, _fmt_ms(v), ha=\"center\", va=\"bottom\", fontsize=9\n )\n else:\n ax_top.text(i, v, _fmt_ms(v), ha=\"center\", va=\"bottom\", fontsize=9)\n\n # Hide spines between axes and draw diagonal break marks\n ax_top.spines[\"bottom\"].set_visible(False)\n ax_bottom.spines[\"top\"].set_visible(False)\n ax_top.tick_params(labeltop=False) # don't put tick labels at the top\n ax_bottom.xaxis.tick_bottom()\n\n # Diagonal lines at the break (matching paper_fig.py style)\n d = 0.015\n kwargs = {\n \"transform\": ax_top.transAxes,\n \"color\": \"k\",\n \"clip_on\": False,\n \"linewidth\": 0.8,\n \"zorder\": 10,\n }\n ax_top.plot((-d, +d), (-d, +d), **kwargs) # top-left diagonal\n ax_top.plot((1 - d, 1 + d), (-d, +d), **kwargs) # top-right diagonal\n kwargs.update({\"transform\": ax_bottom.transAxes})\n ax_bottom.plot((-d, +d), (1 - d, 1 + d), **kwargs) # bottom-left diagonal\n ax_bottom.plot((1 - d, 1 + d), (1 - d, 1 + d), **kwargs) # bottom-right diagonal\n\n ax_bottom.set_xticks(x)\n ax_bottom.set_xticklabels(labels, rotation=0, fontsize=10)\n ax = ax_bottom # for labeling below\n else:\n cap = args.cap_y\n if cap is None and not args.no_auto_cap:\n cap = _auto_cap(values)\n\n plt.figure(figsize=(5.4, 3.15))\n ax = plt.gca()\n\n if cap is not None:\n show_vals = [min(v, cap) for v in values]\n bars = []\n for i, (_label, val, show) in enumerate(zip(labels, values, show_vals)):\n bar = ax.bar(i, show, color=colors[i], width=0.8)\n bars.append(bar[0])\n # Hatch and annotate when capped\n if val > cap:\n bars[-1].set_hatch(\"//\")\n ax.text(i, cap * 1.02, f\"{_fmt_ms(val)}\", ha=\"center\", va=\"bottom\", fontsize=9)\n else:\n ax.text(\n i,\n show + max(1.0, 0.01 * (cap or show)),\n f\"{_fmt_ms(val)}\",\n ha=\"center\",\n va=\"bottom\",\n fontsize=9,\n )\n ax.set_ylim(0, cap * 1.10)\n _add_break_marker(ax, y=0.98)\n ax.legend([bars[1]], [\"capped\"], fontsize=8, frameon=False, loc=\"upper right\") if any(\n v > cap for v in values\n ) else None\n ax.set_xticks(range(len(labels)))\n ax.set_xticklabels(labels, fontsize=11, fontweight=\"bold\")\n else:\n ax.bar(labels, values, color=colors[: len(labels)])\n for idx, val in enumerate(values):\n ax.text(\n idx,\n val + 1.0,\n f\"{_fmt_ms(val)}\",\n ha=\"center\",\n va=\"bottom\",\n fontsize=10,\n fontweight=\"bold\",\n )\n ax.set_xticklabels(labels, fontsize=11, fontweight=\"bold\")\n # Try to extract some context for title\n max_initial = latest_rows[0].get(\"max_initial\", \"?\")\n max_updates = latest_rows[0].get(\"max_updates\", \"?\")\n\n if args.broken_y:\n fig.text(\n 0.02,\n 0.5,\n \"Latency (s)\",\n va=\"center\",\n rotation=\"vertical\",\n fontsize=11,\n fontweight=\"bold\",\n )\n fig.suptitle(\n \"Add Operation Latency\",\n fontsize=11,\n y=0.98,\n fontweight=\"bold\",\n )\n plt.tight_layout(rect=(0.03, 0.04, 1, 0.96))\n else:\n plt.ylabel(\"Latency (s)\", fontsize=11, fontweight=\"bold\")\n plt.title(\"Add Operation Latency\", fontsize=11, fontweight=\"bold\")\n plt.tight_layout()\n\n plt.savefig(args.out, bbox_inches=\"tight\", pad_inches=0.05)\n # Also save PDF for paper\n pdf_out = args.out.with_suffix(\".pdf\")\n plt.savefig(pdf_out, bbox_inches=\"tight\", pad_inches=0.05)\n print(f\"Saved: {args.out}\")\n print(f\"Saved: {pdf_out}\")\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "data/PrideandPrejudice.txt", "content": "The Project Gutenberg eBook of Pride and Prejudice\r\n\r\nThis ebook is for the use of anyone anywhere in the United States and\r\nmost other parts of the world at no cost and with almost no restrictions\r\nwhatsoever. You may copy it, give it away or re-use it under the terms\r\nof the Project Gutenberg License included with this ebook or online\r\nat www.gutenberg.org. If you are not located in the United States,\r\nyou will have to check the laws of the country where you are located\r\nbefore using this eBook.\r\n\r\nTitle: Pride and Prejudice\r\n\r\nAuthor: Jane Austen\r\n\r\nRelease date: June 1, 1998 [eBook #1342]\r\n Most recently updated: October 29, 2024\r\n\r\nLanguage: English\r\n\r\nCredits: Chuck Greif and the Online Distributed Proofreading Team at http://www.pgdp.net (This file was produced from images available at The Internet Archive)\r\n\r\n\r\n*** START OF THE PROJECT GUTENBERG EBOOK PRIDE AND PREJUDICE ***\r\n [Illustration:\r\n\r\n GEORGE ALLEN\r\n PUBLISHER\r\n\r\n 156 CHARING CROSS ROAD\r\n LONDON\r\n\r\n RUSKIN HOUSE\r\n ]\r\n\r\n [Illustration:\r\n\r\n _Reading Jane’s Letters._ _Chap 34._\r\n ]\r\n\r\n\r\n\r\n\r\n PRIDE.\r\n and\r\n PREJUDICE\r\n\r\n by\r\n Jane Austen,\r\n\r\n with a Preface by\r\n George Saintsbury\r\n and\r\n Illustrations by\r\n Hugh Thomson\r\n\r\n [Illustration: 1894]\r\n\r\n Ruskin 156. Charing\r\n House. Cross Road.\r\n\r\n London\r\n George Allen.\r\n\r\n\r\n\r\n\r\n CHISWICK PRESS:--CHARLES WHITTINGHAM AND CO.\r\n TOOKS COURT, CHANCERY LANE, LONDON.\r\n\r\n\r\n\r\n\r\n [Illustration:\r\n\r\n _To J. Comyns Carr\r\n in acknowledgment of all I\r\n owe to his friendship and\r\n advice, these illustrations are\r\n gratefully inscribed_\r\n\r\n _Hugh Thomson_\r\n ]\r\n\r\n\r\n\r\n\r\nPREFACE.\r\n\r\n[Illustration]\r\n\r\n\r\n_Walt Whitman has somewhere a fine and just distinction between “loving\r\nby allowance” and “loving with personal love.” This distinction applies\r\nto books as well as to men and women; and in the case of the not very\r\nnumerous authors who are the objects of the personal affection, it\r\nbrings a curious consequence with it. There is much more difference as\r\nto their best work than in the case of those others who are loved “by\r\nallowance” by convention, and because it is felt to be the right and\r\nproper thing to love them. And in the sect--fairly large and yet\r\nunusually choice--of Austenians or Janites, there would probably be\r\nfound partisans of the claim to primacy of almost every one of the\r\nnovels. To some the delightful freshness and humour of_ Northanger\r\nAbbey, _its completeness, finish, and_ entrain, _obscure the undoubted\r\ncritical facts that its scale is small, and its scheme, after all, that\r\nof burlesque or parody, a kind in which the first rank is reached with\r\ndifficulty._ Persuasion, _relatively faint in tone, and not enthralling\r\nin interest, has devotees who exalt above all the others its exquisite\r\ndelicacy and keeping. The catastrophe of_ Mansfield Park _is admittedly\r\ntheatrical, the hero and heroine are insipid, and the author has almost\r\nwickedly destroyed all romantic interest by expressly admitting that\r\nEdmund only took Fanny because Mary shocked him, and that Fanny might\r\nvery likely have taken Crawford if he had been a little more assiduous;\r\nyet the matchless rehearsal-scenes and the characters of Mrs. Norris and\r\nothers have secured, I believe, a considerable party for it._ Sense and\r\nSensibility _has perhaps the fewest out-and-out admirers; but it does\r\nnot want them._\r\n\r\n_I suppose, however, that the majority of at least competent votes\r\nwould, all things considered, be divided between_ Emma _and the present\r\nbook; and perhaps the vulgar verdict (if indeed a fondness for Miss\r\nAusten be not of itself a patent of exemption from any possible charge\r\nof vulgarity) would go for_ Emma. _It is the larger, the more varied, the\r\nmore popular; the author had by the time of its composition seen rather\r\nmore of the world, and had improved her general, though not her most\r\npeculiar and characteristic dialogue; such figures as Miss Bates, as the\r\nEltons, cannot but unite the suffrages of everybody. On the other hand,\r\nI, for my part, declare for_ Pride and Prejudice _unhesitatingly. It\r\nseems to me the most perfect, the most characteristic, the most\r\neminently quintessential of its author’s works; and for this contention\r\nin such narrow space as is permitted to me, I propose here to show\r\ncause._\r\n\r\n_In the first place, the book (it may be barely necessary to remind the\r\nreader) was in its first shape written very early, somewhere about 1796,\r\nwhen Miss Austen was barely twenty-one; though it was revised and\r\nfinished at Chawton some fifteen years later, and was not published till\r\n1813, only four years before her death. I do not know whether, in this\r\ncombination of the fresh and vigorous projection of youth, and the\r\ncritical revision of middle life, there may be traced the distinct\r\nsuperiority in point of construction, which, as it seems to me, it\r\npossesses over all the others. The plot, though not elaborate, is almost\r\nregular enough for Fielding; hardly a character, hardly an incident\r\ncould be retrenched without loss to the story. The elopement of Lydia\r\nand Wickham is not, like that of Crawford and Mrs. Rushworth, a_ coup de\r\nthéâtre; _it connects itself in the strictest way with the course of the\r\nstory earlier, and brings about the denouement with complete propriety.\r\nAll the minor passages--the loves of Jane and Bingley, the advent of Mr.\r\nCollins, the visit to Hunsford, the Derbyshire tour--fit in after the\r\nsame unostentatious, but masterly fashion. There is no attempt at the\r\nhide-and-seek, in-and-out business, which in the transactions between\r\nFrank Churchill and Jane Fairfax contributes no doubt a good deal to the\r\nintrigue of_ Emma, _but contributes it in a fashion which I do not think\r\nthe best feature of that otherwise admirable book. Although Miss Austen\r\nalways liked something of the misunderstanding kind, which afforded her\r\nopportunities for the display of the peculiar and incomparable talent to\r\nbe noticed presently, she has been satisfied here with the perfectly\r\nnatural occasions provided by the false account of Darcy’s conduct given\r\nby Wickham, and by the awkwardness (arising with equal naturalness) from\r\nthe gradual transformation of Elizabeth’s own feelings from positive\r\naversion to actual love. I do not know whether the all-grasping hand of\r\nthe playwright has ever been laid upon_ Pride and Prejudice; _and I dare\r\nsay that, if it were, the situations would prove not startling or\r\ngarish enough for the footlights, the character-scheme too subtle and\r\ndelicate for pit and gallery. But if the attempt were made, it would\r\ncertainly not be hampered by any of those loosenesses of construction,\r\nwhich, sometimes disguised by the conveniences of which the novelist can\r\navail himself, appear at once on the stage._\r\n\r\n_I think, however, though the thought will doubtless seem heretical to\r\nmore than one school of critics, that construction is not the highest\r\nmerit, the choicest gift, of the novelist. It sets off his other gifts\r\nand graces most advantageously to the critical eye; and the want of it\r\nwill sometimes mar those graces--appreciably, though not quite\r\nconsciously--to eyes by no means ultra-critical. But a very badly-built\r\nnovel which excelled in pathetic or humorous character, or which\r\ndisplayed consummate command of dialogue--perhaps the rarest of all\r\nfaculties--would be an infinitely better thing than a faultless plot\r\nacted and told by puppets with pebbles in their mouths. And despite the\r\nability which Miss Austen has shown in working out the story, I for one\r\nshould put_ Pride and Prejudice _far lower if it did not contain what\r\nseem to me the very masterpieces of Miss Austen’s humour and of her\r\nfaculty of character-creation--masterpieces who may indeed admit John\r\nThorpe, the Eltons, Mrs. Norris, and one or two others to their company,\r\nbut who, in one instance certainly, and perhaps in others, are still\r\nsuperior to them._\r\n\r\n_The characteristics of Miss Austen’s humour are so subtle and delicate\r\nthat they are, perhaps, at all times easier to apprehend than to\r\nexpress, and at any particular time likely to be differently\r\napprehended by different persons. To me this humour seems to possess a\r\ngreater affinity, on the whole, to that of Addison than to any other of\r\nthe numerous species of this great British genus. The differences of\r\nscheme, of time, of subject, of literary convention, are, of course,\r\nobvious enough; the difference of sex does not, perhaps, count for much,\r\nfor there was a distinctly feminine element in “Mr. Spectator,” and in\r\nJane Austen’s genius there was, though nothing mannish, much that was\r\nmasculine. But the likeness of quality consists in a great number of\r\ncommon subdivisions of quality--demureness, extreme minuteness of touch,\r\navoidance of loud tones and glaring effects. Also there is in both a\r\ncertain not inhuman or unamiable cruelty. It is the custom with those\r\nwho judge grossly to contrast the good nature of Addison with the\r\nsavagery of Swift, the mildness of Miss Austen with the boisterousness\r\nof Fielding and Smollett, even with the ferocious practical jokes that\r\nher immediate predecessor, Miss Burney, allowed without very much\r\nprotest. Yet, both in Mr. Addison and in Miss Austen there is, though a\r\nrestrained and well-mannered, an insatiable and ruthless delight in\r\nroasting and cutting up a fool. A man in the early eighteenth century,\r\nof course, could push this taste further than a lady in the early\r\nnineteenth; and no doubt Miss Austen’s principles, as well as her heart,\r\nwould have shrunk from such things as the letter from the unfortunate\r\nhusband in the_ Spectator, _who describes, with all the gusto and all the\r\ninnocence in the world, how his wife and his friend induce him to play\r\nat blind-man’s-buff. But another_ Spectator _letter--that of the damsel\r\nof fourteen who wishes to marry Mr. Shapely, and assures her selected\r\nMentor that “he admires your_ Spectators _mightily”--might have been\r\nwritten by a rather more ladylike and intelligent Lydia Bennet in the\r\ndays of Lydia’s great-grandmother; while, on the other hand, some (I\r\nthink unreasonably) have found “cynicism” in touches of Miss Austen’s\r\nown, such as her satire of Mrs. Musgrove’s self-deceiving regrets over\r\nher son. But this word “cynical” is one of the most misused in the\r\nEnglish language, especially when, by a glaring and gratuitous\r\nfalsification of its original sense, it is applied, not to rough and\r\nsnarling invective, but to gentle and oblique satire. If cynicism means\r\nthe perception of “the other side,” the sense of “the accepted hells\r\nbeneath,” the consciousness that motives are nearly always mixed, and\r\nthat to seem is not identical with to be--if this be cynicism, then\r\nevery man and woman who is not a fool, who does not care to live in a\r\nfool’s paradise, who has knowledge of nature and the world and life, is\r\na cynic. And in that sense Miss Austen certainly was one. She may even\r\nhave been one in the further sense that, like her own Mr. Bennet, she\r\ntook an epicurean delight in dissecting, in displaying, in setting at\r\nwork her fools and her mean persons. I think she did take this delight,\r\nand I do not think at all the worse of her for it as a woman, while she\r\nwas immensely the better for it as an artist._\r\n\r\n_In respect of her art generally, Mr. Goldwin Smith has truly observed\r\nthat “metaphor has been exhausted in depicting the perfection of it,\r\ncombined with the narrowness of her field;” and he has justly added that\r\nwe need not go beyond her own comparison to the art of a miniature\r\npainter. To make this latter observation quite exact we must not use the\r\nterm miniature in its restricted sense, and must think rather of Memling\r\nat one end of the history of painting and Meissonier at the other, than\r\nof Cosway or any of his kind. And I am not so certain that I should\r\nmyself use the word “narrow” in connection with her. If her world is a\r\nmicrocosm, the cosmic quality of it is at least as eminent as the\r\nlittleness. She does not touch what she did not feel herself called to\r\npaint; I am not so sure that she could not have painted what she did not\r\nfeel herself called to touch. It is at least remarkable that in two very\r\nshort periods of writing--one of about three years, and another of not\r\nmuch more than five--she executed six capital works, and has not left a\r\nsingle failure. It is possible that the romantic paste in her\r\ncomposition was defective: we must always remember that hardly\r\nanybody born in her decade--that of the eighteenth-century\r\nseventies--independently exhibited the full romantic quality. Even Scott\r\nrequired hill and mountain and ballad, even Coleridge metaphysics and\r\nGerman to enable them to chip the classical shell. Miss Austen was an\r\nEnglish girl, brought up in a country retirement, at the time when\r\nladies went back into the house if there was a white frost which might\r\npierce their kid shoes, when a sudden cold was the subject of the\r\ngravest fears, when their studies, their ways, their conduct were\r\nsubject to all those fantastic limits and restrictions against which\r\nMary Wollstonecraft protested with better general sense than particular\r\ntaste or judgment. Miss Austen, too, drew back when the white frost\r\ntouched her shoes; but I think she would have made a pretty good journey\r\neven in a black one._\r\n\r\n_For if her knowledge was not very extended, she knew two things which\r\nonly genius knows. The one was humanity, and the other was art. On the\r\nfirst head she could not make a mistake; her men, though limited, are\r\ntrue, and her women are, in the old sense, “absolute.” As to art, if she\r\nhas never tried idealism, her realism is real to a degree which makes\r\nthe false realism of our own day look merely dead-alive. Take almost any\r\nFrenchman, except the late M. de Maupassant, and watch him laboriously\r\npiling up strokes in the hope of giving a complete impression. You get\r\nnone; you are lucky if, discarding two-thirds of what he gives, you can\r\nshape a real impression out of the rest. But with Miss Austen the\r\nmyriad, trivial, unforced strokes build up the picture like magic.\r\nNothing is false; nothing is superfluous. When (to take the present book\r\nonly) Mr. Collins changed his mind from Jane to Elizabeth “while Mrs.\r\nBennet was stirring the fire” (and we know_ how _Mrs. Bennet would have\r\nstirred the fire), when Mr. Darcy “brought his coffee-cup back_\r\nhimself,” _the touch in each case is like that of Swift--“taller by the\r\nbreadth of my nail”--which impressed the half-reluctant Thackeray with\r\njust and outspoken admiration. Indeed, fantastic as it may seem, I\r\nshould put Miss Austen as near to Swift in some ways, as I have put her\r\nto Addison in others._\r\n\r\n_This Swiftian quality appears in the present novel as it appears\r\nnowhere else in the character of the immortal, the ineffable Mr.\r\nCollins. Mr. Collins is really_ great; _far greater than anything Addison\r\never did, almost great enough for Fielding or for Swift himself. It has\r\nbeen said that no one ever was like him. But in the first place,_ he\r\n_was like him; he is there--alive, imperishable, more real than hundreds\r\nof prime ministers and archbishops, of “metals, semi-metals, and\r\ndistinguished philosophers.” In the second place, it is rash, I think,\r\nto conclude that an actual Mr. Collins was impossible or non-existent at\r\nthe end of the eighteenth century. It is very interesting that we\r\npossess, in this same gallery, what may be called a spoiled first\r\ndraught, or an unsuccessful study of him, in John Dashwood. The\r\nformality, the under-breeding, the meanness, are there; but the portrait\r\nis only half alive, and is felt to be even a little unnatural. Mr.\r\nCollins is perfectly natural, and perfectly alive. In fact, for all the\r\n“miniature,” there is something gigantic in the way in which a certain\r\nside, and more than one, of humanity, and especially eighteenth-century\r\nhumanity, its Philistinism, its well-meaning but hide-bound morality,\r\nits formal pettiness, its grovelling respect for rank, its materialism,\r\nits selfishness, receives exhibition. I will not admit that one speech\r\nor one action of this inestimable man is incapable of being reconciled\r\nwith reality, and I should not wonder if many of these words and actions\r\nare historically true._\r\n\r\n_But the greatness of Mr. Collins could not have been so satisfactorily\r\nexhibited if his creatress had not adjusted so artfully to him the\r\nfigures of Mr. Bennet and of Lady Catherine de Bourgh. The latter, like\r\nMr. Collins himself, has been charged with exaggeration. There is,\r\nperhaps, a very faint shade of colour for the charge; but it seems to me\r\nvery faint indeed. Even now I do not think that it would be impossible\r\nto find persons, especially female persons, not necessarily of noble\r\nbirth, as overbearing, as self-centred, as neglectful of good manners,\r\nas Lady Catherine. A hundred years ago, an earl’s daughter, the Lady\r\nPowerful (if not exactly Bountiful) of an out-of-the-way country parish,\r\nrich, long out of marital authority, and so forth, had opportunities of\r\ndeveloping these agreeable characteristics which seldom present\r\nthemselves now. As for Mr. Bennet, Miss Austen, and Mr. Darcy, and even\r\nMiss Elizabeth herself, were, I am inclined to think, rather hard on him\r\nfor the “impropriety” of his conduct. His wife was evidently, and must\r\nalways have been, a quite irreclaimable fool; and unless he had shot her\r\nor himself there was no way out of it for a man of sense and spirit but\r\nthe ironic. From no other point of view is he open to any reproach,\r\nexcept for an excusable and not unnatural helplessness at the crisis of\r\nthe elopement, and his utterances are the most acutely delightful in the\r\nconsciously humorous kind--in the kind that we laugh with, not at--that\r\neven Miss Austen has put into the mouth of any of her characters. It is\r\ndifficult to know whether he is most agreeable when talking to his wife,\r\nor when putting Mr. Collins through his paces; but the general sense of\r\nthe world has probably been right in preferring to the first rank his\r\nconsolation to the former when she maunders over the entail, “My dear,\r\ndo not give way to such gloomy thoughts. Let us hope for better things.\r\nLet us flatter ourselves that_ I _may be the survivor;” and his inquiry\r\nto his colossal cousin as to the compliments which Mr. Collins has just\r\nrelated as made by himself to Lady Catherine, “May I ask whether these\r\npleasing attentions proceed from the impulse of the moment, or are the\r\nresult of previous study?” These are the things which give Miss Austen’s\r\nreaders the pleasant shocks, the delightful thrills, which are felt by\r\nthe readers of Swift, of Fielding, and we may here add, of Thackeray, as\r\nthey are felt by the readers of no other English author of fiction\r\noutside of these four._\r\n\r\n_The goodness of the minor characters in_ Pride and Prejudice _has been\r\nalready alluded to, and it makes a detailed dwelling on their beauties\r\ndifficult in any space, and impossible in this. Mrs. Bennet we have\r\nglanced at, and it is not easy to say whether she is more exquisitely\r\namusing or more horribly true. Much the same may be said of Kitty and\r\nLydia; but it is not every author, even of genius, who would have\r\ndifferentiated with such unerring skill the effects of folly and\r\nvulgarity of intellect and disposition working upon the common\r\nweaknesses of woman at such different ages. With Mary, Miss Austen has\r\ntaken rather less pains, though she has been even more unkind to her;\r\nnot merely in the text, but, as we learn from those interesting\r\ntraditional appendices which Mr. Austen Leigh has given us, in dooming\r\nher privately to marry “one of Mr. Philips’s clerks.” The habits of\r\nfirst copying and then retailing moral sentiments, of playing and\r\nsinging too long in public, are, no doubt, grievous and criminal; but\r\nperhaps poor Mary was rather the scapegoat of the sins of blue stockings\r\nin that Fordyce-belectured generation. It is at any rate difficult not\r\nto extend to her a share of the respect and affection (affection and\r\nrespect of a peculiar kind; doubtless), with which one regards Mr.\r\nCollins, when she draws the moral of Lydia’s fall. I sometimes wish\r\nthat the exigencies of the story had permitted Miss Austen to unite\r\nthese personages, and thus at once achieve a notable mating and soothe\r\npoor Mrs. Bennet’s anguish over the entail._\r\n\r\n_The Bingleys and the Gardiners and the Lucases, Miss Darcy and Miss de\r\nBourgh, Jane, Wickham, and the rest, must pass without special comment,\r\nfurther than the remark that Charlotte Lucas (her egregious papa, though\r\ndelightful, is just a little on the thither side of the line between\r\ncomedy and farce) is a wonderfully clever study in drab of one kind, and\r\nthat Wickham (though something of Miss Austen’s hesitation of touch in\r\ndealing with young men appears) is a not much less notable sketch in\r\ndrab of another. Only genius could have made Charlotte what she is, yet\r\nnot disagreeable; Wickham what he is, without investing him either with\r\na cheap Don Juanish attractiveness or a disgusting rascality. But the\r\nhero and the heroine are not tints to be dismissed._\r\n\r\n_Darcy has always seemed to me by far the best and most interesting of\r\nMiss Austen’s heroes; the only possible competitor being Henry Tilney,\r\nwhose part is so slight and simple that it hardly enters into\r\ncomparison. It has sometimes, I believe, been urged that his pride is\r\nunnatural at first in its expression and later in its yielding, while\r\nhis falling in love at all is not extremely probable. Here again I\r\ncannot go with the objectors. Darcy’s own account of the way in which\r\nhis pride had been pampered, is perfectly rational and sufficient; and\r\nnothing could be, psychologically speaking, a_ causa verior _for its\r\nsudden restoration to healthy conditions than the shock of Elizabeth’s\r\nscornful refusal acting on a nature_ ex hypothesi _generous. Nothing in\r\neven our author is finer and more delicately touched than the change of\r\nhis demeanour at the sudden meeting in the grounds of Pemberley. Had he\r\nbeen a bad prig or a bad coxcomb, he might have been still smarting\r\nunder his rejection, or suspicious that the girl had come\r\nhusband-hunting. His being neither is exactly consistent with the\r\nprobable feelings of a man spoilt in the common sense, but not really\r\ninjured in disposition, and thoroughly in love. As for his being in\r\nlove, Elizabeth has given as just an exposition of the causes of that\r\nphenomenon as Darcy has of the conditions of his unregenerate state,\r\nonly she has of course not counted in what was due to her own personal\r\ncharm._\r\n\r\n_The secret of that charm many men and not a few women, from Miss Austen\r\nherself downwards, have felt, and like most charms it is a thing rather\r\nto be felt than to be explained. Elizabeth of course belongs to the_\r\nallegro _or_ allegra _division of the army of Venus. Miss Austen was\r\nalways provokingly chary of description in regard to her beauties; and\r\nexcept the fine eyes, and a hint or two that she had at any rate\r\nsometimes a bright complexion, and was not very tall, we hear nothing\r\nabout her looks. But her chief difference from other heroines of the\r\nlively type seems to lie first in her being distinctly clever--almost\r\nstrong-minded, in the better sense of that objectionable word--and\r\nsecondly in her being entirely destitute of ill-nature for all her\r\npropensity to tease and the sharpness of her tongue. Elizabeth can give\r\nat least as good as she gets when she is attacked; but she never\r\n“scratches,” and she never attacks first. Some of the merest\r\nobsoletenesses of phrase and manner give one or two of her early\r\nspeeches a slight pertness, but that is nothing, and when she comes to\r\nserious business, as in the great proposal scene with Darcy (which is,\r\nas it should be, the climax of the interest of the book), and in the\r\nfinal ladies’ battle with Lady Catherine, she is unexceptionable. Then\r\ntoo she is a perfectly natural girl. She does not disguise from herself\r\nor anybody that she resents Darcy’s first ill-mannered personality with\r\nas personal a feeling. (By the way, the reproach that the ill-manners of\r\nthis speech are overdone is certainly unjust; for things of the same\r\nkind, expressed no doubt less stiltedly but more coarsely, might have\r\nbeen heard in more than one ball-room during this very year from persons\r\nwho ought to have been no worse bred than Darcy.) And she lets the\r\ninjury done to Jane and the contempt shown to the rest of her family\r\naggravate this resentment in the healthiest way in the world._\r\n\r\n_Still, all this does not explain her charm, which, taking beauty as a\r\ncommon form of all heroines, may perhaps consist in the addition to her\r\nplayfulness, her wit, her affectionate and natural disposition, of a\r\ncertain fearlessness very uncommon in heroines of her type and age.\r\nNearly all of them would have been in speechless awe of the magnificent\r\nDarcy; nearly all of them would have palpitated and fluttered at the\r\nidea of proposals, even naughty ones, from the fascinating Wickham.\r\nElizabeth, with nothing offensive, nothing_ viraginous, _nothing of the\r\n“New Woman” about her, has by nature what the best modern (not “new”)\r\nwomen have by education and experience, a perfect freedom from the idea\r\nthat all men may bully her if they choose, and that most will away with\r\nher if they can. Though not in the least “impudent and mannish grown,”\r\nshe has no mere sensibility, no nasty niceness about her. The form of\r\npassion common and likely to seem natural in Miss Austen’s day was so\r\ninvariably connected with the display of one or the other, or both of\r\nthese qualities, that she has not made Elizabeth outwardly passionate.\r\nBut I, at least, have not the slightest doubt that she would have\r\nmarried Darcy just as willingly without Pemberley as with it, and\r\nanybody who can read between lines will not find the lovers’\r\nconversations in the final chapters so frigid as they might have looked\r\nto the Della Cruscans of their own day, and perhaps do look to the Della\r\nCruscans of this._\r\n\r\n_And, after all, what is the good of seeking for the reason of\r\ncharm?--it is there. There were better sense in the sad mechanic\r\nexercise of determining the reason of its absence where it is not. In\r\nthe novels of the last hundred years there are vast numbers of young\r\nladies with whom it might be a pleasure to fall in love; there are at\r\nleast five with whom, as it seems to me, no man of taste and spirit can\r\nhelp doing so. Their names are, in chronological order, Elizabeth\r\nBennet, Diana Vernon, Argemone Lavington, Beatrix Esmond, and Barbara\r\nGrant. I should have been most in love with Beatrix and Argemone; I\r\nshould, I think, for mere occasional companionship, have preferred Diana\r\nand Barbara. But to live with and to marry, I do not know that any one\r\nof the four can come into competition with Elizabeth._\r\n\r\n_GEORGE SAINTSBURY._\r\n\r\n\r\n\r\n\r\n[Illustration: List of Illustrations.]\r\n\r\n\r\n PAGE\r\n\r\nFrontispiece iv\r\n\r\nTitle-page v\r\n\r\nDedication vii\r\n\r\nHeading to Preface ix\r\n\r\nHeading to List of Illustrations xxv\r\n\r\nHeading to Chapter I. 1\r\n\r\n“He came down to see the place” 2\r\n\r\nMr. and Mrs. Bennet 5\r\n\r\n“I hope Mr. Bingley will like it” 6\r\n\r\n“I’m the tallest” 9\r\n\r\n“He rode a black horse” 10\r\n\r\n“When the party entered” 12\r\n\r\n“She is tolerable” 15\r\n\r\nHeading to Chapter IV. 18\r\n\r\nHeading to Chapter V. 22\r\n\r\n“Without once opening his lips” 24\r\n\r\nTailpiece to Chapter V. 26\r\n\r\nHeading to Chapter VI. 27\r\n\r\n“The entreaties of several” 31\r\n\r\n“A note for Miss Bennet” 36\r\n\r\n“Cheerful prognostics” 40\r\n\r\n“The apothecary came” 43\r\n\r\n“Covering a screen” 45\r\n\r\n“Mrs. Bennet and her two youngest girls” 53\r\n\r\nHeading to Chapter X. 60\r\n\r\n“No, no; stay where you are” 67\r\n\r\n“Piling up the fire” 69\r\n\r\nHeading to Chapter XII. 75\r\n\r\nHeading to Chapter XIII. 78\r\n\r\nHeading to Chapter XIV. 84\r\n\r\n“Protested that he never read novels” 87\r\n\r\nHeading to Chapter XV. 89\r\n\r\nHeading to Chapter XVI. 95\r\n\r\n“The officers of the ----shire” 97\r\n\r\n“Delighted to see their dear friend again” 108\r\n\r\nHeading to Chapter XVIII. 113\r\n\r\n“Such very superior dancing is not often seen” 118\r\n\r\n“To assure you in the most animated language” 132\r\n\r\nHeading to Chapter XX. 139\r\n\r\n“They entered the breakfast-room” 143\r\n\r\nHeading to Chapter XXI. 146\r\n\r\n“Walked back with them” 148\r\n\r\nHeading to Chapter XXII. 154\r\n\r\n“So much love and eloquence” 156\r\n\r\n“Protested he must be entirely mistaken” 161\r\n\r\n“Whenever she spoke in a low voice” 166\r\n\r\nHeading to Chapter XXIV. 168\r\n\r\nHeading to Chapter XXV. 175\r\n\r\n“Offended two or three young ladies” 177\r\n\r\n“Will you come and see me?” 181\r\n\r\n“On the stairs” 189\r\n\r\n“At the door” 194\r\n\r\n“In conversation with the ladies” 198\r\n\r\n“Lady Catherine,” said she, “you have given me a treasure” 200\r\n\r\nHeading to Chapter XXX. 209\r\n\r\n“He never failed to inform them” 211\r\n\r\n“The gentlemen accompanied him” 213\r\n\r\nHeading to Chapter XXXI. 215\r\n\r\nHeading to Chapter XXXII. 221\r\n\r\n“Accompanied by their aunt” 225\r\n\r\n“On looking up” 228\r\n\r\nHeading to Chapter XXXIV. 235\r\n\r\n“Hearing herself called” 243\r\n\r\nHeading to Chapter XXXVI. 253\r\n\r\n“Meeting accidentally in town” 256\r\n\r\n“His parting obeisance” 261\r\n\r\n“Dawson” 263\r\n\r\n“The elevation of his feelings” 267\r\n\r\n“They had forgotten to leave any message” 270\r\n\r\n“How nicely we are crammed in!” 272\r\n\r\nHeading to Chapter XL. 278\r\n\r\n“I am determined never to speak of it again” 283\r\n\r\n“When Colonel Miller’s regiment went away” 285\r\n\r\n“Tenderly flirting” 290\r\n\r\nThe arrival of the Gardiners 294\r\n\r\n“Conjecturing as to the date” 301\r\n\r\nHeading to Chapter XLIV. 318\r\n\r\n“To make herself agreeable to all” 321\r\n\r\n“Engaged by the river” 327\r\n\r\nHeading to Chapter XLVI. 334\r\n\r\n“I have not an instant to lose” 339\r\n\r\n“The first pleasing earnest of their welcome” 345\r\n\r\nThe Post 359\r\n\r\n“To whom I have related the affair” 363\r\n\r\nHeading to Chapter XLIX. 368\r\n\r\n“But perhaps you would like to read it” 370\r\n\r\n“The spiteful old ladies” 377\r\n\r\n“With an affectionate smile” 385\r\n\r\n“I am sure she did not listen” 393\r\n\r\n“Mr. Darcy with him” 404\r\n\r\n“Jane happened to look round” 415\r\n\r\n“Mrs. Long and her nieces” 420\r\n\r\n“Lizzy, my dear, I want to speak to you” 422\r\n\r\nHeading to Chapter LVI. 431\r\n\r\n“After a short survey” 434\r\n\r\n“But now it comes out” 442\r\n\r\n“The efforts of his aunt” 448\r\n\r\n“Unable to utter a syllable” 457\r\n\r\n“The obsequious civility” 466\r\n\r\nHeading to Chapter LXI. 472\r\n\r\nThe End 476\r\n\r\n\r\n\r\n\r\n[Illustration: ·PRIDE AND PREJUDICE·\r\n\r\n\r\n\r\n\r\nChapter I.]\r\n\r\n\r\nIt is a truth universally acknowledged, that a single man in possession\r\nof a good fortune must be in want of a wife.\r\n\r\nHowever little known the feelings or views of such a man may be on his\r\nfirst entering a neighbourhood, this truth is so well fixed in the minds\r\nof the surrounding families, that he is considered as the rightful\r\nproperty of some one or other of their daughters.\r\n\r\n“My dear Mr. Bennet,” said his lady to him one day, “have you heard that\r\nNetherfield Park is let at last?”\r\n\r\nMr. Bennet replied that he had not.\r\n\r\n“But it is,” returned she; “for Mrs. Long has just been here, and she\r\ntold me all about it.”\r\n\r\nMr. Bennet made no answer.\r\n\r\n“Do not you want to know who has taken it?” cried his wife, impatiently.\r\n\r\n“_You_ want to tell me, and I have no objection to hearing it.”\r\n\r\n[Illustration:\r\n\r\n“He came down to see the place”\r\n\r\n[_Copyright 1894 by George Allen._]]\r\n\r\nThis was invitation enough.\r\n\r\n“Why, my dear, you must know, Mrs. Long says that Netherfield is taken\r\nby a young man of large fortune from the north of England; that he came\r\ndown on Monday in a chaise and four to see the place, and was so much\r\ndelighted with it that he agreed with Mr. Morris immediately; that he is\r\nto take possession before Michaelmas, and some of his servants are to be\r\nin the house by the end of next week.”\r\n\r\n“What is his name?”\r\n\r\n“Bingley.”\r\n\r\n“Is he married or single?”\r\n\r\n“Oh, single, my dear, to be sure! A single man of large fortune; four or\r\nfive thousand a year. What a fine thing for our girls!”\r\n\r\n“How so? how can it affect them?”\r\n\r\n“My dear Mr. Bennet,” replied his wife, “how can you be so tiresome? You\r\nmust know that I am thinking of his marrying one of them.”\r\n\r\n“Is that his design in settling here?”\r\n\r\n“Design? Nonsense, how can you talk so! But it is very likely that he\r\n_may_ fall in love with one of them, and therefore you must visit him as\r\nsoon as he comes.”\r\n\r\n“I see no occasion for that. You and the girls may go--or you may send\r\nthem by themselves, which perhaps will be still better; for as you are\r\nas handsome as any of them, Mr. Bingley might like you the best of the\r\nparty.”\r\n\r\n“My dear, you flatter me. I certainly _have_ had my share of beauty, but\r\nI do not pretend to be anything extraordinary now. When a woman has five\r\ngrown-up daughters, she ought to give over thinking of her own beauty.”\r\n\r\n“In such cases, a woman has not often much beauty to think of.”\r\n\r\n“But, my dear, you must indeed go and see Mr. Bingley when he comes into\r\nthe neighbourhood.”\r\n\r\n“It is more than I engage for, I assure you.”\r\n\r\n“But consider your daughters. Only think what an establishment it would\r\nbe for one of them. Sir William and Lady Lucas are determined to go,\r\nmerely on that account; for in general, you know, they visit no new\r\ncomers. Indeed you must go, for it will be impossible for _us_ to visit\r\nhim, if you do not.”\r\n\r\n“You are over scrupulous, surely. I dare say Mr. Bingley will be very\r\nglad to see you; and I will send a few lines by you to assure him of my\r\nhearty consent to his marrying whichever he chooses of the girls--though\r\nI must throw in a good word for my little Lizzy.”\r\n\r\n“I desire you will do no such thing. Lizzy is not a bit better than the\r\nothers: and I am sure she is not half so handsome as Jane, nor half so\r\ngood-humoured as Lydia. But you are always giving _her_ the preference.”\r\n\r\n“They have none of them much to recommend them,” replied he: “they are\r\nall silly and ignorant like other girls; but Lizzy has something more of\r\nquickness than her sisters.”\r\n\r\n“Mr. Bennet, how can you abuse your own children in such a way? You take\r\ndelight in vexing me. You have no compassion on my poor nerves.”\r\n\r\n“You mistake me, my dear. I have a high respect for your nerves. They\r\nare my old friends. I have heard you mention them with consideration\r\nthese twenty years at least.”\r\n\r\n“Ah, you do not know what I suffer.”\r\n\r\n“But I hope you will get over it, and live to see many young men of four\r\nthousand a year come into the neighbourhood.”\r\n\r\n“It will be no use to us, if twenty such should come, since you will not\r\nvisit them.”\r\n\r\n“Depend upon it, my dear, that when there are twenty, I will visit them\r\nall.”\r\n\r\nMr. Bennet was so odd a mixture of quick parts, sarcastic humour,\r\nreserve, and caprice, that the experience of three-and-twenty years had\r\nbeen insufficient to make his wife understand his character. _Her_ mind\r\nwas less difficult to develope. She was a woman of mean understanding,\r\nlittle information, and uncertain temper. When she was discontented, she\r\nfancied herself nervous. The business of her life was to get her\r\ndaughters married: its solace was visiting and news.\r\n\r\n[Illustration: M^{r.} & M^{rs.} Bennet\r\n\r\n[_Copyright 1894 by George Allen._]]\r\n\r\n\r\n\r\n\r\n[Illustration:\r\n\r\n“I hope Mr. Bingley will like it”\r\n\r\n[_Copyright 1894 by George Allen._]]\r\n\r\n\r\n\r\n\r\nCHAPTER II.\r\n\r\n\r\n[Illustration]\r\n\r\nMr. Bennet was among the earliest of those who waited on Mr. Bingley. He\r\nhad always intended to visit him, though to the last always assuring his\r\nwife that he should not go; and till the evening after the visit was\r\npaid she had no knowledge of it. It was then disclosed in the following\r\nmanner. Observing his second daughter employed in trimming a hat, he\r\nsuddenly addressed her with,--\r\n\r\n“I hope Mr. Bingley will like it, Lizzy.”\r\n\r\n“We are not in a way to know _what_ Mr. Bingley likes,” said her mother,\r\nresentfully, “since we are not to visit.”\r\n\r\n“But you forget, mamma,” said Elizabeth, “that we shall meet him at the\r\nassemblies, and that Mrs. Long has promised to introduce him.”\r\n\r\n“I do not believe Mrs. Long will do any such thing. She has two nieces\r\nof her own. She is a selfish, hypocritical woman, and I have no opinion\r\nof her.”\r\n\r\n“No more have I,” said Mr. Bennet; “and I am glad to find that you do\r\nnot depend on her serving you.”\r\n\r\nMrs. Bennet deigned not to make any reply; but, unable to contain\r\nherself, began scolding one of her daughters.\r\n\r\n“Don’t keep coughing so, Kitty, for heaven’s sake! Have a little\r\ncompassion on my nerves. You tear them to pieces.”\r\n\r\n“Kitty has no discretion in her coughs,” said her father; “she times\r\nthem ill.”\r\n\r\n“I do not cough for my own amusement,” replied Kitty, fretfully. “When\r\nis your next ball to be, Lizzy?”\r\n\r\n“To-morrow fortnight.”\r\n\r\n“Ay, so it is,” cried her mother, “and Mrs. Long does not come back till\r\nthe day before; so, it will be impossible for her to introduce him, for\r\nshe will not know him herself.”\r\n\r\n“Then, my dear, you may have the advantage of your friend, and introduce\r\nMr. Bingley to _her_.”\r\n\r\n“Impossible, Mr. Bennet, impossible, when I am not acquainted with him\r\nmyself; how can you be so teasing?”\r\n\r\n“I honour your circumspection. A fortnight’s acquaintance is certainly\r\nvery little. One cannot know what a man really is by the end of a\r\nfortnight. But if _we_ do not venture, somebody else will; and after\r\nall, Mrs. Long and her nieces must stand their chance; and, therefore,\r\nas she will think it an act of kindness, if you decline the office, I\r\nwill take it on myself.”\r\n\r\nThe girls stared at their father. Mrs. Bennet said only, “Nonsense,\r\nnonsense!”\r\n\r\n“What can be the meaning of that emphatic exclamation?” cried he. “Do\r\nyou consider the forms of introduction, and the stress that is laid on\r\nthem, as nonsense? I cannot quite agree with you _there_. What say you,\r\nMary? For you are a young lady of deep reflection, I know, and read\r\ngreat books, and make extracts.”\r\n\r\nMary wished to say something very sensible, but knew not how.\r\n\r\n“While Mary is adjusting her ideas,” he continued, “let us return to Mr.\r\nBingley.”\r\n\r\n“I am sick of Mr. Bingley,” cried his wife.\r\n\r\n“I am sorry to hear _that_; but why did you not tell me so before? If I\r\nhad known as much this morning, I certainly would not have called on\r\nhim. It is very unlucky; but as I have actually paid the visit, we\r\ncannot escape the acquaintance now.”\r\n\r\nThe astonishment of the ladies was just what he wished--that of Mrs.\r\nBennet perhaps surpassing the rest; though when the first tumult of joy\r\nwas over, she began to declare that it was what she had expected all the\r\nwhile.\r\n\r\n“How good it was in you, my dear Mr. Bennet! But I knew I should\r\npersuade you at last. I was sure you loved your girls too well to\r\nneglect such an acquaintance. Well, how pleased I am! And it is such a\r\ngood joke, too, that you should have gone this morning, and never said a\r\nword about it till now.”\r\n\r\n“Now, Kitty, you may cough as much as you choose,” said Mr. Bennet; and,\r\nas he spoke, he left the room, fatigued with the raptures of his wife.\r\n\r\n“What an excellent father you have, girls,” said she, when the door was\r\nshut. “I do not know how you will ever make him amends for his kindness;\r\nor me either, for that matter. At our time of life, it is not so\r\npleasant, I can tell you, to be making new acquaintances every day; but\r\nfor your sakes we would do anything. Lydia, my love, though you _are_\r\nthe youngest, I dare say Mr. Bingley will dance with you at the next\r\nball.”\r\n\r\n“Oh,” said Lydia, stoutly, “I am not afraid; for though I _am_ the\r\nyoungest, I’m the tallest.”\r\n\r\nThe rest of the evening was spent in conjecturing how soon he would\r\nreturn Mr. Bennet’s visit, and determining when they should ask him to\r\ndinner.\r\n\r\n[Illustration: “I’m the tallest”]\r\n\r\n\r\n\r\n\r\n[Illustration:\r\n\r\n “He rode a black horse”\r\n]\r\n\r\n\r\n\r\n\r\nCHAPTER III.\r\n\r\n\r\n[Illustration]\r\n\r\nNot all that Mrs. Bennet, however, with the assistance of her five\r\ndaughters, could ask on the subject, was sufficient to draw from her\r\nhusband any satisfactory description of Mr. Bingley. They attacked him\r\nin various ways, with barefaced questions, ingenious suppositions, and\r\ndistant surmises; but he eluded the skill of them all; and they were at\r\nlast obliged to accept the second-hand intelligence of their neighbour,\r\nLady Lucas. Her report was highly favourable. Sir William had been\r\ndelighted with him. He was quite young, wonderfully handsome, extremely\r\nagreeable, and, to crown the whole, he meant to be at the next assembly\r\nwith a large party. Nothing could be more delightful! To be fond of\r\ndancing was a certain step towards falling in love; and very lively\r\nhopes of Mr. Bingley’s heart were entertained.\r\n\r\n“If I can but see one of my daughters happily settled at Netherfield,”\r\nsaid Mrs. Bennet to her husband, “and all the others equally well\r\nmarried, I shall have nothing to wish for.”\r\n\r\nIn a few days Mr. Bingley returned Mr. Bennet’s visit, and sat about ten\r\nminutes with him in his library. He had entertained hopes of being\r\nadmitted to a sight of the young ladies, of whose beauty he had heard\r\nmuch; but he saw only the father. The ladies were somewhat more\r\nfortunate, for they had the advantage of ascertaining, from an upper\r\nwindow, that he wore a blue coat and rode a black horse.\r\n\r\nAn invitation to dinner was soon afterwards despatched; and already had\r\nMrs. Bennet planned the courses that were to do credit to her\r\nhousekeeping, when an answer arrived which deferred it all. Mr. Bingley\r\nwas obliged to be in town the following day, and consequently unable to\r\naccept the honour of their invitation, etc. Mrs. Bennet was quite\r\ndisconcerted. She could not imagine what business he could have in town\r\nso soon after his arrival in Hertfordshire; and she began to fear that\r\nhe might always be flying about from one place to another, and never\r\nsettled at Netherfield as he ought to be. Lady Lucas quieted her fears a\r\nlittle by starting the idea of his\r\n\r\n[Illustration:\r\n\r\n “When the Party entered”\r\n\r\n[_Copyright 1894 by George Allen._]]\r\n\r\nbeing gone to London only to get a large party for the ball; and a\r\nreport soon followed that Mr. Bingley was to bring twelve ladies and\r\nseven gentlemen with him to the assembly. The girls grieved over such a\r\nnumber of ladies; but were comforted the day before the ball by hearing\r\nthat, instead of twelve, he had brought only six with him from London,\r\nhis five sisters and a cousin. And when the party entered the\r\nassembly-room, it consisted of only five altogether: Mr. Bingley, his\r\ntwo sisters, the husband of the eldest, and another young man.\r\n\r\nMr. Bingley was good-looking and gentlemanlike: he had a pleasant\r\ncountenance, and easy, unaffected manners. His sisters were fine women,\r\nwith an air of decided fashion. His brother-in-law, Mr. Hurst, merely\r\nlooked the gentleman; but his friend Mr. Darcy soon drew the attention\r\nof the room by his fine, tall person, handsome features, noble mien, and\r\nthe report, which was in general circulation within five minutes after\r\nhis entrance, of his having ten thousand a year. The gentlemen\r\npronounced him to be a fine figure of a man, the ladies declared he was\r\nmuch handsomer than Mr. Bingley, and he was looked at with great\r\nadmiration for about half the evening, till his manners gave a disgust\r\nwhich turned the tide of his popularity; for he was discovered to be\r\nproud, to be above his company, and above being pleased; and not all his\r\nlarge estate in Derbyshire could save him from having a most forbidding,\r\ndisagreeable countenance, and being unworthy to be compared with his\r\nfriend.\r\n\r\nMr. Bingley had soon made himself acquainted with all the principal\r\npeople in the room: he was lively and unreserved, danced every dance,\r\nwas angry that the ball closed so early, and talked of giving one\r\nhimself at Netherfield. Such amiable qualities must speak for\r\nthemselves. What a contrast between him and his friend! Mr. Darcy danced\r\nonly once with Mrs. Hurst and once with Miss Bingley, declined being\r\nintroduced to any other lady, and spent the rest of the evening in\r\nwalking about the room, speaking occasionally to one of his own party.\r\nHis character was decided. He was the proudest, most disagreeable man in\r\nthe world, and everybody hoped that he would never come there again.\r\nAmongst the most violent against him was Mrs. Bennet, whose dislike of\r\nhis general behaviour was sharpened into particular resentment by his\r\nhaving slighted one of her daughters.\r\n\r\nElizabeth Bennet had been obliged, by the scarcity of gentlemen, to sit\r\ndown for two dances; and during part of that time, Mr. Darcy had been\r\nstanding near enough for her to overhear a conversation between him and\r\nMr. Bingley, who came from the dance for a few minutes to press his\r\nfriend to join it.\r\n\r\n“Come, Darcy,” said he, “I must have you dance. I hate to see you\r\nstanding about by yourself in this stupid manner. You had much better\r\ndance.”\r\n\r\n“I certainly shall not. You know how I detest it, unless I am\r\nparticularly acquainted with my partner. At such an assembly as this, it\r\nwould be insupportable. Your sisters are engaged, and there is not\r\nanother woman in the room whom it would not be a punishment to me to\r\nstand up with.”\r\n\r\n“I would not be so fastidious as you are,” cried Bingley, “for a\r\nkingdom! Upon my honour, I never met with so many pleasant girls in my\r\nlife as I have this evening; and there are several of them, you see,\r\nuncommonly pretty.”\r\n\r\n“_You_ are dancing with the only handsome girl in the room,” said Mr.\r\nDarcy, looking at the eldest Miss Bennet.\r\n\r\n“Oh, she is the most beautiful creature I ever beheld! But there is one\r\nof her sisters sitting down just behind you, who is very pretty, and I\r\ndare say very agreeable. Do let me ask my partner to introduce you.”\r\n\r\n[Illustration:\r\n\r\n“She is tolerable”\r\n\r\n[_Copyright 1894 by George Allen._]]\r\n\r\n“Which do you mean?” and turning round, he looked for a moment at\r\nElizabeth, till, catching her eye, he withdrew his own, and coldly said,\r\n“She is tolerable: but not handsome enough to tempt _me_; and I am in no\r\nhumour at present to give consequence to young ladies who are slighted\r\nby other men. You had better return to your partner and enjoy her\r\nsmiles, for you are wasting your time with me.”\r\n\r\nMr. Bingley followed his advice. Mr. Darcy walked off; and Elizabeth\r\nremained with no very cordial feelings towards him. She told the story,\r\nhowever, with great spirit among her friends; for she had a lively,\r\nplayful disposition, which delighted in anything ridiculous.\r\n\r\nThe evening altogether passed off pleasantly to the whole family. Mrs.\r\nBennet had seen her eldest daughter much admired by the Netherfield\r\nparty. Mr. Bingley had danced with her twice, and she had been\r\ndistinguished by his sisters. Jane was as much gratified by this as her\r\nmother could be, though in a quieter way. Elizabeth felt Jane’s\r\npleasure. Mary had heard herself mentioned to Miss Bingley as the most\r\naccomplished girl in the neighbourhood; and Catherine and Lydia had been\r\nfortunate enough to be never without partners, which was all that they\r\nhad yet learnt to care for at a ball. They returned, therefore, in good\r\nspirits to Longbourn, the village where they lived, and of which they\r\nwere the principal inhabitants. They found Mr. Bennet still up. With a\r\nbook, he was regardless of time; and on the present occasion he had a\r\ngood deal of curiosity as to the event of an evening which had raised\r\nsuch splendid expectations. He had rather hoped that all his wife’s\r\nviews on the stranger would be disappointed; but he soon found that he\r\nhad a very different story to hear.\r\n\r\n“Oh, my dear Mr. Bennet,” as she entered the room, “we have had a most\r\ndelightful evening, a most excellent ball. I wish you had been there.\r\nJane was so admired, nothing could be like it. Everybody said how well\r\nshe looked; and Mr. Bingley thought her quite beautiful, and danced with\r\nher twice. Only think of _that_, my dear: he actually danced with her\r\ntwice; and she was the only creature in the room that he asked a second\r\ntime. First of all, he asked Miss Lucas. I was so vexed to see him stand\r\nup with her; but, however, he did not admire her at all; indeed, nobody\r\ncan, you know; and he seemed quite struck with Jane as she was going\r\ndown the dance. So he inquired who she was, and got introduced, and\r\nasked her for the two next. Then, the two third he danced with Miss\r\nKing, and the two fourth with Maria Lucas, and the two fifth with Jane\r\nagain, and the two sixth with Lizzy, and the _Boulanger_----”\r\n\r\n“If he had had any compassion for _me_,” cried her husband impatiently,\r\n“he would not have danced half so much! For God’s sake, say no more of\r\nhis partners. O that he had sprained his ancle in the first dance!”\r\n\r\n“Oh, my dear,” continued Mrs. Bennet, “I am quite delighted with him. He\r\nis so excessively handsome! and his sisters are charming women. I never\r\nin my life saw anything more elegant than their dresses. I dare say the\r\nlace upon Mrs. Hurst’s gown----”\r\n\r\nHere she was interrupted again. Mr. Bennet protested against any\r\ndescription of finery. She was therefore obliged to seek another branch\r\nof the subject, and related, with much bitterness of spirit, and some\r\nexaggeration, the shocking rudeness of Mr. Darcy.\r\n\r\n“But I can assure you,” she added, “that Lizzy does not lose much by not\r\nsuiting _his_ fancy; for he is a most disagreeable, horrid man, not at\r\nall worth pleasing. So high and so conceited, that there was no enduring\r\nhim! He walked here, and he walked there, fancying himself so very\r\ngreat! Not handsome enough to dance with! I wish you had been there, my\r\ndear, to have given him one of your set-downs. I quite detest the man.”\r\n\r\n\r\n\r\n\r\n[Illustration]\r\n\r\n\r\n\r\n\r\nCHAPTER IV.\r\n\r\n\r\n[Illustration]\r\n\r\nWhen Jane and Elizabeth were alone, the former, who had been cautious in\r\nher praise of Mr. Bingley before, expressed to her sister how very much\r\nshe admired him.\r\n\r\n“He is just what a young-man ought to be,” said she, “sensible,\r\ngood-humoured, lively; and I never saw such happy manners! so much ease,\r\nwith such perfect good breeding!”\r\n\r\n“He is also handsome,” replied Elizabeth, “which a young man ought\r\nlikewise to be if he possibly can. His character is thereby complete.”\r\n\r\n“I was very much flattered by his asking me to dance a second time. I\r\ndid not expect such a compliment.”\r\n\r\n“Did not you? _I_ did for you. But that is one great difference between\r\nus. Compliments always take _you_ by surprise, and _me_ never. What\r\ncould be more natural than his asking you again? He could not help\r\nseeing that you were about five times as pretty as every other woman in\r\nthe room. No thanks to his gallantry for that. Well, he certainly is\r\nvery agreeable, and I give you leave to like him. You have liked many a\r\nstupider person.”\r\n\r\n“Dear Lizzy!”\r\n\r\n“Oh, you are a great deal too apt, you know, to like people in general.\r\nYou never see a fault in anybody. All the world are good and agreeable\r\nin your eyes. I never heard you speak ill of a human being in my life.”\r\n\r\n“I would wish not to be hasty in censuring anyone; but I always speak\r\nwhat I think.”\r\n\r\n“I know you do: and it is _that_ which makes the wonder. With _your_\r\ngood sense, to be so honestly blind to the follies and nonsense of\r\nothers! Affectation of candour is common enough; one meets with it\r\neverywhere. But to be candid without ostentation or design,--to take the\r\ngood of everybody’s character and make it still better, and say nothing\r\nof the bad,--belongs to you alone. And so, you like this man’s sisters,\r\ntoo, do you? Their manners are not equal to his.”\r\n\r\n“Certainly not, at first; but they are very pleasing women when you\r\nconverse with them. Miss Bingley is to live with her brother, and keep\r\nhis house; and I am much mistaken if we shall not find a very charming\r\nneighbour in her.”\r\n\r\nElizabeth listened in silence, but was not convinced: their behaviour at\r\nthe assembly had not been calculated to please in general; and with more\r\nquickness of observation and less pliancy of temper than her sister, and\r\nwith a judgment, too, unassailed by any attention to herself, she was\r\nvery little disposed to approve them. They were, in fact, very fine\r\nladies; not deficient in good-humour when they were pleased, nor in the\r\npower of being agreeable where they chose it; but proud and conceited.\r\nThey were rather handsome; had been educated in one of the first private\r\nseminaries in town; had a fortune of twenty thousand pounds; were in the\r\nhabit of spending more than they ought, and of associating with people\r\nof rank; and were, therefore, in every respect entitled to think well of\r\nthemselves and meanly of others. They were of a respectable family in\r\nthe north of England; a circumstance more deeply impressed on their\r\nmemories than that their brother’s fortune and their own had been\r\nacquired by trade.\r\n\r\nMr. Bingley inherited property to the amount of nearly a hundred\r\nthousand pounds from his father, who had intended to purchase an estate,\r\nbut did not live to do it. Mr. Bingley intended it likewise, and\r\nsometimes made choice of his county; but, as he was now provided with a\r\ngood house and the liberty of a manor, it was doubtful to many of those\r\nwho best knew the easiness of his temper, whether he might not spend the\r\nremainder of his days at Netherfield, and leave the next generation to\r\npurchase.\r\n\r\nHis sisters were very anxious for his having an estate of his own; but\r\nthough he was now established only as a tenant, Miss Bingley was by no\r\nmeans unwilling to preside at his table; nor was Mrs. Hurst, who had\r\nmarried a man of more fashion than fortune, less disposed to consider\r\nhis house as her home when it suited her. Mr. Bingley had not been of\r\nage two years when he was tempted, by an accidental recommendation, to\r\nlook at Netherfield House. He did look at it, and into it, for half an\r\nhour; was pleased with the situation and the principal rooms, satisfied\r\nwith what the owner said in its praise, and took it immediately.\r\n\r\nBetween him and Darcy there was a very steady friendship, in spite of a\r\ngreat opposition of character. Bingley was endeared to Darcy by the\r\neasiness, openness, and ductility of his temper, though no disposition\r\ncould offer a greater contrast to his own, and though with his own he\r\nnever appeared dissatisfied. On the strength of Darcy’s regard, Bingley\r\nhad the firmest reliance, and of his judgment the highest opinion. In\r\nunderstanding, Darcy was the superior. Bingley was by no means\r\ndeficient; but Darcy was clever. He was at the same time haughty,\r\nreserved, and fastidious; and his manners, though well bred, were not\r\ninviting. In that respect his friend had greatly the advantage. Bingley\r\nwas sure of being liked wherever he appeared; Darcy was continually\r\ngiving offence.\r\n\r\nThe manner in which they spoke of the Meryton assembly was sufficiently\r\ncharacteristic. Bingley had never met with pleasanter people or prettier\r\ngirls in his life; everybody had been most kind and attentive to him;\r\nthere had been no formality, no stiffness; he had soon felt acquainted\r\nwith all the room; and as to Miss Bennet, he could not conceive an angel\r\nmore beautiful. Darcy, on the contrary, had seen a collection of people\r\nin whom there was little beauty and no fashion, for none of whom he had\r\nfelt the smallest interest, and from none received either attention or\r\npleasure. Miss Bennet he acknowledged to be pretty; but she smiled too\r\nmuch.\r\n\r\nMrs. Hurst and her sister allowed it to be so; but still they admired\r\nher and liked her, and pronounced her to be a sweet girl, and one whom\r\nthey should not object to know more of. Miss Bennet was therefore\r\nestablished as a sweet girl; and their brother felt authorized by such\r\ncommendation to think of her as he chose.\r\n\r\n\r\n\r\n\r\n[Illustration: [_Copyright 1894 by George Allen._]]\r\n\r\n\r\n\r\n\r\nCHAPTER V.\r\n\r\n\r\n[Illustration]\r\n\r\nWithin a short walk of Longbourn lived a family with whom the Bennets\r\nwere particularly intimate. Sir William Lucas had been formerly in trade\r\nin Meryton, where he had made a tolerable fortune, and risen to the\r\nhonour of knighthood by an address to the king during his mayoralty. The\r\ndistinction had, perhaps, been felt too strongly. It had given him a\r\ndisgust to his business and to his residence in a small market town;\r\nand, quitting them both, he had removed with his family to a house about\r\na mile from Meryton, denominated from that period Lucas Lodge; where he\r\ncould think with pleasure of his own importance, and, unshackled by\r\nbusiness, occupy himself solely in being civil to all the world. For,\r\nthough elated by his rank, it did not render him supercilious; on the\r\ncontrary, he was all attention to everybody. By nature inoffensive,\r\nfriendly, and obliging, his presentation at St. James’s had made him\r\ncourteous.\r\n\r\nLady Lucas was a very good kind of woman, not too clever to be a\r\nvaluable neighbour to Mrs. Bennet. They had several children. The eldest\r\nof them, a sensible, intelligent young woman, about twenty-seven, was\r\nElizabeth’s intimate friend.\r\n\r\nThat the Miss Lucases and the Miss Bennets should meet to talk over a\r\nball was absolutely necessary; and the morning after the assembly\r\nbrought the former to Longbourn to hear and to communicate.\r\n\r\n“_You_ began the evening well, Charlotte,” said Mrs. Bennet, with civil\r\nself-command, to Miss Lucas. “_You_ were Mr. Bingley’s first choice.”\r\n\r\n“Yes; but he seemed to like his second better.”\r\n\r\n“Oh, you mean Jane, I suppose, because he danced with her twice. To be\r\nsure that _did_ seem as if he admired her--indeed, I rather believe he\r\n_did_--I heard something about it--but I hardly know what--something\r\nabout Mr. Robinson.”\r\n\r\n“Perhaps you mean what I overheard between him and Mr. Robinson: did not\r\nI mention it to you? Mr. Robinson’s asking him how he liked our Meryton\r\nassemblies, and whether he did not think there were a great many pretty\r\nwomen in the room, and _which_ he thought the prettiest? and his\r\nanswering immediately to the last question, ‘Oh, the eldest Miss Bennet,\r\nbeyond a doubt: there cannot be two opinions on that point.’”\r\n\r\n“Upon my word! Well, that was very decided, indeed--that does seem as\r\nif--but, however, it may all come to nothing, you know.”\r\n\r\n“_My_ overhearings were more to the purpose than _yours_, Eliza,” said\r\nCharlotte. “Mr. Darcy is not so well worth listening to as his friend,\r\nis he? Poor Eliza! to be only just _tolerable_.”\r\n\r\n“I beg you will not put it into Lizzy’s head to be vexed by his\r\nill-treatment, for he is such a disagreeable man that it would be quite\r\na misfortune to be liked by him. Mrs. Long told me last night that he\r\nsat close to her for half an hour without once opening his lips.”\r\n\r\n[Illustration: “Without once opening his lips”\r\n\r\n[_Copyright 1894 by George Allen._]]\r\n\r\n“Are you quite sure, ma’am? Is not there a little mistake?” said Jane.\r\n“I certainly saw Mr. Darcy speaking to her.”\r\n\r\n“Ay, because she asked him at last how he liked Netherfield, and he\r\ncould not help answering her; but she said he seemed very angry at being\r\nspoke to.”\r\n\r\n“Miss Bingley told me,” said Jane, “that he never speaks much unless\r\namong his intimate acquaintance. With _them_ he is remarkably\r\nagreeable.”\r\n\r\n“I do not believe a word of it, my dear. If he had been so very\r\nagreeable, he would have talked to Mrs. Long. But I can guess how it\r\nwas; everybody says that he is eat up with pride, and I dare say he had\r\nheard somehow that Mrs. Long does not keep a carriage, and had to come\r\nto the ball in a hack chaise.”\r\n\r\n“I do not mind his not talking to Mrs. Long,” said Miss Lucas, “but I\r\nwish he had danced with Eliza.”\r\n\r\n“Another time, Lizzy,” said her mother, “I would not dance with _him_,\r\nif I were you.”\r\n\r\n“I believe, ma’am, I may safely promise you _never_ to dance with him.”\r\n\r\n“His pride,” said Miss Lucas, “does not offend _me_ so much as pride\r\noften does, because there is an excuse for it. One cannot wonder that so\r\nvery fine a young man, with family, fortune, everything in his favour,\r\nshould think highly of himself. If I may so express it, he has a _right_\r\nto be proud.”\r\n\r\n“That is very true,” replied Elizabeth, “and I could easily forgive\r\n_his_ pride, if he had not mortified _mine_.”\r\n\r\n“Pride,” observed Mary, who piqued herself upon the solidity of her\r\nreflections, “is a very common failing, I believe. By all that I have\r\never read, I am convinced that it is very common indeed; that human\r\nnature is particularly prone to it, and that there are very few of us\r\nwho do not cherish a feeling of self-complacency on the score of some\r\nquality or other, real or imaginary. Vanity and pride are different\r\nthings, though the words are often used synonymously. A person may be\r\nproud without being vain. Pride relates more to our opinion of\r\nourselves; vanity to what we would have others think of us.”\r\n\r\n“If I were as rich as Mr. Darcy,” cried a young Lucas, who came with his\r\nsisters, “I should not care how proud I was. I would keep a pack of\r\nfoxhounds, and drink a bottle of wine every day.”\r\n\r\n“Then you would drink a great deal more than you ought,” said Mrs.\r\nBennet; “and if I were to see you at it, I should take away your bottle\r\ndirectly.”\r\n\r\nThe boy protested that she should not; she continued to declare that she\r\nwould; and the argument ended only with the visit.\r\n\r\n[Illustration]\r\n\r\n\r\n\r\n\r\n[Illustration]\r\n\r\n\r\n\r\n\r\nCHAPTER VI.\r\n\r\n\r\n[Illustration]\r\n\r\nThe ladies of Longbourn soon waited on those of Netherfield. The visit\r\nwas returned in due form. Miss Bennet’s pleasing manners grew on the\r\ngood-will of Mrs. Hurst and Miss Bingley; and though the mother was\r\nfound to be intolerable, and the younger sisters not worth speaking to,\r\na wish of being better acquainted with _them_ was expressed towards the\r\ntwo eldest. By Jane this attention was received with the greatest\r\npleasure; but Elizabeth still saw superciliousness in their treatment of\r\neverybody, hardly excepting even her sister, and could not like them;\r\nthough their kindness to Jane, such as it was, had a value, as arising,\r\nin all probability, from the influence of their brother’s admiration. It\r\nwas generally evident, whenever they met, that he _did_ admire her; and\r\nto _her_ it was equally evident that Jane was yielding to the preference\r\nwhich she had begun to entertain for him from the first, and was in a\r\nway to be very much in love; but she considered with pleasure that it\r\nwas not likely to be discovered by the world in general, since Jane\r\nunited with great strength of feeling, a composure of temper and an\r\nuniform cheerfulness of manner, which would guard her from the\r\nsuspicions of the impertinent. She mentioned this to her friend, Miss\r\nLucas.\r\n\r\n“It may, perhaps, be pleasant,” replied Charlotte, “to be able to impose\r\non the public in such a case; but it is sometimes a disadvantage to be\r\nso very guarded. If a woman conceals her affection with the same skill\r\nfrom the object of it, she may lose the opportunity of fixing him; and\r\nit will then be but poor consolation to believe the world equally in the\r\ndark. There is so much of gratitude or vanity in almost every\r\nattachment, that it is not safe to leave any to itself. We can all\r\n_begin_ freely--a slight preference is natural enough; but there are\r\nvery few of us who have heart enough to be really in love without\r\nencouragement. In nine cases out of ten, a woman had better show _more_\r\naffection than she feels. Bingley likes your sister undoubtedly; but he\r\nmay never do more than like her, if she does not help him on.”\r\n\r\n“But she does help him on, as much as her nature will allow. If _I_ can\r\nperceive her regard for him, he must be a simpleton indeed not to\r\ndiscover it too.”\r\n\r\n“Remember, Eliza, that he does not know Jane’s disposition as you do.”\r\n\r\n“But if a woman is partial to a man, and does not endeavor to conceal\r\nit, he must find it out.”\r\n\r\n“Perhaps he must, if he sees enough of her. But though Bingley and Jane\r\nmeet tolerably often, it is never for many hours together; and as they\r\nalways see each other in large mixed parties, it is impossible that\r\nevery moment should be employed in conversing together. Jane should\r\ntherefore make the most of every half hour in which she can command his\r\nattention. When she is secure of him, there will be leisure for falling\r\nin love as much as she chooses.”\r\n\r\n“Your plan is a good one,” replied Elizabeth, “where nothing is in\r\nquestion but the desire of being well married; and if I were determined\r\nto get a rich husband, or any husband, I dare say I should adopt it. But\r\nthese are not Jane’s feelings; she is not acting by design. As yet she\r\ncannot even be certain of the degree of her own regard, nor of its\r\nreasonableness. She has known him only a fortnight. She danced four\r\ndances with him at Meryton; she saw him one morning at his own house,\r\nand has since dined in company with him four times. This is not quite\r\nenough to make her understand his character.”\r\n\r\n“Not as you represent it. Had she merely _dined_ with him, she might\r\nonly have discovered whether he had a good appetite; but you must\r\nremember that four evenings have been also spent together--and four\r\nevenings may do a great deal.”\r\n\r\n“Yes: these four evenings have enabled them to ascertain that they both\r\nlike Vingt-un better than Commerce, but with respect to any other\r\nleading characteristic, I do not imagine that much has been unfolded.”\r\n\r\n“Well,” said Charlotte, “I wish Jane success with all my heart; and if\r\nshe were married to him to-morrow, I should think she had as good a\r\nchance of happiness as if she were to be studying his character for a\r\ntwelvemonth. Happiness in marriage is entirely a matter of chance. If\r\nthe dispositions of the parties are ever so well known to each other, or\r\never so similar beforehand, it does not advance their felicity in the\r\nleast. They always continue to grow sufficiently unlike afterwards to\r\nhave their share of vexation; and it is better to know as little as\r\npossible of the defects of the person with whom you are to pass your\r\nlife.”\r\n\r\n“You make me laugh, Charlotte; but it is not sound. You know it is not\r\nsound, and that you would never act in this way yourself.”\r\n\r\nOccupied in observing Mr. Bingley’s attention to her sister, Elizabeth\r\nwas far from suspecting that she was herself becoming an object of some\r\ninterest in the eyes of his friend. Mr. Darcy had at first scarcely\r\nallowed her to be pretty: he had looked at her without admiration at the\r\nball; and when they next met, he looked at her only to criticise. But no\r\nsooner had he made it clear to himself and his friends that she had\r\nhardly a good feature in her face, than he began to find it was rendered\r\nuncommonly intelligent by the beautiful expression of her dark eyes. To\r\nthis discovery succeeded some others equally mortifying. Though he had\r\ndetected with a critical eye more than one failure of perfect symmetry\r\nin her form, he was forced to acknowledge her figure to be light and\r\npleasing; and in spite of his asserting that her manners were not those\r\nof the fashionable world, he was caught by their easy playfulness. Of\r\nthis she was perfectly unaware: to her he was only the man who made\r\nhimself agreeable nowhere, and who had not thought her handsome enough\r\nto dance with.\r\n\r\nHe began to wish to know more of her; and, as a step towards conversing\r\nwith her himself, attended to her conversation with others. His doing so\r\ndrew her notice. It was at Sir William Lucas’s, where a large party were\r\nassembled.\r\n\r\n“What does Mr. Darcy mean,” said she to Charlotte, “by listening to my\r\nconversation with Colonel Forster?”\r\n\r\n“That is a question which Mr. Darcy only can answer.”\r\n\r\n“But if he does it any more, I shall certainly let him know that I see\r\nwhat he is about. He has a very satirical eye, and if I do not begin by\r\nbeing impertinent myself, I shall soon grow afraid of him.”\r\n\r\n[Illustration: “The entreaties of several” [_Copyright 1894 by George\r\nAllen._]]\r\n\r\nOn his approaching them soon afterwards, though without seeming to have\r\nany intention of speaking, Miss Lucas defied her friend to mention such\r\na subject to him, which immediately provoking Elizabeth to do it, she\r\nturned to him and said,--\r\n\r\n“Did not you think, Mr. Darcy, that I expressed myself uncommonly well\r\njust now, when I was teasing Colonel Forster to give us a ball at\r\nMeryton?”\r\n\r\n“With great energy; but it is a subject which always makes a lady\r\nenergetic.”\r\n\r\n“You are severe on us.”\r\n\r\n“It will be _her_ turn soon to be teased,” said Miss Lucas. “I am going\r\nto open the instrument, Eliza, and you know what follows.”\r\n\r\n“You are a very strange creature by way of a friend!--always wanting me\r\nto play and sing before anybody and everybody! If my vanity had taken a\r\nmusical turn, you would have been invaluable; but as it is, I would\r\nreally rather not sit down before those who must be in the habit of\r\nhearing the very best performers.” On Miss Lucas’s persevering, however,\r\nshe added, “Very well; if it must be so, it must.” And gravely glancing\r\nat Mr. Darcy, “There is a very fine old saying, which everybody here is\r\nof course familiar with--‘Keep your breath to cool your porridge,’--and\r\nI shall keep mine to swell my song.”\r\n\r\nHer performance was pleasing, though by no means capital. After a song\r\nor two, and before she could reply to the entreaties of several that she\r\nwould sing again, she was eagerly succeeded at the instrument by her\r\nsister Mary, who having, in consequence of being the only plain one in\r\nthe family, worked hard for knowledge and accomplishments, was always\r\nimpatient for display.\r\n\r\nMary had neither genius nor taste; and though vanity had given her\r\napplication, it had given her likewise a pedantic air and conceited\r\nmanner, which would have injured a higher degree of excellence than she\r\nhad reached. Elizabeth, easy and unaffected, had been listened to with\r\nmuch more pleasure, though not playing half so well; and Mary, at the\r\nend of a long concerto, was glad to purchase praise and gratitude by\r\nScotch and Irish airs, at the request of her younger sisters, who with\r\nsome of the Lucases, and two or three officers, joined eagerly in\r\ndancing at one end of the room.\r\n\r\nMr. Darcy stood near them in silent indignation at such a mode of\r\npassing the evening, to the exclusion of all conversation, and was too\r\nmuch engrossed by his own thoughts to perceive that Sir William Lucas\r\nwas his neighbour, till Sir William thus began:--\r\n\r\n“What a charming amusement for young people this is, Mr. Darcy! There is\r\nnothing like dancing, after all. I consider it as one of the first\r\nrefinements of polished societies.”\r\n\r\n“Certainly, sir; and it has the advantage also of being in vogue amongst\r\nthe less polished societies of the world: every savage can dance.”\r\n\r\nSir William only smiled. “Your friend performs delightfully,” he\r\ncontinued, after a pause, on seeing Bingley join the group; “and I doubt\r\nnot that you are an adept in the science yourself, Mr. Darcy.”\r\n\r\n“You saw me dance at Meryton, I believe, sir.”\r\n\r\n“Yes, indeed, and received no inconsiderable pleasure from the sight. Do\r\nyou often dance at St. James’s?”\r\n\r\n“Never, sir.”\r\n\r\n“Do you not think it would be a proper compliment to the place?”\r\n\r\n“It is a compliment which I never pay to any place if I can avoid it.”\r\n\r\n“You have a house in town, I conclude?”\r\n\r\nMr. Darcy bowed.\r\n\r\n“I had once some thoughts of fixing in town myself, for I am fond of\r\nsuperior society; but I did not feel quite certain that the air of\r\nLondon would agree with Lady Lucas.”\r\n\r\nHe paused in hopes of an answer: but his companion was not disposed to\r\nmake any; and Elizabeth at that instant moving towards them, he was\r\nstruck with the notion of doing a very gallant thing, and called out to\r\nher,--\r\n\r\n“My dear Miss Eliza, why are not you dancing? Mr. Darcy, you must allow\r\nme to present this young lady to you as a very desirable partner. You\r\ncannot refuse to dance, I am sure, when so much beauty is before you.”\r\nAnd, taking her hand, he would have given it to Mr. Darcy, who, though\r\nextremely surprised, was not unwilling to receive it, when she instantly\r\ndrew back, and said with some discomposure to Sir William,--\r\n\r\n“Indeed, sir, I have not the least intention of dancing. I entreat you\r\nnot to suppose that I moved this way in order to beg for a partner.”\r\n\r\nMr. Darcy, with grave propriety, requested to be allowed the honour of\r\nher hand, but in vain. Elizabeth was determined; nor did Sir William at\r\nall shake her purpose by his attempt at persuasion.\r\n\r\n“You excel so much in the dance, Miss Eliza, that it is cruel to deny me\r\nthe happiness of seeing you; and though this gentleman dislikes the\r\namusement in general, he can have no objection, I am sure, to oblige us\r\nfor one half hour.”\r\n\r\n“Mr. Darcy is all politeness,” said Elizabeth, smiling.\r\n\r\n“He is, indeed: but considering the inducement, my dear Miss Eliza, we\r\ncannot wonder at his complaisance; for who would object to such a\r\npartner?”\r\n\r\nElizabeth looked archly, and turned away. Her resistance had not injured\r\nher with the gentleman, and he was thinking of her with some\r\ncomplacency, when thus accosted by Miss Bingley,--\r\n\r\n“I can guess the subject of your reverie.”\r\n\r\n“I should imagine not.”\r\n\r\n“You are considering how insupportable it would be to pass many\r\nevenings in this manner,--in such society; and, indeed, I am quite of\r\nyour opinion. I was never more annoyed! The insipidity, and yet the\r\nnoise--the nothingness, and yet the self-importance, of all these\r\npeople! What would I give to hear your strictures on them!”\r\n\r\n“Your conjecture is totally wrong, I assure you. My mind was more\r\nagreeably engaged. I have been meditating on the very great pleasure\r\nwhich a pair of fine eyes in the face of a pretty woman can bestow.”\r\n\r\nMiss Bingley immediately fixed her eyes on his face, and desired he\r\nwould tell her what lady had the credit of inspiring such reflections.\r\nMr. Darcy replied, with great intrepidity,--\r\n\r\n“Miss Elizabeth Bennet.”\r\n\r\n“Miss Elizabeth Bennet!” repeated Miss Bingley. “I am all astonishment.\r\nHow long has she been such a favourite? and pray when am I to wish you\r\njoy?”\r\n\r\n“That is exactly the question which I expected you to ask. A lady’s\r\nimagination is very rapid; it jumps from admiration to love, from love\r\nto matrimony, in a moment. I knew you would be wishing me joy.”\r\n\r\n“Nay, if you are so serious about it, I shall consider the matter as\r\nabsolutely settled. You will have a charming mother-in-law, indeed, and\r\nof course she will be always at Pemberley with you.”\r\n\r\nHe listened to her with perfect indifference, while she chose to\r\nentertain herself in this manner; and as his composure convinced her\r\nthat all was safe, her wit flowed along.\r\n\r\n\r\n\r\n\r\n[Illustration:\r\n\r\n “A note for Miss Bennet”\r\n\r\n[_Copyright 1894 by George Allen._]]\r\n\r\n\r\n\r\n\r\nCHAPTER VII.\r\n\r\n\r\n[Illustration]\r\n\r\nMr. Bennet’s property consisted almost entirely in an estate of two\r\nthousand a year, which, unfortunately for his daughters, was entailed,\r\nin default of heirs male, on a distant relation; and their mother’s\r\nfortune, though ample for her situation in life, could but ill supply\r\nthe deficiency of his. Her father had been an attorney in Meryton, and\r\nhad left her four thousand pounds.\r\n\r\nShe had a sister married to a Mr. Philips, who had been a clerk to their\r\nfather and succeeded him in the business, and a brother settled in\r\nLondon in a respectable line of trade.\r\n\r\nThe village of Longbourn was only one mile from Meryton; a most\r\nconvenient distance for the young ladies, who were usually tempted\r\nthither three or four times a week, to pay their duty to their aunt, and\r\nto a milliner’s shop just over the way. The two youngest of the family,\r\nCatherine and Lydia, were particularly frequent in these attentions:\r\ntheir minds were more vacant than their sisters’, and when nothing\r\nbetter offered, a walk to Meryton was necessary to amuse their morning\r\nhours and furnish conversation for the evening; and, however bare of\r\nnews the country in general might be, they always contrived to learn\r\nsome from their aunt. At present, indeed, they were well supplied both\r\nwith news and happiness by the recent arrival of a militia regiment in\r\nthe neighbourhood; it was to remain the whole winter, and Meryton was\r\nthe head-quarters.\r\n\r\nTheir visits to Mrs. Philips were now productive of the most interesting\r\nintelligence. Every day added something to their knowledge of the\r\nofficers’ names and connections. Their lodgings were not long a secret,\r\nand at length they began to know the officers themselves. Mr. Philips\r\nvisited them all, and this opened to his nieces a source of felicity\r\nunknown before. They could talk of nothing but officers; and Mr.\r\nBingley’s large fortune, the mention of which gave animation to their\r\nmother, was worthless in their eyes when opposed to the regimentals of\r\nan ensign.\r\n\r\nAfter listening one morning to their effusions on this subject, Mr.\r\nBennet coolly observed,--\r\n\r\n“From all that I can collect by your manner of talking, you must be two\r\nof the silliest girls in the country. I have suspected it some time, but\r\nI am now convinced.”\r\n\r\nCatherine was disconcerted, and made no answer; but Lydia, with perfect\r\nindifference, continued to express her admiration of Captain Carter, and\r\nher hope of seeing him in the course of the day, as he was going the\r\nnext morning to London.\r\n\r\n“I am astonished, my dear,” said Mrs. Bennet, “that you should be so\r\nready to think your own children silly. If I wished to think slightingly\r\nof anybody’s children, it should not be of my own, however.”\r\n\r\n“If my children are silly, I must hope to be always sensible of it.”\r\n\r\n“Yes; but as it happens, they are all of them very clever.”\r\n\r\n“This is the only point, I flatter myself, on which we do not agree. I\r\nhad hoped that our sentiments coincided in every particular, but I must\r\nso far differ from you as to think our two youngest daughters uncommonly\r\nfoolish.”\r\n\r\n“My dear Mr. Bennet, you must not expect such girls to have the sense of\r\ntheir father and mother. When they get to our age, I dare say they will\r\nnot think about officers any more than we do. I remember the time when I\r\nliked a red coat myself very well--and, indeed, so I do still at my\r\nheart; and if a smart young colonel, with five or six thousand a year,\r\nshould want one of my girls, I shall not say nay to him; and I thought\r\nColonel Forster looked very becoming the other night at Sir William’s in\r\nhis regimentals.”\r\n\r\n“Mamma,” cried Lydia, “my aunt says that Colonel Forster and Captain\r\nCarter do not go so often to Miss Watson’s as they did when they first\r\ncame; she sees them now very often standing in Clarke’s library.”\r\n\r\nMrs. Bennet was prevented replying by the entrance of the footman with a\r\nnote for Miss Bennet; it came from Netherfield, and the servant waited\r\nfor an answer. Mrs. Bennet’s eyes sparkled with pleasure, and she was\r\neagerly calling out, while her daughter read,--\r\n\r\n“Well, Jane, who is it from? What is it about? What does he say? Well,\r\nJane, make haste and tell us; make haste, my love.”\r\n\r\n“It is from Miss Bingley,” said Jane, and then read it aloud.\r\n\r\n /* NIND “My dear friend, */\r\n\r\n “If you are not so compassionate as to dine to-day with Louisa and\r\n me, we shall be in danger of hating each other for the rest of our\r\n lives; for a whole day’s _tête-à-tête_ between two women can never\r\n end without a quarrel. Come as soon as you can on the receipt of\r\n this. My brother and the gentlemen are to dine with the officers.\r\n Yours ever,\r\n\r\n“CAROLINE BINGLEY.”\r\n\r\n“With the officers!” cried Lydia: “I wonder my aunt did not tell us of\r\n_that_.”\r\n\r\n“Dining out,” said Mrs. Bennet; “that is very unlucky.”\r\n\r\n“Can I have the carriage?” said Jane.\r\n\r\n“No, my dear, you had better go on horseback, because it seems likely to\r\nrain; and then you must stay all night.”\r\n\r\n“That would be a good scheme,” said Elizabeth, “if you were sure that\r\nthey would not offer to send her home.”\r\n\r\n“Oh, but the gentlemen will have Mr. Bingley’s chaise to go to Meryton;\r\nand the Hursts have no horses to theirs.”\r\n\r\n“I had much rather go in the coach.”\r\n\r\n“But, my dear, your father cannot spare the horses, I am sure. They are\r\nwanted in the farm, Mr. Bennet, are not they?”\r\n\r\n[Illustration: Cheerful prognostics]\r\n\r\n“They are wanted in the farm much oftener than I can get them.”\r\n\r\n“But if you have got them to-day,” said Elizabeth, “my mother’s purpose\r\nwill be answered.”\r\n\r\nShe did at last extort from her father an acknowledgment that the horses\r\nwere engaged; Jane was therefore obliged to go on horseback, and her\r\nmother attended her to the door with many cheerful prognostics of a bad\r\nday. Her hopes were answered; Jane had not been gone long before it\r\nrained hard. Her sisters were uneasy for her, but her mother was\r\ndelighted. The rain continued the whole evening without intermission;\r\nJane certainly could not come back.\r\n\r\n“This was a lucky idea of mine, indeed!” said Mrs. Bennet, more than\r\nonce, as if the credit of making it rain were all her own. Till the next\r\nmorning, however, she was not aware of all the felicity of her\r\ncontrivance. Breakfast was scarcely over when a servant from Netherfield\r\nbrought the following note for Elizabeth:--\r\n\r\n /* NIND “My dearest Lizzie, */\r\n\r\n “I find myself very unwell this morning, which, I suppose, is to be\r\n imputed to my getting wet through yesterday. My kind friends will\r\n not hear of my returning home till I am better. They insist also on\r\n my seeing Mr. Jones--therefore do not be alarmed if you should hear\r\n of his having been to me--and, excepting a sore throat and a\r\n headache, there is not much the matter with me.\r\n\r\n“Yours, etc.”\r\n\r\n“Well, my dear,” said Mr. Bennet, when Elizabeth had read the note\r\naloud, “if your daughter should have a dangerous fit of illness--if she\r\nshould die--it would be a comfort to know that it was all in pursuit of\r\nMr. Bingley, and under your orders.”\r\n\r\n“Oh, I am not at all afraid of her dying. People do not die of little\r\ntrifling colds. She will be taken good care of. As long as she stays\r\nthere, it is all very well. I would go and see her if I could have the\r\ncarriage.”\r\n\r\nElizabeth, feeling really anxious, determined to go to her, though the\r\ncarriage was not to be had: and as she was no horsewoman, walking was\r\nher only alternative. She declared her resolution.\r\n\r\n“How can you be so silly,” cried her mother, “as to think of such a\r\nthing, in all this dirt! You will not be fit to be seen when you get\r\nthere.”\r\n\r\n“I shall be very fit to see Jane--which is all I want.”\r\n\r\n“Is this a hint to me, Lizzy,” said her father, “to send for the\r\nhorses?”\r\n\r\n“No, indeed. I do not wish to avoid the walk. The distance is nothing,\r\nwhen one has a motive; only three miles. I shall be back by dinner.”\r\n\r\n“I admire the activity of your benevolence,” observed Mary, “but every\r\nimpulse of feeling should be guided by reason; and, in my opinion,\r\nexertion should always be in proportion to what is required.”\r\n\r\n“We will go as far as Meryton with you,” said Catherine and Lydia.\r\nElizabeth accepted their company, and the three young ladies set off\r\ntogether.\r\n\r\n“If we make haste,” said Lydia, as they walked along, “perhaps we may\r\nsee something of Captain Carter, before he goes.”\r\n\r\nIn Meryton they parted: the two youngest repaired to the lodgings of one\r\nof the officers’ wives, and Elizabeth continued her walk alone, crossing\r\nfield after field at a quick pace, jumping over stiles and springing\r\nover puddles, with impatient activity, and finding herself at last\r\nwithin view of the house, with weary ancles, dirty stockings, and a face\r\nglowing with the warmth of exercise.\r\n\r\nShe was shown into the breakfast parlour, where all but Jane were\r\nassembled, and where her appearance created a great deal of surprise.\r\nThat she should have walked three miles so early in the day in such\r\ndirty weather, and by herself, was almost incredible to Mrs. Hurst and\r\nMiss Bingley; and Elizabeth was convinced that they held her in contempt\r\nfor it. She was received, however, very politely by them; and in their\r\nbrother’s manners there was something better than politeness--there was\r\ngood-humour and kindness. Mr. Darcy said very little, and Mr. Hurst\r\nnothing at all. The former was divided between admiration of the\r\nbrilliancy which exercise had given to her complexion and doubt as to\r\nthe occasion’s justifying her coming so far alone. The latter was\r\nthinking only of his breakfast.\r\n\r\nHer inquiries after her sister were not very favourably answered. Miss\r\nBennet had slept ill, and though up, was very feverish, and not well\r\nenough to leave her room. Elizabeth was glad to be taken to her\r\nimmediately; and Jane, who had only been withheld by the fear of giving\r\nalarm or inconvenience, from expressing in her note how much she longed\r\nfor such a visit, was delighted at her entrance. She was not equal,\r\nhowever, to much conversation; and when Miss Bingley left them together,\r\ncould attempt little beside expressions of gratitude for the\r\nextraordinary kindness she was treated with. Elizabeth silently attended\r\nher.\r\n\r\nWhen breakfast was over, they were joined by the sisters; and Elizabeth\r\nbegan to like them herself, when she saw how much affection and\r\nsolicitude they showed for Jane. The apothecary came; and having\r\nexamined his patient, said, as might be supposed, that she had caught a\r\nviolent cold, and that they must endeavour to get the better of it;\r\nadvised her to return to bed, and promised her some draughts. The advice\r\nwas followed readily, for the feverish symptoms increased, and her head\r\nached acutely. Elizabeth did not quit her room for a moment, nor were\r\nthe other ladies often absent; the gentlemen being out, they had in fact\r\nnothing to do elsewhere.\r\n\r\nWhen the clock struck three, Elizabeth felt that she must go, and very\r\nunwillingly said so. Miss Bingley offered her the carriage, and she only\r\nwanted a little pressing to accept it, when Jane testified such concern\r\nat parting with her that Miss Bingley was obliged to convert the offer\r\nof the chaise into an invitation to remain at Netherfield for the\r\npresent. Elizabeth most thankfully consented, and a servant was\r\ndespatched to Longbourn, to acquaint the family with her stay, and bring\r\nback a supply of clothes.\r\n\r\n[Illustration:\r\n\r\n“The Apothecary came”\r\n]\r\n\r\n\r\n\r\n\r\n[Illustration:\r\n\r\n“covering a screen”\r\n]\r\n\r\n\r\n\r\n\r\nCHAPTER VIII.\r\n\r\n\r\n[Illustration]\r\n\r\nAt five o’clock the two ladies retired to dress, and at half-past six\r\nElizabeth was summoned to dinner. To the civil inquiries which then\r\npoured in, and amongst which she had the pleasure of distinguishing the\r\nmuch superior solicitude of Mr. Bingley, she could not make a very\r\nfavourable answer. Jane was by no means better. The sisters, on hearing\r\nthis, repeated three or four times how much they were grieved, how\r\nshocking it was to have a bad cold, and how excessively they disliked\r\nbeing ill themselves; and then thought no more of the matter: and their\r\nindifference towards Jane, when not immediately before them, restored\r\nElizabeth to the enjoyment of all her original dislike.\r\n\r\nTheir brother, indeed, was the only one of the party whom she could\r\nregard with any complacency. His anxiety for Jane was evident, and his\r\nattentions to herself most pleasing; and they prevented her feeling\r\nherself so much an intruder as she believed she was considered by the\r\nothers. She had very little notice from any but him. Miss Bingley was\r\nengrossed by Mr. Darcy, her sister scarcely less so; and as for Mr.\r\nHurst, by whom Elizabeth sat, he was an indolent man, who lived only to\r\neat, drink, and play at cards, who, when he found her prefer a plain\r\ndish to a ragout, had nothing to say to her.\r\n\r\nWhen dinner was over, she returned directly to Jane, and Miss Bingley\r\nbegan abusing her as soon as she was out of the room. Her manners were\r\npronounced to be very bad indeed,--a mixture of pride and impertinence:\r\nshe had no conversation, no style, no taste, no beauty. Mrs. Hurst\r\nthought the same, and added,--\r\n\r\n“She has nothing, in short, to recommend her, but being an excellent\r\nwalker. I shall never forget her appearance this morning. She really\r\nlooked almost wild.”\r\n\r\n“She did indeed, Louisa. I could hardly keep my countenance. Very\r\nnonsensical to come at all! Why must _she_ be scampering about the\r\ncountry, because her sister had a cold? Her hair so untidy, so blowzy!”\r\n\r\n“Yes, and her petticoat; I hope you saw her petticoat, six inches deep\r\nin mud, I am absolutely certain, and the gown which had been let down to\r\nhide it not doing its office.”\r\n\r\n“Your picture may be very exact, Louisa,” said Bingley; “but this was\r\nall lost upon me. I thought Miss Elizabeth Bennet looked remarkably well\r\nwhen she came into the room this morning. Her dirty petticoat quite\r\nescaped my notice.”\r\n\r\n“_You_ observed it, Mr. Darcy, I am sure,” said Miss Bingley; “and I am\r\ninclined to think that you would not wish to see _your sister_ make such\r\nan exhibition.”\r\n\r\n“Certainly not.”\r\n\r\n“To walk three miles, or four miles, or five miles, or whatever it is,\r\nabove her ancles in dirt, and alone, quite alone! what could she mean by\r\nit? It seems to me to show an abominable sort of conceited independence,\r\na most country-town indifference to decorum.”\r\n\r\n“It shows an affection for her sister that is very pleasing,” said\r\nBingley.\r\n\r\n“I am afraid, Mr. Darcy,” observed Miss Bingley, in a half whisper,\r\n“that this adventure has rather affected your admiration of her fine\r\neyes.”\r\n\r\n“Not at all,” he replied: “they were brightened by the exercise.” A\r\nshort pause followed this speech, and Mrs. Hurst began again,--\r\n\r\n“I have an excessive regard for Jane Bennet,--she is really a very sweet\r\ngirl,--and I wish with all my heart she were well settled. But with such\r\na father and mother, and such low connections, I am afraid there is no\r\nchance of it.”\r\n\r\n“I think I have heard you say that their uncle is an attorney in\r\nMeryton?”\r\n\r\n“Yes; and they have another, who lives somewhere near Cheapside.”\r\n\r\n“That is capital,” added her sister; and they both laughed heartily.\r\n\r\n“If they had uncles enough to fill _all_ Cheapside,” cried Bingley, “it\r\nwould not make them one jot less agreeable.”\r\n\r\n“But it must very materially lessen their chance of marrying men of any\r\nconsideration in the world,” replied Darcy.\r\n\r\nTo this speech Bingley made no answer; but his sisters gave it their\r\nhearty assent, and indulged their mirth for some time at the expense of\r\ntheir dear friend’s vulgar relations.\r\n\r\nWith a renewal of tenderness, however, they repaired to her room on\r\nleaving the dining-parlour, and sat with her till summoned to coffee.\r\nShe was still very poorly, and Elizabeth would not quit her at all, till\r\nlate in the evening, when she had the comfort of seeing her asleep, and\r\nwhen it appeared to her rather right than pleasant that she should go\r\ndown stairs herself. On entering the drawing-room, she found the whole\r\nparty at loo, and was immediately invited to join them; but suspecting\r\nthem to be playing high, she declined it, and making her sister the\r\nexcuse, said she would amuse herself, for the short time she could stay\r\nbelow, with a book. Mr. Hurst looked at her with astonishment.\r\n\r\n“Do you prefer reading to cards?” said he; “that is rather singular.”\r\n\r\n“Miss Eliza Bennet,” said Miss Bingley, “despises cards. She is a great\r\nreader, and has no pleasure in anything else.”\r\n\r\n“I deserve neither such praise nor such censure,” cried Elizabeth; “I\r\nam _not_ a great reader, and I have pleasure in many things.”\r\n\r\n“In nursing your sister I am sure you have pleasure,” said Bingley; “and\r\nI hope it will soon be increased by seeing her quite well.”\r\n\r\nElizabeth thanked him from her heart, and then walked towards a table\r\nwhere a few books were lying. He immediately offered to fetch her\r\nothers; all that his library afforded.\r\n\r\n“And I wish my collection were larger for your benefit and my own\r\ncredit; but I am an idle fellow; and though I have not many, I have more\r\nthan I ever looked into.”\r\n\r\nElizabeth assured him that she could suit herself perfectly with those\r\nin the room.\r\n\r\n“I am astonished,” said Miss Bingley, “that my father should have left\r\nso small a collection of books. What a delightful library you have at\r\nPemberley, Mr. Darcy!”\r\n\r\n“It ought to be good,” he replied: “it has been the work of many\r\ngenerations.”\r\n\r\n“And then you have added so much to it yourself--you are always buying\r\nbooks.”\r\n\r\n“I cannot comprehend the neglect of a family library in such days as\r\nthese.”\r\n\r\n“Neglect! I am sure you neglect nothing that can add to the beauties of\r\nthat noble place. Charles, when you build _your_ house, I wish it may be\r\nhalf as delightful as Pemberley.”\r\n\r\n“I wish it may.”\r\n\r\n“But I would really advise you to make your purchase in that\r\nneighbourhood, and take Pemberley for a kind of model. There is not a\r\nfiner county in England than Derbyshire.”\r\n\r\n“With all my heart: I will buy Pemberley itself, if Darcy will sell it.”\r\n\r\n“I am talking of possibilities, Charles.”\r\n\r\n“Upon my word, Caroline, I should think it more possible to get\r\nPemberley by purchase than by imitation.”\r\n\r\nElizabeth was so much caught by what passed, as to leave her very little\r\nattention for her book; and, soon laying it wholly aside, she drew near\r\nthe card-table, and stationed herself between Mr. Bingley and his eldest\r\nsister, to observe the game.\r\n\r\n“Is Miss Darcy much grown since the spring?” said Miss Bingley: “will\r\nshe be as tall as I am?”\r\n\r\n“I think she will. She is now about Miss Elizabeth Bennet’s height, or\r\nrather taller.”\r\n\r\n“How I long to see her again! I never met with anybody who delighted me\r\nso much. Such a countenance, such manners, and so extremely accomplished\r\nfor her age! Her performance on the pianoforte is exquisite.”\r\n\r\n“It is amazing to me,” said Bingley, “how young ladies can have patience\r\nto be so very accomplished as they all are.”\r\n\r\n“All young ladies accomplished! My dear Charles, what do you mean?”\r\n\r\n“Yes, all of them, I think. They all paint tables, cover screens, and\r\nnet purses. I scarcely know any one who cannot do all this; and I am\r\nsure I never heard a young lady spoken of for the first time, without\r\nbeing informed that she was very accomplished.”\r\n\r\n“Your list of the common extent of accomplishments,” said Darcy, “has\r\ntoo much truth. The word is applied to many a woman who deserves it no\r\notherwise than by netting a purse or covering a screen; but I am very\r\nfar from agreeing with you in your estimation of ladies in general. I\r\ncannot boast of knowing more than half-a-dozen in the whole range of my\r\nacquaintance that are really accomplished.”\r\n\r\n“Nor I, I am sure,” said Miss Bingley.\r\n\r\n“Then,” observed Elizabeth, “you must comprehend a great deal in your\r\nidea of an accomplished woman.”\r\n\r\n“Yes; I do comprehend a great deal in it.”\r\n\r\n“Oh, certainly,” cried his faithful assistant, “no one can be really\r\nesteemed accomplished who does not greatly surpass what is usually met\r\nwith. A woman must have a thorough knowledge of music, singing, drawing,\r\ndancing, and the modern languages, to deserve the word; and, besides all\r\nthis, she must possess a certain something in her air and manner of\r\nwalking, the tone of her voice, her address and expressions, or the word\r\nwill be but half deserved.”\r\n\r\n“All this she must possess,” added Darcy; “and to all she must yet add\r\nsomething more substantial in the improvement of her mind by extensive\r\nreading.”\r\n\r\n“I am no longer surprised at your knowing _only_ six accomplished women.\r\nI rather wonder now at your knowing _any_.”\r\n\r\n“Are you so severe upon your own sex as to doubt the possibility of all\r\nthis?”\r\n\r\n“_I_ never saw such a woman. _I_ never saw such capacity, and taste, and\r\napplication, and elegance, as you describe, united.”\r\n\r\nMrs. Hurst and Miss Bingley both cried out against the injustice of her\r\nimplied doubt, and were both protesting that they knew many women who\r\nanswered this description, when Mr. Hurst called them to order, with\r\nbitter complaints of their inattention to what was going forward. As all\r\nconversation was thereby at an end, Elizabeth soon afterwards left the\r\nroom.\r\n\r\n“Eliza Bennet,” said Miss Bingley, when the door was closed on her, “is\r\none of those young ladies who seek to recommend themselves to the other\r\nsex by undervaluing their own; and with many men, I daresay, it\r\nsucceeds; but, in my opinion, it is a paltry device, a very mean art.”\r\n\r\n“Undoubtedly,” replied Darcy, to whom this remark was chiefly addressed,\r\n“there is meanness in _all_ the arts which ladies sometimes condescend\r\nto employ for captivation. Whatever bears affinity to cunning is\r\ndespicable.”\r\n\r\nMiss Bingley was not so entirely satisfied with this reply as to\r\ncontinue the subject.\r\n\r\nElizabeth joined them again only to say that her sister was worse, and\r\nthat she could not leave her. Bingley urged Mr. Jones’s being sent for\r\nimmediately; while his sisters, convinced that no country advice could\r\nbe of any service, recommended an express to town for one of the most\r\neminent physicians. This she would not hear of; but she was not so\r\nunwilling to comply with their brother’s proposal; and it was settled\r\nthat Mr. Jones should be sent for early in the morning, if Miss Bennet\r\nwere not decidedly better. Bingley was quite uncomfortable; his sisters\r\ndeclared that they were miserable. They solaced their wretchedness,\r\nhowever, by duets after supper; while he could find no better relief to\r\nhis feelings than by giving his housekeeper directions that every\r\npossible attention might be paid to the sick lady and her sister.\r\n\r\n\r\n\r\n\r\n[Illustration:\r\n\r\nM^{rs} Bennet and her two youngest girls\r\n\r\n[_Copyright 1894 by George Allen._]]\r\n\r\n\r\n\r\n\r\nCHAPTER IX.\r\n\r\n\r\n[Illustration]\r\n\r\nElizabeth passed the chief of the night in her sister’s room, and in the\r\nmorning had the pleasure of being able to send a tolerable answer to the\r\ninquiries which she very early received from Mr. Bingley by a housemaid,\r\nand some time afterwards from the two elegant ladies who waited on his\r\nsisters. In spite of this amendment, however, she requested to have a\r\nnote sent to Longbourn, desiring her mother to visit Jane, and form her\r\nown judgment of her situation. The note was immediately despatched, and\r\nits contents as quickly complied with. Mrs. Bennet, accompanied by her\r\ntwo youngest girls, reached Netherfield soon after the family breakfast.\r\n\r\nHad she found Jane in any apparent danger, Mrs. Bennet would have been\r\nvery miserable; but being satisfied on seeing her that her illness was\r\nnot alarming, she had no wish of her recovering immediately, as her\r\nrestoration to health would probably remove her from Netherfield. She\r\nwould not listen, therefore, to her daughter’s proposal of being carried\r\nhome; neither did the apothecary, who arrived about the same time, think\r\nit at all advisable. After sitting a little while with Jane, on Miss\r\nBingley’s appearance and invitation, the mother and three daughters all\r\nattended her into the breakfast parlour. Bingley met them with hopes\r\nthat Mrs. Bennet had not found Miss Bennet worse than she expected.\r\n\r\n“Indeed I have, sir,” was her answer. “She is a great deal too ill to be\r\nmoved. Mr. Jones says we must not think of moving her. We must trespass\r\na little longer on your kindness.”\r\n\r\n“Removed!” cried Bingley. “It must not be thought of. My sister, I am\r\nsure, will not hear of her removal.”\r\n\r\n“You may depend upon it, madam,” said Miss Bingley, with cold civility,\r\n“that Miss Bennet shall receive every possible attention while she\r\nremains with us.”\r\n\r\nMrs. Bennet was profuse in her acknowledgments.\r\n\r\n“I am sure,” she added, “if it was not for such good friends, I do not\r\nknow what would become of her, for she is very ill indeed, and suffers a\r\nvast deal, though with the greatest patience in the world, which is\r\nalways the way with her, for she has, without exception, the sweetest\r\ntemper I ever met with. I often tell my other girls they are nothing to\r\n_her_. You have a sweet room here, Mr. Bingley, and a charming prospect\r\nover that gravel walk. I do not know a place in the country that is\r\nequal to Netherfield. You will not think of quitting it in a hurry, I\r\nhope, though you have but a short lease.”\r\n\r\n“Whatever I do is done in a hurry,” replied he; “and therefore if I\r\nshould resolve to quit Netherfield, I should probably be off in five\r\nminutes. At present, however, I consider myself as quite fixed here.”\r\n\r\n“That is exactly what I should have supposed of you,” said Elizabeth.\r\n\r\n“You begin to comprehend me, do you?” cried he, turning towards her.\r\n\r\n“Oh yes--I understand you perfectly.”\r\n\r\n“I wish I might take this for a compliment; but to be so easily seen\r\nthrough, I am afraid, is pitiful.”\r\n\r\n“That is as it happens. It does not necessarily follow that a deep,\r\nintricate character is more or less estimable than such a one as yours.”\r\n\r\n“Lizzy,” cried her mother, “remember where you are, and do not run on in\r\nthe wild manner that you are suffered to do at home.”\r\n\r\n“I did not know before,” continued Bingley, immediately, “that you were\r\na studier of character. It must be an amusing study.”\r\n\r\n“Yes; but intricate characters are the _most_ amusing. They have at\r\nleast that advantage.”\r\n\r\n“The country,” said Darcy, “can in general supply but few subjects for\r\nsuch a study. In a country neighbourhood you move in a very confined and\r\nunvarying society.”\r\n\r\n“But people themselves alter so much, that there is something new to be\r\nobserved in them for ever.”\r\n\r\n“Yes, indeed,” cried Mrs. Bennet, offended by his manner of mentioning a\r\ncountry neighbourhood. “I assure you there is quite as much of _that_\r\ngoing on in the country as in town.”\r\n\r\nEverybody was surprised; and Darcy, after looking at her for a moment,\r\nturned silently away. Mrs. Bennet, who fancied she had gained a complete\r\nvictory over him, continued her triumph,--\r\n\r\n“I cannot see that London has any great advantage over the country, for\r\nmy part, except the shops and public places. The country is a vast deal\r\npleasanter, is not it, Mr. Bingley?”\r\n\r\n“When I am in the country,” he replied, “I never wish to leave it; and\r\nwhen I am in town, it is pretty much the same. They have each their\r\nadvantages, and I can be equally happy in either.”\r\n\r\n“Ay, that is because you have the right disposition. But that\r\ngentleman,” looking at Darcy, “seemed to think the country was nothing\r\nat all.”\r\n\r\n“Indeed, mamma, you are mistaken,” said Elizabeth, blushing for her\r\nmother. “You quite mistook Mr. Darcy. He only meant that there was not\r\nsuch a variety of people to be met with in the country as in town, which\r\nyou must acknowledge to be true.”\r\n\r\n“Certainly, my dear, nobody said there were; but as to not meeting with\r\nmany people in this neighbourhood, I believe there are few\r\nneighbourhoods larger. I know we dine with four-and-twenty families.”\r\n\r\nNothing but concern for Elizabeth could enable Bingley to keep his\r\ncountenance. His sister was less delicate, and directed her eye towards\r\nMr. Darcy with a very expressive smile. Elizabeth, for the sake of\r\nsaying something that might turn her mother’s thoughts, now asked her if\r\nCharlotte Lucas had been at Longbourn since _her_ coming away.\r\n\r\n“Yes, she called yesterday with her father. What an agreeable man Sir\r\nWilliam is, Mr. Bingley--is not he? so much the man of fashion! so\r\ngenteel and so easy! He has always something to say to everybody. _That_\r\nis my idea of good breeding; and those persons who fancy themselves very\r\nimportant and never open their mouths quite mistake the matter.”\r\n\r\n“Did Charlotte dine with you?”\r\n\r\n“No, she would go home. I fancy she was wanted about the mince-pies. For\r\nmy part, Mr. Bingley, _I_ always keep servants that can do their own\r\nwork; _my_ daughters are brought up differently. But everybody is to\r\njudge for themselves, and the Lucases are a very good sort of girls, I\r\nassure you. It is a pity they are not handsome! Not that _I_ think\r\nCharlotte so _very_ plain; but then she is our particular friend.”\r\n\r\n“She seems a very pleasant young woman,” said Bingley.\r\n\r\n“Oh dear, yes; but you must own she is very plain. Lady Lucas herself\r\nhas often said so, and envied me Jane’s beauty. I do not like to boast\r\nof my own child; but to be sure, Jane--one does not often see anybody\r\nbetter looking. It is what everybody says. I do not trust my own\r\npartiality. When she was only fifteen there was a gentleman at my\r\nbrother Gardiner’s in town so much in love with her, that my\r\nsister-in-law was sure he would make her an offer before we came away.\r\nBut, however, he did not. Perhaps he thought her too young. However, he\r\nwrote some verses on her, and very pretty they were.”\r\n\r\n“And so ended his affection,” said Elizabeth, impatiently. “There has\r\nbeen many a one, I fancy, overcome in the same way. I wonder who first\r\ndiscovered the efficacy of poetry in driving away love!”\r\n\r\n“I have been used to consider poetry as the _food_ of love,” said Darcy.\r\n\r\n“Of a fine, stout, healthy love it may. Everything nourishes what is\r\nstrong already. But if it be only a slight, thin sort of inclination, I\r\nam convinced that one good sonnet will starve it entirely away.”\r\n\r\nDarcy only smiled; and the general pause which ensued made Elizabeth\r\ntremble lest her mother should be exposing herself again. She longed to\r\nspeak, but could think of nothing to say; and after a short silence Mrs.\r\nBennet began repeating her thanks to Mr. Bingley for his kindness to\r\nJane, with an apology for troubling him also with Lizzy. Mr. Bingley was\r\nunaffectedly civil in his answer, and forced his younger sister to be\r\ncivil also, and say what the occasion required. She performed her part,\r\nindeed, without much graciousness, but Mrs. Bennet was satisfied, and\r\nsoon afterwards ordered her carriage. Upon this signal, the youngest of\r\nher daughters put herself forward. The two girls had been whispering to\r\neach other during the whole visit; and the result of it was, that the\r\nyoungest should tax Mr. Bingley with having promised on his first coming\r\ninto the country to give a ball at Netherfield.\r\n\r\nLydia was a stout, well-grown girl of fifteen, with a fine complexion\r\nand good-humoured countenance; a favourite with her mother, whose\r\naffection had brought her into public at an early age. She had high\r\nanimal spirits, and a sort of natural self-consequence, which the\r\nattentions of the officers, to whom her uncle’s good dinners and her\r\nown easy manners recommended her, had increased into assurance. She was\r\nvery equal, therefore, to address Mr. Bingley on the subject of the\r\nball, and abruptly reminded him of his promise; adding, that it would be\r\nthe most shameful thing in the world if he did not keep it. His answer\r\nto this sudden attack was delightful to her mother’s ear.\r\n\r\n“I am perfectly ready, I assure you, to keep my engagement; and, when\r\nyour sister is recovered, you shall, if you please, name the very day of\r\nthe ball. But you would not wish to be dancing while she is ill?”\r\n\r\nLydia declared herself satisfied. “Oh yes--it would be much better to\r\nwait till Jane was well; and by that time, most likely, Captain Carter\r\nwould be at Meryton again. And when you have given _your_ ball,” she\r\nadded, “I shall insist on their giving one also. I shall tell Colonel\r\nForster it will be quite a shame if he does not.”\r\n\r\nMrs. Bennet and her daughters then departed, and Elizabeth returned\r\ninstantly to Jane, leaving her own and her relations’ behaviour to the\r\nremarks of the two ladies and Mr. Darcy; the latter of whom, however,\r\ncould not be prevailed on to join in their censure of _her_, in spite of\r\nall Miss Bingley’s witticisms on _fine eyes_.\r\n\r\n\r\n\r\n\r\n[Illustration]\r\n\r\n\r\n\r\n\r\nCHAPTER X.\r\n\r\n\r\n[Illustration]\r\n\r\nThe day passed much as the day before had done. Mrs. Hurst and Miss\r\nBingley had spent some hours of the morning with the invalid, who\r\ncontinued, though slowly, to mend; and, in the evening, Elizabeth joined\r\ntheir party in the drawing-room. The loo table, however, did not appear.\r\nMr. Darcy was writing, and Miss Bingley, seated near him, was watching\r\nthe progress of his letter, and repeatedly calling off his attention by\r\nmessages to his sister. Mr. Hurst and Mr. Bingley were at piquet, and\r\nMrs. Hurst was observing their game.\r\n\r\nElizabeth took up some needlework, and was sufficiently amused in\r\nattending to what passed between Darcy and his companion. The perpetual\r\ncommendations of the lady either on his hand-writing, or on the evenness\r\nof his lines, or on the length of his letter, with the perfect unconcern\r\nwith which her praises were received, formed a curious dialogue, and was\r\nexactly in unison with her opinion of each.\r\n\r\n“How delighted Miss Darcy will be to receive such a letter!”\r\n\r\nHe made no answer.\r\n\r\n“You write uncommonly fast.”\r\n\r\n“You are mistaken. I write rather slowly.”\r\n\r\n“How many letters you must have occasion to write in the course of a\r\nyear! Letters of business, too! How odious I should think them!”\r\n\r\n“It is fortunate, then, that they fall to my lot instead of to yours.”\r\n\r\n“Pray tell your sister that I long to see her.”\r\n\r\n“I have already told her so once, by your desire.”\r\n\r\n“I am afraid you do not like your pen. Let me mend it for you. I mend\r\npens remarkably well.”\r\n\r\n“Thank you--but I always mend my own.”\r\n\r\n“How can you contrive to write so even?”\r\n\r\nHe was silent.\r\n\r\n“Tell your sister I am delighted to hear of her improvement on the harp,\r\nand pray let her know that I am quite in raptures with her beautiful\r\nlittle design for a table, and I think it infinitely superior to Miss\r\nGrantley’s.”\r\n\r\n“Will you give me leave to defer your raptures till I write again? At\r\npresent I have not room to do them justice.”\r\n\r\n“Oh, it is of no consequence. I shall see her in January. But do you\r\nalways write such charming long letters to her, Mr. Darcy?”\r\n\r\n“They are generally long; but whether always charming, it is not for me\r\nto determine.”\r\n\r\n“It is a rule with me, that a person who can write a long letter with\r\nease cannot write ill.”\r\n\r\n“That will not do for a compliment to Darcy, Caroline,” cried her\r\nbrother, “because he does _not_ write with ease. He studies too much\r\nfor words of four syllables. Do not you, Darcy?”\r\n\r\n“My style of writing is very different from yours.”\r\n\r\n“Oh,” cried Miss Bingley, “Charles writes in the most careless way\r\nimaginable. He leaves out half his words, and blots the rest.”\r\n\r\n“My ideas flow so rapidly that I have not time to express them; by which\r\nmeans my letters sometimes convey no ideas at all to my correspondents.”\r\n\r\n“Your humility, Mr. Bingley,” said Elizabeth, “must disarm reproof.”\r\n\r\n“Nothing is more deceitful,” said Darcy, “than the appearance of\r\nhumility. It is often only carelessness of opinion, and sometimes an\r\nindirect boast.”\r\n\r\n“And which of the two do you call _my_ little recent piece of modesty?”\r\n\r\n“The indirect boast; for you are really proud of your defects in\r\nwriting, because you consider them as proceeding from a rapidity of\r\nthought and carelessness of execution, which, if not estimable, you\r\nthink at least highly interesting. The power of doing anything with\r\nquickness is always much prized by the possessor, and often without any\r\nattention to the imperfection of the performance. When you told Mrs.\r\nBennet this morning, that if you ever resolved on quitting Netherfield\r\nyou should be gone in five minutes, you meant it to be a sort of\r\npanegyric, of compliment to yourself; and yet what is there so very\r\nlaudable in a precipitance which must leave very necessary business\r\nundone, and can be of no real advantage to yourself or anyone else?”\r\n\r\n“Nay,” cried Bingley, “this is too much, to remember at night all the\r\nfoolish things that were said in the morning. And yet, upon my honour, I\r\nbelieved what I said of myself to be true, and I believe it at this\r\nmoment. At least, therefore, I did not assume the character of needless\r\nprecipitance merely to show off before the ladies.”\r\n\r\n“I daresay you believed it; but I am by no means convinced that you\r\nwould be gone with such celerity. Your conduct would be quite as\r\ndependent on chance as that of any man I know; and if, as you were\r\nmounting your horse, a friend were to say, ‘Bingley, you had better stay\r\ntill next week,’ you would probably do it--you would probably not\r\ngo--and, at another word, might stay a month.”\r\n\r\n“You have only proved by this,” cried Elizabeth, “that Mr. Bingley did\r\nnot do justice to his own disposition. You have shown him off now much\r\nmore than he did himself.”\r\n\r\n“I am exceedingly gratified,” said Bingley, “by your converting what my\r\nfriend says into a compliment on the sweetness of my temper. But I am\r\nafraid you are giving it a turn which that gentleman did by no means\r\nintend; for he would certainly think the better of me if, under such a\r\ncircumstance, I were to give a flat denial, and ride off as fast as I\r\ncould.”\r\n\r\n“Would Mr. Darcy then consider the rashness of your original intention\r\nas atoned for by your obstinacy in adhering to it?”\r\n\r\n“Upon my word, I cannot exactly explain the matter--Darcy must speak for\r\nhimself.”\r\n\r\n“You expect me to account for opinions which you choose to call mine,\r\nbut which I have never acknowledged. Allowing the case, however, to\r\nstand according to your representation, you must remember, Miss Bennet,\r\nthat the friend who is supposed to desire his return to the house, and\r\nthe delay of his plan, has merely desired it, asked it without offering\r\none argument in favour of its propriety.”\r\n\r\n“To yield readily--easily--to the _persuasion_ of a friend is no merit\r\nwith you.”\r\n\r\n“To yield without conviction is no compliment to the understanding of\r\neither.”\r\n\r\n“You appear to me, Mr. Darcy, to allow nothing for the influence of\r\nfriendship and affection. A regard for the requester would often make\r\none readily yield to a request, without waiting for arguments to reason\r\none into it. I am not particularly speaking of such a case as you have\r\nsupposed about Mr. Bingley. We may as well wait, perhaps, till the\r\ncircumstance occurs, before we discuss the discretion of his behaviour\r\nthereupon. But in general and ordinary cases, between friend and friend,\r\nwhere one of them is desired by the other to change a resolution of no\r\nvery great moment, should you think ill of that person for complying\r\nwith the desire, without waiting to be argued into it?”\r\n\r\n“Will it not be advisable, before we proceed on this subject, to arrange\r\nwith rather more precision the degree of importance which is to\r\nappertain to this request, as well as the degree of intimacy subsisting\r\nbetween the parties?”\r\n\r\n“By all means,” cried Bingley; “let us hear all the particulars, not\r\nforgetting their comparative height and size, for that will have more\r\nweight in the argument, Miss Bennet, than you may be aware of. I assure\r\nyou that if Darcy were not such a great tall fellow, in comparison with\r\nmyself, I should not pay him half so much deference. I declare I do not\r\nknow a more awful object than Darcy on particular occasions, and in\r\nparticular places; at his own house especially, and of a Sunday evening,\r\nwhen he has nothing to do.”\r\n\r\nMr. Darcy smiled; but Elizabeth thought she could perceive that he was\r\nrather offended, and therefore checked her laugh. Miss Bingley warmly\r\nresented the indignity he had received, in an expostulation with her\r\nbrother for talking such nonsense.\r\n\r\n“I see your design, Bingley,” said his friend. “You dislike an argument,\r\nand want to silence this.”\r\n\r\n“Perhaps I do. Arguments are too much like disputes. If you and Miss\r\nBennet will defer yours till I am out of the room, I shall be very\r\nthankful; and then you may say whatever you like of me.”\r\n\r\n“What you ask,” said Elizabeth, “is no sacrifice on my side; and Mr.\r\nDarcy had much better finish his letter.”\r\n\r\nMr. Darcy took her advice, and did finish his letter.\r\n\r\nWhen that business was over, he applied to Miss Bingley and Elizabeth\r\nfor the indulgence of some music. Miss Bingley moved with alacrity to\r\nthe pianoforte, and after a polite request that Elizabeth would lead the\r\nway, which the other as politely and more earnestly negatived, she\r\nseated herself.\r\n\r\nMrs. Hurst sang with her sister; and while they were thus employed,\r\nElizabeth could not help observing, as she turned over some music-books\r\nthat lay on the instrument, how frequently Mr. Darcy’s eyes were fixed\r\non her. She hardly knew how to suppose that she could be an object of\r\nadmiration to so great a man, and yet that he should look at her because\r\nhe disliked her was still more strange. She could only imagine, however,\r\nat last, that she drew his notice because there was something about her\r\nmore wrong and reprehensible, according to his ideas of right, than in\r\nany other person present. The supposition did not pain her. She liked\r\nhim too little to care for his approbation.\r\n\r\nAfter playing some Italian songs, Miss Bingley varied the charm by a\r\nlively Scotch air; and soon afterwards Mr. Darcy, drawing near\r\nElizabeth, said to her,--\r\n\r\n“Do you not feel a great inclination, Miss Bennet, to seize such an\r\nopportunity of dancing a reel?”\r\n\r\nShe smiled, but made no answer. He repeated the question, with some\r\nsurprise at her silence.\r\n\r\n“Oh,” said she, “I heard you before; but I could not immediately\r\ndetermine what to say in reply. You wanted me, I know, to say ‘Yes,’\r\nthat you might have the pleasure of despising my taste; but I always\r\ndelight in overthrowing those kind of schemes, and cheating a person of\r\ntheir premeditated contempt. I have, therefore, made up my mind to tell\r\nyou that I do not want to dance a reel at all; and now despise me if you\r\ndare.”\r\n\r\n“Indeed I do not dare.”\r\n\r\nElizabeth, having rather expected to affront him, was amazed at his\r\ngallantry; but there was a mixture of sweetness and archness in her\r\nmanner which made it difficult for her to affront anybody, and Darcy had\r\nnever been so bewitched by any woman as he was by her. He really\r\nbelieved that, were it not for the inferiority of her connections, he\r\nshould be in some danger.\r\n\r\nMiss Bingley saw, or suspected, enough to be jealous; and her great\r\nanxiety for the recovery of her dear friend Jane received some\r\nassistance from her desire of getting rid of Elizabeth.\r\n\r\nShe often tried to provoke Darcy into disliking her guest, by talking of\r\ntheir supposed marriage, and planning his happiness in such an alliance.\r\n\r\n“I hope,” said she, as they were walking together in the shrubbery the\r\nnext day, “you will give your mother-in-law a few hints, when this\r\ndesirable event takes place, as to the advantage of holding her tongue;\r\nand if you can compass it, to cure the younger girls of running after\r\nthe officers. And, if I may mention so delicate a subject, endeavour to\r\ncheck that little something, bordering on conceit and impertinence,\r\nwhich your lady possesses.”\r\n\r\n[Illustration:\r\n\r\n “No, no; stay where you are”\r\n\r\n[_Copyright 1894 by George Allen._]]\r\n\r\n“Have you anything else to propose for my domestic felicity?”\r\n\r\n“Oh yes. Do let the portraits of your uncle and aunt Philips be placed\r\nin the gallery at Pemberley. Put them next to your great-uncle the\r\njudge. They are in the same profession, you know, only in different\r\nlines. As for your Elizabeth’s picture, you must not attempt to have it\r\ntaken, for what painter could do justice to those beautiful eyes?”\r\n\r\n“It would not be easy, indeed, to catch their expression; but their\r\ncolour and shape, and the eyelashes, so remarkably fine, might be\r\ncopied.”\r\n\r\nAt that moment they were met from another walk by Mrs. Hurst and\r\nElizabeth herself.\r\n\r\n“I did not know that you intended to walk,” said Miss Bingley, in some\r\nconfusion, lest they had been overheard.\r\n\r\n“You used us abominably ill,” answered Mrs. Hurst, “running away without\r\ntelling us that you were coming out.”\r\n\r\nThen taking the disengaged arm of Mr. Darcy, she left Elizabeth to walk\r\nby herself. The path just admitted three. Mr. Darcy felt their rudeness,\r\nand immediately said,--\r\n\r\n“This walk is not wide enough for our party. We had better go into the\r\navenue.”\r\n\r\nBut Elizabeth, who had not the least inclination to remain with them,\r\nlaughingly answered,--\r\n\r\n“No, no; stay where you are. You are charmingly grouped, and appear to\r\nuncommon advantage. The picturesque would be spoilt by admitting a\r\nfourth. Good-bye.”\r\n\r\nShe then ran gaily off, rejoicing, as she rambled about, in the hope of\r\nbeing at home again in a day or two. Jane was already so much recovered\r\nas to intend leaving her room for a couple of hours that evening.\r\n\r\n\r\n\r\n\r\n[Illustration:\r\n\r\n “Piling up the fire”\r\n\r\n[_Copyright 1894 by George Allen._]]\r\n\r\n\r\n\r\n\r\nCHAPTER XI.\r\n\r\n\r\n[Illustration]\r\n\r\nWhen the ladies removed after dinner Elizabeth ran up to her sister, and\r\nseeing her well guarded from cold, attended her into the drawing-room,\r\nwhere she was welcomed by her two friends with many professions of\r\npleasure; and Elizabeth had never seen them so agreeable as they were\r\nduring the hour which passed before the gentlemen appeared. Their powers\r\nof conversation were considerable. They could describe an entertainment\r\nwith accuracy, relate an anecdote with humour, and laugh at their\r\nacquaintance with spirit.\r\n\r\nBut when the gentlemen entered, Jane was no longer the first object;\r\nMiss Bingley’s eyes were instantly turned towards Darcy, and she had\r\nsomething to say to him before he had advanced many steps. He addressed\r\nhimself directly to Miss Bennet with a polite congratulation; Mr. Hurst\r\nalso made her a slight bow, and said he was “very glad;” but diffuseness\r\nand warmth remained for Bingley’s salutation. He was full of joy and\r\nattention. The first half hour was spent in piling up the fire, lest she\r\nshould suffer from the change of room; and she removed, at his desire,\r\nto the other side of the fireplace, that she might be farther from the\r\ndoor. He then sat down by her, and talked scarcely to anyone else.\r\nElizabeth, at work in the opposite corner, saw it all with great\r\ndelight.\r\n\r\nWhen tea was over Mr. Hurst reminded his sister-in-law of the\r\ncard-table--but in vain. She had obtained private intelligence that Mr.\r\nDarcy did not wish for cards, and Mr. Hurst soon found even his open\r\npetition rejected. She assured him that no one intended to play, and the\r\nsilence of the whole party on the subject seemed to justify her. Mr.\r\nHurst had, therefore, nothing to do but to stretch himself on one of the\r\nsofas and go to sleep. Darcy took up a book. Miss Bingley did the same;\r\nand Mrs. Hurst, principally occupied in playing with her bracelets and\r\nrings, joined now and then in her brother’s conversation with Miss\r\nBennet.\r\n\r\nMiss Bingley’s attention was quite as much engaged in watching Mr.\r\nDarcy’s progress through _his_ book, as in reading her own; and she was\r\nperpetually either making some inquiry, or looking at his page. She\r\ncould not win him, however, to any conversation; he merely answered her\r\nquestion and read on. At length, quite exhausted by the attempt to be\r\namused with her own book, which she had only chosen because it was the\r\nsecond volume of his, she gave a great yawn and said, “How pleasant it\r\nis to spend an evening in this way! I declare, after all, there is no\r\nenjoyment like reading! How much sooner one tires of anything than of a\r\nbook! When I have a house of my own, I shall be miserable if I have not\r\nan excellent library.”\r\n\r\nNo one made any reply. She then yawned again, threw aside her book, and\r\ncast her eyes round the room in quest of some amusement; when, hearing\r\nher brother mentioning a ball to Miss Bennet, she turned suddenly\r\ntowards him and said,--\r\n\r\n“By the bye Charles, are you really serious in meditating a dance at\r\nNetherfield? I would advise you, before you determine on it, to consult\r\nthe wishes of the present party; I am much mistaken if there are not\r\nsome among us to whom a ball would be rather a punishment than a\r\npleasure.”\r\n\r\n“If you mean Darcy,” cried her brother, “he may go to bed, if he\r\nchooses, before it begins; but as for the ball, it is quite a settled\r\nthing, and as soon as Nicholls has made white soup enough I shall send\r\nround my cards.”\r\n\r\n“I should like balls infinitely better,” she replied, “if they were\r\ncarried on in a different manner; but there is something insufferably\r\ntedious in the usual process of such a meeting. It would surely be much\r\nmore rational if conversation instead of dancing made the order of the\r\nday.”\r\n\r\n“Much more rational, my dear Caroline, I dare say; but it would not be\r\nnear so much like a ball.”\r\n\r\nMiss Bingley made no answer, and soon afterwards got up and walked about\r\nthe room. Her figure was elegant, and she walked well; but Darcy, at\r\nwhom it was all aimed, was still inflexibly studious. In the\r\ndesperation of her feelings, she resolved on one effort more; and,\r\nturning to Elizabeth, said,--\r\n\r\n“Miss Eliza Bennet, let me persuade you to follow my example, and take a\r\nturn about the room. I assure you it is very refreshing after sitting so\r\nlong in one attitude.”\r\n\r\nElizabeth was surprised, but agreed to it immediately. Miss Bingley\r\nsucceeded no less in the real object of her civility: Mr. Darcy looked\r\nup. He was as much awake to the novelty of attention in that quarter as\r\nElizabeth herself could be, and unconsciously closed his book. He was\r\ndirectly invited to join their party, but he declined it, observing that\r\nhe could imagine but two motives for their choosing to walk up and down\r\nthe room together, with either of which motives his joining them would\r\ninterfere. What could he mean? She was dying to know what could be his\r\nmeaning--and asked Elizabeth whether she could at all understand him.\r\n\r\n“Not at all,” was her answer; “but, depend upon it, he means to be\r\nsevere on us, and our surest way of disappointing him will be to ask\r\nnothing about it.”\r\n\r\nMiss Bingley, however, was incapable of disappointing Mr. Darcy in\r\nanything, and persevered, therefore, in requiring an explanation of his\r\ntwo motives.\r\n\r\n“I have not the smallest objection to explaining them,” said he, as soon\r\nas she allowed him to speak. “You either choose this method of passing\r\nthe evening because you are in each other’s confidence, and have secret\r\naffairs to discuss, or because you are conscious that your figures\r\nappear to the greatest advantage in walking: if the first, I should be\r\ncompletely in your way; and if the second, I can admire you much better\r\nas I sit by the fire.”\r\n\r\n“Oh, shocking!” cried Miss Bingley. “I never heard anything so\r\nabominable. How shall we punish him for such a speech?”\r\n\r\n“Nothing so easy, if you have but the inclination,” said Elizabeth. “We\r\ncan all plague and punish one another. Tease him--laugh at him. Intimate\r\nas you are, you must know how it is to be done.”\r\n\r\n“But upon my honour I do _not_. I do assure you that my intimacy has not\r\nyet taught me _that_. Tease calmness of temper and presence of mind! No,\r\nno; I feel he may defy us there. And as to laughter, we will not expose\r\nourselves, if you please, by attempting to laugh without a subject. Mr.\r\nDarcy may hug himself.”\r\n\r\n“Mr. Darcy is not to be laughed at!” cried Elizabeth. “That is an\r\nuncommon advantage, and uncommon I hope it will continue, for it would\r\nbe a great loss to _me_ to have many such acquaintance. I dearly love a\r\nlaugh.”\r\n\r\n“Miss Bingley,” said he, “has given me credit for more than can be. The\r\nwisest and best of men,--nay, the wisest and best of their actions,--may\r\nbe rendered ridiculous by a person whose first object in life is a\r\njoke.”\r\n\r\n“Certainly,” replied Elizabeth, “there are such people, but I hope I am\r\nnot one of _them_. I hope I never ridicule what is wise or good. Follies\r\nand nonsense, whims and inconsistencies, _do_ divert me, I own, and I\r\nlaugh at them whenever I can. But these, I suppose, are precisely what\r\nyou are without.”\r\n\r\n“Perhaps that is not possible for anyone. But it has been the study of\r\nmy life to avoid those weaknesses which often expose a strong\r\nunderstanding to ridicule.”\r\n\r\n“Such as vanity and pride.”\r\n\r\n“Yes, vanity is a weakness indeed. But pride--where there is a real\r\nsuperiority of mind--pride will be always under good regulation.”\r\n\r\nElizabeth turned away to hide a smile.\r\n\r\n“Your examination of Mr. Darcy is over, I presume,” said Miss Bingley;\r\n“and pray what is the result?”\r\n\r\n“I am perfectly convinced by it that Mr. Darcy has no defect. He owns it\r\nhimself without disguise.”\r\n\r\n“No,” said Darcy, “I have made no such pretension. I have faults enough,\r\nbut they are not, I hope, of understanding. My temper I dare not vouch\r\nfor. It is, I believe, too little yielding; certainly too little for the\r\nconvenience of the world. I cannot forget the follies and vices of\r\nothers so soon as I ought, nor their offences against myself. My\r\nfeelings are not puffed about with every attempt to move them. My temper\r\nwould perhaps be called resentful. My good opinion once lost is lost for\r\never.”\r\n\r\n“_That_ is a failing, indeed!” cried Elizabeth. “Implacable resentment\r\n_is_ a shade in a character. But you have chosen your fault well. I\r\nreally cannot _laugh_ at it. You are safe from me.”\r\n\r\n“There is, I believe, in every disposition a tendency to some particular\r\nevil, a natural defect, which not even the best education can overcome.”\r\n\r\n“And _your_ defect is a propensity to hate everybody.”\r\n\r\n“And yours,” he replied, with a smile, “is wilfully to misunderstand\r\nthem.”\r\n\r\n“Do let us have a little music,” cried Miss Bingley, tired of a\r\nconversation in which she had no share. “Louisa, you will not mind my\r\nwaking Mr. Hurst.”\r\n\r\nHer sister made not the smallest objection, and the pianoforte was\r\nopened; and Darcy, after a few moments’ recollection, was not sorry for\r\nit. He began to feel the danger of paying Elizabeth too much attention.\r\n\r\n\r\n\r\n\r\n[Illustration]\r\n\r\n\r\n\r\n\r\nCHAPTER XII.\r\n\r\n\r\n[Illustration]\r\n\r\nIn consequence of an agreement between the sisters, Elizabeth wrote the\r\nnext morning to her mother, to beg that the carriage might be sent for\r\nthem in the course of the day. But Mrs. Bennet, who had calculated on\r\nher daughters remaining at Netherfield till the following Tuesday, which\r\nwould exactly finish Jane’s week, could not bring herself to receive\r\nthem with pleasure before. Her answer, therefore, was not propitious, at\r\nleast not to Elizabeth’s wishes, for she was impatient to get home. Mrs.\r\nBennet sent them word that they could not possibly have the carriage\r\nbefore Tuesday; and in her postscript it was added, that if Mr. Bingley\r\nand his sister pressed them to stay longer, she could spare them very\r\nwell. Against staying longer, however, Elizabeth was positively\r\nresolved--nor did she much expect it would be asked; and fearful, on the\r\ncontrary, of being considered as intruding themselves needlessly long,\r\nshe urged Jane to borrow Mr. Bingley’s carriage immediately, and at\r\nlength it was settled that their original design of leaving Netherfield\r\nthat morning should be mentioned, and the request made.\r\n\r\nThe communication excited many professions of concern; and enough was\r\nsaid of wishing them to stay at least till the following day to work on\r\nJane; and till the morrow their going was deferred. Miss Bingley was\r\nthen sorry that she had proposed the delay; for her jealousy and dislike\r\nof one sister much exceeded her affection for the other.\r\n\r\nThe master of the house heard with real sorrow that they were to go so\r\nsoon, and repeatedly tried to persuade Miss Bennet that it would not be\r\nsafe for her--that she was not enough recovered; but Jane was firm where\r\nshe felt herself to be right.\r\n\r\nTo Mr. Darcy it was welcome intelligence: Elizabeth had been at\r\nNetherfield long enough. She attracted him more than he liked; and Miss\r\nBingley was uncivil to _her_ and more teasing than usual to himself. He\r\nwisely resolved to be particularly careful that no sign of admiration\r\nshould _now_ escape him--nothing that could elevate her with the hope of\r\ninfluencing his felicity; sensible that, if such an idea had been\r\nsuggested, his behaviour during the last day must have material weight\r\nin confirming or crushing it. Steady to his purpose, he scarcely spoke\r\nten words to her through the whole of Saturday: and though they were at\r\none time left by themselves for half an hour, he adhered most\r\nconscientiously to his book, and would not even look at her.\r\n\r\nOn Sunday, after morning service, the separation, so agreeable to almost\r\nall, took place. Miss Bingley’s civility to Elizabeth increased at last\r\nvery rapidly, as well as her affection for Jane; and when they parted,\r\nafter assuring the latter of the pleasure it would always give her to\r\nsee her either at Longbourn or Netherfield, and embracing her most\r\ntenderly, she even shook hands with the former. Elizabeth took leave of\r\nthe whole party in the liveliest spirits.\r\n\r\nThey were not welcomed home very cordially by their mother. Mrs. Bennet\r\nwondered at their coming, and thought them very wrong to give so much\r\ntrouble, and was sure Jane would have caught cold again. But their\r\nfather, though very laconic in his expressions of pleasure, was really\r\nglad to see them; he had felt their importance in the family circle. The\r\nevening conversation, when they were all assembled, had lost much of its\r\nanimation, and almost all its sense, by the absence of Jane and\r\nElizabeth.\r\n\r\nThey found Mary, as usual, deep in the study of thorough bass and human\r\nnature; and had some new extracts to admire and some new observations of\r\nthreadbare morality to listen to. Catherine and Lydia had information\r\nfor them of a different sort. Much had been done, and much had been said\r\nin the regiment since the preceding Wednesday; several of the officers\r\nhad dined lately with their uncle; a private had been flogged; and it\r\nhad actually been hinted that Colonel Forster was going to be married.\r\n\r\n\r\n\r\n\r\n[Illustration]\r\n\r\n\r\n\r\n\r\nCHAPTER XIII\r\n\r\n\r\n[Illustration]\r\n\r\n“I hope, my dear,” said Mr. Bennet to his wife, as they were at\r\nbreakfast the next morning, “that you have ordered a good dinner to-day,\r\nbecause I have reason to expect an addition to our family party.”\r\n\r\n“Who do you mean, my dear? I know of nobody that is coming, I am sure,\r\nunless Charlotte Lucas should happen to call in; and I hope _my_ dinners\r\nare good enough for her. I do not believe she often sees such at home.”\r\n\r\n“The person of whom I speak is a gentleman and a stranger.”\r\n\r\nMrs. Bennet’s eyes sparkled. “A gentleman and a stranger! It is Mr.\r\nBingley, I am sure. Why, Jane--you never dropped a word of this--you sly\r\nthing! Well, I am sure I shall be extremely glad to see Mr. Bingley.\r\nBut--good Lord! how unlucky! there is not a bit of fish to be got\r\nto-day. Lydia, my love, ring the bell. I must speak to Hill this\r\nmoment.”\r\n\r\n“It is _not_ Mr. Bingley,” said her husband; “it is a person whom I\r\nnever saw in the whole course of my life.”\r\n\r\nThis roused a general astonishment; and he had the pleasure of being\r\neagerly questioned by his wife and five daughters at once.\r\n\r\nAfter amusing himself some time with their curiosity, he thus\r\nexplained:--“About a month ago I received this letter, and about a\r\nfortnight ago I answered it; for I thought it a case of some delicacy,\r\nand requiring early attention. It is from my cousin, Mr. Collins, who,\r\nwhen I am dead, may turn you all out of this house as soon as he\r\npleases.”\r\n\r\n“Oh, my dear,” cried his wife, “I cannot bear to hear that mentioned.\r\nPray do not talk of that odious man. I do think it is the hardest thing\r\nin the world, that your estate should be entailed away from your own\r\nchildren; and I am sure, if I had been you, I should have tried long ago\r\nto do something or other about it.”\r\n\r\nJane and Elizabeth attempted to explain to her the nature of an entail.\r\nThey had often attempted it before: but it was a subject on which Mrs.\r\nBennet was beyond the reach of reason; and she continued to rail\r\nbitterly against the cruelty of settling an estate away from a family of\r\nfive daughters, in favour of a man whom nobody cared anything about.\r\n\r\n“It certainly is a most iniquitous affair,” said Mr. Bennet; “and\r\nnothing can clear Mr. Collins from the guilt of inheriting Longbourn.\r\nBut if you will listen to his letter, you may, perhaps, be a little\r\nsoftened by his manner of expressing himself.”\r\n\r\n“No, that I am sure I shall not: and I think it was very impertinent of\r\nhim to write to you at all, and very hypocritical. I hate such false\r\nfriends. Why could not he keep on quarrelling with you, as his father\r\ndid before him?”\r\n\r\n“Why, indeed, he does seem to have had some filial scruples on that\r\nhead, as you will hear.”\r\n\r\n /* RIGHT “Hunsford, near Westerham, Kent, _15th October_. */\r\n\r\n“Dear Sir,\r\n\r\n “The disagreement subsisting between yourself and my late honoured\r\n father always gave me much uneasiness; and, since I have had the\r\n misfortune to lose him, I have frequently wished to heal the\r\n breach: but, for some time, I was kept back by my own doubts,\r\n fearing lest it might seem disrespectful to his memory for me to be\r\n on good terms with anyone with whom it had always pleased him to be\r\n at variance.”--‘There, Mrs. Bennet.’--“My mind, however, is now\r\n made up on the subject; for, having received ordination at Easter,\r\n I have been so fortunate as to be distinguished by the patronage of\r\n the Right Honourable Lady Catherine de Bourgh, widow of Sir Lewis\r\n de Bourgh, whose bounty and beneficence has preferred me to the\r\n valuable rectory of this parish, where it shall be my earnest\r\n endeavour to demean myself with grateful respect towards her\r\n Ladyship, and be ever ready to perform those rites and ceremonies\r\n which are instituted by the Church of England. As a clergyman,\r\n moreover, I feel it my duty to promote and establish the blessing\r\n of peace in all families within the reach of my influence; and on\r\n these grounds I flatter myself that my present overtures of\r\n good-will are highly commendable, and that the circumstance of my\r\n being next in the entail of Longbourn estate will be kindly\r\n overlooked on your side, and not lead you to reject the offered\r\n olive branch. I cannot be otherwise than concerned at being the\r\n means of injuring your amiable daughters, and beg leave to\r\n apologize for it, as well as to assure you of my readiness to make\r\n them every possible amends; but of this hereafter. If you should\r\n have no objection to receive me into your house, I propose myself\r\n the satisfaction of waiting on you and your family, Monday,\r\n November 18th, by four o’clock, and shall probably trespass on your\r\n hospitality till the Saturday se’nnight following, which I can do\r\n without any inconvenience, as Lady Catherine is far from objecting\r\n to my occasional absence on a Sunday, provided that some other\r\n clergyman is engaged to do the duty of the day. I remain, dear sir,\r\n with respectful compliments to your lady and daughters, your\r\n well-wisher and friend,\r\n\r\n“WILLIAM COLLINS.”\r\n\r\n“At four o’clock, therefore, we may expect this peace-making gentleman,”\r\nsaid Mr. Bennet, as he folded up the letter. “He seems to be a most\r\nconscientious and polite young man, upon my word; and, I doubt not, will\r\nprove a valuable acquaintance, especially if Lady Catherine should be so\r\nindulgent as to let him come to us again.”\r\n\r\n“There is some sense in what he says about the girls, however; and, if\r\nhe is disposed to make them any amends, I shall not be the person to\r\ndiscourage him.”\r\n\r\n“Though it is difficult,” said Jane, “to guess in what way he can mean\r\nto make us the atonement he thinks our due, the wish is certainly to his\r\ncredit.”\r\n\r\nElizabeth was chiefly struck with his extraordinary deference for Lady\r\nCatherine, and his kind intention of christening, marrying, and burying\r\nhis parishioners whenever it were required.\r\n\r\n“He must be an oddity, I think,” said she. “I cannot make him out. There\r\nis something very pompous in his style. And what can he mean by\r\napologizing for being next in the entail? We cannot suppose he would\r\nhelp it, if he could. Can he be a sensible man, sir?”\r\n\r\n“No, my dear; I think not. I have great hopes of finding him quite the\r\nreverse. There is a mixture of servility and self-importance in his\r\nletter which promises well. I am impatient to see him.”\r\n\r\n“In point of composition,” said Mary, “his letter does not seem\r\ndefective. The idea of the olive branch perhaps is not wholly new, yet I\r\nthink it is well expressed.”\r\n\r\nTo Catherine and Lydia neither the letter nor its writer were in any\r\ndegree interesting. It was next to impossible that their cousin should\r\ncome in a scarlet coat, and it was now some weeks since they had\r\nreceived pleasure from the society of a man in any other colour. As for\r\ntheir mother, Mr. Collins’s letter had done away much of her ill-will,\r\nand she was preparing to see him with a degree of composure which\r\nastonished her husband and daughters.\r\n\r\nMr. Collins was punctual to his time, and was received with great\r\npoliteness by the whole family. Mr. Bennet indeed said little; but the\r\nladies were ready enough to talk, and Mr. Collins seemed neither in need\r\nof encouragement, nor inclined to be silent himself. He was a tall,\r\nheavy-looking young man of five-and-twenty. His air was grave and\r\nstately, and his manners were very formal. He had not been long seated\r\nbefore he complimented Mrs. Bennet on having so fine a family of\r\ndaughters, said he had heard much of their beauty, but that, in this\r\ninstance, fame had fallen short of the truth; and added, that he did not\r\ndoubt her seeing them all in due time well disposed of in marriage. This\r\ngallantry was not much to the taste of some of his hearers; but Mrs.\r\nBennet, who quarrelled with no compliments, answered most readily,--\r\n\r\n“You are very kind, sir, I am sure; and I wish with all my heart it may\r\nprove so; for else they will be destitute enough. Things are settled so\r\noddly.”\r\n\r\n“You allude, perhaps, to the entail of this estate.”\r\n\r\n“Ah, sir, I do indeed. It is a grievous affair to my poor girls, you\r\nmust confess. Not that I mean to find fault with _you_, for such things,\r\nI know, are all chance in this world. There is no knowing how estates\r\nwill go when once they come to be entailed.”\r\n\r\n“I am very sensible, madam, of the hardship to my fair cousins, and\r\ncould say much on the subject, but that I am cautious of appearing\r\nforward and precipitate. But I can assure the young ladies that I come\r\nprepared to admire them. At present I will not say more, but, perhaps,\r\nwhen we are better acquainted----”\r\n\r\nHe was interrupted by a summons to dinner; and the girls smiled on each\r\nother. They were not the only objects of Mr. Collins’s admiration. The\r\nhall, the dining-room, and all its furniture, were examined and praised;\r\nand his commendation of everything would have touched Mrs. Bennet’s\r\nheart, but for the mortifying supposition of his viewing it all as his\r\nown future property. The dinner, too, in its turn, was highly admired;\r\nand he begged to know to which of his fair cousins the excellence of its\r\ncookery was owing. But here he was set right by Mrs. Bennet, who assured\r\nhim, with some asperity, that they were very well able to keep a good\r\ncook, and that her daughters had nothing to do in the kitchen. He begged\r\npardon for having displeased her. In a softened tone she declared\r\nherself not at all offended; but he continued to apologize for about a\r\nquarter of an hour.\r\n\r\n\r\n\r\n\r\n[Illustration]\r\n\r\n\r\n\r\n\r\nCHAPTER XIV\r\n\r\n\r\n[Illustration]\r\n\r\nDuring dinner, Mr. Bennet scarcely spoke at all; but when the servants\r\nwere withdrawn, he thought it time to have some conversation with his\r\nguest, and therefore started a subject in which he expected him to\r\nshine, by observing that he seemed very fortunate in his patroness. Lady\r\nCatherine de Bourgh’s attention to his wishes, and consideration for his\r\ncomfort, appeared very remarkable. Mr. Bennet could not have chosen\r\nbetter. Mr. Collins was eloquent in her praise. The subject elevated him\r\nto more than usual solemnity of manner; and with a most important aspect\r\nhe protested that he had never in his life witnessed such behaviour in a\r\nperson of rank--such affability and condescension, as he had himself\r\nexperienced from Lady Catherine. She had been graciously pleased to\r\napprove of both the discourses which he had already had the honour of\r\npreaching before her. She had also asked him twice to dine at Rosings,\r\nand had sent for him only the Saturday before, to make up her pool of\r\nquadrille in the evening. Lady Catherine was reckoned proud by many\r\npeople, he knew, but _he_ had never seen anything but affability in her.\r\nShe had always spoken to him as she would to any other gentleman; she\r\nmade not the smallest objection to his joining in the society of the\r\nneighbourhood, nor to his leaving his parish occasionally for a week or\r\ntwo to visit his relations. She had even condescended to advise him to\r\nmarry as soon as he could, provided he chose with discretion; and had\r\nonce paid him a visit in his humble parsonage, where she had perfectly\r\napproved all the alterations he had been making, and had even vouchsafed\r\nto suggest some herself,--some shelves in the closets upstairs.\r\n\r\n“That is all very proper and civil, I am sure,” said Mrs. Bennet, “and I\r\ndare say she is a very agreeable woman. It is a pity that great ladies\r\nin general are not more like her. Does she live near you, sir?”\r\n\r\n“The garden in which stands my humble abode is separated only by a lane\r\nfrom Rosings Park, her Ladyship’s residence.”\r\n\r\n“I think you said she was a widow, sir? has she any family?”\r\n\r\n“She has one only daughter, the heiress of Rosings, and of very\r\nextensive property.”\r\n\r\n“Ah,” cried Mrs. Bennet, shaking her head, “then she is better off than\r\nmany girls. And what sort of young lady is she? Is she handsome?”\r\n\r\n“She is a most charming young lady, indeed. Lady Catherine herself says\r\nthat, in point of true beauty, Miss de Bourgh is far superior to the\r\nhandsomest of her sex; because there is that in her features which marks\r\nthe young woman of distinguished birth. She is unfortunately of a sickly\r\nconstitution, which has prevented her making that progress in many\r\naccomplishments which she could not otherwise have failed of, as I am\r\ninformed by the lady who superintended her education, and who still\r\nresides with them. But she is perfectly amiable, and often condescends\r\nto drive by my humble abode in her little phaeton and ponies.”\r\n\r\n“Has she been presented? I do not remember her name among the ladies at\r\ncourt.”\r\n\r\n“Her indifferent state of health unhappily prevents her being in town;\r\nand by that means, as I told Lady Catherine myself one day, has deprived\r\nthe British Court of its brightest ornament. Her Ladyship seemed pleased\r\nwith the idea; and you may imagine that I am happy on every occasion to\r\noffer those little delicate compliments which are always acceptable to\r\nladies. I have more than once observed to Lady Catherine, that her\r\ncharming daughter seemed born to be a duchess; and that the most\r\nelevated rank, instead of giving her consequence, would be adorned by\r\nher. These are the kind of little things which please her Ladyship, and\r\nit is a sort of attention which I conceive myself peculiarly bound to\r\npay.”\r\n\r\n“You judge very properly,” said Mr. Bennet; “and it is happy for you\r\nthat you possess the talent of flattering with delicacy. May I ask\r\nwhether these pleasing attentions proceed from the impulse of the\r\nmoment, or are the result of previous study?”\r\n\r\n“They arise chiefly from what is passing at the time; and though I\r\nsometimes amuse myself with suggesting and arranging such little elegant\r\ncompliments as may be adapted to ordinary occasions, I always wish to\r\ngive them as unstudied an air as possible.”\r\n\r\nMr. Bennet’s expectations were fully answered. His cousin was as absurd\r\nas he had hoped; and he listened to him with the keenest enjoyment,\r\nmaintaining at the same time the most resolute composure of countenance,\r\nand, except in an occasional glance at Elizabeth, requiring no partner\r\nin his pleasure.\r\n\r\nBy tea-time, however, the dose had been enough, and Mr. Bennet was glad\r\nto take his guest into the drawing-room again, and when tea was over,\r\nglad to invite him\r\n\r\n[Illustration:\r\n\r\n“Protested\r\nthat he never read novels” H.T Feb 94\r\n]\r\n\r\nto read aloud to the ladies. Mr. Collins readily assented, and a book\r\nwas produced; but on beholding it (for everything announced it to be\r\nfrom a circulating library) he started back, and, begging pardon,\r\nprotested that he never read novels. Kitty stared at him, and Lydia\r\nexclaimed. Other books were produced, and after some deliberation he\r\nchose “Fordyce’s Sermons.” Lydia gaped as he opened the volume; and\r\nbefore he had, with very monotonous solemnity, read three pages, she\r\ninterrupted him with,--\r\n\r\n“Do you know, mamma, that my uncle Philips talks of turning away\r\nRichard? and if he does, Colonel Forster will hire him. My aunt told me\r\nso herself on Saturday. I shall walk to Meryton to-morrow to hear more\r\nabout it, and to ask when Mr. Denny comes back from town.”\r\n\r\nLydia was bid by her two eldest sisters to hold her tongue; but Mr.\r\nCollins, much offended, laid aside his book, and said,--\r\n\r\n“I have often observed how little young ladies are interested by books\r\nof a serious stamp, though written solely for their benefit. It amazes\r\nme, I confess; for certainly there can be nothing so advantageous to\r\nthem as instruction. But I will no longer importune my young cousin.”\r\n\r\nThen, turning to Mr. Bennet, he offered himself as his antagonist at\r\nbackgammon. Mr. Bennet accepted the challenge, observing that he acted\r\nvery wisely in leaving the girls to their own trifling amusements. Mrs.\r\nBennet and her daughters apologized most civilly for Lydia’s\r\ninterruption, and promised that it should not occur again, if he would\r\nresume his book; but Mr. Collins, after assuring them that he bore his\r\nyoung cousin no ill-will, and should never resent her behaviour as any\r\naffront, seated himself at another table with Mr. Bennet, and prepared\r\nfor backgammon.\r\n\r\n\r\n\r\n\r\n[Illustration]\r\n\r\n\r\n\r\n\r\nCHAPTER XV.\r\n\r\n\r\n[Illustration]\r\n\r\nMr. Collins was not a sensible man, and the deficiency of nature had\r\nbeen but little assisted by education or society; the greatest part of\r\nhis life having been spent under the guidance of an illiterate and\r\nmiserly father; and though he belonged to one of the universities, he\r\nhad merely kept the necessary terms without forming at it any useful\r\nacquaintance. The subjection in which his father had brought him up had\r\ngiven him originally great humility of manner; but it was now a good\r\ndeal counteracted by the self-conceit of a weak head, living in\r\nretirement, and the consequential feelings of early and unexpected\r\nprosperity. A fortunate chance had recommended him to Lady Catherine de\r\nBourgh when the living of Hunsford was vacant; and the respect which he\r\nfelt for her high rank, and his veneration for her as his patroness,\r\nmingling with a very good opinion of himself, of his authority as a\r\nclergyman, and his right as a rector, made him altogether a mixture of\r\npride and obsequiousness, self-importance and humility.\r\n\r\nHaving now a good house and a very sufficient income, he intended to\r\nmarry; and in seeking a reconciliation with the Longbourn family he had\r\na wife in view, as he meant to choose one of the daughters, if he found\r\nthem as handsome and amiable as they were represented by common report.\r\nThis was his plan of amends--of atonement--for inheriting their father’s\r\nestate; and he thought it an excellent one, full of eligibility and\r\nsuitableness, and excessively generous and disinterested on his own\r\npart.\r\n\r\nHis plan did not vary on seeing them. Miss Bennet’s lovely face\r\nconfirmed his views, and established all his strictest notions of what\r\nwas due to seniority; and for the first evening _she_ was his settled\r\nchoice. The next morning, however, made an alteration; for in a quarter\r\nof an hour’s _tête-à-tête_ with Mrs. Bennet before breakfast, a\r\nconversation beginning with his parsonage-house, and leading naturally\r\nto the avowal of his hopes, that a mistress for it might be found at\r\nLongbourn, produced from her, amid very complaisant smiles and general\r\nencouragement, a caution against the very Jane he had fixed on. “As to\r\nher _younger_ daughters, she could not take upon her to say--she could\r\nnot positively answer--but she did not _know_ of any prepossession;--her\r\n_eldest_ daughter she must just mention--she felt it incumbent on her to\r\nhint, was likely to be very soon engaged.”\r\n\r\nMr. Collins had only to change from Jane to Elizabeth--and it was soon\r\ndone--done while Mrs. Bennet was stirring the fire. Elizabeth, equally\r\nnext to Jane in birth and beauty, succeeded her of course.\r\n\r\nMrs. Bennet treasured up the hint, and trusted that she might soon have\r\ntwo daughters married; and the man whom she could not bear to speak of\r\nthe day before, was now high in her good graces.\r\n\r\nLydia’s intention of walking to Meryton was not forgotten: every sister\r\nexcept Mary agreed to go with her; and Mr. Collins was to attend them,\r\nat the request of Mr. Bennet, who was most anxious to get rid of him,\r\nand have his library to himself; for thither Mr. Collins had followed\r\nhim after breakfast, and there he would continue, nominally engaged with\r\none of the largest folios in the collection, but really talking to Mr.\r\nBennet, with little cessation, of his house and garden at Hunsford. Such\r\ndoings discomposed Mr. Bennet exceedingly. In his library he had been\r\nalways sure of leisure and tranquillity; and though prepared, as he told\r\nElizabeth, to meet with folly and conceit in every other room in the\r\nhouse, he was used to be free from them there: his civility, therefore,\r\nwas most prompt in inviting Mr. Collins to join his daughters in their\r\nwalk; and Mr. Collins, being in fact much better fitted for a walker\r\nthan a reader, was extremely well pleased to close his large book, and\r\ngo.\r\n\r\nIn pompous nothings on his side, and civil assents on that of his\r\ncousins, their time passed till they entered Meryton. The attention of\r\nthe younger ones was then no longer to be gained by _him_. Their eyes\r\nwere immediately wandering up the street in quest of the officers, and\r\nnothing less than a very smart bonnet, indeed, or a really new muslin in\r\na shop window, could recall them.\r\n\r\nBut the attention of every lady was soon caught by a young man, whom\r\nthey had never seen before, of most gentlemanlike appearance, walking\r\nwith an officer on the other side of the way. The officer was the very\r\nMr. Denny concerning whose return from London Lydia came to inquire, and\r\nhe bowed as they passed. All were struck with the stranger’s air, all\r\nwondered who he could be; and Kitty and Lydia, determined if possible\r\nto find out, led the way across the street, under pretence of wanting\r\nsomething in an opposite shop, and fortunately had just gained the\r\npavement, when the two gentlemen, turning back, had reached the same\r\nspot. Mr. Denny addressed them directly, and entreated permission to\r\nintroduce his friend, Mr. Wickham, who had returned with him the day\r\nbefore from town, and, he was happy to say, had accepted a commission in\r\ntheir corps. This was exactly as it should be; for the young man wanted\r\nonly regimentals to make him completely charming. His appearance was\r\ngreatly in his favour: he had all the best parts of beauty, a fine\r\ncountenance, a good figure, and very pleasing address. The introduction\r\nwas followed up on his side by a happy readiness of conversation--a\r\nreadiness at the same time perfectly correct and unassuming; and the\r\nwhole party were still standing and talking together very agreeably,\r\nwhen the sound of horses drew their notice, and Darcy and Bingley were\r\nseen riding down the street. On distinguishing the ladies of the group\r\nthe two gentlemen came directly towards them, and began the usual\r\ncivilities. Bingley was the principal spokesman, and Miss Bennet the\r\nprincipal object. He was then, he said, on his way to Longbourn on\r\npurpose to inquire after her. Mr. Darcy corroborated it with a bow, and\r\nwas beginning to determine not to fix his eyes on Elizabeth, when they\r\nwere suddenly arrested by the sight of the stranger; and Elizabeth\r\nhappening to see the countenance of both as they looked at each other,\r\nwas all astonishment at the effect of the meeting. Both changed colour,\r\none looked white, the other red. Mr. Wickham, after a few moments,\r\ntouched his hat--a salutation which Mr. Darcy just deigned to return.\r\nWhat could be the meaning of it? It was impossible to imagine; it was\r\nimpossible not to long to know.\r\n\r\nIn another minute Mr. Bingley, but without seeming to have noticed what\r\npassed, took leave and rode on with his friend.\r\n\r\nMr. Denny and Mr. Wickham walked with the young ladies to the door of\r\nMr. Philips’s house, and then made their bows, in spite of Miss Lydia’s\r\npressing entreaties that they would come in, and even in spite of Mrs.\r\nPhilips’s throwing up the parlour window, and loudly seconding the\r\ninvitation.\r\n\r\nMrs. Philips was always glad to see her nieces; and the two eldest, from\r\ntheir recent absence, were particularly welcome; and she was eagerly\r\nexpressing her surprise at their sudden return home, which, as their own\r\ncarriage had not fetched them, she should have known nothing about, if\r\nshe had not happened to see Mr. Jones’s shopboy in the street, who had\r\ntold her that they were not to send any more draughts to Netherfield,\r\nbecause the Miss Bennets were come away, when her civility was claimed\r\ntowards Mr. Collins by Jane’s introduction of him. She received him with\r\nher very best politeness, which he returned with as much more,\r\napologizing for his intrusion, without any previous acquaintance with\r\nher, which he could not help flattering himself, however, might be\r\njustified by his relationship to the young ladies who introduced him to\r\nher notice. Mrs. Philips was quite awed by such an excess of good\r\nbreeding; but her contemplation of one stranger was soon put an end to\r\nby exclamations and inquiries about the other, of whom, however, she\r\ncould only tell her nieces what they already knew, that Mr. Denny had\r\nbrought him from London, and that he was to have a lieutenant’s\r\ncommission in the ----shire. She had been watching him the last hour,\r\nshe said, as he walked up and down the street,--and had Mr. Wickham\r\nappeared, Kitty and Lydia would certainly have continued the occupation;\r\nbut unluckily no one passed the windows now except a few of the\r\nofficers, who, in comparison with the stranger, were become “stupid,\r\ndisagreeable fellows.” Some of them were to dine with the Philipses the\r\nnext day, and their aunt promised to make her husband call on Mr.\r\nWickham, and give him an invitation also, if the family from Longbourn\r\nwould come in the evening. This was agreed to; and Mrs. Philips\r\nprotested that they would have a nice comfortable noisy game of lottery\r\ntickets, and a little bit of hot supper afterwards. The prospect of such\r\ndelights was very cheering, and they parted in mutual good spirits. Mr.\r\nCollins repeated his apologies in quitting the room, and was assured,\r\nwith unwearying civility, that they were perfectly needless.\r\n\r\nAs they walked home, Elizabeth related to Jane what she had seen pass\r\nbetween the two gentlemen; but though Jane would have defended either or\r\nboth, had they appeared to be wrong, she could no more explain such\r\nbehaviour than her sister.\r\n\r\nMr. Collins on his return highly gratified Mrs. Bennet by admiring Mrs.\r\nPhilips’s manners and politeness. He protested that, except Lady\r\nCatherine and her daughter, he had never seen a more elegant woman; for\r\nshe had not only received him with the utmost civility, but had even\r\npointedly included him in her invitation for the next evening, although\r\nutterly unknown to her before. Something, he supposed, might be\r\nattributed to his connection with them, but yet he had never met with so\r\nmuch attention in the whole course of his life.\r\n\r\n\r\n\r\n\r\n[Illustration]\r\n\r\n\r\n\r\n\r\nCHAPTER XVI.\r\n\r\n\r\n[Illustration]\r\n\r\nAs no objection was made to the young people’s engagement with their\r\naunt, and all Mr. Collins’s scruples of leaving Mr. and Mrs. Bennet for\r\na single evening during his visit were most steadily resisted, the coach\r\nconveyed him and his five cousins at a suitable hour to Meryton; and the\r\ngirls had the pleasure of hearing, as they entered the drawing-room,\r\nthat Mr. Wickham had accepted their uncle’s invitation, and was then in\r\nthe house.\r\n\r\nWhen this information was given, and they had all taken their seats, Mr.\r\nCollins was at leisure to look around him and admire, and he was so much\r\nstruck with the size and furniture of the apartment, that he declared he\r\nmight almost have supposed himself in the small summer breakfast parlour\r\nat Rosings; a comparison that did not at first convey much\r\ngratification; but when Mrs. Philips understood from him what Rosings\r\nwas, and who was its proprietor, when she had listened to the\r\ndescription of only one of Lady Catherine’s drawing-rooms, and found\r\nthat the chimney-piece alone had cost eight hundred pounds, she felt all\r\nthe force of the compliment, and would hardly have resented a comparison\r\nwith the housekeeper’s room.\r\n\r\nIn describing to her all the grandeur of Lady Catherine and her mansion,\r\nwith occasional digressions in praise of his own humble abode, and the\r\nimprovements it was receiving, he was happily employed until the\r\ngentlemen joined them; and he found in Mrs. Philips a very attentive\r\nlistener, whose opinion of his consequence increased with what she\r\nheard, and who was resolving to retail it all among her neighbours as\r\nsoon as she could. To the girls, who could not listen to their cousin,\r\nand who had nothing to do but to wish for an instrument, and examine\r\ntheir own indifferent imitations of china on the mantel-piece, the\r\ninterval of waiting appeared very long. It was over at last, however.\r\nThe gentlemen did approach: and when Mr. Wickham walked into the room,\r\nElizabeth felt that she had neither been seeing him before, nor thinking\r\nof him since, with the smallest degree of unreasonable admiration. The\r\nofficers of the ----shire were in general a very creditable,\r\ngentlemanlike set and the best of them were of the present party; but\r\nMr, Wickham was as far beyond them all in person, countenance, air, and\r\nwalk, as _they_ were superior to the broad-faced stuffy uncle Philips,\r\nbreathing port wine, who followed them into the room.\r\n\r\n[Illustration:\r\n\r\n“The officers of the ----shire”\r\n\r\n[_Copyright 1894 by George Allen._]]\r\n\r\nMr. Wickham was the happy man towards whom almost every female eye was\r\nturned, and Elizabeth was the happy woman by whom he finally seated\r\nhimself; and the agreeable manner in which he immediately fell into\r\nconversation, though it was only on its being a wet night, and on the\r\nprobability of a rainy season, made her feel that the commonest,\r\ndullest, most threadbare topic might be rendered interesting by the\r\nskill of the speaker.\r\n\r\nWith such rivals for the notice of the fair as Mr. Wickham and the\r\nofficers, Mr. Collins seemed to sink into insignificance; to the young\r\nladies he certainly was nothing; but he had still at intervals a kind\r\nlistener in Mrs. Philips, and was, by her watchfulness, most abundantly\r\nsupplied with coffee and muffin.\r\n\r\nWhen the card tables were placed, he had an opportunity of obliging her,\r\nin return, by sitting down to whist.\r\n\r\n“I know little of the game at present,” said he, “but I shall be glad to\r\nimprove myself; for in my situation of life----” Mrs. Philips was very\r\nthankful for his compliance, but could not wait for his reason.\r\n\r\nMr. Wickham did not play at whist, and with ready delight was he\r\nreceived at the other table between Elizabeth and Lydia. At first there\r\nseemed danger of Lydia’s engrossing him entirely, for she was a most\r\ndetermined talker; but being likewise extremely fond of lottery tickets,\r\nshe soon grew too much interested in the game, too eager in making bets\r\nand exclaiming after prizes, to have attention for anyone in particular.\r\nAllowing for the common demands of the game, Mr. Wickham was therefore\r\nat leisure to talk to Elizabeth, and she was very willing to hear him,\r\nthough what she chiefly wished to hear she could not hope to be told,\r\nthe history of his acquaintance with Mr. Darcy. She dared not even\r\nmention that gentleman. Her curiosity, however, was unexpectedly\r\nrelieved. Mr. Wickham began the subject himself. He inquired how far\r\nNetherfield was from Meryton; and, after receiving her answer, asked in\r\na hesitating manner how long Mr. Darcy had been staying there.\r\n\r\n“About a month,” said Elizabeth; and then, unwilling to let the subject\r\ndrop, added, “he is a man of very large property in Derbyshire, I\r\nunderstand.”\r\n\r\n“Yes,” replied Wickham; “his estate there is a noble one. A clear ten\r\nthousand per annum. You could not have met with a person more capable of\r\ngiving you certain information on that head than myself--for I have been\r\nconnected with his family, in a particular manner, from my infancy.”\r\n\r\nElizabeth could not but look surprised.\r\n\r\n“You may well be surprised, Miss Bennet, at such an assertion, after\r\nseeing, as you probably might, the very cold manner of our meeting\r\nyesterday. Are you much acquainted with Mr. Darcy?”\r\n\r\n“As much as I ever wish to be,” cried Elizabeth, warmly. “I have spent\r\nfour days in the same house with him, and I think him very\r\ndisagreeable.”\r\n\r\n“I have no right to give _my_ opinion,” said Wickham, “as to his being\r\nagreeable or otherwise. I am not qualified to form one. I have known him\r\ntoo long and too well to be a fair judge. It is impossible for _me_ to\r\nbe impartial. But I believe your opinion of him would in general\r\nastonish--and, perhaps, you would not express it quite so strongly\r\nanywhere else. Here you are in your own family.”\r\n\r\n“Upon my word I say no more _here_ than I might say in any house in the\r\nneighbourhood, except Netherfield. He is not at all liked in\r\nHertfordshire. Everybody is disgusted with his pride. You will not find\r\nhim more favourably spoken of by anyone.”\r\n\r\n“I cannot pretend to be sorry,” said Wickham, after a short\r\ninterruption, “that he or that any man should not be estimated beyond\r\ntheir deserts; but with _him_ I believe it does not often happen. The\r\nworld is blinded by his fortune and consequence, or frightened by his\r\nhigh and imposing manners, and sees him only as he chooses to be seen.”\r\n\r\n“I should take him, even on _my_ slight acquaintance, to be an\r\nill-tempered man.”\r\n\r\nWickham only shook his head.\r\n\r\n“I wonder,” said he, at the next opportunity of speaking, “whether he is\r\nlikely to be in this country much longer.”\r\n\r\n“I do not at all know; but I _heard_ nothing of his going away when I\r\nwas at Netherfield. I hope your plans in favour of the ----shire will\r\nnot be affected by his being in the neighbourhood.”\r\n\r\n“Oh no--it is not for _me_ to be driven away by Mr. Darcy. If _he_\r\nwishes to avoid seeing _me_ he must go. We are not on friendly terms,\r\nand it always gives me pain to meet him, but I have no reason for\r\navoiding _him_ but what I might proclaim to all the world--a sense of\r\nvery great ill-usage, and most painful regrets at his being what he is.\r\nHis father, Miss Bennet, the late Mr. Darcy, was one of the best men\r\nthat ever breathed, and the truest friend I ever had; and I can never be\r\nin company with this Mr. Darcy without being grieved to the soul by a\r\nthousand tender recollections. His behaviour to myself has been\r\nscandalous; but I verily believe I could forgive him anything and\r\neverything, rather than his disappointing the hopes and disgracing the\r\nmemory of his father.”\r\n\r\nElizabeth found the interest of the subject increase, and listened with\r\nall her heart; but the delicacy of it prevented further inquiry.\r\n\r\nMr. Wickham began to speak on more general topics, Meryton, the\r\nneighbourhood, the society, appearing highly pleased with all that he\r\nhad yet seen, and speaking of the latter, especially, with gentle but\r\nvery intelligible gallantry.\r\n\r\n“It was the prospect of constant society, and good society,” he added,\r\n“which was my chief inducement to enter the ----shire. I know it to be a\r\nmost respectable, agreeable corps; and my friend Denny tempted me\r\nfurther by his account of their present quarters, and the very great\r\nattentions and excellent acquaintance Meryton had procured them.\r\nSociety, I own, is necessary to me. I have been a disappointed man, and\r\nmy spirits will not bear solitude. I _must_ have employment and society.\r\nA military life is not what I was intended for, but circumstances have\r\nnow made it eligible. The church _ought_ to have been my profession--I\r\nwas brought up for the church; and I should at this time have been in\r\npossession of a most valuable living, had it pleased the gentleman we\r\nwere speaking of just now.”\r\n\r\n“Indeed!”\r\n\r\n“Yes--the late Mr. Darcy bequeathed me the next presentation of the best\r\nliving in his gift. He was my godfather, and excessively attached to me.\r\nI cannot do justice to his kindness. He meant to provide for me amply,\r\nand thought he had done it; but when the living fell, it was given\r\nelsewhere.”\r\n\r\n“Good heavens!” cried Elizabeth; “but how could _that_ be? How could his\r\nwill be disregarded? Why did not you seek legal redress?”\r\n\r\n“There was just such an informality in the terms of the bequest as to\r\ngive me no hope from law. A man of honour could not have doubted the\r\nintention, but Mr. Darcy chose to doubt it--or to treat it as a merely\r\nconditional recommendation, and to assert that I had forfeited all claim\r\nto it by extravagance, imprudence, in short, anything or nothing.\r\nCertain it is that the living became vacant two years ago, exactly as I\r\nwas of an age to hold it, and that it was given to another man; and no\r\nless certain is it, that I cannot accuse myself of having really done\r\nanything to deserve to lose it. I have a warm unguarded temper, and I\r\nmay perhaps have sometimes spoken my opinion _of_ him, and _to_ him, too\r\nfreely. I can recall nothing worse. But the fact is, that we are very\r\ndifferent sort of men, and that he hates me.”\r\n\r\n“This is quite shocking! He deserves to be publicly disgraced.”\r\n\r\n“Some time or other he _will_ be--but it shall not be by _me_. Till I\r\ncan forget his father, I can never defy or expose _him_.”\r\n\r\nElizabeth honoured him for such feelings, and thought him handsomer than\r\never as he expressed them.\r\n\r\n“But what,” said she, after a pause, “can have been his motive? what can\r\nhave induced him to behave so cruelly?”\r\n\r\n“A thorough, determined dislike of me--a dislike which I cannot but\r\nattribute in some measure to jealousy. Had the late Mr. Darcy liked me\r\nless, his son might have borne with me better; but his father’s uncommon\r\nattachment to me irritated him, I believe, very early in life. He had\r\nnot a temper to bear the sort of competition in which we stood--the sort\r\nof preference which was often given me.”\r\n\r\n“I had not thought Mr. Darcy so bad as this--though I have never liked\r\nhim, I had not thought so very ill of him--I had supposed him to be\r\ndespising his fellow-creatures in general, but did not suspect him of\r\ndescending to such malicious revenge, such injustice, such inhumanity as\r\nthis!”\r\n\r\nAfter a few minutes’ reflection, however, she continued, “I _do_\r\nremember his boasting one day, at Netherfield, of the implacability of\r\nhis resentments, of his having an unforgiving temper. His disposition\r\nmust be dreadful.”\r\n\r\n“I will not trust myself on the subject,” replied Wickham; “_I_ can\r\nhardly be just to him.”\r\n\r\nElizabeth was again deep in thought, and after a time exclaimed, “To\r\ntreat in such a manner the godson, the friend, the favourite of his\r\nfather!” She could have added, “A young man, too, like _you_, whose very\r\ncountenance may vouch for your being amiable.” But she contented herself\r\nwith--“And one, too, who had probably been his own companion from\r\nchildhood, connected together, as I think you said, in the closest\r\nmanner.”\r\n\r\n“We were born in the same parish, within the same park; the greatest\r\npart of our youth was passed together: inmates of the same house,\r\nsharing the same amusements, objects of the same parental care. _My_\r\nfather began life in the profession which your uncle, Mr. Philips,\r\nappears to do so much credit to; but he gave up everything to be of use\r\nto the late Mr. Darcy, and devoted all his time to the care of the\r\nPemberley property. He was most highly esteemed by Mr. Darcy, a most\r\nintimate, confidential friend. Mr. Darcy often acknowledged himself to\r\nbe under the greatest obligations to my father’s active superintendence;\r\nand when, immediately before my father’s death, Mr. Darcy gave him a\r\nvoluntary promise of providing for me, I am convinced that he felt it\r\nto be as much a debt of gratitude to _him_ as of affection to myself.”\r\n\r\n“How strange!” cried Elizabeth. “How abominable! I wonder that the very\r\npride of this Mr. Darcy has not made him just to you. If from no better\r\nmotive, that he should not have been too proud to be dishonest,--for\r\ndishonesty I must call it.”\r\n\r\n“It _is_ wonderful,” replied Wickham; “for almost all his actions may be\r\ntraced to pride; and pride has often been his best friend. It has\r\nconnected him nearer with virtue than any other feeling. But we are none\r\nof us consistent; and in his behaviour to me there were stronger\r\nimpulses even than pride.”\r\n\r\n“Can such abominable pride as his have ever done him good?”\r\n\r\n“Yes; it has often led him to be liberal and generous; to give his money\r\nfreely, to display hospitality, to assist his tenants, and relieve the\r\npoor. Family pride, and _filial_ pride, for he is very proud of what his\r\nfather was, have done this. Not to appear to disgrace his family, to\r\ndegenerate from the popular qualities, or lose the influence of the\r\nPemberley House, is a powerful motive. He has also _brotherly_ pride,\r\nwhich, with _some_ brotherly affection, makes him a very kind and\r\ncareful guardian of his sister; and you will hear him generally cried up\r\nas the most attentive and best of brothers.”\r\n\r\n“What sort of a girl is Miss Darcy?”\r\n\r\nHe shook his head. “I wish I could call her amiable. It gives me pain to\r\nspeak ill of a Darcy; but she is too much like her brother,--very, very\r\nproud. As a child, she was affectionate and pleasing, and extremely fond\r\nof me; and I have devoted hours and hours to her amusement. But she is\r\nnothing to me now. She is a handsome girl, about fifteen or sixteen,\r\nand, I understand, highly accomplished. Since her father’s death her\r\nhome has been London, where a lady lives with her, and superintends her\r\neducation.”\r\n\r\nAfter many pauses and many trials of other subjects, Elizabeth could not\r\nhelp reverting once more to the first, and saying,--\r\n\r\n“I am astonished at his intimacy with Mr. Bingley. How can Mr. Bingley,\r\nwho seems good-humour itself, and is, I really believe, truly amiable,\r\nbe in friendship with such a man? How can they suit each other? Do you\r\nknow Mr. Bingley?”\r\n\r\n“Not at all.”\r\n\r\n“He is a sweet-tempered, amiable, charming man. He cannot know what Mr.\r\nDarcy is.”\r\n\r\n“Probably not; but Mr. Darcy can please where he chooses. He does not\r\nwant abilities. He can be a conversible companion if he thinks it worth\r\nhis while. Among those who are at all his equals in consequence, he is a\r\nvery different man from what he is to the less prosperous. His pride\r\nnever deserts him; but with the rich he is liberal-minded, just,\r\nsincere, rational, honourable, and, perhaps, agreeable,--allowing\r\nsomething for fortune and figure.”\r\n\r\nThe whist party soon afterwards breaking up, the players gathered round\r\nthe other table, and Mr. Collins took his station between his cousin\r\nElizabeth and Mrs. Philips. The usual inquiries as to his success were\r\nmade by the latter. It had not been very great; he had lost every point;\r\nbut when Mrs. Philips began to express her concern thereupon, he assured\r\nher, with much earnest gravity, that it was not of the least importance;\r\nthat he considered the money as a mere trifle, and begged she would not\r\nmake herself uneasy.\r\n\r\n“I know very well, madam,” said he, “that when persons sit down to a\r\ncard table they must take their chance of these things,--and happily I\r\nam not in such circumstances as to make five shillings any object. There\r\nare, undoubtedly, many who could not say the same; but, thanks to Lady\r\nCatherine de Bourgh, I am removed far beyond the necessity of regarding\r\nlittle matters.”\r\n\r\nMr. Wickham’s attention was caught; and after observing Mr. Collins for\r\na few moments, he asked Elizabeth in a low voice whether her relations\r\nwere very intimately acquainted with the family of De Bourgh.\r\n\r\n“Lady Catherine de Bourgh,” she replied, “has very lately given him a\r\nliving. I hardly know how Mr. Collins was first introduced to her\r\nnotice, but he certainly has not known her long.”\r\n\r\n“You know of course that Lady Catherine de Bourgh and Lady Anne Darcy\r\nwere sisters; consequently that she is aunt to the present Mr. Darcy.”\r\n\r\n“No, indeed, I did not. I knew nothing at all of Lady Catherine’s\r\nconnections. I never heard of her existence till the day before\r\nyesterday.”\r\n\r\n“Her daughter, Miss de Bourgh, will have a very large fortune, and it is\r\nbelieved that she and her cousin will unite the two estates.”\r\n\r\nThis information made Elizabeth smile, as she thought of poor Miss\r\nBingley. Vain indeed must be all her attentions, vain and useless her\r\naffection for his sister and her praise of himself, if he were already\r\nself-destined to another.\r\n\r\n“Mr. Collins,” said she, “speaks highly both of Lady Catherine and her\r\ndaughter; but, from some particulars that he has related of her\r\nLadyship, I suspect his gratitude misleads him; and that, in spite of\r\nher being his patroness, she is an arrogant, conceited woman.”\r\n\r\n“I believe her to be both in a great degree,” replied Wickham; “I have\r\nnot seen her for many years; but I very well remember that I never liked\r\nher, and that her manners were dictatorial and insolent. She has the\r\nreputation of being remarkably sensible and clever; but I rather believe\r\nshe derives part of her abilities from her rank and fortune, part from\r\nher authoritative manner, and the rest from the pride of her nephew, who\r\nchooses that everyone connected with him should have an understanding of\r\nthe first class.”\r\n\r\nElizabeth allowed that he had given a very rational account of it, and\r\nthey continued talking together with mutual satisfaction till supper put\r\nan end to cards, and gave the rest of the ladies their share of Mr.\r\nWickham’s attentions. There could be no conversation in the noise of\r\nMrs. Philips’s supper party, but his manners recommended him to\r\neverybody. Whatever he said, was said well; and whatever he did, done\r\ngracefully. Elizabeth went away with her head full of him. She could\r\nthink of nothing but of Mr. Wickham, and of what he had told her, all\r\nthe way home; but there was not time for her even to mention his name as\r\nthey went, for neither Lydia nor Mr. Collins were once silent. Lydia\r\ntalked incessantly of lottery tickets, of the fish she had lost and the\r\nfish she had won; and Mr. Collins, in describing the civility of Mr. and\r\nMrs. Philips, protesting that he did not in the least regard his losses\r\nat whist, enumerating all the dishes at supper, and repeatedly fearing\r\nthat he crowded his cousins, had more to say than he could well manage\r\nbefore the carriage stopped at Longbourn House.\r\n\r\n\r\n\r\n\r\n[Illustration:\r\n\r\n “delighted to see their dear friend again”\r\n]\r\n\r\n\r\n\r\n\r\nCHAPTER XVII.\r\n\r\n\r\n[Illustration]\r\n\r\nElizabeth related to Jane, the next day, what had passed between Mr.\r\nWickham and herself. Jane listened with astonishment and concern: she\r\nknew not how to believe that Mr. Darcy could be so unworthy of Mr.\r\nBingley’s regard; and yet it was not in her nature to question the\r\nveracity of a young man of such amiable appearance as Wickham. The\r\npossibility of his having really endured such unkindness was enough to\r\ninterest all her tender feelings; and nothing therefore remained to be\r\ndone but to think well of them both, to defend the conduct of each, and\r\nthrow into the account of accident or mistake whatever could not be\r\notherwise explained.\r\n\r\n“They have both,” said she, “been deceived, I dare say, in some way or\r\nother, of which we can form no idea. Interested people have perhaps\r\nmisrepresented each to the other. It is, in short, impossible for us to\r\nconjecture the causes or circumstances which may have alienated them,\r\nwithout actual blame on either side.”\r\n\r\n“Very true, indeed; and now, my dear Jane, what have you got to say in\r\nbehalf of the interested people who have probably been concerned in the\r\nbusiness? Do clear _them_, too, or we shall be obliged to think ill of\r\nsomebody.”\r\n\r\n“Laugh as much as you choose, but you will not laugh me out of my\r\nopinion. My dearest Lizzy, do but consider in what a disgraceful light\r\nit places Mr. Darcy, to be treating his father’s favourite in such a\r\nmanner,--one whom his father had promised to provide for. It is\r\nimpossible. No man of common humanity, no man who had any value for his\r\ncharacter, could be capable of it. Can his most intimate friends be so\r\nexcessively deceived in him? Oh no.”\r\n\r\n“I can much more easily believe Mr. Bingley’s being imposed on than that\r\nMr. Wickham should invent such a history of himself as he gave me last\r\nnight; names, facts, everything mentioned without ceremony. If it be not\r\nso, let Mr. Darcy contradict it. Besides, there was truth in his looks.”\r\n\r\n“It is difficult, indeed--it is distressing. One does not know what to\r\nthink.”\r\n\r\n“I beg your pardon;--one knows exactly what to think.”\r\n\r\nBut Jane could think with certainty on only one point,--that Mr.\r\nBingley, if he _had been_ imposed on, would have much to suffer when\r\nthe affair became public.\r\n\r\nThe two young ladies were summoned from the shrubbery, where this\r\nconversation passed, by the arrival of some of the very persons of whom\r\nthey had been speaking; Mr. Bingley and his sisters came to give their\r\npersonal invitation for the long expected ball at Netherfield, which was\r\nfixed for the following Tuesday. The two ladies were delighted to see\r\ntheir dear friend again, called it an age since they had met, and\r\nrepeatedly asked what she had been doing with herself since their\r\nseparation. To the rest of the family they paid little attention;\r\navoiding Mrs. Bennet as much as possible, saying not much to Elizabeth,\r\nand nothing at all to the others. They were soon gone again, rising from\r\ntheir seats with an activity which took their brother by surprise, and\r\nhurrying off as if eager to escape from Mrs. Bennet’s civilities.\r\n\r\nThe prospect of the Netherfield ball was extremely agreeable to every\r\nfemale of the family. Mrs. Bennet chose to consider it as given in\r\ncompliment to her eldest daughter, and was particularly flattered by\r\nreceiving the invitation from Mr. Bingley himself, instead of a\r\nceremonious card. Jane pictured to herself a happy evening in the\r\nsociety of her two friends, and the attentions of their brother; and\r\nElizabeth thought with pleasure of dancing a great deal with Mr.\r\nWickham, and of seeing a confirmation of everything in Mr. Darcy’s look\r\nand behaviour. The happiness anticipated by Catherine and Lydia depended\r\nless on any single event, or any particular person; for though they\r\neach, like Elizabeth, meant to dance half the evening with Mr. Wickham,\r\nhe was by no means the only partner who could satisfy them, and a ball\r\nwas, at any rate, a ball. And even Mary could assure her family that she\r\nhad no disinclination for it.\r\n\r\n“While I can have my mornings to myself,” said she, “it is enough. I\r\nthink it is no sacrifice to join occasionally in evening engagements.\r\nSociety has claims on us all; and I profess myself one of those who\r\nconsider intervals of recreation and amusement as desirable for\r\neverybody.”\r\n\r\nElizabeth’s spirits were so high on the occasion, that though she did\r\nnot often speak unnecessarily to Mr. Collins, she could not help asking\r\nhim whether he intended to accept Mr. Bingley’s invitation, and if he\r\ndid, whether he would think it proper to join in the evening’s\r\namusement; and she was rather surprised to find that he entertained no\r\nscruple whatever on that head, and was very far from dreading a rebuke,\r\neither from the Archbishop or Lady Catherine de Bourgh, by venturing to\r\ndance.\r\n\r\n“I am by no means of opinion, I assure you,” said he, “that a ball of\r\nthis kind, given by a young man of character, to respectable people, can\r\nhave any evil tendency; and I am so far from objecting to dancing\r\nmyself, that I shall hope to be honoured with the hands of all my fair\r\ncousins in the course of the evening; and I take this opportunity of\r\nsoliciting yours, Miss Elizabeth, for the two first dances especially; a\r\npreference which I trust my cousin Jane will attribute to the right\r\ncause, and not to any disrespect for her.”\r\n\r\nElizabeth felt herself completely taken in. She had fully proposed being\r\nengaged by Wickham for those very dances; and to have Mr. Collins\r\ninstead!--her liveliness had been never worse timed. There was no help\r\nfor it, however. Mr. Wickham’s happiness and her own was perforce\r\ndelayed a little longer, and Mr. Collins’s proposal accepted with as\r\ngood a grace as she could. She was not the better pleased with his\r\ngallantry, from the idea it suggested of something more. It now first\r\nstruck her, that _she_ was selected from among her sisters as worthy of\r\nbeing the mistress of Hunsford Parsonage, and of assisting to form a\r\nquadrille table at Rosings, in the absence of more eligible visitors.\r\nThe idea soon reached to conviction, as she observed his increasing\r\ncivilities towards herself, and heard his frequent attempt at a\r\ncompliment on her wit and vivacity; and though more astonished than\r\ngratified herself by this effect of her charms, it was not long before\r\nher mother gave her to understand that the probability of their marriage\r\nwas exceedingly agreeable to _her_. Elizabeth, however, did not choose\r\nto take the hint, being well aware that a serious dispute must be the\r\nconsequence of any reply. Mr. Collins might never make the offer, and,\r\ntill he did, it was useless to quarrel about him.\r\n\r\nIf there had not been a Netherfield ball to prepare for and talk of, the\r\nyounger Miss Bennets would have been in a pitiable state at this time;\r\nfor, from the day of the invitation to the day of the ball, there was\r\nsuch a succession of rain as prevented their walking to Meryton once. No\r\naunt, no officers, no news could be sought after; the very shoe-roses\r\nfor Netherfield were got by proxy. Even Elizabeth might have found some\r\ntrial of her patience in weather which totally suspended the improvement\r\nof her acquaintance with Mr. Wickham; and nothing less than a dance on\r\nTuesday could have made such a Friday, Saturday, Sunday, and Monday\r\nendurable to Kitty and Lydia.\r\n\r\n\r\n\r\n\r\n[Illustration]\r\n\r\n\r\n\r\n\r\nCHAPTER XVIII.\r\n\r\n\r\n[Illustration]\r\n\r\nTill Elizabeth entered the drawing-room at Netherfield, and looked in\r\nvain for Mr. Wickham among the cluster of red coats there assembled, a\r\ndoubt of his being present had never occurred to her. The certainty of\r\nmeeting him had not been checked by any of those recollections that\r\nmight not unreasonably have alarmed her. She had dressed with more than\r\nusual care, and prepared in the highest spirits for the conquest of all\r\nthat remained unsubdued of his heart, trusting that it was not more than\r\nmight be won in the course of the evening. But in an instant arose the\r\ndreadful suspicion of his being purposely omitted, for Mr. Darcy’s\r\npleasure, in the Bingleys’ invitation to the officers; and though this\r\nwas not exactly the case, the absolute fact of his absence was\r\npronounced by his friend Mr. Denny, to whom Lydia eagerly applied, and\r\nwho told them that Wickham had been obliged to go to town on business\r\nthe day before, and was not yet returned; adding, with a significant\r\nsmile,--\r\n\r\n“I do not imagine his business would have called him away just now, if\r\nhe had not wished to avoid a certain gentleman here.”\r\n\r\nThis part of his intelligence, though unheard by Lydia, was caught by\r\nElizabeth; and, as it assured her that Darcy was not less answerable for\r\nWickham’s absence than if her first surmise had been just, every feeling\r\nof displeasure against the former was so sharpened by immediate\r\ndisappointment, that she could hardly reply with tolerable civility to\r\nthe polite inquiries which he directly afterwards approached to make.\r\nAttention, forbearance, patience with Darcy, was injury to Wickham. She\r\nwas resolved against any sort of conversation with him, and turned away\r\nwith a degree of ill-humour which she could not wholly surmount even in\r\nspeaking to Mr. Bingley, whose blind partiality provoked her.\r\n\r\nBut Elizabeth was not formed for ill-humour; and though every prospect\r\nof her own was destroyed for the evening, it could not dwell long on her\r\nspirits; and, having told all her griefs to Charlotte Lucas, whom she\r\nhad not seen for a week, she was soon able to make a voluntary\r\ntransition to the oddities of her cousin, and to point him out to her\r\nparticular notice. The two first dances, however, brought a return of\r\ndistress: they were dances of mortification. Mr. Collins, awkward and\r\nsolemn, apologizing instead of attending, and often moving wrong\r\nwithout being aware of it, gave her all the shame and misery which a\r\ndisagreeable partner for a couple of dances can give. The moment of her\r\nrelease from him was ecstasy.\r\n\r\nShe danced next with an officer, and had the refreshment of talking of\r\nWickham, and of hearing that he was universally liked. When those dances\r\nwere over, she returned to Charlotte Lucas, and was in conversation with\r\nher, when she found herself suddenly addressed by Mr. Darcy, who took\r\nher so much by surprise in his application for her hand, that, without\r\nknowing what she did, she accepted him. He walked away again\r\nimmediately, and she was left to fret over her own want of presence of\r\nmind: Charlotte tried to console her.\r\n\r\n“I dare say you will find him very agreeable.”\r\n\r\n“Heaven forbid! _That_ would be the greatest misfortune of all! To find\r\na man agreeable whom one is determined to hate! Do not wish me such an\r\nevil.”\r\n\r\nWhen the dancing recommenced, however, and Darcy approached to claim her\r\nhand, Charlotte could not help cautioning her, in a whisper, not to be a\r\nsimpleton, and allow her fancy for Wickham to make her appear unpleasant\r\nin the eyes of a man often times his consequence. Elizabeth made no\r\nanswer, and took her place in the set, amazed at the dignity to which\r\nshe was arrived in being allowed to stand opposite to Mr. Darcy, and\r\nreading in her neighbours’ looks their equal amazement in beholding it.\r\nThey stood for some time without speaking a word; and she began to\r\nimagine that their silence was to last through the two dances, and, at\r\nfirst, was resolved not to break it; till suddenly fancying that it\r\nwould be the greater punishment to her partner to oblige him to talk,\r\nshe made some slight observation on the dance. He replied, and was again\r\nsilent. After a pause of some minutes, she addressed him a second time,\r\nwith--\r\n\r\n“It is _your_ turn to say something now, Mr. Darcy. _I_ talked about the\r\ndance, and _you_ ought to make some kind of remark on the size of the\r\nroom, or the number of couples.”\r\n\r\nHe smiled, and assured her that whatever she wished him to say should be\r\nsaid.\r\n\r\n“Very well; that reply will do for the present. Perhaps, by-and-by, I\r\nmay observe that private balls are much pleasanter than public ones; but\r\n_now_ we may be silent.”\r\n\r\n“Do you talk by rule, then, while you are dancing?”\r\n\r\n“Sometimes. One must speak a little, you know. It would look odd to be\r\nentirely silent for half an hour together; and yet, for the advantage of\r\n_some_, conversation ought to be so arranged as that they may have the\r\ntrouble of saying as little as possible.”\r\n\r\n“Are you consulting your own feelings in the present case, or do you\r\nimagine that you are gratifying mine?”\r\n\r\n“Both,” replied Elizabeth archly; “for I have always seen a great\r\nsimilarity in the turn of our minds. We are each of an unsocial,\r\ntaciturn disposition, unwilling to speak, unless we expect to say\r\nsomething that will amaze the whole room, and be handed down to\r\nposterity with all the _éclat_ of a proverb.”\r\n\r\n“This is no very striking resemblance of your own character, I am sure,”\r\nsaid he. “How near it may be to _mine_, I cannot pretend to say. _You_\r\nthink it a faithful portrait, undoubtedly.”\r\n\r\n“I must not decide on my own performance.”\r\n\r\nHe made no answer; and they were again silent till they had gone down\r\nthe dance, when he asked her if she and her sisters did not very often\r\nwalk to Meryton. She answered in the affirmative; and, unable to resist\r\nthe temptation, added, “When you met us there the other day, we had just\r\nbeen forming a new acquaintance.”\r\n\r\nThe effect was immediate. A deeper shade of _hauteur_ overspread his\r\nfeatures, but he said not a word; and Elizabeth, though blaming herself\r\nfor her own weakness, could not go on. At length Darcy spoke, and in a\r\nconstrained manner said,--\r\n\r\n“Mr. Wickham is blessed with such happy manners as may insure his\r\n_making_ friends; whether he may be equally capable of _retaining_ them,\r\nis less certain.”\r\n\r\n“He has been so unlucky as to lose your friendship,” replied Elizabeth,\r\nwith emphasis, “and in a manner which he is likely to suffer from all\r\nhis life.”\r\n\r\nDarcy made no answer, and seemed desirous of changing the subject. At\r\nthat moment Sir William Lucas appeared close to them, meaning to pass\r\nthrough the set to the other side of the room; but, on perceiving Mr.\r\nDarcy, he stopped, with a bow of superior courtesy, to compliment him on\r\nhis dancing and his partner.\r\n\r\n“I have been most highly gratified, indeed, my dear sir; such very\r\nsuperior dancing is not often seen. It is evident that you belong to the\r\nfirst circles. Allow me to say, however, that your fair partner does not\r\ndisgrace you: and that I must hope to have this pleasure often repeated,\r\nespecially when a certain desirable event, my dear Miss Eliza (glancing\r\nat her sister and Bingley), shall take place. What congratulations will\r\nthen flow in! I appeal to Mr. Darcy;--but let me not interrupt you, sir.\r\nYou will not thank me for detaining you from the bewitching converse of\r\nthat young lady, whose bright eyes are also upbraiding me.”\r\n\r\n[Illustration:\r\n\r\n“Such very superior dancing is not\r\noften seen.”\r\n\r\n[_Copyright 1894 by George Allen._]]\r\n\r\nThe latter part of this address was scarcely heard by Darcy; but Sir\r\nWilliam’s allusion to his friend seemed to strike him forcibly, and his\r\neyes were directed, with a very serious expression, towards Bingley and\r\nJane, who were dancing together. Recovering himself, however, shortly,\r\nhe turned to his partner, and said,--\r\n\r\n“Sir William’s interruption has made me forget what we were talking\r\nof.”\r\n\r\n“I do not think we were speaking at all. Sir William could not have\r\ninterrupted any two people in the room who had less to say for\r\nthemselves. We have tried two or three subjects already without success,\r\nand what we are to talk of next I cannot imagine.”\r\n\r\n“What think you of books?” said he, smiling.\r\n\r\n“Books--oh no!--I am sure we never read the same, or not with the same\r\nfeelings.”\r\n\r\n“I am sorry you think so; but if that be the case, there can at least be\r\nno want of subject. We may compare our different opinions.”\r\n\r\n“No--I cannot talk of books in a ball-room; my head is always full of\r\nsomething else.”\r\n\r\n“The _present_ always occupies you in such scenes--does it?” said he,\r\nwith a look of doubt.\r\n\r\n“Yes, always,” she replied, without knowing what she said; for her\r\nthoughts had wandered far from the subject, as soon afterwards appeared\r\nby her suddenly exclaiming, “I remember hearing you once say, Mr. Darcy,\r\nthat you hardly ever forgave;--that your resentment, once created, was\r\nunappeasable. You are very cautious, I suppose, as to its _being\r\ncreated_?”\r\n\r\n“I am,” said he, with a firm voice.\r\n\r\n“And never allow yourself to be blinded by prejudice?”\r\n\r\n“I hope not.”\r\n\r\n“It is particularly incumbent on those who never change their opinion,\r\nto be secure of judging properly at first.”\r\n\r\n“May I ask to what these questions tend?”\r\n\r\n“Merely to the illustration of _your_ character,” said she, endeavouring\r\nto shake off her gravity. “I am trying to make it out.”\r\n\r\n“And what is your success?”\r\n\r\nShe shook her head. “I do not get on at all. I hear such different\r\naccounts of you as puzzle me exceedingly.”\r\n\r\n“I can readily believe,” answered he, gravely, “that reports may vary\r\ngreatly with respect to me; and I could wish, Miss Bennet, that you were\r\nnot to sketch my character at the present moment, as there is reason to\r\nfear that the performance would reflect no credit on either.”\r\n\r\n“But if I do not take your likeness now, I may never have another\r\nopportunity.”\r\n\r\n“I would by no means suspend any pleasure of yours,” he coldly replied.\r\nShe said no more, and they went down the other dance and parted in\r\nsilence; on each side dissatisfied, though not to an equal degree; for\r\nin Darcy’s breast there was a tolerably powerful feeling towards her,\r\nwhich soon procured her pardon, and directed all his anger against\r\nanother.\r\n\r\nThey had not long separated when Miss Bingley came towards her, and,\r\nwith an expression of civil disdain, thus accosted her,--\r\n\r\n“So, Miss Eliza, I hear you are quite delighted with George Wickham?\r\nYour sister has been talking to me about him, and asking me a thousand\r\nquestions; and I find that the young man forgot to tell you, among his\r\nother communications, that he was the son of old Wickham, the late Mr.\r\nDarcy’s steward. Let me recommend you, however, as a friend, not to give\r\nimplicit confidence to all his assertions; for, as to Mr. Darcy’s using\r\nhim ill, it is perfectly false: for, on the contrary, he has been always\r\nremarkably kind to him, though George Wickham has treated Mr. Darcy in a\r\nmost infamous manner. I do not know the particulars, but I know very\r\nwell that Mr. Darcy is not in the least to blame; that he cannot bear\r\nto hear George Wickham mentioned; and that though my brother thought he\r\ncould not well avoid including him in his invitation to the officers, he\r\nwas excessively glad to find that he had taken himself out of the way.\r\nHis coming into the country at all is a most insolent thing, indeed, and\r\nI wonder how he could presume to do it. I pity you, Miss Eliza, for this\r\ndiscovery of your favourite’s guilt; but really, considering his\r\ndescent, one could not expect much better.”\r\n\r\n“His guilt and his descent appear, by your account, to be the same,”\r\nsaid Elizabeth, angrily; “for I have heard you accuse him of nothing\r\nworse than of being the son of Mr. Darcy’s steward, and of _that_, I can\r\nassure you, he informed me himself.”\r\n\r\n“I beg your pardon,” replied Miss Bingley, turning away with a sneer.\r\n“Excuse my interference; it was kindly meant.”\r\n\r\n“Insolent girl!” said Elizabeth to herself. “You are much mistaken if\r\nyou expect to influence me by such a paltry attack as this. I see\r\nnothing in it but your own wilful ignorance and the malice of Mr.\r\nDarcy.” She then sought her eldest sister, who had undertaken to make\r\ninquiries on the same subject of Bingley. Jane met her with a smile of\r\nsuch sweet complacency, a glow of such happy expression, as sufficiently\r\nmarked how well she was satisfied with the occurrences of the evening.\r\nElizabeth instantly read her feelings; and, at that moment, solicitude\r\nfor Wickham, resentment against his enemies, and everything else, gave\r\nway before the hope of Jane’s being in the fairest way for happiness.\r\n\r\n“I want to know,” said she, with a countenance no less smiling than her\r\nsister’s, “what you have learnt about Mr. Wickham. But perhaps you have\r\nbeen too pleasantly engaged to think of any third person, in which case\r\nyou may be sure of my pardon.”\r\n\r\n“No,” replied Jane, “I have not forgotten him; but I have nothing\r\nsatisfactory to tell you. Mr. Bingley does not know the whole of his\r\nhistory, and is quite ignorant of the circumstances which have\r\nprincipally offended Mr. Darcy; but he will vouch for the good conduct,\r\nthe probity and honour, of his friend, and is perfectly convinced that\r\nMr. Wickham has deserved much less attention from Mr. Darcy than he has\r\nreceived; and I am sorry to say that by his account, as well as his\r\nsister’s, Mr. Wickham is by no means a respectable young man. I am\r\nafraid he has been very imprudent, and has deserved to lose Mr. Darcy’s\r\nregard.”\r\n\r\n“Mr. Bingley does not know Mr. Wickham himself.”\r\n\r\n“No; he never saw him till the other morning at Meryton.”\r\n\r\n“This account then is what he has received from Mr. Darcy. I am\r\nperfectly satisfied. But what does he say of the living?”\r\n\r\n“He does not exactly recollect the circumstances, though he has heard\r\nthem from Mr. Darcy more than once, but he believes that it was left to\r\nhim _conditionally_ only.”\r\n\r\n“I have not a doubt of Mr. Bingley’s sincerity,” said Elizabeth warmly,\r\n“but you must excuse my not being convinced by assurances only. Mr.\r\nBingley’s defence of his friend was a very able one, I dare say; but\r\nsince he is unacquainted with several parts of the story, and has learnt\r\nthe rest from that friend himself, I shall venture still to think of\r\nboth gentlemen as I did before.”\r\n\r\nShe then changed the discourse to one more gratifying to each, and on\r\nwhich there could be no difference of sentiment. Elizabeth listened with\r\ndelight to the happy though modest hopes which Jane entertained of\r\nBingley’s regard, and said all in her power to heighten her confidence\r\nin it. On their being joined by Mr. Bingley himself, Elizabeth withdrew\r\nto Miss Lucas; to whose inquiry after the pleasantness of her last\r\npartner she had scarcely replied, before Mr. Collins came up to them,\r\nand told her with great exultation, that he had just been so fortunate\r\nas to make a most important discovery.\r\n\r\n“I have found out,” said he, “by a singular accident, that there is now\r\nin the room a near relation to my patroness. I happened to overhear the\r\ngentleman himself mentioning to the young lady who does the honours of\r\nthis house the names of his cousin Miss De Bourgh, and of her mother,\r\nLady Catherine. How wonderfully these sort of things occur! Who would\r\nhave thought of my meeting with--perhaps--a nephew of Lady Catherine de\r\nBourgh in this assembly! I am most thankful that the discovery is made\r\nin time for me to pay my respects to him, which I am now going to do,\r\nand trust he will excuse my not having done it before. My total\r\nignorance of the connection must plead my apology.”\r\n\r\n“You are not going to introduce yourself to Mr. Darcy?”\r\n\r\n“Indeed I am. I shall entreat his pardon for not having done it earlier.\r\nI believe him to be Lady Catherine’s _nephew_. It will be in my power to\r\nassure him that her Ladyship was quite well yesterday se’nnight.”\r\n\r\nElizabeth tried hard to dissuade him from such a scheme; assuring him\r\nthat Mr. Darcy would consider his addressing him without introduction as\r\nan impertinent freedom, rather than a compliment to his aunt; that it\r\nwas not in the least necessary there should be any notice on either\r\nside, and that if it were, it must belong to Mr. Darcy, the superior in\r\nconsequence, to begin the acquaintance. Mr. Collins listened to her with\r\nthe determined air of following his own inclination, and when she ceased\r\nspeaking, replied thus,--\r\n\r\n“My dear Miss Elizabeth, I have the highest opinion in the world of your\r\nexcellent judgment in all matters within the scope of your\r\nunderstanding, but permit me to say that there must be a wide difference\r\nbetween the established forms of ceremony amongst the laity and those\r\nwhich regulate the clergy; for, give me leave to observe that I consider\r\nthe clerical office as equal in point of dignity with the highest rank\r\nin the kingdom--provided that a proper humility of behaviour is at the\r\nsame time maintained. You must, therefore, allow me to follow the\r\ndictates of my conscience on this occasion, which lead me to perform\r\nwhat I look on as a point of duty. Pardon me for neglecting to profit by\r\nyour advice, which on every other subject shall be my constant guide,\r\nthough in the case before us I consider myself more fitted by education\r\nand habitual study to decide on what is right than a young lady like\r\nyourself;” and with a low bow he left her to attack Mr. Darcy, whose\r\nreception of his advances she eagerly watched, and whose astonishment at\r\nbeing so addressed was very evident. Her cousin prefaced his speech with\r\na solemn bow, and though she could not hear a word of it, she felt as if\r\nhearing it all, and saw in the motion of his lips the words “apology,”\r\n“Hunsford,” and “Lady Catherine de Bourgh.” It vexed her to see him\r\nexpose himself to such a man. Mr. Darcy was eyeing him with\r\nunrestrained wonder; and when at last Mr. Collins allowed him to speak,\r\nreplied with an air of distant civility. Mr. Collins, however, was not\r\ndiscouraged from speaking again, and Mr. Darcy’s contempt seemed\r\nabundantly increasing with the length of his second speech; and at the\r\nend of it he only made him a slight bow, and moved another way: Mr.\r\nCollins then returned to Elizabeth.\r\n\r\n“I have no reason, I assure you,” said he, “to be dissatisfied with my\r\nreception. Mr. Darcy seemed much pleased with the attention. He answered\r\nme with the utmost civility, and even paid me the compliment of saying,\r\nthat he was so well convinced of Lady Catherine’s discernment as to be\r\ncertain she could never bestow a favour unworthily. It was really a very\r\nhandsome thought. Upon the whole, I am much pleased with him.”\r\n\r\nAs Elizabeth had no longer any interest of her own to pursue, she turned\r\nher attention almost entirely on her sister and Mr. Bingley; and the\r\ntrain of agreeable reflections which her observations gave birth to made\r\nher perhaps almost as happy as Jane. She saw her in idea settled in that\r\nvery house, in all the felicity which a marriage of true affection could\r\nbestow; and she felt capable, under such circumstances, of endeavouring\r\neven to like Bingley’s two sisters. Her mother’s thoughts she plainly\r\nsaw were bent the same way, and she determined not to venture near her,\r\nlest she might hear too much. When they sat down to supper, therefore,\r\nshe considered it a most unlucky perverseness which placed them within\r\none of each other; and deeply was she vexed to find that her mother was\r\ntalking to that one person (Lady Lucas) freely, openly, and of nothing\r\nelse but of her expectation that Jane would be soon married to Mr.\r\nBingley. It was an animating subject, and Mrs. Bennet seemed incapable\r\nof fatigue while enumerating the advantages of the match. His being such\r\na charming young man, and so rich, and living but three miles from them,\r\nwere the first points of self-gratulation; and then it was such a\r\ncomfort to think how fond the two sisters were of Jane, and to be\r\ncertain that they must desire the connection as much as she could do. It\r\nwas, moreover, such a promising thing for her younger daughters, as\r\nJane’s marrying so greatly must throw them in the way of other rich men;\r\nand, lastly, it was so pleasant at her time of life to be able to\r\nconsign her single daughters to the care of their sister, that she might\r\nnot be obliged to go into company more than she liked. It was necessary\r\nto make this circumstance a matter of pleasure, because on such\r\noccasions it is the etiquette; but no one was less likely than Mrs.\r\nBennet to find comfort in staying at home at any period of her life. She\r\nconcluded with many good wishes that Lady Lucas might soon be equally\r\nfortunate, though evidently and triumphantly believing there was no\r\nchance of it.\r\n\r\nIn vain did Elizabeth endeavour to check the rapidity of her mother’s\r\nwords, or persuade her to describe her felicity in a less audible\r\nwhisper; for to her inexpressible vexation she could perceive that the\r\nchief of it was overheard by Mr. Darcy, who sat opposite to them. Her\r\nmother only scolded her for being nonsensical.\r\n\r\n“What is Mr. Darcy to me, pray, that I should be afraid of him? I am\r\nsure we owe him no such particular civility as to be obliged to say\r\nnothing _he_ may not like to hear.”\r\n\r\n“For heaven’s sake, madam, speak lower. What advantage can it be to you\r\nto offend Mr. Darcy? You will never recommend yourself to his friend by\r\nso doing.”\r\n\r\nNothing that she could say, however, had any influence. Her mother would\r\ntalk of her views in the same intelligible tone. Elizabeth blushed and\r\nblushed again with shame and vexation. She could not help frequently\r\nglancing her eye at Mr. Darcy, though every glance convinced her of what\r\nshe dreaded; for though he was not always looking at her mother, she was\r\nconvinced that his attention was invariably fixed by her. The expression\r\nof his face changed gradually from indignant contempt to a composed and\r\nsteady gravity.\r\n\r\nAt length, however, Mrs. Bennet had no more to say; and Lady Lucas, who\r\nhad been long yawning at the repetition of delights which she saw no\r\nlikelihood of sharing, was left to the comforts of cold ham and chicken.\r\nElizabeth now began to revive. But not long was the interval of\r\ntranquillity; for when supper was over, singing was talked of, and she\r\nhad the mortification of seeing Mary, after very little entreaty,\r\npreparing to oblige the company. By many significant looks and silent\r\nentreaties did she endeavour to prevent such a proof of\r\ncomplaisance,--but in vain; Mary would not understand them; such an\r\nopportunity of exhibiting was delightful to her, and she began her song.\r\nElizabeth’s eyes were fixed on her, with most painful sensations; and\r\nshe watched her progress through the several stanzas with an impatience\r\nwhich was very ill rewarded at their close; for Mary, on receiving\r\namongst the thanks of the table the hint of a hope that she might be\r\nprevailed on to favour them again, after the pause of half a minute\r\nbegan another. Mary’s powers were by no means fitted for such a display;\r\nher voice was weak, and her manner affected. Elizabeth was in agonies.\r\nShe looked at Jane to see how she bore it; but Jane was very composedly\r\ntalking to Bingley. She looked at his two sisters, and saw them making\r\nsigns of derision at each other, and at Darcy, who continued, however,\r\nimpenetrably grave. She looked at her father to entreat his\r\ninterference, lest Mary should be singing all night. He took the hint,\r\nand, when Mary had finished her second song, said aloud,--\r\n\r\n“That will do extremely well, child. You have delighted us long enough.\r\nLet the other young ladies have time to exhibit.”\r\n\r\nMary, though pretending not to hear, was somewhat disconcerted; and\r\nElizabeth, sorry for her, and sorry for her father’s speech, was afraid\r\nher anxiety had done no good. Others of the party were now applied to.\r\n\r\n“If I,” said Mr. Collins, “were so fortunate as to be able to sing, I\r\nshould have great pleasure, I am sure, in obliging the company with an\r\nair; for I consider music as a very innocent diversion, and perfectly\r\ncompatible with the profession of a clergyman. I do not mean, however,\r\nto assert that we can be justified in devoting too much of our time to\r\nmusic, for there are certainly other things to be attended to. The\r\nrector of a parish has much to do. In the first place, he must make such\r\nan agreement for tithes as may be beneficial to himself and not\r\noffensive to his patron. He must write his own sermons; and the time\r\nthat remains will not be too much for his parish duties, and the care\r\nand improvement of his dwelling, which he cannot be excused from making\r\nas comfortable as possible. And I do not think it of light importance\r\nthat he should have attentive and conciliatory manners towards\r\neverybody, especially towards those to whom he owes his preferment. I\r\ncannot acquit him of that duty; nor could I think well of the man who\r\nshould omit an occasion of testifying his respect towards anybody\r\nconnected with the family.” And with a bow to Mr. Darcy, he concluded\r\nhis speech, which had been spoken so loud as to be heard by half the\r\nroom. Many stared--many smiled; but no one looked more amused than Mr.\r\nBennet himself, while his wife seriously commended Mr. Collins for\r\nhaving spoken so sensibly, and observed, in a half-whisper to Lady\r\nLucas, that he was a remarkably clever, good kind of young man.\r\n\r\nTo Elizabeth it appeared, that had her family made an agreement to\r\nexpose themselves as much as they could during the evening, it would\r\nhave been impossible for them to play their parts with more spirit, or\r\nfiner success; and happy did she think it for Bingley and her sister\r\nthat some of the exhibition had escaped his notice, and that his\r\nfeelings were not of a sort to be much distressed by the folly which he\r\nmust have witnessed. That his two sisters and Mr. Darcy, however, should\r\nhave such an opportunity of ridiculing her relations was bad enough; and\r\nshe could not determine whether the silent contempt of the gentleman, or\r\nthe insolent smiles of the ladies, were more intolerable.\r\n\r\nThe rest of the evening brought her little amusement. She was teased by\r\nMr. Collins, who continued most perseveringly by her side; and though he\r\ncould not prevail with her to dance with him again, put it out of her\r\npower to dance with others. In vain did she entreat him to stand up with\r\nsomebody else, and offered to introduce him to any young lady in the\r\nroom. He assured her that, as to dancing, he was perfectly indifferent\r\nto it; that his chief object was, by delicate attentions, to recommend\r\nhimself to her; and that he should therefore make a point of remaining\r\nclose to her the whole evening. There was no arguing upon such a\r\nproject. She owed her greatest relief to her friend Miss Lucas, who\r\noften joined them, and good-naturedly engaged Mr. Collins’s conversation\r\nto herself.\r\n\r\nShe was at least free from the offence of Mr. Darcy’s further notice:\r\nthough often standing within a very short distance of her, quite\r\ndisengaged, he never came near enough to speak. She felt it to be the\r\nprobable consequence of her allusions to Mr. Wickham, and rejoiced in\r\nit.\r\n\r\nThe Longbourn party were the last of all the company to depart; and by a\r\nmanœuvre of Mrs. Bennet had to wait for their carriage a quarter of an\r\nhour after everybody else was gone, which gave them time to see how\r\nheartily they were wished away by some of the family. Mrs. Hurst and her\r\nsister scarcely opened their mouths except to complain of fatigue, and\r\nwere evidently impatient to have the house to themselves. They repulsed\r\nevery attempt of Mrs. Bennet at conversation, and, by so doing, threw a\r\nlanguor over the whole party, which was very little relieved by the long\r\nspeeches of Mr. Collins, who was complimenting Mr. Bingley and his\r\nsisters on the elegance of their entertainment, and the hospitality and\r\npoliteness which had marked their behaviour to their guests. Darcy said\r\nnothing at all. Mr. Bennet, in equal silence, was enjoying the scene.\r\nMr. Bingley and Jane were standing together a little detached from the\r\nrest, and talked only to each other. Elizabeth preserved as steady a\r\nsilence as either Mrs. Hurst or Miss Bingley; and even Lydia was too\r\nmuch fatigued to utter more than the occasional exclamation of “Lord,\r\nhow tired I am!” accompanied by a violent yawn.\r\n\r\nWhen at length they arose to take leave, Mrs. Bennet was most pressingly\r\ncivil in her hope of seeing the whole family soon at Longbourn; and\r\naddressed herself particularly to Mr. Bingley, to assure him how happy\r\nhe would make them, by eating a family dinner with them at any time,\r\nwithout the ceremony of a formal invitation. Bingley was all grateful\r\npleasure; and he readily engaged for taking the earliest opportunity of\r\nwaiting on her after his return from London, whither he was obliged to\r\ngo the next day for a short time.\r\n\r\nMrs. Bennet was perfectly satisfied; and quitted the house under the\r\ndelightful persuasion that, allowing for the necessary preparations of\r\nsettlements, new carriages, and wedding clothes, she should undoubtedly\r\nsee her daughter settled at Netherfield in the course of three or four\r\nmonths. Of having another daughter married to Mr. Collins she thought\r\nwith equal certainty, and with considerable, though not equal, pleasure.\r\nElizabeth was the least dear to her of all her children; and though the\r\nman and the match were quite good enough for _her_, the worth of each\r\nwas eclipsed by Mr. Bingley and Netherfield.\r\n\r\n\r\n\r\n\r\n[Illustration:\r\n\r\n “to assure you in the most animated language”\r\n]\r\n\r\n\r\n\r\n\r\nCHAPTER XIX.\r\n\r\n\r\n[Illustration]\r\n\r\nThe next day opened a new scene at Longbourn. Mr. Collins made his\r\ndeclaration in form. Having resolved to do it without loss of time, as\r\nhis leave of absence extended only to the following Saturday, and having\r\nno feelings of diffidence to make it distressing to himself even at the\r\nmoment, he set about it in a very orderly manner, with all the\r\nobservances which he supposed a regular part of the business. On finding\r\nMrs. Bennet, Elizabeth, and one of the younger girls together, soon\r\nafter breakfast, he addressed the mother in these words,--\r\n\r\n“May I hope, madam, for your interest with your fair daughter Elizabeth,\r\nwhen I solicit for the honour of a private audience with her in the\r\ncourse of this morning?”\r\n\r\nBefore Elizabeth had time for anything but a blush of surprise, Mrs.\r\nBennet instantly answered,--\r\n\r\n“Oh dear! Yes, certainly. I am sure Lizzy will be very happy--I am sure\r\nshe can have no objection. Come, Kitty, I want you upstairs.” And\r\ngathering her work together, she was hastening away, when Elizabeth\r\ncalled out,--\r\n\r\n“Dear ma’am, do not go. I beg you will not go. Mr. Collins must excuse\r\nme. He can have nothing to say to me that anybody need not hear. I am\r\ngoing away myself.”\r\n\r\n“No, no, nonsense, Lizzy. I desire you will stay where you are.” And\r\nupon Elizabeth’s seeming really, with vexed and embarrassed looks, about\r\nto escape, she added, “Lizzy, I _insist_ upon your staying and hearing\r\nMr. Collins.”\r\n\r\nElizabeth would not oppose such an injunction; and a moment’s\r\nconsideration making her also sensible that it would be wisest to get it\r\nover as soon and as quietly as possible, she sat down again, and tried\r\nto conceal, by incessant employment, the feelings which were divided\r\nbetween distress and diversion. Mrs. Bennet and Kitty walked off, and as\r\nsoon as they were gone, Mr. Collins began,--\r\n\r\n“Believe me, my dear Miss Elizabeth, that your modesty, so far from\r\ndoing you any disservice, rather adds to your other perfections. You\r\nwould have been less amiable in my eyes had there _not_ been this little\r\nunwillingness; but allow me to assure you that I have your respected\r\nmother’s permission for this address. You can hardly doubt the purport\r\nof my discourse, however your natural delicacy may lead you to\r\ndissemble; my attentions have been too marked to be mistaken. Almost as\r\nsoon as I entered the house I singled you out as the companion of my\r\nfuture life. But before I am run away with by my feelings on this\r\nsubject, perhaps it will be advisable for me to state my reasons for\r\nmarrying--and, moreover, for coming into Hertfordshire with the design\r\nof selecting a wife, as I certainly did.”\r\n\r\nThe idea of Mr. Collins, with all his solemn composure, being run away\r\nwith by his feelings, made Elizabeth so near laughing that she could not\r\nuse the short pause he allowed in any attempt to stop him farther, and\r\nhe continued,--\r\n\r\n“My reasons for marrying are, first, that I think it a right thing for\r\nevery clergyman in easy circumstances (like myself) to set the example\r\nof matrimony in his parish; secondly, that I am convinced it will add\r\nvery greatly to my happiness; and, thirdly, which perhaps I ought to\r\nhave mentioned earlier, that it is the particular advice and\r\nrecommendation of the very noble lady whom I have the honour of calling\r\npatroness. Twice has she condescended to give me her opinion (unasked\r\ntoo!) on this subject; and it was but the very Saturday night before I\r\nleft Hunsford,--between our pools at quadrille, while Mrs. Jenkinson was\r\narranging Miss De Bourgh’s footstool,--that she said, ‘Mr. Collins, you\r\nmust marry. A clergyman like you must marry. Choose properly, choose a\r\ngentlewoman for _my_ sake, and for your _own_; let her be an active,\r\nuseful sort of person, not brought up high, but able to make a small\r\nincome go a good way. This is my advice. Find such a woman as soon as\r\nyou can, bring her to Hunsford, and I will visit her.’ Allow me, by the\r\nway, to observe, my fair cousin, that I do not reckon the notice and\r\nkindness of Lady Catherine de Bourgh as among the least of the\r\nadvantages in my power to offer. You will find her manners beyond\r\nanything I can describe; and your wit and vivacity, I think, must be\r\nacceptable to her, especially when tempered with the silence and respect\r\nwhich her rank will inevitably excite. Thus much for my general\r\nintention in favour of matrimony; it remains to be told why my views\r\nwere directed to Longbourn instead of my own neighbourhood, where I\r\nassure you there are many amiable young women. But the fact is, that\r\nbeing, as I am, to inherit this estate after the death of your honoured\r\nfather (who, however, may live many years longer), I could not satisfy\r\nmyself without resolving to choose a wife from among his daughters, that\r\nthe loss to them might be as little as possible when the melancholy\r\nevent takes place--which, however, as I have already said, may not be\r\nfor several years. This has been my motive, my fair cousin, and I\r\nflatter myself it will not sink me in your esteem. And now nothing\r\nremains for me but to assure you in the most animated language of the\r\nviolence of my affection. To fortune I am perfectly indifferent, and\r\nshall make no demand of that nature on your father, since I am well\r\naware that it could not be complied with; and that one thousand pounds\r\nin the 4 per cents., which will not be yours till after your mother’s\r\ndecease, is all that you may ever be entitled to. On that head,\r\ntherefore, I shall be uniformly silent: and you may assure yourself that\r\nno ungenerous reproach shall ever pass my lips when we are married.”\r\n\r\nIt was absolutely necessary to interrupt him now.\r\n\r\n“You are too hasty, sir,” she cried. “You forget that I have made no\r\nanswer. Let me do it without further loss of time. Accept my thanks for\r\nthe compliment you are paying me. I am very sensible of the honour of\r\nyour proposals, but it is impossible for me to do otherwise than decline\r\nthem.”\r\n\r\n“I am not now to learn,” replied Mr. Collins, with a formal wave of the\r\nhand, “that it is usual with young ladies to reject the addresses of the\r\nman whom they secretly mean to accept, when he first applies for their\r\nfavour; and that sometimes the refusal is repeated a second or even a\r\nthird time. I am, therefore, by no means discouraged by what you have\r\njust said, and shall hope to lead you to the altar ere long.”\r\n\r\n“Upon my word, sir,” cried Elizabeth, “your hope is rather an\r\nextraordinary one after my declaration. I do assure you that I am not\r\none of those young ladies (if such young ladies there are) who are so\r\ndaring as to risk their happiness on the chance of being asked a second\r\ntime. I am perfectly serious in my refusal. You could not make _me_\r\nhappy, and I am convinced that I am the last woman in the world who\r\nwould make _you_ so. Nay, were your friend Lady Catherine to know me, I\r\nam persuaded she would find me in every respect ill qualified for the\r\nsituation.”\r\n\r\n“Were it certain that Lady Catherine would think so,” said Mr. Collins,\r\nvery gravely--“but I cannot imagine that her Ladyship would at all\r\ndisapprove of you. And you may be certain that when I have the honour of\r\nseeing her again I shall speak in the highest terms of your modesty,\r\neconomy, and other amiable qualifications.”\r\n\r\n“Indeed, Mr. Collins, all praise of me will be unnecessary. You must\r\ngive me leave to judge for myself, and pay me the compliment of\r\nbelieving what I say. I wish you very happy and very rich, and by\r\nrefusing your hand, do all in my power to prevent your being otherwise.\r\nIn making me the offer, you must have satisfied the delicacy of your\r\nfeelings with regard to my family, and may take possession of Longbourn\r\nestate whenever it falls, without any self-reproach. This matter may be\r\nconsidered, therefore, as finally settled.” And rising as she thus\r\nspoke, she would have quitted the room, had not Mr. Collins thus\r\naddressed her,--\r\n\r\n“When I do myself the honour of speaking to you next on the subject, I\r\nshall hope to receive a more favourable answer than you have now given\r\nme; though I am far from accusing you of cruelty at present, because I\r\nknow it to be the established custom of your sex to reject a man on the\r\nfirst application, and, perhaps, you have even now said as much to\r\nencourage my suit as would be consistent with the true delicacy of the\r\nfemale character.”\r\n\r\n“Really, Mr. Collins,” cried Elizabeth, with some warmth, “you puzzle me\r\nexceedingly. If what I have hitherto said can appear to you in the form\r\nof encouragement, I know not how to express my refusal in such a way as\r\nmay convince you of its being one.”\r\n\r\n“You must give me leave to flatter myself, my dear cousin, that your\r\nrefusal of my addresses are merely words of course. My reasons for\r\nbelieving it are briefly these:--It does not appear to me that my hand\r\nis unworthy of your acceptance, or that the establishment I can offer\r\nwould be any other than highly desirable. My situation in life, my\r\nconnections with the family of De Bourgh, and my relationship to your\r\nown, are circumstances highly in my favour; and you should take it into\r\nfurther consideration that, in spite of your manifold attractions, it is\r\nby no means certain that another offer of marriage may ever be made you.\r\nYour portion is unhappily so small, that it will in all likelihood undo\r\nthe effects of your loveliness and amiable qualifications. As I must,\r\ntherefore, conclude that you are not serious in your rejection of me, I\r\nshall choose to attribute it to your wish of increasing my love by\r\nsuspense, according to the usual practice of elegant females.”\r\n\r\n“I do assure you, sir, that I have no pretensions whatever to that kind\r\nof elegance which consists in tormenting a respectable man. I would\r\nrather be paid the compliment of being believed sincere. I thank you\r\nagain and again for the honour you have done me in your proposals, but\r\nto accept them is absolutely impossible. My feelings in every respect\r\nforbid it. Can I speak plainer? Do not consider me now as an elegant\r\nfemale intending to plague you, but as a rational creature speaking the\r\ntruth from her heart.”\r\n\r\n“You are uniformly charming!” cried he, with an air of awkward\r\ngallantry; “and I am persuaded that, when sanctioned by the express\r\nauthority of both your excellent parents, my proposals will not fail of\r\nbeing acceptable.”\r\n\r\nTo such perseverance in wilful self-deception Elizabeth would make no\r\nreply, and immediately and in silence withdrew; determined, that if he\r\npersisted in considering her repeated refusals as flattering\r\nencouragement, to apply to her father, whose negative might be uttered\r\nin such a manner as must be decisive, and whose behaviour at least could\r\nnot be mistaken for the affectation and coquetry of an elegant female.\r\n\r\n\r\n\r\n\r\n[Illustration]\r\n\r\n\r\n\r\n\r\nCHAPTER XX.\r\n\r\n\r\n[Illustration]\r\n\r\nMr. Collins was not left long to the silent contemplation of his\r\nsuccessful love; for Mrs. Bennet, having dawdled about in the vestibule\r\nto watch for the end of the conference, no sooner saw Elizabeth open the\r\ndoor and with quick step pass her towards the staircase, than she\r\nentered the breakfast-room, and congratulated both him and herself in\r\nwarm terms on the happy prospect of their nearer connection. Mr. Collins\r\nreceived and returned these felicitations with equal pleasure, and then\r\nproceeded to relate the particulars of their interview, with the result\r\nof which he trusted he had every reason to be satisfied, since the\r\nrefusal which his cousin had steadfastly given him would naturally flow\r\nfrom her bashful modesty and the genuine delicacy of her character.\r\n\r\nThis information, however, startled Mrs. Bennet: she would have been\r\nglad to be equally satisfied that her daughter had meant to encourage\r\nhim by protesting against his proposals, but she dared not believe it,\r\nand could not help saying so.\r\n\r\n“But depend upon it, Mr. Collins,” she added, “that Lizzy shall be\r\nbrought to reason. I will speak to her about it myself directly. She is\r\na very headstrong, foolish girl, and does not know her own interest; but\r\nI will _make_ her know it.”\r\n\r\n“Pardon me for interrupting you, madam,” cried Mr. Collins; “but if she\r\nis really headstrong and foolish, I know not whether she would\r\naltogether be a very desirable wife to a man in my situation, who\r\nnaturally looks for happiness in the marriage state. If, therefore, she\r\nactually persists in rejecting my suit, perhaps it were better not to\r\nforce her into accepting me, because, if liable to such defects of\r\ntemper, she could not contribute much to my felicity.”\r\n\r\n“Sir, you quite misunderstand me,” said Mrs. Bennet, alarmed. “Lizzy is\r\nonly headstrong in such matters as these. In everything else she is as\r\ngood-natured a girl as ever lived. I will go directly to Mr. Bennet, and\r\nwe shall very soon settle it with her, I am sure.”\r\n\r\nShe would not give him time to reply, but hurrying instantly to her\r\nhusband, called out, as she entered the library,--\r\n\r\n“Oh, Mr. Bennet, you are wanted immediately; we are all in an uproar.\r\nYou must come and make Lizzy marry Mr. Collins, for she vows she will\r\nnot have him; and if you do not make haste he will change his mind and\r\nnot have _her_.”\r\n\r\nMr. Bennet raised his eyes from his book as she entered, and fixed them\r\non her face with a calm unconcern, which was not in the least altered by\r\nher communication.\r\n\r\n“I have not the pleasure of understanding you,” said he, when she had\r\nfinished her speech. “Of what are you talking?”\r\n\r\n“Of Mr. Collins and Lizzy. Lizzy declares she will not have Mr. Collins,\r\nand Mr. Collins begins to say that he will not have Lizzy.”\r\n\r\n“And what am I to do on the occasion? It seems a hopeless business.”\r\n\r\n“Speak to Lizzy about it yourself. Tell her that you insist upon her\r\nmarrying him.”\r\n\r\n“Let her be called down. She shall hear my opinion.”\r\n\r\nMrs. Bennet rang the bell, and Miss Elizabeth was summoned to the\r\nlibrary.\r\n\r\n“Come here, child,” cried her father as she appeared. “I have sent for\r\nyou on an affair of importance. I understand that Mr. Collins has made\r\nyou an offer of marriage. Is it true?”\r\n\r\nElizabeth replied that it was.\r\n\r\n“Very well--and this offer of marriage you have refused?”\r\n\r\n“I have, sir.”\r\n\r\n“Very well. We now come to the point. Your mother insists upon your\r\naccepting it. Is it not so, Mrs. Bennet?”\r\n\r\n“Yes, or I will never see her again.”\r\n\r\n“An unhappy alternative is before you, Elizabeth. From this day you must\r\nbe a stranger to one of your parents. Your mother will never see you\r\nagain if you do _not_ marry Mr. Collins, and I will never see you again\r\nif you _do_.”\r\n\r\nElizabeth could not but smile at such a conclusion of such a beginning;\r\nbut Mrs. Bennet, who had persuaded herself that her husband regarded the\r\naffair as she wished, was excessively disappointed.\r\n\r\n“What do you mean, Mr. Bennet, by talking in this way? You promised me\r\nto _insist_ upon her marrying him.”\r\n\r\n“My dear,” replied her husband, “I have two small favours to request.\r\nFirst, that you will allow me the free use of my understanding on the\r\npresent occasion; and, secondly, of my room. I shall be glad to have the\r\nlibrary to myself as soon as may be.”\r\n\r\nNot yet, however, in spite of her disappointment in her husband, did\r\nMrs. Bennet give up the point. She talked to Elizabeth again and again;\r\ncoaxed and threatened her by turns. She endeavoured to secure Jane in\r\nher interest, but Jane, with all possible mildness, declined\r\ninterfering; and Elizabeth, sometimes with real earnestness, and\r\nsometimes with playful gaiety, replied to her attacks. Though her manner\r\nvaried, however, her determination never did.\r\n\r\nMr. Collins, meanwhile, was meditating in solitude on what had passed.\r\nHe thought too well of himself to comprehend on what motive his cousin\r\ncould refuse him; and though his pride was hurt, he suffered in no other\r\nway. His regard for her was quite imaginary; and the possibility of her\r\ndeserving her mother’s reproach prevented his feeling any regret.\r\n\r\nWhile the family were in this confusion, Charlotte Lucas came to spend\r\nthe day with them. She was met in the vestibule by Lydia, who, flying to\r\nher, cried in a half whisper, “I am glad you are come, for there is such\r\nfun here! What do you think has happened this morning? Mr. Collins has\r\nmade an offer to Lizzy, and she will not have him.”\r\n\r\n[Illustration:\r\n\r\n “they entered the breakfast room”\r\n]\r\n\r\nCharlotte had hardly time to answer before they were joined by Kitty,\r\nwho came to tell the same news; and no sooner had they entered the\r\nbreakfast-room, where Mrs. Bennet was alone, than she likewise began on\r\nthe subject, calling on Miss Lucas for her compassion, and entreating\r\nher to persuade her friend Lizzy to comply with the wishes of her\r\nfamily. “Pray do, my dear Miss Lucas,” she added, in a melancholy tone;\r\n“for nobody is on my side, nobody takes part with me; I am cruelly used,\r\nnobody feels for my poor nerves.”\r\n\r\nCharlotte’s reply was spared by the entrance of Jane and Elizabeth.\r\n\r\n“Ay, there she comes,” continued Mrs. Bennet, “looking as unconcerned as\r\nmay be, and caring no more for us than if we were at York, provided she\r\ncan have her own way. But I tell you what, Miss Lizzy, if you take it\r\ninto your head to go on refusing every offer of marriage in this way,\r\nyou will never get a husband at all--and I am sure I do not know who is\r\nto maintain you when your father is dead. _I_ shall not be able to keep\r\nyou--and so I warn you. I have done with you from this very day. I told\r\nyou in the library, you know, that I should never speak to you again,\r\nand you will find me as good as my word. I have no pleasure in talking\r\nto undutiful children. Not that I have much pleasure, indeed, in talking\r\nto anybody. People who suffer as I do from nervous complaints can have\r\nno great inclination for talking. Nobody can tell what I suffer! But it\r\nis always so. Those who do not complain are never pitied.”\r\n\r\nHer daughters listened in silence to this effusion, sensible that any\r\nattempt to reason with or soothe her would only increase the irritation.\r\nShe talked on, therefore, without interruption from any of them till\r\nthey were joined by Mr. Collins, who entered with an air more stately\r\nthan usual, and on perceiving whom, she said to the girls,--\r\n\r\n“Now, I do insist upon it, that you, all of you, hold your tongues, and\r\nlet Mr. Collins and me have a little conversation together.”\r\n\r\nElizabeth passed quietly out of the room, Jane and Kitty followed, but\r\nLydia stood her ground, determined to hear all she could; and Charlotte,\r\ndetained first by the civility of Mr. Collins, whose inquiries after\r\nherself and all her family were very minute, and then by a little\r\ncuriosity, satisfied herself with walking to the window and pretending\r\nnot to hear. In a doleful voice Mrs. Bennet thus began the projected\r\nconversation:--\r\n\r\n“Oh, Mr. Collins!”\r\n\r\n“My dear madam,” replied he, “let us be for ever silent on this point.\r\nFar be it from me,” he presently continued, in a voice that marked his\r\ndispleasure, “to resent the behaviour of your daughter. Resignation to\r\ninevitable evils is the duty of us all: the peculiar duty of a young man\r\nwho has been so fortunate as I have been, in early preferment; and, I\r\ntrust, I am resigned. Perhaps not the less so from feeling a doubt of my\r\npositive happiness had my fair cousin honoured me with her hand; for I\r\nhave often observed, that resignation is never so perfect as when the\r\nblessing denied begins to lose somewhat of its value in our estimation.\r\nYou will not, I hope, consider me as showing any disrespect to your\r\nfamily, my dear madam, by thus withdrawing my pretensions to your\r\ndaughter’s favour, without having paid yourself and Mr. Bennet the\r\ncompliment of requesting you to interpose your authority in my behalf.\r\nMy conduct may, I fear, be objectionable in having accepted my\r\ndismission from your daughter’s lips instead of your own; but we are all\r\nliable to error. I have certainly meant well through the whole affair.\r\nMy object has been to secure an amiable companion for myself, with due\r\nconsideration for the advantage of all your family; and if my _manner_\r\nhas been at all reprehensible, I here beg leave to apologize.”\r\n\r\n\r\n\r\n\r\n[Illustration]\r\n\r\n\r\n\r\n\r\nCHAPTER XXI.\r\n\r\n\r\n[Illustration]\r\n\r\nThe discussion of Mr. Collins’s offer was now nearly at an end, and\r\nElizabeth had only to suffer from the uncomfortable feelings necessarily\r\nattending it, and occasionally from some peevish allusion of her mother.\r\nAs for the gentleman himself, _his_ feelings were chiefly expressed, not\r\nby embarrassment or dejection, or by trying to avoid her, but by\r\nstiffness of manner and resentful silence. He scarcely ever spoke to\r\nher; and the assiduous attentions which he had been so sensible of\r\nhimself were transferred for the rest of the day to Miss Lucas, whose\r\ncivility in listening to him was a seasonable relief to them all, and\r\nespecially to her friend.\r\n\r\nThe morrow produced no abatement of Mrs. Bennet’s ill humour or ill\r\nhealth. Mr. Collins was also in the same state of angry pride. Elizabeth\r\nhad hoped that his resentment might shorten his visit, but his plan did\r\nnot appear in the least affected by it. He was always to have gone on\r\nSaturday, and to Saturday he still meant to stay.\r\n\r\nAfter breakfast, the girls walked to Meryton, to inquire if Mr. Wickham\r\nwere returned, and to lament over his absence from the Netherfield ball.\r\nHe joined them on their entering the town, and attended them to their\r\naunt’s, where his regret and vexation and the concern of everybody were\r\nwell talked over. To Elizabeth, however, he voluntarily acknowledged\r\nthat the necessity of his absence _had_ been self-imposed.\r\n\r\n“I found,” said he, “as the time drew near, that I had better not meet\r\nMr. Darcy;--that to be in the same room, the same party with him for so\r\nmany hours together, might be more than I could bear, and that scenes\r\nmight arise unpleasant to more than myself.”\r\n\r\nShe highly approved his forbearance; and they had leisure for a full\r\ndiscussion of it, and for all the commendations which they civilly\r\nbestowed on each other, as Wickham and another officer walked back with\r\nthem to Longbourn, and during the walk he particularly attended to her.\r\nHis accompanying them was a double advantage: she felt all the\r\ncompliment it offered to herself; and it was most acceptable as an\r\noccasion of introducing him to her father and mother.\r\n\r\n[Illustration: “Walked back with them”\r\n\r\n[_Copyright 1894 by George Allen._]]\r\n\r\nSoon after their return, a letter was delivered to Miss Bennet; it came\r\nfrom Netherfield, and was opened immediately. The envelope contained a\r\nsheet of elegant, little, hot-pressed paper, well covered with a lady’s\r\nfair, flowing hand; and Elizabeth saw her sister’s countenance change as\r\nshe read it, and saw her dwelling intently on some particular passages.\r\nJane recollected herself soon; and putting the letter away, tried to\r\njoin, with her usual cheerfulness, in the general conversation: but\r\nElizabeth felt an anxiety on the subject which drew off her attention\r\neven from Wickham; and no sooner had he and his companion taken leave,\r\nthan a glance from Jane invited her to follow her upstairs. When they\r\nhad gained their own room, Jane, taking out her letter, said, “This is\r\nfrom Caroline Bingley: what it contains has surprised me a good deal.\r\nThe whole party have left Netherfield by this time, and are on their way\r\nto town; and without any intention of coming back again. You shall hear\r\nwhat she says.”\r\n\r\nShe then read the first sentence aloud, which comprised the information\r\nof their having just resolved to follow their brother to town directly,\r\nand of their meaning to dine that day in Grosvenor Street, where Mr.\r\nHurst had a house. The next was in these words:--“‘I do not pretend to\r\nregret anything I shall leave in Hertfordshire except your society, my\r\ndearest friend; but we will hope, at some future period, to enjoy many\r\nreturns of that delightful intercourse we have known, and in the\r\nmeanwhile may lessen the pain of separation by a very frequent and most\r\nunreserved correspondence. I depend on you for that.’” To these\r\nhigh-flown expressions Elizabeth listened with all the insensibility of\r\ndistrust; and though the suddenness of their removal surprised her, she\r\nsaw nothing in it really to lament: it was not to be supposed that their\r\nabsence from Netherfield would prevent Mr. Bingley’s being there; and as\r\nto the loss of their society, she was persuaded that Jane must soon\r\ncease to regard it in the enjoyment of his.\r\n\r\n“It is unlucky,” said she, after a short pause, “that you should not be\r\nable to see your friends before they leave the country. But may we not\r\nhope that the period of future happiness, to which Miss Bingley looks\r\nforward, may arrive earlier than she is aware, and that the delightful\r\nintercourse you have known as friends will be renewed with yet greater\r\nsatisfaction as sisters? Mr. Bingley will not be detained in London by\r\nthem.”\r\n\r\n“Caroline decidedly says that none of the party will return into\r\nHertfordshire this winter. I will read it to you.\r\n\r\n“‘When my brother left us yesterday, he imagined that the business which\r\ntook him to London might be concluded in three or four days; but as we\r\nare certain it cannot be so, and at the same time convinced that when\r\nCharles gets to town he will be in no hurry to leave it again, we have\r\ndetermined on following him thither, that he may not be obliged to spend\r\nhis vacant hours in a comfortless hotel. Many of my acquaintance are\r\nalready there for the winter: I wish I could hear that you, my dearest\r\nfriend, had any intention of making one in the crowd, but of that I\r\ndespair. I sincerely hope your Christmas in Hertfordshire may abound in\r\nthe gaieties which that season generally brings, and that your beaux\r\nwill be so numerous as to prevent your feeling the loss of the three of\r\nwhom we shall deprive you.’\r\n\r\n“It is evident by this,” added Jane, “that he comes back no more this\r\nwinter.”\r\n\r\n“It is only evident that Miss Bingley does not mean he _should_.”\r\n\r\n“Why will you think so? It must be his own doing; he is his own master.\r\nBut you do not know _all_. I _will_ read you the passage which\r\nparticularly hurts me. I will have no reserves from _you_. ‘Mr. Darcy is\r\nimpatient to see his sister; and to confess the truth, _we_ are scarcely\r\nless eager to meet her again. I really do not think Georgiana Darcy has\r\nher equal for beauty, elegance, and accomplishments; and the affection\r\nshe inspires in Louisa and myself is heightened into something still\r\nmore interesting from the hope we dare to entertain of her being\r\nhereafter our sister. I do not know whether I ever before mentioned to\r\nyou my feelings on this subject, but I will not leave the country\r\nwithout confiding them, and I trust you will not esteem them\r\nunreasonable. My brother admires her greatly already; he will have\r\nfrequent opportunity now of seeing her on the most intimate footing; her\r\nrelations all wish the connection as much as his own; and a sister’s\r\npartiality is not misleading me, I think, when I call Charles most\r\ncapable of engaging any woman’s heart. With all these circumstances to\r\nfavour an attachment, and nothing to prevent it, am I wrong, my dearest\r\nJane, in indulging the hope of an event which will secure the happiness\r\nof so many?’ What think you of _this_ sentence, my dear Lizzy?” said\r\nJane, as she finished it. “Is it not clear enough? Does it not expressly\r\ndeclare that Caroline neither expects nor wishes me to be her sister;\r\nthat she is perfectly convinced of her brother’s indifference; and that\r\nif she suspects the nature of my feelings for him she means (most\r\nkindly!) to put me on my guard. Can there be any other opinion on the\r\nsubject?”\r\n\r\n“Yes, there can; for mine is totally different. Will you hear it?”\r\n\r\n“Most willingly.”\r\n\r\n“You shall have it in a few words. Miss Bingley sees that her brother is\r\nin love with you and wants him to marry Miss Darcy. She follows him to\r\ntown in the hope of keeping him there, and tries to persuade you that he\r\ndoes not care about you.”\r\n\r\nJane shook her head.\r\n\r\n“Indeed, Jane, you ought to believe me. No one who has ever seen you\r\ntogether can doubt his affection; Miss Bingley, I am sure, cannot: she\r\nis not such a simpleton. Could she have seen half as much love in Mr.\r\nDarcy for herself, she would have ordered her wedding clothes. But the\r\ncase is this:--we are not rich enough or grand enough for them; and she\r\nis the more anxious to get Miss Darcy for her brother, from the notion\r\nthat when there has been _one_ inter-marriage, she may have less trouble\r\nin achieving a second; in which there is certainly some ingenuity, and I\r\ndare say it would succeed if Miss de Bourgh were out of the way. But, my\r\ndearest Jane, you cannot seriously imagine that, because Miss Bingley\r\ntells you her brother greatly admires Miss Darcy, he is in the smallest\r\ndegree less sensible of _your_ merit than when he took leave of you on\r\nTuesday; or that it will be in her power to persuade him that, instead\r\nof being in love with you, he is very much in love with her friend.”\r\n\r\n“If we thought alike of Miss Bingley,” replied Jane, “your\r\nrepresentation of all this might make me quite easy. But I know the\r\nfoundation is unjust. Caroline is incapable of wilfully deceiving\r\nanyone; and all that I can hope in this case is, that she is deceived\r\nherself.”\r\n\r\n“That is right. You could not have started a more happy idea, since you\r\nwill not take comfort in mine: believe her to be deceived, by all means.\r\nYou have now done your duty by her, and must fret no longer.”\r\n\r\n“But, my dear sister, can I be happy, even supposing the best, in\r\naccepting a man whose sisters and friends are all wishing him to marry\r\nelsewhere?”\r\n\r\n“You must decide for yourself,” said Elizabeth; “and if, upon mature\r\ndeliberation, you find that the misery of disobliging his two sisters is\r\nmore than equivalent to the happiness of being his wife, I advise you,\r\nby all means, to refuse him.”\r\n\r\n“How can you talk so?” said Jane, faintly smiling; “you must know, that,\r\nthough I should be exceedingly grieved at their disapprobation, I could\r\nnot hesitate.”\r\n\r\n“I did not think you would; and that being the case, I cannot consider\r\nyour situation with much compassion.”\r\n\r\n“But if he returns no more this winter, my choice will never be\r\nrequired. A thousand things may arise in six months.”\r\n\r\nThe idea of his returning no more Elizabeth treated with the utmost\r\ncontempt. It appeared to her merely the suggestion of Caroline’s\r\ninterested wishes; and she could not for a moment suppose that those\r\nwishes, however openly or artfully spoken, could influence a young man\r\nso totally independent of everyone.\r\n\r\nShe represented to her sister, as forcibly as possible, what she felt on\r\nthe subject, and had soon the pleasure of seeing its happy effect.\r\nJane’s temper was not desponding; and she was gradually led to hope,\r\nthough the diffidence of affection sometimes overcame the hope, that\r\nBingley would return to Netherfield, and answer every wish of her heart.\r\n\r\nThey agreed that Mrs. Bennet should only hear of the departure of the\r\nfamily, without being alarmed on the score of the gentleman’s conduct;\r\nbut even this partial communication gave her a great deal of concern,\r\nand she bewailed it as exceedingly unlucky that the ladies should happen\r\nto go away just as they were all getting so intimate together. After\r\nlamenting it, however, at some length, she had the consolation of\r\nthinking that Mr. Bingley would be soon down again, and soon dining at\r\nLongbourn; and the conclusion of all was the comfortable declaration,\r\nthat, though he had been invited only to a family dinner, she would take\r\ncare to have two full courses.\r\n\r\n\r\n\r\n\r\n[Illustration]\r\n\r\n\r\n\r\n\r\nCHAPTER XXII.\r\n\r\n\r\n[Illustration]\r\n\r\nThe Bennets were engaged to dine with the Lucases; and again, during the\r\nchief of the day, was Miss Lucas so kind as to listen to Mr. Collins.\r\nElizabeth took an opportunity of thanking her. “It keeps him in good\r\nhumour,” said she, “and I am more obliged to you than I can express.”\r\n\r\nCharlotte assured her friend of her satisfaction in being useful, and\r\nthat it amply repaid her for the little sacrifice of her time. This was\r\nvery amiable; but Charlotte’s kindness extended farther than Elizabeth\r\nhad any conception of:--its object was nothing less than to secure her\r\nfrom any return of Mr. Collins’s addresses, by engaging them towards\r\nherself. Such was Miss Lucas’s scheme; and appearances were so\r\nfavourable, that when they parted at night, she would have felt almost\r\nsure of success if he had not been to leave Hertfordshire so very soon.\r\nBut here she did injustice to the fire and independence of his\r\ncharacter; for it led him to escape out of Longbourn House the next\r\nmorning with admirable slyness, and hasten to Lucas Lodge to throw\r\nhimself at her feet. He was anxious to avoid the notice of his cousins,\r\nfrom a conviction that, if they saw him depart, they could not fail to\r\nconjecture his design, and he was not willing to have the attempt known\r\ntill its success could be known likewise; for, though feeling almost\r\nsecure, and with reason, for Charlotte had been tolerably encouraging,\r\nhe was comparatively diffident since the adventure of Wednesday. His\r\nreception, however, was of the most flattering kind. Miss Lucas\r\nperceived him from an upper window as he walked towards the house, and\r\ninstantly set out to meet him accidentally in the lane. But little had\r\nshe dared to hope that so much love and eloquence awaited her there.\r\n\r\nIn as short a time as Mr. Collins’s long speeches would allow,\r\neverything was settled between them to the satisfaction of both; and as\r\nthey entered the house, he earnestly entreated her to name the day that\r\nwas to make him the happiest of men; and though such a solicitation must\r\nbe waived for the present, the lady felt no inclination to trifle with\r\nhis happiness. The stupidity with which he was favoured by nature must\r\nguard his courtship from any charm that could make a woman wish for its\r\ncontinuance; and Miss Lucas, who accepted him solely from the pure and\r\ndisinterested desire of an establishment, cared not how soon that\r\nestablishment were gained.\r\n\r\nSir William and Lady Lucas were speedily applied to for their consent;\r\nand it was bestowed with a most joyful alacrity. Mr. Collins’s present\r\ncircumstances made it a most eligible match for their daughter, to whom\r\nthey could give little fortune; and his prospects of future wealth were\r\nexceedingly fair. Lady Lucas began directly to calculate, with more\r\ninterest than the matter had ever\r\n\r\n[Illustration:\r\n\r\n “So much love and eloquence”\r\n\r\n[_Copyright 1894 by George Allen._]]\r\n\r\nexcited before, how many years longer Mr. Bennet was likely to live; and\r\nSir William gave it as his decided opinion, that whenever Mr. Collins\r\nshould be in possession of the Longbourn estate, it would be highly\r\nexpedient that both he and his wife should make their appearance at St.\r\nJames’s. The whole family in short were properly overjoyed on the\r\noccasion. The younger girls formed hopes of _coming out_ a year or two\r\nsooner than they might otherwise have done; and the boys were relieved\r\nfrom their apprehension of Charlotte’s dying an old maid. Charlotte\r\nherself was tolerably composed. She had gained her point, and had time\r\nto consider of it. Her reflections were in general satisfactory. Mr.\r\nCollins, to be sure, was neither sensible nor agreeable: his society was\r\nirksome, and his attachment to her must be imaginary. But still he would\r\nbe her husband. Without thinking highly either of men or of matrimony,\r\nmarriage had always been her object: it was the only honourable\r\nprovision for well-educated young women of small fortune, and, however\r\nuncertain of giving happiness, must be their pleasantest preservative\r\nfrom want. This preservative she had now obtained; and at the age of\r\ntwenty-seven, without having ever been handsome, she felt all the good\r\nluck of it. The least agreeable circumstance in the business was the\r\nsurprise it must occasion to Elizabeth Bennet, whose friendship she\r\nvalued beyond that of any other person. Elizabeth would wonder, and\r\nprobably would blame her; and though her resolution was not to be\r\nshaken, her feelings must be hurt by such a disapprobation. She resolved\r\nto give her the information herself; and therefore charged Mr. Collins,\r\nwhen he returned to Longbourn to dinner, to drop no hint of what had\r\npassed before any of the family. A promise of secrecy was of course very\r\ndutifully given, but it could not be kept without difficulty; for the\r\ncuriosity excited by his long absence burst forth in such very direct\r\nquestions on his return, as required some ingenuity to evade, and he was\r\nat the same time exercising great self-denial, for he was longing to\r\npublish his prosperous love.\r\n\r\nAs he was to begin his journey too early on the morrow to see any of\r\nthe family, the ceremony of leave-taking was performed when the ladies\r\nmoved for the night; and Mrs. Bennet, with great politeness and\r\ncordiality, said how happy they should be to see him at Longbourn again,\r\nwhenever his other engagements might allow him to visit them.\r\n\r\n“My dear madam,” he replied, “this invitation is particularly\r\ngratifying, because it is what I have been hoping to receive; and you\r\nmay be very certain that I shall avail myself of it as soon as\r\npossible.”\r\n\r\nThey were all astonished; and Mr. Bennet, who could by no means wish for\r\nso speedy a return, immediately said,--\r\n\r\n“But is there not danger of Lady Catherine’s disapprobation here, my\r\ngood sir? You had better neglect your relations than run the risk of\r\noffending your patroness.”\r\n\r\n“My dear sir,” replied Mr. Collins, “I am particularly obliged to you\r\nfor this friendly caution, and you may depend upon my not taking so\r\nmaterial a step without her Ladyship’s concurrence.”\r\n\r\n“You cannot be too much on your guard. Risk anything rather than her\r\ndispleasure; and if you find it likely to be raised by your coming to us\r\nagain, which I should think exceedingly probable, stay quietly at home,\r\nand be satisfied that _we_ shall take no offence.”\r\n\r\n“Believe me, my dear sir, my gratitude is warmly excited by such\r\naffectionate attention; and, depend upon it, you will speedily receive\r\nfrom me a letter of thanks for this as well as for every other mark of\r\nyour regard during my stay in Hertfordshire. As for my fair cousins,\r\nthough my absence may not be long enough to render it necessary, I shall\r\nnow take the liberty of wishing them health and happiness, not excepting\r\nmy cousin Elizabeth.”\r\n\r\nWith proper civilities, the ladies then withdrew; all of them equally\r\nsurprised to find that he meditated a quick return. Mrs. Bennet wished\r\nto understand by it that he thought of paying his addresses to one of\r\nher younger girls, and Mary might have been prevailed on to accept him.\r\nShe rated his abilities much higher than any of the others: there was a\r\nsolidity in his reflections which often struck her; and though by no\r\nmeans so clever as herself, she thought that, if encouraged to read and\r\nimprove himself by such an example as hers, he might become a very\r\nagreeable companion. But on the following morning every hope of this\r\nkind was done away. Miss Lucas called soon after breakfast, and in a\r\nprivate conference with Elizabeth related the event of the day before.\r\n\r\nThe possibility of Mr. Collins’s fancying himself in love with her\r\nfriend had once occurred to Elizabeth within the last day or two: but\r\nthat Charlotte could encourage him seemed almost as far from possibility\r\nas that she could encourage him herself; and her astonishment was\r\nconsequently so great as to overcome at first the bounds of decorum, and\r\nshe could not help crying out,--\r\n\r\n“Engaged to Mr. Collins! my dear Charlotte, impossible!”\r\n\r\nThe steady countenance which Miss Lucas had commanded in telling her\r\nstory gave way to a momentary confusion here on receiving so direct a\r\nreproach; though, as it was no more than she expected, she soon regained\r\nher composure, and calmly replied,--\r\n\r\n“Why should you be surprised, my dear Eliza? Do you think it incredible\r\nthat Mr. Collins should be able to procure any woman’s good opinion,\r\nbecause he was not so happy as to succeed with you?”\r\n\r\nBut Elizabeth had now recollected herself; and, making a strong effort\r\nfor it, was able to assure her, with tolerable firmness, that the\r\nprospect of their relationship was highly grateful to her, and that she\r\nwished her all imaginable happiness.\r\n\r\n“I see what you are feeling,” replied Charlotte; “you must be surprised,\r\nvery much surprised, so lately as Mr. Collins was wishing to marry you.\r\nBut when you have had time to think it all over, I hope you will be\r\nsatisfied with what I have done. I am not romantic, you know. I never\r\nwas. I ask only a comfortable home; and, considering Mr. Collins’s\r\ncharacter, connections, and situation in life, I am convinced that my\r\nchance of happiness with him is as fair as most people can boast on\r\nentering the marriage state.”\r\n\r\nElizabeth quietly answered “undoubtedly;” and, after an awkward pause,\r\nthey returned to the rest of the family. Charlotte did not stay much\r\nlonger; and Elizabeth was then left to reflect on what she had heard. It\r\nwas a long time before she became at all reconciled to the idea of so\r\nunsuitable a match. The strangeness of Mr. Collins’s making two offers\r\nof marriage within three days was nothing in comparison of his being now\r\naccepted. She had always felt that Charlotte’s opinion of matrimony was\r\nnot exactly like her own; but she could not have supposed it possible\r\nthat, when called into action, she would have sacrificed every better\r\nfeeling to worldly advantage. Charlotte, the wife of Mr. Collins, was a\r\nmost humiliating picture! And to the pang of a friend disgracing\r\nherself, and sunk in her esteem, was added the distressing conviction\r\nthat it was impossible for that friend to be tolerably happy in the lot\r\nshe had chosen.\r\n\r\n\r\n\r\n\r\n[Illustration:\r\n\r\n “Protested he must be entirely mistaken.”\r\n\r\n[_Copyright 1894 by George Allen._]]\r\n\r\n\r\n\r\n\r\nCHAPTER XXIII.\r\n\r\n\r\n[Illustration]\r\n\r\nElizabeth was sitting with her mother and sisters, reflecting on what\r\nshe had heard, and doubting whether she was authorized to mention it,\r\nwhen Sir William Lucas himself appeared, sent by his daughter to\r\nannounce her engagement to the family. With many compliments to them,\r\nand much self-gratulation on the prospect of a connection between the\r\nhouses, he unfolded the matter,--to an audience not merely wondering,\r\nbut incredulous; for Mrs. Bennet, with more perseverance than\r\npoliteness, protested he must be entirely mistaken; and Lydia, always\r\nunguarded and often uncivil, boisterously exclaimed,--\r\n\r\n“Good Lord! Sir William, how can you tell such a story? Do not you know\r\nthat Mr. Collins wants to marry Lizzy?”\r\n\r\nNothing less than the complaisance of a courtier could have borne\r\nwithout anger such treatment: but Sir William’s good-breeding carried\r\nhim through it all; and though he begged leave to be positive as to the\r\ntruth of his information, he listened to all their impertinence with the\r\nmost forbearing courtesy.\r\n\r\nElizabeth, feeling it incumbent on her to relieve him from so unpleasant\r\na situation, now put herself forward to confirm his account, by\r\nmentioning her prior knowledge of it from Charlotte herself; and\r\nendeavoured to put a stop to the exclamations of her mother and sisters,\r\nby the earnestness of her congratulations to Sir William, in which she\r\nwas readily joined by Jane, and by making a variety of remarks on the\r\nhappiness that might be expected from the match, the excellent character\r\nof Mr. Collins, and the convenient distance of Hunsford from London.\r\n\r\nMrs. Bennet was, in fact, too much overpowered to say a great deal while\r\nSir William remained; but no sooner had he left them than her feelings\r\nfound a rapid vent. In the first place, she persisted in disbelieving\r\nthe whole of the matter; secondly, she was very sure that Mr. Collins\r\nhad been taken in; thirdly, she trusted that they would never be happy\r\ntogether; and, fourthly, that the match might be broken off. Two\r\ninferences, however, were plainly deduced from the whole: one, that\r\nElizabeth was the real cause of all the mischief; and the other, that\r\nshe herself had been barbarously used by them all; and on these two\r\npoints she principally dwelt during the rest of the day. Nothing could\r\nconsole and nothing appease her. Nor did that day wear out her\r\nresentment. A week elapsed before she could see Elizabeth without\r\nscolding her: a month passed away before she could speak to Sir William\r\nor Lady Lucas without being rude; and many months were gone before she\r\ncould at all forgive their daughter.\r\n\r\nMr. Bennet’s emotions were much more tranquil on the occasion, and such\r\nas he did experience he pronounced to be of a most agreeable sort; for\r\nit gratified him, he said, to discover that Charlotte Lucas, whom he had\r\nbeen used to think tolerably sensible, was as foolish as his wife, and\r\nmore foolish than his daughter!\r\n\r\nJane confessed herself a little surprised at the match: but she said\r\nless of her astonishment than of her earnest desire for their happiness;\r\nnor could Elizabeth persuade her to consider it as improbable. Kitty and\r\nLydia were far from envying Miss Lucas, for Mr. Collins was only a\r\nclergyman; and it affected them in no other way than as a piece of news\r\nto spread at Meryton.\r\n\r\nLady Lucas could not be insensible of triumph on being able to retort on\r\nMrs. Bennet the comfort of having a daughter well married; and she\r\ncalled at Longbourn rather oftener than usual to say how happy she was,\r\nthough Mrs. Bennet’s sour looks and ill-natured remarks might have been\r\nenough to drive happiness away.\r\n\r\nBetween Elizabeth and Charlotte there was a restraint which kept them\r\nmutually silent on the subject; and Elizabeth felt persuaded that no\r\nreal confidence could ever subsist between them again. Her\r\ndisappointment in Charlotte made her turn with fonder regard to her\r\nsister, of whose rectitude and delicacy she was sure her opinion could\r\nnever be shaken, and for whose happiness she grew daily more anxious, as\r\nBingley had now been gone a week, and nothing was heard of his return.\r\n\r\nJane had sent Caroline an early answer to her letter, and was counting\r\nthe days till she might reasonably hope to hear again. The promised\r\nletter of thanks from Mr. Collins arrived on Tuesday, addressed to their\r\nfather, and written with all the solemnity of gratitude which a\r\ntwelve-month’s abode in the family might have prompted. After\r\ndischarging his conscience on that head, he proceeded to inform them,\r\nwith many rapturous expressions, of his happiness in having obtained the\r\naffection of their amiable neighbour, Miss Lucas, and then explained\r\nthat it was merely with the view of enjoying her society that he had\r\nbeen so ready to close with their kind wish of seeing him again at\r\nLongbourn, whither he hoped to be able to return on Monday fortnight;\r\nfor Lady Catherine, he added, so heartily approved his marriage, that\r\nshe wished it to take place as soon as possible, which he trusted would\r\nbe an unanswerable argument with his amiable Charlotte to name an early\r\nday for making him the happiest of men.\r\n\r\nMr. Collins’s return into Hertfordshire was no longer a matter of\r\npleasure to Mrs. Bennet. On the contrary, she was as much disposed to\r\ncomplain of it as her husband. It was very strange that he should come\r\nto Longbourn instead of to Lucas Lodge; it was also very inconvenient\r\nand exceedingly troublesome. She hated having visitors in the house\r\nwhile her health was so indifferent, and lovers were of all people the\r\nmost disagreeable. Such were the gentle murmurs of Mrs. Bennet, and they\r\ngave way only to the greater distress of Mr. Bingley’s continued\r\nabsence.\r\n\r\nNeither Jane nor Elizabeth were comfortable on this subject. Day after\r\nday passed away without bringing any other tidings of him than the\r\nreport which shortly prevailed in Meryton of his coming no more to\r\nNetherfield the whole winter; a report which highly incensed Mrs.\r\nBennet, and which she never failed to contradict as a most scandalous\r\nfalsehood.\r\n\r\nEven Elizabeth began to fear--not that Bingley was indifferent--but that\r\nhis sisters would be successful in keeping him away. Unwilling as she\r\nwas to admit an idea so destructive to Jane’s happiness, and so\r\ndishonourable to the stability of her lover, she could not prevent its\r\nfrequently recurring. The united efforts of his two unfeeling sisters,\r\nand of his overpowering friend, assisted by the attractions of Miss\r\nDarcy and the amusements of London, might be too much, she feared, for\r\nthe strength of his attachment.\r\n\r\nAs for Jane, _her_ anxiety under this suspense was, of course, more\r\npainful than Elizabeth’s: but whatever she felt she was desirous of\r\nconcealing; and between herself and Elizabeth, therefore, the subject\r\nwas never alluded to. But as no such delicacy restrained her mother, an\r\nhour seldom passed in which she did not talk of Bingley, express her\r\nimpatience for his arrival, or even require Jane to confess that if he\r\ndid not come back she should think herself very ill-used. It needed all\r\nJane’s steady mildness to bear these attacks with tolerable\r\ntranquillity.\r\n\r\nMr. Collins returned most punctually on the Monday fortnight, but his\r\nreception at Longbourn was not quite so gracious as it had been on his\r\nfirst introduction. He was too happy, however, to need much attention;\r\nand, luckily for the others, the business of love-making relieved them\r\nfrom a great deal of his company. The chief of every day was spent by\r\nhim at Lucas Lodge, and he sometimes returned to Longbourn only in time\r\nto make an apology for his absence before the family went to bed.\r\n\r\n[Illustration:\r\n\r\n “_Whenever she spoke in a low voice_”\r\n]\r\n\r\nMrs. Bennet was really in a most pitiable state. The very mention of\r\nanything concerning the match threw her into an agony of ill-humour, and\r\nwherever she went she was sure of hearing it talked of. The sight of\r\nMiss Lucas was odious to her. As her successor in that house, she\r\nregarded her with jealous abhorrence. Whenever Charlotte came to see\r\nthem, she concluded her to be anticipating the hour of possession; and\r\nwhenever she spoke in a low voice to Mr. Collins, was convinced that\r\nthey were talking of the Longbourn estate, and resolving to turn herself\r\nand her daughters out of the house as soon as Mr. Bennet was dead. She\r\ncomplained bitterly of all this to her husband.\r\n\r\n“Indeed, Mr. Bennet,” said she, “it is very hard to think that Charlotte\r\nLucas should ever be mistress of this house, that _I_ should be forced\r\nto make way for _her_, and live to see her take my place in it!”\r\n\r\n“My dear, do not give way to such gloomy thoughts. Let us hope for\r\nbetter things. Let us flatter ourselves that _I_ may be the survivor.”\r\n\r\nThis was not very consoling to Mrs. Bennet; and, therefore, instead of\r\nmaking any answer, she went on as before.\r\n\r\n“I cannot bear to think that they should have all this estate. If it was\r\nnot for the entail, I should not mind it.”\r\n\r\n“What should not you mind?”\r\n\r\n“I should not mind anything at all.”\r\n\r\n“Let us be thankful that you are preserved from a state of such\r\ninsensibility.”\r\n\r\n“I never can be thankful, Mr. Bennet, for anything about the entail. How\r\nanyone could have the conscience to entail away an estate from one’s own\r\ndaughters I cannot understand; and all for the sake of Mr. Collins, too!\r\nWhy should _he_ have it more than anybody else?”\r\n\r\n“I leave it to yourself to determine,” said Mr. Bennet.\r\n\r\n\r\n\r\n\r\n[Illustration]\r\n\r\n\r\n\r\n\r\nCHAPTER XXIV.\r\n\r\n\r\n[Illustration]\r\n\r\nMiss Bingley’s letter arrived, and put an end to doubt. The very first\r\nsentence conveyed the assurance of their being all settled in London for\r\nthe winter, and concluded with her brother’s regret at not having had\r\ntime to pay his respects to his friends in Hertfordshire before he left\r\nthe country.\r\n\r\nHope was over, entirely over; and when Jane could attend to the rest of\r\nthe letter, she found little, except the professed affection of the\r\nwriter, that could give her any comfort. Miss Darcy’s praise occupied\r\nthe chief of it. Her many attractions were again dwelt on; and Caroline\r\nboasted joyfully of their increasing intimacy, and ventured to predict\r\nthe accomplishment of the wishes which had been unfolded in her former\r\nletter. She wrote also with great pleasure of her brother’s being an\r\ninmate of Mr. Darcy’s house, and mentioned with raptures some plans of\r\nthe latter with regard to new furniture.\r\n\r\nElizabeth, to whom Jane very soon communicated the chief of all this,\r\nheard it in silent indignation. Her heart was divided between concern\r\nfor her sister and resentment against all others. To Caroline’s\r\nassertion of her brother’s being partial to Miss Darcy, she paid no\r\ncredit. That he was really fond of Jane, she doubted no more than she\r\nhad ever done; and much as she had always been disposed to like him, she\r\ncould not think without anger, hardly without contempt, on that easiness\r\nof temper, that want of proper resolution, which now made him the slave\r\nof his designing friends, and led him to sacrifice his own happiness to\r\nthe caprice of their inclinations. Had his own happiness, however, been\r\nthe only sacrifice, he might have been allowed to sport with it in\r\nwhatever manner he thought best; but her sister’s was involved in it, as\r\nshe thought he must be sensible himself. It was a subject, in short, on\r\nwhich reflection would be long indulged, and must be unavailing. She\r\ncould think of nothing else; and yet, whether Bingley’s regard had\r\nreally died away, or were suppressed by his friends’ interference;\r\nwhether he had been aware of Jane’s attachment, or whether it had\r\nescaped his observation; whichever were the case, though her opinion of\r\nhim must be materially affected by the difference, her sister’s\r\nsituation remained the same, her peace equally wounded.\r\n\r\nA day or two passed before Jane had courage to speak of her feelings to\r\nElizabeth; but at last, on Mrs. Bennet’s leaving them together, after a\r\nlonger irritation than usual about Netherfield and its master, she could\r\nnot help saying,--\r\n\r\n“O that my dear mother had more command over herself! she can have no\r\nidea of the pain she gives me by her continual reflections on him. But I\r\nwill not repine. It cannot last long. He will be forgot, and we shall\r\nall be as we were before.”\r\n\r\nElizabeth looked at her sister with incredulous solicitude, but said\r\nnothing.\r\n\r\n“You doubt me,” cried Jane, slightly colouring; “indeed, you have no\r\nreason. He may live in my memory as the most amiable man of my\r\nacquaintance but that is all. I have nothing either to hope or fear, and\r\nnothing to reproach him with. Thank God I have not _that_ pain. A little\r\ntime, therefore--I shall certainly try to get the better----”\r\n\r\nWith a stronger voice she soon added, “I have this comfort immediately,\r\nthat it has not been more than an error of fancy on my side, and that it\r\nhas done no harm to anyone but myself.”\r\n\r\n“My dear Jane,” exclaimed Elizabeth, “you are too good. Your sweetness\r\nand disinterestedness are really angelic; I do not know what to say to\r\nyou. I feel as if I had never done you justice, or loved you as you\r\ndeserve.”\r\n\r\nMiss Bennet eagerly disclaimed all extraordinary merit, and threw back\r\nthe praise on her sister’s warm affection.\r\n\r\n“Nay,” said Elizabeth, “this is not fair. _You_ wish to think all the\r\nworld respectable, and are hurt if I speak ill of anybody. _I_ only want\r\nto think _you_ perfect, and you set yourself against it. Do not be\r\nafraid of my running into any excess, of my encroaching on your\r\nprivilege of universal good-will. You need not. There are few people\r\nwhom I really love, and still fewer of whom I think well. The more I see\r\nof the world the more am I dissatisfied with it; and every day confirms\r\nmy belief of the inconsistency of all human characters, and of the\r\nlittle dependence that can be placed on the appearance of either merit\r\nor sense. I have met with two instances lately: one I will not mention,\r\nthe other is Charlotte’s marriage. It is unaccountable! in every view it\r\nis unaccountable!”\r\n\r\n“My dear Lizzy, do not give way to such feelings as these. They will\r\nruin your happiness. You do not make allowance enough for difference of\r\nsituation and temper. Consider Mr. Collins’s respectability, and\r\nCharlotte’s prudent, steady character. Remember that she is one of a\r\nlarge family; that as to fortune it is a most eligible match; and be\r\nready to believe, for everybody’s sake, that she may feel something like\r\nregard and esteem for our cousin.”\r\n\r\n“To oblige you, I would try to believe almost anything, but no one else\r\ncould be benefited by such a belief as this; for were I persuaded that\r\nCharlotte had any regard for him, I should only think worse of her\r\nunderstanding than I now do of her heart. My dear Jane, Mr. Collins is a\r\nconceited, pompous, narrow-minded, silly man: you know he is, as well as\r\nI do; and you must feel, as well as I do, that the woman who marries him\r\ncannot have a proper way of thinking. You shall not defend her, though\r\nit is Charlotte Lucas. You shall not, for the sake of one individual,\r\nchange the meaning of principle and integrity, nor endeavour to persuade\r\nyourself or me, that selfishness is prudence, and insensibility of\r\ndanger security for happiness.”\r\n\r\n“I must think your language too strong in speaking of both,” replied\r\nJane; “and I hope you will be convinced of it, by seeing them happy\r\ntogether. But enough of this. You alluded to something else. You\r\nmentioned _two_ instances. I cannot misunderstand you, but I entreat\r\nyou, dear Lizzy, not to pain me by thinking _that person_ to blame, and\r\nsaying your opinion of him is sunk. We must not be so ready to fancy\r\nourselves intentionally injured. We must not expect a lively young man\r\nto be always so guarded and circumspect. It is very often nothing but\r\nour own vanity that deceives us. Women fancy admiration means more than\r\nit does.”\r\n\r\n“And men take care that they should.”\r\n\r\n“If it is designedly done, they cannot be justified; but I have no idea\r\nof there being so much design in the world as some persons imagine.”\r\n\r\n“I am far from attributing any part of Mr. Bingley’s conduct to design,”\r\nsaid Elizabeth; “but, without scheming to do wrong, or to make others\r\nunhappy, there may be error and there may be misery. Thoughtlessness,\r\nwant of attention to other people’s feelings, and want of resolution,\r\nwill do the business.”\r\n\r\n“And do you impute it to either of those?”\r\n\r\n“Yes; to the last. But if I go on I shall displease you by saying what I\r\nthink of persons you esteem. Stop me, whilst you can.”\r\n\r\n“You persist, then, in supposing his sisters influence him?”\r\n\r\n“Yes, in conjunction with his friend.”\r\n\r\n“I cannot believe it. Why should they try to influence him? They can\r\nonly wish his happiness; and if he is attached to me no other woman can\r\nsecure it.”\r\n\r\n“Your first position is false. They may wish many things besides his\r\nhappiness: they may wish his increase of wealth and consequence; they\r\nmay wish him to marry a girl who has all the importance of money, great\r\nconnections, and pride.”\r\n\r\n“Beyond a doubt they do wish him to choose Miss Darcy,” replied Jane;\r\n“but this may be from better feelings than you are supposing. They have\r\nknown her much longer than they have known me; no wonder if they love\r\nher better. But, whatever may be their own wishes, it is very unlikely\r\nthey should have opposed their brother’s. What sister would think\r\nherself at liberty to do it, unless there were something very\r\nobjectionable? If they believed him attached to me they would not try to\r\npart us; if he were so, they could not succeed. By supposing such an\r\naffection, you make everybody acting unnaturally and wrong, and me most\r\nunhappy. Do not distress me by the idea. I am not ashamed of having been\r\nmistaken--or, at least, it is slight, it is nothing in comparison of\r\nwhat I should feel in thinking ill of him or his sisters. Let me take it\r\nin the best light, in the light in which it may be understood.”\r\n\r\nElizabeth could not oppose such a wish; and from this time Mr. Bingley’s\r\nname was scarcely ever mentioned between them.\r\n\r\nMrs. Bennet still continued to wonder and repine at his returning no\r\nmore; and though a day seldom passed in which Elizabeth did not account\r\nfor it clearly, there seemed little chance of her ever considering it\r\nwith less perplexity. Her daughter endeavoured to convince her of what\r\nshe did not believe herself, that his attentions to Jane had been merely\r\nthe effect of a common and transient liking, which ceased when he saw\r\nher no more; but though the probability of the statement was admitted at\r\nthe time, she had the same story to repeat every day. Mrs. Bennet’s best\r\ncomfort was, that Mr. Bingley must be down again in the summer.\r\n\r\nMr. Bennet treated the matter differently. “So, Lizzy,” said he, one\r\nday, “your sister is crossed in love, I find. I congratulate her. Next\r\nto being married, a girl likes to be crossed in love a little now and\r\nthen. It is something to think of, and gives her a sort of distinction\r\namong her companions. When is your turn to come? You will hardly bear to\r\nbe long outdone by Jane. Now is your time. Here are officers enough at\r\nMeryton to disappoint all the young ladies in the country. Let Wickham\r\nbe your man. He is a pleasant fellow, and would jilt you creditably.”\r\n\r\n“Thank you, sir, but a less agreeable man would satisfy me. We must not\r\nall expect Jane’s good fortune.”\r\n\r\n“True,” said Mr. Bennet; “but it is a comfort to think that, whatever of\r\nthat kind may befall you, you have an affectionate mother who will\r\nalways make the most of it.”\r\n\r\nMr. Wickham’s society was of material service in dispelling the gloom\r\nwhich the late perverse occurrences had thrown on many of the Longbourn\r\nfamily. They saw him often, and to his other recommendations was now\r\nadded that of general unreserve. The whole of what Elizabeth had already\r\nheard, his claims on Mr. Darcy, and all that he had suffered from him,\r\nwas now openly acknowledged and publicly canvassed; and everybody was\r\npleased to think how much they had always disliked Mr. Darcy before they\r\nhad known anything of the matter.\r\n\r\nMiss Bennet was the only creature who could suppose there might be any\r\nextenuating circumstances in the case unknown to the society of\r\nHertfordshire: her mild and steady candour always pleaded for\r\nallowances, and urged the possibility of mistakes; but by everybody else\r\nMr. Darcy was condemned as the worst of men.\r\n\r\n\r\n\r\n\r\n[Illustration]\r\n\r\n\r\n\r\n\r\nCHAPTER XXV.\r\n\r\n\r\n[Illustration]\r\n\r\nAfter a week spent in professions of love and schemes of felicity, Mr.\r\nCollins was called from his amiable Charlotte by the arrival of\r\nSaturday. The pain of separation, however, might be alleviated on his\r\nside by preparations for the reception of his bride, as he had reason to\r\nhope, that shortly after his next return into Hertfordshire, the day\r\nwould be fixed that was to make him the happiest of men. He took leave\r\nof his relations at Longbourn with as much solemnity as before; wished\r\nhis fair cousins health and happiness again, and promised their father\r\nanother letter of thanks.\r\n\r\nOn the following Monday, Mrs. Bennet had the pleasure of receiving her\r\nbrother and his wife, who came, as usual, to spend the Christmas at\r\nLongbourn. Mr. Gardiner was a sensible, gentlemanlike man, greatly\r\nsuperior to his sister, as well by nature as education. The Netherfield\r\nladies would have had difficulty in believing that a man who lived by\r\ntrade, and within view of his own warehouses, could have been so\r\nwell-bred and agreeable. Mrs. Gardiner, who was several years younger\r\nthan Mrs. Bennet and Mrs. Philips, was an amiable, intelligent, elegant\r\nwoman, and a great favourite with her Longbourn nieces. Between the two\r\neldest and herself especially, there subsisted a very particular regard.\r\nThey had frequently been staying with her in town.\r\n\r\nThe first part of Mrs. Gardiner’s business, on her arrival, was to\r\ndistribute her presents and describe the newest fashions. When this was\r\ndone, she had a less active part to play. It became her turn to listen.\r\nMrs. Bennet had many grievances to relate, and much to complain of. They\r\nhad all been very ill-used since she last saw her sister. Two of her\r\ngirls had been on the point of marriage, and after all there was nothing\r\nin it.\r\n\r\n“I do not blame Jane,” she continued, “for Jane would have got Mr.\r\nBingley if she could. But, Lizzy! Oh, sister! it is very hard to think\r\nthat she might have been Mr. Collins’s wife by this time, had not it\r\nbeen for her own perverseness. He made her an offer in this very room,\r\nand she refused him. The consequence of it is, that Lady Lucas will have\r\na daughter married before I have, and that Longbourn estate is just as\r\nmuch entailed as ever. The Lucases are very artful people, indeed,\r\nsister. They are all for what they can get. I am sorry to say it of\r\nthem, but so it is. It makes me very nervous and poorly, to be thwarted\r\nso in my own family, and to have neighbours who think of themselves\r\nbefore anybody else. However, your coming just at this time is the\r\ngreatest of comforts, and I am very glad to hear what you tell us of\r\nlong sleeves.”\r\n\r\nMrs. Gardiner, to whom the chief of this news had been given before, in\r\nthe course of Jane and Elizabeth’s correspondence with her, made her\r\nsister a slight answer, and, in compassion to her nieces, turned the\r\nconversation.\r\n\r\nWhen alone with Elizabeth afterwards, she spoke more on the subject.\r\n“It seems likely to have been a desirable match for Jane,” said she. “I\r\nam sorry it went off. But these things happen so often! A young man,\r\nsuch as you describe Mr. Bingley, so easily falls in love with a pretty\r\ngirl for a few weeks, and, when accident separates them, so easily\r\nforgets her, that these sort of inconstancies are very frequent.”\r\n\r\n[Illustration:\r\n\r\n “Offended two or three young ladies”\r\n\r\n[_Copyright 1894 by George Allen._]]\r\n\r\n“An excellent consolation in its way,” said Elizabeth; “but it will not\r\ndo for _us_. We do not suffer by accident. It does not often happen\r\nthat the interference of friends will persuade a young man of\r\nindependent fortune to think no more of a girl whom he was violently in\r\nlove with only a few days before.”\r\n\r\n“But that expression of ‘violently in love’ is so hackneyed, so\r\ndoubtful, so indefinite, that it gives me very little idea. It is as\r\noften applied to feelings which arise only from a half hour’s\r\nacquaintance, as to a real, strong attachment. Pray, how _violent was_\r\nMr. Bingley’s love?”\r\n\r\n“I never saw a more promising inclination; he was growing quite\r\ninattentive to other people, and wholly engrossed by her. Every time\r\nthey met, it was more decided and remarkable. At his own ball he\r\noffended two or three young ladies by not asking them to dance; and I\r\nspoke to him twice myself without receiving an answer. Could there be\r\nfiner symptoms? Is not general incivility the very essence of love?”\r\n\r\n“Oh, yes! of that kind of love which I suppose him to have felt. Poor\r\nJane! I am sorry for her, because, with her disposition, she may not get\r\nover it immediately. It had better have happened to _you_, Lizzy; you\r\nwould have laughed yourself out of it sooner. But do you think she would\r\nbe prevailed on to go back with us? Change of scene might be of\r\nservice--and perhaps a little relief from home may be as useful as\r\nanything.”\r\n\r\nElizabeth was exceedingly pleased with this proposal, and felt persuaded\r\nof her sister’s ready acquiescence.\r\n\r\n“I hope,” added Mrs. Gardiner, “that no consideration with regard to\r\nthis young man will influence her. We live in so different a part of\r\ntown, all our connections are so different, and, as you well know, we go\r\nout so little, that it is very improbable they should meet at all,\r\nunless he really comes to see her.”\r\n\r\n“And _that_ is quite impossible; for he is now in the custody of his\r\nfriend, and Mr. Darcy would no more suffer him to call on Jane in such a\r\npart of London! My dear aunt, how could you think of it? Mr. Darcy may,\r\nperhaps, have _heard_ of such a place as Gracechurch Street, but he\r\nwould hardly think a month’s ablution enough to cleanse him from its\r\nimpurities, were he once to enter it; and, depend upon it, Mr. Bingley\r\nnever stirs without him.”\r\n\r\n“So much the better. I hope they will not meet at all. But does not Jane\r\ncorrespond with his sister? _She_ will not be able to help calling.”\r\n\r\n“She will drop the acquaintance entirely.”\r\n\r\nBut, in spite of the certainty in which Elizabeth affected to place this\r\npoint, as well as the still more interesting one of Bingley’s being\r\nwithheld from seeing Jane, she felt a solicitude on the subject which\r\nconvinced her, on examination, that she did not consider it entirely\r\nhopeless. It was possible, and sometimes she thought it probable, that\r\nhis affection might be re-animated, and the influence of his friends\r\nsuccessfully combated by the more natural influence of Jane’s\r\nattractions.\r\n\r\nMiss Bennet accepted her aunt’s invitation with pleasure; and the\r\nBingleys were no otherwise in her thoughts at the same time than as she\r\nhoped, by Caroline’s not living in the same house with her brother, she\r\nmight occasionally spend a morning with her, without any danger of\r\nseeing him.\r\n\r\nThe Gardiners stayed a week at Longbourn; and what with the Philipses,\r\nthe Lucases, and the officers, there was not a day without its\r\nengagement. Mrs. Bennet had so carefully provided for the entertainment\r\nof her brother and sister, that they did not once sit down to a family\r\ndinner. When the engagement was for home, some of the officers always\r\nmade part of it, of which officers Mr. Wickham was sure to be one; and\r\non these occasions Mrs. Gardiner, rendered suspicious by Elizabeth’s\r\nwarm commendation of him, narrowly observed them both. Without supposing\r\nthem, from what she saw, to be very seriously in love, their preference\r\nof each other was plain enough to make her a little uneasy; and she\r\nresolved to speak to Elizabeth on the subject before she left\r\nHertfordshire, and represent to her the imprudence of encouraging such\r\nan attachment.\r\n\r\nTo Mrs. Gardiner, Wickham had one means of affording pleasure,\r\nunconnected with his general powers. About ten or a dozen years ago,\r\nbefore her marriage, she had spent a considerable time in that very part\r\nof Derbyshire to which he belonged. They had, therefore, many\r\nacquaintance in common; and, though Wickham had been little there since\r\nthe death of Darcy’s father, five years before, it was yet in his power\r\nto give her fresher intelligence of her former friends than she had been\r\nin the way of procuring.\r\n\r\nMrs. Gardiner had seen Pemberley, and known the late Mr. Darcy by\r\ncharacter perfectly well. Here, consequently, was an inexhaustible\r\nsubject of discourse. In comparing her recollection of Pemberley with\r\nthe minute description which Wickham could give, and in bestowing her\r\ntribute of praise on the character of its late possessor, she was\r\ndelighting both him and herself. On being made acquainted with the\r\npresent Mr. Darcy’s treatment of him, she tried to remember something of\r\nthat gentleman’s reputed disposition, when quite a lad, which might\r\nagree with it; and was confident, at last, that she recollected having\r\nheard Mr. Fitzwilliam Darcy formerly spoken of as a very proud,\r\nill-natured boy.\r\n\r\n\r\n\r\n\r\n[Illustration:\r\n\r\n “Will you come and see me?”\r\n]\r\n\r\n\r\n\r\n\r\nCHAPTER XXVI.\r\n\r\n\r\n[Illustration]\r\n\r\nMrs. Gardiner’s caution to Elizabeth was punctually and kindly given on\r\nthe first favourable opportunity of speaking to her alone: after\r\nhonestly telling her what she thought, she thus went on:--\r\n\r\n“You are too sensible a girl, Lizzy, to fall in love merely because you\r\nare warned against it; and, therefore, I am not afraid of speaking\r\nopenly. Seriously, I would have you be on your guard. Do not involve\r\nyourself, or endeavour to involve him, in an affection which the want of\r\nfortune would make so very imprudent. I have nothing to say against\r\n_him_: he is a most interesting young man; and if he had the fortune he\r\nought to have, I should think you could not do better. But as it is--you\r\nmust not let your fancy run away with you. You have sense, and we all\r\nexpect you to use it. Your father would depend on _your_ resolution and\r\ngood conduct, I am sure. You must not disappoint your father.”\r\n\r\n“My dear aunt, this is being serious indeed.”\r\n\r\n“Yes, and I hope to engage you to be serious likewise.”\r\n\r\n“Well, then, you need not be under any alarm. I will take care of\r\nmyself, and of Mr. Wickham too. He shall not be in love with me, if I\r\ncan prevent it.”\r\n\r\n“Elizabeth, you are not serious now.”\r\n\r\n“I beg your pardon. I will try again. At present I am not in love with\r\nMr. Wickham; no, I certainly am not. But he is, beyond all comparison,\r\nthe most agreeable man I ever saw--and if he becomes really attached to\r\nme--I believe it will be better that he should not. I see the imprudence\r\nof it. Oh, _that_ abominable Mr. Darcy! My father’s opinion of me does\r\nme the greatest honour; and I should be miserable to forfeit it. My\r\nfather, however, is partial to Mr. Wickham. In short, my dear aunt, I\r\nshould be very sorry to be the means of making any of you unhappy; but\r\nsince we see, every day, that where there is affection young people are\r\nseldom withheld, by immediate want of fortune, from entering into\r\nengagements with each other, how can I promise to be wiser than so many\r\nof my fellow-creatures, if I am tempted, or how am I even to know that\r\nit would be wiser to resist? All that I can promise you, therefore, is\r\nnot to be in a hurry. I will not be in a hurry to believe myself his\r\nfirst object. When I am in company with him, I will not be wishing. In\r\nshort, I will do my best.”\r\n\r\n“Perhaps it will be as well if you discourage his coming here so very\r\noften. At least you should not _remind_ your mother of inviting him.”\r\n\r\n“As I did the other day,” said Elizabeth, with a conscious smile; “very\r\ntrue, it will be wise in me to refrain from _that_. But do not imagine\r\nthat he is always here so often. It is on your account that he has been\r\nso frequently invited this week. You know my mother’s ideas as to the\r\nnecessity of constant company for her friends. But really, and upon my\r\nhonour, I will try to do what I think to be wisest; and now I hope you\r\nare satisfied.”\r\n\r\nHer aunt assured her that she was; and Elizabeth, having thanked her for\r\nthe kindness of her hints, they parted,--a wonderful instance of advice\r\nbeing given on such a point without being resented.\r\n\r\nMr. Collins returned into Hertfordshire soon after it had been quitted\r\nby the Gardiners and Jane; but, as he took up his abode with the\r\nLucases, his arrival was no great inconvenience to Mrs. Bennet. His\r\nmarriage was now fast approaching; and she was at length so far resigned\r\nas to think it inevitable, and even repeatedly to say, in an ill-natured\r\ntone, that she “_wished_ they might be happy.” Thursday was to be the\r\nwedding-day, and on Wednesday Miss Lucas paid her farewell visit; and\r\nwhen she rose to take leave, Elizabeth, ashamed of her mother’s\r\nungracious and reluctant good wishes, and sincerely affected herself,\r\naccompanied her out of the room. As they went down stairs together,\r\nCharlotte said,--\r\n\r\n“I shall depend on hearing from you very often, Eliza.”\r\n\r\n“_That_ you certainly shall.”\r\n\r\n“And I have another favour to ask. Will you come and see me?”\r\n\r\n“We shall often meet, I hope, in Hertfordshire.”\r\n\r\n“I am not likely to leave Kent for some time. Promise me, therefore, to\r\ncome to Hunsford.”\r\n\r\nElizabeth could not refuse, though she foresaw little pleasure in the\r\nvisit.\r\n\r\n“My father and Maria are to come to me in March,” added Charlotte, “and\r\nI hope you will consent to be of the party. Indeed, Eliza, you will be\r\nas welcome to me as either of them.”\r\n\r\nThe wedding took place: the bride and bridegroom set off for Kent from\r\nthe church door, and everybody had as much to say or to hear on the\r\nsubject as usual. Elizabeth soon heard from her friend, and their\r\ncorrespondence was as regular and frequent as it ever had been: that it\r\nshould be equally unreserved was impossible. Elizabeth could never\r\naddress her without feeling that all the comfort of intimacy was over;\r\nand, though determined not to slacken as a correspondent, it was for the\r\nsake of what had been rather than what was. Charlotte’s first letters\r\nwere received with a good deal of eagerness: there could not but be\r\ncuriosity to know how she would speak of her new home, how she would\r\nlike Lady Catherine, and how happy she would dare pronounce herself to\r\nbe; though, when the letters were read, Elizabeth felt that Charlotte\r\nexpressed herself on every point exactly as she might have foreseen. She\r\nwrote cheerfully, seemed surrounded with comforts, and mentioned nothing\r\nwhich she could not praise. The house, furniture, neighbourhood, and\r\nroads, were all to her taste, and Lady Catherine’s behaviour was most\r\nfriendly and obliging. It was Mr. Collins’s picture of Hunsford and\r\nRosings rationally softened; and Elizabeth perceived that she must wait\r\nfor her own visit there, to know the rest.\r\n\r\nJane had already written a few lines to her sister, to announce their\r\nsafe arrival in London; and when she wrote again, Elizabeth hoped it\r\nwould be in her power to say something of the Bingleys.\r\n\r\nHer impatience for this second letter was as well rewarded as impatience\r\ngenerally is. Jane had been a week in town, without either seeing or\r\nhearing from Caroline. She accounted for it, however, by supposing that\r\nher last letter to her friend from Longbourn had by some accident been\r\nlost.\r\n\r\n“My aunt,” she continued, “is going to-morrow into that part of the\r\ntown, and I shall take the opportunity of calling in Grosvenor Street.”\r\n\r\nShe wrote again when the visit was paid, and she had seen Miss Bingley.\r\n“I did not think Caroline in spirits,” were her words, “but she was very\r\nglad to see me, and reproached me for giving her no notice of my coming\r\nto London. I was right, therefore; my last letter had never reached her.\r\nI inquired after their brother, of course. He was well, but so much\r\nengaged with Mr. Darcy that they scarcely ever saw him. I found that\r\nMiss Darcy was expected to dinner: I wish I could see her. My visit was\r\nnot long, as Caroline and Mrs. Hurst were going out. I dare say I shall\r\nsoon see them here.”\r\n\r\nElizabeth shook her head over this letter. It convinced her that\r\naccident only could discover to Mr. Bingley her sister’s being in town.\r\n\r\nFour weeks passed away, and Jane saw nothing of him. She endeavoured to\r\npersuade herself that she did not regret it; but she could no longer be\r\nblind to Miss Bingley’s inattention. After waiting at home every morning\r\nfor a fortnight, and inventing every evening a fresh excuse for her, the\r\nvisitor did at last appear; but the shortness of her stay, and, yet\r\nmore, the alteration of her manner, would allow Jane to deceive herself\r\nno longer. The letter which she wrote on this occasion to her sister\r\nwill prove what she felt:--\r\n\r\n “My dearest Lizzy will, I am sure, be incapable of triumphing in\r\n her better judgment, at my expense, when I confess myself to have\r\n been entirely deceived in Miss Bingley’s regard for me. But, my\r\n dear sister, though the event has proved you right, do not think me\r\n obstinate if I still assert that, considering what her behaviour\r\n was, my confidence was as natural as your suspicion. I do not at\r\n all comprehend her reason for wishing to be intimate with me; but,\r\n if the same circumstances were to happen again, I am sure I should\r\n be deceived again. Caroline did not return my visit till yesterday;\r\n and not a note, not a line, did I receive in the meantime. When she\r\n did come, it was very evident that she had no pleasure in it; she\r\n made a slight, formal apology for not calling before, said not a\r\n word of wishing to see me again, and was, in every respect, so\r\n altered a creature, that when she went away I was perfectly\r\n resolved to continue the acquaintance no longer. I pity, though I\r\n cannot help blaming, her. She was very wrong in singling me out as\r\n she did; I can safely say, that every advance to intimacy began on\r\n her side. But I pity her, because she must feel that she has been\r\n acting wrong, and because I am very sure that anxiety for her\r\n brother is the cause of it. I need not explain myself farther; and\r\n though _we_ know this anxiety to be quite needless, yet if she\r\n feels it, it will easily account for her behaviour to me; and so\r\n deservedly dear as he is to his sister, whatever anxiety she may\r\n feel on his behalf is natural and amiable. I cannot but wonder,\r\n however, at her having any such fears now, because if he had at all\r\n cared about me, we must have met long, long ago. He knows of my\r\n being in town, I am certain, from something she said herself; and\r\n yet it would seem, by her manner of talking, as if she wanted to\r\n persuade herself that he is really partial to Miss Darcy. I cannot\r\n understand it. If I were not afraid of judging harshly, I should be\r\n almost tempted to say, that there is a strong appearance of\r\n duplicity in all this. I will endeavour to banish every painful\r\n thought, and think only of what will make me happy, your affection,\r\n and the invariable kindness of my dear uncle and aunt. Let me hear\r\n from you very soon. Miss Bingley said something of his never\r\n returning to Netherfield again, of giving up the house, but not\r\n with any certainty. We had better not mention it. I am extremely\r\n glad that you have such pleasant accounts from our friends at\r\n Hunsford. Pray go to see them, with Sir William and Maria. I am\r\n sure you will be very comfortable there.\r\n\r\n“Yours, etc.”\r\n\r\nThis letter gave Elizabeth some pain; but her spirits returned, as she\r\nconsidered that Jane would no longer be duped, by the sister at least.\r\nAll expectation from the brother was now absolutely over. She would not\r\neven wish for any renewal of his attentions. His character sunk on every\r\nreview of it; and, as a punishment for him, as well as a possible\r\nadvantage to Jane, she seriously hoped he might really soon marry Mr.\r\nDarcy’s sister, as, by Wickham’s account, she would make him abundantly\r\nregret what he had thrown away.\r\n\r\nMrs. Gardiner about this time reminded Elizabeth of her promise\r\nconcerning that gentleman, and required information; and Elizabeth had\r\nsuch to send as might rather give contentment to her aunt than to\r\nherself. His apparent partiality had subsided, his attentions were over,\r\nhe was the admirer of some one else. Elizabeth was watchful enough to\r\nsee it all, but she could see it and write of it without material pain.\r\nHer heart had been but slightly touched, and her vanity was satisfied\r\nwith believing that _she_ would have been his only choice, had fortune\r\npermitted it. The sudden acquisition of ten thousand pounds was the most\r\nremarkable charm of the young lady to whom he was now rendering himself\r\nagreeable; but Elizabeth, less clear-sighted perhaps in this case than\r\nin Charlotte’s, did not quarrel with him for his wish of independence.\r\nNothing, on the contrary, could be more natural; and, while able to\r\nsuppose that it cost him a few struggles to relinquish her, she was\r\nready to allow it a wise and desirable measure for both, and could very\r\nsincerely wish him happy.\r\n\r\nAll this was acknowledged to Mrs. Gardiner; and, after relating the\r\ncircumstances, she thus went on:--“I am now convinced, my dear aunt,\r\nthat I have never been much in love; for had I really experienced that\r\npure and elevating passion, I should at present detest his very name,\r\nand wish him all manner of evil. But my feelings are not only cordial\r\ntowards _him_, they are even impartial towards Miss King. I cannot find\r\nout that I hate her at all, or that I am in the least unwilling to think\r\nher a very good sort of girl. There can be no love in all this. My\r\nwatchfulness has been effectual; and though I should certainly be a more\r\ninteresting object to all my acquaintance, were I distractedly in love\r\nwith him, I cannot say that I regret my comparative insignificance.\r\nImportance may sometimes be purchased too dearly. Kitty and Lydia take\r\nhis defection much more to heart than I do. They are young in the ways\r\nof the world, and not yet open to the mortifying conviction that\r\nhandsome young men must have something to live on as well as the\r\nplain.”\r\n\r\n\r\n\r\n\r\n[Illustration:\r\n\r\n “On the Stairs”\r\n]\r\n\r\n\r\n\r\n\r\nCHAPTER XXVII.\r\n\r\n\r\n[Illustration]\r\n\r\nWith no greater events than these in the Longbourn family, and otherwise\r\ndiversified by little beyond the walks to Meryton, sometimes dirty and\r\nsometimes cold, did January and February pass away. March was to take\r\nElizabeth to Hunsford. She had not at first thought very seriously of\r\ngoing thither; but Charlotte, she soon found, was depending on the\r\nplan, and she gradually learned to consider it herself with greater\r\npleasure as well as greater certainty. Absence had increased her desire\r\nof seeing Charlotte again, and weakened her disgust of Mr. Collins.\r\nThere was novelty in the scheme; and as, with such a mother and such\r\nuncompanionable sisters, home could not be faultless, a little change\r\nwas not unwelcome for its own sake. The journey would, moreover, give\r\nher a peep at Jane; and, in short, as the time drew near, she would have\r\nbeen very sorry for any delay. Everything, however, went on smoothly,\r\nand was finally settled according to Charlotte’s first sketch. She was\r\nto accompany Sir William and his second daughter. The improvement of\r\nspending a night in London was added in time, and the plan became as\r\nperfect as plan could be.\r\n\r\nThe only pain was in leaving her father, who would certainly miss her,\r\nand who, when it came to the point, so little liked her going, that he\r\ntold her to write to him, and almost promised to answer her letter.\r\n\r\nThe farewell between herself and Mr. Wickham was perfectly friendly; on\r\nhis side even more. His present pursuit could not make him forget that\r\nElizabeth had been the first to excite and to deserve his attention, the\r\nfirst to listen and to pity, the first to be admired; and in his manner\r\nof bidding her adieu, wishing her every enjoyment, reminding her of what\r\nshe was to expect in Lady Catherine de Bourgh, and trusting their\r\nopinion of her--their opinion of everybody--would always coincide, there\r\nwas a solicitude, an interest, which she felt must ever attach her to\r\nhim with a most sincere regard; and she parted from him convinced, that,\r\nwhether married or single, he must always be her model of the amiable\r\nand pleasing.\r\n\r\nHer fellow-travellers the next day were not of a kind to make her think\r\nhim less agreeable. Sir William Lucas, and his daughter Maria, a\r\ngood-humoured girl, but as empty-headed as himself, had nothing to say\r\nthat could be worth hearing, and were listened to with about as much\r\ndelight as the rattle of the chaise. Elizabeth loved absurdities, but\r\nshe had known Sir William’s too long. He could tell her nothing new of\r\nthe wonders of his presentation and knighthood; and his civilities were\r\nworn out, like his information.\r\n\r\nIt was a journey of only twenty-four miles, and they began it so early\r\nas to be in Gracechurch Street by noon. As they drove to Mr. Gardiner’s\r\ndoor, Jane was at a drawing-room window watching their arrival: when\r\nthey entered the passage, she was there to welcome them, and Elizabeth,\r\nlooking earnestly in her face, was pleased to see it healthful and\r\nlovely as ever. On the stairs were a troop of little boys and girls,\r\nwhose eagerness for their cousin’s appearance would not allow them to\r\nwait in the drawing-room, and whose shyness, as they had not seen her\r\nfor a twelvemonth, prevented their coming lower. All was joy and\r\nkindness. The day passed most pleasantly away; the morning in bustle and\r\nshopping, and the evening at one of the theatres.\r\n\r\nElizabeth then contrived to sit by her aunt. Their first subject was her\r\nsister; and she was more grieved than astonished to hear, in reply to\r\nher minute inquiries, that though Jane always struggled to support her\r\nspirits, there were periods of dejection. It was reasonable, however, to\r\nhope that they would not continue long. Mrs. Gardiner gave her the\r\nparticulars also of Miss Bingley’s visit in Gracechurch Street, and\r\nrepeated conversations occurring at different times between Jane and\r\nherself, which proved that the former had, from her heart, given up the\r\nacquaintance.\r\n\r\nMrs. Gardiner then rallied her niece on Wickham’s desertion, and\r\ncomplimented her on bearing it so well.\r\n\r\n“But, my dear Elizabeth,” she added, “what sort of girl is Miss King? I\r\nshould be sorry to think our friend mercenary.”\r\n\r\n“Pray, my dear aunt, what is the difference in matrimonial affairs,\r\nbetween the mercenary and the prudent motive? Where does discretion end,\r\nand avarice begin? Last Christmas you were afraid of his marrying me,\r\nbecause it would be imprudent; and now, because he is trying to get a\r\ngirl with only ten thousand pounds, you want to find out that he is\r\nmercenary.”\r\n\r\n“If you will only tell me what sort of girl Miss King is, I shall know\r\nwhat to think.”\r\n\r\n“She is a very good kind of girl, I believe. I know no harm of her.”\r\n\r\n“But he paid her not the smallest attention till her grandfather’s death\r\nmade her mistress of this fortune?”\r\n\r\n“No--why should he? If it were not allowable for him to gain _my_\r\naffections, because I had no money, what occasion could there be for\r\nmaking love to a girl whom he did not care about, and who was equally\r\npoor?”\r\n\r\n“But there seems indelicacy in directing his attentions towards her so\r\nsoon after this event.”\r\n\r\n“A man in distressed circumstances has not time for all those elegant\r\ndecorums which other people may observe. If _she_ does not object to it,\r\nwhy should _we_?”\r\n\r\n“_Her_ not objecting does not justify _him_. It only shows her being\r\ndeficient in something herself--sense or feeling.”\r\n\r\n“Well,” cried Elizabeth, “have it as you choose. _He_ shall be\r\nmercenary, and _she_ shall be foolish.”\r\n\r\n“No, Lizzy, that is what I do _not_ choose. I should be sorry, you know,\r\nto think ill of a young man who has lived so long in Derbyshire.”\r\n\r\n“Oh, if that is all, I have a very poor opinion of young men who live in\r\nDerbyshire; and their intimate friends who live in Hertfordshire are not\r\nmuch better. I am sick of them all. Thank heaven! I am going to-morrow\r\nwhere I shall find a man who has not one agreeable quality, who has\r\nneither manners nor sense to recommend him. Stupid men are the only ones\r\nworth knowing, after all.”\r\n\r\n“Take care, Lizzy; that speech savours strongly of disappointment.”\r\n\r\nBefore they were separated by the conclusion of the play, she had the\r\nunexpected happiness of an invitation to accompany her uncle and aunt in\r\na tour of pleasure which they proposed taking in the summer.\r\n\r\n“We have not quite determined how far it shall carry us,” said Mrs.\r\nGardiner; “but perhaps, to the Lakes.”\r\n\r\nNo scheme could have been more agreeable to Elizabeth, and her\r\nacceptance of the invitation was most ready and grateful. “My dear, dear\r\naunt,” she rapturously cried, “what delight! what felicity! You give me\r\nfresh life and vigour. Adieu to disappointment and spleen. What are men\r\nto rocks and mountains? Oh, what hours of transport we shall spend! And\r\nwhen we _do_ return, it shall not be like other travellers, without\r\nbeing able to give one accurate idea of anything. We _will_ know where\r\nwe have gone--we _will_ recollect what we have seen. Lakes, mountains,\r\nand rivers, shall not be jumbled together in our imaginations; nor, when\r\nwe attempt to describe any particular scene, will we begin quarrelling\r\nabout its relative situation. Let _our_ first effusions be less\r\ninsupportable than those of the generality of travellers.”\r\n\r\n\r\n\r\n\r\n[Illustration:\r\n\r\n “At the door”\r\n]\r\n\r\n\r\n\r\n\r\nCHAPTER XXVIII.\r\n\r\n\r\n[Illustration]\r\n\r\nEvery object in the next day’s journey was new and interesting to\r\nElizabeth; and her spirits were in a state of enjoyment; for she had\r\nseen her sister looking so well as to banish all fear for her health,\r\nand the prospect of her northern tour was a constant source of delight.\r\n\r\nWhen they left the high road for the lane to Hunsford, every eye was in\r\nsearch of the Parsonage, and every turning expected to bring it in view.\r\nThe paling of Rosings park was their boundary on one side. Elizabeth\r\nsmiled at the recollection of all that she had heard of its inhabitants.\r\n\r\nAt length the Parsonage was discernible. The garden sloping to the\r\nroad, the house standing in it, the green pales and the laurel hedge,\r\neverything declared they were arriving. Mr. Collins and Charlotte\r\nappeared at the door, and the carriage stopped at the small gate, which\r\nled by a short gravel walk to the house, amidst the nods and smiles of\r\nthe whole party. In a moment they were all out of the chaise, rejoicing\r\nat the sight of each other. Mrs. Collins welcomed her friend with the\r\nliveliest pleasure, and Elizabeth was more and more satisfied with\r\ncoming, when she found herself so affectionately received. She saw\r\ninstantly that her cousin’s manners were not altered by his marriage:\r\nhis formal civility was just what it had been; and he detained her some\r\nminutes at the gate to hear and satisfy his inquiries after all her\r\nfamily. They were then, with no other delay than his pointing out the\r\nneatness of the entrance, taken into the house; and as soon as they were\r\nin the parlour, he welcomed them a second time, with ostentatious\r\nformality, to his humble abode, and punctually repeated all his wife’s\r\noffers of refreshment.\r\n\r\nElizabeth was prepared to see him in his glory; and she could not help\r\nfancying that in displaying the good proportion of the room, its aspect,\r\nand its furniture, he addressed himself particularly to her, as if\r\nwishing to make her feel what she had lost in refusing him. But though\r\neverything seemed neat and comfortable, she was not able to gratify him\r\nby any sigh of repentance; and rather looked with wonder at her friend,\r\nthat she could have so cheerful an air with such a companion. When Mr.\r\nCollins said anything of which his wife might reasonably be ashamed,\r\nwhich certainly was not seldom, she involuntarily turned her eye on\r\nCharlotte. Once or twice she could discern a faint blush; but in general\r\nCharlotte wisely did not hear. After sitting long enough to admire\r\nevery article of furniture in the room, from the sideboard to the\r\nfender, to give an account of their journey, and of all that had\r\nhappened in London, Mr. Collins invited them to take a stroll in the\r\ngarden, which was large and well laid out, and to the cultivation of\r\nwhich he attended himself. To work in his garden was one of his most\r\nrespectable pleasures; and Elizabeth admired the command of countenance\r\nwith which Charlotte talked of the healthfulness of the exercise, and\r\nowned she encouraged it as much as possible. Here, leading the way\r\nthrough every walk and cross walk, and scarcely allowing them an\r\ninterval to utter the praises he asked for, every view was pointed out\r\nwith a minuteness which left beauty entirely behind. He could number the\r\nfields in every direction, and could tell how many trees there were in\r\nthe most distant clump. But of all the views which his garden, or which\r\nthe country or the kingdom could boast, none were to be compared with\r\nthe prospect of Rosings, afforded by an opening in the trees that\r\nbordered the park nearly opposite the front of his house. It was a\r\nhandsome modern building, well situated on rising ground.\r\n\r\nFrom his garden, Mr. Collins would have led them round his two meadows;\r\nbut the ladies, not having shoes to encounter the remains of a white\r\nfrost, turned back; and while Sir William accompanied him, Charlotte\r\ntook her sister and friend over the house, extremely well pleased,\r\nprobably, to have the opportunity of showing it without her husband’s\r\nhelp. It was rather small, but well built and convenient; and everything\r\nwas fitted up and arranged with a neatness and consistency, of which\r\nElizabeth gave Charlotte all the credit. When Mr. Collins could be\r\nforgotten, there was really a great air of comfort throughout, and by\r\nCharlotte’s evident enjoyment of it, Elizabeth supposed he must be often\r\nforgotten.\r\n\r\nShe had already learnt that Lady Catherine was still in the country. It\r\nwas spoken of again while they were at dinner, when Mr. Collins joining\r\nin, observed,--\r\n\r\n“Yes, Miss Elizabeth, you will have the honour of seeing Lady Catherine\r\nde Bourgh on the ensuing Sunday at church, and I need not say you will\r\nbe delighted with her. She is all affability and condescension, and I\r\ndoubt not but you will be honoured with some portion of her notice when\r\nservice is over. I have scarcely any hesitation in saying that she will\r\ninclude you and my sister Maria in every invitation with which she\r\nhonours us during your stay here. Her behaviour to my dear Charlotte is\r\ncharming. We dine at Rosings twice every week, and are never allowed to\r\nwalk home. Her Ladyship’s carriage is regularly ordered for us. I\r\n_should_ say, one of her Ladyship’s carriages, for she has several.”\r\n\r\n“Lady Catherine is a very respectable, sensible woman, indeed,” added\r\nCharlotte, “and a most attentive neighbour.”\r\n\r\n“Very true, my dear, that is exactly what I say. She is the sort of\r\nwoman whom one cannot regard with too much deference.”\r\n\r\nThe evening was spent chiefly in talking over Hertfordshire news, and\r\ntelling again what had been already written; and when it closed,\r\nElizabeth, in the solitude of her chamber, had to meditate upon\r\nCharlotte’s degree of contentment, to understand her address in guiding,\r\nand composure in bearing with, her husband, and to acknowledge that it\r\nwas all done very well. She had also to anticipate how her visit would\r\npass, the quiet tenour of their usual employments, the vexatious\r\ninterruptions of Mr. Collins, and the gaieties of their intercourse\r\nwith Rosings. A lively imagination soon settled it all.\r\n\r\nAbout the middle of the next day, as she was in her room getting ready\r\nfor a walk, a sudden noise below seemed to speak the whole house in\r\nconfusion; and, after listening a moment, she heard somebody running\r\nupstairs in a violent hurry, and calling loudly after her. She opened\r\nthe door, and met Maria in the landing-place, who, breathless with\r\nagitation, cried out,--\r\n\r\n[Illustration:\r\n\r\n “In Conversation with the ladies”\r\n\r\n[Copyright 1894 by George Allen.]]\r\n\r\n“Oh, my dear Eliza! pray make haste and come into the dining-room, for\r\nthere is such a sight to be seen! I will not tell you what it is. Make\r\nhaste, and come down this moment.”\r\n\r\nElizabeth asked questions in vain; Maria would tell her nothing more;\r\nand down they ran into the dining-room which fronted the lane, in quest\r\nof this wonder; it was two ladies, stopping in a low phaeton at the\r\ngarden gate.\r\n\r\n“And is this all?” cried Elizabeth. “I expected at least that the pigs\r\nwere got into the garden, and here is nothing but Lady Catherine and her\r\ndaughter!”\r\n\r\n“La! my dear,” said Maria, quite shocked at the mistake, “it is not Lady\r\nCatherine. The old lady is Mrs. Jenkinson, who lives with them. The\r\nother is Miss De Bourgh. Only look at her. She is quite a little\r\ncreature. Who would have thought she could be so thin and small!”\r\n\r\n“She is abominably rude to keep Charlotte out of doors in all this wind.\r\nWhy does she not come in?”\r\n\r\n“Oh, Charlotte says she hardly ever does. It is the greatest of favours\r\nwhen Miss De Bourgh comes in.”\r\n\r\n“I like her appearance,” said Elizabeth, struck with other ideas. “She\r\nlooks sickly and cross. Yes, she will do for him very well. She will\r\nmake him a very proper wife.”\r\n\r\nMr. Collins and Charlotte were both standing at the gate in conversation\r\nwith the ladies; and Sir William, to Elizabeth’s high diversion, was\r\nstationed in the doorway, in earnest contemplation of the greatness\r\nbefore him, and constantly bowing whenever Miss De Bourgh looked that\r\nway.\r\n\r\nAt length there was nothing more to be said; the ladies drove on, and\r\nthe others returned into the house. Mr. Collins no sooner saw the two\r\ngirls than he began to congratulate them on their good fortune, which\r\nCharlotte explained by letting them know that the whole party was asked\r\nto dine at Rosings the next day.\r\n\r\n\r\n\r\n\r\n[Illustration:\r\n\r\n ‘Lady Catherine, said she, you have given me a treasure.’\r\n\r\n[_Copyright 1894 by George Allen._]]\r\n\r\n\r\n\r\n\r\nCHAPTER XXIX.\r\n\r\n\r\n[Illustration]\r\n\r\nMr. Collins’s triumph, in consequence of this invitation, was complete.\r\nThe power of displaying the grandeur of his patroness to his wondering\r\nvisitors, and of letting them see her civility towards himself and his\r\nwife, was exactly what he had wished for; and that an opportunity of\r\ndoing it should be given so soon was such an instance of Lady\r\nCatherine’s condescension as he knew not how to admire enough.\r\n\r\n“I confess,” said he, “that I should not have been at all surprised by\r\nher Ladyship’s asking us on Sunday to drink tea and spend the evening\r\nat Rosings. I rather expected, from my knowledge of her affability, that\r\nit would happen. But who could have foreseen such an attention as this?\r\nWho could have imagined that we should receive an invitation to dine\r\nthere (an invitation, moreover, including the whole party) so\r\nimmediately after your arrival?”\r\n\r\n“I am the less surprised at what has happened,” replied Sir William,\r\n“from that knowledge of what the manners of the great really are, which\r\nmy situation in life has allowed me to acquire. About the court, such\r\ninstances of elegant breeding are not uncommon.”\r\n\r\nScarcely anything was talked of the whole day or next morning but their\r\nvisit to Rosings. Mr. Collins was carefully instructing them in what\r\nthey were to expect, that the sight of such rooms, so many servants, and\r\nso splendid a dinner, might not wholly overpower them.\r\n\r\nWhen the ladies were separating for the toilette, he said to\r\nElizabeth,--\r\n\r\n“Do not make yourself uneasy, my dear cousin, about your apparel. Lady\r\nCatherine is far from requiring that elegance of dress in us which\r\nbecomes herself and daughter. I would advise you merely to put on\r\nwhatever of your clothes is superior to the rest--there is no occasion\r\nfor anything more. Lady Catherine will not think the worse of you for\r\nbeing simply dressed. She likes to have the distinction of rank\r\npreserved.”\r\n\r\nWhile they were dressing, he came two or three times to their different\r\ndoors, to recommend their being quick, as Lady Catherine very much\r\nobjected to be kept waiting for her dinner. Such formidable accounts of\r\nher Ladyship, and her manner of living, quite frightened Maria Lucas,\r\nwho had been little used to company; and she looked forward to her\r\nintroduction at Rosings with as much apprehension as her father had done\r\nto his presentation at St. James’s.\r\n\r\nAs the weather was fine, they had a pleasant walk of about half a mile\r\nacross the park. Every park has its beauty and its prospects; and\r\nElizabeth saw much to be pleased with, though she could not be in such\r\nraptures as Mr. Collins expected the scene to inspire, and was but\r\nslightly affected by his enumeration of the windows in front of the\r\nhouse, and his relation of what the glazing altogether had originally\r\ncost Sir Lewis de Bourgh.\r\n\r\nWhen they ascended the steps to the hall, Maria’s alarm was every moment\r\nincreasing, and even Sir William did not look perfectly calm.\r\nElizabeth’s courage did not fail her. She had heard nothing of Lady\r\nCatherine that spoke her awful from any extraordinary talents or\r\nmiraculous virtue, and the mere stateliness of money and rank she\r\nthought she could witness without trepidation.\r\n\r\nFrom the entrance hall, of which Mr. Collins pointed out, with a\r\nrapturous air, the fine proportion and finished ornaments, they followed\r\nthe servants through an antechamber to the room where Lady Catherine,\r\nher daughter, and Mrs. Jenkinson were sitting. Her Ladyship, with great\r\ncondescension, arose to receive them; and as Mrs. Collins had settled it\r\nwith her husband that the office of introduction should be hers, it was\r\nperformed in a proper manner, without any of those apologies and thanks\r\nwhich he would have thought necessary.\r\n\r\nIn spite of having been at St. James’s, Sir William was so completely\r\nawed by the grandeur surrounding him, that he had but just courage\r\nenough to make a very low bow, and take his seat without saying a word;\r\nand his daughter, frightened almost out of her senses, sat on the edge\r\nof her chair, not knowing which way to look. Elizabeth found herself\r\nquite equal to the scene, and could observe the three ladies before her\r\ncomposedly. Lady Catherine was a tall, large woman, with strongly-marked\r\nfeatures, which might once have been handsome. Her air was not\r\nconciliating, nor was her manner of receiving them such as to make her\r\nvisitors forget their inferior rank. She was not rendered formidable by\r\nsilence: but whatever she said was spoken in so authoritative a tone as\r\nmarked her self-importance, and brought Mr. Wickham immediately to\r\nElizabeth’s mind; and, from the observation of the day altogether, she\r\nbelieved Lady Catherine to be exactly what he had represented.\r\n\r\nWhen, after examining the mother, in whose countenance and deportment\r\nshe soon found some resemblance of Mr. Darcy, she turned her eyes on the\r\ndaughter, she could almost have joined in Maria’s astonishment at her\r\nbeing so thin and so small. There was neither in figure nor face any\r\nlikeness between the ladies. Miss de Bourgh was pale and sickly: her\r\nfeatures, though not plain, were insignificant; and she spoke very\r\nlittle, except in a low voice, to Mrs. Jenkinson, in whose appearance\r\nthere was nothing remarkable, and who was entirely engaged in listening\r\nto what she said, and placing a screen in the proper direction before\r\nher eyes.\r\n\r\nAfter sitting a few minutes, they were all sent to one of the windows to\r\nadmire the view, Mr. Collins attending them to point out its beauties,\r\nand Lady Catherine kindly informing them that it was much better worth\r\nlooking at in the summer.\r\n\r\nThe dinner was exceedingly handsome, and there were all the servants,\r\nand all the articles of plate which Mr. Collins had promised; and, as he\r\nhad likewise foretold, he took his seat at the bottom of the table, by\r\nher Ladyship’s desire, and looked as if he felt that life could furnish\r\nnothing greater. He carved and ate and praised with delighted alacrity;\r\nand every dish was commended first by him, and then by Sir William, who\r\nwas now enough recovered to echo whatever his son-in-law said, in a\r\nmanner which Elizabeth wondered Lady Catherine could bear. But Lady\r\nCatherine seemed gratified by their excessive admiration, and gave most\r\ngracious smiles, especially when any dish on the table proved a novelty\r\nto them. The party did not supply much conversation. Elizabeth was ready\r\nto speak whenever there was an opening, but she was seated between\r\nCharlotte and Miss de Bourgh--the former of whom was engaged in\r\nlistening to Lady Catherine, and the latter said not a word to her all\r\nthe dinnertime. Mrs. Jenkinson was chiefly employed in watching how\r\nlittle Miss de Bourgh ate, pressing her to try some other dish and\r\nfearing she was indisposed. Maria thought speaking out of the question,\r\nand the gentlemen did nothing but eat and admire.\r\n\r\nWhen the ladies returned to the drawing-room, there was little to be\r\ndone but to hear Lady Catherine talk, which she did without any\r\nintermission till coffee came in, delivering her opinion on every\r\nsubject in so decisive a manner as proved that she was not used to have\r\nher judgment controverted. She inquired into Charlotte’s domestic\r\nconcerns familiarly and minutely, and gave her a great deal of advice as\r\nto the management of them all; told her how everything ought to be\r\nregulated in so small a family as hers, and instructed her as to the\r\ncare of her cows and her poultry. Elizabeth found that nothing was\r\nbeneath this great lady’s attention which could furnish her with an\r\noccasion for dictating to others. In the intervals of her discourse with\r\nMrs. Collins, she addressed a variety of questions to Maria and\r\nElizabeth, but especially to the latter, of whose connections she knew\r\nthe least, and who, she observed to Mrs. Collins, was a very genteel,\r\npretty kind of girl. She asked her at different times how many sisters\r\nshe had, whether they were older or younger than herself, whether any of\r\nthem were likely to be married, whether they were handsome, where they\r\nhad been educated, what carriage her father kept, and what had been her\r\nmother’s maiden name? Elizabeth felt all the impertinence of her\r\nquestions, but answered them very composedly. Lady Catherine then\r\nobserved,--\r\n\r\n“Your father’s estate is entailed on Mr. Collins, I think? For your\r\nsake,” turning to Charlotte, “I am glad of it; but otherwise I see no\r\noccasion for entailing estates from the female line. It was not thought\r\nnecessary in Sir Lewis de Bourgh’s family. Do you play and sing, Miss\r\nBennet?”\r\n\r\n“A little.”\r\n\r\n“Oh then--some time or other we shall be happy to hear you. Our\r\ninstrument is a capital one, probably superior to ---- you shall try it\r\nsome day. Do your sisters play and sing?”\r\n\r\n“One of them does.”\r\n\r\n“Why did not you all learn? You ought all to have learned. The Miss\r\nWebbs all play, and their father has not so good an income as yours. Do\r\nyou draw?”\r\n\r\n“No, not at all.”\r\n\r\n“What, none of you?”\r\n\r\n“Not one.”\r\n\r\n“That is very strange. But I suppose you had no opportunity. Your mother\r\nshould have taken you to town every spring for the benefit of masters.”\r\n\r\n“My mother would have no objection, but my father hates London.”\r\n\r\n“Has your governess left you?”\r\n\r\n“We never had any governess.”\r\n\r\n“No governess! How was that possible? Five daughters brought up at home\r\nwithout a governess! I never heard of such a thing. Your mother must\r\nhave been quite a slave to your education.”\r\n\r\nElizabeth could hardly help smiling, as she assured her that had not\r\nbeen the case.\r\n\r\n“Then who taught you? who attended to you? Without a governess, you must\r\nhave been neglected.”\r\n\r\n“Compared with some families, I believe we were; but such of us as\r\nwished to learn never wanted the means. We were always encouraged to\r\nread, and had all the masters that were necessary. Those who chose to be\r\nidle certainly might.”\r\n\r\n“Ay, no doubt: but that is what a governess will prevent; and if I had\r\nknown your mother, I should have advised her most strenuously to engage\r\none. I always say that nothing is to be done in education without steady\r\nand regular instruction, and nobody but a governess can give it. It is\r\nwonderful how many families I have been the means of supplying in that\r\nway. I am always glad to get a young person well placed out. Four nieces\r\nof Mrs. Jenkinson are most delightfully situated through my means; and\r\nit was but the other day that I recommended another young person, who\r\nwas merely accidentally mentioned to me, and the family are quite\r\ndelighted with her. Mrs. Collins, did I tell you of Lady Metcalfe’s\r\ncalling yesterday to thank me? She finds Miss Pope a treasure. ‘Lady\r\nCatherine,’ said she, ‘you have given me a treasure.’ Are any of your\r\nyounger sisters out, Miss Bennet?”\r\n\r\n“Yes, ma’am, all.”\r\n\r\n“All! What, all five out at once? Very odd! And you only the second. The\r\nyounger ones out before the elder are married! Your younger sisters must\r\nbe very young?”\r\n\r\n“Yes, my youngest is not sixteen. Perhaps _she_ is full young to be much\r\nin company. But really, ma’am, I think it would be very hard upon\r\nyounger sisters that they should not have their share of society and\r\namusement, because the elder may not have the means or inclination to\r\nmarry early. The last born has as good a right to the pleasures of youth\r\nas the first. And to be kept back on _such_ a motive! I think it would\r\nnot be very likely to promote sisterly affection or delicacy of mind.”\r\n\r\n“Upon my word,” said her Ladyship, “you give your opinion very decidedly\r\nfor so young a person. Pray, what is your age?”\r\n\r\n“With three younger sisters grown up,” replied Elizabeth, smiling, “your\r\nLadyship can hardly expect me to own it.”\r\n\r\nLady Catherine seemed quite astonished at not receiving a direct answer;\r\nand Elizabeth suspected herself to be the first creature who had ever\r\ndared to trifle with so much dignified impertinence.\r\n\r\n“You cannot be more than twenty, I am sure,--therefore you need not\r\nconceal your age.”\r\n\r\n“I am not one-and-twenty.”\r\n\r\nWhen the gentlemen had joined them, and tea was over, the card tables\r\nwere placed. Lady Catherine, Sir William, and Mr. and Mrs. Collins sat\r\ndown to quadrille; and as Miss De Bourgh chose to play at cassino, the\r\ntwo girls had the honour of assisting Mrs. Jenkinson to make up her\r\nparty. Their table was superlatively stupid. Scarcely a syllable was\r\nuttered that did not relate to the game, except when Mrs. Jenkinson\r\nexpressed her fears of Miss De Bourgh’s being too hot or too cold, or\r\nhaving too much or too little light. A great deal more passed at the\r\nother table. Lady Catherine was generally speaking--stating the mistakes\r\nof the three others, or relating some anecdote of herself. Mr. Collins\r\nwas employed in agreeing to everything her Ladyship said, thanking her\r\nfor every fish he won, and apologizing if he thought he won too many.\r\nSir William did not say much. He was storing his memory with anecdotes\r\nand noble names.\r\n\r\nWhen Lady Catherine and her daughter had played as long as they chose,\r\nthe tables were broken up, the carriage was offered to Mrs. Collins,\r\ngratefully accepted, and immediately ordered. The party then gathered\r\nround the fire to hear Lady Catherine determine what weather they were\r\nto have on the morrow. From these instructions they were summoned by the\r\narrival of the coach; and with many speeches of thankfulness on Mr.\r\nCollins’s side, and as many bows on Sir William’s, they departed. As\r\nsoon as they had driven from the door, Elizabeth was called on by her\r\ncousin to give her opinion of all that she had seen at Rosings, which,\r\nfor Charlotte’s sake, she made more favourable than it really was. But\r\nher commendation, though costing her some trouble, could by no means\r\nsatisfy Mr. Collins, and he was very soon obliged to take her Ladyship’s\r\npraise into his own hands.\r\n\r\n\r\n\r\n\r\n[Illustration]\r\n\r\n\r\n\r\n\r\nCHAPTER XXX.\r\n\r\n\r\n[Illustration]\r\n\r\nSir William stayed only a week at Hunsford; but his visit was long\r\nenough to convince him of his daughter’s being most comfortably settled,\r\nand of her possessing such a husband and such a neighbour as were not\r\noften met with. While Sir William was with them, Mr. Collins devoted his\r\nmornings to driving him out in his gig, and showing him the country: but\r\nwhen he went away, the whole family returned to their usual employments,\r\nand Elizabeth was thankful to find that they did not see more of her\r\ncousin by the alteration; for the chief of the time between breakfast\r\nand dinner was now passed by him either at work in the garden, or in\r\nreading and writing, and looking out of window in his own book room,\r\nwhich fronted the road. The room in which the ladies sat was backwards.\r\nElizabeth at first had rather wondered that Charlotte should not prefer\r\nthe dining parlour for common use; it was a better sized room, and had a\r\npleasanter aspect: but she soon saw that her friend had an excellent\r\nreason for what she did, for Mr. Collins would undoubtedly have been\r\nmuch less in his own apartment had they sat in one equally lively; and\r\nshe gave Charlotte credit for the arrangement.\r\n\r\nFrom the drawing-room they could distinguish nothing in the lane, and\r\nwere indebted to Mr. Collins for the knowledge of what carriages went\r\nalong, and how often especially Miss De Bourgh drove by in her phaeton,\r\nwhich he never failed coming to inform them of, though it happened\r\nalmost every day. She not unfrequently stopped at the Parsonage, and had\r\na few minutes’ conversation with Charlotte, but was scarcely ever\r\nprevailed on to get out.\r\n\r\nVery few days passed in which Mr. Collins did not walk to Rosings, and\r\nnot many in which his wife did not think it necessary to go likewise;\r\nand till Elizabeth recollected that there might be other family livings\r\nto be disposed of, she could not understand the sacrifice of so many\r\nhours. Now and then they were honoured with a call from her Ladyship,\r\nand nothing escaped her observation that was passing in the room during\r\nthese visits. She examined into their employments, looked at their work,\r\nand advised them to do it differently; found fault with the arrangement\r\nof the furniture, or detected the housemaid in negligence; and if she\r\naccepted any refreshment, seemed to do it only for the sake of finding\r\nout that Mrs. Collins’s joints of meat were too large for her family.\r\n\r\nElizabeth soon perceived, that though this great lady was not in the\r\ncommission of the peace for the county, she was a most active magistrate\r\nin her own parish, the minutest concerns of which were carried to her by\r\nMr. Collins; and whenever any of the cottagers were disposed to be\r\nquarrelsome, discontented, or too poor, she sallied forth into the\r\nvillage to settle their differences, silence their complaints, and scold\r\nthem into harmony and plenty.\r\n\r\n[Illustration:\r\n\r\n “he never failed to inform them”\r\n]\r\n\r\nThe entertainment of dining at Rosings was repeated about twice a week;\r\nand, allowing for the loss of Sir William, and there being only one\r\ncard-table in the evening, every such entertainment was the counterpart\r\nof the first. Their other engagements were few, as the style of living\r\nof the neighbourhood in general was beyond the Collinses’ reach. This,\r\nhowever, was no evil to Elizabeth, and upon the whole she spent her time\r\ncomfortably enough: there were half hours of pleasant conversation with\r\nCharlotte, and the weather was so fine for the time of year, that she\r\nhad often great enjoyment out of doors. Her favourite walk, and where\r\nshe frequently went while the others were calling on Lady Catherine, was\r\nalong the open grove which edged that side of the park, where there was\r\na nice sheltered path, which no one seemed to value but herself, and\r\nwhere she felt beyond the reach of Lady Catherine’s curiosity.\r\n\r\nIn this quiet way the first fortnight of her visit soon passed away.\r\nEaster was approaching, and the week preceding it was to bring an\r\naddition to the family at Rosings, which in so small a circle must be\r\nimportant. Elizabeth had heard, soon after her arrival, that Mr. Darcy\r\nwas expected there in the course of a few weeks; and though there were\r\nnot many of her acquaintance whom she did not prefer, his coming would\r\nfurnish one comparatively new to look at in their Rosings parties, and\r\nshe might be amused in seeing how hopeless Miss Bingley’s designs on him\r\nwere, by his behaviour to his cousin, for whom he was evidently destined\r\nby Lady Catherine, who talked of his coming with the greatest\r\nsatisfaction, spoke of him in terms of the highest admiration, and\r\nseemed almost angry to find that he had already been frequently seen by\r\nMiss Lucas and herself.\r\n\r\nHis arrival was soon known at the Parsonage; for Mr. Collins was walking\r\nthe whole morning within view of the lodges opening into Hunsford Lane,\r\nin order to have\r\n\r\n[Illustration:\r\n\r\n“The gentlemen accompanied him.”\r\n\r\n[_Copyright 1894 by George Allen._]]\r\n\r\nthe earliest assurance of it; and, after making his bow as the carriage\r\nturned into the park, hurried home with the great intelligence. On the\r\nfollowing morning he hastened to Rosings to pay his respects. There were\r\ntwo nephews of Lady Catherine to require them, for Mr. Darcy had brought\r\nwith him a Colonel Fitzwilliam, the younger son of his uncle, Lord ----;\r\nand, to the great surprise of all the party, when Mr. Collins returned,\r\nthe gentlemen accompanied him. Charlotte had seen them from her\r\nhusband’s room, crossing the road, and immediately running into the\r\nother, told the girls what an honour they might expect, adding,--\r\n\r\n“I may thank you, Eliza, for this piece of civility. Mr. Darcy would\r\nnever have come so soon to wait upon me.”\r\n\r\nElizabeth had scarcely time to disclaim all right to the compliment\r\nbefore their approach was announced by the door-bell, and shortly\r\nafterwards the three gentlemen entered the room. Colonel Fitzwilliam,\r\nwho led the way, was about thirty, not handsome, but in person and\r\naddress most truly the gentleman. Mr. Darcy looked just as he had been\r\nused to look in Hertfordshire, paid his compliments, with his usual\r\nreserve, to Mrs. Collins; and whatever might be his feelings towards her\r\nfriend, met her with every appearance of composure. Elizabeth merely\r\ncourtesied to him, without saying a word.\r\n\r\nColonel Fitzwilliam entered into conversation directly, with the\r\nreadiness and ease of a well-bred man, and talked very pleasantly; but\r\nhis cousin, after having addressed a slight observation on the house and\r\ngarden to Mrs. Collins, sat for some time without speaking to anybody.\r\nAt length, however, his civility was so far awakened as to inquire of\r\nElizabeth after the health of her family. She answered him in the usual\r\nway; and, after a moment’s pause, added,--\r\n\r\n“My eldest sister has been in town these three months. Have you never\r\nhappened to see her there?”\r\n\r\nShe was perfectly sensible that he never had: but she wished to see\r\nwhether he would betray any consciousness of what had passed between the\r\nBingleys and Jane; and she thought he looked a little confused as he\r\nanswered that he had never been so fortunate as to meet Miss Bennet. The\r\nsubject was pursued no further, and the gentlemen soon afterwards went\r\naway.\r\n\r\n\r\n\r\n\r\n[Illustration:\r\n\r\n“At Church”\r\n]\r\n\r\n\r\n\r\n\r\nCHAPTER XXXI.\r\n\r\n\r\n[Illustration]\r\n\r\nColonel Fitzwilliam’s manners were very much admired at the Parsonage,\r\nand the ladies all felt that he must add considerably to the pleasure of\r\ntheir engagements at Rosings. It was some days, however, before they\r\nreceived any invitation thither, for while there were visitors in the\r\nhouse they could not be necessary; and it was not till Easter-day,\r\nalmost a week after the gentlemen’s arrival, that they were honoured by\r\nsuch an attention, and then they were merely asked on leaving church to\r\ncome there in the evening. For the last week they had seen very little\r\nof either Lady Catherine or her daughter. Colonel Fitzwilliam had called\r\nat the Parsonage more than once during the time, but Mr. Darcy they had\r\nonly seen at church.\r\n\r\nThe invitation was accepted, of course, and at a proper hour they joined\r\nthe party in Lady Catherine’s drawing-room. Her Ladyship received them\r\ncivilly, but it was plain that their company was by no means so\r\nacceptable as when she could get nobody else; and she was, in fact,\r\nalmost engrossed by her nephews, speaking to them, especially to Darcy,\r\nmuch more than to any other person in the room.\r\n\r\nColonel Fitzwilliam seemed really glad to see them: anything was a\r\nwelcome relief to him at Rosings; and Mrs. Collins’s pretty friend had,\r\nmoreover, caught his fancy very much. He now seated himself by her, and\r\ntalked so agreeably of Kent and Hertfordshire, of travelling and staying\r\nat home, of new books and music, that Elizabeth had never been half so\r\nwell entertained in that room before; and they conversed with so much\r\nspirit and flow as to draw the attention of Lady Catherine herself, as\r\nwell as of Mr. Darcy. _His_ eyes had been soon and repeatedly turned\r\ntowards them with a look of curiosity; and that her Ladyship, after a\r\nwhile, shared the feeling, was more openly acknowledged, for she did not\r\nscruple to call out,--\r\n\r\n“What is that you are saying, Fitzwilliam? What is it you are talking\r\nof? What are you telling Miss Bennet? Let me hear what it is.”\r\n\r\n“We were talking of music, madam,” said he, when no longer able to avoid\r\na reply.\r\n\r\n“Of music! Then pray speak aloud. It is of all subjects my delight. I\r\nmust have my share in the conversation, if you are speaking of music.\r\nThere are few people in England, I suppose, who have more true\r\nenjoyment of music than myself, or a better natural taste. If I had ever\r\nlearnt, I should have been a great proficient. And so would Anne, if her\r\nhealth had allowed her to apply. I am confident that she would have\r\nperformed delightfully. How does Georgiana get on, Darcy?”\r\n\r\nMr. Darcy spoke with affectionate praise of his sister’s proficiency.\r\n\r\n“I am very glad to hear such a good account of her,” said Lady\r\nCatherine; “and pray tell her from me, that she cannot expect to excel,\r\nif she does not practise a great deal.”\r\n\r\n“I assure you, madam,” he replied, “that she does not need such advice.\r\nShe practises very constantly.”\r\n\r\n“So much the better. It cannot be done too much; and when I next write\r\nto her, I shall charge her not to neglect it on any account. I often\r\ntell young ladies, that no excellence in music is to be acquired without\r\nconstant practice. I have told Miss Bennet several times, that she will\r\nnever play really well, unless she practises more; and though Mrs.\r\nCollins has no instrument, she is very welcome, as I have often told\r\nher, to come to Rosings every day, and play on the pianoforte in Mrs.\r\nJenkinson’s room. She would be in nobody’s way, you know, in that part\r\nof the house.”\r\n\r\nMr. Darcy looked a little ashamed of his aunt’s ill-breeding, and made\r\nno answer.\r\n\r\nWhen coffee was over, Colonel Fitzwilliam reminded Elizabeth of having\r\npromised to play to him; and she sat down directly to the instrument. He\r\ndrew a chair near her. Lady Catherine listened to half a song, and then\r\ntalked, as before, to her other nephew; till the latter walked away from\r\nher, and moving with his usual deliberation towards the pianoforte,\r\nstationed himself so as to command a full view of the fair performer’s\r\ncountenance. Elizabeth saw what he was doing, and at the first\r\nconvenient pause turned to him with an arch smile, and said,--\r\n\r\n“You mean to frighten me, Mr. Darcy, by coming in all this state to hear\r\nme. But I will not be alarmed, though your sister _does_ play so well.\r\nThere is a stubbornness about me that never can bear to be frightened at\r\nthe will of others. My courage always rises with every attempt to\r\nintimidate me.”\r\n\r\n“I shall not say that you are mistaken,” he replied, “because you could\r\nnot really believe me to entertain any design of alarming you; and I\r\nhave had the pleasure of your acquaintance long enough to know, that you\r\nfind great enjoyment in occasionally professing opinions which, in fact,\r\nare not your own.”\r\n\r\nElizabeth laughed heartily at this picture of herself, and said to\r\nColonel Fitzwilliam, “Your cousin will give you a very pretty notion of\r\nme, and teach you not to believe a word I say. I am particularly unlucky\r\nin meeting with a person so well able to expose my real character, in a\r\npart of the world where I had hoped to pass myself off with some degree\r\nof credit. Indeed, Mr. Darcy, it is very ungenerous in you to mention\r\nall that you knew to my disadvantage in Hertfordshire--and, give me\r\nleave to say, very impolitic too--for it is provoking me to retaliate,\r\nand such things may come out as will shock your relations to hear.”\r\n\r\n“I am not afraid of you,” said he, smilingly.\r\n\r\n“Pray let me hear what you have to accuse him of,” cried Colonel\r\nFitzwilliam. “I should like to know how he behaves among strangers.”\r\n\r\n“You shall hear, then--but prepare for something very dreadful. The\r\nfirst time of my ever seeing him in Hertfordshire, you must know, was at\r\na ball--and at this ball, what do you think he did? He danced only four\r\ndances! I am sorry to pain you, but so it was. He danced only four\r\ndances, though gentlemen were scarce; and, to my certain knowledge, more\r\nthan one young lady was sitting down in want of a partner. Mr. Darcy,\r\nyou cannot deny the fact.”\r\n\r\n“I had not at that time the honour of knowing any lady in the assembly\r\nbeyond my own party.”\r\n\r\n“True; and nobody can ever be introduced in a ball-room. Well, Colonel\r\nFitzwilliam, what do I play next? My fingers wait your orders.”\r\n\r\n“Perhaps,” said Darcy, “I should have judged better had I sought an\r\nintroduction, but I am ill-qualified to recommend myself to strangers.”\r\n\r\n“Shall we ask your cousin the reason of this?” said Elizabeth, still\r\naddressing Colonel Fitzwilliam. “Shall we ask him why a man of sense and\r\neducation, and who has lived in the world, is ill-qualified to recommend\r\nhimself to strangers?”\r\n\r\n“I can answer your question,” said Fitzwilliam, “without applying to\r\nhim. It is because he will not give himself the trouble.”\r\n\r\n“I certainly have not the talent which some people possess,” said Darcy,\r\n“of conversing easily with those I have never seen before. I cannot\r\ncatch their tone of conversation, or appear interested in their\r\nconcerns, as I often see done.”\r\n\r\n“My fingers,” said Elizabeth, “do not move over this instrument in the\r\nmasterly manner which I see so many women’s do. They have not the same\r\nforce or rapidity, and do not produce the same expression. But then I\r\nhave always supposed it to be my own fault--because I would not take\r\nthe trouble of practising. It is not that I do not believe _my_ fingers\r\nas capable as any other woman’s of superior execution.”\r\n\r\nDarcy smiled and said, “You are perfectly right. You have employed your\r\ntime much better. No one admitted to the privilege of hearing you can\r\nthink anything wanting. We neither of us perform to strangers.”\r\n\r\nHere they were interrupted by Lady Catherine, who called out to know\r\nwhat they were talking of. Elizabeth immediately began playing again.\r\nLady Catherine approached, and, after listening for a few minutes, said\r\nto Darcy,--\r\n\r\n“Miss Bennet would not play at all amiss if she practised more, and\r\ncould have the advantage of a London master. She has a very good notion\r\nof fingering, though her taste is not equal to Anne’s. Anne would have\r\nbeen a delightful performer, had her health allowed her to learn.”\r\n\r\nElizabeth looked at Darcy, to see how cordially he assented to his\r\ncousin’s praise: but neither at that moment nor at any other could she\r\ndiscern any symptom of love; and from the whole of his behaviour to Miss\r\nDe Bourgh she derived this comfort for Miss Bingley, that he might have\r\nbeen just as likely to marry _her_, had she been his relation.\r\n\r\nLady Catherine continued her remarks on Elizabeth’s performance, mixing\r\nwith them many instructions on execution and taste. Elizabeth received\r\nthem with all the forbearance of civility; and at the request of the\r\ngentlemen remained at the instrument till her Ladyship’s carriage was\r\nready to take them all home.\r\n\r\n\r\n\r\n\r\n[Illustration]\r\n\r\n\r\n\r\n\r\nCHAPTER XXXII.\r\n\r\n\r\n[Illustration]\r\n\r\nElizabeth was sitting by herself the next morning, and writing to Jane,\r\nwhile Mrs. Collins and Maria were gone on business into the village,\r\nwhen she was startled by a ring at the door, the certain signal of a\r\nvisitor. As she had heard no carriage, she thought it not unlikely to be\r\nLady Catherine; and under that apprehension was putting away her\r\nhalf-finished letter, that she might escape all impertinent questions,\r\nwhen the door opened, and to her very great surprise Mr. Darcy, and Mr.\r\nDarcy only, entered the room.\r\n\r\nHe seemed astonished too on finding her alone, and apologized for his\r\nintrusion, by letting her know that he had understood all the ladies to\r\nbe within.\r\n\r\nThey then sat down, and when her inquiries after Rosings were made,\r\nseemed in danger of sinking into total silence. It was absolutely\r\nnecessary, therefore, to think of something; and in this emergency\r\nrecollecting _when_ she had seen him last in Hertfordshire, and feeling\r\ncurious to know what he would say on the subject of their hasty\r\ndeparture, she observed,--\r\n\r\n“How very suddenly you all quitted Netherfield last November, Mr. Darcy!\r\nIt must have been a most agreeable surprise to Mr. Bingley to see you\r\nall after him so soon; for, if I recollect right, he went but the day\r\nbefore. He and his sisters were well, I hope, when you left London?”\r\n\r\n“Perfectly so, I thank you.”\r\n\r\nShe found that she was to receive no other answer; and, after a short\r\npause, added,--\r\n\r\n“I think I have understood that Mr. Bingley has not much idea of ever\r\nreturning to Netherfield again?”\r\n\r\n“I have never heard him say so; but it is probable that he may spend\r\nvery little of his time there in future. He has many friends, and he is\r\nat a time of life when friends and engagements are continually\r\nincreasing.”\r\n\r\n“If he means to be but little at Netherfield, it would be better for the\r\nneighbourhood that he should give up the place entirely, for then we\r\nmight possibly get a settled family there. But, perhaps, Mr. Bingley did\r\nnot take the house so much for the convenience of the neighbourhood as\r\nfor his own, and we must expect him to keep or quit it on the same\r\nprinciple.”\r\n\r\n“I should not be surprised,” said Darcy, “if he were to give it up as\r\nsoon as any eligible purchase offers.”\r\n\r\nElizabeth made no answer. She was afraid of talking longer of his\r\nfriend; and, having nothing else to say, was now determined to leave the\r\ntrouble of finding a subject to him.\r\n\r\nHe took the hint and soon began with, “This seems a very comfortable\r\nhouse. Lady Catherine, I believe, did a great deal to it when Mr.\r\nCollins first came to Hunsford.”\r\n\r\n“I believe she did--and I am sure she could not have bestowed her\r\nkindness on a more grateful object.”\r\n\r\n“Mr. Collins appears very fortunate in his choice of a wife.”\r\n\r\n“Yes, indeed; his friends may well rejoice in his having met with one of\r\nthe very few sensible women who would have accepted him, or have made\r\nhim happy if they had. My friend has an excellent understanding--though\r\nI am not certain that I consider her marrying Mr. Collins as the wisest\r\nthing she ever did. She seems perfectly happy, however; and, in a\r\nprudential light, it is certainly a very good match for her.”\r\n\r\n“It must be very agreeable to her to be settled within so easy a\r\ndistance of her own family and friends.”\r\n\r\n“An easy distance do you call it? It is nearly fifty miles.”\r\n\r\n“And what is fifty miles of good road? Little more than half a day’s\r\njourney. Yes, I call it a very easy distance.”\r\n\r\n“I should never have considered the distance as one of the _advantages_\r\nof the match,” cried Elizabeth. “I should never have said Mrs. Collins\r\nwas settled _near_ her family.”\r\n\r\n“It is a proof of your own attachment to Hertfordshire. Anything beyond\r\nthe very neighbourhood of Longbourn, I suppose, would appear far.”\r\n\r\nAs he spoke there was a sort of smile, which Elizabeth fancied she\r\nunderstood; he must be supposing her to be thinking of Jane and\r\nNetherfield, and she blushed as she answered,--\r\n\r\n“I do not mean to say that a woman may not be settled too near her\r\nfamily. The far and the near must be relative, and depend on many\r\nvarying circumstances. Where there is fortune to make the expense of\r\ntravelling unimportant, distance becomes no evil. But that is not the\r\ncase _here_. Mr. and Mrs. Collins have a comfortable income, but not\r\nsuch a one as will allow of frequent journeys--and I am persuaded my\r\nfriend would not call herself _near_ her family under less than _half_\r\nthe present distance.”\r\n\r\nMr. Darcy drew his chair a little towards her, and said, “_You_ cannot\r\nhave a right to such very strong local attachment. _You_ cannot have\r\nbeen always at Longbourn.”\r\n\r\nElizabeth looked surprised. The gentleman experienced some change of\r\nfeeling; he drew back his chair, took a newspaper from the table, and,\r\nglancing over it, said, in a colder voice,--\r\n\r\n“Are you pleased with Kent?”\r\n\r\nA short dialogue on the subject of the country ensued, on either side\r\ncalm and concise--and soon put an end to by the entrance of Charlotte\r\nand her sister, just returned from their walk. The _tête-à-tête_\r\nsurprised them. Mr. Darcy related the mistake which had occasioned his\r\nintruding on Miss Bennet, and, after sitting a few minutes longer,\r\nwithout saying much to anybody, went away.\r\n\r\n[Illustration: “Accompanied by their aunt”\r\n\r\n[_Copyright 1894 by George Allen._]]\r\n\r\n“What can be the meaning of this?” said Charlotte, as soon as he was\r\ngone. “My dear Eliza, he must be in love with you, or he would never\r\nhave called on us in this familiar way.”\r\n\r\nBut when Elizabeth told of his silence, it did not seem very likely,\r\neven to Charlotte’s wishes, to be the case; and, after various\r\nconjectures, they could at last only suppose his visit to proceed from\r\nthe difficulty of finding anything to do, which was the more probable\r\nfrom the time of year. All field sports were over. Within doors there\r\nwas Lady Catherine, books, and a billiard table, but gentlemen cannot be\r\nalways within doors; and in the nearness of the Parsonage, or the\r\npleasantness of the walk to it, or of the people who lived in it, the\r\ntwo cousins found a temptation from this period of walking thither\r\nalmost every day. They called at various times of the morning, sometimes\r\nseparately, sometimes together, and now and then accompanied by their\r\naunt. It was plain to them all that Colonel Fitzwilliam came because he\r\nhad pleasure in their society, a persuasion which of course recommended\r\nhim still more; and Elizabeth was reminded by her own satisfaction in\r\nbeing with him, as well as by his evident admiration, of her former\r\nfavourite, George Wickham; and though, in comparing them, she saw there\r\nwas less captivating softness in Colonel Fitzwilliam’s manners, she\r\nbelieved he might have the best informed mind.\r\n\r\nBut why Mr. Darcy came so often to the Parsonage it was more difficult\r\nto understand. It could not be for society, as he frequently sat there\r\nten minutes together without opening his lips; and when he did speak, it\r\nseemed the effect of necessity rather than of choice--a sacrifice to\r\npropriety, not a pleasure to himself. He seldom appeared really\r\nanimated. Mrs. Collins knew not what to make of him. Colonel\r\nFitzwilliam’s occasionally laughing at his stupidity proved that he was\r\ngenerally different, which her own knowledge of him could not have told\r\nher; and as she would have liked to believe this change the effect of\r\nlove, and the object of that love her friend Eliza, she set herself\r\nseriously to work to find it out: she watched him whenever they were at\r\nRosings, and whenever he came to Hunsford; but without much success. He\r\ncertainly looked at her friend a great deal, but the expression of that\r\nlook was disputable. It was an earnest, steadfast gaze, but she often\r\ndoubted whether there were much admiration in it, and sometimes it\r\nseemed nothing but absence of mind.\r\n\r\nShe had once or twice suggested to Elizabeth the possibility of his\r\nbeing partial to her, but Elizabeth always laughed at the idea; and Mrs.\r\nCollins did not think it right to press the subject, from the danger of\r\nraising expectations which might only end in disappointment; for in her\r\nopinion it admitted not of a doubt, that all her friend’s dislike would\r\nvanish, if she could suppose him to be in her power.\r\n\r\nIn her kind schemes for Elizabeth, she sometimes planned her marrying\r\nColonel Fitzwilliam. He was, beyond comparison, the pleasantest man: he\r\ncertainly admired her, and his situation in life was most eligible; but,\r\nto counterbalance these advantages, Mr. Darcy had considerable patronage\r\nin the church, and his cousin could have none at all.\r\n\r\n\r\n\r\n\r\n[Illustration: “On looking up”]\r\n\r\n\r\n\r\n\r\nCHAPTER XXXIII.\r\n\r\n\r\n[Illustration]\r\n\r\nMore than once did Elizabeth, in her ramble within the park,\r\nunexpectedly meet Mr. Darcy. She felt all the perverseness of the\r\nmischance that should bring him where no one else was brought; and, to\r\nprevent its ever happening again, took care to inform him, at first,\r\nthat it was a favourite haunt of hers. How it could occur a second time,\r\ntherefore, was very odd! Yet it did, and even the third. It seemed like\r\nwilful ill-nature, or a voluntary penance; for on these occasions it was\r\nnot merely a few formal inquiries and an awkward pause and then away,\r\nbut he actually thought it necessary to turn back and walk with her. He\r\nnever said a great deal, nor did she give herself the trouble of talking\r\nor of listening much; but it struck her in the course of their third\r\nencounter that he was asking some odd unconnected questions--about her\r\npleasure in being at Hunsford, her love of solitary walks, and her\r\nopinion of Mr. and Mrs. Collins’s happiness; and that in speaking of\r\nRosings, and her not perfectly understanding the house, he seemed to\r\nexpect that whenever she came into Kent again she would be staying\r\n_there_ too. His words seemed to imply it. Could he have Colonel\r\nFitzwilliam in his thoughts? She supposed, if he meant anything, he must\r\nmean an allusion to what might arise in that quarter. It distressed her\r\na little, and she was quite glad to find herself at the gate in the\r\npales opposite the Parsonage.\r\n\r\nShe was engaged one day, as she walked, in re-perusing Jane’s last\r\nletter, and dwelling on some passages which proved that Jane had not\r\nwritten in spirits, when, instead of being again surprised by Mr. Darcy,\r\nshe saw, on looking up, that Colonel Fitzwilliam was meeting her.\r\nPutting away the letter immediately, and forcing a smile, she said,--\r\n\r\n“I did not know before that you ever walked this way.”\r\n\r\n“I have been making the tour of the park,” he replied, “as I generally\r\ndo every year, and intended to close it with a call at the Parsonage.\r\nAre you going much farther?”\r\n\r\n“No, I should have turned in a moment.”\r\n\r\nAnd accordingly she did turn, and they walked towards the Parsonage\r\ntogether.\r\n\r\n“Do you certainly leave Kent on Saturday?” said she.\r\n\r\n“Yes--if Darcy does not put it off again. But I am at his disposal. He\r\narranges the business just as he pleases.”\r\n\r\n“And if not able to please himself in the arrangement, he has at least\r\ngreat pleasure in the power of choice. I do not know anybody who seems\r\nmore to enjoy the power of doing what he likes than Mr. Darcy.”\r\n\r\n“He likes to have his own way very well,” replied Colonel Fitzwilliam.\r\n“But so we all do. It is only that he has better means of having it than\r\nmany others, because he is rich, and many others are poor. I speak\r\nfeelingly. A younger son, you know, must be inured to self-denial and\r\ndependence.”\r\n\r\n“In my opinion, the younger son of an earl can know very little of\r\neither. Now, seriously, what have you ever known of self-denial and\r\ndependence? When have you been prevented by want of money from going\r\nwherever you chose or procuring anything you had a fancy for?”\r\n\r\n“These are home questions--and perhaps I cannot say that I have\r\nexperienced many hardships of that nature. But in matters of greater\r\nweight, I may suffer from the want of money. Younger sons cannot marry\r\nwhere they like.”\r\n\r\n“Unless where they like women of fortune, which I think they very often\r\ndo.”\r\n\r\n“Our habits of expense make us too dependent, and there are not many in\r\nmy rank of life who can afford to marry without some attention to\r\nmoney.”\r\n\r\n“Is this,” thought Elizabeth, “meant for me?” and she coloured at the\r\nidea; but, recovering herself, said in a lively tone, “And pray, what is\r\nthe usual price of an earl’s younger son? Unless the elder brother is\r\nvery sickly, I suppose you would not ask above fifty thousand pounds.”\r\n\r\nHe answered her in the same style, and the subject dropped. To interrupt\r\na silence which might make him fancy her affected with what had passed,\r\nshe soon afterwards said,--\r\n\r\n“I imagine your cousin brought you down with him chiefly for the sake of\r\nhaving somebody at his disposal. I wonder he does not marry, to secure a\r\nlasting convenience of that kind. But, perhaps, his sister does as well\r\nfor the present; and, as she is under his sole care, he may do what he\r\nlikes with her.”\r\n\r\n“No,” said Colonel Fitzwilliam, “that is an advantage which he must\r\ndivide with me. I am joined with him in the guardianship of Miss Darcy.”\r\n\r\n“Are you, indeed? And pray what sort of a guardian do you make? Does\r\nyour charge give you much trouble? Young ladies of her age are sometimes\r\na little difficult to manage; and if she has the true Darcy spirit, she\r\nmay like to have her own way.”\r\n\r\nAs she spoke, she observed him looking at her earnestly; and the manner\r\nin which he immediately asked her why she supposed Miss Darcy likely to\r\ngive them any uneasiness, convinced her that she had somehow or other\r\ngot pretty near the truth. She directly replied,--\r\n\r\n“You need not be frightened. I never heard any harm of her; and I dare\r\nsay she is one of the most tractable creatures in the world. She is a\r\nvery great favourite with some ladies of my acquaintance, Mrs. Hurst and\r\nMiss Bingley. I think I have heard you say that you know them.”\r\n\r\n“I know them a little. Their brother is a pleasant, gentlemanlike\r\nman--he is a great friend of Darcy’s.”\r\n\r\n“Oh yes,” said Elizabeth drily--“Mr. Darcy is uncommonly kind to Mr.\r\nBingley, and takes a prodigious deal of care of him.”\r\n\r\n“Care of him! Yes, I really believe Darcy _does_ take care of him in\r\nthose points where he most wants care. From something that he told me\r\nin our journey hither, I have reason to think Bingley very much indebted\r\nto him. But I ought to beg his pardon, for I have no right to suppose\r\nthat Bingley was the person meant. It was all conjecture.”\r\n\r\n“What is it you mean?”\r\n\r\n“It is a circumstance which Darcy of course could not wish to be\r\ngenerally known, because if it were to get round to the lady’s family it\r\nwould be an unpleasant thing.”\r\n\r\n“You may depend upon my not mentioning it.”\r\n\r\n“And remember that I have not much reason for supposing it to be\r\nBingley. What he told me was merely this: that he congratulated himself\r\non having lately saved a friend from the inconveniences of a most\r\nimprudent marriage, but without mentioning names or any other\r\nparticulars; and I only suspected it to be Bingley from believing him\r\nthe kind of young man to get into a scrape of that sort, and from\r\nknowing them to have been together the whole of last summer.”\r\n\r\n“Did Mr. Darcy give you his reasons for this interference?”\r\n\r\n“I understood that there were some very strong objections against the\r\nlady.”\r\n\r\n“And what arts did he use to separate them?”\r\n\r\n“He did not talk to me of his own arts,” said Fitzwilliam, smiling. “He\r\nonly told me what I have now told you.”\r\n\r\nElizabeth made no answer, and walked on, her heart swelling with\r\nindignation. After watching her a little, Fitzwilliam asked her why she\r\nwas so thoughtful.\r\n\r\n“I am thinking of what you have been telling me,” said she. “Your\r\ncousin’s conduct does not suit my feelings. Why was he to be the\r\njudge?”\r\n\r\n“You are rather disposed to call his interference officious?”\r\n\r\n“I do not see what right Mr. Darcy had to decide on the propriety of his\r\nfriend’s inclination; or why, upon his own judgment alone, he was to\r\ndetermine and direct in what manner that friend was to be happy. But,”\r\nshe continued, recollecting herself, “as we know none of the\r\nparticulars, it is not fair to condemn him. It is not to be supposed\r\nthat there was much affection in the case.”\r\n\r\n“That is not an unnatural surmise,” said Fitzwilliam; “but it is\r\nlessening the honour of my cousin’s triumph very sadly.”\r\n\r\nThis was spoken jestingly, but it appeared to her so just a picture of\r\nMr. Darcy, that she would not trust herself with an answer; and,\r\ntherefore, abruptly changing the conversation, talked on indifferent\r\nmatters till they reached the Parsonage. There, shut into her own room,\r\nas soon as their visitor left them, she could think without interruption\r\nof all that she had heard. It was not to be supposed that any other\r\npeople could be meant than those with whom she was connected. There\r\ncould not exist in the world _two_ men over whom Mr. Darcy could have\r\nsuch boundless influence. That he had been concerned in the measures\r\ntaken to separate Mr. Bingley and Jane, she had never doubted; but she\r\nhad always attributed to Miss Bingley the principal design and\r\narrangement of them. If his own vanity, however, did not mislead him,\r\n_he_ was the cause--his pride and caprice were the cause--of all that\r\nJane had suffered, and still continued to suffer. He had ruined for a\r\nwhile every hope of happiness for the most affectionate, generous heart\r\nin the world; and no one could say how lasting an evil he might have\r\ninflicted.\r\n\r\n“There were some very strong objections against the lady,” were Colonel\r\nFitzwilliam’s words; and these strong objections probably were, her\r\nhaving one uncle who was a country attorney, and another who was in\r\nbusiness in London.\r\n\r\n“To Jane herself,” she exclaimed, “there could be no possibility of\r\nobjection,--all loveliness and goodness as she is! Her understanding\r\nexcellent, her mind improved, and her manners captivating. Neither could\r\nanything be urged against my father, who, though with some\r\npeculiarities, has abilities which Mr. Darcy himself need not disdain,\r\nand respectability which he will probably never reach.” When she thought\r\nof her mother, indeed, her confidence gave way a little; but she would\r\nnot allow that any objections _there_ had material weight with Mr.\r\nDarcy, whose pride, she was convinced, would receive a deeper wound from\r\nthe want of importance in his friend’s connections than from their want\r\nof sense; and she was quite decided, at last, that he had been partly\r\ngoverned by this worst kind of pride, and partly by the wish of\r\nretaining Mr. Bingley for his sister.\r\n\r\nThe agitation and tears which the subject occasioned brought on a\r\nheadache; and it grew so much worse towards the evening that, added to\r\nher unwillingness to see Mr. Darcy, it determined her not to attend her\r\ncousins to Rosings, where they were engaged to drink tea. Mrs. Collins,\r\nseeing that she was really unwell, did not press her to go, and as much\r\nas possible prevented her husband from pressing her; but Mr. Collins\r\ncould not conceal his apprehension of Lady Catherine’s being rather\r\ndispleased by her staying at home.\r\n\r\n\r\n\r\n\r\n[Illustration]\r\n\r\n\r\n\r\n\r\nCHAPTER XXXIV.\r\n\r\n\r\n[Illustration]\r\n\r\nWhen they were gone, Elizabeth, as if intending to exasperate herself as\r\nmuch as possible against Mr. Darcy, chose for her employment the\r\nexamination of all the letters which Jane had written to her since her\r\nbeing in Kent. They contained no actual complaint, nor was there any\r\nrevival of past occurrences, or any communication of present suffering.\r\nBut in all, and in almost every line of each, there was a want of that\r\ncheerfulness which had been used to characterize her style, and which,\r\nproceeding from the serenity of a mind at ease with itself, and kindly\r\ndisposed towards everyone, had been scarcely ever clouded. Elizabeth\r\nnoticed every sentence conveying the idea of uneasiness, with an\r\nattention which it had hardly received on the first perusal. Mr. Darcy’s\r\nshameful boast of what misery he had been able to inflict gave her a\r\nkeener sense of her sister’s sufferings. It was some consolation to\r\nthink that his visit to Rosings was to end on the day after the next,\r\nand a still greater that in less than a fortnight she should herself be\r\nwith Jane again, and enabled to contribute to the recovery of her\r\nspirits, by all that affection could do.\r\n\r\nShe could not think of Darcy’s leaving Kent without remembering that his\r\ncousin was to go with him; but Colonel Fitzwilliam had made it clear\r\nthat he had no intentions at all, and, agreeable as he was, she did not\r\nmean to be unhappy about him.\r\n\r\nWhile settling this point, she was suddenly roused by the sound of the\r\ndoor-bell; and her spirits were a little fluttered by the idea of its\r\nbeing Colonel Fitzwilliam himself, who had once before called late in\r\nthe evening, and might now come to inquire particularly after her. But\r\nthis idea was soon banished, and her spirits were very differently\r\naffected, when, to her utter amazement, she saw Mr. Darcy walk into the\r\nroom. In a hurried manner he immediately began an inquiry after her\r\nhealth, imputing his visit to a wish of hearing that she were better.\r\nShe answered him with cold civility. He sat down for a few moments, and\r\nthen getting up walked about the room. Elizabeth was surprised, but\r\nsaid not a word. After a silence of several minutes, he came towards her\r\nin an agitated manner, and thus began:--\r\n\r\n“In vain have I struggled. It will not do. My feelings will not be\r\nrepressed. You must allow me to tell you how ardently I admire and love\r\nyou.”\r\n\r\nElizabeth’s astonishment was beyond expression. She stared, coloured,\r\ndoubted, and was silent. This he considered sufficient encouragement,\r\nand the avowal of all that he felt and had long felt for her immediately\r\nfollowed. He spoke well; but there were feelings besides those of the\r\nheart to be detailed, and he was not more eloquent on the subject of\r\ntenderness than of pride. His sense of her inferiority, of its being a\r\ndegradation, of the family obstacles which judgment had always opposed\r\nto inclination, were dwelt on with a warmth which seemed due to the\r\nconsequence he was wounding, but was very unlikely to recommend his\r\nsuit.\r\n\r\nIn spite of her deeply-rooted dislike, she could not be insensible to\r\nthe compliment of such a man’s affection, and though her intentions did\r\nnot vary for an instant, she was at first sorry for the pain he was to\r\nreceive; till roused to resentment by his subsequent language, she lost\r\nall compassion in anger. She tried, however, to compose herself to\r\nanswer him with patience, when he should have done. He concluded with\r\nrepresenting to her the strength of that attachment which in spite of\r\nall his endeavours he had found impossible to conquer; and with\r\nexpressing his hope that it would now be rewarded by her acceptance of\r\nhis hand. As he said this she could easily see that he had no doubt of a\r\nfavourable answer. He _spoke_ of apprehension and anxiety, but his\r\ncountenance expressed real security. Such a circumstance could only\r\nexasperate farther; and when he ceased the colour rose into her cheeks\r\nand she said,--\r\n\r\n“In such cases as this, it is, I believe, the established mode to\r\nexpress a sense of obligation for the sentiments avowed, however\r\nunequally they may be returned. It is natural that obligation should be\r\nfelt, and if I could _feel_ gratitude, I would now thank you. But I\r\ncannot--I have never desired your good opinion, and you have certainly\r\nbestowed it most unwillingly. I am sorry to have occasioned pain to\r\nanyone. It has been most unconsciously done, however, and I hope will be\r\nof short duration. The feelings which you tell me have long prevented\r\nthe acknowledgment of your regard can have little difficulty in\r\novercoming it after this explanation.”\r\n\r\nMr. Darcy, who was leaning against the mantel-piece with his eyes fixed\r\non her face, seemed to catch her words with no less resentment than\r\nsurprise. His complexion became pale with anger, and the disturbance of\r\nhis mind was visible in every feature. He was struggling for the\r\nappearance of composure, and would not open his lips till he believed\r\nhimself to have attained it. The pause was to Elizabeth’s feelings\r\ndreadful. At length, in a voice of forced calmness, he said,--\r\n\r\n“And this is all the reply which I am to have the honour of expecting! I\r\nmight, perhaps, wish to be informed why, with so little _endeavour_ at\r\ncivility, I am thus rejected. But it is of small importance.”\r\n\r\n“I might as well inquire,” replied she, “why, with so evident a design\r\nof offending and insulting me, you chose to tell me that you liked me\r\nagainst your will, against your reason, and even against your character?\r\nWas not this some excuse for incivility, if I _was_ uncivil? But I have\r\nother provocations. You know I have. Had not my own feelings decided\r\nagainst you, had they been indifferent, or had they even been\r\nfavourable, do you think that any consideration would tempt me to accept\r\nthe man who has been the means of ruining, perhaps for ever, the\r\nhappiness of a most beloved sister?”\r\n\r\nAs she pronounced these words, Mr. Darcy changed colour; but the emotion\r\nwas short, and he listened without attempting to interrupt her while she\r\ncontinued,--\r\n\r\n“I have every reason in the world to think ill of you. No motive can\r\nexcuse the unjust and ungenerous part you acted _there_. You dare not,\r\nyou cannot deny that you have been the principal, if not the only means\r\nof dividing them from each other, of exposing one to the censure of the\r\nworld for caprice and instability, the other to its derision for\r\ndisappointed hopes, and involving them both in misery of the acutest\r\nkind.”\r\n\r\nShe paused, and saw with no slight indignation that he was listening\r\nwith an air which proved him wholly unmoved by any feeling of remorse.\r\nHe even looked at her with a smile of affected incredulity.\r\n\r\n“Can you deny that you have done it?” she repeated.\r\n\r\nWith assumed tranquillity he then replied, “I have no wish of denying\r\nthat I did everything in my power to separate my friend from your\r\nsister, or that I rejoice in my success. Towards _him_ I have been\r\nkinder than towards myself.”\r\n\r\nElizabeth disdained the appearance of noticing this civil reflection,\r\nbut its meaning did not escape, nor was it likely to conciliate her.\r\n\r\n“But it is not merely this affair,” she continued, “on which my dislike\r\nis founded. Long before it had taken place, my opinion of you was\r\ndecided. Your character was unfolded in the recital which I received\r\nmany months ago from Mr. Wickham. On this subject, what can you have to\r\nsay? In what imaginary act of friendship can you here defend yourself?\r\nor under what misrepresentation can you here impose upon others?”\r\n\r\n“You take an eager interest in that gentleman’s concerns,” said Darcy,\r\nin a less tranquil tone, and with a heightened colour.\r\n\r\n“Who that knows what his misfortunes have been can help feeling an\r\ninterest in him?”\r\n\r\n“His misfortunes!” repeated Darcy, contemptuously,--“yes, his\r\nmisfortunes have been great indeed.”\r\n\r\n“And of your infliction,” cried Elizabeth, with energy; “You have\r\nreduced him to his present state of poverty--comparative poverty. You\r\nhave withheld the advantages which you must know to have been designed\r\nfor him. You have deprived the best years of his life of that\r\nindependence which was no less his due than his desert. You have done\r\nall this! and yet you can treat the mention of his misfortunes with\r\ncontempt and ridicule.”\r\n\r\n“And this,” cried Darcy, as he walked with quick steps across the room,\r\n“is your opinion of me! This is the estimation in which you hold me! I\r\nthank you for explaining it so fully. My faults, according to this\r\ncalculation, are heavy indeed! But, perhaps,” added he, stopping in his\r\nwalk, and turning towards her, “these offences might have been\r\noverlooked, had not your pride been hurt by my honest confession of the\r\nscruples that had long prevented my forming any serious design. These\r\nbitter accusations might have been suppressed, had I, with greater\r\npolicy, concealed my struggles, and flattered you into the belief of my\r\nbeing impelled by unqualified, unalloyed inclination; by reason, by\r\nreflection, by everything. But disguise of every sort is my abhorrence.\r\nNor am I ashamed of the feelings I related. They were natural and just.\r\nCould you expect me to rejoice in the inferiority of your\r\nconnections?--to congratulate myself on the hope of relations whose\r\ncondition in life is so decidedly beneath my own?”\r\n\r\nElizabeth felt herself growing more angry every moment; yet she tried to\r\nthe utmost to speak with composure when she said,--\r\n\r\n“You are mistaken, Mr. Darcy, if you suppose that the mode of your\r\ndeclaration affected me in any other way than as it spared me the\r\nconcern which I might have felt in refusing you, had you behaved in a\r\nmore gentlemanlike manner.”\r\n\r\nShe saw him start at this; but he said nothing, and she continued,--\r\n\r\n“You could not have made me the offer of your hand in any possible way\r\nthat would have tempted me to accept it.”\r\n\r\nAgain his astonishment was obvious; and he looked at her with an\r\nexpression of mingled incredulity and mortification. She went on,--\r\n\r\n“From the very beginning, from the first moment, I may almost say, of my\r\nacquaintance with you, your manners impressing me with the fullest\r\nbelief of your arrogance, your conceit, and your selfish disdain of the\r\nfeelings of others, were such as to form that groundwork of\r\ndisapprobation, on which succeeding events have built so immovable a\r\ndislike; and I had not known you a month before I felt that you were the\r\nlast man in the world whom I could ever be prevailed on to marry.”\r\n\r\n“You have said quite enough, madam. I perfectly comprehend your\r\nfeelings, and have now only to be ashamed of what my own have been.\r\nForgive me for having taken up so much of your time, and accept my best\r\nwishes for your health and happiness.”\r\n\r\nAnd with these words he hastily left the room, and Elizabeth heard him\r\nthe next moment open the front door and quit the house. The tumult of\r\nher mind was now painfully great. She knew not how to support herself,\r\nand, from actual weakness, sat down and cried for half an hour. Her\r\nastonishment, as she reflected on what had passed, was increased by\r\nevery review of it. That she should receive an offer of marriage from\r\nMr. Darcy! that he should have been in love with her for so many months!\r\nso much in love as to wish to marry her in spite of all the objections\r\nwhich had made him prevent his friend’s marrying her sister, and which\r\nmust appear at least with equal force in his own case, was almost\r\nincredible! it was gratifying to have inspired unconsciously so strong\r\nan affection. But his pride, his abominable pride, his shameless avowal\r\nof what he had done with respect to Jane, his unpardonable assurance in\r\nacknowledging, though he could not justify it, and the unfeeling manner\r\nwhich he had mentioned Mr. Wickham, his cruelty towards whom he had not\r\nattempted to deny, soon overcame the pity which the consideration of his\r\nattachment had for a moment excited.\r\n\r\nShe continued in very agitating reflections till the sound of Lady\r\nCatherine’s carriage made her feel how unequal she was to encounter\r\nCharlotte’s observation, and hurried her away to her room.\r\n\r\n\r\n\r\n\r\n[Illustration:\r\n\r\n“Hearing herself called”\r\n]\r\n\r\n\r\n\r\n\r\nCHAPTER XXXV.\r\n\r\n\r\n[Illustration]\r\n\r\nElizabeth awoke the next morning to the same thoughts and meditations\r\nwhich had at length closed her eyes. She could not yet recover from the\r\nsurprise of what had happened: it was impossible to think of anything\r\nelse; and, totally indisposed for employment, she resolved soon after\r\nbreakfast to indulge herself in air and exercise. She was proceeding\r\ndirectly to her favourite walk, when the recollection of Mr. Darcy’s\r\nsometimes coming there stopped her, and instead of entering the park,\r\nshe turned up the lane which led her farther from the turnpike road. The\r\npark paling was still the boundary on one side, and she soon passed one\r\nof the gates into the ground.\r\n\r\nAfter walking two or three times along that part of the lane, she was\r\ntempted, by the pleasantness of the morning, to stop at the gates and\r\nlook into the park. The five weeks which she had now passed in Kent had\r\nmade a great difference in the country, and every day was adding to the\r\nverdure of the early trees. She was on the point of continuing her\r\nwalk, when she caught a glimpse of a gentleman within the sort of grove\r\nwhich edged the park: he was moving that way; and fearful of its being\r\nMr. Darcy, she was directly retreating. But the person who advanced was\r\nnow near enough to see her, and stepping forward with eagerness,\r\npronounced her name. She had turned away; but on hearing herself called,\r\nthough in a voice which proved it to be Mr. Darcy, she moved again\r\ntowards the gate. He had by that time reached it also; and, holding out\r\na letter, which she instinctively took, said, with a look of haughty\r\ncomposure, “I have been walking in the grove some time, in the hope of\r\nmeeting you. Will you do me the honour of reading that letter?” and\r\nthen, with a slight bow, turned again into the plantation, and was soon\r\nout of sight.\r\n\r\nWith no expectation of pleasure, but with the strongest curiosity,\r\nElizabeth opened the letter, and to her still increasing wonder,\r\nperceived an envelope containing two sheets of letter paper, written\r\nquite through, in a very close hand. The envelope itself was likewise\r\nfull. Pursuing her way along the lane, she then began it. It was dated\r\nfrom Rosings, at eight o’clock in the morning, and was as follows:--\r\n\r\n“Be not alarmed, madam, on receiving this letter, by the apprehension of\r\nits containing any repetition of those sentiments, or renewal of those\r\noffers, which were last night so disgusting to you. I write without any\r\nintention of paining you, or humbling myself, by dwelling on wishes,\r\nwhich, for the happiness of both, cannot be too soon forgotten; and the\r\neffort which the formation and the perusal of this letter must occasion,\r\nshould have been spared, had not my character required it to be written\r\nand read. You must, therefore, pardon the freedom with which I demand\r\nyour attention; your feelings, I know, will bestow it unwillingly, but I\r\ndemand it of your justice.\r\n\r\n“Two offences of a very different nature, and by no means of equal\r\nmagnitude, you last night laid to my charge. The first mentioned was,\r\nthat, regardless of the sentiments of either, I had detached Mr. Bingley\r\nfrom your sister,--and the other, that I had, in defiance of various\r\nclaims, in defiance of honour and humanity, ruined the immediate\r\nprosperity and blasted the prospects of Mr. Wickham. Wilfully and\r\nwantonly to have thrown off the companion of my youth, the acknowledged\r\nfavourite of my father, a young man who had scarcely any other\r\ndependence than on our patronage, and who had been brought up to expect\r\nits exertion, would be a depravity, to which the separation of two young\r\npersons whose affection could be the growth of only a few weeks, could\r\nbear no comparison. But from the severity of that blame which was last\r\nnight so liberally bestowed, respecting each circumstance, I shall hope\r\nto be in future secured, when the following account of my actions and\r\ntheir motives has been read. If, in the explanation of them which is due\r\nto myself, I am under the necessity of relating feelings which may be\r\noffensive to yours, I can only say that I am sorry. The necessity must\r\nbe obeyed, and further apology would be absurd. I had not been long in\r\nHertfordshire before I saw, in common with others, that Bingley\r\npreferred your elder sister to any other young woman in the country. But\r\nit was not till the evening of the dance at Netherfield that I had any\r\napprehension of his feeling a serious attachment. I had often seen him\r\nin love before. At that ball, while I had the honour of dancing with\r\nyou, I was first made acquainted, by Sir William Lucas’s accidental\r\ninformation, that Bingley’s attentions to your sister had given rise to\r\na general expectation of their marriage. He spoke of it as a certain\r\nevent, of which the time alone could be undecided. From that moment I\r\nobserved my friend’s behaviour attentively; and I could then perceive\r\nthat his partiality for Miss Bennet was beyond what I had ever witnessed\r\nin him. Your sister I also watched. Her look and manners were open,\r\ncheerful, and engaging as ever, but without any symptom of peculiar\r\nregard; and I remained convinced, from the evening’s scrutiny, that\r\nthough she received his attentions with pleasure, she did not invite\r\nthem by any participation of sentiment. If _you_ have not been mistaken\r\nhere, _I_ must have been in an error. Your superior knowledge of your\r\nsister must make the latter probable. If it be so, if I have been misled\r\nby such error to inflict pain on her, your resentment has not been\r\nunreasonable. But I shall not scruple to assert, that the serenity of\r\nyour sister’s countenance and air was such as might have given the most\r\nacute observer a conviction that, however amiable her temper, her heart\r\nwas not likely to be easily touched. That I was desirous of believing\r\nher indifferent is certain; but I will venture to say that my\r\ninvestigations and decisions are not usually influenced by my hopes or\r\nfears. I did not believe her to be indifferent because I wished it; I\r\nbelieved it on impartial conviction, as truly as I wished it in reason.\r\nMy objections to the marriage were not merely those which I last night\r\nacknowledged to have required the utmost force of passion to put aside\r\nin my own case; the want of connection could not be so great an evil to\r\nmy friend as to me. But there were other causes of repugnance; causes\r\nwhich, though still existing, and existing to an equal degree in both\r\ninstances, I had myself endeavoured to forget, because they were not\r\nimmediately before me. These causes must be stated, though briefly. The\r\nsituation of your mother’s family, though objectionable, was nothing in\r\ncomparison of that total want of propriety so frequently, so almost\r\nuniformly betrayed by herself, by your three younger sisters, and\r\noccasionally even by your father:--pardon me,--it pains me to offend\r\nyou. But amidst your concern for the defects of your nearest relations,\r\nand your displeasure at this representation of them, let it give you\r\nconsolation to consider that to have conducted yourselves so as to avoid\r\nany share of the like censure is praise no less generally bestowed on\r\nyou and your eldest sister than it is honourable to the sense and\r\ndisposition of both. I will only say, farther, that from what passed\r\nthat evening my opinion of all parties was confirmed, and every\r\ninducement heightened, which could have led me before to preserve my\r\nfriend from what I esteemed a most unhappy connection. He left\r\nNetherfield for London on the day following, as you, I am certain,\r\nremember, with the design of soon returning. The part which I acted is\r\nnow to be explained. His sisters’ uneasiness had been equally excited\r\nwith my own: our coincidence of feeling was soon discovered; and, alike\r\nsensible that no time was to be lost in detaching their brother, we\r\nshortly resolved on joining him directly in London. We accordingly\r\nwent--and there I readily engaged in the office of pointing out to my\r\nfriend the certain evils of such a choice. I described and enforced them\r\nearnestly. But however this remonstrance might have staggered or delayed\r\nhis determination, I do not suppose that it would ultimately have\r\nprevented the marriage, had it not been seconded by the assurance, which\r\nI hesitated not in giving, of your sister’s indifference. He had before\r\nbelieved her to return his affection with sincere, if not with equal,\r\nregard. But Bingley has great natural modesty, with a stronger\r\ndependence on my judgment than on his own. To convince him, therefore,\r\nthat he had deceived himself was no very difficult point. To persuade\r\nhim against returning into Hertfordshire, when that conviction had been\r\ngiven, was scarcely the work of a moment. I cannot blame myself for\r\nhaving done thus much. There is but one part of my conduct, in the whole\r\naffair, on which I do not reflect with satisfaction; it is that I\r\ncondescended to adopt the measures of art so far as to conceal from him\r\nyour sister’s being in town. I knew it myself, as it was known to Miss\r\nBingley; but her brother is even yet ignorant of it. That they might\r\nhave met without ill consequence is, perhaps, probable; but his regard\r\ndid not appear to me enough extinguished for him to see her without some\r\ndanger. Perhaps this concealment, this disguise, was beneath me. It is\r\ndone, however, and it was done for the best. On this subject I have\r\nnothing more to say, no other apology to offer. If I have wounded your\r\nsister’s feelings, it was unknowingly done; and though the motives which\r\ngoverned me may to you very naturally appear insufficient, I have not\r\nyet learnt to condemn them.--With respect to that other, more weighty\r\naccusation, of having injured Mr. Wickham, I can only refute it by\r\nlaying before you the whole of his connection with my family. Of what he\r\nhas _particularly_ accused me I am ignorant; but of the truth of what I\r\nshall relate I can summon more than one witness of undoubted veracity.\r\nMr. Wickham is the son of a very respectable man, who had for many years\r\nthe management of all the Pemberley estates, and whose good conduct in\r\nthe discharge of his trust naturally inclined my father to be of service\r\nto him; and on George Wickham, who was his godson, his kindness was\r\ntherefore liberally bestowed. My father supported him at school, and\r\nafterwards at Cambridge; most important assistance, as his own father,\r\nalways poor from the extravagance of his wife, would have been unable to\r\ngive him a gentleman’s education. My father was not only fond of this\r\nyoung man’s society, whose manners were always engaging, he had also the\r\nhighest opinion of him, and hoping the church would be his profession,\r\nintended to provide for him in it. As for myself, it is many, many years\r\nsince I first began to think of him in a very different manner. The\r\nvicious propensities, the want of principle, which he was careful to\r\nguard from the knowledge of his best friend, could not escape the\r\nobservation of a young man of nearly the same age with himself, and who\r\nhad opportunities of seeing him in unguarded moments, which Mr. Darcy\r\ncould not have. Here again I shall give you pain--to what degree you\r\nonly can tell. But whatever may be the sentiments which Mr. Wickham has\r\ncreated, a suspicion of their nature shall not prevent me from unfolding\r\nhis real character. It adds even another motive. My excellent father\r\ndied about five years ago; and his attachment to Mr. Wickham was to the\r\nlast so steady, that in his will he particularly recommended it to me to\r\npromote his advancement in the best manner that his profession might\r\nallow, and if he took orders, desired that a valuable family living\r\nmight be his as soon as it became vacant. There was also a legacy of\r\none thousand pounds. His own father did not long survive mine; and\r\nwithin half a year from these events Mr. Wickham wrote to inform me\r\nthat, having finally resolved against taking orders, he hoped I should\r\nnot think it unreasonable for him to expect some more immediate\r\npecuniary advantage, in lieu of the preferment, by which he could not be\r\nbenefited. He had some intention, he added, of studying the law, and I\r\nmust be aware that the interest of one thousand pounds would be a very\r\ninsufficient support therein. I rather wished than believed him to be\r\nsincere; but, at any rate, was perfectly ready to accede to his\r\nproposal. I knew that Mr. Wickham ought not to be a clergyman. The\r\nbusiness was therefore soon settled. He resigned all claim to assistance\r\nin the church, were it possible that he could ever be in a situation to\r\nreceive it, and accepted in return three thousand pounds. All connection\r\nbetween us seemed now dissolved. I thought too ill of him to invite him\r\nto Pemberley, or admit his society in town. In town, I believe, he\r\nchiefly lived, but his studying the law was a mere pretence; and being\r\nnow free from all restraint, his life was a life of idleness and\r\ndissipation. For about three years I heard little of him; but on the\r\ndecease of the incumbent of the living which had been designed for him,\r\nhe applied to me again by letter for the presentation. His\r\ncircumstances, he assured me, and I had no difficulty in believing it,\r\nwere exceedingly bad. He had found the law a most unprofitable study,\r\nand was now absolutely resolved on being ordained, if I would present\r\nhim to the living in question--of which he trusted there could be little\r\ndoubt, as he was well assured that I had no other person to provide for,\r\nand I could not have forgotten my revered father’s intentions. You will\r\nhardly blame me for refusing to comply with this entreaty, or for\r\nresisting every repetition of it. His resentment was in proportion to\r\nthe distress of his circumstances--and he was doubtless as violent in\r\nhis abuse of me to others as in his reproaches to myself. After this\r\nperiod, every appearance of acquaintance was dropped. How he lived, I\r\nknow not. But last summer he was again most painfully obtruded on my\r\nnotice. I must now mention a circumstance which I would wish to forget\r\nmyself, and which no obligation less than the present should induce me\r\nto unfold to any human being. Having said thus much, I feel no doubt of\r\nyour secrecy. My sister, who is more than ten years my junior, was left\r\nto the guardianship of my mother’s nephew, Colonel Fitzwilliam, and\r\nmyself. About a year ago, she was taken from school, and an\r\nestablishment formed for her in London; and last summer she went with\r\nthe lady who presided over it to Ramsgate; and thither also went Mr.\r\nWickham, undoubtedly by design; for there proved to have been a prior\r\nacquaintance between him and Mrs. Younge, in whose character we were\r\nmost unhappily deceived; and by her connivance and aid he so far\r\nrecommended himself to Georgiana, whose affectionate heart retained a\r\nstrong impression of his kindness to her as a child, that she was\r\npersuaded to believe herself in love and to consent to an elopement. She\r\nwas then but fifteen, which must be her excuse; and after stating her\r\nimprudence, I am happy to add, that I owed the knowledge of it to\r\nherself. I joined them unexpectedly a day or two before the intended\r\nelopement; and then Georgiana, unable to support the idea of grieving\r\nand offending a brother whom she almost looked up to as a father,\r\nacknowledged the whole to me. You may imagine what I felt and how I\r\nacted. Regard for my sister’s credit and feelings prevented any public\r\nexposure; but I wrote to Mr. Wickham, who left the place immediately,\r\nand Mrs. Younge was of course removed from her charge. Mr. Wickham’s\r\nchief object was unquestionably my sister’s fortune, which is thirty\r\nthousand pounds; but I cannot help supposing that the hope of revenging\r\nhimself on me was a strong inducement. His revenge would have been\r\ncomplete indeed. This, madam, is a faithful narrative of every event in\r\nwhich we have been concerned together; and if you do not absolutely\r\nreject it as false, you will, I hope, acquit me henceforth of cruelty\r\ntowards Mr. Wickham. I know not in what manner, under what form of\r\nfalsehood, he has imposed on you; but his success is not perhaps to be\r\nwondered at, ignorant as you previously were of everything concerning\r\neither. Detection could not be in your power, and suspicion certainly\r\nnot in your inclination. You may possibly wonder why all this was not\r\ntold you last night. But I was not then master enough of myself to know\r\nwhat could or ought to be revealed. For the truth of everything here\r\nrelated, I can appeal more particularly to the testimony of Colonel\r\nFitzwilliam, who, from our near relationship and constant intimacy, and\r\nstill more as one of the executors of my father’s will, has been\r\nunavoidably acquainted with every particular of these transactions. If\r\nyour abhorrence of _me_ should make _my_ assertions valueless, you\r\ncannot be prevented by the same cause from confiding in my cousin; and\r\nthat there may be the possibility of consulting him, I shall endeavour\r\nto find some opportunity of putting this letter in your hands in the\r\ncourse of the morning. I will only add, God bless you.\r\n\r\n“FITZWILLIAM DARCY.”\r\n\r\n\r\n\r\n\r\n[Illustration]\r\n\r\n\r\n\r\n\r\nCHAPTER XXXVI.\r\n\r\n\r\n[Illustration]\r\n\r\nElizabeth, when Mr. Darcy gave her the letter, did not expect it to\r\ncontain a renewal of his offers, she had formed no expectation at all of\r\nits contents. But such as they were, it may be well supposed how eagerly\r\nshe went through them, and what a contrariety of emotion they excited.\r\nHer feelings as she read were scarcely to be defined. With amazement did\r\nshe first understand that he believed any apology to be in his power;\r\nand steadfastly was she persuaded, that he could have no explanation to\r\ngive, which a just sense of shame would not conceal. With a strong\r\nprejudice against everything he might say, she began his account of\r\nwhat had happened at Netherfield. She read with an eagerness which\r\nhardly left her power of comprehension; and from impatience of knowing\r\nwhat the next sentence might bring, was incapable of attending to the\r\nsense of the one before her eyes. His belief of her sister’s\r\ninsensibility she instantly resolved to be false; and his account of the\r\nreal, the worst objections to the match, made her too angry to have any\r\nwish of doing him justice. He expressed no regret for what he had done\r\nwhich satisfied her; his style was not penitent, but haughty. It was all\r\npride and insolence.\r\n\r\nBut when this subject was succeeded by his account of Mr. Wickham--when\r\nshe read, with somewhat clearer attention, a relation of events which,\r\nif true, must overthrow every cherished opinion of his worth, and which\r\nbore so alarming an affinity to his own history of himself--her feelings\r\nwere yet more acutely painful and more difficult of definition.\r\nAstonishment, apprehension, and even horror, oppressed her. She wished\r\nto discredit it entirely, repeatedly exclaiming, “This must be false!\r\nThis cannot be! This must be the grossest falsehood!”--and when she had\r\ngone through the whole letter, though scarcely knowing anything of the\r\nlast page or two, put it hastily away, protesting that she would not\r\nregard it, that she would never look in it again.\r\n\r\nIn this perturbed state of mind, with thoughts that could rest on\r\nnothing, she walked on; but it would not do: in half a minute the letter\r\nwas unfolded again; and collecting herself as well as she could, she\r\nagain began the mortifying perusal of all that related to Wickham, and\r\ncommanded herself so far as to examine the meaning of every sentence.\r\nThe account of his connection with the Pemberley family was exactly\r\nwhat he had related himself; and the kindness of the late Mr. Darcy,\r\nthough she had not before known its extent, agreed equally well with his\r\nown words. So far each recital confirmed the other; but when she came to\r\nthe will, the difference was great. What Wickham had said of the living\r\nwas fresh in her memory; and as she recalled his very words, it was\r\nimpossible not to feel that there was gross duplicity on one side or the\r\nother, and, for a few moments, she flattered herself that her wishes did\r\nnot err. But when she read and re-read, with the closest attention, the\r\nparticulars immediately following of Wickham’s resigning all pretensions\r\nto the living, of his receiving in lieu so considerable a sum as three\r\nthousand pounds, again was she forced to hesitate. She put down the\r\nletter, weighed every circumstance with what she meant to be\r\nimpartiality--deliberated on the probability of each statement--but with\r\nlittle success. On both sides it was only assertion. Again she read on.\r\nBut every line proved more clearly that the affair, which she had\r\nbelieved it impossible that any contrivance could so represent as to\r\nrender Mr. Darcy’s conduct in it less than infamous, was capable of a\r\nturn which must make him entirely blameless throughout the whole.\r\n\r\nThe extravagance and general profligacy which he scrupled not to lay to\r\nMr. Wickham’s charge exceedingly shocked her; the more so, as she could\r\nbring no proof of its injustice. She had never heard of him before his\r\nentrance into the ----shire militia, in which he had engaged at the\r\npersuasion of the young man, who, on meeting him accidentally in town,\r\nhad there renewed a slight acquaintance. Of his former way of life,\r\nnothing had been known in Hertfordshire but what he told\r\n\r\n[Illustration:\r\n\r\n “Meeting accidentally in Town”\r\n\r\n[_Copyright 1894 by George Allen._]]\r\n\r\nhimself. As to his real character, had information been in her power,\r\nshe had never felt a wish of inquiring. His countenance, voice, and\r\nmanner, had established him at once in the possession of every virtue.\r\nShe tried to recollect some instance of goodness, some distinguished\r\ntrait of integrity or benevolence, that might rescue him from the\r\nattacks of Mr. Darcy; or at least, by the predominance of virtue, atone\r\nfor those casual errors, under which she would endeavour to class what\r\nMr. Darcy had described as the idleness and vice of many years’\r\ncontinuance. But no such recollection befriended her. She could see him\r\ninstantly before her, in every charm of air and address, but she could\r\nremember no more substantial good than the general approbation of the\r\nneighbourhood, and the regard which his social powers had gained him in\r\nthe mess. After pausing on this point a considerable while, she once\r\nmore continued to read. But, alas! the story which followed, of his\r\ndesigns on Miss Darcy, received some confirmation from what had passed\r\nbetween Colonel Fitzwilliam and herself only the morning before; and at\r\nlast she was referred for the truth of every particular to Colonel\r\nFitzwilliam himself--from whom she had previously received the\r\ninformation of his near concern in all his cousin’s affairs and whose\r\ncharacter she had no reason to question. At one time she had almost\r\nresolved on applying to him, but the idea was checked by the awkwardness\r\nof the application, and at length wholly banished by the conviction that\r\nMr. Darcy would never have hazarded such a proposal, if he had not been\r\nwell assured of his cousin’s corroboration.\r\n\r\nShe perfectly remembered everything that had passed in conversation\r\nbetween Wickham and herself in their first evening at Mr. Philips’s.\r\nMany of his expressions were still fresh in her memory. She was _now_\r\nstruck with the impropriety of such communications to a stranger, and\r\nwondered it had escaped her before. She saw the indelicacy of putting\r\nhimself forward as he had done, and the inconsistency of his professions\r\nwith his conduct. She remembered that he had boasted of having no fear\r\nof seeing Mr. Darcy--that Mr. Darcy might leave the country, but that\r\n_he_ should stand his ground; yet he had avoided the Netherfield ball\r\nthe very next week. She remembered, also, that till the Netherfield\r\nfamily had quitted the country, he had told his story to no one but\r\nherself; but that after their removal, it had been everywhere discussed;\r\nthat he had then no reserves, no scruples in sinking Mr. Darcy’s\r\ncharacter, though he had assured her that respect for the father would\r\nalways prevent his exposing the son.\r\n\r\nHow differently did everything now appear in which he was concerned! His\r\nattentions to Miss King were now the consequence of views solely and\r\nhatefully mercenary; and the mediocrity of her fortune proved no longer\r\nthe moderation of his wishes, but his eagerness to grasp at anything.\r\nHis behaviour to herself could now have had no tolerable motive: he had\r\neither been deceived with regard to her fortune, or had been gratifying\r\nhis vanity by encouraging the preference which she believed she had most\r\nincautiously shown. Every lingering struggle in his favour grew fainter\r\nand fainter; and in further justification of Mr. Darcy, she could not\r\nbut allow that Mr. Bingley, when questioned by Jane, had long ago\r\nasserted his blamelessness in the affair;--that, proud and repulsive as\r\nwere his manners, she had never, in the whole course of their\r\nacquaintance--an acquaintance which had latterly brought them much\r\ntogether, and given her a sort of intimacy with his ways--seen anything\r\nthat betrayed him to be unprincipled or unjust--anything that spoke him\r\nof irreligious or immoral habits;--that among his own connections he was\r\nesteemed and valued;--that even Wickham had allowed him merit as a\r\nbrother, and that she had often heard him speak so affectionately of his\r\nsister as to prove him capable of some amiable feeling;--that had his\r\nactions been what Wickham represented them, so gross a violation of\r\neverything right could hardly have been concealed from the world; and\r\nthat friendship between a person capable of it and such an amiable man\r\nas Mr. Bingley was incomprehensible.\r\n\r\nShe grew absolutely ashamed of herself. Of neither Darcy nor Wickham\r\ncould she think, without feeling that she had been blind, partial,\r\nprejudiced, absurd.\r\n\r\n“How despicably have I acted!” she cried. “I, who have prided myself on\r\nmy discernment! I, who have valued myself on my abilities! who have\r\noften disdained the generous candour of my sister, and gratified my\r\nvanity in useless or blameless distrust. How humiliating is this\r\ndiscovery! Yet, how just a humiliation! Had I been in love, I could not\r\nhave been more wretchedly blind. But vanity, not love, has been my\r\nfolly. Pleased with the preference of one, and offended by the neglect\r\nof the other, on the very beginning of our acquaintance, I have courted\r\nprepossession and ignorance, and driven reason away where either were\r\nconcerned. Till this moment, I never knew myself.”\r\n\r\nFrom herself to Jane, from Jane to Bingley, her thoughts were in a line\r\nwhich soon brought to her recollection that Mr. Darcy’s explanation\r\n_there_ had appeared very insufficient; and she read it again. Widely\r\ndifferent was the effect of a second perusal. How could she deny that\r\ncredit to his assertions, in one instance, which she had been obliged to\r\ngive in the other? He declared himself to have been totally unsuspicious\r\nof her sister’s attachment; and she could not help remembering what\r\nCharlotte’s opinion had always been. Neither could she deny the justice\r\nof his description of Jane. She felt that Jane’s feelings, though\r\nfervent, were little displayed, and that there was a constant\r\ncomplacency in her air and manner, not often united with great\r\nsensibility.\r\n\r\nWhen she came to that part of the letter in which her family were\r\nmentioned, in tones of such mortifying, yet merited, reproach, her sense\r\nof shame was severe. The justice of the charge struck her too forcibly\r\nfor denial; and the circumstances to which he particularly alluded, as\r\nhaving passed at the Netherfield ball, and as confirming all his first\r\ndisapprobation, could not have made a stronger impression on his mind\r\nthan on hers.\r\n\r\nThe compliment to herself and her sister was not unfelt. It soothed, but\r\nit could not console her for the contempt which had been thus\r\nself-attracted by the rest of her family; and as she considered that\r\nJane’s disappointment had, in fact, been the work of her nearest\r\nrelations, and reflected how materially the credit of both must be hurt\r\nby such impropriety of conduct, she felt depressed beyond anything she\r\nhad ever known before.\r\n\r\nAfter wandering along the lane for two hours, giving way to every\r\nvariety of thought, reconsidering events, determining probabilities, and\r\nreconciling herself, as well as she could, to a change so sudden and so\r\nimportant, fatigue, and a recollection of her long absence, made her at\r\nlength return home; and she entered the house with the wish of appearing\r\ncheerful as usual, and the resolution of repressing such reflections as\r\nmust make her unfit for conversation.\r\n\r\nShe was immediately told, that the two gentlemen from Rosings had each\r\ncalled during her absence; Mr. Darcy, only for a few minutes, to take\r\nleave, but that Colonel Fitzwilliam had been sitting with them at least\r\nan hour, hoping for her return, and almost resolving to walk after her\r\ntill she could be found. Elizabeth could but just _affect_ concern in\r\nmissing him; she really rejoiced at it. Colonel Fitzwilliam was no\r\nlonger an object. She could think only of her letter.\r\n\r\n\r\n\r\n\r\n[Illustration:\r\n\r\n“His parting obeisance”\r\n]\r\n\r\n\r\n\r\n\r\nCHAPTER XXXVII.\r\n\r\n\r\n[Illustration]\r\n\r\nThe two gentlemen left Rosings the next morning; and Mr. Collins having\r\nbeen in waiting near the lodges, to make them his parting obeisance, was\r\nable to bring home the pleasing intelligence of their appearing in very\r\ngood health, and in as tolerable spirits as could be expected, after the\r\nmelancholy scene so lately gone through at Rosings. To Rosings he then\r\nhastened to console Lady Catherine and her daughter; and on his return\r\nbrought back, with great satisfaction, a message from her Ladyship,\r\nimporting that she felt herself so dull as to make her very desirous of\r\nhaving them all to dine with her.\r\n\r\nElizabeth could not see Lady Catherine without recollecting that, had\r\nshe chosen it, she might by this time have been presented to her as her\r\nfuture niece; nor could she think, without a smile, of what her\r\nLadyship’s indignation would have been. “What would she have said? how\r\nwould she have behaved?” were the questions with which she amused\r\nherself.\r\n\r\nTheir first subject was the diminution of the Rosings’ party. “I assure\r\nyou, I feel it exceedingly,” said Lady Catherine; “I believe nobody\r\nfeels the loss of friends so much as I do. But I am particularly\r\nattached to these young men; and know them to be so much attached to me!\r\nThey were excessively sorry to go! But so they always are. The dear\r\nColonel rallied his spirits tolerably till just at last; but Darcy\r\nseemed to feel it most acutely--more, I think, than last year. His\r\nattachment to Rosings certainly increases.”\r\n\r\nMr. Collins had a compliment and an allusion to throw in here, which\r\nwere kindly smiled on by the mother and daughter.\r\n\r\nLady Catherine observed, after dinner, that Miss Bennet seemed out of\r\nspirits; and immediately accounting for it herself, by supposing that\r\nshe did not like to go home again so soon, she added,--\r\n\r\n“But if that is the case, you must write to your mother to beg that you\r\nmay stay a little longer. Mrs. Collins will be very glad of your\r\ncompany, I am sure.”\r\n\r\n“I am much obliged to your Ladyship for your kind invitation,” replied\r\nElizabeth; “but it is not in my power to accept it. I must be in town\r\nnext Saturday.”\r\n\r\n“Why, at that rate, you will have been here only six weeks. I expected\r\nyou to stay two months. I told Mrs. Collins so before you came. There\r\ncan be no occasion for your going so soon. Mrs. Bennet could certainly\r\nspare you for another fortnight.”\r\n\r\n“But my father cannot. He wrote last week to hurry my return.”\r\n\r\n[Illustration:\r\n\r\n“Dawson”\r\n\r\n[_Copyright 1894 by George Allen._]]\r\n\r\n“Oh, your father, of course, may spare you, if your mother can.\r\nDaughters are never of so much consequence to a father. And if you will\r\nstay another _month_ complete, it will be in my power to take one of you\r\nas far as London, for I am going there early in June, for a week; and\r\nas Dawson does not object to the barouche-box, there will be very good\r\nroom for one of you--and, indeed, if the weather should happen to be\r\ncool, I should not object to taking you both, as you are neither of you\r\nlarge.”\r\n\r\n“You are all kindness, madam; but I believe we must abide by our\r\noriginal plan.”\r\n\r\nLady Catherine seemed resigned. “Mrs. Collins, you must send a servant\r\nwith them. You know I always speak my mind, and I cannot bear the idea\r\nof two young women travelling post by themselves. It is highly improper.\r\nYou must contrive to send somebody. I have the greatest dislike in the\r\nworld to that sort of thing. Young women should always be properly\r\nguarded and attended, according to their situation in life. When my\r\nniece Georgiana went to Ramsgate last summer, I made a point of her\r\nhaving two men-servants go with her. Miss Darcy, the daughter of Mr.\r\nDarcy of Pemberley, and Lady Anne, could not have appeared with\r\npropriety in a different manner. I am excessively attentive to all those\r\nthings. You must send John with the young ladies, Mrs. Collins. I am\r\nglad it occurred to me to mention it; for it would really be\r\ndiscreditable to _you_ to let them go alone.”\r\n\r\n“My uncle is to send a servant for us.”\r\n\r\n“Oh! Your uncle! He keeps a man-servant, does he? I am very glad you\r\nhave somebody who thinks of those things. Where shall you change horses?\r\nOh, Bromley, of course. If you mention my name at the Bell, you will be\r\nattended to.”\r\n\r\nLady Catherine had many other questions to ask respecting their journey;\r\nand as she did not answer them all herself attention was\r\nnecessary--which Elizabeth believed to be lucky for her; or, with a\r\nmind so occupied, she might have forgotten where she was. Reflection\r\nmust be reserved for solitary hours: whenever she was alone, she gave\r\nway to it as the greatest relief; and not a day went by without a\r\nsolitary walk, in which she might indulge in all the delight of\r\nunpleasant recollections.\r\n\r\nMr. Darcy’s letter she was in a fair way of soon knowing by heart. She\r\nstudied every sentence; and her feelings towards its writer were at\r\ntimes widely different. When she remembered the style of his address,\r\nshe was still full of indignation: but when she considered how unjustly\r\nshe had condemned and upbraided him, her anger was turned against\r\nherself; and his disappointed feelings became the object of compassion.\r\nHis attachment excited gratitude, his general character respect: but she\r\ncould not approve him; nor could she for a moment repent her refusal, or\r\nfeel the slightest inclination ever to see him again. In her own past\r\nbehaviour, there was a constant source of vexation and regret: and in\r\nthe unhappy defects of her family, a subject of yet heavier chagrin.\r\nThey were hopeless of remedy. Her father, contented with laughing at\r\nthem, would never exert himself to restrain the wild giddiness of his\r\nyoungest daughters; and her mother, with manners so far from right\r\nherself, was entirely insensible of the evil. Elizabeth had frequently\r\nunited with Jane in an endeavour to check the imprudence of Catherine\r\nand Lydia; but while they were supported by their mother’s indulgence,\r\nwhat chance could there be of improvement? Catherine, weak-spirited,\r\nirritable, and completely under Lydia’s guidance, had been always\r\naffronted by their advice; and Lydia, self-willed and careless, would\r\nscarcely give them a hearing. They were ignorant, idle, and vain. While\r\nthere was an officer in Meryton, they would flirt with him; and while\r\nMeryton was within a walk of Longbourn, they would be going there for\r\never.\r\n\r\nAnxiety on Jane’s behalf was another prevailing concern; and Mr. Darcy’s\r\nexplanation, by restoring Bingley to all her former good opinion,\r\nheightened the sense of what Jane had lost. His affection was proved to\r\nhave been sincere, and his conduct cleared of all blame, unless any\r\ncould attach to the implicitness of his confidence in his friend. How\r\ngrievous then was the thought that, of a situation so desirable in every\r\nrespect, so replete with advantage, so promising for happiness, Jane had\r\nbeen deprived, by the folly and indecorum of her own family!\r\n\r\nWhen to these recollections was added the development of Wickham’s\r\ncharacter, it may be easily believed that the happy spirits which had\r\nseldom been depressed before were now so much affected as to make it\r\nalmost impossible for her to appear tolerably cheerful.\r\n\r\nTheir engagements at Rosings were as frequent during the last week of\r\nher stay as they had been at first. The very last evening was spent\r\nthere; and her Ladyship again inquired minutely into the particulars of\r\ntheir journey, gave them directions as to the best method of packing,\r\nand was so urgent on the necessity of placing gowns in the only right\r\nway, that Maria thought herself obliged, on her return, to undo all the\r\nwork of the morning, and pack her trunk afresh.\r\n\r\nWhen they parted, Lady Catherine, with great condescension, wished them\r\na good journey, and invited them to come to Hunsford again next year;\r\nand Miss de Bourgh exerted herself so far as to courtesy and hold out\r\nher hand to both.\r\n\r\n\r\n\r\n\r\n[Illustration:\r\n\r\n“The elevation of his feelings.”\r\n]\r\n\r\n\r\n\r\n\r\nCHAPTER XXXVIII.\r\n\r\n\r\n[Illustration]\r\n\r\nOn Saturday morning Elizabeth and Mr. Collins met for breakfast a few\r\nminutes before the others appeared; and he took the opportunity of\r\npaying the parting civilities which he deemed indispensably necessary.\r\n\r\n“I know not, Miss Elizabeth,” said he, “whether Mrs. Collins has yet\r\nexpressed her sense of your kindness in coming to us; but I am very\r\ncertain you will not leave the house without receiving her thanks for\r\nit. The favour of your company has been much felt, I assure you. We know\r\nhow little there is to tempt anyone to our humble abode. Our plain\r\nmanner of living, our small rooms, and few domestics, and the little we\r\nsee of the world, must make Hunsford extremely dull to a young lady like\r\nyourself; but I hope you will believe us grateful for the condescension,\r\nand that we have done everything in our power to prevent you spending\r\nyour time unpleasantly.”\r\n\r\nElizabeth was eager with her thanks and assurances of happiness. She had\r\nspent six weeks with great enjoyment; and the pleasure of being with\r\nCharlotte, and the kind attention she had received, must make _her_ feel\r\nthe obliged. Mr. Collins was gratified; and with a more smiling\r\nsolemnity replied,--\r\n\r\n“It gives me the greatest pleasure to hear that you have passed your\r\ntime not disagreeably. We have certainly done our best; and most\r\nfortunately having it in our power to introduce you to very superior\r\nsociety, and from our connection with Rosings, the frequent means of\r\nvarying the humble home scene, I think we may flatter ourselves that\r\nyour Hunsford visit cannot have been entirely irksome. Our situation\r\nwith regard to Lady Catherine’s family is, indeed, the sort of\r\nextraordinary advantage and blessing which few can boast. You see on\r\nwhat a footing we are. You see how continually we are engaged there. In\r\ntruth, I must acknowledge, that, with all the disadvantages of this\r\nhumble parsonage, I should not think anyone abiding in it an object of\r\ncompassion, while they are sharers of our intimacy at Rosings.”\r\n\r\nWords were insufficient for the elevation of his feelings; and he was\r\nobliged to walk about the room, while Elizabeth tried to unite civility\r\nand truth in a few short sentences.\r\n\r\n“You may, in fact, carry a very favourable report of us into\r\nHertfordshire, my dear cousin. I flatter myself, at least, that you will\r\nbe able to do so. Lady Catherine’s great attentions to Mrs. Collins you\r\nhave been a daily witness of; and altogether I trust it does not appear\r\nthat your friend has drawn an unfortunate--but on this point it will be\r\nas well to be silent. Only let me assure you, my dear Miss Elizabeth,\r\nthat I can from my heart most cordially wish you equal felicity in\r\nmarriage. My dear Charlotte and I have but one mind and one way of\r\nthinking. There is in everything a most remarkable resemblance of\r\ncharacter and ideas between us. We seem to have been designed for each\r\nother.”\r\n\r\nElizabeth could safely say that it was a great happiness where that was\r\nthe case, and with equal sincerity could add, that she firmly believed\r\nand rejoiced in his domestic comforts. She was not sorry, however, to\r\nhave the recital of them interrupted by the entrance of the lady from\r\nwhom they sprang. Poor Charlotte! it was melancholy to leave her to such\r\nsociety! But she had chosen it with her eyes open; and though evidently\r\nregretting that her visitors were to go, she did not seem to ask for\r\ncompassion. Her home and her housekeeping, her parish and her poultry,\r\nand all their dependent concerns, had not yet lost their charms.\r\n\r\nAt length the chaise arrived, the trunks were fastened on, the parcels\r\nplaced within, and it was pronounced to be ready. After an affectionate\r\nparting between the friends, Elizabeth was attended to the carriage by\r\nMr. Collins; and as they walked down the garden, he was commissioning\r\nher with his best respects to all her family, not forgetting his thanks\r\nfor the kindness he had received at Longbourn in the winter, and his\r\ncompliments to Mr. and Mrs. Gardiner, though unknown. He then handed\r\nher in, Maria followed, and the door was on the point of being closed,\r\nwhen he suddenly reminded them, with some consternation, that they had\r\nhitherto forgotten to leave any message for the ladies of Rosings.\r\n\r\n[Illustration:\r\n\r\n“They had forgotten to leave any message”\r\n]\r\n\r\n“But,” he added, “you will of course wish to have your humble respects\r\ndelivered to them, with your grateful thanks for their kindness to you\r\nwhile you have been here.”\r\n\r\nElizabeth made no objection: the door was then allowed to be shut, and\r\nthe carriage drove off.\r\n\r\n“Good gracious!” cried Maria, after a few minutes’ silence, “it seems\r\nbut a day or two since we first came! and yet how many things have\r\nhappened!”\r\n\r\n“A great many indeed,” said her companion, with a sigh.\r\n\r\n“We have dined nine times at Rosings, besides drinking tea there twice!\r\nHow much I shall have to tell!”\r\n\r\nElizabeth privately added, “And how much I shall have to conceal!”\r\n\r\nTheir journey was performed without much conversation, or any alarm; and\r\nwithin four hours of their leaving Hunsford they reached Mr. Gardiner’s\r\nhouse, where they were to remain a few days.\r\n\r\nJane looked well, and Elizabeth had little opportunity of studying her\r\nspirits, amidst the various engagements which the kindness of her aunt\r\nhad reserved for them. But Jane was to go home with her, and at\r\nLongbourn there would be leisure enough for observation.\r\n\r\nIt was not without an effort, meanwhile, that she could wait even for\r\nLongbourn, before she told her sister of Mr. Darcy’s proposals. To know\r\nthat she had the power of revealing what would so exceedingly astonish\r\nJane, and must, at the same time, so highly gratify whatever of her own\r\nvanity she had not yet been able to reason away, was such a temptation\r\nto openness as nothing could have conquered, but the state of indecision\r\nin which she remained as to the extent of what she should communicate,\r\nand her fear, if she once entered on the subject, of being hurried into\r\nrepeating something of Bingley, which might only grieve her sister\r\nfurther.\r\n\r\n\r\n\r\n\r\n[Illustration:\r\n\r\n “How nicely we are crammed in”\r\n]\r\n\r\n\r\n\r\n\r\nCHAPTER XXXIX.\r\n\r\n\r\n[Illustration]\r\n\r\nIt was the second week in May, in which the three young ladies set out\r\ntogether from Gracechurch Street for the town of ----, in Hertfordshire;\r\nand, as they drew near the appointed inn where Mr. Bennet’s carriage was\r\nto meet them, they quickly perceived, in token of the coachman’s\r\npunctuality, both Kitty and Lydia looking out of a dining-room upstairs.\r\nThese two girls had been above an hour in the place, happily employed\r\nin visiting an opposite milliner, watching the sentinel on guard, and\r\ndressing a salad and cucumber.\r\n\r\nAfter welcoming their sisters, they triumphantly displayed a table set\r\nout with such cold meat as an inn larder usually affords, exclaiming,\r\n“Is not this nice? is not this an agreeable surprise?”\r\n\r\n“And we mean to treat you all,” added Lydia; “but you must lend us the\r\nmoney, for we have just spent ours at the shop out there.” Then showing\r\nher purchases,--“Look here, I have bought this bonnet. I do not think it\r\nis very pretty; but I thought I might as well buy it as not. I shall\r\npull it to pieces as soon as I get home, and see if I can make it up any\r\nbetter.”\r\n\r\nAnd when her sisters abused it as ugly, she added, with perfect\r\nunconcern, “Oh, but there were two or three much uglier in the shop; and\r\nwhen I have bought some prettier-coloured satin to trim it with fresh, I\r\nthink it will be very tolerable. Besides, it will not much signify what\r\none wears this summer, after the ----shire have left Meryton, and they\r\nare going in a fortnight.”\r\n\r\n“Are they, indeed?” cried Elizabeth, with the greatest satisfaction.\r\n\r\n“They are going to be encamped near Brighton; and I do so want papa to\r\ntake us all there for the summer! It would be such a delicious scheme,\r\nand I dare say would hardly cost anything at all. Mamma would like to\r\ngo, too, of all things! Only think what a miserable summer else we shall\r\nhave!”\r\n\r\n“Yes,” thought Elizabeth; “_that_ would be a delightful scheme, indeed,\r\nand completely do for us at once. Good Heaven! Brighton and a whole\r\ncampful of soldiers, to us, who have been overset already by one poor\r\nregiment of militia, and the monthly balls of Meryton!”\r\n\r\n“Now I have got some news for you,” said Lydia, as they sat down to\r\ntable. “What do you think? It is excellent news, capital news, and about\r\na certain person that we all like.”\r\n\r\nJane and Elizabeth looked at each other, and the waiter was told that he\r\nneed not stay. Lydia laughed, and said,--\r\n\r\n“Ay, that is just like your formality and discretion. You thought the\r\nwaiter must not hear, as if he cared! I dare say he often hears worse\r\nthings said than I am going to say. But he is an ugly fellow! I am glad\r\nhe is gone. I never saw such a long chin in my life. Well, but now for\r\nmy news: it is about dear Wickham; too good for the waiter, is not it?\r\nThere is no danger of Wickham’s marrying Mary King--there’s for you! She\r\nis gone down to her uncle at Liverpool; gone to stay. Wickham is safe.”\r\n\r\n“And Mary King is safe!” added Elizabeth; “safe from a connection\r\nimprudent as to fortune.”\r\n\r\n“She is a great fool for going away, if she liked him.”\r\n\r\n“But I hope there is no strong attachment on either side,” said Jane.\r\n\r\n“I am sure there is not on _his_. I will answer for it, he never cared\r\nthree straws about her. Who _could_ about such a nasty little freckled\r\nthing?”\r\n\r\nElizabeth was shocked to think that, however incapable of such\r\ncoarseness of _expression_ herself, the coarseness of the _sentiment_\r\nwas little other than her own breast had formerly harboured and fancied\r\nliberal!\r\n\r\nAs soon as all had ate, and the elder ones paid, the carriage was\r\nordered; and, after some contrivance, the whole party, with all their\r\nboxes, workbags, and parcels, and the unwelcome addition of Kitty’s and\r\nLydia’s purchases, were seated in it.\r\n\r\n“How nicely we are crammed in!” cried Lydia. “I am glad I brought my\r\nbonnet, if it is only for the fun of having another band-box! Well, now\r\nlet us be quite comfortable and snug, and talk and laugh all the way\r\nhome. And in the first place, let us hear what has happened to you all\r\nsince you went away. Have you seen any pleasant men? Have you had any\r\nflirting? I was in great hopes that one of you would have got a husband\r\nbefore you came back. Jane will be quite an old maid soon, I declare.\r\nShe is almost three-and-twenty! Lord! how ashamed I should be of not\r\nbeing married before three-and-twenty! My aunt Philips wants you so to\r\nget husbands you can’t think. She says Lizzy had better have taken Mr.\r\nCollins; but _I_ do not think there would have been any fun in it. Lord!\r\nhow I should like to be married before any of you! and then I would\r\n_chaperon_ you about to all the balls. Dear me! we had such a good piece\r\nof fun the other day at Colonel Forster’s! Kitty and me were to spend\r\nthe day there, and Mrs. Forster promised to have a little dance in the\r\nevening; (by-the-bye, Mrs. Forster and me are _such_ friends!) and so\r\nshe asked the two Harringtons to come: but Harriet was ill, and so Pen\r\nwas forced to come by herself; and then, what do you think we did? We\r\ndressed up Chamberlayne in woman’s clothes, on purpose to pass for a\r\nlady,--only think what fun! Not a soul knew of it, but Colonel and Mrs.\r\nForster, and Kitty and me, except my aunt, for we were forced to borrow\r\none of her gowns; and you cannot imagine how well he looked! When Denny,\r\nand Wickham, and Pratt, and two or three more of the men came in, they\r\ndid not know him in the least. Lord! how I laughed! and so did Mrs.\r\nForster. I thought I should have died. And _that_ made the men suspect\r\nsomething, and then they soon found out what was the matter.”\r\n\r\nWith such kind of histories of their parties and good jokes did Lydia,\r\nassisted by Kitty’s hints and additions, endeavour to amuse her\r\ncompanions all the way to Longbourn. Elizabeth listened as little as she\r\ncould, but there was no escaping the frequent mention of Wickham’s name.\r\n\r\nTheir reception at home was most kind. Mrs. Bennet rejoiced to see Jane\r\nin undiminished beauty; and more than once during dinner did Mr. Bennet\r\nsay voluntarily to Elizabeth,----\r\n\r\n“I am glad you are come back, Lizzy.”\r\n\r\nTheir party in the dining-room was large, for almost all the Lucases\r\ncame to meet Maria and hear the news; and various were the subjects\r\nwhich occupied them: Lady Lucas was inquiring of Maria, across the\r\ntable, after the welfare and poultry of her eldest daughter; Mrs. Bennet\r\nwas doubly engaged, on one hand collecting an account of the present\r\nfashions from Jane, who sat some way below her, and on the other,\r\nretailing them all to the younger Miss Lucases; and Lydia, in a voice\r\nrather louder than any other person’s, was enumerating the various\r\npleasures of the morning to anybody who would hear her.\r\n\r\n“Oh, Mary,” said she, “I wish you had gone with us, for we had such fun!\r\nas we went along Kitty and me drew up all the blinds, and pretended\r\nthere was nobody in the coach; and I should have gone so all the way, if\r\nKitty had not been sick; and when we got to the George, I do think we\r\nbehaved very handsomely, for we treated the other three with the nicest\r\ncold luncheon in the world, and if you would have gone, we would have\r\ntreated you too. And then when we came away it was such fun! I thought\r\nwe never should have got into the coach. I was ready to die of laughter.\r\nAnd then we were so merry all the way home! we talked and laughed so\r\nloud, that anybody might have heard us ten miles off!”\r\n\r\nTo this, Mary very gravely replied, “Far be it from me, my dear sister,\r\nto depreciate such pleasures. They would doubtless be congenial with the\r\ngenerality of female minds. But I confess they would have no charms for\r\n_me_. I should infinitely prefer a book.”\r\n\r\nBut of this answer Lydia heard not a word. She seldom listened to\r\nanybody for more than half a minute, and never attended to Mary at all.\r\n\r\nIn the afternoon Lydia was urgent with the rest of the girls to walk to\r\nMeryton, and see how everybody went on; but Elizabeth steadily opposed\r\nthe scheme. It should not be said, that the Miss Bennets could not be at\r\nhome half a day before they were in pursuit of the officers. There was\r\nanother reason, too, for her opposition. She dreaded seeing Wickham\r\nagain, and was resolved to avoid it as long as possible. The comfort to\r\n_her_, of the regiment’s approaching removal, was indeed beyond\r\nexpression. In a fortnight they were to go, and once gone, she hoped\r\nthere could be nothing more to plague her on his account.\r\n\r\nShe had not been many hours at home, before she found that the Brighton\r\nscheme, of which Lydia had given them a hint at the inn, was under\r\nfrequent discussion between her parents. Elizabeth saw directly that her\r\nfather had not the smallest intention of yielding; but his answers were\r\nat the same time so vague and equivocal, that her mother, though often\r\ndisheartened, had never yet despaired of succeeding at last.\r\n\r\n\r\n\r\n\r\n[Illustration]\r\n\r\n\r\n\r\n\r\nCHAPTER XL.\r\n\r\n\r\n[Illustration]\r\n\r\nElizabeth’s impatience to acquaint Jane with what had happened could no\r\nlonger be overcome; and at length resolving to suppress every particular\r\nin which her sister was concerned, and preparing her to be surprised,\r\nshe related to her the next morning the chief of the scene between Mr.\r\nDarcy and herself.\r\n\r\nMiss Bennet’s astonishment was soon lessened by the strong sisterly\r\npartiality which made any admiration of Elizabeth appear perfectly\r\nnatural; and all surprise was shortly lost in other feelings. She was\r\nsorry that Mr. Darcy should have delivered his sentiments in a manner so\r\nlittle suited to recommend them; but still more was she grieved for the\r\nunhappiness which her sister’s refusal must have given him.\r\n\r\n“His being so sure of succeeding was wrong,” said she, “and certainly\r\nought not to have appeared; but consider how much it must increase his\r\ndisappointment.”\r\n\r\n“Indeed,” replied Elizabeth, “I am heartily sorry for him; but he has\r\nother feelings which will probably soon drive away his regard for me.\r\nYou do not blame me, however, for refusing him?”\r\n\r\n“Blame you! Oh, no.”\r\n\r\n“But you blame me for having spoken so warmly of Wickham?”\r\n\r\n“No--I do not know that you were wrong in saying what you did.”\r\n\r\n“But you _will_ know it, when I have told you what happened the very\r\nnext day.”\r\n\r\nShe then spoke of the letter, repeating the whole of its contents as far\r\nas they concerned George Wickham. What a stroke was this for poor Jane,\r\nwho would willingly have gone through the world without believing that\r\nso much wickedness existed in the whole race of mankind as was here\r\ncollected in one individual! Nor was Darcy’s vindication, though\r\ngrateful to her feelings, capable of consoling her for such discovery.\r\nMost earnestly did she labour to prove the probability of error, and\r\nseek to clear one, without involving the other.\r\n\r\n“This will not do,” said Elizabeth; “you never will be able to make both\r\nof them good for anything. Take your choice, but you must be satisfied\r\nwith only one. There is but such a quantity of merit between them; just\r\nenough to make one good sort of man; and of late it has been shifting\r\nabout pretty much. For my part, I am inclined to believe it all Mr.\r\nDarcy’s, but you shall do as you choose.”\r\n\r\nIt was some time, however, before a smile could be extorted from Jane.\r\n\r\n“I do not know when I have been more shocked,” said she. “Wickham so\r\nvery bad! It is almost past belief. And poor Mr. Darcy! dear Lizzy,\r\nonly consider what he must have suffered. Such a disappointment! and\r\nwith the knowledge of your ill opinion too! and having to relate such a\r\nthing of his sister! It is really too distressing, I am sure you must\r\nfeel it so.”\r\n\r\n“Oh no, my regret and compassion are all done away by seeing you so full\r\nof both. I know you will do him such ample justice, that I am growing\r\nevery moment more unconcerned and indifferent. Your profusion makes me\r\nsaving; and if you lament over him much longer, my heart will be as\r\nlight as a feather.”\r\n\r\n“Poor Wickham! there is such an expression of goodness in his\r\ncountenance! such an openness and gentleness in his manner.”\r\n\r\n“There certainly was some great mismanagement in the education of those\r\ntwo young men. One has got all the goodness, and the other all the\r\nappearance of it.”\r\n\r\n“I never thought Mr. Darcy so deficient in the _appearance_ of it as you\r\nused to do.”\r\n\r\n“And yet I meant to be uncommonly clever in taking so decided a dislike\r\nto him, without any reason. It is such a spur to one’s genius, such an\r\nopening for wit, to have a dislike of that kind. One may be continually\r\nabusive without saying anything just; but one cannot be always laughing\r\nat a man without now and then stumbling on something witty.”\r\n\r\n“Lizzy, when you first read that letter, I am sure you could not treat\r\nthe matter as you do now.”\r\n\r\n“Indeed, I could not. I was uncomfortable enough, I was very\r\nuncomfortable--I may say unhappy. And with no one to speak to of what I\r\nfelt, no Jane to comfort me, and say that I had not been so very weak,\r\nand vain, and nonsensical, as I knew I had! Oh, how I wanted you!”\r\n\r\n“How unfortunate that you should have used such very strong expressions\r\nin speaking of Wickham to Mr. Darcy, for now they _do_ appear wholly\r\nundeserved.”\r\n\r\n“Certainly. But the misfortune of speaking with bitterness is a most\r\nnatural consequence of the prejudices I had been encouraging. There is\r\none point on which I want your advice. I want to be told whether I\r\nought, or ought not, to make our acquaintance in general understand\r\nWickham’s character.”\r\n\r\nMiss Bennet paused a little, and then replied, “Surely there can be no\r\noccasion for exposing him so dreadfully. What is your own opinion?”\r\n\r\n“That it ought not to be attempted. Mr. Darcy has not authorized me to\r\nmake his communication public. On the contrary, every particular\r\nrelative to his sister was meant to be kept as much as possible to\r\nmyself; and if I endeavour to undeceive people as to the rest of his\r\nconduct, who will believe me? The general prejudice against Mr. Darcy is\r\nso violent, that it would be the death of half the good people in\r\nMeryton, to attempt to place him in an amiable light. I am not equal to\r\nit. Wickham will soon be gone; and, therefore, it will not signify to\r\nanybody here what he really is. Some time hence it will be all found\r\nout, and then we may laugh at their stupidity in not knowing it before.\r\nAt present I will say nothing about it.”\r\n\r\n“You are quite right. To have his errors made public might ruin him for\r\never. He is now, perhaps, sorry for what he has done, and anxious to\r\nre-establish a character. We must not make him desperate.”\r\n\r\nThe tumult of Elizabeth’s mind was allayed by this conversation. She\r\nhad got rid of two of the secrets which had weighed on her for a\r\nfortnight, and was certain of a willing listener in Jane, whenever she\r\nmight wish to talk again of either. But there was still something\r\nlurking behind, of which prudence forbade the disclosure. She dared not\r\nrelate the other half of Mr. Darcy’s letter, nor explain to her sister\r\nhow sincerely she had been valued by his friend. Here was knowledge in\r\nwhich no one could partake; and she was sensible that nothing less than\r\na perfect understanding between the parties could justify her in\r\nthrowing off this last encumbrance of mystery. “And then,” said she, “if\r\nthat very improbable event should ever take place, I shall merely be\r\nable to tell what Bingley may tell in a much more agreeable manner\r\nhimself. The liberty of communication cannot be mine till it has lost\r\nall its value!”\r\n\r\nShe was now, on being settled at home, at leisure to observe the real\r\nstate of her sister’s spirits. Jane was not happy. She still cherished a\r\nvery tender affection for Bingley. Having never even fancied herself in\r\nlove before, her regard had all the warmth of first attachment, and from\r\nher age and disposition, greater steadiness than first attachments often\r\nboast; and so fervently did she value his remembrance, and prefer him to\r\nevery other man, that all her good sense, and all her attention to the\r\nfeelings of her friends, were requisite to check the indulgence of those\r\nregrets which must have been injurious to her own health and their\r\ntranquillity.\r\n\r\n“Well, Lizzy,” said Mrs. Bennet, one day, “what is your opinion _now_ of\r\nthis sad business of Jane’s? For my part, I am determined never to speak\r\nof it again to anybody. I told my sister Philips so the other day. But I\r\ncannot find out that Jane saw anything of him in London. Well, he is a\r\nvery undeserving young man--and I do not suppose there is the least\r\nchance in the world of her ever getting him now. There is no talk of his\r\ncoming to Netherfield again in the summer; and I have inquired of\r\neverybody, too, who is likely to know.”\r\n\r\n[Illustration:\r\n\r\n “I am determined never to speak of it again”\r\n]\r\n\r\n“I do not believe that he will ever live at Netherfield any more.”\r\n\r\n“Oh, well! it is just as he chooses. Nobody wants him to come; though I\r\nshall always say that he used my daughter extremely ill; and, if I was\r\nher, I would not have put up with it. Well, my comfort is, I am sure\r\nJane will die of a broken heart, and then he will be sorry for what he\r\nhas done.”\r\n\r\nBut as Elizabeth could not receive comfort from any such expectation she\r\nmade no answer.\r\n\r\n“Well, Lizzy,” continued her mother, soon afterwards, “and so the\r\nCollinses live very comfortable, do they? Well, well, I only hope it\r\nwill last. And what sort of table do they keep? Charlotte is an\r\nexcellent manager, I dare say. If she is half as sharp as her mother,\r\nshe is saving enough. There is nothing extravagant in _their_\r\nhousekeeping, I dare say.”\r\n\r\n“No, nothing at all.”\r\n\r\n“A great deal of good management, depend upon it. Yes, yes. _They_ will\r\ntake care not to outrun their income. _They_ will never be distressed\r\nfor money. Well, much good may it do them! And so, I suppose, they often\r\ntalk of having Longbourn when your father is dead. They look upon it\r\nquite as their own, I dare say, whenever that happens.”\r\n\r\n“It was a subject which they could not mention before me.”\r\n\r\n“No; it would have been strange if they had. But I make no doubt they\r\noften talk of it between themselves. Well, if they can be easy with an\r\nestate that is not lawfully their own, so much the better. _I_ should be\r\nashamed of having one that was only entailed on me.”\r\n\r\n\r\n\r\n\r\n[Illustration:\r\n\r\n“When Colonel Miller’s regiment went away”\r\n\r\n[_Copyright 1894 by George Allen._]]\r\n\r\n\r\n\r\n\r\nCHAPTER XLI.\r\n\r\n\r\n[Illustration]\r\n\r\nThe first week of their return was soon gone. The second began. It was\r\nthe last of the regiment’s stay in Meryton, and all the young ladies in\r\nthe neighbourhood were drooping apace. The dejection was almost\r\nuniversal. The elder Miss Bennets alone were still able to eat, drink,\r\nand sleep, and pursue the usual course of their employments. Very\r\nfrequently were they reproached for this insensibility by Kitty and\r\nLydia, whose own misery was extreme, and who could not comprehend such\r\nhard-heartedness in any of the family.\r\n\r\n“Good Heaven! What is to become of us? What are we to do?” would they\r\noften exclaim in the bitterness of woe. “How can you be smiling so,\r\nLizzy?”\r\n\r\nTheir affectionate mother shared all their grief; she remembered what\r\nshe had herself endured on a similar occasion five-and-twenty years ago.\r\n\r\n“I am sure,” said she, “I cried for two days together when Colonel\r\nMiller’s regiment went away. I thought I should have broke my heart.”\r\n\r\n“I am sure I shall break _mine_,” said Lydia.\r\n\r\n“If one could but go to Brighton!” observed Mrs. Bennet.\r\n\r\n“Oh yes!--if one could but go to Brighton! But papa is so disagreeable.”\r\n\r\n“A little sea-bathing would set me up for ever.”\r\n\r\n“And my aunt Philips is sure it would do _me_ a great deal of good,”\r\nadded Kitty.\r\n\r\nSuch were the kind of lamentations resounding perpetually through\r\nLongbourn House. Elizabeth tried to be diverted by them; but all sense\r\nof pleasure was lost in shame. She felt anew the justice of Mr. Darcy’s\r\nobjections; and never had she before been so much disposed to pardon his\r\ninterference in the views of his friend.\r\n\r\nBut the gloom of Lydia’s prospect was shortly cleared away; for she\r\nreceived an invitation from Mrs. Forster, the wife of the colonel of the\r\nregiment, to accompany her to Brighton. This invaluable friend was a\r\nvery young woman, and very lately married. A resemblance in good-humour\r\nand good spirits had recommended her and Lydia to each other, and out of\r\ntheir _three_ months’ acquaintance they had been intimate _two_.\r\n\r\nThe rapture of Lydia on this occasion, her adoration of Mrs. Forster,\r\nthe delight of Mrs. Bennet, and the mortification of Kitty, are scarcely\r\nto be described. Wholly inattentive to her sister’s feelings, Lydia flew\r\nabout the house in restless ecstasy, calling for everyone’s\r\ncongratulations, and laughing and talking with more violence than ever;\r\nwhilst the luckless Kitty continued in the parlour repining at her fate\r\nin terms as unreasonable as her accent was peevish.\r\n\r\n“I cannot see why Mrs. Forster should not ask _me_ as well as Lydia,”\r\nsaid she, “though I am _not_ her particular friend. I have just as much\r\nright to be asked as she has, and more too, for I am two years older.”\r\n\r\nIn vain did Elizabeth attempt to make her reasonable, and Jane to make\r\nher resigned. As for Elizabeth herself, this invitation was so far from\r\nexciting in her the same feelings as in her mother and Lydia, that she\r\nconsidered it as the death-warrant of all possibility of common sense\r\nfor the latter; and detestable as such a step must make her, were it\r\nknown, she could not help secretly advising her father not to let her\r\ngo. She represented to him all the improprieties of Lydia’s general\r\nbehaviour, the little advantage she could derive from the friendship of\r\nsuch a woman as Mrs. Forster, and the probability of her being yet more\r\nimprudent with such a companion at Brighton, where the temptations must\r\nbe greater than at home. He heard her attentively, and then said,--\r\n\r\n“Lydia will never be easy till she has exposed herself in some public\r\nplace or other, and we can never expect her to do it with so little\r\nexpense or inconvenience to her family as under the present\r\ncircumstances.”\r\n\r\n“If you were aware,” said Elizabeth, “of the very great disadvantage to\r\nus all, which must arise from the public notice of Lydia’s unguarded and\r\nimprudent manner, nay, which has already arisen from it, I am sure you\r\nwould judge differently in the affair.”\r\n\r\n“Already arisen!” repeated Mr. Bennet. “What! has she frightened away\r\nsome of your lovers? Poor little Lizzy! But do not be cast down. Such\r\nsqueamish youths as cannot bear to be connected with a little absurdity\r\nare not worth a regret. Come, let me see the list of the pitiful fellows\r\nwho have been kept aloof by Lydia’s folly.”\r\n\r\n“Indeed, you are mistaken. I have no such injuries to resent. It is not\r\nof peculiar, but of general evils, which I am now complaining. Our\r\nimportance, our respectability in the world, must be affected by the\r\nwild volatility, the assurance and disdain of all restraint which mark\r\nLydia’s character. Excuse me,--for I must speak plainly. If you, my dear\r\nfather, will not take the trouble of checking her exuberant spirits, and\r\nof teaching her that her present pursuits are not to be the business of\r\nher life, she will soon be beyond the reach of amendment. Her character\r\nwill be fixed; and she will, at sixteen, be the most determined flirt\r\nthat ever made herself and her family ridiculous;--a flirt, too, in the\r\nworst and meanest degree of flirtation; without any attraction beyond\r\nyouth and a tolerable person; and, from the ignorance and emptiness of\r\nher mind, wholly unable to ward off any portion of that universal\r\ncontempt which her rage for admiration will excite. In this danger Kitty\r\nis also comprehended. She will follow wherever Lydia leads. Vain,\r\nignorant, idle, and absolutely uncontrolled! Oh, my dear father, can you\r\nsuppose it possible that they will not be censured and despised wherever\r\nthey are known, and that their sisters will not be often involved in the\r\ndisgrace?”\r\n\r\nMr. Bennet saw that her whole heart was in the subject; and,\r\naffectionately taking her hand, said, in reply,--\r\n\r\n“Do not make yourself uneasy, my love. Wherever you and Jane are known,\r\nyou must be respected and valued; and you will not appear to less\r\nadvantage for having a couple of--or I may say, three--very silly\r\nsisters. We shall have no peace at Longbourn if Lydia does not go to\r\nBrighton. Let her go, then. Colonel Forster is a sensible man, and will\r\nkeep her out of any real mischief; and she is luckily too poor to be an\r\nobject of prey to anybody. At Brighton she will be of less importance\r\neven as a common flirt than she has been here. The officers will find\r\nwomen better worth their notice. Let us hope, therefore, that her being\r\nthere may teach her her own insignificance. At any rate, she cannot grow\r\nmany degrees worse, without authorizing us to lock her up for the rest\r\nof her life.”\r\n\r\nWith this answer Elizabeth was forced to be content; but her own opinion\r\ncontinued the same, and she left him disappointed and sorry. It was not\r\nin her nature, however, to increase her vexations by dwelling on them.\r\nShe was confident of having performed her duty; and to fret over\r\nunavoidable evils, or augment them by anxiety, was no part of her\r\ndisposition.\r\n\r\nHad Lydia and her mother known the substance of her conference with her\r\nfather, their indignation would hardly have found expression in their\r\nunited volubility. In Lydia’s imagination, a visit to Brighton comprised\r\nevery possibility of earthly happiness. She saw, with the creative eye\r\nof fancy, the streets of that gay bathing-place covered with officers.\r\nShe saw herself the object of attention to tens and to scores of them at\r\npresent unknown. She saw all the glories of the camp: its tents\r\nstretched forth in beauteous uniformity of lines, crowded with the young\r\nand the gay, and dazzling with scarlet; and, to complete the view, she\r\nsaw herself seated beneath a tent, tenderly flirting with at least six\r\nofficers at once.\r\n\r\n[Illustration:\r\n\r\n“Tenderly flirting”\r\n\r\n[_Copyright 1894 by George Allen._]]\r\n\r\nHad she known that her sister sought to tear her from such prospects and\r\nsuch realities as these, what would have been her sensations? They could\r\nhave been understood only by her mother, who might have felt nearly the\r\nsame. Lydia’s going to Brighton was all that consoled her for the\r\nmelancholy conviction of her husband’s never intending to go there\r\nhimself.\r\n\r\nBut they were entirely ignorant of what had passed; and their raptures\r\ncontinued, with little intermission, to the very day of Lydia’s leaving\r\nhome.\r\n\r\nElizabeth was now to see Mr. Wickham for the last time. Having been\r\nfrequently in company with him since her return, agitation was pretty\r\nwell over; the agitations of former partiality entirely so. She had even\r\nlearnt to detect, in the very gentleness which had first delighted her,\r\nan affectation and a sameness to disgust and weary. In his present\r\nbehaviour to herself, moreover, she had a fresh source of displeasure;\r\nfor the inclination he soon testified of renewing those attentions which\r\nhad marked the early part of their acquaintance could only serve, after\r\nwhat had since passed, to provoke her. She lost all concern for him in\r\nfinding herself thus selected as the object of such idle and frivolous\r\ngallantry; and while she steadily repressed it, could not but feel the\r\nreproof contained in his believing, that however long, and for whatever\r\ncause, his attentions had been withdrawn, her vanity would be gratified,\r\nand her preference secured, at any time, by their renewal.\r\n\r\nOn the very last day of the regiment’s remaining in Meryton, he dined,\r\nwith others of the officers, at Longbourn; and so little was Elizabeth\r\ndisposed to part from him in good-humour, that, on his making some\r\ninquiry as to the manner in which her time had passed at Hunsford, she\r\nmentioned Colonel Fitzwilliam’s and Mr. Darcy’s having both spent three\r\nweeks at Rosings, and asked him if he were acquainted with the former.\r\n\r\nHe looked surprised, displeased, alarmed; but, with a moment’s\r\nrecollection, and a returning smile, replied, that he had formerly seen\r\nhim often; and, after observing that he was a very gentlemanlike man,\r\nasked her how she had liked him. Her answer was warmly in his favour.\r\nWith an air of indifference, he soon afterwards added, “How long did you\r\nsay that he was at Rosings?”\r\n\r\n“Nearly three weeks.”\r\n\r\n“And you saw him frequently?”\r\n\r\n“Yes, almost every day.”\r\n\r\n“His manners are very different from his cousin’s.”\r\n\r\n“Yes, very different; but I think Mr. Darcy improves on acquaintance.”\r\n\r\n“Indeed!” cried Wickham, with a look which did not escape her. “And pray\r\nmay I ask--” but checking himself, he added, in a gayer tone, “Is it in\r\naddress that he improves? Has he deigned to add aught of civility to his\r\nordinary style? for I dare not hope,” he continued, in a lower and more\r\nserious tone, “that he is improved in essentials.”\r\n\r\n“Oh, no!” said Elizabeth. “In essentials, I believe, he is very much\r\nwhat he ever was.”\r\n\r\nWhile she spoke, Wickham looked as if scarcely knowing whether to\r\nrejoice over her words or to distrust their meaning. There was a\r\nsomething in her countenance which made him listen with an apprehensive\r\nand anxious attention, while she added,--\r\n\r\n“When I said that he improved on acquaintance, I did not mean that\r\neither his mind or manners were in a state of improvement; but that,\r\nfrom knowing him better, his disposition was better understood.”\r\n\r\nWickham’s alarm now appeared in a heightened complexion and agitated\r\nlook; for a few minutes he was silent; till, shaking off his\r\nembarrassment, he turned to her again, and said in the gentlest of\r\naccents,--\r\n\r\n“You, who so well know my feelings towards Mr. Darcy, will readily\r\ncomprehend how sincerely I must rejoice that he is wise enough to assume\r\neven the _appearance_ of what is right. His pride, in that direction,\r\nmay be of service, if not to himself, to many others, for it must deter\r\nhim from such foul misconduct as I have suffered by. I only fear that\r\nthe sort of cautiousness to which you, I imagine, have been alluding, is\r\nmerely adopted on his visits to his aunt, of whose good opinion and\r\njudgment he stands much in awe. His fear of her has always operated, I\r\nknow, when they were together; and a good deal is to be imputed to his\r\nwish of forwarding the match with Miss de Bourgh, which I am certain he\r\nhas very much at heart.”\r\n\r\nElizabeth could not repress a smile at this, but she answered only by a\r\nslight inclination of the head. She saw that he wanted to engage her on\r\nthe old subject of his grievances, and she was in no humour to indulge\r\nhim. The rest of the evening passed with the _appearance_, on his side,\r\nof usual cheerfulness, but with no further attempt to distinguish\r\nElizabeth; and they parted at last with mutual civility, and possibly a\r\nmutual desire of never meeting again.\r\n\r\nWhen the party broke up, Lydia returned with Mrs. Forster to Meryton,\r\nfrom whence they were to set out early the next morning. The separation\r\nbetween her and her family was rather noisy than pathetic. Kitty was the\r\nonly one who shed tears; but she did weep from vexation and envy. Mrs.\r\nBennet was diffuse in her good wishes for the felicity of her daughter,\r\nand impressive in her injunctions that she would not miss the\r\nopportunity of enjoying herself as much as possible,--advice which there\r\nwas every reason to believe would be attended to; and, in the clamorous\r\nhappiness of Lydia herself in bidding farewell, the more gentle adieus\r\nof her sisters were uttered without being heard.\r\n\r\n\r\n\r\n\r\n[Illustration:\r\n\r\nThe arrival of the\r\nGardiners\r\n]\r\n\r\n\r\n\r\n\r\nCHAPTER XLII.\r\n\r\n\r\n[Illustration]\r\n\r\nHad Elizabeth’s opinion been all drawn from her own family, she could\r\nnot have formed a very pleasing picture of conjugal felicity or domestic\r\ncomfort. Her father, captivated by youth and beauty, and that appearance\r\nof good-humour which youth and beauty generally give, had married a\r\nwoman whose weak understanding and illiberal mind had very early in\r\ntheir marriage put an end to all real affection for her. Respect,\r\nesteem, and confidence had vanished for ever; and all his views of\r\ndomestic happiness were overthrown. But Mr. Bennet was not of a\r\ndisposition to seek comfort for the disappointment which his own\r\nimprudence had brought on in any of those pleasures which too often\r\nconsole the unfortunate for their folly or their vice. He was fond of\r\nthe country and of books; and from these tastes had arisen his principal\r\nenjoyments. To his wife he was very little otherwise indebted than as\r\nher ignorance and folly had contributed to his amusement. This is not\r\nthe sort of happiness which a man would in general wish to owe to his\r\nwife; but where other powers of entertainment are wanting, the true\r\nphilosopher will derive benefit from such as are given.\r\n\r\nElizabeth, however, had never been blind to the impropriety of her\r\nfather’s behaviour as a husband. She had always seen it with pain; but\r\nrespecting his abilities, and grateful for his affectionate treatment of\r\nherself, she endeavoured to forget what she could not overlook, and to\r\nbanish from her thoughts that continual breach of conjugal obligation\r\nand decorum which, in exposing his wife to the contempt of her own\r\nchildren, was so highly reprehensible. But she had never felt so\r\nstrongly as now the disadvantages which must attend the children of so\r\nunsuitable a marriage, nor ever been so fully aware of the evils arising\r\nfrom so ill-judged a direction of talents--talents which, rightly used,\r\nmight at least have preserved the respectability of his daughters, even\r\nif incapable of enlarging the mind of his wife.\r\n\r\nWhen Elizabeth had rejoiced over Wickham’s departure, she found little\r\nother cause for satisfaction in the loss of the regiment. Their parties\r\nabroad were less varied than before; and at home she had a mother and\r\nsister, whose constant repinings at the dulness of everything around\r\nthem threw a real gloom over their domestic circle; and, though Kitty\r\nmight in time regain her natural degree of sense, since the disturbers\r\nof her brain were removed, her other sister, from whose disposition\r\ngreater evil might be apprehended, was likely to be hardened in all her\r\nfolly and assurance, by a situation of such double danger as a\r\nwatering-place and a camp. Upon the whole, therefore, she found, what\r\nhas been sometimes found before, that an event to which she had looked\r\nforward with impatient desire, did not, in taking place, bring all the\r\nsatisfaction she had promised herself. It was consequently necessary to\r\nname some other period for the commencement of actual felicity; to have\r\nsome other point on which her wishes and hopes might be fixed, and by\r\nagain enjoying the pleasure of anticipation, console herself for the\r\npresent, and prepare for another disappointment. Her tour to the Lakes\r\nwas now the object of her happiest thoughts: it was her best consolation\r\nfor all the uncomfortable hours which the discontentedness of her mother\r\nand Kitty made inevitable; and could she have included Jane in the\r\nscheme, every part of it would have been perfect.\r\n\r\n“But it is fortunate,” thought she, “that I have something to wish for.\r\nWere the whole arrangement complete, my disappointment would be certain.\r\nBut here, by carrying with me one ceaseless source of regret in my\r\nsister’s absence, I may reasonably hope to have all my expectations of\r\npleasure realized. A scheme of which every part promises delight can\r\nnever be successful; and general disappointment is only warded off by\r\nthe defence of some little peculiar vexation.”\r\n\r\nWhen Lydia went away she promised to write very often and very minutely\r\nto her mother and Kitty; but her letters were always long expected, and\r\nalways very short. Those to her mother contained little else than that\r\nthey were just returned from the library, where such and such officers\r\nhad attended them, and where she had seen such beautiful ornaments as\r\nmade her quite wild; that she had a new gown, or a new parasol, which\r\nshe would have described more fully, but was obliged to leave off in a\r\nviolent hurry, as Mrs. Forster called her, and they were going to the\r\ncamp; and from her correspondence with her sister there was still less\r\nto be learnt, for her letters to Kitty, though rather longer, were much\r\ntoo full of lines under the words to be made public.\r\n\r\nAfter the first fortnight or three weeks of her absence, health,\r\ngood-humour, and cheerfulness began to reappear at Longbourn. Everything\r\nwore a happier aspect. The families who had been in town for the winter\r\ncame back again, and summer finery and summer engagements arose. Mrs.\r\nBennet was restored to her usual querulous serenity; and by the middle\r\nof June Kitty was so much recovered as to be able to enter Meryton\r\nwithout tears,--an event of such happy promise as to make Elizabeth\r\nhope, that by the following Christmas she might be so tolerably\r\nreasonable as not to mention an officer above once a day, unless, by\r\nsome cruel and malicious arrangement at the War Office, another regiment\r\nshould be quartered in Meryton.\r\n\r\nThe time fixed for the beginning of their northern tour was now fast\r\napproaching; and a fortnight only was wanting of it, when a letter\r\narrived from Mrs. Gardiner, which at once delayed its commencement and\r\ncurtailed its extent. Mr. Gardiner would be prevented by business from\r\nsetting out till a fortnight later in July, and must be in London again\r\nwithin a month; and as that left too short a period for them to go so\r\nfar, and see so much as they had proposed, or at least to see it with\r\nthe leisure and comfort they had built on, they were obliged to give up\r\nthe Lakes, and substitute a more contracted tour; and, according to the\r\npresent plan, were to go no farther northward than Derbyshire. In that\r\ncounty there was enough to be seen to occupy the chief of their three\r\nweeks; and to Mrs. Gardiner it had a peculiarly strong attraction. The\r\ntown where she had formerly passed some years of her life, and where\r\nthey were now to spend a few days, was probably as great an object of\r\nher curiosity as all the celebrated beauties of Matlock, Chatsworth,\r\nDovedale, or the Peak.\r\n\r\nElizabeth was excessively disappointed: she had set her heart on seeing\r\nthe Lakes; and still thought there might have been time enough. But it\r\nwas her business to be satisfied--and certainly her temper to be happy;\r\nand all was soon right again.\r\n\r\nWith the mention of Derbyshire, there were many ideas connected. It was\r\nimpossible for her to see the word without thinking of Pemberley and its\r\nowner. “But surely,” said she, “I may enter his county with impunity,\r\nand rob it of a few petrified spars, without his perceiving me.”\r\n\r\nThe period of expectation was now doubled. Four weeks were to pass away\r\nbefore her uncle and aunt’s arrival. But they did pass away, and Mr. and\r\nMrs. Gardiner, with their four children, did at length appear at\r\nLongbourn. The children, two girls of six and eight years old, and two\r\nyounger boys, were to be left under the particular care of their cousin\r\nJane, who was the general favourite, and whose steady sense and\r\nsweetness of temper exactly adapted her for attending to them in every\r\nway--teaching them, playing with them, and loving them.\r\n\r\nThe Gardiners stayed only one night at Longbourn, and set off the next\r\nmorning with Elizabeth in pursuit of novelty and amusement. One\r\nenjoyment was certain--that of suitableness as companions; a\r\nsuitableness which comprehended health and temper to bear\r\ninconveniences--cheerfulness to enhance every pleasure--and affection\r\nand intelligence, which might supply it among themselves if there were\r\ndisappointments abroad.\r\n\r\nIt is not the object of this work to give a description of Derbyshire,\r\nnor of any of the remarkable places through which their route thither\r\nlay--Oxford, Blenheim, Warwick, Kenilworth, Birmingham, etc., are\r\nsufficiently known. A small part of Derbyshire is all the present\r\nconcern. To the little town of Lambton, the scene of Mrs. Gardiner’s\r\nformer residence, and where she had lately learned that some\r\nacquaintance still remained, they bent their steps, after having seen\r\nall the principal wonders of the country; and within five miles of\r\nLambton, Elizabeth found, from her aunt, that Pemberley was situated. It\r\nwas not in their direct road; nor more than a mile or two out of it. In\r\ntalking over their route the evening before, Mrs. Gardiner expressed an\r\ninclination to see the place again. Mr. Gardiner declared his\r\nwillingness, and Elizabeth was applied to for her approbation.\r\n\r\n“My love, should not you like to see a place of which you have heard so\r\nmuch?” said her aunt. “A place, too, with which so many of your\r\nacquaintance are connected. Wickham passed all his youth there, you\r\nknow.”\r\n\r\nElizabeth was distressed. She felt that she had no business at\r\nPemberley, and was obliged to assume a disinclination for seeing it. She\r\nmust own that she was tired of great houses: after going over so many,\r\nshe really had no pleasure in fine carpets or satin curtains.\r\n\r\nMrs. Gardiner abused her stupidity. “If it were merely a fine house\r\nrichly furnished,” said she, “I should not care about it myself; but the\r\ngrounds are delightful. They have some of the finest woods in the\r\ncountry.”\r\n\r\nElizabeth said no more; but her mind could not acquiesce. The\r\npossibility of meeting Mr. Darcy, while viewing the place, instantly\r\noccurred. It would be dreadful! She blushed at the very idea; and\r\nthought it would be better to speak openly to her aunt, than to run such\r\na risk. But against this there were objections; and she finally resolved\r\nthat it could be the last resource, if her private inquiries as to the\r\nabsence of the family were unfavourably answered.\r\n\r\nAccordingly, when she retired at night, she asked the chambermaid\r\nwhether Pemberley were not a very fine place, what was the name of its\r\nproprietor, and, with no little alarm, whether the family were down for\r\nthe summer? A most welcome negative followed the last question; and her\r\nalarms being now removed, she was at leisure to feel a great deal of\r\ncuriosity to see the house herself; and when the subject was revived the\r\nnext morning, and she was again applied to, could readily answer, and\r\nwith a proper air of indifference, that she had not really any dislike\r\nto the scheme.\r\n\r\nTo Pemberley, therefore, they were to go.\r\n\r\n\r\n\r\n\r\n[Illustration:\r\n\r\n “Conjecturing as to the date”\r\n]\r\n\r\n\r\n\r\n\r\nCHAPTER XLIII.\r\n\r\n\r\n[Illustration]\r\n\r\nElizabeth, as they drove along, watched for the first appearance of\r\nPemberley Woods with some perturbation; and when at length they turned\r\nin at the lodge, her spirits were in a high flutter.\r\n\r\nThe park was very large, and contained great variety of ground. They\r\nentered it in one of its lowest points, and drove for some time through\r\na beautiful wood stretching over a wide extent.\r\n\r\nElizabeth’s mind was too full for conversation, but she saw and admired\r\nevery remarkable spot and point of view. They gradually ascended for\r\nhalf a mile, and then found themselves at the top of a considerable\r\neminence, where the wood ceased, and the eye was instantly caught by\r\nPemberley House, situated on the opposite side of the valley, into which\r\nthe road with some abruptness wound. It was a large, handsome stone\r\nbuilding, standing well on rising ground, and backed by a ridge of high\r\nwoody hills; and in front a stream of some natural importance was\r\nswelled into greater, but without any artificial appearance. Its banks\r\nwere neither formal nor falsely adorned. Elizabeth was delighted. She\r\nhad never seen a place for which nature had done more, or where natural\r\nbeauty had been so little counteracted by an awkward taste. They were\r\nall of them warm in their admiration; and at that moment she felt that\r\nto be mistress of Pemberley might be something!\r\n\r\nThey descended the hill, crossed the bridge, and drove to the door; and,\r\nwhile examining the nearer aspect of the house, all her apprehension of\r\nmeeting its owner returned. She dreaded lest the chambermaid had been\r\nmistaken. On applying to see the place, they were admitted into the\r\nhall; and Elizabeth, as they waited for the housekeeper, had leisure to\r\nwonder at her being where she was.\r\n\r\nThe housekeeper came; a respectable looking elderly woman, much less\r\nfine, and more civil, than she had any notion of finding her. They\r\nfollowed her into the dining-parlour. It was a large, well-proportioned\r\nroom, handsomely fitted up. Elizabeth, after slightly surveying it, went\r\nto a window to enjoy its prospect. The hill, crowned with wood, from\r\nwhich they had descended, receiving increased abruptness from the\r\ndistance, was a beautiful object. Every disposition of the ground was\r\ngood; and she looked on the whole scene, the river, the trees scattered\r\non its banks, and the winding of the valley, as far as she could trace\r\nit, with delight. As they passed into other rooms, these objects were\r\ntaking different positions; but from every window there were beauties\r\nto be seen. The rooms were lofty and handsome, and their furniture\r\nsuitable to the fortune of their proprietor; but Elizabeth saw, with\r\nadmiration of his taste, that it was neither gaudy nor uselessly\r\nfine,--with less of splendour, and more real elegance, than the\r\nfurniture of Rosings.\r\n\r\n“And of this place,” thought she, “I might have been mistress! With\r\nthese rooms I might have now been familiarly acquainted! Instead of\r\nviewing them as a stranger, I might have rejoiced in them as my own, and\r\nwelcomed to them as visitors my uncle and aunt. But, no,” recollecting\r\nherself, “that could never be; my uncle and aunt would have been lost to\r\nme; I should not have been allowed to invite them.”\r\n\r\nThis was a lucky recollection--it saved her from something like regret.\r\n\r\nShe longed to inquire of the housekeeper whether her master were really\r\nabsent, but had not courage for it. At length, however, the question was\r\nasked by her uncle; and she turned away with alarm, while Mrs. Reynolds\r\nreplied, that he was; adding, “But we expect him to-morrow, with a large\r\nparty of friends.” How rejoiced was Elizabeth that their own journey had\r\nnot by any circumstance been delayed a day!\r\n\r\nHer aunt now called her to look at a picture. She approached, and saw\r\nthe likeness of Mr. Wickham, suspended, amongst several other\r\nminiatures, over the mantel-piece. Her aunt asked her, smilingly, how\r\nshe liked it. The housekeeper came forward, and told them it was the\r\npicture of a young gentleman, the son of her late master’s steward, who\r\nhad been brought up by him at his own expense. “He is now gone into the\r\narmy,” she added; “but I am afraid he has turned out very wild.”\r\n\r\nMrs. Gardiner looked at her niece with a smile, but Elizabeth could not\r\nreturn it.\r\n\r\n“And that,” said Mrs. Reynolds, pointing to another of the miniatures,\r\n“is my master--and very like him. It was drawn at the same time as the\r\nother--about eight years ago.”\r\n\r\n“I have heard much of your master’s fine person,” said Mrs. Gardiner,\r\nlooking at the picture; “it is a handsome face. But, Lizzy, you can tell\r\nus whether it is like or not.”\r\n\r\nMrs. Reynolds’ respect for Elizabeth seemed to increase on this\r\nintimation of her knowing her master.\r\n\r\n“Does that young lady know Mr. Darcy?”\r\n\r\nElizabeth coloured, and said, “A little.”\r\n\r\n“And do not you think him a very handsome gentleman, ma’am?”\r\n\r\n“Yes, very handsome.”\r\n\r\n“I am sure _I_ know none so handsome; but in the gallery upstairs you\r\nwill see a finer, larger picture of him than this. This room was my late\r\nmaster’s favourite room, and these miniatures are just as they used to\r\nbe then. He was very fond of them.”\r\n\r\nThis accounted to Elizabeth for Mr. Wickham’s being among them.\r\n\r\nMrs. Reynolds then directed their attention to one of Miss Darcy, drawn\r\nwhen she was only eight years old.\r\n\r\n“And is Miss Darcy as handsome as her brother?” said Mr. Gardiner.\r\n\r\n“Oh, yes--the handsomest young lady that ever was seen; and so\r\naccomplished! She plays and sings all day long. In the next room is a\r\nnew instrument just come down for her--a present from my master: she\r\ncomes here to-morrow with him.”\r\n\r\nMr. Gardiner, whose manners were easy and pleasant, encouraged her\r\ncommunicativeness by his questions and remarks: Mrs. Reynolds, either\r\nfrom pride or attachment, had evidently great pleasure in talking of her\r\nmaster and his sister.\r\n\r\n“Is your master much at Pemberley in the course of the year?”\r\n\r\n“Not so much as I could wish, sir: but I dare say he may spend half his\r\ntime here; and Miss Darcy is always down for the summer months.”\r\n\r\n“Except,” thought Elizabeth, “when she goes to Ramsgate.”\r\n\r\n“If your master would marry, you might see more of him.”\r\n\r\n“Yes, sir; but I do not know when _that_ will be. I do not know who is\r\ngood enough for him.”\r\n\r\nMr. and Mrs. Gardiner smiled. Elizabeth could not help saying, “It is\r\nvery much to his credit, I am sure, that you should think so.”\r\n\r\n“I say no more than the truth, and what everybody will say that knows\r\nhim,” replied the other. Elizabeth thought this was going pretty far;\r\nand she listened with increasing astonishment as the housekeeper added,\r\n“I have never had a cross word from him in my life, and I have known him\r\never since he was four years old.”\r\n\r\nThis was praise of all others most extraordinary, most opposite to her\r\nideas. That he was not a good-tempered man had been her firmest opinion.\r\nHer keenest attention was awakened: she longed to hear more; and was\r\ngrateful to her uncle for saying,--\r\n\r\n“There are very few people of whom so much can be said. You are lucky in\r\nhaving such a master.”\r\n\r\n“Yes, sir, I know I am. If I were to go through the world, I could not\r\nmeet with a better. But I have always observed, that they who are\r\ngood-natured when children, are good-natured when they grow up; and he\r\nwas always the sweetest tempered, most generous-hearted boy in the\r\nworld.”\r\n\r\nElizabeth almost stared at her. “Can this be Mr. Darcy?” thought she.\r\n\r\n“His father was an excellent man,” said Mrs. Gardiner.\r\n\r\n“Yes, ma’am, that he was indeed; and his son will be just like him--just\r\nas affable to the poor.”\r\n\r\nElizabeth listened, wondered, doubted, and was impatient for more. Mrs.\r\nReynolds could interest her on no other point. She related the subjects\r\nof the pictures, the dimensions of the rooms, and the price of the\r\nfurniture in vain. Mr. Gardiner, highly amused by the kind of family\r\nprejudice, to which he attributed her excessive commendation of her\r\nmaster, soon led again to the subject; and she dwelt with energy on his\r\nmany merits, as they proceeded together up the great staircase.\r\n\r\n“He is the best landlord, and the best master,” said she, “that ever\r\nlived. Not like the wild young men now-a-days, who think of nothing but\r\nthemselves. There is not one of his tenants or servants but what will\r\ngive him a good name. Some people call him proud; but I am sure I never\r\nsaw anything of it. To my fancy, it is only because he does not rattle\r\naway like other young men.”\r\n\r\n“In what an amiable light does this place him!” thought Elizabeth.\r\n\r\n“This fine account of him,” whispered her aunt as they walked, “is not\r\nquite consistent with his behaviour to our poor friend.”\r\n\r\n“Perhaps we might be deceived.”\r\n\r\n“That is not very likely; our authority was too good.”\r\n\r\nOn reaching the spacious lobby above, they were shown into a very pretty\r\nsitting-room, lately fitted up with greater elegance and lightness than\r\nthe apartments below; and were informed that it was but just done to\r\ngive pleasure to Miss Darcy, who had taken a liking to the room, when\r\nlast at Pemberley.\r\n\r\n“He is certainly a good brother,” said Elizabeth, as she walked towards\r\none of the windows.\r\n\r\nMrs. Reynolds anticipated Miss Darcy’s delight, when she should enter\r\nthe room. “And this is always the way with him,” she added. “Whatever\r\ncan give his sister any pleasure, is sure to be done in a moment. There\r\nis nothing he would not do for her.”\r\n\r\nThe picture gallery, and two or three of the principal bed-rooms, were\r\nall that remained to be shown. In the former were many good paintings:\r\nbut Elizabeth knew nothing of the art; and from such as had been already\r\nvisible below, she had willingly turned to look at some drawings of Miss\r\nDarcy’s, in crayons, whose subjects were usually more interesting, and\r\nalso more intelligible.\r\n\r\nIn the gallery there were many family portraits, but they could have\r\nlittle to fix the attention of a stranger. Elizabeth walked on in quest\r\nof the only face whose features would be known to her. At last it\r\narrested her--and she beheld a striking resemblance of Mr. Darcy, with\r\nsuch a smile over the face, as she remembered to have sometimes seen,\r\nwhen he looked at her. She stood several minutes before the picture, in\r\nearnest contemplation, and returned to it again before they quitted the\r\ngallery. Mrs. Reynolds informed them, that it had been taken in his\r\nfather’s lifetime.\r\n\r\nThere was certainly at this moment, in Elizabeth’s mind, a more gentle\r\nsensation towards the original than she had ever felt in the height of\r\ntheir acquaintance. The commendation bestowed on him by Mrs. Reynolds\r\nwas of no trifling nature. What praise is more valuable than the praise\r\nof an intelligent servant? As a brother, a landlord, a master, she\r\nconsidered how many people’s happiness were in his guardianship! How\r\nmuch of pleasure or pain it was in his power to bestow! How much of good\r\nor evil must be done by him! Every idea that had been brought forward by\r\nthe housekeeper was favourable to his character; and as she stood before\r\nthe canvas, on which he was represented, and fixed his eyes upon\r\nherself, she thought of his regard with a deeper sentiment of gratitude\r\nthan it had ever raised before: she remembered its warmth, and softened\r\nits impropriety of expression.\r\n\r\nWhen all of the house that was open to general inspection had been seen,\r\nthey returned down stairs; and, taking leave of the housekeeper, were\r\nconsigned over to the gardener, who met them at the hall door.\r\n\r\nAs they walked across the lawn towards the river, Elizabeth turned back\r\nto look again; her uncle and aunt stopped also; and while the former was\r\nconjecturing as to the date of the building, the owner of it himself\r\nsuddenly came forward from the road which led behind it to the stables.\r\n\r\nThey were within twenty yards of each other; and so abrupt was his\r\nappearance, that it was impossible to avoid his sight. Their eyes\r\ninstantly met, and the cheeks of each were overspread with the deepest\r\nblush. He absolutely started, and for a moment seemed immovable from\r\nsurprise; but shortly recovering himself, advanced towards the party,\r\nand spoke to Elizabeth, if not in terms of perfect composure, at least\r\nof perfect civility.\r\n\r\nShe had instinctively turned away; but stopping on his approach,\r\nreceived his compliments with an embarrassment impossible to be\r\novercome. Had his first appearance, or his resemblance to the picture\r\nthey had just been examining, been insufficient to assure the other two\r\nthat they now saw Mr. Darcy, the gardener’s expression of surprise, on\r\nbeholding his master, must immediately have told it. They stood a little\r\naloof while he was talking to their niece, who, astonished and confused,\r\nscarcely dared lift her eyes to his face, and knew not what answer she\r\nreturned to his civil inquiries after her family. Amazed at the\r\nalteration of his manner since they last parted, every sentence that he\r\nuttered was increasing her embarrassment; and every idea of the\r\nimpropriety of her being found there recurring to her mind, the few\r\nminutes in which they continued together were some of the most\r\nuncomfortable of her life. Nor did he seem much more at ease; when he\r\nspoke, his accent had none of its usual sedateness; and he repeated his\r\ninquiries as to the time of her having left Longbourn, and of her stay\r\nin Derbyshire, so often, and in so hurried a way, as plainly spoke the\r\ndistraction of his thoughts.\r\n\r\nAt length, every idea seemed to fail him; and after standing a few\r\nmoments without saying a word, he suddenly recollected himself, and took\r\nleave.\r\n\r\nThe others then joined her, and expressed their admiration of his\r\nfigure; but Elizabeth heard not a word, and, wholly engrossed by her own\r\nfeelings, followed them in silence. She was overpowered by shame and\r\nvexation. Her coming there was the most unfortunate, the most ill-judged\r\nthing in the world! How strange must it appear to him! In what a\r\ndisgraceful light might it not strike so vain a man! It might seem as if\r\nshe had purposely thrown herself in his way again! Oh! why did she come?\r\nor, why did he thus come a day before he was expected? Had they been\r\nonly ten minutes sooner, they should have been beyond the reach of his\r\ndiscrimination; for it was plain that he was that moment arrived, that\r\nmoment alighted from his horse or his carriage. She blushed again and\r\nagain over the perverseness of the meeting. And his behaviour, so\r\nstrikingly altered,--what could it mean? That he should even speak to\r\nher was amazing!--but to speak with such civility, to inquire after her\r\nfamily! Never in her life had she seen his manners so little dignified,\r\nnever had he spoken with such gentleness as on this unexpected meeting.\r\nWhat a contrast did it offer to his last address in Rosings Park, when\r\nhe put his letter into her hand! She knew not what to think, or how to\r\naccount for it.\r\n\r\nThey had now entered a beautiful walk by the side of the water, and\r\nevery step was bringing forward a nobler fall of ground, or a finer\r\nreach of the woods to which they were approaching: but it was some time\r\nbefore Elizabeth was sensible of any of it; and, though she answered\r\nmechanically to the repeated appeals of her uncle and aunt, and seemed\r\nto direct her eyes to such objects as they pointed out, she\r\ndistinguished no part of the scene. Her thoughts were all fixed on that\r\none spot of Pemberley House, whichever it might be, where Mr. Darcy then\r\nwas. She longed to know what at that moment was passing in his mind; in\r\nwhat manner he thought of her, and whether, in defiance of everything,\r\nshe was still dear to him. Perhaps he had been civil only because he\r\nfelt himself at ease; yet there had been _that_ in his voice, which was\r\nnot like ease. Whether he had felt more of pain or of pleasure in seeing\r\nher, she could not tell, but he certainly had not seen her with\r\ncomposure.\r\n\r\nAt length, however, the remarks of her companions on her absence of mind\r\nroused her, and she felt the necessity of appearing more like herself.\r\n\r\nThey entered the woods, and, bidding adieu to the river for a while,\r\nascended some of the higher grounds; whence, in spots where the opening\r\nof the trees gave the eye power to wander, were many charming views of\r\nthe valley, the opposite hills, with the long range of woods\r\noverspreading many, and occasionally part of the stream. Mr. Gardiner\r\nexpressed a wish of going round the whole park, but feared it might be\r\nbeyond a walk. With a triumphant smile, they were told, that it was ten\r\nmiles round. It settled the matter; and they pursued the accustomed\r\ncircuit; which brought them again, after some time, in a descent among\r\nhanging woods, to the edge of the water, and one of its narrowest parts.\r\nThey crossed it by a simple bridge, in character with the general air of\r\nthe scene: it was a spot less adorned than any they had yet visited; and\r\nthe valley, here contracted into a glen, allowed room only for the\r\nstream, and a narrow walk amidst the rough coppice-wood which bordered\r\nit. Elizabeth longed to explore its windings; but when they had crossed\r\nthe bridge, and perceived their distance from the house, Mrs. Gardiner,\r\nwho was not a great walker, could go no farther, and thought only of\r\nreturning to the carriage as quickly as possible. Her niece was,\r\ntherefore, obliged to submit, and they took their way towards the house\r\non the opposite side of the river, in the nearest direction; but their\r\nprogress was slow, for Mr. Gardiner, though seldom able to indulge the\r\ntaste, was very fond of fishing, and was so much engaged in watching the\r\noccasional appearance of some trout in the water, and talking to the man\r\nabout them, that he advanced but little. Whilst wandering on in this\r\nslow manner, they were again surprised, and Elizabeth’s astonishment was\r\nquite equal to what it had been at first, by the sight of Mr. Darcy\r\napproaching them, and at no great distance. The walk being here less\r\nsheltered than on the other side, allowed them to see him before they\r\nmet. Elizabeth, however astonished, was at least more prepared for an\r\ninterview than before, and resolved to appear and to speak with\r\ncalmness, if he really intended to meet them. For a few moments, indeed,\r\nshe felt that he would probably strike into some other path. The idea\r\nlasted while a turning in the walk concealed him from their view; the\r\nturning past, he was immediately before them. With a glance she saw that\r\nhe had lost none of his recent civility; and, to imitate his politeness,\r\nshe began as they met to admire the beauty of the place; but she had not\r\ngot beyond the words “delightful,” and “charming,” when some unlucky\r\nrecollections obtruded, and she fancied that praise of Pemberley from\r\nher might be mischievously construed. Her colour changed, and she said\r\nno more.\r\n\r\nMrs. Gardiner was standing a little behind; and on her pausing, he asked\r\nher if she would do him the honour of introducing him to her friends.\r\nThis was a stroke of civility for which she was quite unprepared; and\r\nshe could hardly suppress a smile at his being now seeking the\r\nacquaintance of some of those very people, against whom his pride had\r\nrevolted, in his offer to herself. “What will be his surprise,” thought\r\nshe, “when he knows who they are! He takes them now for people of\r\nfashion.”\r\n\r\nThe introduction, however, was immediately made; and as she named their\r\nrelationship to herself, she stole a sly look at him, to see how he bore\r\nit; and was not without the expectation of his decamping as fast as he\r\ncould from such disgraceful companions. That he was _surprised_ by the\r\nconnection was evident: he sustained it, however, with fortitude: and,\r\nso far from going away, turned back with them, and entered into\r\nconversation with Mr. Gardiner. Elizabeth could not but be pleased,\r\ncould not but triumph. It was consoling that he should know she had some\r\nrelations for whom there was no need to blush. She listened most\r\nattentively to all that passed between them, and gloried in every\r\nexpression, every sentence of her uncle, which marked his intelligence,\r\nhis taste, or his good manners.\r\n\r\nThe conversation soon turned upon fishing; and she heard Mr. Darcy\r\ninvite him, with the greatest civility, to fish there as often as he\r\nchose, while he continued in the neighbourhood, offering at the same\r\ntime to supply him with fishing tackle, and pointing out those parts of\r\nthe stream where there was usually most sport. Mrs. Gardiner, who was\r\nwalking arm in arm with Elizabeth, gave her a look expressive of her\r\nwonder. Elizabeth said nothing, but it gratified her exceedingly; the\r\ncompliment must be all for herself. Her astonishment, however, was\r\nextreme; and continually was she repeating, “Why is he so altered? From\r\nwhat can it proceed? It cannot be for _me_, it cannot be for _my_ sake\r\nthat his manners are thus softened. My reproofs at Hunsford could not\r\nwork such a change as this. It is impossible that he should still love\r\nme.”\r\n\r\nAfter walking some time in this way, the two ladies in front, the two\r\ngentlemen behind, on resuming their places, after descending to the\r\nbrink of the river for the better inspection of some curious\r\nwater-plant, there chanced to be a little alteration. It originated in\r\nMrs. Gardiner, who, fatigued by the exercise of the morning, found\r\nElizabeth’s arm inadequate to her support, and consequently preferred\r\nher husband’s. Mr. Darcy took her place by her niece, and they walked on\r\ntogether. After a short silence the lady first spoke. She wished him to\r\nknow that she had been assured of his absence before she came to the\r\nplace, and accordingly began by observing, that his arrival had been\r\nvery unexpected--“for your housekeeper,” she added, “informed us that\r\nyou would certainly not be here till to-morrow; and, indeed, before we\r\nleft Bakewell, we understood that you were not immediately expected in\r\nthe country.” He acknowledged the truth of it all; and said that\r\nbusiness with his steward had occasioned his coming forward a few hours\r\nbefore the rest of the party with whom he had been travelling. “They\r\nwill join me early to-morrow,” he continued, “and among them are some\r\nwho will claim an acquaintance with you,--Mr. Bingley and his sisters.”\r\n\r\nElizabeth answered only by a slight bow. Her thoughts were instantly\r\ndriven back to the time when Mr. Bingley’s name had been last mentioned\r\nbetween them; and if she might judge from his complexion, _his_ mind was\r\nnot very differently engaged.\r\n\r\n“There is also one other person in the party,” he continued after a\r\npause, “who more particularly wishes to be known to you. Will you allow\r\nme, or do I ask too much, to introduce my sister to your acquaintance\r\nduring your stay at Lambton?”\r\n\r\nThe surprise of such an application was great indeed; it was too great\r\nfor her to know in what manner she acceded to it. She immediately felt\r\nthat whatever desire Miss Darcy might have of being acquainted with her,\r\nmust be the work of her brother, and without looking farther, it was\r\nsatisfactory; it was gratifying to know that his resentment had not made\r\nhim think really ill of her.\r\n\r\nThey now walked on in silence; each of them deep in thought. Elizabeth\r\nwas not comfortable; that was impossible; but she was flattered and\r\npleased. His wish of introducing his sister to her was a compliment of\r\nthe highest kind. They soon outstripped the others; and when they had\r\nreached the carriage, Mr. and Mrs. Gardiner were half a quarter of a\r\nmile behind.\r\n\r\nHe then asked her to walk into the house--but she declared herself not\r\ntired, and they stood together on the lawn. At such a time much might\r\nhave been said, and silence was very awkward. She wanted to talk, but\r\nthere seemed an embargo on every subject. At last she recollected that\r\nshe had been travelling, and they talked of Matlock and Dovedale with\r\ngreat perseverance. Yet time and her aunt moved slowly--and her patience\r\nand her ideas were nearly worn out before the _tête-à-tête_ was over.\r\n\r\nOn Mr. and Mrs. Gardiner’s coming up they were all pressed to go into\r\nthe house and take some refreshment; but this was declined, and they\r\nparted on each side with the utmost politeness. Mr. Darcy handed the\r\nladies into the carriage; and when it drove off, Elizabeth saw him\r\nwalking slowly towards the house.\r\n\r\nThe observations of her uncle and aunt now began; and each of them\r\npronounced him to be infinitely superior to anything they had expected.\r\n\r\n“He is perfectly well-behaved, polite, and unassuming,” said her uncle.\r\n\r\n“There _is_ something a little stately in him, to be sure,” replied her\r\naunt; “but it is confined to his air, and is not unbecoming. I can now\r\nsay with the housekeeper, that though some people may call him proud,\r\n_I_ have seen nothing of it.”\r\n\r\n“I was never more surprised than by his behaviour to us. It was more\r\nthan civil; it was really attentive; and there was no necessity for such\r\nattention. His acquaintance with Elizabeth was very trifling.”\r\n\r\n“To be sure, Lizzy,” said her aunt, “he is not so handsome as Wickham;\r\nor rather he has not Wickham’s countenance, for his features are\r\nperfectly good. But how came you to tell us that he was so\r\ndisagreeable?”\r\n\r\nElizabeth excused herself as well as she could: said that she had liked\r\nhim better when they met in Kent than before, and that she had never\r\nseen him so pleasant as this morning.\r\n\r\n“But perhaps he may be a little whimsical in his civilities,” replied\r\nher uncle. “Your great men often are; and therefore I shall not take him\r\nat his word about fishing, as he might change his mind another day, and\r\nwarn me off his grounds.”\r\n\r\nElizabeth felt that they had entirely mistaken his character, but said\r\nnothing.\r\n\r\n“From what we have seen of him,” continued Mrs. Gardiner, “I really\r\nshould not have thought that he could have behaved in so cruel a way by\r\nanybody as he has done by poor Wickham. He has not an ill-natured look.\r\nOn the contrary, there is something pleasing about his mouth when he\r\nspeaks. And there is something of dignity in his countenance, that would\r\nnot give one an unfavourable idea of his heart. But, to be sure, the\r\ngood lady who showed us the house did give him a most flaming character!\r\nI could hardly help laughing aloud sometimes. But he is a liberal\r\nmaster, I suppose, and _that_, in the eye of a servant, comprehends\r\nevery virtue.”\r\n\r\nElizabeth here felt herself called on to say something in vindication of\r\nhis behaviour to Wickham; and, therefore, gave them to understand, in as\r\nguarded a manner as she could, that by what she had heard from his\r\nrelations in Kent, his actions were capable of a very different\r\nconstruction; and that his character was by no means so faulty, nor\r\nWickham’s so amiable, as they had been considered in Hertfordshire. In\r\nconfirmation of this, she related the particulars of all the pecuniary\r\ntransactions in which they had been connected, without actually naming\r\nher authority, but stating it to be such as might be relied on.\r\n\r\nMrs. Gardiner was surprised and concerned: but as they were now\r\napproaching the scene of her former pleasures, every idea gave way to\r\nthe charm of recollection; and she was too much engaged in pointing out\r\nto her husband all the interesting spots in its environs, to think of\r\nanything else. Fatigued as she had been by the morning’s walk, they had\r\nno sooner dined than she set off again in quest of her former\r\nacquaintance, and the evening was spent in the satisfactions of an\r\nintercourse renewed after many years’ discontinuance.\r\n\r\nThe occurrences of the day were too full of interest to leave Elizabeth\r\nmuch attention for any of these new friends; and she could do nothing\r\nbut think, and think with wonder, of Mr. Darcy’s civility, and, above\r\nall, of his wishing her to be acquainted with his sister.\r\n\r\n\r\n\r\n\r\n[Illustration]\r\n\r\n\r\n\r\n\r\nCHAPTER XLIV.\r\n\r\n\r\n[Illustration]\r\n\r\nElizabeth had settled it that Mr. Darcy would bring his sister to visit\r\nher the very day after her reaching Pemberley; and was, consequently,\r\nresolved not to be out of sight of the inn the whole of that morning.\r\nBut her conclusion was false; for on the very morning after their own\r\narrival at Lambton these visitors came. They had been walking about the\r\nplace with some of their new friends, and were just returned to the inn\r\nto dress themselves for dining with the same family, when the sound of a\r\ncarriage drew them to a window, and they saw a gentleman and lady in a\r\ncurricle driving up the street. Elizabeth, immediately recognizing the\r\nlivery, guessed what it meant, and imparted no small degree of surprise\r\nto her relations, by acquainting them with the honour which she\r\nexpected. Her uncle and aunt were all amazement; and the embarrassment\r\nof her manner as she spoke, joined to the circumstance itself, and many\r\nof the circumstances of the preceding day, opened to them a new idea on\r\nthe business. Nothing had ever suggested it before, but they now felt\r\nthat there was no other way of accounting for such attentions from such\r\na quarter than by supposing a partiality for their niece. While these\r\nnewly-born notions were passing in their heads, the perturbation of\r\nElizabeth’s feelings was every moment increasing. She was quite amazed\r\nat her own discomposure; but, amongst other causes of disquiet, she\r\ndreaded lest the partiality of the brother should have said too much in\r\nher favour; and, more than commonly anxious to please, she naturally\r\nsuspected that every power of pleasing would fail her.\r\n\r\nShe retreated from the window, fearful of being seen; and as she walked\r\nup and down the room, endeavouring to compose herself, saw such looks of\r\ninquiring surprise in her uncle and aunt as made everything worse.\r\n\r\nMiss Darcy and her brother appeared, and this formidable introduction\r\ntook place. With astonishment did Elizabeth see that her new\r\nacquaintance was at least as much embarrassed as herself. Since her\r\nbeing at Lambton, she had heard that Miss Darcy was exceedingly proud;\r\nbut the observation of a very few minutes convinced her that she was\r\nonly exceedingly shy. She found it difficult to obtain even a word from\r\nher beyond a monosyllable.\r\n\r\nMiss Darcy was tall, and on a larger scale than Elizabeth; and, though\r\nlittle more than sixteen, her figure was formed, and her appearance\r\nwomanly and graceful. She was less handsome than her brother, but there\r\nwas sense and good-humour in her face, and her manners were perfectly\r\nunassuming and gentle. Elizabeth, who had expected to find in her as\r\nacute and unembarrassed an observer as ever Mr. Darcy had been, was much\r\nrelieved by discerning such different feelings.\r\n\r\nThey had not been long together before Darcy told her that Bingley was\r\nalso coming to wait on her; and she had barely time to express her\r\nsatisfaction, and prepare for such a visitor, when Bingley’s quick step\r\nwas heard on the stairs, and in a moment he entered the room. All\r\nElizabeth’s anger against him had been long done away; but had she still\r\nfelt any, it could hardly have stood its ground against the unaffected\r\ncordiality with which he expressed himself on seeing her again. He\r\ninquired in a friendly, though general, way, after her family, and\r\nlooked and spoke with the same good-humoured ease that he had ever done.\r\n\r\nTo Mr. and Mrs. Gardiner he was scarcely a less interesting personage\r\nthan to herself. They had long wished to see him. The whole party before\r\nthem, indeed, excited a lively attention. The suspicions which had just\r\narisen of Mr. Darcy and their niece, directed their observation towards\r\neach with an earnest, though guarded, inquiry; and they soon drew from\r\nthose inquiries the full conviction that one of them at least knew what\r\nit was to love. Of the lady’s sensations they remained a little in\r\ndoubt; but that the gentleman was overflowing with admiration was\r\nevident enough.\r\n\r\nElizabeth, on her side, had much to do. She wanted to ascertain the\r\nfeelings of each of her visitors, she wanted to compose her own, and to\r\nmake herself agreeable to all; and in the latter object, where she\r\nfeared most to fail, she was most sure of success, for those to whom\r\nshe endeavoured to give pleasure were pre-possessed in her favour.\r\nBingley was ready, Georgiana was eager, and Darcy determined, to be\r\npleased.\r\n\r\n[Illustration:\r\n\r\n “To make herself agreeable to all”\r\n\r\n[_Copyright 1894 by George Allen._]]\r\n\r\nIn seeing Bingley, her thoughts naturally flew to her sister; and oh!\r\nhow ardently did she long to know whether any of his were directed in a\r\nlike manner. Sometimes she could fancy that he talked less than on\r\nformer occasions, and once or twice pleased herself with the notion\r\nthat, as he looked at her, he was trying to trace a resemblance. But,\r\nthough this might be imaginary, she could not be deceived as to his\r\nbehaviour to Miss Darcy, who had been set up as a rival to Jane. No\r\nlook appeared on either side that spoke particular regard. Nothing\r\noccurred between them that could justify the hopes of his sister. On\r\nthis point she was soon satisfied; and two or three little circumstances\r\noccurred ere they parted, which, in her anxious interpretation, denoted\r\na recollection of Jane, not untinctured by tenderness, and a wish of\r\nsaying more that might lead to the mention of her, had he dared. He\r\nobserved to her, at a moment when the others were talking together, and\r\nin a tone which had something of real regret, that it “was a very long\r\ntime since he had had the pleasure of seeing her;” and, before she could\r\nreply, he added, “It is above eight months. We have not met since the\r\n26th of November, when we were all dancing together at Netherfield.”\r\n\r\nElizabeth was pleased to find his memory so exact; and he afterwards\r\ntook occasion to ask her, when unattended to by any of the rest, whether\r\n_all_ her sisters were at Longbourn. There was not much in the question,\r\nnor in the preceding remark; but there was a look and a manner which\r\ngave them meaning.\r\n\r\nIt was not often that she could turn her eyes on Mr. Darcy himself; but\r\nwhenever she did catch a glimpse she saw an expression of general\r\ncomplaisance, and in all that he said, she heard an accent so far\r\nremoved from _hauteur_ or disdain of his companions, as convinced her\r\nthat the improvement of manners which she had yesterday witnessed,\r\nhowever temporary its existence might prove, had at least outlived one\r\nday. When she saw him thus seeking the acquaintance, and courting the\r\ngood opinion of people with whom any intercourse a few months ago would\r\nhave been a disgrace; when she saw him thus civil, not only to herself,\r\nbut to the very relations whom he had openly disdained, and recollected\r\ntheir last lively scene in Hunsford Parsonage, the difference, the\r\nchange was so great, and struck so forcibly on her mind, that she could\r\nhardly restrain her astonishment from being visible. Never, even in the\r\ncompany of his dear friends at Netherfield, or his dignified relations\r\nat Rosings, had she seen him so desirous to please, so free from\r\nself-consequence or unbending reserve, as now, when no importance could\r\nresult from the success of his endeavours, and when even the\r\nacquaintance of those to whom his attentions were addressed, would draw\r\ndown the ridicule and censure of the ladies both of Netherfield and\r\nRosings.\r\n\r\nTheir visitors stayed with them above half an hour; and when they arose\r\nto depart, Mr. Darcy called on his sister to join him in expressing\r\ntheir wish of seeing Mr. and Mrs. Gardiner, and Miss Bennet, to dinner\r\nat Pemberley, before they left the country. Miss Darcy, though with a\r\ndiffidence which marked her little in the habit of giving invitations,\r\nreadily obeyed. Mrs. Gardiner looked at her niece, desirous of knowing\r\nhow _she_, whom the invitation most concerned, felt disposed as to its\r\nacceptance, but Elizabeth had turned away her head. Presuming, however,\r\nthat this studied avoidance spoke rather a momentary embarrassment than\r\nany dislike of the proposal, and seeing in her husband, who was fond of\r\nsociety, a perfect willingness to accept it, she ventured to engage for\r\nher attendance, and the day after the next was fixed on.\r\n\r\nBingley expressed great pleasure in the certainty of seeing Elizabeth\r\nagain, having still a great deal to say to her, and many inquiries to\r\nmake after all their Hertfordshire friends. Elizabeth, construing all\r\nthis into a wish of hearing her speak of her sister, was pleased; and\r\non this account, as well as some others, found herself, when their\r\nvisitors left them, capable of considering the last half hour with some\r\nsatisfaction, though while it was passing the enjoyment of it had been\r\nlittle. Eager to be alone, and fearful of inquiries or hints from her\r\nuncle and aunt, she stayed with them only long enough to hear their\r\nfavourable opinion of Bingley, and then hurried away to dress.\r\n\r\nBut she had no reason to fear Mr. and Mrs. Gardiner’s curiosity; it was\r\nnot their wish to force her communication. It was evident that she was\r\nmuch better acquainted with Mr. Darcy than they had before any idea of;\r\nit was evident that he was very much in love with her. They saw much to\r\ninterest, but nothing to justify inquiry.\r\n\r\nOf Mr. Darcy it was now a matter of anxiety to think well; and, as far\r\nas their acquaintance reached, there was no fault to find. They could\r\nnot be untouched by his politeness; and had they drawn his character\r\nfrom their own feelings and his servant’s report, without any reference\r\nto any other account, the circle in Hertfordshire to which he was known\r\nwould not have recognized it for Mr. Darcy. There was now an interest,\r\nhowever, in believing the housekeeper; and they soon became sensible\r\nthat the authority of a servant, who had known him since he was four\r\nyears old, and whose own manners indicated respectability, was not to be\r\nhastily rejected. Neither had anything occurred in the intelligence of\r\ntheir Lambton friends that could materially lessen its weight. They had\r\nnothing to accuse him of but pride; pride he probably had, and if not,\r\nit would certainly be imputed by the inhabitants of a small market town\r\nwhere the family did not visit. It was acknowledged, however, that he\r\nwas a liberal man, and did much good among the poor.\r\n\r\nWith respect to Wickham, the travellers soon found that he was not held\r\nthere in much estimation; for though the chief of his concerns with the\r\nson of his patron were imperfectly understood, it was yet a well-known\r\nfact that, on his quitting Derbyshire, he had left many debts behind\r\nhim, which Mr. Darcy afterwards discharged.\r\n\r\nAs for Elizabeth, her thoughts were at Pemberley this evening more than\r\nthe last; and the evening, though as it passed it seemed long, was not\r\nlong enough to determine her feelings towards _one_ in that mansion; and\r\nshe lay awake two whole hours, endeavouring to make them out. She\r\ncertainly did not hate him. No; hatred had vanished long ago, and she\r\nhad almost as long been ashamed of ever feeling a dislike against him,\r\nthat could be so called. The respect created by the conviction of his\r\nvaluable qualities, though at first unwillingly admitted, had for some\r\ntime ceased to be repugnant to her feelings; and it was now heightened\r\ninto somewhat of a friendlier nature by the testimony so highly in his\r\nfavour, and bringing forward his disposition in so amiable a light,\r\nwhich yesterday had produced. But above all, above respect and esteem,\r\nthere was a motive within her of good-will which could not be\r\noverlooked. It was gratitude;--gratitude, not merely for having once\r\nloved her, but for loving her still well enough to forgive all the\r\npetulance and acrimony of her manner in rejecting him, and all the\r\nunjust accusations accompanying her rejection. He who, she had been\r\npersuaded, would avoid her as his greatest enemy, seemed, on this\r\naccidental meeting, most eager to preserve the acquaintance; and\r\nwithout any indelicate display of regard, or any peculiarity of manner,\r\nwhere their two selves only were concerned, was soliciting the good\r\nopinion of her friends, and bent on making her known to his sister. Such\r\na change in a man of so much pride excited not only astonishment but\r\ngratitude--for to love, ardent love, it must be attributed; and, as\r\nsuch, its impression on her was of a sort to be encouraged, as by no\r\nmeans unpleasing, though it could not be exactly defined. She respected,\r\nshe esteemed, she was grateful to him, she felt a real interest in his\r\nwelfare; and she only wanted to know how far she wished that welfare to\r\ndepend upon herself, and how far it would be for the happiness of both\r\nthat she should employ the power, which her fancy told her she still\r\npossessed, of bringing on the renewal of his addresses.\r\n\r\nIt had been settled in the evening, between the aunt and niece, that\r\nsuch a striking civility as Miss Darcy’s, in coming to them on the very\r\nday of her arrival at Pemberley--for she had reached it only to a late\r\nbreakfast--ought to be imitated, though it could not be equalled, by\r\nsome exertion of politeness on their side; and, consequently, that it\r\nwould be highly expedient to wait on her at Pemberley the following\r\nmorning. They were, therefore, to go. Elizabeth was pleased; though when\r\nshe asked herself the reason, she had very little to say in reply.\r\n\r\nMr. Gardiner left them soon after breakfast. The fishing scheme had been\r\nrenewed the day before, and a positive engagement made of his meeting\r\nsome of the gentlemen at Pemberley by noon.\r\n\r\n\r\n\r\n\r\n[Illustration:\r\n\r\n “Engaged by the river”\r\n]\r\n\r\n\r\n\r\n\r\nCHAPTER XLV.\r\n\r\n\r\n[Illustration]\r\n\r\nConvinced as Elizabeth now was that Miss Bingley’s dislike of her had\r\noriginated in jealousy, she could not help feeling how very unwelcome\r\nher appearance at Pemberley must be to her, and was curious to know\r\nwith how much civility on that lady’s side the acquaintance would now\r\nbe renewed.\r\n\r\nOn reaching the house, they were shown through the hall into the saloon,\r\nwhose northern aspect rendered it delightful for summer. Its windows,\r\nopening to the ground, admitted a most refreshing view of the high woody\r\nhills behind the house, and of the beautiful oaks and Spanish chestnuts\r\nwhich were scattered over the intermediate lawn.\r\n\r\nIn this room they were received by Miss Darcy, who was sitting there\r\nwith Mrs. Hurst and Miss Bingley, and the lady with whom she lived in\r\nLondon. Georgiana’s reception of them was very civil, but attended with\r\nall that embarrassment which, though proceeding from shyness and the\r\nfear of doing wrong, would easily give to those who felt themselves\r\ninferior the belief of her being proud and reserved. Mrs. Gardiner and\r\nher niece, however, did her justice, and pitied her.\r\n\r\nBy Mrs. Hurst and Miss Bingley they were noticed only by a courtesy; and\r\non their being seated, a pause, awkward as such pauses must always be,\r\nsucceeded for a few moments. It was first broken by Mrs. Annesley, a\r\ngenteel, agreeable-looking woman, whose endeavour to introduce some kind\r\nof discourse proved her to be more truly well-bred than either of the\r\nothers; and between her and Mrs. Gardiner, with occasional help from\r\nElizabeth, the conversation was carried on. Miss Darcy looked as if she\r\nwished for courage enough to join in it; and sometimes did venture a\r\nshort sentence, when there was least danger of its being heard.\r\n\r\nElizabeth soon saw that she was herself closely watched by Miss Bingley,\r\nand that she could not speak a word, especially to Miss Darcy, without\r\ncalling her attention. This observation would not have prevented her\r\nfrom trying to talk to the latter, had they not been seated at an\r\ninconvenient distance; but she was not sorry to be spared the necessity\r\nof saying much: her own thoughts were employing her. She expected every\r\nmoment that some of the gentlemen would enter the room: she wished, she\r\nfeared, that the master of the house might be amongst them; and whether\r\nshe wished or feared it most, she could scarcely determine. After\r\nsitting in this manner a quarter of an hour, without hearing Miss\r\nBingley’s voice, Elizabeth was roused by receiving from her a cold\r\ninquiry after the health of her family. She answered with equal\r\nindifference and brevity, and the other said no more.\r\n\r\nThe next variation which their visit afforded was produced by the\r\nentrance of servants with cold meat, cake, and a variety of all the\r\nfinest fruits in season; but this did not take place till after many a\r\nsignificant look and smile from Mrs. Annesley to Miss Darcy had been\r\ngiven, to remind her of her post. There was now employment for the whole\r\nparty; for though they could not all talk, they could all eat; and the\r\nbeautiful pyramids of grapes, nectarines, and peaches, soon collected\r\nthem round the table.\r\n\r\nWhile thus engaged, Elizabeth had a fair opportunity of deciding whether\r\nshe most feared or wished for the appearance of Mr. Darcy, by the\r\nfeelings which prevailed on his entering the room; and then, though but\r\na moment before she had believed her wishes to predominate, she began to\r\nregret that he came.\r\n\r\nHe had been some time with Mr. Gardiner, who, with two or three other\r\ngentlemen from the house, was engaged by the river; and had left him\r\nonly on learning that the ladies of the family intended a visit to\r\nGeorgiana that morning. No sooner did he appear, than Elizabeth wisely\r\nresolved to be perfectly easy and unembarrassed;--a resolution the more\r\nnecessary to be made, but perhaps not the more easily kept, because she\r\nsaw that the suspicions of the whole party were awakened against them,\r\nand that there was scarcely an eye which did not watch his behaviour\r\nwhen he first came into the room. In no countenance was attentive\r\ncuriosity so strongly marked as in Miss Bingley’s, in spite of the\r\nsmiles which overspread her face whenever she spoke to one of its\r\nobjects; for jealousy had not yet made her desperate, and her attentions\r\nto Mr. Darcy were by no means over. Miss Darcy, on her brother’s\r\nentrance, exerted herself much more to talk; and Elizabeth saw that he\r\nwas anxious for his sister and herself to get acquainted, and forwarded,\r\nas much as possible, every attempt at conversation on either side. Miss\r\nBingley saw all this likewise; and, in the imprudence of anger, took the\r\nfirst opportunity of saying, with sneering civility,--\r\n\r\n“Pray, Miss Eliza, are not the ----shire militia removed from Meryton?\r\nThey must be a great loss to _your_ family.”\r\n\r\nIn Darcy’s presence she dared not mention Wickham’s name: but Elizabeth\r\ninstantly comprehended that he was uppermost in her thoughts; and the\r\nvarious recollections connected with him gave her a moment’s distress;\r\nbut, exerting herself vigorously to repel the ill-natured attack, she\r\npresently answered the question in a tolerably disengaged tone. While\r\nshe spoke, an involuntary glance showed her Darcy with a heightened\r\ncomplexion, earnestly looking at her, and his sister overcome with\r\nconfusion, and unable to lift up her eyes. Had Miss Bingley known what\r\npain she was then giving her beloved friend, she undoubtedly would have\r\nrefrained from the hint; but she had merely intended to discompose\r\nElizabeth, by bringing forward the idea of a man to whom she believed\r\nher partial, to make her betray a sensibility which might injure her in\r\nDarcy’s opinion, and, perhaps, to remind the latter of all the follies\r\nand absurdities by which some part of her family were connected with\r\nthat corps. Not a syllable had ever reached her of Miss Darcy’s\r\nmeditated elopement. To no creature had it been revealed, where secrecy\r\nwas possible, except to Elizabeth; and from all Bingley’s connections\r\nher brother was particularly anxious to conceal it, from that very wish\r\nwhich Elizabeth had long ago attributed to him, of their becoming\r\nhereafter her own. He had certainly formed such a plan; and without\r\nmeaning that it should affect his endeavour to separate him from Miss\r\nBennet, it is probable that it might add something to his lively concern\r\nfor the welfare of his friend.\r\n\r\nElizabeth’s collected behaviour, however, soon quieted his emotion; and\r\nas Miss Bingley, vexed and disappointed, dared not approach nearer to\r\nWickham, Georgiana also recovered in time, though not enough to be able\r\nto speak any more. Her brother, whose eye she feared to meet, scarcely\r\nrecollected her interest in the affair; and the very circumstance which\r\nhad been designed to turn his thoughts from Elizabeth, seemed to have\r\nfixed them on her more and more cheerfully.\r\n\r\nTheir visit did not continue long after the question and answer above\r\nmentioned; and while Mr. Darcy was attending them to their carriage,\r\nMiss Bingley was venting her feelings in criticisms on Elizabeth’s\r\nperson, behaviour, and dress. But Georgiana would not join her. Her\r\nbrother’s recommendation was enough to insure her favour: his judgment\r\ncould not err; and he had spoken in such terms of Elizabeth, as to leave\r\nGeorgiana without the power of finding her otherwise than lovely and\r\namiable. When Darcy returned to the saloon, Miss Bingley could not help\r\nrepeating to him some part of what she had been saying to his sister.\r\n\r\n“How very ill Eliza Bennet looks this morning, Mr. Darcy,” she cried: “I\r\nnever in my life saw anyone so much altered as she is since the winter.\r\nShe is grown so brown and coarse! Louisa and I were agreeing that we\r\nshould not have known her again.”\r\n\r\nHowever little Mr. Darcy might have liked such an address, he contented\r\nhimself with coolly replying, that he perceived no other alteration than\r\nher being rather tanned,--no miraculous consequence of travelling in the\r\nsummer.\r\n\r\n“For my own part,” she rejoined, “I must confess that I never could see\r\nany beauty in her. Her face is too thin; her complexion has no\r\nbrilliancy; and her features are not at all handsome. Her nose wants\r\ncharacter; there is nothing marked in its lines. Her teeth are\r\ntolerable, but not out of the common way; and as for her eyes, which\r\nhave sometimes been called so fine, I never could perceive anything\r\nextraordinary in them. They have a sharp, shrewish look, which I do not\r\nlike at all; and in her air altogether, there is a self-sufficiency\r\nwithout fashion, which is intolerable.”\r\n\r\nPersuaded as Miss Bingley was that Darcy admired Elizabeth, this was not\r\nthe best method of recommending herself; but angry people are not always\r\nwise; and in seeing him at last look somewhat nettled, she had all the\r\nsuccess she expected. He was resolutely silent, however; and, from a\r\ndetermination of making him speak, she continued,--\r\n\r\n“I remember, when we first knew her in Hertfordshire, how amazed we all\r\nwere to find that she was a reputed beauty; and I particularly recollect\r\nyour saying one night, after they had been dining at Netherfield, ‘_She_\r\na beauty! I should as soon call her mother a wit.’ But afterwards she\r\nseemed to improve on you, and I believe you thought her rather pretty at\r\none time.”\r\n\r\n“Yes,” replied Darcy, who could contain himself no longer, “but _that_\r\nwas only when I first knew her; for it is many months since I have\r\nconsidered her as one of the handsomest women of my acquaintance.”\r\n\r\nHe then went away, and Miss Bingley was left to all the satisfaction of\r\nhaving forced him to say what gave no one any pain but herself.\r\n\r\nMrs. Gardiner and Elizabeth talked of all that had occurred during their\r\nvisit, as they returned, except what had particularly interested them\r\nboth. The looks and behaviour of everybody they had seen were discussed,\r\nexcept of the person who had mostly engaged their attention. They talked\r\nof his sister, his friends, his house, his fruit, of everything but\r\nhimself; yet Elizabeth was longing to know what Mrs. Gardiner thought of\r\nhim, and Mrs. Gardiner would have been highly gratified by her niece’s\r\nbeginning the subject.\r\n\r\n\r\n\r\n\r\n[Illustration]\r\n\r\n\r\n\r\n\r\nChapter XLVI.\r\n\r\n\r\n[Illustration]\r\n\r\nElizabeth had been a good deal disappointed in not finding a letter from\r\nJane on their first arrival at Lambton; and this disappointment had been\r\nrenewed on each of the mornings that had now been spent there; but on\r\nthe third her repining was over, and her sister justified, by the\r\nreceipt of two letters from her at once, on one of which was marked that\r\nit had been mis-sent elsewhere. Elizabeth was not surprised at it, as\r\nJane had written the direction remarkably ill.\r\n\r\nThey had just been preparing to walk as the letters came in; and her\r\nuncle and aunt, leaving her to enjoy them in quiet, set off by\r\nthemselves. The one mis-sent must be first attended to; it had been\r\nwritten five days ago. The beginning contained an account of all their\r\nlittle parties and engagements, with such news as the country afforded;\r\nbut the latter half, which was dated a day later, and written in evident\r\nagitation, gave more important intelligence. It was to this effect:--\r\n\r\n“Since writing the above, dearest Lizzy, something has occurred of a\r\nmost unexpected and serious nature; but I am afraid of alarming you--be\r\nassured that we are all well. What I have to say relates to poor Lydia.\r\nAn express came at twelve last night, just as we were all gone to bed,\r\nfrom Colonel Forster, to inform us that she was gone off to Scotland\r\nwith one of his officers; to own the truth, with Wickham! Imagine our\r\nsurprise. To Kitty, however, it does not seem so wholly unexpected. I am\r\nvery, very sorry. So imprudent a match on both sides! But I am willing\r\nto hope the best, and that his character has been misunderstood.\r\nThoughtless and indiscreet I can easily believe him, but this step (and\r\nlet us rejoice over it) marks nothing bad at heart. His choice is\r\ndisinterested at least, for he must know my father can give her nothing.\r\nOur poor mother is sadly grieved. My father bears it better. How\r\nthankful am I, that we never let them know what has been said against\r\nhim; we must forget it ourselves. They were off Saturday night about\r\ntwelve, as is conjectured, but were not missed till yesterday morning at\r\neight. The express was sent off directly. My dear Lizzy, they must have\r\npassed within ten miles of us. Colonel Forster gives us reason to expect\r\nhim here soon. Lydia left a few lines for his wife, informing her of\r\ntheir intention. I must conclude, for I cannot be long from my poor\r\nmother. I am afraid you will not be able to make it out, but I hardly\r\nknow what I have written.”\r\n\r\nWithout allowing herself time for consideration, and scarcely knowing\r\nwhat she felt, Elizabeth, on finishing this letter, instantly seized the\r\nother, and opening it with the utmost impatience, read as follows: it\r\nhad been written a day later than the conclusion of the first.\r\n\r\n“By this time, my dearest sister, you have received my hurried letter; I\r\nwish this may be more intelligible, but though not confined for time, my\r\nhead is so bewildered that I cannot answer for being coherent. Dearest\r\nLizzy, I hardly know what I would write, but I have bad news for you,\r\nand it cannot be delayed. Imprudent as a marriage between Mr. Wickham\r\nand our poor Lydia would be, we are now anxious to be assured it has\r\ntaken place, for there is but too much reason to fear they are not gone\r\nto Scotland. Colonel Forster came yesterday, having left Brighton the\r\nday before, not many hours after the express. Though Lydia’s short\r\nletter to Mrs. F. gave them to understand that they were going to Gretna\r\nGreen, something was dropped by Denny expressing his belief that W.\r\nnever intended to go there, or to marry Lydia at all, which was repeated\r\nto Colonel F., who, instantly taking the alarm, set off from B.,\r\nintending to trace their route. He did trace them easily to Clapham, but\r\nno farther; for on entering that place, they removed into a\r\nhackney-coach, and dismissed the chaise that brought them from Epsom.\r\nAll that is known after this is, that they were seen to continue the\r\nLondon road. I know not what to think. After making every possible\r\ninquiry on that side of London, Colonel F. came on into Hertfordshire,\r\nanxiously renewing them at all the turnpikes, and at the inns in Barnet\r\nand Hatfield, but without any success,--no such people had been seen to\r\npass through. With the kindest concern he came on to Longbourn, and\r\nbroke his apprehensions to us in a manner most creditable to his heart.\r\nI am sincerely grieved for him and Mrs. F.; but no one can throw any\r\nblame on them. Our distress, my dear Lizzy, is very great. My father and\r\nmother believe the worst, but I cannot think so ill of him. Many\r\ncircumstances might make it more eligible for them to be married\r\nprivately in town than to pursue their first plan; and even if _he_\r\ncould form such a design against a young woman of Lydia’s connections,\r\nwhich is not likely, can I suppose her so lost to everything?\r\nImpossible! I grieve to find, however, that Colonel F. is not disposed\r\nto depend upon their marriage: he shook his head when I expressed my\r\nhopes, and said he feared W. was not a man to be trusted. My poor mother\r\nis really ill, and keeps her room. Could she exert herself, it would be\r\nbetter, but this is not to be expected; and as to my father, I never in\r\nmy life saw him so affected. Poor Kitty has anger for having concealed\r\ntheir attachment; but as it was a matter of confidence, one cannot\r\nwonder. I am truly glad, dearest Lizzy, that you have been spared\r\nsomething of these distressing scenes; but now, as the first shock is\r\nover, shall I own that I long for your return? I am not so selfish,\r\nhowever, as to press for it, if inconvenient. Adieu! I take up my pen\r\nagain to do, what I have just told you I would not; but circumstances\r\nare such, that I cannot help earnestly begging you all to come here as\r\nsoon as possible. I know my dear uncle and aunt so well, that I am not\r\nafraid of requesting it, though I have still something more to ask of\r\nthe former. My father is going to London with Colonel Forster instantly,\r\nto try to discover her. What he means to do, I am sure I know not; but\r\nhis excessive distress will not allow him to pursue any measure in the\r\nbest and safest way, and Colonel Forster is obliged to be at Brighton\r\nagain to-morrow evening. In such an exigence my uncle’s advice and\r\nassistance would be everything in the world; he will immediately\r\ncomprehend what I must feel, and I rely upon his goodness.”\r\n\r\n“Oh! where, where is my uncle?” cried Elizabeth, darting from her seat\r\nas she finished the letter, in eagerness to follow him, without losing a\r\nmoment of the time so precious; but as she reached the door, it was\r\nopened by a servant, and Mr. Darcy appeared. Her pale face and\r\nimpetuous manner made him start, and before he could recover himself\r\nenough to speak, she, in whose mind every idea was superseded by Lydia’s\r\nsituation, hastily exclaimed, “I beg your pardon, but I must leave you.\r\nI must find Mr. Gardiner this moment on business that cannot be delayed;\r\nI have not an instant to lose.”\r\n\r\n“Good God! what is the matter?” cried he, with more feeling than\r\npoliteness; then recollecting himself, “I will not detain you a minute;\r\nbut let me, or let the servant, go after Mr. and Mrs. Gardiner. You are\r\nnot well enough; you cannot go yourself.”\r\n\r\nElizabeth hesitated; but her knees trembled under her, and she felt how\r\nlittle would be gained by her attempting to pursue them. Calling back\r\nthe servant, therefore, she commissioned him, though in so breathless an\r\naccent as made her almost unintelligible, to fetch his master and\r\nmistress home instantly.\r\n\r\nOn his quitting the room, she sat down, unable to support herself, and\r\nlooking so miserably ill, that it was impossible for Darcy to leave her,\r\nor to refrain from saying, in a tone of gentleness and commiseration,\r\n“Let me call your maid. Is there nothing you could take to give you\r\npresent relief? A glass of wine; shall I get you one? You are very ill.”\r\n\r\n“No, I thank you,” she replied, endeavouring to recover herself. “There\r\nis nothing the matter with me. I am quite well, I am only distressed by\r\nsome dreadful news which I have just received from Longbourn.”\r\n\r\nShe burst into tears as she alluded to it, and for a few minutes could\r\nnot speak another word. Darcy, in wretched suspense, could only say\r\nsomething indistinctly of his\r\n\r\n[Illustration:\r\n\r\n “I have not an instant to lose”\r\n]\r\n\r\nconcern, and observe her in compassionate silence. At length she spoke\r\nagain. “I have just had a letter from Jane, with such dreadful news. It\r\ncannot be concealed from anyone. My youngest sister has left all her\r\nfriends--has eloped; has thrown herself into the power of--of Mr.\r\nWickham. They are gone off together from Brighton. _You_ know him too\r\nwell to doubt the rest. She has no money, no connections, nothing that\r\ncan tempt him to--she is lost for ever.”\r\n\r\nDarcy was fixed in astonishment.\r\n\r\n“When I consider,” she added, in a yet more agitated voice, “that _I_\r\nmight have prevented it! _I_ who knew what he was. Had I but explained\r\nsome part of it only--some part of what I learnt, to my own family! Had\r\nhis character been known, this could not have happened. But it is all,\r\nall too late now.”\r\n\r\n“I am grieved, indeed,” cried Darcy: “grieved--shocked. But is it\r\ncertain, absolutely certain?”\r\n\r\n“Oh, yes! They left Brighton together on Sunday night, and were traced\r\nalmost to London, but not beyond: they are certainly not gone to\r\nScotland.”\r\n\r\n“And what has been done, what has been attempted, to recover her?”\r\n\r\n“My father has gone to London, and Jane has written to beg my uncle’s\r\nimmediate assistance, and we shall be off, I hope, in half an hour. But\r\nnothing can be done; I know very well that nothing can be done. How is\r\nsuch a man to be worked on? How are they even to be discovered? I have\r\nnot the smallest hope. It is every way horrible!”\r\n\r\nDarcy shook his head in silent acquiescence.\r\n\r\n“When _my_ eyes were opened to his real character, oh! had I known what\r\nI ought, what I dared to do! But I knew not--I was afraid of doing too\r\nmuch. Wretched, wretched mistake!”\r\n\r\nDarcy made no answer. He seemed scarcely to hear her, and was walking up\r\nand down the room in earnest meditation; his brow contracted, his air\r\ngloomy. Elizabeth soon observed, and instantly understood it. Her power\r\nwas sinking; everything _must_ sink under such a proof of family\r\nweakness, such an assurance of the deepest disgrace. She could neither\r\nwonder nor condemn; but the belief of his self-conquest brought nothing\r\nconsolatory to her bosom, afforded no palliation of her distress. It\r\nwas, on the contrary, exactly calculated to make her understand her own\r\nwishes; and never had she so honestly felt that she could have loved\r\nhim, as now, when all love must be vain.\r\n\r\nBut self, though it would intrude, could not engross her. Lydia--the\r\nhumiliation, the misery she was bringing on them all--soon swallowed up\r\nevery private care; and covering her face with her handkerchief,\r\nElizabeth was soon lost to everything else; and, after a pause of\r\nseveral minutes, was only recalled to a sense of her situation by the\r\nvoice of her companion, who, in a manner which, though it spoke\r\ncompassion, spoke likewise restraint, said,--\r\n\r\n“I am afraid you have been long desiring my absence, nor have I anything\r\nto plead in excuse of my stay, but real, though unavailing concern.\r\nWould to Heaven that anything could be either said or done on my part,\r\nthat might offer consolation to such distress! But I will not torment\r\nyou with vain wishes, which may seem purposely to ask for your thanks.\r\nThis unfortunate affair will, I fear, prevent my sister’s having the\r\npleasure of seeing you at Pemberley to-day.”\r\n\r\n“Oh, yes! Be so kind as to apologize for us to Miss Darcy. Say that\r\nurgent business calls us home immediately. Conceal the unhappy truth as\r\nlong as it is possible. I know it cannot be long.”\r\n\r\nHe readily assured her of his secrecy, again expressed his sorrow for\r\nher distress, wished it a happier conclusion than there was at present\r\nreason to hope, and, leaving his compliments for her relations, with\r\nonly one serious parting look, went away.\r\n\r\nAs he quitted the room, Elizabeth felt how improbable it was that they\r\nshould ever see each other again on such terms of cordiality as had\r\nmarked their several meetings in Derbyshire; and as she threw a\r\nretrospective glance over the whole of their acquaintance, so full of\r\ncontradictions and varieties, sighed at the perverseness of those\r\nfeelings which would now have promoted its continuance, and would\r\nformerly have rejoiced in its termination.\r\n\r\nIf gratitude and esteem are good foundations of affection, Elizabeth’s\r\nchange of sentiment will be neither improbable nor faulty. But if\r\notherwise, if the regard springing from such sources is unreasonable or\r\nunnatural, in comparison of what is so often described as arising on a\r\nfirst interview with its object, and even before two words have been\r\nexchanged, nothing can be said in her defence, except that she had given\r\nsomewhat of a trial to the latter method, in her partiality for Wickham,\r\nand that its ill success might, perhaps, authorize her to seek the other\r\nless interesting mode of attachment. Be that as it may, she saw him go\r\nwith regret; and in this early example of what Lydia’s infamy must\r\nproduce, found additional anguish as she reflected on that wretched\r\nbusiness. Never since reading Jane’s second letter had she entertained a\r\nhope of Wickham’s meaning to marry her. No one but Jane, she thought,\r\ncould flatter herself with such an expectation. Surprise was the least\r\nof all her feelings on this development. While the contents of the first\r\nletter remained on her mind, she was all surprise, all astonishment,\r\nthat Wickham should marry a girl whom it was impossible he could marry\r\nfor money; and how Lydia could ever have attached him had appeared\r\nincomprehensible. But now it was all too natural. For such an attachment\r\nas this, she might have sufficient charms; and though she did not\r\nsuppose Lydia to be deliberately engaging in an elopement, without the\r\nintention of marriage, she had no difficulty in believing that neither\r\nher virtue nor her understanding would preserve her from falling an easy\r\nprey.\r\n\r\nShe had never perceived, while the regiment was in Hertfordshire, that\r\nLydia had any partiality for him; but she was convinced that Lydia had\r\nwanted only encouragement to attach herself to anybody. Sometimes one\r\nofficer, sometimes another, had been her favourite, as their attentions\r\nraised them in her opinion. Her affections had been continually\r\nfluctuating, but never without an object. The mischief of neglect and\r\nmistaken indulgence towards such a girl--oh! how acutely did she now\r\nfeel it!\r\n\r\nShe was wild to be at home--to hear, to see, to be upon the spot to\r\nshare with Jane in the cares that must now fall wholly upon her, in a\r\nfamily so deranged; a father absent, a mother incapable of exertion, and\r\nrequiring constant attendance; and though almost persuaded that nothing\r\ncould be done for Lydia, her uncle’s interference seemed of the utmost\r\nimportance, and till he entered the room the misery of her impatience\r\nwas severe. Mr. and Mrs. Gardiner had hurried back in alarm, supposing,\r\nby the servant’s account, that their niece was taken suddenly ill; but\r\nsatisfying them instantly on that head, she eagerly communicated the\r\ncause of their summons, reading the two letters aloud, and dwelling on\r\nthe postscript of the last with trembling energy. Though Lydia had never\r\nbeen a favourite with them, Mr. and Mrs. Gardiner could not but be\r\ndeeply affected. Not Lydia only, but all were concerned in it; and after\r\nthe first exclamations of surprise and horror, Mr. Gardiner readily\r\npromised every assistance in his power. Elizabeth, though expecting no\r\nless, thanked him with tears of gratitude; and all three being actuated\r\nby one spirit, everything relating to their journey was speedily\r\nsettled. They were to be off as soon as possible. “But what is to be\r\ndone about Pemberley?” cried Mrs. Gardiner. “John told us Mr. Darcy was\r\nhere when you sent for us;--was it so?”\r\n\r\n“Yes; and I told him we should not be able to keep our engagement.\r\n_That_ is all settled.”\r\n\r\n“What is all settled?” repeated the other, as she ran into her room to\r\nprepare. “And are they upon such terms as for her to disclose the real\r\ntruth? Oh, that I knew how it was!”\r\n\r\nBut wishes were vain; or, at best, could serve only to amuse her in the\r\nhurry and confusion of the following hour. Had Elizabeth been at leisure\r\nto be idle, she would have remained certain that all employment was\r\nimpossible to one so wretched as herself; but she had her share of\r\nbusiness as well as her aunt, and amongst the rest there were notes to\r\nbe written to all their friends at Lambton, with false excuses for their\r\nsudden departure. An hour, however, saw the whole completed; and Mr.\r\nGardiner, meanwhile, having settled his account at the inn, nothing\r\nremained to be done but to go; and Elizabeth, after all the misery of\r\nthe morning, found herself, in a shorter space of time than she could\r\nhave supposed, seated in the carriage, and on the road to Longbourn.\r\n\r\n\r\n\r\n\r\n[Illustration:\r\n\r\n “The first pleasing earnest of their welcome”\r\n]\r\n\r\n\r\n\r\n\r\nCHAPTER XLVII.\r\n\r\n\r\n[Illustration]\r\n\r\n“I have been thinking it over again, Elizabeth,” said her uncle, as they\r\ndrove from the town; “and really, upon serious consideration, I am much\r\nmore inclined than I was to judge as your eldest sister does of the\r\nmatter. It appears to me so very unlikely that any young man should form\r\nsuch a design against a girl who is by no means unprotected or\r\nfriendless, and who was actually staying in his Colonel’s family, that I\r\nam strongly inclined to hope the best. Could he expect that her friends\r\nwould not step forward? Could he expect to be noticed again by the\r\nregiment, after such an affront to Colonel Forster? His temptation is\r\nnot adequate to the risk.”\r\n\r\n“Do you really think so?” cried Elizabeth, brightening up for a moment.\r\n\r\n“Upon my word,” said Mrs. Gardiner, “I begin to be of your uncle’s\r\nopinion. It is really too great a violation of decency, honour, and\r\ninterest, for him to be guilty of it. I cannot think so very ill of\r\nWickham. Can you, yourself, Lizzie, so wholly give him up, as to believe\r\nhim capable of it?”\r\n\r\n“Not perhaps of neglecting his own interest. But of every other neglect\r\nI can believe him capable. If, indeed, it should be so! But I dare not\r\nhope it. Why should they not go on to Scotland, if that had been the\r\ncase?”\r\n\r\n“In the first place,” replied Mr. Gardiner, “there is no absolute proof\r\nthat they are not gone to Scotland.”\r\n\r\n“Oh, but their removing from the chaise into a hackney coach is such a\r\npresumption! And, besides, no traces of them were to be found on the\r\nBarnet road.”\r\n\r\n“Well, then,--supposing them to be in London--they may be there, though\r\nfor the purpose of concealment, for no more exceptionable purpose. It is\r\nnot likely that money should be very abundant on either side; and it\r\nmight strike them that they could be more economically, though less\r\nexpeditiously, married in London, than in Scotland.”\r\n\r\n“But why all this secrecy? Why any fear of detection? Why must their\r\nmarriage be private? Oh, no, no--this is not likely. His most particular\r\nfriend, you see by Jane’s account, was persuaded of his never intending\r\nto marry her. Wickham will never marry a woman without some money. He\r\ncannot afford it. And what claims has Lydia, what attractions has she\r\nbeyond youth, health, and good humour, that could make him for her sake\r\nforego every chance of benefiting himself by marrying well? As to what\r\nrestraint the apprehensions of disgrace in the corps might throw on a\r\ndishonourable elopement with her, I am not able to judge; for I know\r\nnothing of the effects that such a step might produce. But as to your\r\nother objection, I am afraid it will hardly hold good. Lydia has no\r\nbrothers to step forward; and he might imagine, from my father’s\r\nbehaviour, from his indolence and the little attention he has ever\r\nseemed to give to what was going forward in his family, that _he_ would\r\ndo as little and think as little about it, as any father could do, in\r\nsuch a matter.”\r\n\r\n“But can you think that Lydia is so lost to everything but love of him,\r\nas to consent to live with him on any other terms than marriage?”\r\n\r\n“It does seem, and it is most shocking, indeed,” replied Elizabeth, with\r\ntears in her eyes, “that a sister’s sense of decency and virtue in such\r\na point should admit of doubt. But, really, I know not what to say.\r\nPerhaps I am not doing her justice. But she is very young: she has never\r\nbeen taught to think on serious subjects; and for the last half year,\r\nnay, for a twelvemonth, she has been given up to nothing but amusement\r\nand vanity. She has been allowed to dispose of her time in the most idle\r\nand frivolous manner, and to adopt any opinions that came in her way.\r\nSince the ----shire were first quartered in Meryton, nothing but love,\r\nflirtation, and officers, have been in her head. She has been doing\r\neverything in her power, by thinking and talking on the subject, to give\r\ngreater--what shall I call it?--susceptibility to her feelings; which\r\nare naturally lively enough. And we all know that Wickham has every\r\ncharm of person and address that can captivate a woman.”\r\n\r\n“But you see that Jane,” said her aunt, “does not think so ill of\r\nWickham, as to believe him capable of the attempt.”\r\n\r\n“Of whom does Jane ever think ill? And who is there, whatever might be\r\ntheir former conduct, that she would believe capable of such an attempt,\r\ntill it were proved against them? But Jane knows, as well as I do, what\r\nWickham really is. We both know that he has been profligate in every\r\nsense of the word; that he has neither integrity nor honour; that he is\r\nas false and deceitful as he is insinuating.”\r\n\r\n“And do you really know all this?” cried Mrs. Gardiner, whose curiosity\r\nas to the mode of her intelligence was all alive.\r\n\r\n“I do, indeed,” replied Elizabeth, colouring. “I told you the other day\r\nof his infamous behaviour to Mr. Darcy; and you, yourself, when last at\r\nLongbourn, heard in what manner he spoke of the man who had behaved with\r\nsuch forbearance and liberality towards him. And there are other\r\ncircumstances which I am not at liberty--which it is not worth while to\r\nrelate; but his lies about the whole Pemberley family are endless. From\r\nwhat he said of Miss Darcy, I was thoroughly prepared to see a proud,\r\nreserved, disagreeable girl. Yet he knew to the contrary himself. He\r\nmust know that she was as amiable and unpretending as we have found\r\nher.”\r\n\r\n“But does Lydia know nothing of this? can she be ignorant of what you\r\nand Jane seem so well to understand?”\r\n\r\n“Oh, yes!--that, that is the worst of all. Till I was in Kent, and saw\r\nso much both of Mr. Darcy and his relation Colonel Fitzwilliam, I was\r\nignorant of the truth myself. And when I returned home the ----shire\r\nwas to leave Meryton in a week or fortnight’s time. As that was the\r\ncase, neither Jane, to whom I related the whole, nor I, thought it\r\nnecessary to make our knowledge public; for of what use could it\r\napparently be to anyone, that the good opinion, which all the\r\nneighbourhood had of him, should then be overthrown? And even when it\r\nwas settled that Lydia should go with Mrs. Forster, the necessity of\r\nopening her eyes to his character never occurred to me. That _she_ could\r\nbe in any danger from the deception never entered my head. That such a\r\nconsequence as _this_ should ensue, you may easily believe was far\r\nenough from my thoughts.”\r\n\r\n“When they all removed to Brighton, therefore, you had no reason, I\r\nsuppose, to believe them fond of each other?”\r\n\r\n“Not the slightest. I can remember no symptom of affection on either\r\nside; and had anything of the kind been perceptible, you must be aware\r\nthat ours is not a family on which it could be thrown away. When first\r\nhe entered the corps, she was ready enough to admire him; but so we all\r\nwere. Every girl in or near Meryton was out of her senses about him for\r\nthe first two months: but he never distinguished _her_ by any particular\r\nattention; and, consequently, after a moderate period of extravagant and\r\nwild admiration, her fancy for him gave way, and others of the regiment,\r\nwho treated her with more distinction, again became her favourites.”\r\n\r\nIt may be easily believed, that however little of novelty could be added\r\nto their fears, hopes, and conjectures, on this interesting subject by\r\nits repeated discussion, no other could detain them from it long, during\r\nthe whole of the journey. From Elizabeth’s thoughts it was never absent.\r\nFixed there by the keenest of all anguish, self-reproach, she could\r\nfind no interval of ease or forgetfulness.\r\n\r\nThey travelled as expeditiously as possible; and sleeping one night on\r\nthe road, reached Longbourn by dinnertime the next day. It was a comfort\r\nto Elizabeth to consider that Jane could not have been wearied by long\r\nexpectations.\r\n\r\nThe little Gardiners, attracted by the sight of a chaise, were standing\r\non the steps of the house, as they entered the paddock; and when the\r\ncarriage drove up to the door, the joyful surprise that lighted up their\r\nfaces and displayed itself over their whole bodies, in a variety of\r\ncapers and frisks, was the first pleasing earnest of their welcome.\r\n\r\nElizabeth jumped out; and after giving each of them a hasty kiss,\r\nhurried into the vestibule, where Jane, who came running downstairs from\r\nher mother’s apartment, immediately met her.\r\n\r\nElizabeth, as she affectionately embraced her, whilst tears filled the\r\neyes of both, lost not a moment in asking whether anything had been\r\nheard of the fugitives.\r\n\r\n“Not yet,” replied Jane. “But now that my dear uncle is come, I hope\r\neverything will be well.”\r\n\r\n“Is my father in town?”\r\n\r\n“Yes, he went on Tuesday, as I wrote you word.”\r\n\r\n“And have you heard from him often?”\r\n\r\n“We have heard only once. He wrote me a few lines on Wednesday, to say\r\nthat he had arrived in safety, and to give me his directions, which I\r\nparticularly begged him to do. He merely added, that he should not write\r\nagain, till he had something of importance to mention.”\r\n\r\n“And my mother--how is she? How are you all?”\r\n\r\n“My mother is tolerably well, I trust; though her spirits are greatly\r\nshaken. She is upstairs, and will have great satisfaction in seeing you\r\nall. She does not yet leave her dressing-room. Mary and Kitty, thank\r\nHeaven! are quite well.”\r\n\r\n“But you--how are you?” cried Elizabeth. “You look pale. How much you\r\nmust have gone through!”\r\n\r\nHer sister, however, assured her of her being perfectly well; and their\r\nconversation, which had been passing while Mr. and Mrs. Gardiner were\r\nengaged with their children, was now put an end to by the approach of\r\nthe whole party. Jane ran to her uncle and aunt, and welcomed and\r\nthanked them both, with alternate smiles and tears.\r\n\r\nWhen they were all in the drawing-room, the questions which Elizabeth\r\nhad already asked were of course repeated by the others, and they soon\r\nfound that Jane had no intelligence to give. The sanguine hope of good,\r\nhowever, which the benevolence of her heart suggested, had not yet\r\ndeserted her; she still expected that it would all end well, and that\r\nevery morning would bring some letter, either from Lydia or her father,\r\nto explain their proceedings, and, perhaps, announce the marriage.\r\n\r\nMrs. Bennet, to whose apartment they all repaired, after a few minutes’\r\nconversation together, received them exactly as might be expected; with\r\ntears and lamentations of regret, invectives against the villainous\r\nconduct of Wickham, and complaints of her own sufferings and ill-usage;\r\nblaming everybody but the person to whose ill-judging indulgence the\r\nerrors of her daughter must be principally owing.\r\n\r\n“If I had been able,” said she, “to carry my point in going to Brighton\r\nwith all my family, _this_ would not have happened: but poor dear Lydia\r\nhad nobody to take care of her. Why did the Forsters ever let her go out\r\nof their sight? I am sure there was some great neglect or other on their\r\nside, for she is not the kind of girl to do such a thing, if she had\r\nbeen well looked after. I always thought they were very unfit to have\r\nthe charge of her; but I was over-ruled, as I always am. Poor, dear\r\nchild! And now here’s Mr. Bennet gone away, and I know he will fight\r\nWickham, wherever he meets him, and then he will be killed, and what is\r\nto become of us all? The Collinses will turn us out, before he is cold\r\nin his grave; and if you are not kind to us, brother, I do not know what\r\nwe shall do.”\r\n\r\nThey all exclaimed against such terrific ideas; and Mr. Gardiner, after\r\ngeneral assurances of his affection for her and all her family, told her\r\nthat he meant to be in London the very next day, and would assist Mr.\r\nBennet in every endeavour for recovering Lydia.\r\n\r\n“Do not give way to useless alarm,” added he: “though it is right to be\r\nprepared for the worst, there is no occasion to look on it as certain.\r\nIt is not quite a week since they left Brighton. In a few days more, we\r\nmay gain some news of them; and till we know that they are not married,\r\nand have no design of marrying, do not let us give the matter over as\r\nlost. As soon as I get to town, I shall go to my brother, and make him\r\ncome home with me to Gracechurch Street, and then we may consult\r\ntogether as to what is to be done.”\r\n\r\n“Oh, my dear brother,” replied Mrs. Bennet, “that is exactly what I\r\ncould most wish for. And now do, when you get to town, find them out,\r\nwherever they may be; and if they are not married already, _make_ them\r\nmarry. And as for wedding clothes, do not let them wait for that, but\r\ntell Lydia she shall have as much money as she chooses to buy them,\r\nafter they are married. And, above all things, keep Mr. Bennet from\r\nfighting. Tell him what a dreadful state I am in--that I am frightened\r\nout of my wits; and have such tremblings, such flutterings all over me,\r\nsuch spasms in my side, and pains in my head, and such beatings at my\r\nheart, that I can get no rest by night nor by day. And tell my dear\r\nLydia not to give any directions about her clothes till she has seen me,\r\nfor she does not know which are the best warehouses. Oh, brother, how\r\nkind you are! I know you will contrive it all.”\r\n\r\nBut Mr. Gardiner, though he assured her again of his earnest endeavours\r\nin the cause, could not avoid recommending moderation to her, as well in\r\nher hopes as her fears; and after talking with her in this manner till\r\ndinner was on table, they left her to vent all her feelings on the\r\nhousekeeper, who attended in the absence of her daughters.\r\n\r\nThough her brother and sister were persuaded that there was no real\r\noccasion for such a seclusion from the family, they did not attempt to\r\noppose it; for they knew that she had not prudence enough to hold her\r\ntongue before the servants, while they waited at table, and judged it\r\nbetter that _one_ only of the household, and the one whom they could\r\nmost trust, should comprehend all her fears and solicitude on the\r\nsubject.\r\n\r\nIn the dining-room they were soon joined by Mary and Kitty, who had been\r\ntoo busily engaged in their separate apartments to make their appearance\r\nbefore. One came from her books, and the other from her toilette. The\r\nfaces of both, however, were tolerably calm; and no change was visible\r\nin either, except that the loss of her favourite sister, or the anger\r\nwhich she had herself incurred in the business, had given something more\r\nof fretfulness than usual to the accents of Kitty. As for Mary, she was\r\nmistress enough of herself to whisper to Elizabeth, with a countenance\r\nof grave reflection, soon after they were seated at table,--\r\n\r\n“This is a most unfortunate affair, and will probably be much talked of.\r\nBut we must stem the tide of malice, and pour into the wounded bosoms of\r\neach other the balm of sisterly consolation.”\r\n\r\nThen perceiving in Elizabeth no inclination of replying, she added,\r\n“Unhappy as the event must be for Lydia, we may draw from it this useful\r\nlesson:--that loss of virtue in a female is irretrievable, that one\r\nfalse step involves her in endless ruin, that her reputation is no less\r\nbrittle than it is beautiful, and that she cannot be too much guarded in\r\nher behaviour towards the undeserving of the other sex.”\r\n\r\nElizabeth lifted up her eyes in amazement, but was too much oppressed to\r\nmake any reply. Mary, however, continued to console herself with such\r\nkind of moral extractions from the evil before them.\r\n\r\nIn the afternoon, the two elder Miss Bennets were able to be for half an\r\nhour by themselves; and Elizabeth instantly availed herself of the\r\nopportunity of making any inquiries which Jane was equally eager to\r\nsatisfy. After joining in general lamentations over the dreadful sequel\r\nof this event, which Elizabeth considered as all but certain, and Miss\r\nBennet could not assert to be wholly impossible, the former continued\r\nthe subject by saying, “But tell me all and everything about it which I\r\nhave not already heard. Give me further particulars. What did Colonel\r\nForster say? Had they no apprehension of anything before the elopement\r\ntook place? They must have seen them together for ever.”\r\n\r\n“Colonel Forster did own that he had often suspected some partiality,\r\nespecially on Lydia’s side, but nothing to give him any alarm. I am so\r\ngrieved for him. His behaviour was attentive and kind to the utmost. He\r\n_was_ coming to us, in order to assure us of his concern, before he had\r\nany idea of their not being gone to Scotland: when that apprehension\r\nfirst got abroad, it hastened his journey.”\r\n\r\n“And was Denny convinced that Wickham would not marry? Did he know of\r\ntheir intending to go off? Had Colonel Forster seen Denny himself?”\r\n\r\n“Yes; but when questioned by _him_, Denny denied knowing anything of\r\ntheir plan, and would not give his real opinion about it. He did not\r\nrepeat his persuasion of their not marrying, and from _that_ I am\r\ninclined to hope he might have been misunderstood before.”\r\n\r\n“And till Colonel Forster came himself, not one of you entertained a\r\ndoubt, I suppose, of their being really married?”\r\n\r\n“How was it possible that such an idea should enter our brains? I felt a\r\nlittle uneasy--a little fearful of my sister’s happiness with him in\r\nmarriage, because I knew that his conduct had not been always quite\r\nright. My father and mother knew nothing of that; they only felt how\r\nimprudent a match it must be. Kitty then owned, with a very natural\r\ntriumph on knowing more than the rest of us, that in Lydia’s last letter\r\nshe had prepared her for such a step. She had known, it seems, of their\r\nbeing in love with each other many weeks.”\r\n\r\n“But not before they went to Brighton?”\r\n\r\n“No, I believe not.”\r\n\r\n“And did Colonel Forster appear to think ill of Wickham himself? Does he\r\nknow his real character?”\r\n\r\n“I must confess that he did not speak so well of Wickham as he formerly\r\ndid. He believed him to be imprudent and extravagant; and since this sad\r\naffair has taken place, it is said that he left Meryton greatly in debt:\r\nbut I hope this may be false.”\r\n\r\n“Oh, Jane, had we been less secret, had we told what we knew of him,\r\nthis could not have happened!”\r\n\r\n“Perhaps it would have been better,” replied her sister.\r\n\r\n“But to expose the former faults of any person, without knowing what\r\ntheir present feelings were, seemed unjustifiable.”\r\n\r\n“We acted with the best intentions.”\r\n\r\n“Could Colonel Forster repeat the particulars of Lydia’s note to his\r\nwife?”\r\n\r\n“He brought it with him for us to see.”\r\n\r\nJane then took it from her pocket-book, and gave it to Elizabeth. These\r\nwere the contents:--\r\n\r\n /* NIND “My dear Harriet, */\r\n\r\n “You will laugh when you know where I am gone, and I cannot help\r\n laughing myself at your surprise to-morrow morning, as soon as I am\r\n missed. I am going to Gretna Green, and if you cannot guess with\r\n who, I shall think you a simpleton, for there is but one man in the\r\n world I love, and he is an angel. I should never be happy without\r\n him, so think it no harm to be off. You need not send them word at\r\n Longbourn of my going, if you do not like it, for it will make the\r\n surprise the greater when I write to them, and sign my name Lydia\r\n Wickham. What a good joke it will be! I can hardly write for\r\n laughing. Pray make my excuses to Pratt for not keeping my\r\n engagement, and dancing with him to-night. Tell him I hope he will\r\n excuse me when he knows all, and tell him I will dance with him at\r\n the next ball we meet with great pleasure. I shall send for my\r\n clothes when I get to Longbourn; but I wish you would tell Sally to\r\n mend a great slit in my worked muslin gown before they are packed\r\n up. Good-bye. Give my love to Colonel Forster. I hope you will\r\n drink to our good journey.\r\n\r\n“Your affectionate friend,\r\n\r\n“LYDIA BENNET.”\r\n\r\n\r\n“Oh, thoughtless, thoughtless Lydia!” cried Elizabeth when she had\r\nfinished it. “What a letter is this, to be written at such a moment! But\r\nat least it shows that _she_ was serious in the object of her journey.\r\nWhatever he might afterwards persuade her to, it was not on her side a\r\n_scheme_ of infamy. My poor father! how he must have felt it!”\r\n\r\n“I never saw anyone so shocked. He could not speak a word for full ten\r\nminutes. My mother was taken ill immediately, and the whole house in\r\nsuch confusion!”\r\n\r\n“Oh, Jane,” cried Elizabeth, “was there a servant belonging to it who\r\ndid not know the whole story before the end of the day?”\r\n\r\n“I do not know: I hope there was. But to be guarded at such a time is\r\nvery difficult. My mother was in hysterics; and though I endeavoured to\r\ngive her every assistance in my power, I am afraid I did not do so much\r\nas I might have done. But the horror of what might possibly happen\r\nalmost took from me my faculties.”\r\n\r\n“Your attendance upon her has been too much for you. You do not look\r\nwell. Oh that I had been with you! you have had every care and anxiety\r\nupon yourself alone.”\r\n\r\n“Mary and Kitty have been very kind, and would have shared in every\r\nfatigue, I am sure, but I did not think it right for either of them.\r\nKitty is slight and delicate, and Mary studies so much that her hours of\r\nrepose should not be broken in on. My aunt Philips came to Longbourn on\r\nTuesday, after my father went away; and was so good as to stay till\r\nThursday with me. She was of great use and comfort to us all, and Lady\r\nLucas has been very kind: she walked here on Wednesday morning to\r\ncondole with us, and offered her services, or any of her daughters, if\r\nthey could be of use to us.”\r\n\r\n“She had better have stayed at home,” cried Elizabeth: “perhaps she\r\n_meant_ well, but, under such a misfortune as this, one cannot see too\r\nlittle of one’s neighbours. Assistance is impossible; condolence,\r\ninsufferable. Let them triumph over us at a distance, and be satisfied.”\r\n\r\nShe then proceeded to inquire into the measures which her father had\r\nintended to pursue, while in town, for the recovery of his daughter.\r\n\r\n“He meant, I believe,” replied Jane, “to go to Epsom, the place where\r\nthey last changed horses, see the postilions, and try if anything could\r\nbe made out from them. His principal object must be to discover the\r\nnumber of the hackney coach which took them from Clapham. It had come\r\nwith a fare from London; and as he thought the circumstance of a\r\ngentleman and lady’s removing from one carriage into another might be\r\nremarked, he meant to make inquiries at Clapham. If he could anyhow\r\ndiscover at what house the coachman had before set down his fare, he\r\ndetermined to make inquiries there, and hoped it might not be impossible\r\nto find out the stand and number of the coach. I do not know of any\r\nother designs that he had formed; but he was in such a hurry to be gone,\r\nand his spirits so greatly discomposed, that I had difficulty in finding\r\nout even so much as this.”\r\n\r\n\r\n\r\n\r\n[Illustration:\r\n\r\n The Post\r\n]\r\n\r\n\r\n\r\n\r\nCHAPTER XLVIII.\r\n\r\n\r\n[Illustration]\r\n\r\nThe whole party were in hopes of a letter from Mr. Bennet the next\r\nmorning, but the post came in without bringing a single line from him.\r\nHis family knew him to be, on all common occasions, a most negligent and\r\ndilatory correspondent; but at such a time they had hoped for exertion.\r\nThey were forced to conclude, that he had no pleasing intelligence to\r\nsend; but even of _that_ they would have been glad to be certain. Mr.\r\nGardiner had waited only for the letters before he set off.\r\n\r\nWhen he was gone, they were certain at least of receiving constant\r\ninformation of what was going on; and their uncle promised, at parting,\r\nto prevail on Mr. Bennet to return to Longbourn as soon as he could, to\r\nthe great consolation of his sister, who considered it as the only\r\nsecurity for her husband’s not being killed in a duel.\r\n\r\nMrs. Gardiner and the children were to remain in Hertfordshire a few\r\ndays longer, as the former thought her presence might be serviceable to\r\nher nieces. She shared in their attendance on Mrs. Bennet, and was a\r\ngreat comfort to them in their hours of freedom. Their other aunt also\r\nvisited them frequently, and always, as she said, with the design of\r\ncheering and heartening them up--though, as she never came without\r\nreporting some fresh instance of Wickham’s extravagance or irregularity,\r\nshe seldom went away without leaving them more dispirited than she found\r\nthem.\r\n\r\nAll Meryton seemed striving to blacken the man who, but three months\r\nbefore, had been almost an angel of light. He was declared to be in debt\r\nto every tradesman in the place, and his intrigues, all honoured with\r\nthe title of seduction, had been extended into every tradesman’s family.\r\nEverybody declared that he was the wickedest young man in the world; and\r\neverybody began to find out that they had always distrusted the\r\nappearance of his goodness. Elizabeth, though she did not credit above\r\nhalf of what was said, believed enough to make her former assurance of\r\nher sister’s ruin still more certain; and even Jane, who believed still\r\nless of it, became almost hopeless, more especially as the time was now\r\ncome, when, if they had gone to Scotland, which she had never before\r\nentirely despaired of, they must in all probability have gained some\r\nnews of them.\r\n\r\nMr. Gardiner left Longbourn on Sunday; on Tuesday, his wife received a\r\nletter from him: it told them, that on his arrival he had immediately\r\nfound out his brother, and persuaded him to come to Gracechurch Street.\r\nThat Mr. Bennet had been to Epsom and Clapham, before his arrival, but\r\nwithout gaining any satisfactory information; and that he was now\r\ndetermined to inquire at all the principal hotels in town, as Mr. Bennet\r\nthought it possible they might have gone to one of them, on their first\r\ncoming to London, before they procured lodgings. Mr. Gardiner himself\r\ndid not expect any success from this measure; but as his brother was\r\neager in it, he meant to assist him in pursuing it. He added, that Mr.\r\nBennet seemed wholly disinclined at present to leave London, and\r\npromised to write again very soon. There was also a postscript to this\r\neffect:--\r\n\r\n“I have written to Colonel Forster to desire him to find out, if\r\npossible, from some of the young man’s intimates in the regiment,\r\nwhether Wickham has any relations or connections who would be likely to\r\nknow in what part of the town he has now concealed himself. If there\r\nwere anyone that one could apply to, with a probability of gaining such\r\na clue as that, it might be of essential consequence. At present we have\r\nnothing to guide us. Colonel Forster will, I dare say, do everything in\r\nhis power to satisfy us on this head. But, on second thoughts, perhaps\r\nLizzy could tell us what relations he has now living better than any\r\nother person.”\r\n\r\nElizabeth was at no loss to understand from whence this deference for\r\nher authority proceeded; but it was not in her power to give any\r\ninformation of so satisfactory a nature as the compliment deserved.\r\n\r\nShe had never heard of his having had any relations, except a father\r\nand mother, both of whom had been dead many years. It was possible,\r\nhowever, that some of his companions in the ----shire might be able to\r\ngive more information; and though she was not very sanguine in expecting\r\nit, the application was a something to look forward to.\r\n\r\nEvery day at Longbourn was now a day of anxiety; but the most anxious\r\npart of each was when the post was expected. The arrival of letters was\r\nthe first grand object of every morning’s impatience. Through letters,\r\nwhatever of good or bad was to be told would be communicated; and every\r\nsucceeding day was expected to bring some news of importance.\r\n\r\nBut before they heard again from Mr. Gardiner, a letter arrived for\r\ntheir father, from a different quarter, from Mr. Collins; which, as Jane\r\nhad received directions to open all that came for him in his absence,\r\nshe accordingly read; and Elizabeth, who knew what curiosities his\r\nletters always were, looked over her, and read it likewise. It was as\r\nfollows:--\r\n\r\n /* “My dear Sir, */\r\n\r\n “I feel myself called upon, by our relationship, and my situation\r\n in life, to condole with you on the grievous affliction you are now\r\n suffering under, of which we were yesterday informed by a letter\r\n from Hertfordshire. Be assured, my dear sir, that Mrs. Collins and\r\n myself sincerely sympathize with you, and all your respectable\r\n family, in your present distress, which must be of the bitterest\r\n kind, because proceeding from a cause which no time can remove. No\r\n arguments shall be wanting on my part, that can alleviate so severe\r\n a misfortune; or that may comfort you, under a circumstance that\r\n must be, of all others, most afflicting to a parent’s mind. The\r\n death of your daughter would have been a blessing in comparison of\r\n this. And it is the more to be lamented, because there is reason to\r\n suppose, as my dear Charlotte informs me, that this licentiousness\r\n of behaviour in your\r\n\r\n [Illustration:\r\n\r\n“To whom I have related the affair”\r\n\r\n [_Copyright 1894 by George Allen._]]\r\n\r\n daughter has proceeded from a faulty degree of indulgence; though,\r\n at the same time, for the consolation of yourself and Mrs. Bennet,\r\n I am inclined to think that her own disposition must be naturally\r\n bad, or she could not be guilty of such an enormity, at so early an\r\n age. Howsoever that may be, you are grievously to be pitied; in\r\n which opinion I am not only joined by Mrs. Collins, but likewise by\r\n Lady Catherine and her daughter, to whom I have related the affair.\r\n They agree with me in apprehending that this false step in one\r\n daughter will be injurious to the fortunes of all the others: for\r\n who, as Lady Catherine herself condescendingly says, will connect\r\n themselves with such a family? And this consideration leads me,\r\n moreover, to reflect, with augmented satisfaction, on a certain\r\n event of last November; for had it been otherwise, I must have been\r\n involved in all your sorrow and disgrace. Let me advise you, then,\r\n my dear sir, to console yourself as much as possible, to throw off\r\n your unworthy child from your affection for ever, and leave her to\r\n reap the fruits of her own heinous offence.\r\n\r\n“I am, dear sir,” etc., etc.\r\n\r\nMr. Gardiner did not write again, till he had received an answer from\r\nColonel Forster; and then he had nothing of a pleasant nature to send.\r\nIt was not known that Wickham had a single relation with whom he kept up\r\nany connection, and it was certain that he had no near one living. His\r\nformer acquaintance had been numerous; but since he had been in the\r\nmilitia, it did not appear that he was on terms of particular friendship\r\nwith any of them. There was no one, therefore, who could be pointed out\r\nas likely to give any news of him. And in the wretched state of his own\r\nfinances, there was a very powerful motive for secrecy, in addition to\r\nhis fear of discovery by Lydia’s relations; for it had just transpired\r\nthat he had left gaming debts behind him to a very considerable amount.\r\nColonel Forster believed that more than a thousand pounds would be\r\nnecessary to clear his expenses at Brighton. He owed a good deal in the\r\ntown, but his debts of honour were still more formidable. Mr. Gardiner\r\ndid not attempt to conceal these particulars from the Longbourn family;\r\nJane heard them with horror. “A gamester!” she cried. “This is wholly\r\nunexpected; I had not an idea of it.”\r\n\r\nMr. Gardiner added, in his letter, that they might expect to see their\r\nfather at home on the following day, which was Saturday. Rendered\r\nspiritless by the ill success of all their endeavours, he had yielded to\r\nhis brother-in-law’s entreaty that he would return to his family and\r\nleave it to him to do whatever occasion might suggest to be advisable\r\nfor continuing their pursuit. When Mrs. Bennet was told of this, she did\r\nnot express so much satisfaction as her children expected, considering\r\nwhat her anxiety for his life had been before.\r\n\r\n“What! is he coming home, and without poor Lydia?” she cried. “Sure he\r\nwill not leave London before he has found them. Who is to fight Wickham,\r\nand make him marry her, if he comes away?”\r\n\r\nAs Mrs. Gardiner began to wish to be at home, it was settled that she\r\nand her children should go to London at the same time that Mr. Bennet\r\ncame from it. The coach, therefore, took them the first stage of their\r\njourney, and brought its master back to Longbourn.\r\n\r\nMrs. Gardiner went away in all the perplexity about Elizabeth and her\r\nDerbyshire friend, that had attended her from that part of the world.\r\nHis name had never been voluntarily mentioned before them by her niece;\r\nand the kind of half-expectation which Mrs. Gardiner had formed, of\r\ntheir being followed by a letter from him, had ended in nothing.\r\nElizabeth had received none since her return, that could come from\r\nPemberley.\r\n\r\nThe present unhappy state of the family rendered any other excuse for\r\nthe lowness of her spirits unnecessary; nothing, therefore, could be\r\nfairly conjectured from _that_,--though Elizabeth, who was by this time\r\ntolerably well acquainted with her own feelings, was perfectly aware\r\nthat, had she known nothing of Darcy, she could have borne the dread of\r\nLydia’s infamy somewhat better. It would have spared her, she thought,\r\none sleepless night out of two.\r\n\r\nWhen Mr. Bennet arrived, he had all the appearance of his usual\r\nphilosophic composure. He said as little as he had ever been in the\r\nhabit of saying; made no mention of the business that had taken him\r\naway; and it was some time before his daughters had courage to speak of\r\nit.\r\n\r\nIt was not till the afternoon, when he joined them at tea, that\r\nElizabeth ventured to introduce the subject; and then, on her briefly\r\nexpressing her sorrow for what he must have endured, he replied, “Say\r\nnothing of that. Who should suffer but myself? It has been my own doing,\r\nand I ought to feel it.”\r\n\r\n“You must not be too severe upon yourself,” replied Elizabeth.\r\n\r\n“You may well warn me against such an evil. Human nature is so prone to\r\nfall into it! No, Lizzy, let me once in my life feel how much I have\r\nbeen to blame. I am not afraid of being overpowered by the impression.\r\nIt will pass away soon enough.”\r\n\r\n“Do you suppose them to be in London?”\r\n\r\n“Yes; where else can they be so well concealed?”\r\n\r\n“And Lydia used to want to go to London,” added Kitty.\r\n\r\n“She is happy, then,” said her father, drily; “and her residence there\r\nwill probably be of some duration.”\r\n\r\nThen, after a short silence, he continued, “Lizzy, I bear you no\r\nill-will for being justified in your advice to me last May, which,\r\nconsidering the event, shows some greatness of mind.”\r\n\r\nThey were interrupted by Miss Bennet, who came to fetch her mother’s\r\ntea.\r\n\r\n“This is a parade,” cried he, “which does one good; it gives such an\r\nelegance to misfortune! Another day I will do the same; I will sit in my\r\nlibrary, in my nightcap and powdering gown, and give as much trouble as\r\nI can,--or perhaps I may defer it till Kitty runs away.”\r\n\r\n“I am not going to run away, papa,” said Kitty, fretfully. “If _I_\r\nshould ever go to Brighton, I would behave better than Lydia.”\r\n\r\n“_You_ go to Brighton! I would not trust you so near it as Eastbourne,\r\nfor fifty pounds! No, Kitty, I have at least learnt to be cautious, and\r\nyou will feel the effects of it. No officer is ever to enter my house\r\nagain, nor even to pass through the village. Balls will be absolutely\r\nprohibited, unless you stand up with one of your sisters. And you are\r\nnever to stir out of doors, till you can prove that you have spent ten\r\nminutes of every day in a rational manner.”\r\n\r\nKitty, who took all these threats in a serious light, began to cry.\r\n\r\n“Well, well,” said he, “do not make yourself unhappy. If you are a good\r\ngirl for the next ten years, I will take you to a review at the end of\r\nthem.”\r\n\r\n\r\n\r\n\r\n[Illustration]\r\n\r\n\r\n\r\n\r\nCHAPTER XLIX.\r\n\r\n\r\n[Illustration]\r\n\r\nTwo days after Mr. Bennet’s return, as Jane and Elizabeth were walking\r\ntogether in the shrubbery behind the house, they saw the housekeeper\r\ncoming towards them, and concluding that she came to call them to their\r\nmother, went forward to meet her; but instead of the expected summons,\r\nwhen they approached her, she said to Miss Bennet, “I beg your pardon,\r\nmadam, for interrupting you, but I was in hopes you might have got some\r\ngood news from town, so I took the liberty of coming to ask.”\r\n\r\n“What do you mean, Hill? We have heard nothing from town.”\r\n\r\n“Dear madam,” cried Mrs. Hill, in great astonishment, “don’t you know\r\nthere is an express come for master from Mr. Gardiner? He has been here\r\nthis half hour, and master has had a letter.”\r\n\r\nAway ran the girls, too eager to get in to have time for speech. They\r\nran through the vestibule into the breakfast-room; from thence to the\r\nlibrary;--their father was in neither; and they were on the point of\r\nseeking him upstairs with their mother, when they were met by the\r\nbutler, who said,--\r\n\r\n“If you are looking for my master, ma’am, he is walking towards the\r\nlittle copse.”\r\n\r\nUpon this information, they instantly passed through the hall once more,\r\nand ran across the lawn after their father, who was deliberately\r\npursuing his way towards a small wood on one side of the paddock.\r\n\r\nJane, who was not so light, nor so much in the habit of running as\r\nElizabeth, soon lagged behind, while her sister, panting for breath,\r\ncame up with him, and eagerly cried out,--\r\n\r\n“Oh, papa, what news? what news? have you heard from my uncle?”\r\n\r\n“Yes, I have had a letter from him by express.”\r\n\r\n“Well, and what news does it bring--good or bad?”\r\n\r\n“What is there of good to be expected?” said he, taking the letter from\r\nhis pocket; “but perhaps you would like to read it.”\r\n\r\nElizabeth impatiently caught it from his hand. Jane now came up.\r\n\r\n“Read it aloud,” said their father, “for I hardly know myself what it is\r\nabout.”\r\n\r\n /* RIGHT “Gracechurch Street, _Monday, August 2_. */\r\n\r\n“My dear Brother,\r\n\r\n “At last I am able to send you some tidings of my niece, and such\r\n as, upon the whole, I hope will give you satisfaction. Soon after\r\n you left me on Saturday, I was fortunate enough to find out in what\r\n part of London they were. The particulars I reserve till we meet.\r\n It is enough to know they are discovered: I have seen them\r\n both----”\r\n\r\n [Illustration:\r\n\r\n“But perhaps you would like to read it”\r\n\r\n [_Copyright 1894 by George Allen._]]\r\n\r\n “Then it is as I always hoped,” cried Jane: “they are married!”\r\n\r\n Elizabeth read on: “I have seen them both. They are not married,\r\n nor can I find there was any intention of being so; but if you are\r\n willing to perform the engagements which I have ventured to make on\r\n your side, I hope it will not be long before they are. All that is\r\n required of you is, to assure to your daughter, by settlement, her\r\n equal share of the five thousand pounds, secured among your\r\n children after the decease of yourself and my sister; and,\r\n moreover, to enter into an engagement of allowing her, during your\r\n life, one hundred pounds per annum. These are conditions which,\r\n considering everything, I had no hesitation in complying with, as\r\n far as I thought myself privileged, for you. I shall send this by\r\n express, that no time may be lost in bringing me your answer. You\r\n will easily comprehend, from these particulars, that Mr. Wickham’s\r\n circumstances are not so hopeless as they are generally believed to\r\n be. The world has been deceived in that respect; and I am happy to\r\n say, there will be some little money, even when all his debts are\r\n discharged, to settle on my niece, in addition to her own fortune.\r\n If, as I conclude will be the case, you send me full powers to act\r\n in your name throughout the whole of this business, I will\r\n immediately give directions to Haggerston for preparing a proper\r\n settlement. There will not be the smallest occasion for your coming\r\n to town again; therefore stay quietly at Longbourn, and depend on\r\n my diligence and care. Send back your answer as soon as you can,\r\n and be careful to write explicitly. We have judged it best that my\r\n niece should be married from this house, of which I hope you will\r\n approve. She comes to us to-day. I shall write again as soon as\r\n anything more is determined on. Yours, etc.\r\n\r\n“EDW. GARDINER.”\r\n\r\n“Is it possible?” cried Elizabeth, when she had finished. “Can it be\r\npossible that he will marry her?”\r\n\r\n“Wickham is not so undeserving, then, as we have thought him,” said her\r\nsister. “My dear father, I congratulate you.”\r\n\r\n“And have you answered the letter?” said Elizabeth.\r\n\r\n“No; but it must be done soon.”\r\n\r\nMost earnestly did she then entreat him to lose no more time before he\r\nwrote.\r\n\r\n“Oh! my dear father,” she cried, “come back and write immediately.\r\nConsider how important every moment is in such a case.”\r\n\r\n“Let me write for you,” said Jane, “if you dislike the trouble\r\nyourself.”\r\n\r\n“I dislike it very much,” he replied; “but it must be done.”\r\n\r\nAnd so saying, he turned back with them, and walked towards the house.\r\n\r\n“And--may I ask?” said Elizabeth; “but the terms, I suppose, must be\r\ncomplied with.”\r\n\r\n“Complied with! I am only ashamed of his asking so little.”\r\n\r\n“And they _must_ marry! Yet he is _such_ a man.”\r\n\r\n“Yes, yes, they must marry. There is nothing else to be done. But there\r\nare two things that I want very much to know:--one is, how much money\r\nyour uncle has laid down to bring it about; and the other, how I am ever\r\nto pay him.”\r\n\r\n“Money! my uncle!” cried Jane, “what do you mean, sir?”\r\n\r\n“I mean that no man in his proper senses would marry Lydia on so slight\r\na temptation as one hundred a year during my life, and fifty after I am\r\ngone.”\r\n\r\n“That is very true,” said Elizabeth; “though it had not occurred to me\r\nbefore. His debts to be discharged, and something still to remain! Oh,\r\nit must be my uncle’s doings! Generous, good man, I am afraid he has\r\ndistressed himself. A small sum could not do all this.”\r\n\r\n“No,” said her father. “Wickham’s a fool if he takes her with a farthing\r\nless than ten thousand pounds: I should be sorry to think so ill of him,\r\nin the very beginning of our relationship.”\r\n\r\n“Ten thousand pounds! Heaven forbid! How is half such a sum to be\r\nrepaid?”\r\n\r\nMr. Bennet made no answer; and each of them, deep in thought, continued\r\nsilent till they reached the house. Their father then went to the\r\nlibrary to write, and the girls walked into the breakfast-room.\r\n\r\n“And they are really to be married!” cried Elizabeth, as soon as they\r\nwere by themselves. “How strange this is! and for _this_ we are to be\r\nthankful. That they should marry, small as is their chance of happiness,\r\nand wretched as is his character, we are forced to rejoice! Oh, Lydia!”\r\n\r\n“I comfort myself with thinking,” replied Jane, “that he certainly would\r\nnot marry Lydia, if he had not a real regard for her. Though our kind\r\nuncle has done something towards clearing him, I cannot believe that ten\r\nthousand pounds, or anything like it, has been advanced. He has children\r\nof his own, and may have more. How could he spare half ten thousand\r\npounds?”\r\n\r\n“If we are ever able to learn what Wickham’s debts have been,” said\r\nElizabeth, “and how much is settled on his side on our sister, we shall\r\nexactly know what Mr. Gardiner has done for them, because Wickham has\r\nnot sixpence of his own. The kindness of my uncle and aunt can never be\r\nrequited. Their taking her home, and affording her their personal\r\nprotection and countenance, is such a sacrifice to her advantage as\r\nyears of gratitude cannot enough acknowledge. By this time she is\r\nactually with them! If such goodness does not make her miserable now,\r\nshe will never deserve to be happy! What a meeting for her, when she\r\nfirst sees my aunt!”\r\n\r\n“We must endeavour to forget all that has passed on either side,” said\r\nJane: “I hope and trust they will yet be happy. His consenting to marry\r\nher is a proof, I will believe, that he is come to a right way of\r\nthinking. Their mutual affection will steady them; and I flatter myself\r\nthey will settle so quietly, and live in so rational a manner, as may in\r\ntime make their past imprudence forgotten.”\r\n\r\n“Their conduct has been such,” replied Elizabeth, “as neither you, nor\r\nI, nor anybody, can ever forget. It is useless to talk of it.”\r\n\r\nIt now occurred to the girls that their mother was in all likelihood\r\nperfectly ignorant of what had happened. They went to the library,\r\ntherefore, and asked their father whether he would not wish them to make\r\nit known to her. He was writing, and, without raising his head, coolly\r\nreplied,--\r\n\r\n“Just as you please.”\r\n\r\n“May we take my uncle’s letter to read to her?”\r\n\r\n“Take whatever you like, and get away.”\r\n\r\nElizabeth took the letter from his writing-table, and they went upstairs\r\ntogether. Mary and Kitty were both with Mrs. Bennet: one communication\r\nwould, therefore, do for all. After a slight preparation for good news,\r\nthe letter was read aloud. Mrs. Bennet could hardly contain herself. As\r\nsoon as Jane had read Mr. Gardiner’s hope of Lydia’s being soon married,\r\nher joy burst forth, and every following sentence added to its\r\nexuberance. She was now in an irritation as violent from delight as she\r\nhad ever been fidgety from alarm and vexation. To know that her daughter\r\nwould be married was enough. She was disturbed by no fear for her\r\nfelicity, nor humbled by any remembrance of her misconduct.\r\n\r\n“My dear, dear Lydia!” she cried: “this is delightful indeed! She will\r\nbe married! I shall see her again! She will be married at sixteen! My\r\ngood, kind brother! I knew how it would be--I knew he would manage\r\neverything. How I long to see her! and to see dear Wickham too! But the\r\nclothes, the wedding clothes! I will write to my sister Gardiner about\r\nthem directly. Lizzy, my dear, run down to your father, and ask him how\r\nmuch he will give her. Stay, stay, I will go myself. Ring the bell,\r\nKitty, for Hill. I will put on my things in a moment. My dear, dear\r\nLydia! How merry we shall be together when we meet!”\r\n\r\nHer eldest daughter endeavoured to give some relief to the violence of\r\nthese transports, by leading her thoughts to the obligations which Mr.\r\nGardiner’s behaviour laid them all under.\r\n\r\n“For we must attribute this happy conclusion,” she added, “in a great\r\nmeasure to his kindness. We are persuaded that he has pledged himself to\r\nassist Mr. Wickham with money.”\r\n\r\n“Well,” cried her mother, “it is all very right; who should do it but\r\nher own uncle? If he had not had a family of his own, I and my children\r\nmust have had all his money, you know; and it is the first time we have\r\never had anything from him except a few presents. Well! I am so happy.\r\nIn a short time, I shall have a daughter married. Mrs. Wickham! How well\r\nit sounds! And she was only sixteen last June. My dear Jane, I am in\r\nsuch a flutter, that I am sure I can’t write; so I will dictate, and you\r\nwrite for me. We will settle with your father about the money\r\nafterwards; but the things should be ordered immediately.”\r\n\r\nShe was then proceeding to all the particulars of calico, muslin, and\r\ncambric, and would shortly have dictated some very plentiful orders, had\r\nnot Jane, though with some difficulty, persuaded her to wait till her\r\nfather was at leisure to be consulted. One day’s delay, she observed,\r\nwould be of small importance; and her mother was too happy to be quite\r\nso obstinate as usual. Other schemes, too, came into her head.\r\n\r\n“I will go to Meryton,” said she, “as soon as I am dressed, and tell the\r\ngood, good news to my sister Philips. And as I come back, I can call on\r\nLady Lucas and Mrs. Long. Kitty, run down and order the carriage. An\r\nairing would do me a great deal of good, I am sure. Girls, can I do\r\nanything for you in Meryton? Oh! here comes Hill. My dear Hill, have you\r\nheard the good news? Miss Lydia is going to be married; and you shall\r\nall have a bowl of punch to make merry at her wedding.”\r\n\r\nMrs. Hill began instantly to express her joy. Elizabeth received her\r\ncongratulations amongst the rest, and then, sick of this folly, took\r\nrefuge in her own room, that she might think with freedom. Poor Lydia’s\r\nsituation must, at best, be bad enough; but that it was no worse, she\r\nhad need to be thankful. She felt it so; and though, in looking forward,\r\nneither rational happiness, nor worldly prosperity could be justly\r\nexpected for her sister, in looking back to what they had feared, only\r\ntwo hours ago, she felt all the advantages of what they had gained.\r\n\r\n\r\n\r\n\r\n[Illustration:\r\n\r\n“The spiteful old ladies”\r\n\r\n[_Copyright 1894 by George Allen._]]\r\n\r\n\r\n\r\n\r\nCHAPTER L.\r\n\r\n\r\n[Illustration]\r\n\r\nMr. Bennet had very often wished, before this period of his life, that,\r\ninstead of spending his whole income, he had laid by an annual sum, for\r\nthe better provision of his children, and of his wife, if she survived\r\nhim. He now wished it more than ever. Had he done his duty in that\r\nrespect, Lydia need not have been indebted to her uncle for whatever of\r\nhonour or credit could now be purchased for her. The satisfaction of\r\nprevailing on one of the most worthless young men in Great Britain to\r\nbe her husband might then have rested in its proper place.\r\n\r\nHe was seriously concerned that a cause of so little advantage to anyone\r\nshould be forwarded at the sole expense of his brother-in-law; and he\r\nwas determined, if possible, to find out the extent of his assistance,\r\nand to discharge the obligation as soon as he could.\r\n\r\nWhen first Mr. Bennet had married, economy was held to be perfectly\r\nuseless; for, of course, they were to have a son. This son was to join\r\nin cutting off the entail, as soon as he should be of age, and the widow\r\nand younger children would by that means be provided for. Five daughters\r\nsuccessively entered the world, but yet the son was to come; and Mrs.\r\nBennet, for many years after Lydia’s birth, had been certain that he\r\nwould. This event had at last been despaired of, but it was then too\r\nlate to be saving. Mrs. Bennet had no turn for economy; and her\r\nhusband’s love of independence had alone prevented their exceeding their\r\nincome.\r\n\r\nFive thousand pounds was settled by marriage articles on Mrs. Bennet and\r\nthe children. But in what proportions it should be divided amongst the\r\nlatter depended on the will of the parents. This was one point, with\r\nregard to Lydia at least, which was now to be settled, and Mr. Bennet\r\ncould have no hesitation in acceding to the proposal before him. In\r\nterms of grateful acknowledgment for the kindness of his brother, though\r\nexpressed most concisely, he then delivered on paper his perfect\r\napprobation of all that was done, and his willingness to fulfil the\r\nengagements that had been made for him. He had never before supposed\r\nthat, could Wickham be prevailed on to marry his daughter, it would be\r\ndone with so little inconvenience to himself as by the present\r\narrangement. He would scarcely be ten pounds a year the loser, by the\r\nhundred that was to be paid them; for, what with her board and pocket\r\nallowance, and the continual presents in money which passed to her\r\nthrough her mother’s hands, Lydia’s expenses had been very little within\r\nthat sum.\r\n\r\nThat it would be done with such trifling exertion on his side, too, was\r\nanother very welcome surprise; for his chief wish at present was to have\r\nas little trouble in the business as possible. When the first transports\r\nof rage which had produced his activity in seeking her were over, he\r\nnaturally returned to all his former indolence. His letter was soon\r\ndespatched; for though dilatory in undertaking business, he was quick in\r\nits execution. He begged to know further particulars of what he was\r\nindebted to his brother; but was too angry with Lydia to send any\r\nmessage to her.\r\n\r\nThe good news quickly spread through the house; and with proportionate\r\nspeed through the neighbourhood. It was borne in the latter with decent\r\nphilosophy. To be sure, it would have been more for the advantage of\r\nconversation, had Miss Lydia Bennet come upon the town; or, as the\r\nhappiest alternative, been secluded from the world in some distant\r\nfarm-house. But there was much to be talked of, in marrying her; and the\r\ngood-natured wishes for her well-doing, which had proceeded before from\r\nall the spiteful old ladies in Meryton, lost but little of their spirit\r\nin this change of circumstances, because with such a husband her misery\r\nwas considered certain.\r\n\r\nIt was a fortnight since Mrs. Bennet had been down stairs, but on this\r\nhappy day she again took her seat at the head of her table, and in\r\nspirits oppressively high. No sentiment of shame gave a damp to her\r\ntriumph. The marriage of a daughter, which had been the first object of\r\nher wishes since Jane was sixteen, was now on the point of\r\naccomplishment, and her thoughts and her words ran wholly on those\r\nattendants of elegant nuptials, fine muslins, new carriages, and\r\nservants. She was busily searching through the neighbourhood for a\r\nproper situation for her daughter; and, without knowing or considering\r\nwhat their income might be, rejected many as deficient in size and\r\nimportance.\r\n\r\n“Haye Park might do,” said she, “if the Gouldings would quit it, or the\r\ngreat house at Stoke, if the drawing-room were larger; but Ashworth is\r\ntoo far off. I could not bear to have her ten miles from me; and as for\r\nPurvis Lodge, the attics are dreadful.”\r\n\r\nHer husband allowed her to talk on without interruption while the\r\nservants remained. But when they had withdrawn, he said to her, “Mrs.\r\nBennet, before you take any, or all of these houses, for your son and\r\ndaughter, let us come to a right understanding. Into _one_ house in this\r\nneighbourhood they shall never have admittance. I will not encourage the\r\nimprudence of either, by receiving them at Longbourn.”\r\n\r\nA long dispute followed this declaration; but Mr. Bennet was firm: it\r\nsoon led to another; and Mrs. Bennet found, with amazement and horror,\r\nthat her husband would not advance a guinea to buy clothes for his\r\ndaughter. He protested that she should receive from him no mark of\r\naffection whatever on the occasion. Mrs. Bennet could hardly comprehend\r\nit. That his anger could be carried to such a point of inconceivable\r\nresentment as to refuse his daughter a privilege, without which her\r\nmarriage would scarcely seem valid, exceeded all that she could believe\r\npossible. She was more alive to the disgrace, which her want of new\r\nclothes must reflect on her daughter’s nuptials, than to any sense of\r\nshame at her eloping and living with Wickham a fortnight before they\r\ntook place.\r\n\r\nElizabeth was now most heartily sorry that she had, from the distress of\r\nthe moment, been led to make Mr. Darcy acquainted with their fears for\r\nher sister; for since her marriage would so shortly give the proper\r\ntermination to the elopement, they might hope to conceal its\r\nunfavourable beginning from all those who were not immediately on the\r\nspot.\r\n\r\nShe had no fear of its spreading farther, through his means. There were\r\nfew people on whose secrecy she would have more confidently depended;\r\nbut at the same time there was no one whose knowledge of a sister’s\r\nfrailty would have mortified her so much. Not, however, from any fear of\r\ndisadvantage from it individually to herself; for at any rate there\r\nseemed a gulf impassable between them. Had Lydia’s marriage been\r\nconcluded on the most honourable terms, it was not to be supposed that\r\nMr. Darcy would connect himself with a family, where to every other\r\nobjection would now be added an alliance and relationship of the nearest\r\nkind with the man whom he so justly scorned.\r\n\r\nFrom such a connection she could not wonder that he should shrink. The\r\nwish of procuring her regard, which she had assured herself of his\r\nfeeling in Derbyshire, could not in rational expectation survive such a\r\nblow as this. She was humbled, she was grieved; she repented, though she\r\nhardly knew of what. She became jealous of his esteem, when she could no\r\nlonger hope to be benefited by it. She wanted to hear of him, when there\r\nseemed the least chance of gaining intelligence. She was convinced that\r\nshe could have been happy with him, when it was no longer likely they\r\nshould meet.\r\n\r\nWhat a triumph for him, as she often thought, could he know that the\r\nproposals which she had proudly spurned only four months ago would now\r\nhave been gladly and gratefully received! He was as generous, she\r\ndoubted not, as the most generous of his sex. But while he was mortal,\r\nthere must be a triumph.\r\n\r\nShe began now to comprehend that he was exactly the man who, in\r\ndisposition and talents, would most suit her. His understanding and\r\ntemper, though unlike her own, would have answered all her wishes. It\r\nwas an union that must have been to the advantage of both: by her ease\r\nand liveliness, his mind might have been softened, his manners improved;\r\nand from his judgment, information, and knowledge of the world, she must\r\nhave received benefit of greater importance.\r\n\r\nBut no such happy marriage could now teach the admiring multitude what\r\nconnubial felicity really was. An union of a different tendency, and\r\nprecluding the possibility of the other, was soon to be formed in their\r\nfamily.\r\n\r\nHow Wickham and Lydia were to be supported in tolerable independence she\r\ncould not imagine. But how little of permanent happiness could belong to\r\na couple who were only brought together because their passions were\r\nstronger than their virtue, she could easily conjecture.\r\n\r\nMr. Gardiner soon wrote again to his brother. To Mr. Bennet’s\r\nacknowledgments he briefly replied, with assurances of his eagerness to\r\npromote the welfare of any of his family; and concluded with entreaties\r\nthat the subject might never be mentioned to him again. The principal\r\npurport of his letter was to inform them, that Mr. Wickham had resolved\r\non quitting the militia.\r\n\r\n“It was greatly my wish that he should do so,” he added, “as soon as his\r\nmarriage was fixed on. And I think you will agree with me, in\r\nconsidering a removal from that corps as highly advisable, both on his\r\naccount and my niece’s. It is Mr. Wickham’s intention to go into the\r\nRegulars; and, among his former friends, there are still some who are\r\nable and willing to assist him in the army. He has the promise of an\r\nensigncy in General----’s regiment, now quartered in the north. It is\r\nan advantage to have it so far from this part of the kingdom. He\r\npromises fairly; and I hope among different people, where they may each\r\nhave a character to preserve, they will both be more prudent. I have\r\nwritten to Colonel Forster, to inform him of our present arrangements,\r\nand to request that he will satisfy the various creditors of Mr. Wickham\r\nin and near Brighton with assurances of speedy payment, for which I have\r\npledged myself. And will you give yourself the trouble of carrying\r\nsimilar assurances to his creditors in Meryton, of whom I shall subjoin\r\na list, according to his information? He has given in all his debts; I\r\nhope at least he has not deceived us. Haggerston has our directions, and\r\nall will be completed in a week. They will then join his regiment,\r\nunless they are first invited to Longbourn; and I understand from Mrs.\r\nGardiner that my niece is very desirous of seeing you all before she\r\nleaves the south. She is well, and begs to be dutifully remembered to\r\nyou and her mother.--Yours, etc.\r\n\r\n“E. GARDINER.”\r\n\r\nMr. Bennet and his daughters saw all the advantages of Wickham’s\r\nremoval from the ----shire, as clearly as Mr. Gardiner could do. But\r\nMrs. Bennet was not so well pleased with it. Lydia’s being settled in\r\nthe north, just when she had expected most pleasure and pride in her\r\ncompany, for she had by no means given up her plan of their residing in\r\nHertfordshire, was a severe disappointment; and, besides, it was such a\r\npity that Lydia should be taken from a regiment where she was acquainted\r\nwith everybody, and had so many favourites.\r\n\r\n“She is so fond of Mrs. Forster,” said she, “it will be quite shocking\r\nto send her away! And there are several of the young men, too, that she\r\nlikes very much. The officers may not be so pleasant in General----’s\r\nregiment.”\r\n\r\nHis daughter’s request, for such it might be considered, of being\r\nadmitted into her family again, before she set off for the north,\r\nreceived at first an absolute negative. But Jane and Elizabeth, who\r\nagreed in wishing, for the sake of their sister’s feelings and\r\nconsequence, that she should be noticed on her marriage by her parents,\r\nurged him so earnestly, yet so rationally and so mildly, to receive her\r\nand her husband at Longbourn, as soon as they were married, that he was\r\nprevailed on to think as they thought, and act as they wished. And their\r\nmother had the satisfaction of knowing, that she should be able to show\r\nher married daughter in the neighbourhood, before she was banished to\r\nthe north. When Mr. Bennet wrote again to his brother, therefore, he\r\nsent his permission for them to come; and it was settled, that, as soon\r\nas the ceremony was over, they should proceed to Longbourn. Elizabeth\r\nwas surprised, however, that Wickham should consent to such a scheme;\r\nand, had she consulted only her own inclination, any meeting with him\r\nwould have been the last object of her wishes.\r\n\r\n\r\n\r\n\r\n[Illustration:\r\n\r\n“With an affectionate smile”\r\n\r\n[_Copyright 1894 by George Allen._]]\r\n\r\n\r\n\r\n\r\nCHAPTER LI.\r\n\r\n\r\n[Illustration]\r\n\r\nTheir sister’s wedding-day arrived; and Jane and Elizabeth felt for her\r\nprobably more than she felt for herself. The carriage was sent to meet\r\nthem at----, and they were to return in it by dinnertime. Their arrival\r\nwas dreaded by the elder Miss Bennets--and Jane more especially, who\r\ngave Lydia the feelings which would have attended herself, had _she_\r\nbeen the culprit, and was wretched in the thought of what her sister\r\nmust endure.\r\n\r\nThey came. The family were assembled in the breakfast-room to receive\r\nthem. Smiles decked the face of Mrs. Bennet, as the carriage drove up to\r\nthe door; her husband looked impenetrably grave; her daughters, alarmed,\r\nanxious, uneasy.\r\n\r\nLydia’s voice was heard in the vestibule; the door was thrown open, and\r\nshe ran into the room. Her mother stepped forwards, embraced her, and\r\nwelcomed her with rapture; gave her hand with an affectionate smile to\r\nWickham, who followed his lady; and wished them both joy, with an\r\nalacrity which showed no doubt of their happiness.\r\n\r\nTheir reception from Mr. Bennet, to whom they then turned, was not quite\r\nso cordial. His countenance rather gained in austerity; and he scarcely\r\nopened his lips. The easy assurance of the young couple, indeed, was\r\nenough to provoke him.\r\n\r\nElizabeth was disgusted, and even Miss Bennet was shocked. Lydia was\r\nLydia still; untamed, unabashed, wild, noisy, and fearless. She turned\r\nfrom sister to sister, demanding their congratulations; and when at\r\nlength they all sat down, looked eagerly round the room, took notice of\r\nsome little alteration in it, and observed, with a laugh, that it was a\r\ngreat while since she had been there.\r\n\r\nWickham was not at all more distressed than herself; but his manners\r\nwere always so pleasing, that, had his character and his marriage been\r\nexactly what they ought, his smiles and his easy address, while he\r\nclaimed their relationship, would have delighted them all. Elizabeth\r\nhad not before believed him quite equal to such assurance; but she sat\r\ndown, resolving within herself to draw no limits in future to the\r\nimpudence of an impudent man. _She_ blushed, and Jane blushed; but the\r\ncheeks of the two who caused their confusion suffered no variation of\r\ncolour.\r\n\r\nThere was no want of discourse. The bride and her mother could neither\r\nof them talk fast enough; and Wickham, who happened to sit near\r\nElizabeth, began inquiring after his acquaintance in that neighbourhood,\r\nwith a good-humoured ease, which she felt very unable to equal in her\r\nreplies. They seemed each of them to have the happiest memories in the\r\nworld. Nothing of the past was recollected with pain; and Lydia led\r\nvoluntarily to subjects which her sisters would not have alluded to for\r\nthe world.\r\n\r\n“Only think of its being three months,” she cried, “since I went away:\r\nit seems but a fortnight, I declare; and yet there have been things\r\nenough happened in the time. Good gracious! when I went away, I am sure\r\nI had no more idea of being married till I came back again! though I\r\nthought it would be very good fun if I was.”\r\n\r\nHer father lifted up his eyes, Jane was distressed, Elizabeth looked\r\nexpressively at Lydia; but she, who never heard nor saw anything of\r\nwhich she chose to be insensible, gaily continued,--\r\n\r\n“Oh, mamma, do the people hereabouts know I am married to-day? I was\r\nafraid they might not; and we overtook William Goulding in his curricle,\r\nso I was determined he should know it, and so I let down the side glass\r\nnext to him, and took off my glove and let my hand just rest upon the\r\nwindow frame, so that he might see the ring, and then I bowed and\r\nsmiled like anything.”\r\n\r\nElizabeth could bear it no longer. She got up and ran out of the room;\r\nand returned no more, till she heard them passing through the hall to\r\nthe dining-parlour. She then joined them soon enough to see Lydia, with\r\nanxious parade, walk up to her mother’s right hand, and hear her say to\r\nher eldest sister,--\r\n\r\n“Ah, Jane, I take your place now, and you must go lower, because I am a\r\nmarried woman.”\r\n\r\nIt was not to be supposed that time would give Lydia that embarrassment\r\nfrom which she had been so wholly free at first. Her ease and good\r\nspirits increased. She longed to see Mrs. Philips, the Lucases, and all\r\ntheir other neighbours, and to hear herself called “Mrs. Wickham” by\r\neach of them; and in the meantime she went after dinner to show her ring\r\nand boast of being married to Mrs. Hill and the two housemaids.\r\n\r\n“Well, mamma,” said she, when they were all returned to the\r\nbreakfast-room, “and what do you think of my husband? Is not he a\r\ncharming man? I am sure my sisters must all envy me. I only hope they\r\nmay have half my good luck. They must all go to Brighton. That is the\r\nplace to get husbands. What a pity it is, mamma, we did not all go!”\r\n\r\n“Very true; and if I had my will we should. But, my dear Lydia, I don’t\r\nat all like your going such a way off. Must it be so?”\r\n\r\n“Oh, Lord! yes; there is nothing in that. I shall like it of all things.\r\nYou and papa, and my sisters, must come down and see us. We shall be at\r\nNewcastle all the winter, and I dare say there will be some balls, and I\r\nwill take care to get good partners for them all.”\r\n\r\n“I should like it beyond anything!” said her mother.\r\n\r\n“And then when you go away, you may leave one or two of my sisters\r\nbehind you; and I dare say I shall get husbands for them before the\r\nwinter is over.”\r\n\r\n“I thank you for my share of the favour,” said Elizabeth; “but I do not\r\nparticularly like your way of getting husbands.”\r\n\r\nTheir visitors were not to remain above ten days with them. Mr. Wickham\r\nhad received his commission before he left London, and he was to join\r\nhis regiment at the end of a fortnight.\r\n\r\nNo one but Mrs. Bennet regretted that their stay would be so short; and\r\nshe made the most of the time by visiting about with her daughter, and\r\nhaving very frequent parties at home. These parties were acceptable to\r\nall; to avoid a family circle was even more desirable to such as did\r\nthink than such as did not.\r\n\r\nWickham’s affection for Lydia was just what Elizabeth had expected to\r\nfind it; not equal to Lydia’s for him. She had scarcely needed her\r\npresent observation to be satisfied, from the reason of things, that\r\ntheir elopement had been brought on by the strength of her love rather\r\nthan by his; and she would have wondered why, without violently caring\r\nfor her, he chose to elope with her at all, had she not felt certain\r\nthat his flight was rendered necessary by distress of circumstances; and\r\nif that were the case, he was not the young man to resist an opportunity\r\nof having a companion.\r\n\r\nLydia was exceedingly fond of him. He was her dear Wickham on every\r\noccasion; no one was to be put in competition with him. He did\r\neverything best in the world; and she was sure he would kill more birds\r\non the first of September than anybody else in the country.\r\n\r\nOne morning, soon after their arrival, as she was sitting with her two\r\nelder sisters, she said to Elizabeth,--\r\n\r\n“Lizzy, I never gave _you_ an account of my wedding, I believe. You were\r\nnot by, when I told mamma, and the others, all about it. Are not you\r\ncurious to hear how it was managed?”\r\n\r\n“No, really,” replied Elizabeth; “I think there cannot be too little\r\nsaid on the subject.”\r\n\r\n“La! You are so strange! But I must tell you how it went off. We were\r\nmarried, you know, at St. Clement’s, because Wickham’s lodgings were in\r\nthat parish. And it was settled that we should all be there by eleven\r\no’clock. My uncle and aunt and I were to go together; and the others\r\nwere to meet us at the church.\r\n\r\n“Well, Monday morning came, and I was in such a fuss! I was so afraid,\r\nyou know, that something would happen to put it off, and then I should\r\nhave gone quite distracted. And there was my aunt, all the time I was\r\ndressing, preaching and talking away just as if she was reading a\r\nsermon. However, I did not hear above one word in ten, for I was\r\nthinking, you may suppose, of my dear Wickham. I longed to know whether\r\nhe would be married in his blue coat.\r\n\r\n“Well, and so we breakfasted at ten as usual: I thought it would never\r\nbe over; for, by the bye, you are to understand that my uncle and aunt\r\nwere horrid unpleasant all the time I was with them. If you’ll believe\r\nme, I did not once put my foot out of doors, though I was there a\r\nfortnight. Not one party, or scheme, or anything! To be sure, London was\r\nrather thin, but, however, the Little Theatre was open.\r\n\r\n“Well, and so, just as the carriage came to the door, my uncle was\r\ncalled away upon business to that horrid man Mr. Stone. And then, you\r\nknow, when once they get together, there is no end of it. Well, I was so\r\nfrightened I did not know what to do, for my uncle was to give me away;\r\nand if we were beyond the hour we could not be married all day. But,\r\nluckily, he came back again in ten minutes’ time, and then we all set\r\nout. However, I recollected afterwards, that if he _had_ been prevented\r\ngoing, the wedding need not be put off, for Mr. Darcy might have done as\r\nwell.”\r\n\r\n“Mr. Darcy!” repeated Elizabeth, in utter amazement.\r\n\r\n“Oh, yes! he was to come there with Wickham, you know. But, gracious me!\r\nI quite forgot! I ought not to have said a word about it. I promised\r\nthem so faithfully! What will Wickham say? It was to be such a secret!”\r\n\r\n“If it was to be a secret,” said Jane, “say not another word on the\r\nsubject. You may depend upon my seeking no further.”\r\n\r\n“Oh, certainly,” said Elizabeth, though burning with curiosity; “we will\r\nask you no questions.”\r\n\r\n“Thank you,” said Lydia; “for if you did, I should certainly tell you\r\nall, and then Wickham would be so angry.”\r\n\r\nOn such encouragement to ask, Elizabeth was forced to put it out of her\r\npower, by running away.\r\n\r\nBut to live in ignorance on such a point was impossible; or at least it\r\nwas impossible not to try for information. Mr. Darcy had been at her\r\nsister’s wedding. It was exactly a scene, and exactly among people,\r\nwhere he had apparently least to do, and least temptation to go.\r\nConjectures as to the meaning of it, rapid and wild, hurried into her\r\nbrain; but she was satisfied with none. Those that best pleased her, as\r\nplacing his conduct in the noblest light, seemed most improbable. She\r\ncould not bear such suspense; and hastily seizing a sheet of paper,\r\nwrote a short letter to her aunt, to request an explanation of what\r\nLydia had dropped, if it were compatible with the secrecy which had been\r\nintended.\r\n\r\n“You may readily comprehend,” she added, “what my curiosity must be to\r\nknow how a person unconnected with any of us, and, comparatively\r\nspeaking, a stranger to our family, should have been amongst you at such\r\na time. Pray write instantly, and let me understand it--unless it is,\r\nfor very cogent reasons, to remain in the secrecy which Lydia seems to\r\nthink necessary; and then I must endeavour to be satisfied with\r\nignorance.”\r\n\r\n“Not that I _shall_, though,” she added to herself, and she finished the\r\nletter; “and, my dear aunt, if you do not tell me in an honourable\r\nmanner, I shall certainly be reduced to tricks and stratagems to find it\r\nout.”\r\n\r\nJane’s delicate sense of honour would not allow her to speak to\r\nElizabeth privately of what Lydia had let fall; Elizabeth was glad of\r\nit:--till it appeared whether her inquiries would receive any\r\nsatisfaction, she had rather be without a confidante.\r\n\r\n\r\n\r\n\r\n[Illustration:\r\n\r\n“I am sure she did not listen.”\r\n]\r\n\r\n\r\n\r\n\r\nCHAPTER LII.\r\n\r\n\r\n[Illustration]\r\n\r\nElizabeth had the satisfaction of receiving an answer to her letter as\r\nsoon as she possibly could. She was no sooner in possession of it, than\r\nhurrying into the little copse, where she was least likely to be\r\ninterrupted, she sat down on one of the benches, and prepared to be\r\nhappy; for the length of the letter convinced her that it did not\r\ncontain a denial.\r\n\r\n /* RIGHT “Gracechurch Street, _Sept. 6_. */\r\n\r\n“My dear Niece,\r\n\r\n “I have just received your letter, and shall devote this whole\r\n morning to answering it, as I foresee that a _little_ writing will\r\n not comprise what I have to tell you. I must confess myself\r\n surprised by your application; I did not expect it from _you_.\r\n Don’t think me angry, however, for I only mean to let you know,\r\n that I had not imagined such inquiries to be necessary on _your_\r\n side. If you do not choose to understand me, forgive my\r\n impertinence. Your uncle is as much surprised as I am; and nothing\r\n but the belief of your being a party concerned would have allowed\r\n him to act as he has done. But if you are really innocent and\r\n ignorant, I must be more explicit. On the very day of my coming\r\n home from Longbourn, your uncle had a most unexpected visitor. Mr.\r\n Darcy called, and was shut up with him several hours. It was all\r\n over before I arrived; so my curiosity was not so dreadfully racked\r\n as _yours_ seems to have been. He came to tell Mr. Gardiner that he\r\n had found out where your sister and Mr. Wickham were, and that he\r\n had seen and talked with them both--Wickham repeatedly, Lydia once.\r\n From what I can collect, he left Derbyshire only one day after\r\n ourselves, and came to town with the resolution of hunting for\r\n them. The motive professed was his conviction of its being owing to\r\n himself that Wickham’s worthlessness had not been so well known as\r\n to make it impossible for any young woman of character to love or\r\n confide in him. He generously imputed the whole to his mistaken\r\n pride, and confessed that he had before thought it beneath him to\r\n lay his private actions open to the world. His character was to\r\n speak for itself. He called it, therefore, his duty to step\r\n forward, and endeavour to remedy an evil which had been brought on\r\n by himself. If he _had another_ motive, I am sure it would never\r\n disgrace him. He had been some days in town before he was able to\r\n discover them; but he had something to direct his search, which was\r\n more than _we_ had; and the consciousness of this was another\r\n reason for his resolving to follow us. There is a lady, it seems, a\r\n Mrs. Younge, who was some time ago governess to Miss Darcy, and was\r\n dismissed from her charge on some cause of disapprobation, though\r\n he did not say what. She then took a large house in Edward Street,\r\n and has since maintained herself by letting lodgings. This Mrs.\r\n Younge was, he knew, intimately acquainted with Wickham; and he\r\n went to her for intelligence of him, as soon as he got to town. But\r\n it was two or three days before he could get from her what he\r\n wanted. She would not betray her trust, I suppose, without bribery\r\n and corruption, for she really did know where her friend was to be\r\n found. Wickham, indeed, had gone to her on their first arrival in\r\n London; and had she been able to receive them into her house, they\r\n would have taken up their abode with her. At length, however, our\r\n kind friend procured the wished-for direction. They were in ----\r\n Street. He saw Wickham, and afterwards insisted on seeing Lydia.\r\n His first object with her, he acknowledged, had been to persuade\r\n her to quit her present disgraceful situation, and return to her\r\n friends as soon as they could be prevailed on to receive her,\r\n offering his assistance as far as it would go. But he found Lydia\r\n absolutely resolved on remaining where she was. She cared for none\r\n of her friends; she wanted no help of his; she would not hear of\r\n leaving Wickham. She was sure they should be married some time or\r\n other, and it did not much signify when. Since such were her\r\n feelings, it only remained, he thought, to secure and expedite a\r\n marriage, which, in his very first conversation with Wickham, he\r\n easily learnt had never been _his_ design. He confessed himself\r\n obliged to leave the regiment on account of some debts of honour\r\n which were very pressing; and scrupled not to lay all the ill\r\n consequences of Lydia’s flight on her own folly alone. He meant to\r\n resign his commission immediately; and as to his future situation,\r\n he could conjecture very little about it. He must go somewhere, but\r\n he did not know where, and he knew he should have nothing to live\r\n on. Mr. Darcy asked why he did not marry your sister at once.\r\n Though Mr. Bennet was not imagined to be very rich, he would have\r\n been able to do something for him, and his situation must have been\r\n benefited by marriage. But he found, in reply to this question,\r\n that Wickham still cherished the hope of more effectually making\r\n his fortune by marriage, in some other country. Under such\r\n circumstances, however, he was not likely to be proof against the\r\n temptation of immediate relief. They met several times, for there\r\n was much to be discussed. Wickham, of course, wanted more than he\r\n could get; but at length was reduced to be reasonable. Everything\r\n being settled between _them_, Mr. Darcy’s next step was to make\r\n your uncle acquainted with it, and he first called in Gracechurch\r\n Street the evening before I came home. But Mr. Gardiner could not\r\n be seen; and Mr. Darcy found, on further inquiry, that your father\r\n was still with him, but would quit town the next morning. He did\r\n not judge your father to be a person whom he could so properly\r\n consult as your uncle, and therefore readily postponed seeing him\r\n till after the departure of the former. He did not leave his name,\r\n and till the next day it was only known that a gentleman had called\r\n on business. On Saturday he came again. Your father was gone, your\r\n uncle at home, and, as I said before, they had a great deal of talk\r\n together. They met again on Sunday, and then _I_ saw him too. It\r\n was not all settled before Monday: as soon as it was, the express\r\n was sent off to Longbourn. But our visitor was very obstinate. I\r\n fancy, Lizzy, that obstinacy is the real defect of his character,\r\n after all. He has been accused of many faults at different times;\r\n but _this_ is the true one. Nothing was to be done that he did not\r\n do himself; though I am sure (and I do not speak it to be thanked,\r\n therefore say nothing about it) your uncle would most readily have\r\n settled the whole. They battled it together for a long time, which\r\n was more than either the gentleman or lady concerned in it\r\n deserved. But at last your uncle was forced to yield, and instead\r\n of being allowed to be of use to his niece, was forced to put up\r\n with only having the probable credit of it, which went sorely\r\n against the grain; and I really believe your letter this morning\r\n gave him great pleasure, because it required an explanation that\r\n would rob him of his borrowed feathers, and give the praise where\r\n it was due. But, Lizzy, this must go no further than yourself, or\r\n Jane at most. You know pretty well, I suppose, what has been done\r\n for the young people. His debts are to be paid, amounting, I\r\n believe, to considerably more than a thousand pounds, another\r\n thousand in addition to her own settled upon _her_, and his\r\n commission purchased. The reason why all this was to be done by him\r\n alone, was such as I have given above. It was owing to him, to his\r\n reserve and want of proper consideration, that Wickham’s character\r\n had been so misunderstood, and consequently that he had been\r\n received and noticed as he was. Perhaps there was some truth in\r\n _this_; though I doubt whether _his_ reserve, or _anybody’s_\r\n reserve can be answerable for the event. But in spite of all this\r\n fine talking, my dear Lizzy, you may rest perfectly assured that\r\n your uncle would never have yielded, if we had not given him credit\r\n for _another interest_ in the affair. When all this was resolved\r\n on, he returned again to his friends, who were still staying at\r\n Pemberley; but it was agreed that he should be in London once more\r\n when the wedding took place, and all money matters were then to\r\n receive the last finish. I believe I have now told you everything.\r\n It is a relation which you tell me is to give you great surprise; I\r\n hope at least it will not afford you any displeasure. Lydia came to\r\n us, and Wickham had constant admission to the house. _He_ was\r\n exactly what he had been when I knew him in Hertfordshire; but I\r\n would not tell you how little I was satisfied with _her_ behaviour\r\n while she stayed with us, if I had not perceived, by Jane’s letter\r\n last Wednesday, that her conduct on coming home was exactly of a\r\n piece with it, and therefore what I now tell you can give you no\r\n fresh pain. I talked to her repeatedly in the most serious manner,\r\n representing to her the wickedness of what she had done, and all\r\n the unhappiness she had brought on her family. If she heard me, it\r\n was by good luck, for I am sure she did not listen. I was sometimes\r\n quite provoked; but then I recollected my dear Elizabeth and Jane,\r\n and for their sakes had patience with her. Mr. Darcy was punctual\r\n in his return, and, as Lydia informed you, attended the wedding. He\r\n dined with us the next day, and was to leave town again on\r\n Wednesday or Thursday. Will you be very angry with me, my dear\r\n Lizzy, if I take this opportunity of saying (what I was never bold\r\n enough to say before) how much I like him? His behaviour to us has,\r\n in every respect, been as pleasing as when we were in Derbyshire.\r\n His understanding and opinions all please me; he wants nothing but\r\n a little more liveliness, and _that_, if he marry _prudently_, his\r\n wife may teach him. I thought him very sly; he hardly ever\r\n mentioned your name. But slyness seems the fashion. Pray forgive\r\n me, if I have been very presuming, or at least do not punish me so\r\n far as to exclude me from P. I shall never be quite happy till I\r\n have been all round the park. A low phaeton with a nice little pair\r\n of ponies would be the very thing. But I must write no more. The\r\n children have been wanting me this half hour.\r\n\r\n“Yours, very sincerely,\r\n\r\n“M. GARDINER.”\r\n\r\n\r\nThe contents of this letter threw Elizabeth into a flutter of spirits,\r\nin which it was difficult to determine whether pleasure or pain bore the\r\ngreatest share. The vague and unsettled suspicions which uncertainty had\r\nproduced, of what Mr. Darcy might have been doing to forward her\r\nsister’s match--which she had feared to encourage, as an exertion of\r\ngoodness too great to be probable, and at the same time dreaded to be\r\njust, from the pain of obligation--were proved beyond their greatest\r\nextent to be true! He had followed them purposely to town, he had taken\r\non himself all the trouble and mortification attendant on such a\r\nresearch; in which supplication had been necessary to a woman whom he\r\nmust abominate and despise, and where he was reduced to meet, frequently\r\nmeet, reason with, persuade, and finally bribe the man whom he always\r\nmost wished to avoid, and whose very name it was punishment to him to\r\npronounce. He had done all this for a girl whom he could neither regard\r\nnor esteem. Her heart did whisper that he had done it for her. But it\r\nwas a hope shortly checked by other considerations; and she soon felt\r\nthat even her vanity was insufficient, when required to depend on his\r\naffection for her, for a woman who had already refused him, as able to\r\novercome a sentiment so natural as abhorrence against relationship with\r\nWickham. Brother-in-law of Wickham! Every kind of pride must revolt from\r\nthe connection. He had, to be sure, done much. She was ashamed to think\r\nhow much. But he had given a reason for his interference, which asked no\r\nextraordinary stretch of belief. It was reasonable that he should feel\r\nhe had been wrong; he had liberality, and he had the means of exercising\r\nit; and though she would not place herself as his principal inducement,\r\nshe could perhaps believe, that remaining partiality for her might\r\nassist his endeavours in a cause where her peace of mind must be\r\nmaterially concerned. It was painful, exceedingly painful, to know that\r\nthey were under obligations to a person who could never receive a\r\nreturn. They owed the restoration of Lydia, her character, everything to\r\nhim. Oh, how heartily did she grieve over every ungracious sensation she\r\nhad ever encouraged, every saucy speech she had ever directed towards\r\nhim! For herself she was humbled; but she was proud of him,--proud that\r\nin a cause of compassion and honour he had been able to get the better\r\nof himself. She read over her aunt’s commendation of him again and\r\nagain. It was hardly enough; but it pleased her. She was even sensible\r\nof some pleasure, though mixed with regret, on finding how steadfastly\r\nboth she and her uncle had been persuaded that affection and confidence\r\nsubsisted between Mr. Darcy and herself.\r\n\r\nShe was roused from her seat and her reflections, by someone’s approach;\r\nand, before she could strike into another path, she was overtaken by\r\nWickham.\r\n\r\n“I am afraid I interrupt your solitary ramble, my dear sister?” said he,\r\nas he joined her.\r\n\r\n“You certainly do,” she replied with a smile; “but it does not follow\r\nthat the interruption must be unwelcome.”\r\n\r\n“I should be sorry, indeed, if it were. _We_ were always good friends,\r\nand now we are better.”\r\n\r\n“True. Are the others coming out?”\r\n\r\n“I do not know. Mrs. Bennet and Lydia are going in the carriage to\r\nMeryton. And so, my dear sister, I find, from our uncle and aunt, that\r\nyou have actually seen Pemberley.”\r\n\r\nShe replied in the affirmative.\r\n\r\n“I almost envy you the pleasure, and yet I believe it would be too much\r\nfor me, or else I could take it in my way to Newcastle. And you saw the\r\nold housekeeper, I suppose? Poor Reynolds, she was always very fond of\r\nme. But of course she did not mention my name to you.”\r\n\r\n“Yes, she did.”\r\n\r\n“And what did she say?”\r\n\r\n“That you were gone into the army, and she was afraid had--not turned\r\nout well. At such a distance as _that_, you know, things are strangely\r\nmisrepresented.”\r\n\r\n“Certainly,” he replied, biting his lips. Elizabeth hoped she had\r\nsilenced him; but he soon afterwards said,--\r\n\r\n“I was surprised to see Darcy in town last month. We passed each other\r\nseveral times. I wonder what he can be doing there.”\r\n\r\n“Perhaps preparing for his marriage with Miss de Bourgh,” said\r\nElizabeth. “It must be something particular to take him there at this\r\ntime of year.”\r\n\r\n“Undoubtedly. Did you see him while you were at Lambton? I thought I\r\nunderstood from the Gardiners that you had.”\r\n\r\n“Yes; he introduced us to his sister.”\r\n\r\n“And do you like her?”\r\n\r\n“Very much.”\r\n\r\n“I have heard, indeed, that she is uncommonly improved within this year\r\nor two. When I last saw her, she was not very promising. I am very glad\r\nyou liked her. I hope she will turn out well.”\r\n\r\n“I dare say she will; she has got over the most trying age.”\r\n\r\n“Did you go by the village of Kympton?”\r\n\r\n“I do not recollect that we did.”\r\n\r\n“I mention it because it is the living which I ought to have had. A most\r\ndelightful place! Excellent parsonage-house! It would have suited me in\r\nevery respect.”\r\n\r\n“How should you have liked making sermons?”\r\n\r\n“Exceedingly well. I should have considered it as part of my duty, and\r\nthe exertion would soon have been nothing. One ought not to repine; but,\r\nto be sure, it would have been such a thing for me! The quiet, the\r\nretirement of such a life, would have answered all my ideas of\r\nhappiness! But it was not to be. Did you ever hear Darcy mention the\r\ncircumstance when you were in Kent?”\r\n\r\n“I _have_ heard from authority, which I thought _as good_, that it was\r\nleft you conditionally only, and at the will of the present patron.”\r\n\r\n“You have! Yes, there was something in _that_; I told you so from the\r\nfirst, you may remember.”\r\n\r\n“I _did_ hear, too, that there was a time when sermon-making was not so\r\npalatable to you as it seems to be at present; that you actually\r\ndeclared your resolution of never taking orders, and that the business\r\nhad been compromised accordingly.”\r\n\r\n“You did! and it was not wholly without foundation. You may remember\r\nwhat I told you on that point, when first we talked of it.”\r\n\r\nThey were now almost at the door of the house, for she had walked fast\r\nto get rid of him; and unwilling, for her sister’s sake, to provoke him,\r\nshe only said in reply, with a good-humoured smile,--\r\n\r\n“Come, Mr. Wickham, we are brother and sister, you know. Do not let us\r\nquarrel about the past. In future, I hope we shall be always of one\r\nmind.”\r\n\r\nShe held out her hand: he kissed it with affectionate gallantry, though\r\nhe hardly knew how to look, and they entered the house.\r\n\r\n\r\n\r\n\r\n[Illustration:\r\n\r\n“Mr. Darcy with him.”\r\n]\r\n\r\n\r\n\r\n\r\nCHAPTER LIII.\r\n\r\n\r\n[Illustration]\r\n\r\nMr. Wickham was so perfectly satisfied with this conversation, that he\r\nnever again distressed himself, or provoked his dear sister Elizabeth,\r\nby introducing the subject of it; and she was pleased to find that she\r\nhad said enough to keep him quiet.\r\n\r\nThe day of his and Lydia’s departure soon came; and Mrs. Bennet was\r\nforced to submit to a separation, which, as her husband by no means\r\nentered into her scheme of their all going to Newcastle, was likely to\r\ncontinue at least a twelvemonth.\r\n\r\n“Oh, my dear Lydia,” she cried, “when shall we meet again?”\r\n\r\n“Oh, Lord! I don’t know. Not these two or three years, perhaps.”\r\n\r\n“Write to me very often, my dear.”\r\n\r\n“As often as I can. But you know married women have never much time for\r\nwriting. My sisters may write to _me_. They will have nothing else to\r\ndo.”\r\n\r\nMr. Wickham’s adieus were much more affectionate than his wife’s. He\r\nsmiled, looked handsome, and said many pretty things.\r\n\r\n“He is as fine a fellow,” said Mr. Bennet, as soon as they were out of\r\nthe house, “as ever I saw. He simpers, and smirks, and makes love to us\r\nall. I am prodigiously proud of him. I defy even Sir William Lucas\r\nhimself to produce a more valuable son-in-law.”\r\n\r\nThe loss of her daughter made Mrs. Bennet very dull for several days.\r\n\r\n“I often think,” said she, “that there is nothing so bad as parting with\r\none’s friends. One seems so forlorn without them.”\r\n\r\n“This is the consequence, you see, madam, of marrying a daughter,” said\r\nElizabeth. “It must make you better satisfied that your other four are\r\nsingle.”\r\n\r\n“It is no such thing. Lydia does not leave me because she is married;\r\nbut only because her husband’s regiment happens to be so far off. If\r\nthat had been nearer, she would not have gone so soon.”\r\n\r\nBut the spiritless condition which this event threw her into was shortly\r\nrelieved, and her mind opened again to the agitation of hope, by an\r\narticle of news which then began to be in circulation. The housekeeper\r\nat Netherfield had received orders to prepare for the arrival of her\r\nmaster, who was coming down in a day or two, to shoot there for several\r\nweeks. Mrs. Bennet was quite in the fidgets. She looked at Jane, and\r\nsmiled, and shook her head, by turns.\r\n\r\n“Well, well, and so Mr. Bingley is coming down, sister,” (for Mrs.\r\nPhilips first brought her the news). “Well, so much the better. Not that\r\nI care about it, though. He is nothing to us, you know, and I am sure I\r\nnever want to see him again. But, however, he is very welcome to come to\r\nNetherfield, if he likes it. And who knows what _may_ happen? But that\r\nis nothing to us. You know, sister, we agreed long ago never to mention\r\na word about it. And so, it is quite certain he is coming?”\r\n\r\n“You may depend on it,” replied the other, “for Mrs. Nichols was in\r\nMeryton last night: I saw her passing by, and went out myself on purpose\r\nto know the truth of it; and she told me that it was certainly true. He\r\ncomes down on Thursday, at the latest, very likely on Wednesday. She was\r\ngoing to the butcher’s, she told me, on purpose to order in some meat on\r\nWednesday, and she has got three couple of ducks just fit to be killed.”\r\n\r\nMiss Bennet had not been able to hear of his coming without changing\r\ncolour. It was many months since she had mentioned his name to\r\nElizabeth; but now, as soon as they were alone together, she said,--\r\n\r\n“I saw you look at me to-day, Lizzy, when my aunt told us of the present\r\nreport; and I know I appeared distressed; but don’t imagine it was from\r\nany silly cause. I was only confused for the moment, because I felt that\r\nI _should_ be looked at. I do assure you that the news does not affect\r\nme either with pleasure or pain. I am glad of one thing, that he comes\r\nalone; because we shall see the less of him. Not that I am afraid of\r\n_myself_, but I dread other people’s remarks.”\r\n\r\nElizabeth did not know what to make of it. Had she not seen him in\r\nDerbyshire, she might have supposed him capable of coming there with no\r\nother view than what was acknowledged; but she still thought him partial\r\nto Jane, and she wavered as to the greater probability of his coming\r\nthere _with_ his friend’s permission, or being bold enough to come\r\nwithout it.\r\n\r\n“Yet it is hard,” she sometimes thought, “that this poor man cannot come\r\nto a house, which he has legally hired, without raising all this\r\nspeculation! I _will_ leave him to himself.”\r\n\r\nIn spite of what her sister declared, and really believed to be her\r\nfeelings, in the expectation of his arrival, Elizabeth could easily\r\nperceive that her spirits were affected by it. They were more disturbed,\r\nmore unequal, than she had often seen them.\r\n\r\nThe subject which had been so warmly canvassed between their parents,\r\nabout a twelvemonth ago, was now brought forward again.\r\n\r\n“As soon as ever Mr. Bingley comes, my dear,” said Mrs. Bennet, “you\r\nwill wait on him, of course.”\r\n\r\n“No, no. You forced me into visiting him last year, and promised, if I\r\nwent to see him, he should marry one of my daughters. But it ended in\r\nnothing, and I will not be sent on a fool’s errand again.”\r\n\r\nHis wife represented to him how absolutely necessary such an attention\r\nwould be from all the neighbouring gentlemen, on his returning to\r\nNetherfield.\r\n\r\n“’Tis an _etiquette_ I despise,” said he. “If he wants our society, let\r\nhim seek it. He knows where we live. I will not spend _my_ hours in\r\nrunning after my neighbours every time they go away and come back\r\nagain.”\r\n\r\n“Well, all I know is, that it will be abominably rude if you do not wait\r\non him. But, however, that shan’t prevent my asking him to dine here, I\r\nam determined. We must have Mrs. Long and the Gouldings soon. That will\r\nmake thirteen with ourselves, so there will be just room at table for\r\nhim.”\r\n\r\nConsoled by this resolution, she was the better able to bear her\r\nhusband’s incivility; though it was very mortifying to know that her\r\nneighbours might all see Mr. Bingley, in consequence of it, before\r\n_they_ did. As the day of his arrival drew near,--\r\n\r\n“I begin to be sorry that he comes at all,” said Jane to her sister. “It\r\nwould be nothing; I could see him with perfect indifference; but I can\r\nhardly bear to hear it thus perpetually talked of. My mother means well;\r\nbut she does not know, no one can know, how much I suffer from what she\r\nsays. Happy shall I be when his stay at Netherfield is over!”\r\n\r\n“I wish I could say anything to comfort you,” replied Elizabeth; “but it\r\nis wholly out of my power. You must feel it; and the usual satisfaction\r\nof preaching patience to a sufferer is denied me, because you have\r\nalways so much.”\r\n\r\nMr. Bingley arrived. Mrs. Bennet, through the assistance of servants,\r\ncontrived to have the earliest tidings of it, that the period of anxiety\r\nand fretfulness on her side be as long as it could. She counted the days\r\nthat must intervene before their invitation could be sent--hopeless of\r\nseeing him before. But on the third morning after his arrival in\r\nHertfordshire, she saw him from her dressing-room window enter the\r\npaddock, and ride towards the house.\r\n\r\nHer daughters were eagerly called to partake of her joy. Jane resolutely\r\nkept her place at the table; but Elizabeth, to satisfy her mother, went\r\nto the window--she looked--she saw Mr. Darcy with him, and sat down\r\nagain by her sister.\r\n\r\n“There is a gentleman with him, mamma,” said Kitty; “who can it be?”\r\n\r\n“Some acquaintance or other, my dear, I suppose; I am sure I do not\r\nknow.”\r\n\r\n“La!” replied Kitty, “it looks just like that man that used to be with\r\nhim before. Mr. what’s his name--that tall, proud man.”\r\n\r\n“Good gracious! Mr. Darcy!--and so it does, I vow. Well, any friend of\r\nMr. Bingley’s will always be welcome here, to be sure; but else I must\r\nsay that I hate the very sight of him.”\r\n\r\nJane looked at Elizabeth with surprise and concern. She knew but little\r\nof their meeting in Derbyshire, and therefore felt for the awkwardness\r\nwhich must attend her sister, in seeing him almost for the first time\r\nafter receiving his explanatory letter. Both sisters were uncomfortable\r\nenough. Each felt for the other, and of course for themselves; and their\r\nmother talked on of her dislike of Mr. Darcy, and her resolution to be\r\ncivil to him only as Mr. Bingley’s friend, without being heard by either\r\nof them. But Elizabeth had sources of uneasiness which could not yet be\r\nsuspected by Jane, to whom she had never yet had courage to show Mrs.\r\nGardiner’s letter, or to relate her own change of sentiment towards\r\nhim. To Jane, he could be only a man whose proposals she had refused,\r\nand whose merits she had undervalued; but to her own more extensive\r\ninformation, he was the person to whom the whole family were indebted\r\nfor the first of benefits, and whom she regarded herself with an\r\ninterest, if not quite so tender, at least as reasonable and just, as\r\nwhat Jane felt for Bingley. Her astonishment at his coming--at his\r\ncoming to Netherfield, to Longbourn, and voluntarily seeking her again,\r\nwas almost equal to what she had known on first witnessing his altered\r\nbehaviour in Derbyshire.\r\n\r\nThe colour which had been driven from her face returned for half a\r\nminute with an additional glow, and a smile of delight added lustre to\r\nher eyes, as she thought for that space of time that his affection and\r\nwishes must still be unshaken; but she would not be secure.\r\n\r\n“Let me first see how he behaves,” said she; “it will then be early\r\nenough for expectation.”\r\n\r\nShe sat intently at work, striving to be composed, and without daring to\r\nlift up her eyes, till anxious curiosity carried them to the face of her\r\nsister as the servant was approaching the door. Jane looked a little\r\npaler than usual, but more sedate than Elizabeth had expected. On the\r\ngentlemen’s appearing, her colour increased; yet she received them with\r\ntolerable ease, and with a propriety of behaviour equally free from any\r\nsymptom of resentment, or any unnecessary complaisance.\r\n\r\nElizabeth said as little to either as civility would allow, and sat down\r\nagain to her work, with an eagerness which it did not often command. She\r\nhad ventured only one glance at Darcy. He looked serious as usual; and,\r\nshe thought, more as he had been used to look in Hertfordshire, than as\r\nshe had seen him at Pemberley. But, perhaps, he could not in her\r\nmother’s presence be what he was before her uncle and aunt. It was a\r\npainful, but not an improbable, conjecture.\r\n\r\nBingley she had likewise seen for an instant, and in that short period\r\nsaw him looking both pleased and embarrassed. He was received by Mrs.\r\nBennet with a degree of civility which made her two daughters ashamed,\r\nespecially when contrasted with the cold and ceremonious politeness of\r\nher courtesy and address of his friend.\r\n\r\nElizabeth particularly, who knew that her mother owed to the latter the\r\npreservation of her favourite daughter from irremediable infamy, was\r\nhurt and distressed to a most painful degree by a distinction so ill\r\napplied.\r\n\r\nDarcy, after inquiring of her how Mr. and Mrs. Gardiner did--a question\r\nwhich she could not answer without confusion--said scarcely anything. He\r\nwas not seated by her: perhaps that was the reason of his silence; but\r\nit had not been so in Derbyshire. There he had talked to her friends\r\nwhen he could not to herself. But now several minutes elapsed, without\r\nbringing the sound of his voice; and when occasionally, unable to resist\r\nthe impulse of curiosity, she raised her eyes to his face, she as often\r\nfound him looking at Jane as at herself, and frequently on no object but\r\nthe ground. More thoughtfulness and less anxiety to please, than when\r\nthey last met, were plainly expressed. She was disappointed, and angry\r\nwith herself for being so.\r\n\r\n“Could I expect it to be otherwise?” said she. “Yet why did he come?”\r\n\r\nShe was in no humour for conversation with anyone but himself; and to\r\nhim she had hardly courage to speak.\r\n\r\nShe inquired after his sister, but could do no more.\r\n\r\n“It is a long time, Mr. Bingley, since you went away,” said Mrs. Bennet.\r\n\r\nHe readily agreed to it.\r\n\r\n“I began to be afraid you would never come back again. People _did_ say,\r\nyou meant to quit the place entirely at Michaelmas; but, however, I hope\r\nit is not true. A great many changes have happened in the neighbourhood\r\nsince you went away. Miss Lucas is married and settled: and one of my\r\nown daughters. I suppose you have heard of it; indeed, you must have\r\nseen it in the papers. It was in the ‘Times’ and the ‘Courier,’ I know;\r\nthough it was not put in as it ought to be. It was only said, ‘Lately,\r\nGeorge Wickham, Esq., to Miss Lydia Bennet,’ without there being a\r\nsyllable said of her father, or the place where she lived, or anything.\r\nIt was my brother Gardiner’s drawing up, too, and I wonder how he came\r\nto make such an awkward business of it. Did you see it?”\r\n\r\nBingley replied that he did, and made his congratulations. Elizabeth\r\ndared not lift up her eyes. How Mr. Darcy looked, therefore, she could\r\nnot tell.\r\n\r\n“It is a delightful thing, to be sure, to have a daughter well married,”\r\ncontinued her mother; “but at the same time, Mr. Bingley, it is very\r\nhard to have her taken away from me. They are gone down to Newcastle, a\r\nplace quite northward it seems, and there they are to stay, I do not\r\nknow how long. His regiment is there; for I suppose you have heard of\r\nhis leaving the ----shire, and of his being gone into the Regulars.\r\nThank heaven! he has _some_ friends, though, perhaps, not so many as he\r\ndeserves.”\r\n\r\nElizabeth, who knew this to be levelled at Mr. Darcy, was in such misery\r\nof shame that she could hardly keep her seat. It drew from her, however,\r\nthe exertion of speaking, which nothing else had so effectually done\r\nbefore; and she asked Bingley whether he meant to make any stay in the\r\ncountry at present. A few weeks, he believed.\r\n\r\n“When you have killed all your own birds, Mr. Bingley,” said her mother,\r\n“I beg you will come here and shoot as many as you please on Mr.\r\nBennet’s manor. I am sure he will be vastly happy to oblige you, and\r\nwill save all the best of the coveys for you.”\r\n\r\nElizabeth’s misery increased at such unnecessary, such officious\r\nattention! Were the same fair prospect to arise at present, as had\r\nflattered them a year ago, everything, she was persuaded, would be\r\nhastening to the same vexatious conclusion. At that instant she felt,\r\nthat years of happiness could not make Jane or herself amends for\r\nmoments of such painful confusion.\r\n\r\n“The first wish of my heart,” said she to herself, “is never more to be\r\nin company with either of them. Their society can afford no pleasure\r\nthat will atone for such wretchedness as this! Let me never see either\r\none or the other again!”\r\n\r\nYet the misery, for which years of happiness were to offer no\r\ncompensation, received soon afterwards material relief, from observing\r\nhow much the beauty of her sister rekindled the admiration of her former\r\nlover. When first he came in, he had spoken to her but little, but every\r\nfive minutes seemed to be giving her more of his attention. He found her\r\nas handsome as she had been last year; as good-natured, and as\r\nunaffected, though not quite so chatty. Jane was anxious that no\r\ndifference should be perceived in her at all, and was really persuaded\r\nthat she talked as much as ever; but her mind was so busily engaged,\r\nthat she did not always know when she was silent.\r\n\r\nWhen the gentlemen rose to go away, Mrs. Bennet was mindful of her\r\nintended civility, and they were invited and engaged to dine at\r\nLongbourn in a few days’ time.\r\n\r\n“You are quite a visit in my debt, Mr. Bingley,” she added; “for when\r\nyou went to town last winter, you promised to take a family dinner with\r\nus as soon as you returned. I have not forgot, you see; and I assure you\r\nI was very much disappointed that you did not come back and keep your\r\nengagement.”\r\n\r\nBingley looked a little silly at this reflection, and said something of\r\nhis concern at having been prevented by business. They then went away.\r\n\r\nMrs. Bennet had been strongly inclined to ask them to stay and dine\r\nthere that day; but, though she always kept a very good table, she did\r\nnot think anything less than two courses could be good enough for a man\r\non whom she had such anxious designs, or satisfy the appetite and pride\r\nof one who had ten thousand a year.\r\n\r\n\r\n\r\n\r\n[Illustration:\r\n\r\n “Jane happened to look round”\r\n]\r\n\r\n\r\n\r\n\r\nCHAPTER LIV.\r\n\r\n\r\n[Illustration]\r\n\r\nAs soon as they were gone, Elizabeth walked out to recover her spirits;\r\nor, in other words, to dwell without interruption on those subjects\r\nwhich must deaden them more. Mr. Darcy’s behaviour astonished and vexed\r\nher.\r\n\r\n“Why, if he came only to be silent, grave, and indifferent,” said she,\r\n“did he come at all?”\r\n\r\nShe could settle it in no way that gave her pleasure.\r\n\r\n“He could be still amiable, still pleasing to my uncle and aunt, when he\r\nwas in town; and why not to me? If he fears me, why come hither? If he\r\nno longer cares for me, why silent? Teasing, teasing man! I will think\r\nno more about him.”\r\n\r\nHer resolution was for a short time involuntarily kept by the approach\r\nof her sister, who joined her with a cheerful look which showed her\r\nbetter satisfied with their visitors than Elizabeth.\r\n\r\n“Now,” said she, “that this first meeting is over, I feel perfectly\r\neasy. I know my own strength, and I shall never be embarrassed again by\r\nhis coming. I am glad he dines here on Tuesday. It will then be publicly\r\nseen, that on both sides we meet only as common and indifferent\r\nacquaintance.”\r\n\r\n“Yes, very indifferent, indeed,” said Elizabeth, laughingly. “Oh, Jane!\r\ntake care.”\r\n\r\n“My dear Lizzy, you cannot think me so weak as to be in danger now.”\r\n\r\n“I think you are in very great danger of making him as much in love with\r\nyou as ever.”\r\n\r\nThey did not see the gentlemen again till Tuesday; and Mrs. Bennet, in\r\nthe meanwhile, was giving way to all the happy schemes which the\r\ngood-humour and common politeness of Bingley, in half an hour’s visit,\r\nhad revived.\r\n\r\nOn Tuesday there was a large party assembled at Longbourn; and the two\r\nwho were most anxiously expected, to the credit of their punctuality as\r\nsportsmen, were in very good time. When they repaired to the\r\ndining-room, Elizabeth eagerly watched to see whether Bingley would take\r\nthe place which, in all their former parties, had belonged to him, by\r\nher sister. Her prudent mother, occupied by the same ideas, forbore to\r\ninvite him to sit by herself. On entering the room, he seemed to\r\nhesitate; but Jane happened to look round, and happened to smile: it was\r\ndecided. He placed himself by her.\r\n\r\nElizabeth, with a triumphant sensation, looked towards his friend. He\r\nbore it with noble indifference; and she would have imagined that\r\nBingley had received his sanction to be happy, had she not seen his eyes\r\nlikewise turned towards Mr. Darcy, with an expression of half-laughing\r\nalarm.\r\n\r\nHis behaviour to her sister was such during dinnertime as showed an\r\nadmiration of her, which, though more guarded than formerly, persuaded\r\nElizabeth, that, if left wholly to himself, Jane’s happiness, and his\r\nown, would be speedily secured. Though she dared not depend upon the\r\nconsequence, she yet received pleasure from observing his behaviour. It\r\ngave her all the animation that her spirits could boast; for she was in\r\nno cheerful humour. Mr. Darcy was almost as far from her as the table\r\ncould divide them. He was on one side of her mother. She knew how little\r\nsuch a situation would give pleasure to either, or make either appear to\r\nadvantage. She was not near enough to hear any of their discourse; but\r\nshe could see how seldom they spoke to each other, and how formal and\r\ncold was their manner whenever they did. Her mother’s ungraciousness\r\nmade the sense of what they owed him more painful to Elizabeth’s mind;\r\nand she would, at times, have given anything to be privileged to tell\r\nhim, that his kindness was neither unknown nor unfelt by the whole of\r\nthe family.\r\n\r\nShe was in hopes that the evening would afford some opportunity of\r\nbringing them together; that the whole of the visit would not pass away\r\nwithout enabling them to enter into something more of conversation,\r\nthan the mere ceremonious salutation attending his entrance. Anxious and\r\nuneasy, the period which passed in the drawing-room before the gentlemen\r\ncame, was wearisome and dull to a degree that almost made her uncivil.\r\nShe looked forward to their entrance as the point on which all her\r\nchance of pleasure for the evening must depend.\r\n\r\n“If he does not come to me, _then_,” said she, “I shall give him up for\r\never.”\r\n\r\nThe gentlemen came; and she thought he looked as if he would have\r\nanswered her hopes; but, alas! the ladies had crowded round the table,\r\nwhere Miss Bennet was making tea, and Elizabeth pouring out the coffee,\r\nin so close a confederacy, that there was not a single vacancy near her\r\nwhich would admit of a chair. And on the gentlemen’s approaching, one of\r\nthe girls moved closer to her than ever, and said, in a whisper,--\r\n\r\n“The men shan’t come and part us, I am determined. We want none of them;\r\ndo we?”\r\n\r\nDarcy had walked away to another part of the room. She followed him with\r\nher eyes, envied everyone to whom he spoke, had scarcely patience enough\r\nto help anybody to coffee, and then was enraged against herself for\r\nbeing so silly!\r\n\r\n“A man who has once been refused! How could I ever be foolish enough to\r\nexpect a renewal of his love? Is there one among the sex who would not\r\nprotest against such a weakness as a second proposal to the same woman?\r\nThere is no indignity so abhorrent to their feelings.”\r\n\r\nShe was a little revived, however, by his bringing back his coffee-cup\r\nhimself; and she seized the opportunity of saying,--\r\n\r\n“Is your sister at Pemberley still?”\r\n\r\n“Yes; she will remain there till Christmas.”\r\n\r\n“And quite alone? Have all her friends left her?”\r\n\r\n“Mrs. Annesley is with her. The others have been gone on to Scarborough\r\nthese three weeks.”\r\n\r\nShe could think of nothing more to say; but if he wished to converse\r\nwith her, he might have better success. He stood by her, however, for\r\nsome minutes, in silence; and, at last, on the young lady’s whispering\r\nto Elizabeth again, he walked away.\r\n\r\nWhen the tea things were removed, and the card tables placed, the ladies\r\nall rose; and Elizabeth was then hoping to be soon joined by him, when\r\nall her views were overthrown, by seeing him fall a victim to her\r\nmother’s rapacity for whist players, and in a few moments after seated\r\nwith the rest of the party. She now lost every expectation of pleasure.\r\nThey were confined for the evening at different tables; and she had\r\nnothing to hope, but that his eyes were so often turned towards her side\r\nof the room, as to make him play as unsuccessfully as herself.\r\n\r\nMrs. Bennet had designed to keep the two Netherfield gentlemen to\r\nsupper; but their carriage was, unluckily, ordered before any of the\r\nothers, and she had no opportunity of detaining them.\r\n\r\n“Well, girls,” said she, as soon as they were left to themselves, “what\r\nsay you to the day? I think everything has passed off uncommonly well, I\r\nassure you. The dinner was as well dressed as any I ever saw. The\r\nvenison was roasted to a turn--and everybody said, they never saw so fat\r\na haunch. The soup was fifty times better than what we had at the\r\nLucases’ last week; and even Mr. Darcy acknowledged that the partridges\r\nwere remarkably well done; and I suppose he has two or three French\r\ncooks at least. And, my dear Jane, I never saw you look in greater\r\nbeauty. Mrs. Long said so too, for I asked her whether you did not. And\r\nwhat do you think she said besides? ‘Ah! Mrs. Bennet, we shall have her\r\nat Netherfield at last!’ She did, indeed. I do think Mrs. Long is as\r\ngood a creature as ever lived--and her nieces are very pretty behaved\r\ngirls, and not at all handsome: I like them prodigiously.”\r\n\r\n[Illustration:\r\n\r\n “M^{rs}. Long and her nieces.”\r\n]\r\n\r\nMrs. Bennet, in short, was in very great spirits: she had seen enough of\r\nBingley’s behaviour to Jane to be convinced that she would get him at\r\nlast; and her expectations of advantage to her family, when in a happy\r\nhumour, were so far beyond reason, that she was quite disappointed at\r\nnot seeing him there again the next day, to make his proposals.\r\n\r\n“It has been a very agreeable day,” said Miss Bennet to Elizabeth. “The\r\nparty seemed so well selected, so suitable one with the other. I hope we\r\nmay often meet again.”\r\n\r\nElizabeth smiled.\r\n\r\n“Lizzy, you must not do so. You must not suspect me. It mortifies me. I\r\nassure you that I have now learnt to enjoy his conversation as an\r\nagreeable and sensible young man without having a wish beyond it. I am\r\nperfectly satisfied, from what his manners now are, that he never had\r\nany design of engaging my affection. It is only that he is blessed with\r\ngreater sweetness of address, and a stronger desire of generally\r\npleasing, than any other man.”\r\n\r\n“You are very cruel,” said her sister, “you will not let me smile, and\r\nare provoking me to it every moment.”\r\n\r\n“How hard it is in some cases to be believed! And how impossible in\r\nothers! But why should you wish to persuade me that I feel more than I\r\nacknowledge?”\r\n\r\n“That is a question which I hardly know how to answer. We all love to\r\ninstruct, though we can teach only what is not worth knowing. Forgive\r\nme; and if you persist in indifference, do not make _me_ your\r\nconfidante.”\r\n\r\n\r\n\r\n\r\n[Illustration:\r\n\r\n “Lizzy, my dear, I want to speak to you.”\r\n]\r\n\r\n\r\n\r\n\r\nCHAPTER LV.\r\n\r\n\r\n[Illustration]\r\n\r\nA few days after this visit, Mr. Bingley called again, and alone. His\r\nfriend had left him that morning for London, but was to return home in\r\nten days’ time. He sat with them above an hour, and was in remarkably\r\ngood spirits. Mrs. Bennet invited him to dine with them; but, with many\r\nexpressions of concern, he confessed himself engaged elsewhere.\r\n\r\n“Next time you call,” said she, “I hope we shall be more lucky.”\r\n\r\nHe should be particularly happy at any time, etc., etc.; and if she\r\nwould give him leave, would take an early opportunity of waiting on\r\nthem.\r\n\r\n“Can you come to-morrow?”\r\n\r\nYes, he had no engagement at all for to-morrow; and her invitation was\r\naccepted with alacrity.\r\n\r\nHe came, and in such very good time, that the ladies were none of them\r\ndressed. In ran Mrs. Bennet to her daughters’ room, in her\r\ndressing-gown, and with her hair half finished, crying out,--\r\n\r\n“My dear Jane, make haste and hurry down. He is come--Mr. Bingley is\r\ncome. He is, indeed. Make haste, make haste. Here, Sarah, come to Miss\r\nBennet this moment, and help her on with her gown. Never mind Miss\r\nLizzy’s hair.”\r\n\r\n“We will be down as soon as we can,” said Jane; “but I dare say Kitty is\r\nforwarder than either of us, for she went upstairs half an hour ago.”\r\n\r\n“Oh! hang Kitty! what has she to do with it? Come, be quick, be quick!\r\nwhere is your sash, my dear?”\r\n\r\nBut when her mother was gone, Jane would not be prevailed on to go down\r\nwithout one of her sisters.\r\n\r\nThe same anxiety to get them by themselves was visible again in the\r\nevening. After tea, Mr. Bennet retired to the library, as was his\r\ncustom, and Mary went upstairs to her instrument. Two obstacles of the\r\nfive being thus removed, Mrs. Bennet sat looking and winking at\r\nElizabeth and Catherine for a considerable time, without making any\r\nimpression on them. Elizabeth would not observe her; and when at last\r\nKitty did, she very innocently said, “What is the matter, mamma? What do\r\nyou keep winking at me for? What am I to do?”\r\n\r\n“Nothing, child, nothing. I did not wink at you.” She then sat still\r\nfive minutes longer; but unable to waste such a precious occasion, she\r\nsuddenly got up, and saying to Kitty,--\r\n\r\n“Come here, my love, I want to speak to you,” took her out of the room.\r\nJane instantly gave a look at Elizabeth which spoke her distress at such\r\npremeditation, and her entreaty that _she_ would not give in to it. In a\r\nfew minutes, Mrs. Bennet half opened the door and called out,--\r\n\r\n“Lizzy, my dear, I want to speak with you.”\r\n\r\nElizabeth was forced to go.\r\n\r\n“We may as well leave them by themselves, you know,” said her mother as\r\nsoon as she was in the hall. “Kitty and I are going upstairs to sit in\r\nmy dressing-room.”\r\n\r\nElizabeth made no attempt to reason with her mother, but remained\r\nquietly in the hall till she and Kitty were out of sight, then returned\r\ninto the drawing-room.\r\n\r\nMrs. Bennet’s schemes for this day were ineffectual. Bingley was\r\neverything that was charming, except the professed lover of her\r\ndaughter. His ease and cheerfulness rendered him a most agreeable\r\naddition to their evening party; and he bore with the ill-judged\r\nofficiousness of the mother, and heard all her silly remarks with a\r\nforbearance and command of countenance particularly grateful to the\r\ndaughter.\r\n\r\nHe scarcely needed an invitation to stay supper; and before he went away\r\nan engagement was formed, chiefly through his own and Mrs. Bennet’s\r\nmeans, for his coming next morning to shoot with her husband.\r\n\r\nAfter this day, Jane said no more of her indifference. Not a word passed\r\nbetween the sisters concerning Bingley; but Elizabeth went to bed in the\r\nhappy belief that all must speedily be concluded, unless Mr. Darcy\r\nreturned within the stated time. Seriously, however, she felt tolerably\r\npersuaded that all this must have taken place with that gentleman’s\r\nconcurrence.\r\n\r\nBingley was punctual to his appointment; and he and Mr. Bennet spent the\r\nmorning together, as had been agreed on. The latter was much more\r\nagreeable than his companion expected. There was nothing of presumption\r\nor folly in Bingley that could provoke his ridicule, or disgust him into\r\nsilence; and he was more communicative, and less eccentric, than the\r\nother had ever seen him. Bingley of course returned with him to dinner;\r\nand in the evening Mrs. Bennet’s invention was again at work to get\r\neverybody away from him and her daughter. Elizabeth, who had a letter to\r\nwrite, went into the breakfast-room for that purpose soon after tea; for\r\nas the others were all going to sit down to cards, she could not be\r\nwanted to counteract her mother’s schemes.\r\n\r\nBut on her returning to the drawing-room, when her letter was finished,\r\nshe saw, to her infinite surprise, there was reason to fear that her\r\nmother had been too ingenious for her. On opening the door, she\r\nperceived her sister and Bingley standing together over the hearth, as\r\nif engaged in earnest conversation; and had this led to no suspicion,\r\nthe faces of both, as they hastily turned round and moved away from each\r\nother, would have told it all. _Their_ situation was awkward enough; but\r\n_hers_ she thought was still worse. Not a syllable was uttered by\r\neither; and Elizabeth was on the point of going away again, when\r\nBingley, who as well as the other had sat down, suddenly rose, and,\r\nwhispering a few words to her sister, ran out of the room.\r\n\r\nJane could have no reserves from Elizabeth, where confidence would give\r\npleasure; and, instantly embracing her, acknowledged, with the liveliest\r\nemotion, that she was the happiest creature in the world.\r\n\r\n“’Tis too much!” she added, “by far too much. I do not deserve it. Oh,\r\nwhy is not everybody as happy?”\r\n\r\nElizabeth’s congratulations were given with a sincerity, a warmth, a\r\ndelight, which words could but poorly express. Every sentence of\r\nkindness was a fresh source of happiness to Jane. But she would not\r\nallow herself to stay with her sister, or say half that remained to be\r\nsaid, for the present.\r\n\r\n“I must go instantly to my mother,” she cried. “I would not on any\r\naccount trifle with her affectionate solicitude, or allow her to hear it\r\nfrom anyone but myself. He is gone to my father already. Oh, Lizzy, to\r\nknow that what I have to relate will give such pleasure to all my dear\r\nfamily! how shall I bear so much happiness?”\r\n\r\nShe then hastened away to her mother, who had purposely broken up the\r\ncard-party, and was sitting upstairs with Kitty.\r\n\r\nElizabeth, who was left by herself, now smiled at the rapidity and ease\r\nwith which an affair was finally settled, that had given them so many\r\nprevious months of suspense and vexation.\r\n\r\n“And this,” said she, “is the end of all his friend’s anxious\r\ncircumspection! of all his sister’s falsehood and contrivance! the\r\nhappiest, wisest, and most reasonable end!”\r\n\r\nIn a few minutes she was joined by Bingley, whose conference with her\r\nfather had been short and to the purpose.\r\n\r\n“Where is your sister?” said he hastily, as he opened the door.\r\n\r\n“With my mother upstairs. She will be down in a moment, I dare say.”\r\n\r\nHe then shut the door, and, coming up to her, claimed the good wishes\r\nand affection of a sister. Elizabeth honestly and heartily expressed her\r\ndelight in the prospect of their relationship. They shook hands with\r\ngreat cordiality; and then, till her sister came down, she had to listen\r\nto all he had to say of his own happiness, and of Jane’s perfections;\r\nand in spite of his being a lover, Elizabeth really believed all his\r\nexpectations of felicity to be rationally founded, because they had for\r\nbasis the excellent understanding and super-excellent disposition of\r\nJane, and a general similarity of feeling and taste between her and\r\nhimself.\r\n\r\nIt was an evening of no common delight to them all; the satisfaction of\r\nMiss Bennet’s mind gave such a glow of sweet animation to her face, as\r\nmade her look handsomer than ever. Kitty simpered and smiled, and hoped\r\nher turn was coming soon. Mrs. Bennet could not give her consent, or\r\nspeak her approbation in terms warm enough to satisfy her feelings,\r\nthough she talked to Bingley of nothing else, for half an hour; and when\r\nMr. Bennet joined them at supper, his voice and manner plainly showed\r\nhow really happy he was.\r\n\r\nNot a word, however, passed his lips in allusion to it, till their\r\nvisitor took his leave for the night; but as soon as he was gone, he\r\nturned to his daughter and said,--\r\n\r\n“Jane, I congratulate you. You will be a very happy woman.”\r\n\r\nJane went to him instantly, kissed him, and thanked him for his\r\ngoodness.\r\n\r\n“You are a good girl,” he replied, “and I have great pleasure in\r\nthinking you will be so happily settled. I have not a doubt of your\r\ndoing very well together. Your tempers are by no means unlike. You are\r\neach of you so complying, that nothing will ever be resolved on; so\r\neasy, that every servant will cheat you; and so generous, that you will\r\nalways exceed your income.”\r\n\r\n“I hope not so. Imprudence or thoughtlessness in money matters would be\r\nunpardonable in _me_.”\r\n\r\n“Exceed their income! My dear Mr. Bennet,” cried his wife, “what are you\r\ntalking of? Why, he has four or five thousand a year, and very likely\r\nmore.” Then addressing her daughter, “Oh, my dear, dear Jane, I am so\r\nhappy! I am sure I shan’t get a wink of sleep all night. I knew how it\r\nwould be. I always said it must be so, at last. I was sure you could not\r\nbe so beautiful for nothing! I remember, as soon as ever I saw him, when\r\nhe first came into Hertfordshire last year, I thought how likely it was\r\nthat you should come together. Oh, he is the handsomest young man that\r\never was seen!”\r\n\r\nWickham, Lydia, were all forgotten. Jane was beyond competition her\r\nfavourite child. At that moment she cared for no other. Her younger\r\nsisters soon began to make interest with her for objects of happiness\r\nwhich she might in future be able to dispense.\r\n\r\nMary petitioned for the use of the library at Netherfield; and Kitty\r\nbegged very hard for a few balls there every winter.\r\n\r\nBingley, from this time, was of course a daily visitor at Longbourn;\r\ncoming frequently before breakfast, and always remaining till after\r\nsupper; unless when some barbarous neighbour, who could not be enough\r\ndetested, had given him an invitation to dinner, which he thought\r\nhimself obliged to accept.\r\n\r\nElizabeth had now but little time for conversation with her sister; for\r\nwhile he was present Jane had no attention to bestow on anyone else: but\r\nshe found herself considerably useful to both of them, in those hours of\r\nseparation that must sometimes occur. In the absence of Jane, he always\r\nattached himself to Elizabeth for the pleasure of talking of her; and\r\nwhen Bingley was gone, Jane constantly sought the same means of relief.\r\n\r\n“He has made me so happy,” said she, one evening, “by telling me that he\r\nwas totally ignorant of my being in town last spring! I had not believed\r\nit possible.”\r\n\r\n“I suspected as much,” replied Elizabeth. “But how did he account for\r\nit?”\r\n\r\n“It must have been his sisters’ doing. They were certainly no friends to\r\nhis acquaintance with me, which I cannot wonder at, since he might have\r\nchosen so much more advantageously in many respects. But when they see,\r\nas I trust they will, that their brother is happy with me, they will\r\nlearn to be contented, and we shall be on good terms again: though we\r\ncan never be what we once were to each other.”\r\n\r\n“That is the most unforgiving speech,” said Elizabeth, “that I ever\r\nheard you utter. Good girl! It would vex me, indeed, to see you again\r\nthe dupe of Miss Bingley’s pretended regard.”\r\n\r\n“Would you believe it, Lizzy, that when he went to town last November he\r\nreally loved me, and nothing but a persuasion of _my_ being indifferent\r\nwould have prevented his coming down again?”\r\n\r\n“He made a little mistake, to be sure; but it is to the credit of his\r\nmodesty.”\r\n\r\nThis naturally introduced a panegyric from Jane on his diffidence, and\r\nthe little value he put on his own good qualities.\r\n\r\nElizabeth was pleased to find that he had not betrayed the interference\r\nof his friend; for, though Jane had the most generous and forgiving\r\nheart in the world, she knew it was a circumstance which must prejudice\r\nher against him.\r\n\r\n“I am certainly the most fortunate creature that ever existed!” cried\r\nJane. “Oh, Lizzy, why am I thus singled from my family, and blessed\r\nabove them all? If I could but see you as happy! If there were but such\r\nanother man for you!”\r\n\r\n“If you were to give me forty such men I never could be so happy as you.\r\nTill I have your disposition, your goodness, I never can have your\r\nhappiness. No, no, let me shift for myself; and, perhaps, if I have very\r\ngood luck, I may meet with another Mr. Collins in time.”\r\n\r\nThe situation of affairs in the Longbourn family could not be long a\r\nsecret. Mrs. Bennet was privileged to whisper it to Mrs. Philips, and\r\nshe ventured, without any permission, to do the same by all her\r\nneighbours in Meryton.\r\n\r\nThe Bennets were speedily pronounced to be the luckiest family in the\r\nworld; though only a few weeks before, when Lydia had first run away,\r\nthey had been generally proved to be marked out for misfortune.\r\n\r\n\r\n\r\n\r\n[Illustration]\r\n\r\n\r\n\r\n\r\nCHAPTER LVI.\r\n\r\n\r\n[Illustration]\r\n\r\nOne morning, about a week after Bingley’s engagement with Jane had been\r\nformed, as he and the females of the family were sitting together in the\r\ndining-room, their attention was suddenly drawn to the window by the\r\nsound of a carriage; and they perceived a chaise and four driving up the\r\nlawn. It was too early in the morning for visitors; and besides, the\r\nequipage did not answer to that of any of their neighbours. The horses\r\nwere post; and neither the carriage, nor the livery of the servant who\r\npreceded it, were familiar to them. As it was certain, however, that\r\nsomebody was coming, Bingley instantly prevailed on Miss Bennet to avoid\r\nthe confinement of such an intrusion, and walk away with him into the\r\nshrubbery. They both set off; and the conjectures of the remaining three\r\ncontinued, though with little satisfaction, till the door was thrown\r\nopen, and their visitor entered. It was Lady Catherine de Bourgh.\r\n\r\nThey were of course all intending to be surprised: but their\r\nastonishment was beyond their expectation; and on the part of Mrs.\r\nBennet and Kitty, though she was perfectly unknown to them, even\r\ninferior to what Elizabeth felt.\r\n\r\nShe entered the room with an air more than usually ungracious, made no\r\nother reply to Elizabeth’s salutation than a slight inclination of the\r\nhead, and sat down without saying a word. Elizabeth had mentioned her\r\nname to her mother on her Ladyship’s entrance, though no request of\r\nintroduction had been made.\r\n\r\nMrs. Bennet, all amazement, though flattered by having a guest of such\r\nhigh importance, received her with the utmost politeness. After sitting\r\nfor a moment in silence, she said, very stiffly, to Elizabeth,--\r\n\r\n“I hope you are well, Miss Bennet. That lady, I suppose, is your\r\nmother?”\r\n\r\nElizabeth replied very concisely that she was.\r\n\r\n“And _that_, I suppose, is one of your sisters?”\r\n\r\n“Yes, madam,” said Mrs. Bennet, delighted to speak to a Lady Catherine.\r\n“She is my youngest girl but one. My youngest of all is lately married,\r\nand my eldest is somewhere about the ground, walking with a young man,\r\nwho, I believe, will soon become a part of the family.”\r\n\r\n“You have a very small park here,” returned Lady Catherine, after a\r\nshort silence.\r\n\r\n“It is nothing in comparison of Rosings, my Lady, I dare say; but, I\r\nassure you, it is much larger than Sir William Lucas’s.”\r\n\r\n“This must be a most inconvenient sitting-room for the evening in\r\nsummer: the windows are full west.”\r\n\r\nMrs. Bennet assured her that they never sat there after dinner; and then\r\nadded,--\r\n\r\n“May I take the liberty of asking your Ladyship whether you left Mr. and\r\nMrs. Collins well?”\r\n\r\n“Yes, very well. I saw them the night before last.”\r\n\r\nElizabeth now expected that she would produce a letter for her from\r\nCharlotte, as it seemed the only probable motive for her calling. But no\r\nletter appeared, and she was completely puzzled.\r\n\r\nMrs. Bennet, with great civility, begged her Ladyship to take some\r\nrefreshment: but Lady Catherine very resolutely, and not very politely,\r\ndeclined eating anything; and then, rising up, said to Elizabeth,--\r\n\r\n“Miss Bennet, there seemed to be a prettyish kind of a little wilderness\r\non one side of your lawn. I should be glad to take a turn in it, if you\r\nwill favour me with your company.”\r\n\r\n“Go, my dear,” cried her mother, “and show her Ladyship about the\r\ndifferent walks. I think she will be pleased with the hermitage.”\r\n\r\nElizabeth obeyed; and, running into her own room for her parasol,\r\nattended her noble guest downstairs. As they passed through the hall,\r\nLady Catherine opened the doors into the dining-parlour and\r\ndrawing-room, and pronouncing them, after a short survey, to be\r\ndecent-looking rooms, walked on.\r\n\r\nHer carriage remained at the door, and Elizabeth saw that her\r\nwaiting-woman was in it. They proceeded in silence along the gravel walk\r\nthat led to the copse; Elizabeth was determined to make no effort for\r\nconversation with a woman who was now more than usually insolent and\r\ndisagreeable.\r\n\r\n[Illustration:\r\n\r\n“After a short survey”\r\n\r\n[_Copyright 1894 by George Allen._]]\r\n\r\n“How could I ever think her like her nephew?” said she, as she looked in\r\nher face.\r\n\r\nAs soon as they entered the copse, Lady Catherine began in the following\r\nmanner:--\r\n\r\n“You can be at no loss, Miss Bennet, to understand the reason of my\r\njourney hither. Your own heart, your own conscience, must tell you why I\r\ncome.”\r\n\r\nElizabeth looked with unaffected astonishment.\r\n\r\n“Indeed, you are mistaken, madam; I have not been at all able to account\r\nfor the honour of seeing you here.”\r\n\r\n“Miss Bennet,” replied her Ladyship, in an angry tone, “you ought to\r\nknow that I am not to be trifled with. But however insincere _you_ may\r\nchoose to be, you shall not find _me_ so. My character has ever been\r\ncelebrated for its sincerity and frankness; and in a cause of such\r\nmoment as this, I shall certainly not depart from it. A report of a most\r\nalarming nature reached me two days ago. I was told, that not only your\r\nsister was on the point of being most advantageously married, but that\r\n_you_--that Miss Elizabeth Bennet would, in all likelihood, be soon\r\nafterwards united to my nephew--my own nephew, Mr. Darcy. Though I\r\n_know_ it must be a scandalous falsehood, though I would not injure him\r\nso much as to suppose the truth of it possible, I instantly resolved on\r\nsetting off for this place, that I might make my sentiments known to\r\nyou.”\r\n\r\n“If you believed it impossible to be true,” said Elizabeth, colouring\r\nwith astonishment and disdain, “I wonder you took the trouble of coming\r\nso far. What could your Ladyship propose by it?”\r\n\r\n“At once to insist upon having such a report universally contradicted.”\r\n\r\n“Your coming to Longbourn, to see me and my family,” said Elizabeth\r\ncoolly, “will be rather a confirmation of it--if, indeed, such a report\r\nis in existence.”\r\n\r\n“If! do you then pretend to be ignorant of it? Has it not been\r\nindustriously circulated by yourselves? Do you not know that such a\r\nreport is spread abroad?”\r\n\r\n“I never heard that it was.”\r\n\r\n“And can you likewise declare, that there is no _foundation_ for it?”\r\n\r\n“I do not pretend to possess equal frankness with your Ladyship. _You_\r\nmay ask questions which _I_ shall not choose to answer.”\r\n\r\n“This is not to be borne. Miss Bennet, I insist on being satisfied. Has\r\nhe, has my nephew, made you an offer of marriage?”\r\n\r\n“Your Ladyship has declared it to be impossible.”\r\n\r\n“It ought to be so; it must be so, while he retains the use of his\r\nreason. But _your_ arts and allurements may, in a moment of infatuation,\r\nhave made him forget what he owes to himself and to all his family. You\r\nmay have drawn him in.”\r\n\r\n“If I have, I shall be the last person to confess it.”\r\n\r\n“Miss Bennet, do you know who I am? I have not been accustomed to such\r\nlanguage as this. I am almost the nearest relation he has in the world,\r\nand am entitled to know all his dearest concerns.”\r\n\r\n“But you are not entitled to know _mine_; nor will such behaviour as\r\nthis ever induce me to be explicit.”\r\n\r\n“Let me be rightly understood. This match, to which you have the\r\npresumption to aspire, can never take place. No, never. Mr. Darcy is\r\nengaged to _my daughter_. Now, what have you to say?”\r\n\r\n“Only this,--that if he is so, you can have no reason to suppose he will\r\nmake an offer to me.”\r\n\r\nLady Catherine hesitated for a moment, and then replied,--\r\n\r\n“The engagement between them is of a peculiar kind. From their infancy,\r\nthey have been intended for each other. It was the favourite wish of\r\n_his_ mother, as well as of hers. While in their cradles we planned the\r\nunion; and now, at the moment when the wishes of both sisters would be\r\naccomplished, is their marriage to be prevented by a young woman of\r\ninferior birth, of no importance in the world, and wholly unallied to\r\nthe family? Do you pay no regard to the wishes of his friends--to his\r\ntacit engagement with Miss de Bourgh? Are you lost to every feeling of\r\npropriety and delicacy? Have you not heard me say, that from his\r\nearliest hours he was destined for his cousin?”\r\n\r\n“Yes; and I had heard it before. But what is that to me? If there is no\r\nother objection to my marrying your nephew, I shall certainly not be\r\nkept from it by knowing that his mother and aunt wished him to marry\r\nMiss de Bourgh. You both did as much as you could in planning the\r\nmarriage. Its completion depended on others. If Mr. Darcy is neither by\r\nhonour nor inclination confined to his cousin, why is not he to make\r\nanother choice? And if I am that choice, why may not I accept him?”\r\n\r\n“Because honour, decorum, prudence--nay, interest--forbid it. Yes, Miss\r\nBennet, interest; for do not expect to be noticed by his family or\r\nfriends, if you wilfully act against the inclinations of all. You will\r\nbe censured, slighted, and despised, by everyone connected with him.\r\nYour alliance will be a disgrace; your name will never even be mentioned\r\nby any of us.”\r\n\r\n“These are heavy misfortunes,” replied Elizabeth. “But the wife of Mr.\r\nDarcy must have such extraordinary sources of happiness necessarily\r\nattached to her situation, that she could, upon the whole, have no cause\r\nto repine.”\r\n\r\n“Obstinate, headstrong girl! I am ashamed of you! Is this your gratitude\r\nfor my attentions to you last spring? Is nothing due to me on that\r\nscore? Let us sit down. You are to understand, Miss Bennet, that I came\r\nhere with the determined resolution of carrying my purpose; nor will I\r\nbe dissuaded from it. I have not been used to submit to any person’s\r\nwhims. I have not been in the habit of brooking disappointment.”\r\n\r\n“_That_ will make your Ladyship’s situation at present more pitiable;\r\nbut it will have no effect on _me_.”\r\n\r\n“I will not be interrupted! Hear me in silence. My daughter and my\r\nnephew are formed for each other. They are descended, on the maternal\r\nside, from the same noble line; and, on the father’s, from respectable,\r\nhonourable, and ancient, though untitled, families. Their fortune on\r\nboth sides is splendid. They are destined for each other by the voice of\r\nevery member of their respective houses; and what is to divide\r\nthem?--the upstart pretensions of a young woman without family,\r\nconnections, or fortune! Is this to be endured? But it must not, shall\r\nnot be! If you were sensible of your own good, you would not wish to\r\nquit the sphere in which you have been brought up.”\r\n\r\n“In marrying your nephew, I should not consider myself as quitting that\r\nsphere. He is a gentleman; I am a gentleman’s daughter; so far we are\r\nequal.”\r\n\r\n“True. You _are_ a gentleman’s daughter. But what was your mother? Who\r\nare your uncles and aunts? Do not imagine me ignorant of their\r\ncondition.”\r\n\r\n“Whatever my connections may be,” said Elizabeth, “if your nephew does\r\nnot object to them, they can be nothing to _you_.”\r\n\r\n“Tell me, once for all, are you engaged to him?”\r\n\r\nThough Elizabeth would not, for the mere purpose of obliging Lady\r\nCatherine, have answered this question, she could not but say, after a\r\nmoment’s deliberation,--\r\n\r\n“I am not.”\r\n\r\nLady Catherine seemed pleased.\r\n\r\n“And will you promise me never to enter into such an engagement?”\r\n\r\n“I will make no promise of the kind.”\r\n\r\n“Miss Bennet, I am shocked and astonished. I expected to find a more\r\nreasonable young woman. But do not deceive yourself into a belief that I\r\nwill ever recede. I shall not go away till you have given me the\r\nassurance I require.”\r\n\r\n“And I certainly _never_ shall give it. I am not to be intimidated into\r\nanything so wholly unreasonable. Your Ladyship wants Mr. Darcy to marry\r\nyour daughter; but would my giving you the wished-for promise make\r\n_their_ marriage at all more probable? Supposing him to be attached to\r\nme, would _my_ refusing to accept his hand make him wish to bestow it on\r\nhis cousin? Allow me to say, Lady Catherine, that the arguments with\r\nwhich you have supported this extraordinary application have been as\r\nfrivolous as the application was ill-judged. You have widely mistaken my\r\ncharacter, if you think I can be worked on by such persuasions as these.\r\nHow far your nephew might approve of your interference in _his_ affairs,\r\nI cannot tell; but you have certainly no right to concern yourself in\r\nmine. I must beg, therefore, to be importuned no further on the\r\nsubject.”\r\n\r\n“Not so hasty, if you please. I have by no means done. To all the\r\nobjections I have already urged I have still another to add. I am no\r\nstranger to the particulars of your youngest sister’s infamous\r\nelopement. I know it all; that the young man’s marrying her was a\r\npatched-up business, at the expense of your father and uncle. And is\r\n_such_ a girl to be my nephew’s sister? Is _her_ husband, who is the son\r\nof his late father’s steward, to be his brother? Heaven and earth!--of\r\nwhat are you thinking? Are the shades of Pemberley to be thus polluted?”\r\n\r\n“You can _now_ have nothing further to say,” she resentfully answered.\r\n“You have insulted me, in every possible method. I must beg to return to\r\nthe house.”\r\n\r\nAnd she rose as she spoke. Lady Catherine rose also, and they turned\r\nback. Her Ladyship was highly incensed.\r\n\r\n“You have no regard, then, for the honour and credit of my nephew!\r\nUnfeeling, selfish girl! Do you not consider that a connection with you\r\nmust disgrace him in the eyes of everybody?”\r\n\r\n“Lady Catherine, I have nothing further to say. You know my sentiments.”\r\n\r\n“You are then resolved to have him?”\r\n\r\n“I have said no such thing. I am only resolved to act in that manner,\r\nwhich will, in my own opinion, constitute my happiness, without\r\nreference to _you_, or to any person so wholly unconnected with me.”\r\n\r\n“It is well. You refuse, then, to oblige me. You refuse to obey the\r\nclaims of duty, honour, and gratitude. You are determined to ruin him in\r\nthe opinion of all his friends, and make him the contempt of the world.”\r\n\r\n“Neither duty, nor honour, nor gratitude,” replied Elizabeth, “has any\r\npossible claim on me, in the present instance. No principle of either\r\nwould be violated by my marriage with Mr. Darcy. And with regard to the\r\nresentment of his family, or the indignation of the world, if the former\r\n_were_ excited by his marrying me, it would not give me one moment’s\r\nconcern--and the world in general would have too much sense to join in\r\nthe scorn.”\r\n\r\n“And this is your real opinion! This is your final resolve! Very well. I\r\nshall now know how to act. Do not imagine, Miss Bennet, that your\r\nambition will ever be gratified. I came to try you. I hoped to find you\r\nreasonable; but depend upon it I will carry my point.”\r\n\r\nIn this manner Lady Catherine talked on till they were at the door of\r\nthe carriage, when, turning hastily round, she added,--\r\n\r\n“I take no leave of you, Miss Bennet. I send no compliments to your\r\nmother. You deserve no such attention. I am most seriously displeased.”\r\n\r\nElizabeth made no answer; and without attempting to persuade her\r\nLadyship to return into the house, walked quietly into it herself. She\r\nheard the carriage drive away as she proceeded upstairs. Her mother\r\nimpatiently met her at the door of her dressing-room, to ask why Lady\r\nCatherine would not come in again and rest herself.\r\n\r\n“She did not choose it,” said her daughter; “she would go.”\r\n\r\n“She is a very fine-looking woman! and her calling here was prodigiously\r\ncivil! for she only came, I suppose, to tell us the Collinses were well.\r\nShe is on her road somewhere, I dare say; and so, passing through\r\nMeryton, thought she might as well call on you. I suppose she had\r\nnothing particular to say to you, Lizzy?”\r\n\r\nElizabeth was forced to give in to a little falsehood here; for to\r\nacknowledge the substance of their conversation was impossible.\r\n\r\n\r\n\r\n\r\n[Illustration:\r\n\r\n “But now it comes out”\r\n]\r\n\r\n\r\n\r\n\r\nCHAPTER LVII.\r\n\r\n\r\n[Illustration]\r\n\r\nThe discomposure of spirits which this extraordinary visit threw\r\nElizabeth into could not be easily overcome; nor could she for many\r\nhours learn to think of it less than incessantly. Lady Catherine, it\r\nappeared, had actually taken the trouble of this journey from Rosings\r\nfor the sole purpose of breaking off her supposed engagement with Mr.\r\nDarcy. It was a rational scheme, to be sure! but from what the report of\r\ntheir engagement could originate, Elizabeth was at a loss to imagine;\r\ntill she recollected that _his_ being the intimate friend of Bingley,\r\nand _her_ being the sister of Jane, was enough, at a time when the\r\nexpectation of one wedding made everybody eager for another, to supply\r\nthe idea. She had not herself forgotten to feel that the marriage of her\r\nsister must bring them more frequently together. And her neighbours at\r\nLucas Lodge, therefore, (for through their communication with the\r\nCollinses, the report, she concluded, had reached Lady Catherine,) had\r\nonly set _that_ down as almost certain and immediate which _she_ had\r\nlooked forward to as possible at some future time.\r\n\r\nIn revolving Lady Catherine’s expressions, however, she could not help\r\nfeeling some uneasiness as to the possible consequence of her persisting\r\nin this interference. From what she had said of her resolution to\r\nprevent the marriage, it occurred to Elizabeth that she must meditate an\r\napplication to her nephew; and how he might take a similar\r\nrepresentation of the evils attached to a connection with her she dared\r\nnot pronounce. She knew not the exact degree of his affection for his\r\naunt, or his dependence on her judgment, but it was natural to suppose\r\nthat he thought much higher of her Ladyship than _she_ could do; and it\r\nwas certain, that in enumerating the miseries of a marriage with _one_\r\nwhose immediate connections were so unequal to his own, his aunt would\r\naddress him on his weakest side. With his notions of dignity, he would\r\nprobably feel that the arguments, which to Elizabeth had appeared weak\r\nand ridiculous, contained much good sense and solid reasoning.\r\n\r\nIf he had been wavering before, as to what he should do, which had often\r\nseemed likely, the advice and entreaty of so near a relation might\r\nsettle every doubt, and determine him at once to be as happy as dignity\r\nunblemished could make him. In that case he would return no more. Lady\r\nCatherine might see him in her way through town; and his engagement to\r\nBingley of coming again to Netherfield must give way.\r\n\r\n“If, therefore, an excuse for not keeping his promise should come to his\r\nfriend within a few days,” she added, “I shall know how to understand\r\nit. I shall then give over every expectation, every wish of his\r\nconstancy. If he is satisfied with only regretting me, when he might\r\nhave obtained my affections and hand, I shall soon cease to regret him\r\nat all.”\r\n\r\nThe surprise of the rest of the family, on hearing who their visitor had\r\nbeen, was very great: but they obligingly satisfied it with the same\r\nkind of supposition which had appeased Mrs. Bennet’s curiosity; and\r\nElizabeth was spared from much teasing on the subject.\r\n\r\nThe next morning, as she was going down stairs, she was met by her\r\nfather, who came out of his library with a letter in his hand.\r\n\r\n“Lizzy,” said he, “I was going to look for you: come into my room.”\r\n\r\nShe followed him thither; and her curiosity to know what he had to tell\r\nher was heightened by the supposition of its being in some manner\r\nconnected with the letter he held. It suddenly struck her that it might\r\nbe from Lady Catherine, and she anticipated with dismay all the\r\nconsequent explanations.\r\n\r\nShe followed her father to the fireplace, and they both sat down. He\r\nthen said,--\r\n\r\n“I have received a letter this morning that has astonished me\r\nexceedingly. As it principally concerns yourself, you ought to know its\r\ncontents. I did not know before that I had _two_ daughters on the brink\r\nof matrimony. Let me congratulate you on a very important conquest.”\r\n\r\nThe colour now rushed into Elizabeth’s cheeks in the instantaneous\r\nconviction of its being a letter from the nephew, instead of the aunt;\r\nand she was undetermined whether most to be pleased that he explained\r\nhimself at all, or offended that his letter was not rather addressed to\r\nherself, when her father continued,--\r\n\r\n“You look conscious. Young ladies have great penetration in such matters\r\nas these; but I think I may defy even _your_ sagacity to discover the\r\nname of your admirer. This letter is from Mr. Collins.”\r\n\r\n“From Mr. Collins! and what can _he_ have to say?”\r\n\r\n“Something very much to the purpose, of course. He begins with\r\ncongratulations on the approaching nuptials of my eldest daughter, of\r\nwhich, it seems, he has been told by some of the good-natured, gossiping\r\nLucases. I shall not sport with your impatience by reading what he says\r\non that point. What relates to yourself is as follows:--‘Having thus\r\noffered you the sincere congratulations of Mrs. Collins and myself on\r\nthis happy event, let me now add a short hint on the subject of another,\r\nof which we have been advertised by the same authority. Your daughter\r\nElizabeth, it is presumed, will not long bear the name of Bennet, after\r\nher eldest sister has resigned it; and the chosen partner of her fate\r\nmay be reasonably looked up to as one of the most illustrious personages\r\nin this land.’ Can you possibly guess, Lizzy, who is meant by this?\r\n‘This young gentleman is blessed, in a peculiar way, with everything the\r\nheart of mortal can most desire,--splendid property, noble kindred, and\r\nextensive patronage. Yet, in spite of all these temptations, let me warn\r\nmy cousin Elizabeth, and yourself, of what evils you may incur by a\r\nprecipitate closure with this gentleman’s proposals, which, of course,\r\nyou will be inclined to take immediate advantage of.’ Have you any idea,\r\nLizzy, who this gentleman is? But now it comes out. ‘My motive for\r\ncautioning you is as follows:--We have reason to imagine that his aunt,\r\nLady Catherine de Bourgh, does not look on the match with a friendly\r\neye.’ _Mr. Darcy_, you see, is the man! Now, Lizzy, I think I _have_\r\nsurprised you. Could he, or the Lucases, have pitched on any man, within\r\nthe circle of our acquaintance, whose name would have given the lie more\r\neffectually to what they related? Mr. Darcy, who never looks at any\r\nwoman but to see a blemish, and who probably never looked at _you_ in\r\nhis life! It is admirable!”\r\n\r\nElizabeth tried to join in her father’s pleasantry, but could only force\r\none most reluctant smile. Never had his wit been directed in a manner so\r\nlittle agreeable to her.\r\n\r\n“Are you not diverted?”\r\n\r\n“Oh, yes. Pray read on.”\r\n\r\n“‘After mentioning the likelihood of this marriage to her Ladyship last\r\nnight, she immediately, with her usual condescension, expressed what she\r\nfelt on the occasion; when it became apparent, that, on the score of\r\nsome family objections on the part of my cousin, she would never give\r\nher consent to what she termed so disgraceful a match. I thought it my\r\nduty to give the speediest intelligence of this to my cousin, that she\r\nand her noble admirer may be aware of what they are about, and not run\r\nhastily into a marriage which has not been properly sanctioned.’ Mr.\r\nCollins, moreover, adds, ‘I am truly rejoiced that my cousin Lydia’s sad\r\nbusiness has been so well hushed up, and am only concerned that their\r\nliving together before the marriage took place should be so generally\r\nknown. I must not, however, neglect the duties of my station, or refrain\r\nfrom declaring my amazement, at hearing that you received the young\r\ncouple into your house as soon as they were married. It was an\r\nencouragement of vice; and had I been the rector of Longbourn, I should\r\nvery strenuously have opposed it. You ought certainly to forgive them as\r\na Christian, but never to admit them in your sight, or allow their\r\nnames to be mentioned in your hearing.’ _That_ is his notion of\r\nChristian forgiveness! The rest of his letter is only about his dear\r\nCharlotte’s situation, and his expectation of a young olive-branch. But,\r\nLizzy, you look as if you did not enjoy it. You are not going to be\r\n_missish_, I hope, and pretend to be affronted at an idle report. For\r\nwhat do we live, but to make sport for our neighbours, and laugh at them\r\nin our turn?”\r\n\r\n“Oh,” cried Elizabeth, “I am exceedingly diverted. But it is so\r\nstrange!”\r\n\r\n“Yes, _that_ is what makes it amusing. Had they fixed on any other man\r\nit would have been nothing; but _his_ perfect indifference and _your_\r\npointed dislike make it so delightfully absurd! Much as I abominate\r\nwriting, I would not give up Mr. Collins’s correspondence for any\r\nconsideration. Nay, when I read a letter of his, I cannot help giving\r\nhim the preference even over Wickham, much as I value the impudence and\r\nhypocrisy of my son-in-law. And pray, Lizzy, what said Lady Catherine\r\nabout this report? Did she call to refuse her consent?”\r\n\r\nTo this question his daughter replied only with a laugh; and as it had\r\nbeen asked without the least suspicion, she was not distressed by his\r\nrepeating it. Elizabeth had never been more at a loss to make her\r\nfeelings appear what they were not. It was necessary to laugh when she\r\nwould rather have cried. Her father had most cruelly mortified her by\r\nwhat he said of Mr. Darcy’s indifference; and she could do nothing but\r\nwonder at such a want of penetration, or fear that, perhaps, instead of\r\nhis seeing too _little_, she might have fancied too _much_.\r\n\r\n\r\n\r\n\r\n[Illustration:\r\n\r\n“The efforts of his aunt”\r\n\r\n[_Copyright 1894 by George Allen._]]\r\n\r\n\r\n\r\n\r\nCHAPTER LVIII.\r\n\r\n\r\n[Illustration]\r\n\r\nInstead of receiving any such letter of excuse from his friend, as\r\nElizabeth half expected Mr. Bingley to do, he was able to bring Darcy\r\nwith him to Longbourn before many days had passed after Lady Catherine’s\r\nvisit. The gentlemen arrived early; and, before Mrs. Bennet had time to\r\ntell him of their having seen his aunt, of which her daughter sat in\r\nmomentary dread, Bingley, who wanted to be alone with Jane, proposed\r\ntheir all walking out. It was agreed to. Mrs. Bennet was not in the\r\nhabit of walking, Mary could never spare time, but the remaining five\r\nset off together. Bingley and Jane, however, soon allowed the others to\r\noutstrip them. They lagged behind, while Elizabeth, Kitty, and Darcy\r\nwere to entertain each other. Very little was said by either; Kitty was\r\ntoo much afraid of him to talk; Elizabeth was secretly forming a\r\ndesperate resolution; and, perhaps, he might be doing the same.\r\n\r\nThey walked towards the Lucases’, because Kitty wished to call upon\r\nMaria; and as Elizabeth saw no occasion for making it a general concern,\r\nwhen Kitty left them she went boldly on with him alone. Now was the\r\nmoment for her resolution to be executed; and while her courage was\r\nhigh, she immediately said,--\r\n\r\n“Mr. Darcy, I am a very selfish creature, and for the sake of giving\r\nrelief to my own feelings care not how much I may be wounding yours. I\r\ncan no longer help thanking you for your unexampled kindness to my poor\r\nsister. Ever since I have known it I have been most anxious to\r\nacknowledge to you how gratefully I feel it. Were it known to the rest\r\nof my family I should not have merely my own gratitude to express.”\r\n\r\n“I am sorry, exceedingly sorry,” replied Darcy, in a tone of surprise\r\nand emotion, “that you have ever been informed of what may, in a\r\nmistaken light, have given you uneasiness. I did not think Mrs. Gardiner\r\nwas so little to be trusted.”\r\n\r\n“You must not blame my aunt. Lydia’s thoughtlessness first betrayed to\r\nme that you had been concerned in the matter; and, of course, I could\r\nnot rest till I knew the particulars. Let me thank you again and again,\r\nin the name of all my family, for that generous compassion which induced\r\nyou to take so much trouble, and bear so many mortifications, for the\r\nsake of discovering them.”\r\n\r\n“If you _will_ thank me,” he replied, “let it be for yourself alone.\r\nThat the wish of giving happiness to you might add force to the other\r\ninducements which led me on, I shall not attempt to deny. But your\r\n_family_ owe me nothing. Much as I respect them, I believe I thought\r\nonly of _you_.”\r\n\r\nElizabeth was too much embarrassed to say a word. After a short pause,\r\nher companion added, “You are too generous to trifle with me. If your\r\nfeelings are still what they were last April, tell me so at once. _My_\r\naffections and wishes are unchanged; but one word from you will silence\r\nme on this subject for ever.”\r\n\r\nElizabeth, feeling all the more than common awkwardness and anxiety of\r\nhis situation, now forced herself to speak; and immediately, though not\r\nvery fluently, gave him to understand that her sentiments had undergone\r\nso material a change since the period to which he alluded, as to make\r\nher receive with gratitude and pleasure his present assurances. The\r\nhappiness which this reply produced was such as he had probably never\r\nfelt before; and he expressed himself on the occasion as sensibly and as\r\nwarmly as a man violently in love can be supposed to do. Had Elizabeth\r\nbeen able to encounter his eyes, she might have seen how well the\r\nexpression of heartfelt delight diffused over his face became him: but\r\nthough she could not look she could listen; and he told her of feelings\r\nwhich, in proving of what importance she was to him, made his affection\r\nevery moment more valuable.\r\n\r\nThey walked on without knowing in what direction. There was too much to\r\nbe thought, and felt, and said, for attention to any other objects. She\r\nsoon learnt that they were indebted for their present good understanding\r\nto the efforts of his aunt, who _did_ call on him in her return through\r\nLondon, and there relate her journey to Longbourn, its motive, and the\r\nsubstance of her conversation with Elizabeth; dwelling emphatically on\r\nevery expression of the latter, which, in her Ladyship’s apprehension,\r\npeculiarly denoted her perverseness and assurance, in the belief that\r\nsuch a relation must assist her endeavours to obtain that promise from\r\nher nephew which _she_ had refused to give. But, unluckily for her\r\nLadyship, its effect had been exactly contrariwise.\r\n\r\n“It taught me to hope,” said he, “as I had scarcely ever allowed myself\r\nto hope before. I knew enough of your disposition to be certain, that\r\nhad you been absolutely, irrevocably decided against me, you would have\r\nacknowledged it to Lady Catherine frankly and openly.”\r\n\r\nElizabeth coloured and laughed as she replied, “Yes, you know enough of\r\nmy _frankness_ to believe me capable of _that_. After abusing you so\r\nabominably to your face, I could have no scruple in abusing you to all\r\nyour relations.”\r\n\r\n“What did you say of me that I did not deserve? For though your\r\naccusations were ill-founded, formed on mistaken premises, my behaviour\r\nto you at the time had merited the severest reproof. It was\r\nunpardonable. I cannot think of it without abhorrence.”\r\n\r\n“We will not quarrel for the greater share of blame annexed to that\r\nevening,” said Elizabeth. “The conduct of neither, if strictly\r\nexamined, will be irreproachable; but since then we have both, I hope,\r\nimproved in civility.”\r\n\r\n“I cannot be so easily reconciled to myself. The recollection of what I\r\nthen said, of my conduct, my manners, my expressions during the whole of\r\nit, is now, and has been many months, inexpressibly painful to me. Your\r\nreproof, so well applied, I shall never forget: ‘Had you behaved in a\r\nmore gentlemanlike manner.’ Those were your words. You know not, you can\r\nscarcely conceive, how they have tortured me; though it was some time, I\r\nconfess, before I was reasonable enough to allow their justice.”\r\n\r\n“I was certainly very far from expecting them to make so strong an\r\nimpression. I had not the smallest idea of their being ever felt in such\r\na way.”\r\n\r\n“I can easily believe it. You thought me then devoid of every proper\r\nfeeling, I am sure you did. The turn of your countenance I shall never\r\nforget, as you said that I could not have addressed you in any possible\r\nway that would induce you to accept me.”\r\n\r\n“Oh, do not repeat what I then said. These recollections will not do at\r\nall. I assure you that I have long been most heartily ashamed of it.”\r\n\r\nDarcy mentioned his letter. “Did it,” said he,--“did it _soon_ make you\r\nthink better of me? Did you, on reading it, give any credit to its\r\ncontents?”\r\n\r\nShe explained what its effects on her had been, and how gradually all\r\nher former prejudices had been removed.\r\n\r\n“I knew,” said he, “that what I wrote must give you pain, but it was\r\nnecessary. I hope you have destroyed the letter. There was one part,\r\nespecially the opening of it, which I should dread your having the power\r\nof reading again. I can remember some expressions which might justly\r\nmake you hate me.”\r\n\r\n“The letter shall certainly be burnt, if you believe it essential to the\r\npreservation of my regard; but, though we have both reason to think my\r\nopinions not entirely unalterable, they are not, I hope, quite so easily\r\nchanged as that implies.”\r\n\r\n“When I wrote that letter,” replied Darcy, “I believed myself perfectly\r\ncalm and cool; but I am since convinced that it was written in a\r\ndreadful bitterness of spirit.”\r\n\r\n“The letter, perhaps, began in bitterness, but it did not end so. The\r\nadieu is charity itself. But think no more of the letter. The feelings\r\nof the person who wrote and the person who received it are now so widely\r\ndifferent from what they were then, that every unpleasant circumstance\r\nattending it ought to be forgotten. You must learn some of my\r\nphilosophy. Think only of the past as its remembrance gives you\r\npleasure.”\r\n\r\n“I cannot give you credit for any philosophy of the kind. _Your_\r\nretrospections must be so totally void of reproach, that the contentment\r\narising from them is not of philosophy, but, what is much better, of\r\nignorance. But with _me_, it is not so. Painful recollections will\r\nintrude, which cannot, which ought not to be repelled. I have been a\r\nselfish being all my life, in practice, though not in principle. As a\r\nchild I was taught what was _right_, but I was not taught to correct my\r\ntemper. I was given good principles, but left to follow them in pride\r\nand conceit. Unfortunately an only son (for many years an only _child_),\r\nI was spoiled by my parents, who, though good themselves, (my father\r\nparticularly, all that was benevolent and amiable,) allowed, encouraged,\r\nalmost taught me to be selfish and overbearing, to care for none beyond\r\nmy own family circle, to think meanly of all the rest of the world, to\r\n_wish_ at least to think meanly of their sense and worth compared with\r\nmy own. Such I was, from eight to eight-and-twenty; and such I might\r\nstill have been but for you, dearest, loveliest Elizabeth! What do I not\r\nowe you! You taught me a lesson, hard indeed at first, but most\r\nadvantageous. By you, I was properly humbled. I came to you without a\r\ndoubt of my reception. You showed me how insufficient were all my\r\npretensions to please a woman worthy of being pleased.”\r\n\r\n“Had you then persuaded yourself that I should?”\r\n\r\n“Indeed I had. What will you think of my vanity? I believed you to be\r\nwishing, expecting my addresses.”\r\n\r\n“My manners must have been in fault, but not intentionally, I assure\r\nyou. I never meant to deceive you, but my spirits might often lead me\r\nwrong. How you must have hated me after _that_ evening!”\r\n\r\n“Hate you! I was angry, perhaps, at first, but my anger soon began to\r\ntake a proper direction.”\r\n\r\n“I am almost afraid of asking what you thought of me when we met at\r\nPemberley. You blamed me for coming?”\r\n\r\n“No, indeed, I felt nothing but surprise.”\r\n\r\n“Your surprise could not be greater than _mine_ in being noticed by you.\r\nMy conscience told me that I deserved no extraordinary politeness, and I\r\nconfess that I did not expect to receive _more_ than my due.”\r\n\r\n“My object _then_,” replied Darcy, “was to show you, by every civility\r\nin my power, that I was not so mean as to resent the past; and I hoped\r\nto obtain your forgiveness, to lessen your ill opinion, by letting you\r\nsee that your reproofs had been attended to. How soon any other wishes\r\nintroduced themselves, I can hardly tell, but I believe in about half\r\nan hour after I had seen you.”\r\n\r\nHe then told her of Georgiana’s delight in her acquaintance, and of her\r\ndisappointment at its sudden interruption; which naturally leading to\r\nthe cause of that interruption, she soon learnt that his resolution of\r\nfollowing her from Derbyshire in quest of her sister had been formed\r\nbefore he quitted the inn, and that his gravity and thoughtfulness there\r\nhad arisen from no other struggles than what such a purpose must\r\ncomprehend.\r\n\r\nShe expressed her gratitude again, but it was too painful a subject to\r\neach to be dwelt on farther.\r\n\r\nAfter walking several miles in a leisurely manner, and too busy to know\r\nanything about it, they found at last, on examining their watches, that\r\nit was time to be at home.\r\n\r\n“What could have become of Mr. Bingley and Jane?” was a wonder which\r\nintroduced the discussion of _their_ affairs. Darcy was delighted with\r\ntheir engagement; his friend had given him the earliest information of\r\nit.\r\n\r\n“I must ask whether you were surprised?” said Elizabeth.\r\n\r\n“Not at all. When I went away, I felt that it would soon happen.”\r\n\r\n“That is to say, you had given your permission. I guessed as much.” And\r\nthough he exclaimed at the term, she found that it had been pretty much\r\nthe case.\r\n\r\n“On the evening before my going to London,” said he, “I made a\r\nconfession to him, which I believe I ought to have made long ago. I told\r\nhim of all that had occurred to make my former interference in his\r\naffairs absurd and impertinent. His surprise was great. He had never had\r\nthe slightest suspicion. I told him, moreover, that I believed myself\r\nmistaken in supposing, as I had done, that your sister was indifferent\r\nto him; and as I could easily perceive that his attachment to her was\r\nunabated, I felt no doubt of their happiness together.”\r\n\r\nElizabeth could not help smiling at his easy manner of directing his\r\nfriend.\r\n\r\n“Did you speak from your own observation,” said she, “when you told him\r\nthat my sister loved him, or merely from my information last spring?”\r\n\r\n“From the former. I had narrowly observed her, during the two visits\r\nwhich I had lately made her here; and I was convinced of her affection.”\r\n\r\n“And your assurance of it, I suppose, carried immediate conviction to\r\nhim.”\r\n\r\n“It did. Bingley is most unaffectedly modest. His diffidence had\r\nprevented his depending on his own judgment in so anxious a case, but\r\nhis reliance on mine made everything easy. I was obliged to confess one\r\nthing, which for a time, and not unjustly, offended him. I could not\r\nallow myself to conceal that your sister had been in town three months\r\nlast winter, that I had known it, and purposely kept it from him. He was\r\nangry. But his anger, I am persuaded, lasted no longer than he remained\r\nin any doubt of your sister’s sentiments. He has heartily forgiven me\r\nnow.”\r\n\r\nElizabeth longed to observe that Mr. Bingley had been a most delightful\r\nfriend; so easily guided that his worth was invaluable; but she checked\r\nherself. She remembered that he had yet to learn to be laughed at, and\r\nit was rather too early to begin. In anticipating the happiness of\r\nBingley, which of course was to be inferior only to his own, he\r\ncontinued the conversation till they reached the house. In the hall they\r\nparted.\r\n\r\n\r\n\r\n\r\n[Illustration:\r\n\r\n “Unable to utter a syllable”\r\n\r\n[_Copyright 1894 by George Allen._]]\r\n\r\n\r\n\r\n\r\nCHAPTER LIX.\r\n\r\n\r\n[Illustration]\r\n\r\n“My dear Lizzy, where can you have been walking to?” was a question\r\nwhich Elizabeth received from Jane as soon as she entered the room, and\r\nfrom all the others when they sat down to table. She had only to say in\r\nreply, that they had wandered about till she was beyond her own\r\nknowledge. She coloured as she spoke; but neither that, nor anything\r\nelse, awakened a suspicion of the truth.\r\n\r\nThe evening passed quietly, unmarked by anything extraordinary. The\r\nacknowledged lovers talked and laughed; the unacknowledged were silent.\r\nDarcy was not of a disposition in which happiness overflows in mirth;\r\nand Elizabeth, agitated and confused, rather _knew_ that she was happy\r\nthan _felt_ herself to be so; for, besides the immediate embarrassment,\r\nthere were other evils before her. She anticipated what would be felt in\r\nthe family when her situation became known: she was aware that no one\r\nliked him but Jane; and even feared that with the others it was a\r\n_dislike_ which not all his fortune and consequence might do away.\r\n\r\nAt night she opened her heart to Jane. Though suspicion was very far\r\nfrom Miss Bennet’s general habits, she was absolutely incredulous here.\r\n\r\n“You are joking, Lizzy. This cannot be! Engaged to Mr. Darcy! No, no,\r\nyou shall not deceive me: I know it to be impossible.”\r\n\r\n“This is a wretched beginning, indeed! My sole dependence was on you;\r\nand I am sure nobody else will believe me, if you do not. Yet, indeed, I\r\nam in earnest. I speak nothing but the truth. He still loves me, and we\r\nare engaged.”\r\n\r\nJane looked at her doubtingly. “Oh, Lizzy! it cannot be. I know how much\r\nyou dislike him.”\r\n\r\n“You know nothing of the matter. _That_ is all to be forgot. Perhaps I\r\ndid not always love him so well as I do now; but in such cases as these\r\na good memory is unpardonable. This is the last time I shall ever\r\nremember it myself.”\r\n\r\nMiss Bennet still looked all amazement. Elizabeth again, and more\r\nseriously, assured her of its truth.\r\n\r\n“Good heaven! can it be really so? Yet now I must believe you,” cried\r\nJane. “My dear, dear Lizzy, I would, I do congratulate you; but are you\r\ncertain--forgive the question--are you quite certain that you can be\r\nhappy with him?”\r\n\r\n“There can be no doubt of that. It is settled between us already that we\r\nare to be the happiest couple in the world. But are you pleased, Jane?\r\nShall you like to have such a brother?”\r\n\r\n“Very, very much. Nothing could give either Bingley or myself more\r\ndelight. But we considered it, we talked of it as impossible. And do you\r\nreally love him quite well enough? Oh, Lizzy! do anything rather than\r\nmarry without affection. Are you quite sure that you feel what you ought\r\nto do?”\r\n\r\n“Oh, yes! You will only think I feel _more_ than I ought to do when I\r\ntell you all.”\r\n\r\n“What do you mean?”\r\n\r\n“Why, I must confess that I love him better than I do Bingley. I am\r\nafraid you will be angry.”\r\n\r\n“My dearest sister, now be, _be_ serious. I want to talk very seriously.\r\nLet me know everything that I am to know without delay. Will you tell me\r\nhow long you have loved him?”\r\n\r\n“It has been coming on so gradually, that I hardly know when it began;\r\nbut I believe I must date it from my first seeing his beautiful grounds\r\nat Pemberley.”\r\n\r\nAnother entreaty that she would be serious, however, produced the\r\ndesired effect; and she soon satisfied Jane by her solemn assurances of\r\nattachment. When convinced on that article, Miss Bennet had nothing\r\nfurther to wish.\r\n\r\n“Now I am quite happy,” said she, “for you will be as happy as myself. I\r\nalways had a value for him. Were it for nothing but his love of you, I\r\nmust always have esteemed him; but now, as Bingley’s friend and your\r\nhusband, there can be only Bingley and yourself more dear to me. But,\r\nLizzy, you have been very sly, very reserved with me. How little did you\r\ntell me of what passed at Pemberley and Lambton! I owe all that I know\r\nof it to another, not to you.”\r\n\r\nElizabeth told her the motives of her secrecy. She had been unwilling to\r\nmention Bingley; and the unsettled state of her own feelings had made\r\nher equally avoid the name of his friend: but now she would no longer\r\nconceal from her his share in Lydia’s marriage. All was acknowledged,\r\nand half the night spent in conversation.\r\n\r\n“Good gracious!” cried Mrs. Bennet, as she stood at a window the next\r\nmorning, “if that disagreeable Mr. Darcy is not coming here again with\r\nour dear Bingley! What can he mean by being so tiresome as to be always\r\ncoming here? I had no notion but he would go a-shooting, or something or\r\nother, and not disturb us with his company. What shall we do with him?\r\nLizzy, you must walk out with him again, that he may not be in Bingley’s\r\nway.”\r\n\r\nElizabeth could hardly help laughing at so convenient a proposal; yet\r\nwas really vexed that her mother should be always giving him such an\r\nepithet.\r\n\r\nAs soon as they entered, Bingley looked at her so expressively, and\r\nshook hands with such warmth, as left no doubt of his good information;\r\nand he soon afterwards said aloud, “Mrs. Bennet, have you no more lanes\r\nhereabouts in which Lizzy may lose her way again to-day?”\r\n\r\n“I advise Mr. Darcy, and Lizzy, and Kitty,” said Mrs. Bennet, “to walk\r\nto Oakham Mount this morning. It is a nice long walk, and Mr. Darcy has\r\nnever seen the view.”\r\n\r\n“It may do very well for the others,” replied Mr. Bingley; “but I am\r\nsure it will be too much for Kitty. Won’t it, Kitty?”\r\n\r\nKitty owned that she had rather stay at home. Darcy professed a great\r\ncuriosity to see the view from the Mount, and Elizabeth silently\r\nconsented. As she went upstairs to get ready, Mrs. Bennet followed her,\r\nsaying,--\r\n\r\n“I am quite sorry, Lizzy, that you should be forced to have that\r\ndisagreeable man all to yourself; but I hope you will not mind it. It is\r\nall for Jane’s sake, you know; and there is no occasion for talking to\r\nhim except just now and then; so do not put yourself to inconvenience.”\r\n\r\nDuring their walk, it was resolved that Mr. Bennet’s consent should be\r\nasked in the course of the evening: Elizabeth reserved to herself the\r\napplication for her mother’s. She could not determine how her mother\r\nwould take it; sometimes doubting whether all his wealth and grandeur\r\nwould be enough to overcome her abhorrence of the man; but whether she\r\nwere violently set against the match, or violently delighted with it, it\r\nwas certain that her manner would be equally ill adapted to do credit to\r\nher sense; and she could no more bear that Mr. Darcy should hear the\r\nfirst raptures of her joy, than the first vehemence of her\r\ndisapprobation.\r\n\r\nIn the evening, soon after Mr. Bennet withdrew to the library, she saw\r\nMr. Darcy rise also and follow him, and her agitation on seeing it was\r\nextreme. She did not fear her father’s opposition, but he was going to\r\nbe made unhappy, and that it should be through her means; that _she_,\r\nhis favourite child, should be distressing him by her choice, should be\r\nfilling him with fears and regrets in disposing of her, was a wretched\r\nreflection, and she sat in misery till Mr. Darcy appeared again, when,\r\nlooking at him, she was a little relieved by his smile. In a few minutes\r\nhe approached the table where she was sitting with Kitty; and, while\r\npretending to admire her work, said in a whisper, “Go to your father; he\r\nwants you in the library.” She was gone directly.\r\n\r\nHer father was walking about the room, looking grave and anxious.\r\n“Lizzy,” said he, “what are you doing? Are you out of your senses to be\r\naccepting this man? Have not you always hated him?”\r\n\r\nHow earnestly did she then wish that her former opinions had been more\r\nreasonable, her expressions more moderate! It would have spared her from\r\nexplanations and professions which it was exceedingly awkward to give;\r\nbut they were now necessary, and she assured him, with some confusion,\r\nof her attachment to Mr. Darcy.\r\n\r\n“Or, in other words, you are determined to have him. He is rich, to be\r\nsure, and you may have more fine clothes and fine carriages than Jane.\r\nBut will they make you happy?”\r\n\r\n“Have you any other objection,” said Elizabeth, “than your belief of my\r\nindifference?”\r\n\r\n“None at all. We all know him to be a proud, unpleasant sort of man; but\r\nthis would be nothing if you really liked him.”\r\n\r\n“I do, I do like him,” she replied, with tears in her eyes; “I love him.\r\nIndeed he has no improper pride. He is perfectly amiable. You do not\r\nknow what he really is; then pray do not pain me by speaking of him in\r\nsuch terms.”\r\n\r\n“Lizzy,” said her father, “I have given him my consent. He is the kind\r\nof man, indeed, to whom I should never dare refuse anything, which he\r\ncondescended to ask. I now give it to _you_, if you are resolved on\r\nhaving him. But let me advise you to think better of it. I know your\r\ndisposition, Lizzy. I know that you could be neither happy nor\r\nrespectable, unless you truly esteemed your husband, unless you looked\r\nup to him as a superior. Your lively talents would place you in the\r\ngreatest danger in an unequal marriage. You could scarcely escape\r\ndiscredit and misery. My child, let me not have the grief of seeing\r\n_you_ unable to respect your partner in life. You know not what you are\r\nabout.”\r\n\r\nElizabeth, still more affected, was earnest and solemn in her reply;\r\nand, at length, by repeated assurances that Mr. Darcy was really the\r\nobject of her choice, by explaining the gradual change which her\r\nestimation of him had undergone, relating her absolute certainty that\r\nhis affection was not the work of a day, but had stood the test of many\r\nmonths’ suspense, and enumerating with energy all his good qualities,\r\nshe did conquer her father’s incredulity, and reconcile him to the\r\nmatch.\r\n\r\n“Well, my dear,” said he, when she ceased speaking, “I have no more to\r\nsay. If this be the case, he deserves you. I could not have parted with\r\nyou, my Lizzy, to anyone less worthy.”\r\n\r\nTo complete the favourable impression, she then told him what Mr. Darcy\r\nhad voluntarily done for Lydia. He heard her with astonishment.\r\n\r\n“This is an evening of wonders, indeed! And so, Darcy did everything;\r\nmade up the match, gave the money, paid the fellow’s debts, and got him\r\nhis commission! So much the better. It will save me a world of trouble\r\nand economy. Had it been your uncle’s doing, I must and _would_ have\r\npaid him; but these violent young lovers carry everything their own\r\nway. I shall offer to pay him to-morrow, he will rant and storm about\r\nhis love for you, and there will be an end of the matter.”\r\n\r\nHe then recollected her embarrassment a few days before on his reading\r\nMr. Collins’s letter; and after laughing at her some time, allowed her\r\nat last to go, saying, as she quitted the room, “If any young men come\r\nfor Mary or Kitty, send them in, for I am quite at leisure.”\r\n\r\nElizabeth’s mind was now relieved from a very heavy weight; and, after\r\nhalf an hour’s quiet reflection in her own room, she was able to join\r\nthe others with tolerable composure. Everything was too recent for\r\ngaiety, but the evening passed tranquilly away; there was no longer\r\nanything material to be dreaded, and the comfort of ease and familiarity\r\nwould come in time.\r\n\r\nWhen her mother went up to her dressing-room at night, she followed her,\r\nand made the important communication. Its effect was most extraordinary;\r\nfor, on first hearing it, Mrs. Bennet sat quite still, and unable to\r\nutter a syllable. Nor was it under many, many minutes, that she could\r\ncomprehend what she heard, though not in general backward to credit what\r\nwas for the advantage of her family, or that came in the shape of a\r\nlover to any of them. She began at length to recover, to fidget about in\r\nher chair, get up, sit down again, wonder, and bless herself.\r\n\r\n“Good gracious! Lord bless me! only think! dear me! Mr. Darcy! Who would\r\nhave thought it? And is it really true? Oh, my sweetest Lizzy! how rich\r\nand how great you will be! What pin-money, what jewels, what carriages\r\nyou will have! Jane’s is nothing to it--nothing at all. I am so\r\npleased--so happy. Such a charming man! so handsome! so tall! Oh, my\r\ndear Lizzy! pray apologize for my having disliked him so much before. I\r\nhope he will overlook it. Dear, dear Lizzy. A house in town! Everything\r\nthat is charming! Three daughters married! Ten thousand a year! Oh,\r\nLord! what will become of me? I shall go distracted.”\r\n\r\nThis was enough to prove that her approbation need not be doubted; and\r\nElizabeth, rejoicing that such an effusion was heard only by herself,\r\nsoon went away. But before she had been three minutes in her own room,\r\nher mother followed her.\r\n\r\n“My dearest child,” she cried, “I can think of nothing else. Ten\r\nthousand a year, and very likely more! ’Tis as good as a lord! And a\r\nspecial licence--you must and shall be married by a special licence.\r\nBut, my dearest love, tell me what dish Mr. Darcy is particularly fond\r\nof, that I may have it to-morrow.”\r\n\r\nThis was a sad omen of what her mother’s behaviour to the gentleman\r\nhimself might be; and Elizabeth found that, though in the certain\r\npossession of his warmest affection, and secure of her relations’\r\nconsent, there was still something to be wished for. But the morrow\r\npassed off much better than she expected; for Mrs. Bennet luckily stood\r\nin such awe of her intended son-in-law, that she ventured not to speak\r\nto him, unless it was in her power to offer him any attention, or mark\r\nher deference for his opinion.\r\n\r\nElizabeth had the satisfaction of seeing her father taking pains to get\r\nacquainted with him; and Mr. Bennet soon assured her that he was rising\r\nevery hour in his esteem.\r\n\r\n“I admire all my three sons-in-law highly,” said he. “Wickham, perhaps,\r\nis my favourite; but I think I shall like _your_ husband quite as well\r\nas Jane’s.”\r\n\r\n\r\n\r\n\r\n[Illustration:\r\n\r\n“The obsequious civility.”\r\n\r\n[_Copyright 1894 by George Allen._]]\r\n\r\n\r\n\r\n\r\nCHAPTER LX.\r\n\r\n\r\n[Illustration]\r\n\r\nElizabeth’s spirits soon rising to playfulness again, she wanted Mr.\r\nDarcy to account for his having ever fallen in love with her. “How could\r\nyou begin?” said she. “I can comprehend your going on charmingly, when\r\nyou had once made a beginning; but what could set you off in the first\r\nplace?”\r\n\r\n“I cannot fix on the hour, or the spot, or the look, or the words, which\r\nlaid the foundation. It is too long ago. I was in the middle before I\r\nknew that I _had_ begun.”\r\n\r\n“My beauty you had early withstood, and as for my manners--my behaviour\r\nto _you_ was at least always bordering on the uncivil, and I never spoke\r\nto you without rather wishing to give you pain than not. Now, be\r\nsincere; did you admire me for my impertinence?”\r\n\r\n“For the liveliness of your mind I did.”\r\n\r\n“You may as well call it impertinence at once. It was very little less.\r\nThe fact is, that you were sick of civility, of deference, of officious\r\nattention. You were disgusted with the women who were always speaking,\r\nand looking, and thinking for _your_ approbation alone. I roused and\r\ninterested you, because I was so unlike _them_. Had you not been really\r\namiable you would have hated me for it: but in spite of the pains you\r\ntook to disguise yourself, your feelings were always noble and just; and\r\nin your heart you thoroughly despised the persons who so assiduously\r\ncourted you. There--I have saved you the trouble of accounting for it;\r\nand really, all things considered, I begin to think it perfectly\r\nreasonable. To be sure you know no actual good of me--but nobody thinks\r\nof _that_ when they fall in love.”\r\n\r\n“Was there no good in your affectionate behaviour to Jane, while she was\r\nill at Netherfield?”\r\n\r\n“Dearest Jane! who could have done less for her? But make a virtue of it\r\nby all means. My good qualities are under your protection, and you are\r\nto exaggerate them as much as possible; and, in return, it belongs to me\r\nto find occasions for teasing and quarrelling with you as often as may\r\nbe; and I shall begin directly, by asking you what made you so unwilling\r\nto come to the point at last? What made you so shy of me, when you\r\nfirst called, and afterwards dined here? Why, especially, when you\r\ncalled, did you look as if you did not care about me?”\r\n\r\n“Because you were grave and silent, and gave me no encouragement.”\r\n\r\n“But I was embarrassed.”\r\n\r\n“And so was I.”\r\n\r\n“You might have talked to me more when you came to dinner.”\r\n\r\n“A man who had felt less might.”\r\n\r\n“How unlucky that you should have a reasonable answer to give, and that\r\nI should be so reasonable as to admit it! But I wonder how long you\r\n_would_ have gone on, if you had been left to yourself. I wonder when\r\nyou _would_ have spoken if I had not asked you! My resolution of\r\nthanking you for your kindness to Lydia had certainly great effect. _Too\r\nmuch_, I am afraid; for what becomes of the moral, if our comfort\r\nsprings from a breach of promise, for I ought not to have mentioned the\r\nsubject? This will never do.”\r\n\r\n“You need not distress yourself. The moral will be perfectly fair. Lady\r\nCatherine’s unjustifiable endeavours to separate us were the means of\r\nremoving all my doubts. I am not indebted for my present happiness to\r\nyour eager desire of expressing your gratitude. I was not in a humour to\r\nwait for an opening of yours. My aunt’s intelligence had given me hope,\r\nand I was determined at once to know everything.”\r\n\r\n“Lady Catherine has been of infinite use, which ought to make her happy,\r\nfor she loves to be of use. But tell me, what did you come down to\r\nNetherfield for? Was it merely to ride to Longbourn and be embarrassed?\r\nor had you intended any more serious consequences?”\r\n\r\n“My real purpose was to see _you_, and to judge, if I could, whether I\r\nmight ever hope to make you love me. My avowed one, or what I avowed to\r\nmyself, was to see whether your sister was still partial to Bingley, and\r\nif she were, to make the confession to him which I have since made.”\r\n\r\n“Shall you ever have courage to announce to Lady Catherine what is to\r\nbefall her?”\r\n\r\n“I am more likely to want time than courage, Elizabeth. But it ought to\r\nbe done; and if you will give me a sheet of paper it shall be done\r\ndirectly.”\r\n\r\n“And if I had not a letter to write myself, I might sit by you, and\r\nadmire the evenness of your writing, as another young lady once did. But\r\nI have an aunt, too, who must not be longer neglected.”\r\n\r\nFrom an unwillingness to confess how much her intimacy with Mr. Darcy\r\nhad been overrated, Elizabeth had never yet answered Mrs. Gardiner’s\r\nlong letter; but now, having _that_ to communicate which she knew would\r\nbe most welcome, she was almost ashamed to find that her uncle and aunt\r\nhad already lost three days of happiness, and immediately wrote as\r\nfollows:--\r\n\r\n“I would have thanked you before, my dear aunt, as I ought to have done,\r\nfor your long, kind, satisfactory detail of particulars; but, to say the\r\ntruth, I was too cross to write. You supposed more than really existed.\r\nBut _now_ suppose as much as you choose; give a loose to your fancy,\r\nindulge your imagination in every possible flight which the subject will\r\nafford, and unless you believe me actually married, you cannot greatly\r\nerr. You must write again very soon, and praise him a great deal more\r\nthan you did in your last. I thank you again and again, for not going to\r\nthe Lakes. How could I be so silly as to wish it! Your idea of the\r\nponies is delightful. We will go round the park every day. I am the\r\nhappiest creature in the world. Perhaps other people have said so\r\nbefore, but no one with such justice. I am happier even than Jane; she\r\nonly smiles, I laugh. Mr. Darcy sends you all the love in the world that\r\ncan be spared from me. You are all to come to Pemberley at Christmas.\r\nYours,” etc.\r\n\r\nMr. Darcy’s letter to Lady Catherine was in a different style, and still\r\ndifferent from either was what Mr. Bennet sent to Mr. Collins, in return\r\nfor his last.\r\n\r\n /* “Dear Sir, */\r\n\r\n “I must trouble you once more for congratulations. Elizabeth will\r\n soon be the wife of Mr. Darcy. Console Lady Catherine as well as\r\n you can. But, if I were you, I would stand by the nephew. He has\r\n more to give.\r\n\r\n“Yours sincerely,” etc.\r\n\r\nMiss Bingley’s congratulations to her brother on his approaching\r\nmarriage were all that was affectionate and insincere. She wrote even to\r\nJane on the occasion, to express her delight, and repeat all her former\r\nprofessions of regard. Jane was not deceived, but she was affected; and\r\nthough feeling no reliance on her, could not help writing her a much\r\nkinder answer than she knew was deserved.\r\n\r\nThe joy which Miss Darcy expressed on receiving similar information was\r\nas sincere as her brother’s in sending it. Four sides of paper were\r\ninsufficient to contain all her delight, and all her earnest desire of\r\nbeing loved by her sister.\r\n\r\nBefore any answer could arrive from Mr. Collins, or any congratulations\r\nto Elizabeth from his wife, the Longbourn family heard that the\r\nCollinses were come themselves to Lucas Lodge. The reason of this\r\nsudden removal was soon evident. Lady Catherine had been rendered so\r\nexceedingly angry by the contents of her nephew’s letter, that\r\nCharlotte, really rejoicing in the match, was anxious to get away till\r\nthe storm was blown over. At such a moment, the arrival of her friend\r\nwas a sincere pleasure to Elizabeth, though in the course of their\r\nmeetings she must sometimes think the pleasure dearly bought, when she\r\nsaw Mr. Darcy exposed to all the parading and obsequious civility of her\r\nhusband. He bore it, however, with admirable calmness. He could even\r\nlisten to Sir William Lucas, when he complimented him on carrying away\r\nthe brightest jewel of the country, and expressed his hopes of their all\r\nmeeting frequently at St. James’s, with very decent composure. If he did\r\nshrug his shoulders, it was not till Sir William was out of sight.\r\n\r\nMrs. Philips’s vulgarity was another, and, perhaps, a greater tax on his\r\nforbearance; and though Mrs. Philips, as well as her sister, stood in\r\ntoo much awe of him to speak with the familiarity which Bingley’s\r\ngood-humour encouraged; yet, whenever she _did_ speak, she must be\r\nvulgar. Nor was her respect for him, though it made her more quiet, at\r\nall likely to make her more elegant. Elizabeth did all she could to\r\nshield him from the frequent notice of either, and was ever anxious to\r\nkeep him to herself, and to those of her family with whom he might\r\nconverse without mortification; and though the uncomfortable feelings\r\narising from all this took from the season of courtship much of its\r\npleasure, it added to the hope of the future; and she looked forward\r\nwith delight to the time when they should be removed from society so\r\nlittle pleasing to either, to all the comfort and elegance of their\r\nfamily party at Pemberley.\r\n\r\n\r\n\r\n\r\n[Illustration]\r\n\r\n\r\n\r\n\r\nCHAPTER LXI.\r\n\r\n\r\n[Illustration]\r\n\r\nHappy for all her maternal feelings was the day on which Mrs. Bennet got\r\nrid of her two most deserving daughters. With what delighted pride she\r\nafterwards visited Mrs. Bingley, and talked of Mrs. Darcy, may be\r\nguessed. I wish I could say, for the sake of her family, that the\r\naccomplishment of her earnest desire in the establishment of so many of\r\nher children produced so happy an effect as to make her a sensible,\r\namiable, well-informed woman for the rest of her life; though, perhaps,\r\nit was lucky for her husband, who might not have relished domestic\r\nfelicity in so unusual a form, that she still was occasionally nervous\r\nand invariably silly.\r\n\r\nMr. Bennet missed his second daughter exceedingly; his affection for her\r\ndrew him oftener from home than anything else could do. He delighted in\r\ngoing to Pemberley, especially when he was least expected.\r\n\r\nMr. Bingley and Jane remained at Netherfield only a twelvemonth. So near\r\na vicinity to her mother and Meryton relations was not desirable even to\r\n_his_ easy temper, or _her_ affectionate heart. The darling wish of his\r\nsisters was then gratified: he bought an estate in a neighbouring county\r\nto Derbyshire; and Jane and Elizabeth, in addition to every other source\r\nof happiness, were within thirty miles of each other.\r\n\r\nKitty, to her very material advantage, spent the chief of her time with\r\nher two elder sisters. In society so superior to what she had generally\r\nknown, her improvement was great. She was not of so ungovernable a\r\ntemper as Lydia; and, removed from the influence of Lydia’s example, she\r\nbecame, by proper attention and management, less irritable, less\r\nignorant, and less insipid. From the further disadvantage of Lydia’s\r\nsociety she was of course carefully kept; and though Mrs. Wickham\r\nfrequently invited her to come and stay with her, with the promise of\r\nballs and young men, her father would never consent to her going.\r\n\r\nMary was the only daughter who remained at home; and she was necessarily\r\ndrawn from the pursuit of accomplishments by Mrs. Bennet’s being quite\r\nunable to sit alone. Mary was obliged to mix more with the world, but\r\nshe could still moralize over every morning visit; and as she was no\r\nlonger mortified by comparisons between her sisters’ beauty and her own,\r\nit was suspected by her father that she submitted to the change without\r\nmuch reluctance.\r\n\r\nAs for Wickham and Lydia, their characters suffered no revolution from\r\nthe marriage of her sisters. He bore with philosophy the conviction that\r\nElizabeth must now become acquainted with whatever of his ingratitude\r\nand falsehood had before been unknown to her; and, in spite of\r\neverything, was not wholly without hope that Darcy might yet be\r\nprevailed on to make his fortune. The congratulatory letter which\r\nElizabeth received from Lydia on her marriage explained to her that, by\r\nhis wife at least, if not by himself, such a hope was cherished. The\r\nletter was to this effect:--\r\n\r\n /* “My dear Lizzy, */\r\n\r\n “I wish you joy. If you love Mr. Darcy half so well as I do my dear\r\n Wickham, you must be very happy. It is a great comfort to have you\r\n so rich; and when you have nothing else to do, I hope you will\r\n think of us. I am sure Wickham would like a place at court very\r\n much; and I do not think we shall have quite money enough to live\r\n upon without some help. Any place would do of about three or four\r\n hundred a year; but, however, do not speak to Mr. Darcy about it,\r\n if you had rather not.\r\n\r\n“Yours,” etc.\r\n\r\nAs it happened that Elizabeth had much rather not, she endeavoured in\r\nher answer to put an end to every entreaty and expectation of the kind.\r\nSuch relief, however, as it was in her power to afford, by the practice\r\nof what might be called economy in her own private expenses, she\r\nfrequently sent them. It had always been evident to her that such an\r\nincome as theirs, under the direction of two persons so extravagant in\r\ntheir wants, and heedless of the future, must be very insufficient to\r\ntheir support; and whenever they changed their quarters, either Jane or\r\nherself were sure of being applied to for some little assistance towards\r\ndischarging their bills. Their manner of living, even when the\r\nrestoration of peace dismissed them to a home, was unsettled in the\r\nextreme. They were always moving from place to place in quest of a\r\ncheap situation, and always spending more than they ought. His affection\r\nfor her soon sunk into indifference: hers lasted a little longer; and,\r\nin spite of her youth and her manners, she retained all the claims to\r\nreputation which her marriage had given her. Though Darcy could never\r\nreceive _him_ at Pemberley, yet, for Elizabeth’s sake, he assisted him\r\nfurther in his profession. Lydia was occasionally a visitor there, when\r\nher husband was gone to enjoy himself in London or Bath; and with the\r\nBingleys they both of them frequently stayed so long, that even\r\nBingley’s good-humour was overcome, and he proceeded so far as to _talk_\r\nof giving them a hint to be gone.\r\n\r\nMiss Bingley was very deeply mortified by Darcy’s marriage; but as she\r\nthought it advisable to retain the right of visiting at Pemberley, she\r\ndropped all her resentment; was fonder than ever of Georgiana, almost as\r\nattentive to Darcy as heretofore, and paid off every arrear of civility\r\nto Elizabeth.\r\n\r\nPemberley was now Georgiana’s home; and the attachment of the sisters\r\nwas exactly what Darcy had hoped to see. They were able to love each\r\nother, even as well as they intended. Georgiana had the highest opinion\r\nin the world of Elizabeth; though at first she often listened with an\r\nastonishment bordering on alarm at her lively, sportive manner of\r\ntalking to her brother. He, who had always inspired in herself a respect\r\nwhich almost overcame her affection, she now saw the object of open\r\npleasantry. Her mind received knowledge which had never before fallen in\r\nher way. By Elizabeth’s instructions she began to comprehend that a\r\nwoman may take liberties with her husband, which a brother will not\r\nalways allow in a sister more than ten years younger than himself.\r\n\r\nLady Catherine was extremely indignant on the marriage of her nephew;\r\nand as she gave way to all the genuine frankness of her character, in\r\nher reply to the letter which announced its arrangement, she sent him\r\nlanguage so very abusive, especially of Elizabeth, that for some time\r\nall intercourse was at an end. But at length, by Elizabeth’s persuasion,\r\nhe was prevailed on to overlook the offence, and seek a reconciliation;\r\nand, after a little further resistance on the part of his aunt, her\r\nresentment gave way, either to her affection for him, or her curiosity\r\nto see how his wife conducted herself; and she condescended to wait on\r\nthem at Pemberley, in spite of that pollution which its woods had\r\nreceived, not merely from the presence of such a mistress, but the\r\nvisits of her uncle and aunt from the city.\r\n\r\nWith the Gardiners they were always on the most intimate terms. Darcy,\r\nas well as Elizabeth, really loved them; and they were both ever\r\nsensible of the warmest gratitude towards the persons who, by bringing\r\nher into Derbyshire, had been the means of uniting them.\r\n\r\n [Illustration:\r\n\r\n THE\r\n END\r\n ]\r\n\r\n\r\n\r\n\r\n CHISWICK PRESS:--CHARLES WHITTINGHAM AND CO.\r\n TOOKS COURT, CHANCERY LANE, LONDON.\r\n\r\n\r\n\r\n\r\n*** END OF THE PROJECT GUTENBERG EBOOK PRIDE AND PREJUDICE ***\r\n\r\n\r\n\r\n\r\nUpdated editions will replace the previous one—the old editions will\r\nbe renamed.\r\n\r\nCreating the works from print editions not protected by U.S. copyright\r\nlaw means that no one owns a United States copyright in these works,\r\nso the Foundation (and you!) can copy and distribute it in the United\r\nStates without permission and without paying copyright\r\nroyalties. Special rules, set forth in the General Terms of Use part\r\nof this license, apply to copying and distributing Project\r\nGutenberg™ electronic works to protect the PROJECT GUTENBERG™\r\nconcept and trademark. Project Gutenberg is a registered trademark,\r\nand may not be used if you charge for an eBook, except by following\r\nthe terms of the trademark license, including paying royalties for use\r\nof the Project Gutenberg trademark. If you do not charge anything for\r\ncopies of this eBook, complying with the trademark license is very\r\neasy. You may use this eBook for nearly any purpose such as creation\r\nof derivative works, reports, performances and research. Project\r\nGutenberg eBooks may be modified and printed and given away—you may\r\ndo practically ANYTHING in the United States with eBooks not protected\r\nby U.S. copyright law. Redistribution is subject to the trademark\r\nlicense, especially commercial redistribution.\r\n\r\n\r\nSTART: FULL LICENSE\r\n\r\nTHE FULL PROJECT GUTENBERG LICENSE\r\n\r\nPLEASE READ THIS BEFORE YOU DISTRIBUTE OR USE THIS WORK\r\n\r\nTo protect the Project Gutenberg™ mission of promoting the free\r\ndistribution of electronic works, by using or distributing this work\r\n(or any other work associated in any way with the phrase “Project\r\nGutenberg”), you agree to comply with all the terms of the Full\r\nProject Gutenberg™ License available with this file or online at\r\nwww.gutenberg.org/license.\r\n\r\nSection 1. General Terms of Use and Redistributing Project Gutenberg™\r\nelectronic works\r\n\r\n1.A. By reading or using any part of this Project Gutenberg™\r\nelectronic work, you indicate that you have read, understand, agree to\r\nand accept all the terms of this license and intellectual property\r\n(trademark/copyright) agreement. If you do not agree to abide by all\r\nthe terms of this agreement, you must cease using and return or\r\ndestroy all copies of Project Gutenberg™ electronic works in your\r\npossession. If you paid a fee for obtaining a copy of or access to a\r\nProject Gutenberg™ electronic work and you do not agree to be bound\r\nby the terms of this agreement, you may obtain a refund from the person\r\nor entity to whom you paid the fee as set forth in paragraph 1.E.8.\r\n\r\n1.B. “Project Gutenberg” is a registered trademark. It may only be\r\nused on or associated in any way with an electronic work by people who\r\nagree to be bound by the terms of this agreement. There are a few\r\nthings that you can do with most Project Gutenberg™ electronic works\r\neven without complying with the full terms of this agreement. See\r\nparagraph 1.C below. There are a lot of things you can do with Project\r\nGutenberg™ electronic works if you follow the terms of this\r\nagreement and help preserve free future access to Project Gutenberg™\r\nelectronic works. See paragraph 1.E below.\r\n\r\n1.C. The Project Gutenberg Literary Archive Foundation (“the\r\nFoundation” or PGLAF), owns a compilation copyright in the collection\r\nof Project Gutenberg™ electronic works. Nearly all the individual\r\nworks in the collection are in the public domain in the United\r\nStates. If an individual work is unprotected by copyright law in the\r\nUnited States and you are located in the United States, we do not\r\nclaim a right to prevent you from copying, distributing, performing,\r\ndisplaying or creating derivative works based on the work as long as\r\nall references to Project Gutenberg are removed. Of course, we hope\r\nthat you will support the Project Gutenberg™ mission of promoting\r\nfree access to electronic works by freely sharing Project Gutenberg™\r\nworks in compliance with the terms of this agreement for keeping the\r\nProject Gutenberg™ name associated with the work. You can easily\r\ncomply with the terms of this agreement by keeping this work in the\r\nsame format with its attached full Project Gutenberg™ License when\r\nyou share it without charge with others.\r\n\r\n1.D. The copyright laws of the place where you are located also govern\r\nwhat you can do with this work. Copyright laws in most countries are\r\nin a constant state of change. If you are outside the United States,\r\ncheck the laws of your country in addition to the terms of this\r\nagreement before downloading, copying, displaying, performing,\r\ndistributing or creating derivative works based on this work or any\r\nother Project Gutenberg™ work. The Foundation makes no\r\nrepresentations concerning the copyright status of any work in any\r\ncountry other than the United States.\r\n\r\n1.E. Unless you have removed all references to Project Gutenberg:\r\n\r\n1.E.1. The following sentence, with active links to, or other\r\nimmediate access to, the full Project Gutenberg™ License must appear\r\nprominently whenever any copy of a Project Gutenberg™ work (any work\r\non which the phrase “Project Gutenberg” appears, or with which the\r\nphrase “Project Gutenberg” is associated) is accessed, displayed,\r\nperformed, viewed, copied or distributed:\r\n\r\n This eBook is for the use of anyone anywhere in the United States and most\r\n other parts of the world at no cost and with almost no restrictions\r\n whatsoever. You may copy it, give it away or re-use it under the terms\r\n of the Project Gutenberg License included with this eBook or online\r\n at www.gutenberg.org. If you\r\n are not located in the United States, you will have to check the laws\r\n of the country where you are located before using this eBook.\r\n\r\n1.E.2. If an individual Project Gutenberg™ electronic work is\r\nderived from texts not protected by U.S. copyright law (does not\r\ncontain a notice indicating that it is posted with permission of the\r\ncopyright holder), the work can be copied and distributed to anyone in\r\nthe United States without paying any fees or charges. If you are\r\nredistributing or providing access to a work with the phrase “Project\r\nGutenberg” associated with or appearing on the work, you must comply\r\neither with the requirements of paragraphs 1.E.1 through 1.E.7 or\r\nobtain permission for the use of the work and the Project Gutenberg™\r\ntrademark as set forth in paragraphs 1.E.8 or 1.E.9.\r\n\r\n1.E.3. If an individual Project Gutenberg™ electronic work is posted\r\nwith the permission of the copyright holder, your use and distribution\r\nmust comply with both paragraphs 1.E.1 through 1.E.7 and any\r\nadditional terms imposed by the copyright holder. Additional terms\r\nwill be linked to the Project Gutenberg™ License for all works\r\nposted with the permission of the copyright holder found at the\r\nbeginning of this work.\r\n\r\n1.E.4. Do not unlink or detach or remove the full Project Gutenberg™\r\nLicense terms from this work, or any files containing a part of this\r\nwork or any other work associated with Project Gutenberg™.\r\n\r\n1.E.5. Do not copy, display, perform, distribute or redistribute this\r\nelectronic work, or any part of this electronic work, without\r\nprominently displaying the sentence set forth in paragraph 1.E.1 with\r\nactive links or immediate access to the full terms of the Project\r\nGutenberg™ License.\r\n\r\n1.E.6. You may convert to and distribute this work in any binary,\r\ncompressed, marked up, nonproprietary or proprietary form, including\r\nany word processing or hypertext form. However, if you provide access\r\nto or distribute copies of a Project Gutenberg™ work in a format\r\nother than “Plain Vanilla ASCII” or other format used in the official\r\nversion posted on the official Project Gutenberg™ website\r\n(www.gutenberg.org), you must, at no additional cost, fee or expense\r\nto the user, provide a copy, a means of exporting a copy, or a means\r\nof obtaining a copy upon request, of the work in its original “Plain\r\nVanilla ASCII” or other form. Any alternate format must include the\r\nfull Project Gutenberg™ License as specified in paragraph 1.E.1.\r\n\r\n1.E.7. Do not charge a fee for access to, viewing, displaying,\r\nperforming, copying or distributing any Project Gutenberg™ works\r\nunless you comply with paragraph 1.E.8 or 1.E.9.\r\n\r\n1.E.8. You may charge a reasonable fee for copies of or providing\r\naccess to or distributing Project Gutenberg™ electronic works\r\nprovided that:\r\n\r\n • You pay a royalty fee of 20% of the gross profits you derive from\r\n the use of Project Gutenberg™ works calculated using the method\r\n you already use to calculate your applicable taxes. The fee is owed\r\n to the owner of the Project Gutenberg™ trademark, but he has\r\n agreed to donate royalties under this paragraph to the Project\r\n Gutenberg Literary Archive Foundation. Royalty payments must be paid\r\n within 60 days following each date on which you prepare (or are\r\n legally required to prepare) your periodic tax returns. Royalty\r\n payments should be clearly marked as such and sent to the Project\r\n Gutenberg Literary Archive Foundation at the address specified in\r\n Section 4, “Information about donations to the Project Gutenberg\r\n Literary Archive Foundation.”\r\n\r\n • You provide a full refund of any money paid by a user who notifies\r\n you in writing (or by e-mail) within 30 days of receipt that s/he\r\n does not agree to the terms of the full Project Gutenberg™\r\n License. You must require such a user to return or destroy all\r\n copies of the works possessed in a physical medium and discontinue\r\n all use of and all access to other copies of Project Gutenberg™\r\n works.\r\n\r\n • You provide, in accordance with paragraph 1.F.3, a full refund of\r\n any money paid for a work or a replacement copy, if a defect in the\r\n electronic work is discovered and reported to you within 90 days of\r\n receipt of the work.\r\n\r\n • You comply with all other terms of this agreement for free\r\n distribution of Project Gutenberg™ works.\r\n\r\n\r\n1.E.9. If you wish to charge a fee or distribute a Project\r\nGutenberg™ electronic work or group of works on different terms than\r\nare set forth in this agreement, you must obtain permission in writing\r\nfrom the Project Gutenberg Literary Archive Foundation, the manager of\r\nthe Project Gutenberg™ trademark. Contact the Foundation as set\r\nforth in Section 3 below.\r\n\r\n1.F.\r\n\r\n1.F.1. Project Gutenberg volunteers and employees expend considerable\r\neffort to identify, do copyright research on, transcribe and proofread\r\nworks not protected by U.S. copyright law in creating the Project\r\nGutenberg™ collection. Despite these efforts, Project Gutenberg™\r\nelectronic works, and the medium on which they may be stored, may\r\ncontain “Defects,” such as, but not limited to, incomplete, inaccurate\r\nor corrupt data, transcription errors, a copyright or other\r\nintellectual property infringement, a defective or damaged disk or\r\nother medium, a computer virus, or computer codes that damage or\r\ncannot be read by your equipment.\r\n\r\n1.F.2. LIMITED WARRANTY, DISCLAIMER OF DAMAGES - Except for the “Right\r\nof Replacement or Refund” described in paragraph 1.F.3, the Project\r\nGutenberg Literary Archive Foundation, the owner of the Project\r\nGutenberg™ trademark, and any other party distributing a Project\r\nGutenberg™ electronic work under this agreement, disclaim all\r\nliability to you for damages, costs and expenses, including legal\r\nfees. YOU AGREE THAT YOU HAVE NO REMEDIES FOR NEGLIGENCE, STRICT\r\nLIABILITY, BREACH OF WARRANTY OR BREACH OF CONTRACT EXCEPT THOSE\r\nPROVIDED IN PARAGRAPH 1.F.3. YOU AGREE THAT THE FOUNDATION, THE\r\nTRADEMARK OWNER, AND ANY DISTRIBUTOR UNDER THIS AGREEMENT WILL NOT BE\r\nLIABLE TO YOU FOR ACTUAL, DIRECT, INDIRECT, CONSEQUENTIAL, PUNITIVE OR\r\nINCIDENTAL DAMAGES EVEN IF YOU GIVE NOTICE OF THE POSSIBILITY OF SUCH\r\nDAMAGE.\r\n\r\n1.F.3. LIMITED RIGHT OF REPLACEMENT OR REFUND - If you discover a\r\ndefect in this electronic work within 90 days of receiving it, you can\r\nreceive a refund of the money (if any) you paid for it by sending a\r\nwritten explanation to the person you received the work from. If you\r\nreceived the work on a physical medium, you must return the medium\r\nwith your written explanation. The person or entity that provided you\r\nwith the defective work may elect to provide a replacement copy in\r\nlieu of a refund. If you received the work electronically, the person\r\nor entity providing it to you may choose to give you a second\r\nopportunity to receive the work electronically in lieu of a refund. If\r\nthe second copy is also defective, you may demand a refund in writing\r\nwithout further opportunities to fix the problem.\r\n\r\n1.F.4. Except for the limited right of replacement or refund set forth\r\nin paragraph 1.F.3, this work is provided to you ‘AS-IS’, WITH NO\r\nOTHER WARRANTIES OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT\r\nLIMITED TO WARRANTIES OF MERCHANTABILITY OR FITNESS FOR ANY PURPOSE.\r\n\r\n1.F.5. Some states do not allow disclaimers of certain implied\r\nwarranties or the exclusion or limitation of certain types of\r\ndamages. If any disclaimer or limitation set forth in this agreement\r\nviolates the law of the state applicable to this agreement, the\r\nagreement shall be interpreted to make the maximum disclaimer or\r\nlimitation permitted by the applicable state law. The invalidity or\r\nunenforceability of any provision of this agreement shall not void the\r\nremaining provisions.\r\n\r\n1.F.6. INDEMNITY - You agree to indemnify and hold the Foundation, the\r\ntrademark owner, any agent or employee of the Foundation, anyone\r\nproviding copies of Project Gutenberg™ electronic works in\r\naccordance with this agreement, and any volunteers associated with the\r\nproduction, promotion and distribution of Project Gutenberg™\r\nelectronic works, harmless from all liability, costs and expenses,\r\nincluding legal fees, that arise directly or indirectly from any of\r\nthe following which you do or cause to occur: (a) distribution of this\r\nor any Project Gutenberg™ work, (b) alteration, modification, or\r\nadditions or deletions to any Project Gutenberg™ work, and (c) any\r\nDefect you cause.\r\n\r\nSection 2. Information about the Mission of Project Gutenberg™\r\n\r\nProject Gutenberg™ is synonymous with the free distribution of\r\nelectronic works in formats readable by the widest variety of\r\ncomputers including obsolete, old, middle-aged and new computers. It\r\nexists because of the efforts of hundreds of volunteers and donations\r\nfrom people in all walks of life.\r\n\r\nVolunteers and financial support to provide volunteers with the\r\nassistance they need are critical to reaching Project Gutenberg™’s\r\ngoals and ensuring that the Project Gutenberg™ collection will\r\nremain freely available for generations to come. In 2001, the Project\r\nGutenberg Literary Archive Foundation was created to provide a secure\r\nand permanent future for Project Gutenberg™ and future\r\ngenerations. To learn more about the Project Gutenberg Literary\r\nArchive Foundation and how your efforts and donations can help, see\r\nSections 3 and 4 and the Foundation information page at www.gutenberg.org.\r\n\r\nSection 3. Information about the Project Gutenberg Literary Archive Foundation\r\n\r\nThe Project Gutenberg Literary Archive Foundation is a non-profit\r\n501(c)(3) educational corporation organized under the laws of the\r\nstate of Mississippi and granted tax exempt status by the Internal\r\nRevenue Service. The Foundation’s EIN or federal tax identification\r\nnumber is 64-6221541. Contributions to the Project Gutenberg Literary\r\nArchive Foundation are tax deductible to the full extent permitted by\r\nU.S. federal laws and your state’s laws.\r\n\r\nThe Foundation’s business office is located at 809 North 1500 West,\r\nSalt Lake City, UT 84116, (801) 596-1887. Email contact links and up\r\nto date contact information can be found at the Foundation’s website\r\nand official page at www.gutenberg.org/contact\r\n\r\nSection 4. Information about Donations to the Project Gutenberg\r\nLiterary Archive Foundation\r\n\r\nProject Gutenberg™ depends upon and cannot survive without widespread\r\npublic support and donations to carry out its mission of\r\nincreasing the number of public domain and licensed works that can be\r\nfreely distributed in machine-readable form accessible by the widest\r\narray of equipment including outdated equipment. Many small donations\r\n($1 to $5,000) are particularly important to maintaining tax exempt\r\nstatus with the IRS.\r\n\r\nThe Foundation is committed to complying with the laws regulating\r\ncharities and charitable donations in all 50 states of the United\r\nStates. Compliance requirements are not uniform and it takes a\r\nconsiderable effort, much paperwork and many fees to meet and keep up\r\nwith these requirements. We do not solicit donations in locations\r\nwhere we have not received written confirmation of compliance. To SEND\r\nDONATIONS or determine the status of compliance for any particular state\r\nvisit www.gutenberg.org/donate.\r\n\r\nWhile we cannot and do not solicit contributions from states where we\r\nhave not met the solicitation requirements, we know of no prohibition\r\nagainst accepting unsolicited donations from donors in such states who\r\napproach us with offers to donate.\r\n\r\nInternational donations are gratefully accepted, but we cannot make\r\nany statements concerning tax treatment of donations received from\r\noutside the United States. U.S. laws alone swamp our small staff.\r\n\r\nPlease check the Project Gutenberg web pages for current donation\r\nmethods and addresses. Donations are accepted in a number of other\r\nways including checks, online payments and credit card donations. To\r\ndonate, please visit: www.gutenberg.org/donate.\r\n\r\nSection 5. General Information About Project Gutenberg™ electronic works\r\n\r\nProfessor Michael S. Hart was the originator of the Project\r\nGutenberg™ concept of a library of electronic works that could be\r\nfreely shared with anyone. For forty years, he produced and\r\ndistributed Project Gutenberg™ eBooks with only a loose network of\r\nvolunteer support.\r\n\r\nProject Gutenberg™ eBooks are often created from several printed\r\neditions, all of which are confirmed as not protected by copyright in\r\nthe U.S. unless a copyright notice is included. Thus, we do not\r\nnecessarily keep eBooks in compliance with any particular paper\r\nedition.\r\n\r\nMost people start at our website which has the main PG search\r\nfacility: www.gutenberg.org.\r\n\r\nThis website includes information about Project Gutenberg™,\r\nincluding how to make donations to the Project Gutenberg Literary\r\nArchive Foundation, how to help produce our new eBooks, and how to\r\nsubscribe to our email newsletter to hear about new eBooks.\r\n" }, { "path": "data/huawei_pangu.md", "content": "# 盘古之殇：华为诺亚盘古大模型研发历程的心酸与黑暗\n\n各位好，\n\n我是一名盘古大模型团队，华为诺亚方舟实验室的员工。\n\n首先为自证身份，列举一些细节：\n\n1. 现诺亚主任，前算法应用部部长，后改名为小模型实验室的主任王云鹤。前诺亚主任：姚骏（大家称姚老师）。几个实验室主任：唐睿明（明哥，明队，已离职），尚利峰，张维（维哥），郝建业（郝老师），刘武龙（称呼为武龙所）等。其他骨干成员和专家陆续有很多人离职。\n2. 我们隶属于“四野”这个组织。四野下属有许多纵队，基础语言大模型是四纵。王云鹤的小模型是十六纵队。我们参加过苏州的集结，有各种月份的时间节点。在苏州攻关会颁发任务令，需要在节点前达成目标。苏州集结会把各地的人员都集中在苏州研究所，平常住宾馆，比如在甪直的酒店，与家人孩子天各一方。\n3. 在苏州集结的时候周六默认上班，非常辛苦，不过周六有下午茶，有一次还有小龙虾。在苏州研究所的工位搬迁过一次，从一栋楼换到了另一栋。苏州研究所楼栋都是欧式装修，门口有大坡，里面景色很不错。去苏州集结一般至少要去一周，甚至更久，多的人甚至一两个月都回不了家。\n4. 诺亚曾经传说是研究型的，但是来了之后因为在四野做大模型项目，项目成员完全变成了交付型的，且充满了例会，评审，汇报。很多时候做实验都要申请。团队需要对接终端小艺，华为云，ICT等诸多业务线，交付压力不小。\n5. 诺亚研发的盘古模型早期内部代号叫做“盘古智子”，一开始只有内部需要申请试用的网页版，到后续迫于压力在welink上接入和公测开放。\n\n这些天发生关于质疑盘古大模型抄袭千问的事情闹的沸沸扬扬。作为一个盘古团队的成员，我最近夜夜辗转反侧，难以入眠。盘古的品牌受到如此大的影响，一方面，我自私的为我的职业发展担忧，也为自己过去的努力工作感到不值。另一方面，由于有人开始揭露这些事情我内心又感到大快人心。在多少个日日夜夜，我们对内部某些人一次次靠着造假而又获得了无数利益的行为咬牙切齿而又无能为力。这种压抑和羞辱也逐渐消磨了我对华为的感情，让我在这里的时日逐渐浑浑噩噩，迷茫无措，时常怀疑自己的人生和自我价值。\n\n我承认我是一个懦弱的人，作为一个小小的打工人，我不仅不敢和王云鹤等内部手眼通天的人做对，更不敢和华为这样的庞然大物做对。我很怕失去我的工作，毕竟我也有家人和孩子，所以我打心眼里很佩服揭露者。但是，看到内部还在试图洗地掩盖事实，蒙蔽公众的时候，我实在不能容忍了。我也希望勇敢一次，顺从自己本心。就算自损八百，我也希望能伤敌一千。我决定把我在这里的所见所闻（部分来自于同事口述）公布出来，关于盘古大模型的“传奇故事”：\n\n华为确实主要在昇腾卡上训练大模型（小模型实验室有不少英伟达的卡，他们之前也会用来训练，后面转移到昇腾）。曾经我被华为“打造世界第二选择”的决心而折服，我本身也曾经对华为有深厚的感情。我们陪着昇腾一步步摸爬滚打，从充满bug到现在能训出模型，付出了巨大的心血和代价。\n\n最初我们的算力非常有限，在910A上训练模型。那会只支持fp16，训练的稳定性远不如bf16。盘古的moe开始很早，23年就主要是训练38Bmoe模型和后续的71B dense模型。71B的dense模型通过扩增变成了第一代的135Bdense模型，后面主力模型也逐渐在910B上训练。\n\n71B和135B模型都有一个巨大的硬伤就是tokenizer。当时使用的tokenizer编码效率极低，每个单个的符号，数字，空格，乃至汉字都会占用一个token。可想而知这会非常浪费算力，且使得模型的效果很差。这时候小模型实验室正好有个自己训的词表。姚老师当时怀疑是不是模型的tokenizer不好（虽然事后来看，他的怀疑是无疑正确的），于是就决定，让71B和135B换tokenizer，因为小模型实验室曾经尝试过。团队缝合了两个tokenizer，开始了tokenizer的更换。71B模型的更换失败了，而135B因为采用了更精细的embedding初始化策略，续训了至少1T的数据后词表总算更换成功，但可想而知，效果并不会变好。\n\n于此同期，阿里和智谱等国内其他公司在GPU上训练，且已经摸索出了正确的方法，盘古和竞品的差距越来越大。内部一个230B从头训练的dense模型又因为各种原因训练失败，导致项目的状况几乎陷入绝境。面临几个节点的压力以及内部对盘古的强烈质疑时，团队的士气低迷到了极点。团队在算力极其有限的时候，做出了很多努力和挣扎。比如，团队偶然发现当时的38B moe并没有预期moe的效果。于是去掉了moe参数，还原为了13B的dense模型。由于38B的moe源自很早的pangu alpha 13B，架构相对落后，团队进行了一系列的操作，比如切换绝对位置编码到rope，去掉bias，切换为rmsnorm。同时鉴于tokenizer的一些失败和换词表的经验，这个模型的词表也更换为了王云鹤的小模型实验室7B模型所使用的词表。后面这个13B模型进行了扩增续训，变成了第二代38B dense模型（在几个月内这个模型都是主要的盘古中档位模型），曾经具有一定的竞争力。但是，由于更大的135B模型架构落后，且更换词表模型损伤巨大（后续分析发现当时更换的缝合词表有更严重的bug），续训后也与千问等当时国内领先模型存在很大差距。这时由于内部的质疑声和领导的压力也越来越大。团队的状态几乎陷入了绝境。\n\n在这种情况下，王云鹤和他的小模型实验室出手了。他们声称是从旧的135B参数继承改造而来，通过训练短短的几百B数据，各项指标平均提升了十个点左右。实际上，这就是他们套壳应用到大模型的第一次杰作。华为的外行领导内行，使得领导完全对于这种扯淡的事情没有概念，他们只会觉得肯定是有什么算法创新。经过内部的分析，他们实际上是使用Qwen 1.5 110B续训而来，通过加层，扩增ffn维度，添加盘古pi论文的一些机制得来，凑够了大概135B的参数。实际上，旧的135B有107层，而这个模型只有82层，各种配置也都不一样。新的来路不明的135B训练完很多参数的分布也和Qwen 110B几乎一模一样。连模型代码的类名当时都是Qwen，甚至懒得改名。后续这个模型就是所谓的135B V2。而这个模型当时也提供给了很多下游，甚至包括外部客户。\n\n这件事对于我们这些认真诚实做事的同事们带来了巨大的冲击，内部很多人其实都知道这件事，甚至包括终端和华为云。我们都戏称以后别叫盘古模型了，叫千古吧。当时团队成员就想向bcg举报了，毕竟这已经是重大的业务造假了。但是后面据说被领导拦了下来，因为更高级别的领导（比如姚老师，以及可能熊总和查老）其实后面也知道了，但是并不管，因为通过套壳拿出好的结果，对他们也是有利的。这件事使得当时团队几位最强的同事开始心灰意冷，离职跑路也逐渐成为挂在嘴边的事。\n\n此时，盘古似乎迎来了转机。由于前面所述的这些盘古模型基本都是续训和改造而来，当时诺亚完全没有掌握从头训练的技术，何况还是在昇腾的NPU上进行训练。在当时团队的核心成员的极力争取下，盘古开始了第三代模型的训练，付出了巨大的努力后，在数据架构和训练算法方面都与业界逐渐接轨，而这其中的艰辛和小模型实验室的人一点关系都没有。\n\n一开始团队成员毫无信心，只从一个13B的模型开始训练，但是后面发现效果还不错，于是这个模型后续再次进行了一次参数扩增，变成了第三代的38B，代号38B V3。想必很多产品线的兄弟都对这个模型很熟悉。当时这个模型的tokenizer是基于llama的词表进行扩展的（也是业界常见的做法）。而当时王云鹤的实验室做出来了另一个词表（也就是后续pangu系列的词表）。当时两个词表还被迫进行了一次赛马，最终没有明显的好坏结论。于是，领导当即决定，应该统一词表，使用王云鹤他们的。于是，在后续从头训练的135B V3（也就是对外的Pangu Ultra），便是采用了这个tokenizer。这也解释了很多使用我们模型的兄弟的疑惑，为什么当时同为V3代的两个不同档位的模型，会使用不同的tokenizer。\n\n\n我们打心眼里觉得，135B V3是我们四纵团队当时的骄傲。这是第一个真正意义上的，华为全栈自研，正经从头训练的千亿级别的模型，且效果与24年同期竞品可比的。写到这里我已经热泪盈眶，太不容易了。当时为了稳定训练，团队做了大量实验对比，并且多次在模型梯度出现异常的时候进行及时回退重启。这个模型真正做到了后面技术报告所说的训练全程没有一个loss spike。我们克服了不知道多少困难，我们做到了，我们愿用生命和荣誉保证这个模型训练的真实性。多少个凌晨，我们为了它的训练而不眠。在被内部心声骂的一文不值的时候，我们有多么不甘，有多少的委屈，我们挺住了。\n\n我们这帮人是真的在为打磨国产算力底座燃烧自己的青春啊……客居他乡，我们放弃了家庭，放弃了假期，放弃了健康，放弃了娱乐，抛头颅洒热血，其中的艰辛与困苦，寥寥数笔不足以概括其万一。在各种动员大会上，当时口号中喊出的盘古必胜，华为必胜，我们心里是真的深深被感动。\n\n然而，我们的所有辛苦的成果，经常被小模型实验室轻飘飘的拿走了。数据，直接要走。代码，直接要走，还要求我们配合适配到能一键运行。我们当时戏称小模型实验室为点鼠标实验室。我们付出辛苦，他们取得荣耀。果然应了那句话，你在负重前行是因为有人替你岁月静好。在这种情况下，越来越多的战友再也坚持不下去了，选择了离开。看到身边那些优秀的同事一个个离职，我的内心又感叹又难过。在这种作战一样的环境下，我们比起同事来说更像是战友。他们在技术上也有无数值得我学习的地方，堪称良师。看到他们去了诸如字节Seed，Deepseek，月之暗面，腾讯和快手等等很多出色的团队，我打心眼里为他们高兴和祝福，脱离了这个辛苦却肮脏的地方。我至今还对一位离职同事的话记忆犹新，ta说：“来这里是我技术生涯中的耻辱，在这里再呆每一天都是浪费生命”。话虽难听却让我无言以对。我担心我自己技术方面的积累不足，以及没法适应互联网公司高淘汰的环境，让我多次想离职的心始终没有迈出这一步。\n\n盘古除了dense模型，后续也启动了moe的探索。一开始训练的是一个224B的moe模型。而与之平行的，小模型实验室也开启了第二次主要的套壳行动（次要的插曲可能还包括一些别的模型，比如math模型），即这次流传甚广的pangu pro moe 72B。这个模型内部自称是从小模型实验室的7B扩增上来的（就算如此，这也与技术报告不符，何况是套壳qwen 2.5的14b续训）。还记得他们训了没几天，内部的评测就立刻追上了当时的38B V3。AI系统实验室很多兄弟因为需要适配模型，都知道他们的套壳行动，只是迫于各种原因，无法伸张正义。实际上，对于后续训了很久很久的这个模型，Honestagi能够分析出这个量级的相似性我已经很诧异了，因为这个模型为了续训洗参数，所付出的算力甚至早就足够从头训一个同档位的模型了。听同事说他们为了洗掉千问的水印，采取了不少办法，甚至包括故意训了脏数据。这也为学术界研究模型血缘提供了一个前所未有的特殊模范吧。以后新的血缘方法提出可以拿出来溜溜。\n\n24年底和25年初，在Deepseek v3和r1发布之后，由于其惊艳的技术水平，团队受到了巨大的冲击，也受到了更大的质疑。于是为了紧跟潮流，盘古模仿Deepseek的模型尺寸，开启了718B moe的训练。这个时候，小模型实验室再次出手了。他们选择了套壳Deepseekv3续训。他们通过冻住Deepseek加载的参数，进行训练。连任务加载ckpt的目录都是deepseekv3，改都不改，何其嚣张？与之相反，一些有真正技术信仰的同事，在从头训练另一个718B的moe。但其中出现了各种各样的问题。但是很显然，这个模型怎么可能比直接套壳的好呢？如果不是团队leader坚持，早就被叫停了。\n\n华为的流程管理之繁重，严重拖累了大模型的研发节奏，例如版本管理，模型血缘，各种流程化，各种可追溯。讽刺的是，小模型实验室的模型似乎从来不受这些流程的约束，想套壳就套壳，想续训就续训，算力源源不断的伸手拿走。这种强烈到近乎魔幻的对比，说明了当前流程管理的情况：只许州官放火，不许百姓点灯。何其可笑？何其可悲？何其可恶？何其可耻！\n\nHonestAGI的事情出来后，内部让大家不停的研讨分析，如何公关和“回应”。诚然，这个原文的分析也许不够有力，给了王云鹤与小模型实验室他们狡辩和颠倒黑白的机会。为此，这两天我内心感到作呕，时时怀疑自己的人生意义以及苍天无眼。我不奉陪了，我要离职了，同时我也在申请从盘古部分技术报告的作者名单中移除。曾经在这些技术报告上署名是我一生都无法抹除的污点。当时我没想到，他们竟然猖狂到敢开源。我没想到，他们敢如此愚弄世人，大肆宣发。当时，我也许是存了侥幸心理，没有拒绝署名。我相信很多扎实做事的战友，也只是被迫上了贼船，或者不知情。但这件事已经无法挽回，我希望我的余生能够坚持扎实做真正有意义的事，为我当时的软弱和不坚定赎罪。\n\n深夜写到这里，我已经泪流满面，泣不成声。还记得一些出色的同事离职时，我苦笑问他们要不要发个长长的心声惯例帖，揭露一下现状。对方说：不了，浪费时间，而且我也怕揭露出来你们过的更糟。我当时一下黯然神伤，因为曾经共同为了理想奋斗过的战友已经彻底对华为彻底灰心了。当时大家调侃，我们用着当年共产党的小米加步枪，组织却有着堪比当年国民党的作风。\n\n曾几何时，我为我们用着小米加步枪打败洋枪洋炮而自豪。\n\n现在，我累了，我想投降。\n\n其实时至今日，我还是真心希望华为能认真吸取教训，能做好盘古，把盘古做到世界一流，把昇腾变成英伟达的水平。内部的劣币驱逐良币，使得诺亚乃至华为在短时间内急剧流失了大量出色的大模型人才。相信他们也正在如Deepseek等各个团队闪耀着，施展着他们的抱负才华，为中美在AI的激烈竞赛中奉献力量。我时常感叹，华为不是没有人才，而是根本不知道怎么留住人才。如果给这些人合适的环境，合适的资源，更少的枷锁，更少的政治斗争，盘古何愁不成？\n\n最后：我以生命，人格和荣誉发誓，我写的以上所有内容均为真实（至少在我有限的认知范围内）。我没有那么高的技术水平以及机会去做详尽扎实的分析，也不敢直接用内部记录举证，怕因为信息安全抓到。但是我相信我很多曾经的战友，会为我作证。在华为内部的兄弟，包括我们曾经服务过的产品线兄弟们，相信本文的无数细节能和你们的印象对照，印证我的说法。你们可能也曾经被蒙骗，但这些残酷的真相不会被尘封。我们奋战过的痕迹，也不应该被扭曲和埋葬。\n\n写了这么多，某些人肯定想把我找出来，抹杀掉。公司搞不好也想让我噤声乃至追责。如果真的这样，我，乃至我的家人的人身乃至生命安全可能都会受到威胁。为了自我保护，我近期每天会跟大家报平安。\n\n如果我消失了，就当是我为了真理和理想，为了华为乃至中国能够更好地发展算力和AI而牺牲了吧，我愿埋葬于那片曾经奋斗过的地方。\n\n诺亚，再见\n\n2025年7月6日凌晨 写于深圳\n\n---\n\n各位好，\n\n感谢大家的关心与祝福。我目前暂时安全，但公司应该在进行排查与某些名单收集，后续情况未知。\n\n我补充一些细节，以免某些人继续颠倒黑白。\n\n关于135B V2，小模型实验室在迅速地完成套壳并拿完所有套壳带来的好处后（比如任务令表彰和及时激励），因为不想继续支撑下游应用和模型迭代，又把这个烫手山芋甩给了四纵。确实技高一筹，直接把四纵的兄弟们拉下水。同事提供过去一个老旧的模型，最终拿回了一个当时一个魔改的先进的千问。做大模型的人，自己做的模型就像自己孩子一样熟悉，不要把别人都当傻子。就像自家儿子出门一趟，回来个别人家孩子。\n\n盘古report的署名是不符合学术规范的。例如，135B V3有不少有技术贡献的人，因为作者名额数量限制，劳动成果没有得到应有的回报，团队内曾经有不小的意见。这个模型当时是大家智慧和汗水的结晶，甚至是团队当时的精神支柱，支撑着不少兄弟们继续留在诺亚。所谓的名额限制，以及挂名了一些毫无技术贡献的人（如一些小模型实验室的人），让兄弟们何其心寒。\n\n---\n\n暂时平安。另外，支持我勇于说出真相的战友们 https://github.com/HW-whistleblower/True-Story-of-Pangu/issues/317\n" }, { "path": "demo.ipynb", "content": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"# Quick Start \\n\",\n \"\\n\",\n \"**Home GitHub Repository:** [LEANN on GitHub](https://github.com/yichuan-w/LEANN)\\n\",\n \"\\n\",\n \"**Important for Colab users:** Set your runtime type to T4 GPU for optimal performance. Go to Runtime → Change runtime type → Hardware accelerator → T4 GPU.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"# install this if you are using colab\\n\",\n \"! uv pip install leann-core leann-backend-hnsw --no-deps\\n\",\n \"! uv pip install leann --no-deps\\n\",\n \"# For Colab environment, we need to set some environment variables\\n\",\n \"import os\\n\",\n \"\\n\",\n \"os.environ[\\\"LEANN_LOG_LEVEL\\\"] = \\\"INFO\\\" # Enable more detailed logging\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"from pathlib import Path\\n\",\n \"\\n\",\n \"INDEX_DIR = Path(\\\"./\\\").resolve()\\n\",\n \"INDEX_PATH = str(INDEX_DIR / \\\"demo.leann\\\")\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## Build the index\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"from leann.api import LeannBuilder\\n\",\n \"\\n\",\n \"builder = LeannBuilder(backend_name=\\\"hnsw\\\")\\n\",\n \"builder.add_text(\\\"C# is a powerful programming language and it is good at game development\\\")\\n\",\n \"builder.add_text(\\n\",\n \" \\\"Python is a powerful programming language and it is good at machine learning tasks\\\"\\n\",\n \")\\n\",\n \"builder.add_text(\\\"Machine learning transforms industries\\\")\\n\",\n \"builder.add_text(\\\"Neural networks process complex data\\\")\\n\",\n \"builder.add_text(\\\"Leann is a great storage saving engine for RAG on your MacBook\\\")\\n\",\n \"builder.build_index(INDEX_PATH)\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## Search with real-time embeddings\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"from leann.api import LeannSearcher\\n\",\n \"\\n\",\n \"searcher = LeannSearcher(INDEX_PATH)\\n\",\n \"results = searcher.search(\\\"programming languages\\\", top_k=2)\\n\",\n \"results\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## Chat with LEANN using retrieved results\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": null,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"from leann.api import LeannChat\\n\",\n \"\\n\",\n \"llm_config = {\\n\",\n \" \\\"type\\\": \\\"hf\\\",\\n\",\n \" \\\"model\\\": \\\"Qwen/Qwen3-0.6B\\\",\\n\",\n \"}\\n\",\n \"\\n\",\n \"chat = LeannChat(index_path=INDEX_PATH, llm_config=llm_config)\\n\",\n \"response = chat.ask(\\n\",\n \" \\\"Compare the two retrieved programming languages and tell me their advantages.\\\",\\n\",\n \" top_k=2,\\n\",\n \" llm_kwargs={\\\"max_tokens\\\": 128},\\n\",\n \")\\n\",\n \"response\"\n ]\n }\n ],\n \"metadata\": {\n \"kernelspec\": {\n \"display_name\": \".venv\",\n \"language\": \"python\",\n \"name\": \"python3\"\n },\n \"language_info\": {\n \"codemirror_mode\": {\n \"name\": \"ipython\",\n \"version\": 3\n },\n \"file_extension\": \".py\",\n \"mimetype\": \"text/x-python\",\n \"name\": \"python\",\n \"nbconvert_exporter\": \"python\",\n \"pygments_lexer\": \"ipython3\",\n \"version\": \"3.11.12\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 2\n}\n" }, { "path": "docker/Dockerfile", "content": "ARG PYTHON_VERSION=3.11\nFROM python:${PYTHON_VERSION}-slim\n\nARG LEANN_VERSION=0.3.6\n\nENV PIP_NO_CACHE_DIR=1 \\\n PIP_DISABLE_PIP_VERSION_CHECK=1 \\\n PIP_ROOT_USER_ACTION=ignore\n\n# Keep runtime image minimal while ensuring common C++ runtime libs are present.\nRUN apt-get update \\\n && apt-get install -y --no-install-recommends \\\n libgomp1 \\\n libstdc++6 \\\n && rm -rf /var/lib/apt/lists/*\n\nRUN python -m pip install --upgrade pip \\\n && pip install --prefer-binary \"leann==${LEANN_VERSION}\" \\\n && python -c \"import leann; import leann_backend_hnsw; import leann_backend_diskann\"\n\nWORKDIR /workspace\n\nCMD [\"python\", \"-c\", \"import leann; print('LEANN installed and importable.')\"]\n" }, { "path": "docker/Dockerfile.cpu", "content": "ARG PYTHON_VERSION=3.11\nFROM python:${PYTHON_VERSION}-slim\n\nARG LEANN_VERSION=0.3.6\n\nENV PIP_NO_CACHE_DIR=1 \\\n PIP_DISABLE_PIP_VERSION_CHECK=1 \\\n PIP_ROOT_USER_ACTION=ignore \\\n PIP_INDEX_URL=https://download.pytorch.org/whl/cpu \\\n PIP_EXTRA_INDEX_URL=https://pypi.org/simple\n\n# Keep runtime image minimal while ensuring common C++ runtime libs are present.\nRUN apt-get update \\\n && apt-get install -y --no-install-recommends \\\n libgomp1 \\\n libstdc++6 \\\n && rm -rf /var/lib/apt/lists/*\n\nRUN python -m pip install --upgrade pip \\\n && pip install --prefer-binary \"leann==${LEANN_VERSION}\" \\\n && python -c \"import leann; import leann_backend_hnsw; import leann_backend_diskann\"\n\nWORKDIR /workspace\n\nCMD [\"python\", \"-c\", \"import leann; print('LEANN installed and importable (CPU image).')\"]\n" }, { "path": "docker/Dockerfile.dev", "content": "ARG PYTHON_VERSION=3.11\nFROM python:${PYTHON_VERSION}-slim\n\nENV DEBIAN_FRONTEND=noninteractive \\\n PIP_NO_CACHE_DIR=1 \\\n PIP_DISABLE_PIP_VERSION_CHECK=1 \\\n PIP_ROOT_USER_ACTION=ignore\n\n# Build + test toolchain for local development in container.\nRUN apt-get update \\\n && apt-get install -y --no-install-recommends \\\n build-essential \\\n cmake \\\n git \\\n swig \\\n libomp-dev \\\n libboost-all-dev \\\n protobuf-compiler \\\n libzmq3-dev \\\n pkg-config \\\n libabsl-dev \\\n libaio-dev \\\n libprotobuf-dev \\\n libopenblas-dev \\\n liblapack-dev \\\n liblapacke-dev \\\n && rm -rf /var/lib/apt/lists/*\n\nRUN python -m pip install --upgrade pip \\\n && pip install uv\n\nWORKDIR /workspace\nCOPY . /workspace\n\n# Keep dependency install explicit; default includes lint + test groups.\nRUN uv sync --group lint --group test\n\nCMD [\"/bin/bash\"]\n" }, { "path": "docker/README.md", "content": "# Docker Setup\n\nThis folder provides reference Docker images for LEANN.\n\n## Build (default)\n\nDefault image (no forced CPU index):\n\n```bash\ndocker build -f docker/Dockerfile docker -t leann:latest\n```\n\n## Build (CPU)\n\nCPU-optimized image (forces PyTorch CPU index):\n\n```bash\ndocker build \\\n --build-arg PYTHON_VERSION=3.12 \\\n --build-arg LEANN_VERSION=0.3.6 \\\n -f docker/Dockerfile.cpu docker -t leann:cpu\n```\n\n## Build (development)\n\nDevelopment image (source tree + build/test toolchain + `uv sync`):\n\n```bash\ndocker build -f docker/Dockerfile.dev . -t leann:dev\n```\n\n## Run\n\n```bash\ndocker run --rm leann:latest\ndocker run --rm leann:cpu\ndocker run --rm -it leann:dev\n```\n\nExpected output:\n\n```text\nLEANN installed and importable.\n```\n\n## Notes\n\n- Both images keep LEANN package semantics unchanged (`pip install leann` with both backends).\n- `Dockerfile.cpu` uses the PyTorch CPU index to avoid downloading CUDA wheels.\n- `Dockerfile.dev` is for local development, not production deployment.\n" }, { "path": "docs/COLQWEN_GUIDE.md", "content": "# ColQwen Integration Guide\n\nEasy-to-use multimodal PDF retrieval with ColQwen2/ColPali models.\n\n## Quick Start\n\n> **🍎 Mac Users**: ColQwen is optimized for Apple Silicon with MPS acceleration for faster inference!\n\n### 1. Install Dependencies\n```bash\nuv pip install colpali_engine pdf2image pillow matplotlib qwen_vl_utils einops seaborn\nbrew install poppler # macOS only, for PDF processing\n```\n\n### 2. Basic Usage\n```bash\n# Build index from PDFs\npython -m apps.colqwen_rag build --pdfs ./my_papers/ --index research_papers\n\n# Search with text queries\npython -m apps.colqwen_rag search research_papers \"How does attention mechanism work?\"\n\n# Interactive Q&A\npython -m apps.colqwen_rag ask research_papers --interactive\n```\n\n## Commands\n\n### Build Index\n```bash\npython -m apps.colqwen_rag build \\\n --pdfs ./pdf_directory/ \\\n --index my_index \\\n --model colqwen2 \\\n --pages-dir ./page_images/ # Optional: save page images\n```\n\n**Options:**\n- `--pdfs`: Directory containing PDF files (or single PDF path)\n- `--index`: Name for the index (required)\n- `--model`: `colqwen2` (default) or `colpali`\n- `--pages-dir`: Directory to save page images (optional)\n\n### Search Index\n```bash\npython -m apps.colqwen_rag search my_index \"your question here\" --top-k 5\n```\n\n**Options:**\n- `--top-k`: Number of results to return (default: 5)\n- `--model`: Model used for search (should match build model)\n\n### Interactive Q&A\n```bash\npython -m apps.colqwen_rag ask my_index --interactive\n```\n\n**Commands in interactive mode:**\n- Type your questions naturally\n- `help`: Show available commands\n- `quit`/`exit`/`q`: Exit interactive mode\n\n## 🧪 Test & Reproduce Results\n\nRun the reproduction test for issue #119:\n```bash\npython test_colqwen_reproduction.py\n```\n\nThis will:\n1. ✅ Check dependencies\n2. 📥 Download sample PDF (Attention Is All You Need paper)\n3. 🏗️ Build test index\n4. 🔍 Run sample queries\n5. 📊 Show how to generate similarity maps\n\n## 🎨 Advanced: Similarity Maps\n\nFor visual similarity analysis, use the existing advanced script:\n```bash\ncd apps/multimodal/vision-based-pdf-multi-vector/\npython multi-vector-leann-similarity-map.py\n```\n\nEdit the script to customize:\n- `QUERY`: Your question\n- `MODEL`: \"colqwen2\" or \"colpali\"\n- `USE_HF_DATASET`: Use HuggingFace dataset or local PDFs\n- `SIMILARITY_MAP`: Generate heatmaps\n- `ANSWER`: Enable Qwen-VL answer generation\n\n## 🔧 How It Works\n\n### ColQwen2 vs ColPali\n- **ColQwen2** (`vidore/colqwen2-v1.0`): Latest vision-language model\n- **ColPali** (`vidore/colpali-v1.2`): Proven multimodal retriever\n\n### Architecture\n1. **PDF → Images**: Convert PDF pages to images (150 DPI)\n2. **Vision Encoding**: Process images with ColQwen2/ColPali\n3. **Multi-Vector Index**: Build LEANN HNSW index with multiple embeddings per page\n4. **Query Processing**: Encode text queries with same model\n5. **Similarity Search**: Find most relevant pages/regions\n6. **Visual Maps**: Generate attention heatmaps (optional)\n\n### Device Support\n- **CUDA**: Best performance with GPU acceleration\n- **MPS**: Apple Silicon Mac support\n- **CPU**: Fallback for any system (slower)\n\nAuto-detection: CUDA > MPS > CPU\n\n## 📊 Performance Tips\n\n### For Best Performance:\n```bash\n# Use ColQwen2 for latest features\n--model colqwen2\n\n# Save page images for reuse\n--pages-dir ./cached_pages/\n\n# Adjust batch size based on GPU memory\n# (automatically handled)\n```\n\n### For Large Document Sets:\n- Process PDFs in batches\n- Use SSD storage for index files\n- Consider using CUDA if available\n\n## 🔗 Related Resources\n\n- **Fast-PLAID**: https://github.com/lightonai/fast-plaid\n- **Pylate**: https://github.com/lightonai/pylate\n- **ColBERT**: https://github.com/stanford-futuredata/ColBERT\n- **ColPali Paper**: Vision-Language Models for Document Retrieval\n- **Issue #119**: https://github.com/yichuan-w/LEANN/issues/119\n\n## 🐛 Troubleshooting\n\n### PDF Conversion Issues (macOS)\n```bash\n# Install poppler\nbrew install poppler\nwhich pdfinfo && pdfinfo -v\n```\n\n### Memory Issues\n- Reduce batch size (automatically handled)\n- Use CPU instead of GPU: `export CUDA_VISIBLE_DEVICES=\"\"`\n- Process fewer PDFs at once\n\n### Model Download Issues\n- Ensure internet connection for first run\n- Models are cached after first download\n- Use HuggingFace mirrors if needed\n\n### Import Errors\n```bash\n# Ensure all dependencies installed\nuv pip install colpali_engine pdf2image pillow matplotlib qwen_vl_utils einops seaborn\n\n# Check PyTorch installation\npython -c \"import torch; print(torch.__version__)\"\n```\n\n## 💡 Examples\n\n### Research Paper Analysis\n```bash\n# Index your research papers\npython -m apps.colqwen_rag build --pdfs ~/Papers/AI/ --index ai_papers\n\n# Ask research questions\npython -m apps.colqwen_rag search ai_papers \"What are the limitations of transformer models?\"\npython -m apps.colqwen_rag search ai_papers \"How does BERT compare to GPT?\"\n```\n\n### Document Q&A\n```bash\n# Index business documents\npython -m apps.colqwen_rag build --pdfs ~/Documents/Reports/ --index reports\n\n# Interactive analysis\npython -m apps.colqwen_rag ask reports --interactive\n```\n\n### Visual Analysis\n```bash\n# Generate similarity maps for specific queries\ncd apps/multimodal/vision-based-pdf-multi-vector/\n# Edit multi-vector-leann-similarity-map.py with your query\npython multi-vector-leann-similarity-map.py\n# Check ./figures/ for generated heatmaps\n```\n\n---\n\n**🎯 This integration makes ColQwen as easy to use as other LEANN features while maintaining the full power of multimodal document understanding!**\n" }, { "path": "docs/CONTRIBUTING.md", "content": "# 🤝 Contributing\n\nWe welcome contributions! Leann is built by the community, for the community.\n\n## Ways to Contribute\n\n- 🐛 **Bug Reports**: Found an issue? Let us know!\n- 💡 **Feature Requests**: Have an idea? We'd love to hear it!\n- 🔧 **Code Contributions**: PRs welcome for all skill levels\n- 📖 **Documentation**: Help make Leann more accessible\n- 🧪 **Benchmarks**: Share your performance results\n\n## 🚀 Development Setup\n\n### Prerequisites\n\n1. **Install uv** (fast Python package installer):\n ```bash\n curl -LsSf https://astral.sh/uv/install.sh | sh\n ```\n\n2. **Clone the repository**:\n ```bash\n git clone https://github.com/yichuan-w/LEANN.git leann\n git submodule update --init --recursive\n cd leann\n ```\n\n3. **Install system dependencies**:\n\n **macOS:**\n ```bash\n brew install llvm libomp boost protobuf zeromq pkgconf\n ```\n\n **Ubuntu/Debian:**\n ```bash\n sudo apt-get install libomp-dev libboost-all-dev protobuf-compiler \\\n libabsl-dev libmkl-full-dev libaio-dev libzmq3-dev\n ```\n\n4. **Build from source**:\n ```bash\n # macOS\n CC=$(brew --prefix llvm)/bin/clang CXX=$(brew --prefix llvm)/bin/clang++ uv sync\n\n # Ubuntu/Debian\n uv sync\n ```\n\n## 🔨 Pre-commit Hooks\n\nWe use pre-commit hooks to ensure code quality and consistency. This runs automatically before each commit.\n\n### Setup Pre-commit\n\n1. **Install pre-commit tools**:\n ```bash\n uv sync --group lint\n ```\n\n2. **Install the git hooks**:\n ```bash\n pre-commit install\n ```\n\n3. **Run pre-commit manually** (optional):\n ```bash\n uv run pre-commit run --all-files\n ```\n\n### Pre-commit Checks\n\nOur pre-commit configuration includes:\n- **Trailing whitespace removal**\n- **End-of-file fixing**\n- **YAML validation**\n- **Large file prevention**\n- **Merge conflict detection**\n- **Debug statement detection**\n- **Code formatting with ruff**\n- **Code linting with ruff**\n\n## 🧪 Testing\n\n### Running Tests\n\n```bash\n# Install test tools only (no project runtime)\nuv sync --group test\n\n# Run all tests\nuv run pytest\n\n# Run specific test file\nuv run pytest test/test_filename.py\n\n# Run with coverage\nuv run pytest --cov=leann\n```\n\n### Writing Tests\n\n- Place tests in the `test/` directory\n- Follow the naming convention `test_*.py`\n- Use descriptive test names that explain what's being tested\n- Include both positive and negative test cases\n\n## 📝 Code Style\n\nWe use `ruff` for both linting and formatting to ensure consistent code style.\n\n### Format Your Code\n\n```bash\n# Format all files\nruff format\n\n# Check formatting without changing files\nruff format --check\n```\n\n### Lint Your Code\n\n```bash\n# Run linter with auto-fix\nruff check --fix\n\n# Just check without fixing\nruff check\n```\n\n### Style Guidelines\n\n- Follow PEP 8 conventions\n- Use descriptive variable names\n- Add type hints where appropriate\n- Write docstrings for all public functions and classes\n- Keep functions focused and single-purpose\n\n## 🚦 CI/CD\n\nOur CI pipeline runs automatically on all pull requests. It includes:\n\n1. **Linting and Formatting**: Ensures code follows our style guidelines\n2. **Multi-platform builds**: Tests on Ubuntu and macOS\n3. **Python version matrix**: Tests on Python 3.9-3.13\n4. **Wheel building**: Ensures packages can be built and distributed\n\n### CI Commands\n\nThe CI uses the same commands as pre-commit to ensure consistency:\n```bash\n# Linting\nruff check .\n\n# Format checking\nruff format --check .\n```\n\nMake sure your code passes these checks locally before pushing!\n\n## 🔄 Pull Request Process\n\n1. **Fork the repository** and create your branch from `main`:\n ```bash\n git checkout -b feature/your-feature-name\n ```\n\n2. **Make your changes**:\n - Write clean, documented code\n - Add tests for new functionality\n - Update documentation as needed\n\n3. **Run pre-commit checks**:\n ```bash\n pre-commit run --all-files\n ```\n\n4. **Test your changes**:\n ```bash\n uv run pytest\n ```\n\n5. **Commit with descriptive messages**:\n ```bash\n git commit -m \"feat: add new search algorithm\"\n ```\n\n Follow [Conventional Commits](https://www.conventionalcommits.org/):\n - `feat:` for new features\n - `fix:` for bug fixes\n - `docs:` for documentation changes\n - `test:` for test additions/changes\n - `refactor:` for code refactoring\n - `perf:` for performance improvements\n\n6. **Push and create a pull request**:\n - Provide a clear description of your changes\n - Reference any related issues\n - Include examples or screenshots if applicable\n\n## 📚 Documentation\n\nWhen adding new features or making significant changes:\n\n1. Update relevant documentation in `/docs`\n2. Add docstrings to new functions/classes\n3. Update README.md if needed\n4. Include usage examples\n\n## 🤔 Getting Help\n\n- **Discord**: Join our community for discussions\n- **Issues**: Check existing issues or create a new one\n- **Discussions**: For general questions and ideas\n\n## 📄 License\n\nBy contributing, you agree that your contributions will be licensed under the same license as the project (MIT).\n\n---\n\nThank you for contributing to LEANN! Every contribution, no matter how small, helps make the project better for everyone. 🌟\n" }, { "path": "docs/RELEASE.md", "content": "# Release Guide\n\n## Setup (One-time)\n\nAdd `PYPI_API_TOKEN` to GitHub Secrets:\n1. Get token: https://pypi.org/manage/account/token/\n2. Add to secrets: Settings → Secrets → Actions → `PYPI_API_TOKEN`\n\n## Release (One-click)\n\n1. Go to: https://github.com/yichuan-w/LEANN/actions/workflows/release-manual.yml\n2. Click \"Run workflow\"\n3. Enter version: `0.1.2`\n4. Click green \"Run workflow\" button\n\nThat's it! The workflow will automatically:\n- ✅ Update version in all packages\n- ✅ Build all packages\n- ✅ Publish to PyPI\n- ✅ Create GitHub tag and release\n\nCheck progress: https://github.com/yichuan-w/LEANN/actions\n" }, { "path": "docs/THINKING_BUDGET_FEATURE.md", "content": "# Thinking Budget Feature Implementation\n\n## Overview\n\nThis document describes the implementation of the **thinking budget** feature for LEANN, which allows users to control the computational effort for reasoning models like GPT-Oss:20b.\n\n## Feature Description\n\nThe thinking budget feature provides three levels of computational effort for reasoning models:\n- **`low`**: Fast responses, basic reasoning (default for simple queries)\n- **`medium`**: Balanced speed and reasoning depth\n- **`high`**: Maximum reasoning effort, best for complex analytical questions\n\n## Implementation Details\n\n### 1. Command Line Interface\n\nAdded `--thinking-budget` parameter to both CLI and RAG examples:\n\n```bash\n# LEANN CLI\nleann ask my-index --llm ollama --model gpt-oss:20b --thinking-budget high\n\n# RAG Examples\npython apps/email_rag.py --llm ollama --llm-model gpt-oss:20b --thinking-budget high\npython apps/document_rag.py --llm openai --llm-model o3 --thinking-budget medium\n```\n\n### 2. LLM Backend Support\n\n#### Ollama Backend (`packages/leann-core/src/leann/chat.py`)\n\n```python\ndef ask(self, prompt: str, **kwargs) -> str:\n # Handle thinking budget for reasoning models\n options = kwargs.copy()\n thinking_budget = kwargs.get(\"thinking_budget\")\n if thinking_budget:\n options.pop(\"thinking_budget\", None)\n if thinking_budget in [\"low\", \"medium\", \"high\"]:\n options[\"reasoning\"] = {\"effort\": thinking_budget, \"exclude\": False}\n```\n\n**API Format**: Uses Ollama's `reasoning` parameter with `effort` and `exclude` fields.\n\n#### OpenAI Backend (`packages/leann-core/src/leann/chat.py`)\n\n```python\ndef ask(self, prompt: str, **kwargs) -> str:\n # Handle thinking budget for reasoning models\n thinking_budget = kwargs.get(\"thinking_budget\")\n if thinking_budget and thinking_budget in [\"low\", \"medium\", \"high\"]:\n # Check if this is an o-series model\n o_series_models = [\"o3\", \"o3-mini\", \"o4-mini\", \"o1\", \"o3-pro\", \"o3-deep-research\"]\n if any(model in self.model for model in o_series_models):\n params[\"reasoning_effort\"] = thinking_budget\n```\n\n**API Format**: Uses OpenAI's `reasoning_effort` parameter for o-series models.\n\n### 3. Parameter Propagation\n\nThe thinking budget parameter is properly propagated through the LEANN architecture:\n\n1. **CLI** (`packages/leann-core/src/leann/cli.py`): Captures `--thinking-budget` argument\n2. **Base RAG** (`apps/base_rag_example.py`): Adds parameter to argument parser\n3. **LeannChat** (`packages/leann-core/src/leann/api.py`): Passes `llm_kwargs` to LLM\n4. **LLM Interface**: Handles the parameter in backend-specific implementations\n\n## Files Modified\n\n### Core Implementation\n- `packages/leann-core/src/leann/chat.py`: Added thinking budget support to OllamaChat and OpenAIChat\n- `packages/leann-core/src/leann/cli.py`: Added `--thinking-budget` argument\n- `apps/base_rag_example.py`: Added thinking budget parameter to RAG examples\n\n### Documentation\n- `README.md`: Added thinking budget parameter to usage examples\n- `docs/configuration-guide.md`: Added detailed documentation and usage guidelines\n\n### Examples\n- `examples/thinking_budget_demo.py`: Comprehensive demo script with usage examples\n\n## Usage Examples\n\n### Basic Usage\n```bash\n# High reasoning effort for complex questions\nleann ask my-index --llm ollama --model gpt-oss:20b --thinking-budget high\n\n# Medium reasoning for balanced performance\nleann ask my-index --llm openai --model gpt-4o --thinking-budget medium\n\n# Low reasoning for fast responses\nleann ask my-index --llm ollama --model gpt-oss:20b --thinking-budget low\n```\n\n### RAG Examples\n```bash\n# Email RAG with high reasoning\npython apps/email_rag.py --llm ollama --llm-model gpt-oss:20b --thinking-budget high\n\n# Document RAG with medium reasoning\npython apps/document_rag.py --llm openai --llm-model gpt-4o --thinking-budget medium\n```\n\n## Supported Models\n\n### Ollama Models\n- **GPT-Oss:20b**: Primary target model with reasoning capabilities\n- **Other reasoning models**: Any Ollama model that supports the `reasoning` parameter\n\n### OpenAI Models\n- **o3, o3-mini, o4-mini, o1**: o-series reasoning models with `reasoning_effort` parameter\n- **GPT-OSS models**: Models that support reasoning capabilities\n\n## Testing\n\nThe implementation includes comprehensive testing:\n- Parameter handling verification\n- Backend-specific API format validation\n- CLI argument parsing tests\n- Integration with existing LEANN architecture\n" }, { "path": "docs/ast_chunking_guide.md", "content": "# AST-Aware Code chunking guide\n\n## Overview\n\nThis guide covers best practices for using AST-aware code chunking in LEANN. AST chunking provides better semantic understanding of code structure compared to traditional text-based chunking.\n\n## Quick Start\n\n### Basic Usage\n\n```bash\n# Enable AST chunking for mixed content (code + docs)\npython -m apps.document_rag --enable-code-chunking --data-dir ./my_project\n\n# Specialized code repository indexing\npython -m apps.code_rag --repo-dir ./my_codebase\n\n# Global CLI with AST support\nleann build my-code-index --docs ./src --use-ast-chunking\n```\n\n### Installation\n\n```bash\n# Install LEANN with AST chunking support\nuv pip install -e \".\"\n```\n\n#### For normal users (PyPI install)\n- Use `pip install leann` or `uv pip install leann`.\n- `astchunk` is pulled automatically from PyPI as a dependency; no extra steps.\n\n#### For developers (from source, editable)\n```bash\ngit clone https://github.com/yichuan-w/LEANN.git leann\ncd leann\ngit submodule update --init --recursive\nuv sync\n```\n- This repo vendors `astchunk` as a git submodule at `packages/astchunk-leann` (our fork).\n- `[tool.uv.sources]` maps the `astchunk` package to that path in editable mode.\n- You can edit code under `packages/astchunk-leann` and Python will use your changes immediately (no separate `pip install astchunk` needed).\n\n## Best Practices\n\n### When to Use AST Chunking\n\n✅ **Recommended for:**\n- Code repositories with multiple languages\n- Mixed documentation and code content\n- Complex codebases with deep function/class hierarchies\n- When working with Claude Code for code assistance\n\n❌ **Not recommended for:**\n- Pure text documents\n- Very large files (>1MB)\n- Languages not supported by tree-sitter\n\n### Optimal Configuration\n\n```bash\n# Recommended settings for most codebases\npython -m apps.code_rag \\\n --repo-dir ./src \\\n --ast-chunk-size 768 \\\n --ast-chunk-overlap 96 \\\n --exclude-dirs .git __pycache__ node_modules build dist\n```\n\n### Supported Languages\n\n| Extension | Language | Status |\n|-----------|----------|--------|\n| `.py` | Python | ✅ Full support |\n| `.java` | Java | ✅ Full support |\n| `.cs` | C# | ✅ Full support |\n| `.ts`, `.tsx` | TypeScript | ✅ Full support |\n| `.js`, `.jsx` | JavaScript | ✅ Via TypeScript parser |\n\n## Integration Examples\n\n### Document RAG with Code Support\n\n```python\n# Enable code chunking in document RAG\npython -m apps.document_rag \\\n --enable-code-chunking \\\n --data-dir ./project \\\n --query \"How does authentication work in the codebase?\"\n```\n\n### Claude Code Integration\n\nWhen using with Claude Code MCP server, AST chunking provides better context for:\n- Code completion and suggestions\n- Bug analysis and debugging\n- Architecture understanding\n- Refactoring assistance\n\n## Troubleshooting\n\n### Common Issues\n\n1. **Fallback to Traditional Chunking**\n - Normal behavior for unsupported languages\n - Check logs for specific language support\n\n2. **Performance with Large Files**\n - Adjust `--max-file-size` parameter\n - Use `--exclude-dirs` to skip unnecessary directories\n\n3. **Quality Issues**\n - Try different `--ast-chunk-size` values (512, 768, 1024)\n - Adjust overlap for better context preservation\n\n### Debug Mode\n\n```bash\nexport LEANN_LOG_LEVEL=DEBUG\npython -m apps.code_rag --repo-dir ./my_code\n```\n\n## Migration from Traditional Chunking\n\nExisting workflows continue to work without changes. To enable AST chunking:\n\n```bash\n# Before\npython -m apps.document_rag --chunk-size 256\n\n# After (maintains traditional chunking for non-code files)\npython -m apps.document_rag --enable-code-chunking --chunk-size 256 --ast-chunk-size 768\n```\n\n## References\n\n- [astchunk GitHub Repository](https://github.com/yilinjz/astchunk)\n- [LEANN MCP Integration](../packages/leann-mcp/README.md)\n- [Research Paper](https://arxiv.org/html/2506.15655v1)\n\n---\n\n**Note**: AST chunking maintains full backward compatibility while enhancing code understanding capabilities.\n" }, { "path": "docs/code/embedding_model_compare.py", "content": "\"\"\"\nComparison between Sentence Transformers and OpenAI embeddings\n\nThis example shows how different embedding models handle complex queries\nand demonstrates the differences between local and API-based embeddings.\n\"\"\"\n\nimport numpy as np\nfrom leann.embedding_compute import compute_embeddings\n\n# OpenAI API key should be set as environment variable\n# export OPENAI_API_KEY=\"your-api-key-here\"\n\n# Test data\nconference_text = \"[Title]: COLING 2025 Conference\\n[URL]: https://coling2025.org/\"\nbrowser_text = \"[Title]: Browser Use Tool\\n[URL]: https://github.com/browser-use\"\n\n# Two queries with same intent but different wording\nquery1 = \"Tell me my browser history about some conference i often visit\"\nquery2 = \"browser history about conference I often visit\"\n\ntexts = [query1, query2, conference_text, browser_text]\n\n\ndef cosine_similarity(a, b):\n return np.dot(a, b) # Already normalized\n\n\ndef analyze_embeddings(embeddings, model_name):\n print(f\"\\n=== {model_name} Results ===\")\n\n # Results for Query 1\n sim1_conf = cosine_similarity(embeddings[0], embeddings[2])\n sim1_browser = cosine_similarity(embeddings[0], embeddings[3])\n\n print(f\"Query 1: '{query1}'\")\n print(f\" → Conference similarity: {sim1_conf:.4f} {'✓' if sim1_conf > sim1_browser else ''}\")\n print(\n f\" → Browser similarity: {sim1_browser:.4f} {'✓' if sim1_browser > sim1_conf else ''}\"\n )\n print(f\" Winner: {'Conference' if sim1_conf > sim1_browser else 'Browser'}\")\n\n # Results for Query 2\n sim2_conf = cosine_similarity(embeddings[1], embeddings[2])\n sim2_browser = cosine_similarity(embeddings[1], embeddings[3])\n\n print(f\"\\nQuery 2: '{query2}'\")\n print(f\" → Conference similarity: {sim2_conf:.4f} {'✓' if sim2_conf > sim2_browser else ''}\")\n print(\n f\" → Browser similarity: {sim2_browser:.4f} {'✓' if sim2_browser > sim2_conf else ''}\"\n )\n print(f\" Winner: {'Conference' if sim2_conf > sim2_browser else 'Browser'}\")\n\n # Show the impact\n print(\"\\n=== Impact Analysis ===\")\n print(f\"Conference similarity change: {sim2_conf - sim1_conf:+.4f}\")\n print(f\"Browser similarity change: {sim2_browser - sim1_browser:+.4f}\")\n\n if sim1_conf > sim1_browser and sim2_browser > sim2_conf:\n print(\"❌ FLIP: Adding 'browser history' flips winner from Conference to Browser!\")\n elif sim1_conf > sim1_browser and sim2_conf > sim2_browser:\n print(\"✅ STABLE: Conference remains winner in both queries\")\n elif sim1_browser > sim1_conf and sim2_browser > sim2_conf:\n print(\"✅ STABLE: Browser remains winner in both queries\")\n else:\n print(\"🔄 MIXED: Results vary between queries\")\n\n return {\n \"query1_conf\": sim1_conf,\n \"query1_browser\": sim1_browser,\n \"query2_conf\": sim2_conf,\n \"query2_browser\": sim2_browser,\n }\n\n\n# Test Sentence Transformers\nprint(\"Testing Sentence Transformers (facebook/contriever)...\")\ntry:\n st_embeddings = compute_embeddings(texts, \"facebook/contriever\", mode=\"sentence-transformers\")\n st_results = analyze_embeddings(st_embeddings, \"Sentence Transformers (facebook/contriever)\")\nexcept Exception as e:\n print(f\"❌ Sentence Transformers failed: {e}\")\n st_results = None\n\n# Test OpenAI\nprint(\"\\n\" + \"=\" * 60)\nprint(\"Testing OpenAI (text-embedding-3-small)...\")\ntry:\n openai_embeddings = compute_embeddings(texts, \"text-embedding-3-small\", mode=\"openai\")\n openai_results = analyze_embeddings(openai_embeddings, \"OpenAI (text-embedding-3-small)\")\nexcept Exception as e:\n print(f\"❌ OpenAI failed: {e}\")\n openai_results = None\n\n# Compare results\nif st_results and openai_results:\n print(\"\\n\" + \"=\" * 60)\n print(\"=== COMPARISON SUMMARY ===\")\n" }, { "path": "docs/configuration-guide.md", "content": "# LEANN Configuration Guide\n\nThis guide helps you optimize LEANN for different use cases and understand the trade-offs between various configuration options.\n\n## Getting Started: Simple is Better\n\nWhen first trying LEANN, start with a small dataset to quickly validate your approach:\n\n**For document RAG**: The default `data/` directory works perfectly - includes 2 AI research papers, Pride and Prejudice literature, and a technical report\n```bash\npython -m apps.document_rag --query \"What techniques does LEANN use?\"\n```\n\n**For other data sources**: Limit the dataset size for quick testing\n```bash\n# WeChat: Test with recent messages only\npython -m apps.wechat_rag --max-items 100 --query \"What did we discuss about the project timeline?\"\n\n# Browser history: Last few days\npython -m apps.browser_rag --max-items 500 --query \"Find documentation about vector databases\"\n\n# Email: Recent inbox\npython -m apps.email_rag --max-items 200 --query \"Who sent updates about the deployment status?\"\n```\n\nOnce validated, scale up gradually:\n- 100 documents → 1,000 → 10,000 → full dataset (`--max-items -1`)\n- This helps identify issues early before committing to long processing times\n\n## Embedding Model Selection: Understanding the Trade-offs\n\nBased on our experience developing LEANN, embedding models fall into three categories:\n\n### Small Models (< 100M parameters)\n**Example**: `sentence-transformers/all-MiniLM-L6-v2` (22M params)\n- **Pros**: Lightweight, fast for both indexing and inference\n- **Cons**: Lower semantic understanding, may miss nuanced relationships\n- **Use when**: Speed is critical, handling simple queries, interactive mode, or just experimenting with LEANN. If time is not a constraint, consider using a larger/better embedding model\n\n### Medium Models (100M-500M parameters)\n**Example**: `facebook/contriever` (110M params), `BAAI/bge-base-en-v1.5` (110M params)\n- **Pros**: Balanced performance, good multilingual support, reasonable speed\n- **Cons**: Requires more compute than small models\n- **Use when**: Need quality results without extreme compute requirements, general-purpose RAG applications\n\n### Large Models (500M+ parameters)\n**Example**: `Qwen/Qwen3-Embedding-0.6B` (600M params), `intfloat/multilingual-e5-large` (560M params)\n- **Pros**: Best semantic understanding, captures complex relationships, excellent multilingual support. **Qwen3-Embedding-0.6B achieves nearly OpenAI API performance!**\n- **Cons**: Slower inference, longer index build times\n- **Use when**: Quality is paramount and you have sufficient compute resources. **Highly recommended** for production use\n\n### Quick Start: Cloud and Local Embedding Options\n\n**OpenAI Embeddings (Fastest Setup)**\nFor immediate testing without local model downloads(also if you [do not have GPU](https://github.com/yichuan-w/LEANN/issues/43) and do not care that much about your document leak, you should use this, we compute the embedding and recompute using openai API):\n```bash\n# Set OpenAI embeddings (requires OPENAI_API_KEY)\n--embedding-mode openai --embedding-model text-embedding-3-small\n```\n\n**Ollama Embeddings (Privacy-Focused)**\nFor local embeddings with complete privacy:\n```bash\n# First, pull an embedding model\nollama pull nomic-embed-text\n\n# Use Ollama embeddings\n--embedding-mode ollama --embedding-model nomic-embed-text\n```\n\n
\nCloud vs Local Trade-offs\n\n**OpenAI Embeddings** (`text-embedding-3-small/large`)\n- **Pros**: No local compute needed, consistently fast, high quality\n- **Cons**: Requires API key, costs money, data leaves your system, [known limitations with certain languages](https://yichuan-w.github.io/blog/lessons_learned_in_dev_leann/)\n- **When to use**: Prototyping, non-sensitive data, need immediate results\n\n**Local Embeddings**\n- **Pros**: Complete privacy, no ongoing costs, full control, can sometimes outperform OpenAI embeddings\n- **Cons**: Slower than cloud APIs, requires local compute resources\n- **When to use**: Production systems, sensitive data, cost-sensitive applications\n\n
\n\n## Local & Remote Inference Endpoints\n\n> Applies to both LLMs (`leann ask`) and embeddings (`leann build`).\n\nLEANN now treats Ollama, LM Studio, and other OpenAI-compatible runtimes as first-class providers. You can point LEANN at any compatible endpoint – either on the same machine or across the network – with a couple of flags or environment variables.\n\n### One-Time Environment Setup\n\n```bash\n# Works for OpenAI-compatible runtimes such as LM Studio, vLLM, SGLang, llamafile, etc.\nexport OPENAI_API_KEY=\"your-key\" # or leave unset for local servers that do not check keys\nexport OPENAI_BASE_URL=\"http://localhost:1234/v1\"\n\n# Ollama-compatible runtimes (Ollama, Ollama on another host, llamacpp-server, etc.)\nexport LEANN_OLLAMA_HOST=\"http://localhost:11434\" # falls back to OLLAMA_HOST or LOCAL_LLM_ENDPOINT\n```\n\nLEANN also recognises `LEANN_LOCAL_LLM_HOST` (highest priority), `LEANN_OPENAI_BASE_URL`, and `LOCAL_OPENAI_BASE_URL`, so existing scripts continue to work.\n\n### Passing Hosts Per Command\n\n```bash\n# Build an index with a remote embedding server\nleann build my-notes \\\n --docs ./notes \\\n --embedding-mode openai \\\n --embedding-model text-embedding-qwen3-embedding-0.6b \\\n --embedding-api-base http://192.168.1.50:1234/v1 \\\n --embedding-api-key local-dev-key\n\n# Query using a local LM Studio instance via OpenAI-compatible API\nleann ask my-notes \\\n --llm openai \\\n --llm-model qwen3-8b \\\n --api-base http://localhost:1234/v1 \\\n --api-key local-dev-key\n\n# Query an Ollama instance running on another box\nleann ask my-notes \\\n --llm ollama \\\n --llm-model qwen3:14b \\\n --host http://192.168.1.101:11434\n```\n\n⚠️ **Make sure the endpoint is reachable**: when your inference server runs on a home/workstation and the index/search job runs in the cloud, the server must be able to reach the host you configured. Typical options include:\n\n- Expose a public IP (and open the relevant port) on the machine that hosts LM Studio/Ollama.\n- Configure router or cloud provider port forwarding.\n- Tunnel traffic through tools like `tailscale`, `cloudflared`, or `ssh -R`.\n\nWhen you set these options while building an index, LEANN stores them in `meta.json`. Any subsequent `leann ask` or searcher process automatically reuses the same provider settings – even when we spawn background embedding servers. This makes the “server without GPU talking to my local workstation” workflow from [issue #80](https://github.com/yichuan-w/LEANN/issues/80#issuecomment-2287230548) work out-of-the-box.\n\n**Tip:** If your runtime does not require an API key (many local stacks don’t), leave `--api-key` unset. LEANN will skip injecting credentials.\n\n### Python API Usage\n\nYou can pass the same configuration from Python:\n\n```python\nfrom leann.api import LeannBuilder\n\nbuilder = LeannBuilder(\n backend_name=\"hnsw\",\n embedding_mode=\"openai\",\n embedding_model=\"text-embedding-qwen3-embedding-0.6b\",\n embedding_options={\n \"base_url\": \"http://192.168.1.50:1234/v1\",\n \"api_key\": \"local-dev-key\",\n },\n)\nbuilder.build_index(\"./indexes/my-notes\", chunks)\n```\n\n`embedding_options` is persisted to the index `meta.json`, so subsequent `LeannSearcher` or `LeannChat` sessions automatically reuse the same provider settings (the embedding server manager forwards them to the provider for you).\n\n## Optional Embedding Features\n\n### Task-Specific Prompt Templates\n\nSome embedding models are trained with task-specific prompts to differentiate between documents and queries. The most notable example is **Google's EmbeddingGemma**, which requires different prompts depending on the use case:\n\n- **Indexing documents**: `\"title: none | text: \"`\n- **Search queries**: `\"task: search result | query: \"`\n\nLEANN supports automatic prompt prepending via the `--embedding-prompt-template` flag:\n\n```bash\n# Build index with EmbeddingGemma (via LM Studio or Ollama)\nleann build my-docs \\\n --docs ./documents \\\n --embedding-mode openai \\\n --embedding-model text-embedding-embeddinggemma-300m-qat \\\n --embedding-api-base http://localhost:1234/v1 \\\n --embedding-prompt-template \"title: none | text: \" \\\n --force\n\n# Search with query-specific prompt\nleann search my-docs \\\n --query \"What is quantum computing?\" \\\n --embedding-prompt-template \"task: search result | query: \"\n```\n\nA full example that is used for building the LEANN's repo during dev:\n```\nsource \"$LEANN_PATH/.venv/bin/activate\" && \\\nleann build --docs $(git ls-files | grep -Ev '\\.(png|jpg|jpeg|gif|yml|yaml|sh|pdf|JPG)$') --embedding-mode openai \\\n--embedding-model text-embedding-embeddinggemma-300m-qat \\\n--embedding-prompt-template \"title: none | text: \" \\\n--query-prompt-template \"task: search result | query: \" \\\n--embedding-api-key local-dev-key \\\n--embedding-api-base http://localhost:1234/v1 \\\n--doc-chunk-size 1024 --doc-chunk-overlap 100 \\\n--code-chunk-size 1024 --code-chunk-overlap 100 \\\n--ast-chunk-size 1024 --ast-chunk-overlap 100 \\\n--force --use-ast-chunking --no-compact --no-recompute\n```\n\n**Important Notes:**\n- **Only use with compatible models**: EmbeddingGemma and similar task-specific models\n- **NOT for regular models**: Adding prompts to models like `nomic-embed-text`, `text-embedding-3-small`, or `bge-base-en-v1.5` will corrupt embeddings\n- **Template is saved**: Build-time templates are saved to `.meta.json` for reference; you can add both `--embedding-prompt-template` and `--query-prompt-template` values during building phase, and this way the mcp query will automatically pick up the query template\n- **Flexible prompts**: You can use any prompt string, or leave it empty (`\"\"`)\n\n**Python API:**\n```python\nfrom leann.api import LeannBuilder\n\nbuilder = LeannBuilder(\n embedding_mode=\"openai\",\n embedding_model=\"text-embedding-embeddinggemma-300m-qat\",\n embedding_options={\n \"base_url\": \"http://localhost:1234/v1\",\n \"api_key\": \"lm-studio\",\n \"prompt_template\": \"title: none | text: \",\n },\n)\nbuilder.build_index(\"./indexes/my-docs\", chunks)\n```\n\n**References:**\n- [HuggingFace Blog: EmbeddingGemma](https://huggingface.co/blog/embeddinggemma) - Technical details\n\n### LM Studio Auto-Detection (Optional)\n\nWhen using LM Studio with the OpenAI-compatible API, LEANN can optionally auto-detect model context lengths via the LM Studio SDK. This eliminates manual configuration for token limits.\n\n**Prerequisites:**\n```bash\n# Install Node.js (if not already installed)\n# Then install the LM Studio SDK globally\nnpm install -g @lmstudio/sdk\n```\n\n**How it works:**\n1. LEANN detects LM Studio URLs (`:1234`, `lmstudio` in URL)\n2. Queries model metadata via Node.js subprocess\n3. Automatically unloads model after query (respects your JIT auto-evict settings)\n4. Falls back to static registry if SDK unavailable\n\n**No configuration needed** - it works automatically when SDK is installed:\n\n```bash\nleann build my-docs \\\n --docs ./documents \\\n --embedding-mode openai \\\n --embedding-model text-embedding-nomic-embed-text-v1.5 \\\n --embedding-api-base http://localhost:1234/v1\n # Context length auto-detected if SDK available\n # Falls back to registry (2048) if not\n```\n\n**Benefits:**\n- ✅ Automatic token limit detection\n- ✅ Respects LM Studio JIT auto-evict settings\n- ✅ No manual registry maintenance\n- ✅ Graceful fallback if SDK unavailable\n\n**Note:** This is completely optional. LEANN works perfectly fine without the SDK using the built-in token limit registry.\n\n## Index Selection: Matching Your Scale\n\n### HNSW (Hierarchical Navigable Small World)\n**Best for**: Small to medium datasets (< 10M vectors) - **Default and recommended for extreme low storage**\n- Full recomputation required\n- High memory usage during build phase\n- Excellent recall (95%+)\n\n```bash\n# Optimal for most use cases\n--backend-name hnsw --graph-degree 32 --build-complexity 64\n```\n\n### DiskANN\n**Best for**: Large datasets, especially when you want `recompute=True`.\n\n**Key advantages:**\n- **Faster search** on large datasets (3x+ speedup vs HNSW in many cases)\n- **Smart storage**: `recompute=True` enables automatic graph partitioning for smaller indexes\n- **Better scaling**: Designed for 100k+ documents\n\n**Recompute behavior:**\n- `recompute=True` (recommended): Pure PQ traversal + final reranking - faster and enables partitioning\n- `recompute=False`: PQ + partial real distances during traversal - slower but higher accuracy\n\n```bash\n# Recommended for most use cases\n--backend-name diskann --graph-degree 32 --build-complexity 64\n```\n\n**Performance Benchmark**: Run `uv run benchmarks/diskann_vs_hnsw_speed_comparison.py` to compare DiskANN and HNSW on your system.\n\n## LLM Selection: Engine and Model Comparison\n\n### LLM Engines\n\n**OpenAI** (`--llm openai`)\n- **Pros**: Best quality, consistent performance, no local resources needed\n- **Cons**: Costs money ($0.15-2.5 per million tokens), requires internet, data privacy concerns\n- **Models**: `gpt-4o-mini` (fast, cheap), `gpt-4o` (best quality), `o3` (reasoning), `o3-mini` (reasoning, cheaper)\n- **Thinking Budget**: Use `--thinking-budget low/medium/high` for o-series reasoning models (o3, o3-mini, o4-mini)\n- **Note**: Our current default, but we recommend switching to Ollama for most use cases\n\n**Ollama** (`--llm ollama`)\n- **Pros**: Fully local, free, privacy-preserving, good model variety\n- **Cons**: Requires local GPU/CPU resources, slower than cloud APIs, need to install extra [ollama app](https://github.com/ollama/ollama?tab=readme-ov-file#ollama) and pre-download models by `ollama pull`\n- **Models**: `qwen3:0.6b` (ultra-fast), `qwen3:1.7b` (balanced), `qwen3:4b` (good quality), `qwen3:7b` (high quality), `deepseek-r1:1.5b` (reasoning)\n- **Thinking Budget**: Use `--thinking-budget low/medium/high` for reasoning models like GPT-Oss:20b\n\n**HuggingFace** (`--llm hf`)\n- **Pros**: Free tier available, huge model selection, direct model loading (vs Ollama's server-based approach)\n- **Cons**: More complex initial setup\n- **Models**: `Qwen/Qwen3-1.7B-FP8`\n\n## Parameter Tuning Guide\n\n### Search Complexity Parameters\n\n**`--build-complexity`** (index building)\n- Controls thoroughness during index construction\n- Higher = better recall but slower build\n- Recommendations:\n - 32: Quick prototyping\n - 64: Balanced (default)\n - 128: Production systems\n - 256: Maximum quality\n\n**`--search-complexity`** (query time)\n- Controls search thoroughness\n- Higher = better results but slower\n- Recommendations:\n - 16: Fast/Interactive search\n - 32: High quality with diversity\n - 64+: Maximum accuracy\n\n### Top-K Selection\n\n**`--top-k`** (number of retrieved chunks)\n- More chunks = better context but slower LLM processing\n- Should be always smaller than `--search-complexity`\n- Guidelines:\n - 10-20: General questions (default: 20)\n - 30+: Complex multi-hop reasoning requiring comprehensive context\n\n**Trade-off formula**:\n- Retrieval time ∝ log(n) × search_complexity\n- LLM processing time ∝ top_k × chunk_size\n- Total context = top_k × chunk_size tokens\n\n### Thinking Budget for Reasoning Models\n\n**`--thinking-budget`** (reasoning effort level)\n- Controls the computational effort for reasoning models\n- Options: `low`, `medium`, `high`\n- Guidelines:\n - `low`: Fast responses, basic reasoning (default for simple queries)\n - `medium`: Balanced speed and reasoning depth\n - `high`: Maximum reasoning effort, best for complex analytical questions\n- **Supported Models**:\n - **Ollama**: `gpt-oss:20b`, `gpt-oss:120b`\n - **OpenAI**: `o3`, `o3-mini`, `o4-mini`, `o1` (o-series reasoning models)\n- **Note**: Models without reasoning support will show a warning and proceed without reasoning parameters\n- **Example**: `--thinking-budget high` for complex analytical questions\n\n**📖 For detailed usage examples and implementation details, check out [Thinking Budget Documentation](THINKING_BUDGET_FEATURE.md)**\n\n**💡 Quick Examples:**\n```bash\n# OpenAI o-series reasoning model\npython apps/document_rag.py --query \"What are the main techniques LEANN explores?\" \\\n --index-dir hnswbuild --backend hnsw \\\n --llm openai --llm-model o3 --thinking-budget medium\n\n# Ollama reasoning model\npython apps/document_rag.py --query \"What are the main techniques LEANN explores?\" \\\n --index-dir hnswbuild --backend hnsw \\\n --llm ollama --llm-model gpt-oss:20b --thinking-budget high\n```\n\n### Graph Degree (HNSW/DiskANN)\n\n**`--graph-degree`**\n- Number of connections per node in the graph\n- Higher = better recall but more memory\n- HNSW: 16-32 (default: 32)\n- DiskANN: 32-128 (default: 64)\n\n\n## Performance Optimization Checklist\n\n### If Embedding is Too Slow\n\n1. **Switch to smaller model**:\n ```bash\n # From large model\n --embedding-model Qwen/Qwen3-Embedding-0.6B\n # To small model\n --embedding-model sentence-transformers/all-MiniLM-L6-v2\n ```\n\n2. **Limit dataset size for testing**:\n ```bash\n --max-items 1000 # Process first 1k items only\n ```\n\n3. **Use MLX on Apple Silicon** (optional optimization):\n ```bash\n --embedding-mode mlx --embedding-model mlx-community/Qwen3-Embedding-0.6B-8bit\n ```\n MLX might not be the best choice, as we tested and found that it only offers 1.3x acceleration compared to HF, so maybe using ollama is a better choice for embedding generation\n\n4. **Use Ollama**\n ```bash\n --embedding-mode ollama --embedding-model nomic-embed-text\n ```\n To discover additional embedding models in ollama, check out https://ollama.com/search?c=embedding or read more about embedding models at https://ollama.com/blog/embedding-models, please do check the model size that works best for you\n### If Search Quality is Poor\n\n1. **Increase retrieval count**:\n ```bash\n --top-k 30 # Retrieve more candidates\n ```\n\n2. **Upgrade embedding model**:\n ```bash\n # For English\n --embedding-model BAAI/bge-base-en-v1.5\n # For multilingual\n --embedding-model intfloat/multilingual-e5-large\n ```\n\n## Understanding the Trade-offs\n\nEvery configuration choice involves trade-offs:\n\n| Factor | Small/Fast | Large/Quality |\n|--------|------------|---------------|\n| Embedding Model | `all-MiniLM-L6-v2` | `Qwen/Qwen3-Embedding-0.6B` |\n| Chunk Size | 512 tokens | 128 tokens |\n| Index Type | HNSW | DiskANN |\n| LLM | `qwen3:1.7b` | `gpt-4o` |\n\nThe key is finding the right balance for your specific use case. Start small and simple, measure performance, then scale up only where needed.\n\n## Low-resource setups\n\nIf you don’t have a local GPU or builds/searches are too slow, use one or more of the options below.\n\n### 1) Use OpenAI embeddings (no local compute)\n\nFastest path with zero local GPU requirements. Set your API key and use OpenAI embeddings during build and search:\n\n```bash\nexport OPENAI_API_KEY=sk-...\n\n# Build with OpenAI embeddings\nleann build my-index \\\n --embedding-mode openai \\\n --embedding-model text-embedding-3-small\n\n# Search with OpenAI embeddings (recompute at query time)\nleann search my-index \"your query\" \\\n --recompute\n```\n\n### 2) Run remote builds with SkyPilot (cloud GPU)\n\nOffload embedding generation and index building to a GPU VM using [SkyPilot](https://docs.skypilot.co/en/latest/docs/index.html). A template is provided at `sky/leann-build.yaml`.\n\n```bash\n# One-time: install and configure SkyPilot\npip install skypilot\n\n# Launch with defaults (L4:1) and mount ./data to ~/leann-data; the build runs automatically\nsky launch -c leann-gpu sky/leann-build.yaml\n\n# Override parameters via -e key=value (optional)\nsky launch -c leann-gpu sky/leann-build.yaml \\\n -e index_name=my-index \\\n -e backend=hnsw \\\n -e embedding_mode=sentence-transformers \\\n -e embedding_model=Qwen/Qwen3-Embedding-0.6B\n\n# Copy the built index back to your local .leann (use rsync)\nrsync -Pavz leann-gpu:~/.leann/indexes/my-index ./.leann/indexes/\n```\n\n### 3) Disable recomputation to trade storage for speed\n\nIf you need lower latency and have more storage/memory, disable recomputation. This stores full embeddings and avoids recomputing at search time.\n\n```bash\n# Build without recomputation (HNSW requires non-compact in this mode)\nleann build my-index --no-recompute --no-compact\n\n# Search without recomputation\nleann search my-index \"your query\" --no-recompute\n```\n\nWhen to use:\n- Extreme low latency requirements (high QPS, interactive assistants)\n- Read-heavy workloads where storage is cheaper than latency\n- No always-available GPU\n\nConstraints:\n- HNSW: when `--no-recompute` is set, LEANN automatically disables compact mode during build\n- DiskANN: supported; `--no-recompute` skips selective recompute during search\n\nStorage impact:\n- Storing N embeddings of dimension D with float32 requires approximately N × D × 4 bytes\n- Example: 1,000,000 chunks × 768 dims × 4 bytes ≈ 2.86 GB (plus graph/metadata)\n\nConverting an existing index (rebuild required):\n```bash\n# Rebuild in-place (ensure you still have original docs or can regenerate chunks)\nleann build my-index --force --no-recompute --no-compact\n```\n\nPython API usage:\n```python\nfrom leann import LeannSearcher\n\nsearcher = LeannSearcher(\"/path/to/my-index.leann\")\nresults = searcher.search(\"your query\", top_k=10, recompute_embeddings=False)\n```\n\nTrade-offs:\n- Lower latency and fewer network hops at query time\n- Significantly higher storage (10–100× vs selective recomputation)\n- Slightly larger memory footprint during build and search\n\nQuick benchmark results (`benchmarks/benchmark_no_recompute.py` with 5k texts, complexity=32):\n\n- HNSW\n\n ```text\n recompute=True: search_time=0.818s, size=1.1MB\n recompute=False: search_time=0.012s, size=16.6MB\n ```\n\n- DiskANN\n\n ```text\n recompute=True: search_time=0.041s, size=5.9MB\n recompute=False: search_time=0.013s, size=24.6MB\n ```\n\nConclusion:\n- **HNSW**: `no-recompute` is significantly faster (no embedding recomputation) but requires much more storage (stores all embeddings)\n- **DiskANN**: `no-recompute` uses PQ + partial real distances during traversal (slower but higher accuracy), while `recompute=True` uses pure PQ traversal + final reranking (faster traversal, enables build-time partitioning for smaller storage)\n\n\n\n## Further Reading\n\n- [Lessons Learned Developing LEANN](https://yichuan-w.github.io/blog/lessons_learned_in_dev_leann/)\n- [LEANN Technical Paper](https://arxiv.org/abs/2506.08276)\n- [DiskANN Original Paper](https://suhasjs.github.io/files/diskann_neurips19.pdf)\n- [SSD-based Graph Partitioning](https://github.com/SonglinLife/SSD_BASED_PLAN)\n" }, { "path": "docs/faq.md", "content": "# FAQ\n\n## 1. My building time seems long\n\nYou can speed up the process by using a lightweight embedding model. Add this to your arguments:\n\n```bash\n--embedding-model sentence-transformers/all-MiniLM-L6-v2\n```\n**Model sizes:** `all-MiniLM-L6-v2` (30M parameters), `facebook/contriever` (~100M parameters), `Qwen3-0.6B` (600M parameters)\n\n## 2. When should I use prompt templates?\n\n**Use prompt templates ONLY with task-specific embedding models** like Google's EmbeddingGemma. These models are specially trained to use different prompts for documents vs queries.\n\n**DO NOT use with regular models** like `nomic-embed-text`, `text-embedding-3-small`, or `bge-base-en-v1.5` - adding prompts to these models will corrupt the embeddings.\n\n**Example usage with EmbeddingGemma:**\n```bash\n# Build with document prompt\nleann build my-docs --embedding-prompt-template \"title: none | text: \"\n\n# Search with query prompt\nleann search my-docs --query \"your question\" --embedding-prompt-template \"task: search result | query: \"\n```\n\nSee the [Configuration Guide: Task-Specific Prompt Templates](configuration-guide.md#task-specific-prompt-templates) for detailed usage.\n\n## 3. Why is LM Studio loading multiple copies of my model?\n\nThis was fixed in recent versions. LEANN now properly unloads models after querying metadata, respecting your LM Studio JIT auto-evict settings.\n\n**If you still see duplicates:**\n- Update to the latest LEANN version\n- Restart LM Studio to clear loaded models\n- Check that you have JIT auto-evict enabled in LM Studio settings\n\n**How it works now:**\n1. LEANN loads model temporarily to get context length\n2. Immediately unloads after query\n3. LM Studio JIT loads model on-demand for actual embeddings\n4. Auto-evicts per your settings\n\n## 4. Do I need Node.js and @lmstudio/sdk?\n\n**No, it's completely optional.** LEANN works perfectly fine without them using a built-in token limit registry.\n\n**Benefits if you install it:**\n- Automatic context length detection for LM Studio models\n- No manual registry maintenance\n- Always gets accurate token limits from the model itself\n\n**To install (optional):**\n```bash\nnpm install -g @lmstudio/sdk\n```\n\nSee [Configuration Guide: LM Studio Auto-Detection](configuration-guide.md#lm-studio-auto-detection-optional) for details.\n" }, { "path": "docs/features.md", "content": "# ✨ Detailed Features\n\n## 🔥 Core Features\n\n- **🔄 Real-time Embeddings** - Eliminate heavy embedding storage with dynamic computation using optimized ZMQ servers and highly optimized search paradigm (overlapping and batching) with highly optimized embedding engine\n- **🧠 AST-Aware Code Chunking** - Intelligent code chunking that preserves semantic boundaries (functions, classes, methods) for Python, Java, C#, and TypeScript files\n- **📈 Scalable Architecture** - Handles millions of documents on consumer hardware; the larger your dataset, the more LEANN can save\n- **🎯 Graph Pruning** - Advanced techniques to minimize the storage overhead of vector search to a limited footprint\n- **🏗️ Pluggable Backends** - HNSW/FAISS (default), with optional DiskANN for large-scale deployments\n\n## 🛠️ Technical Highlights\n- **🔄 Recompute Mode** - Highest accuracy scenarios while eliminating vector storage overhead\n- **⚡ Zero-copy Operations** - Minimize IPC overhead by transferring distances instead of embeddings\n- **🚀 High-throughput Embedding Pipeline** - Optimized batched processing for maximum efficiency\n- **🎯 Two-level Search** - Novel coarse-to-fine search overlap for accelerated query processing (optional)\n- **💾 Memory-mapped Indices** - Fast startup with raw text mapping to reduce memory overhead\n- **🚀 MLX Support** - Ultra-fast recompute/build with quantized embedding models, accelerating building and search ([minimal example](../examples/mlx_demo.py))\n\n## 🎨 Developer Experience\n\n- **Simple Python API** - Get started in minutes\n- **Extensible backend system** - Easy to add new algorithms\n- **Comprehensive examples** - From basic usage to production deployment\n" }, { "path": "docs/grep_search.md", "content": "# LEANN Grep Search Usage Guide\n\n## Overview\n\nLEANN's grep search functionality provides exact text matching for finding specific code patterns, error messages, function names, or exact phrases in your indexed documents.\n\n## Basic Usage\n\n### Simple Grep Search\n\n```python\nfrom leann.api import LeannSearcher\n\nsearcher = LeannSearcher(\"your_index_path\")\n\n# Exact text search\nresults = searcher.search(\"def authenticate_user\", use_grep=True, top_k=5)\n\nfor result in results:\n print(f\"Score: {result.score}\")\n print(f\"Text: {result.text[:100]}...\")\n print(\"-\" * 40)\n```\n\n### Comparison: Semantic vs Grep Search\n\n```python\n# Semantic search - finds conceptually similar content\nsemantic_results = searcher.search(\"machine learning algorithms\", top_k=3)\n\n# Grep search - finds exact text matches\ngrep_results = searcher.search(\"def train_model\", use_grep=True, top_k=3)\n```\n\n## When to Use Grep Search\n\n### Use Cases\n\n- **Code Search**: Finding specific function definitions, class names, or variable references\n- **Error Debugging**: Locating exact error messages or stack traces\n- **Documentation**: Finding specific API endpoints or exact terminology\n\n### Examples\n\n```python\n# Find function definitions\nfunctions = searcher.search(\"def __init__\", use_grep=True)\n\n# Find import statements\nimports = searcher.search(\"from sklearn import\", use_grep=True)\n\n# Find specific error types\nerrors = searcher.search(\"FileNotFoundError\", use_grep=True)\n\n# Find TODO comments\ntodos = searcher.search(\"TODO:\", use_grep=True)\n\n# Find configuration entries\nconfigs = searcher.search(\"server_port=\", use_grep=True)\n```\n\n## Technical Details\n\n### How It Works\n\n1. **File Location**: Grep search operates on the raw text stored in `.jsonl` files\n2. **Command Execution**: Uses the system `grep` command with case-insensitive search\n3. **Result Processing**: Parses JSON lines and extracts text and metadata\n4. **Scoring**: Simple frequency-based scoring based on query term occurrences\n\n### Search Process\n\n```\nQuery: \"def train_model\"\n ↓\ngrep -i -n \"def train_model\" documents.leann.passages.jsonl\n ↓\nParse matching JSON lines\n ↓\nCalculate scores based on term frequency\n ↓\nReturn top_k results\n```\n\n### Scoring Algorithm\n\n```python\n# Term frequency in document\nscore = text.lower().count(query.lower())\n```\n\nResults are ranked by score (highest first), with higher scores indicating more occurrences of the search term.\n\n## Error Handling\n\n### Common Issues\n\n#### Grep Command Not Found\n```\nRuntimeError: grep command not found. Please install grep or use semantic search.\n```\n\n**Solution**: Install grep on your system:\n- **Ubuntu/Debian**: `sudo apt-get install grep`\n- **macOS**: grep is pre-installed\n- **Windows**: Use WSL or install grep via Git Bash/MSYS2\n\n#### No Results Found\n```python\n# Check if your query exists in the raw data\nresults = searcher.search(\"your_query\", use_grep=True)\nif not results:\n print(\"No exact matches found. Try:\")\n print(\"1. Check spelling and case\")\n print(\"2. Use partial terms\")\n print(\"3. Switch to semantic search\")\n```\n\n## Complete Example\n\n```python\n#!/usr/bin/env python3\n\"\"\"\nGrep Search Example\nDemonstrates grep search for exact text matching.\n\"\"\"\n\nfrom leann.api import LeannSearcher\n\ndef demonstrate_grep_search():\n # Initialize searcher\n searcher = LeannSearcher(\"my_index\")\n\n print(\"=== Function Search ===\")\n functions = searcher.search(\"def __init__\", use_grep=True, top_k=5)\n for i, result in enumerate(functions, 1):\n print(f\"{i}. Score: {result.score}\")\n print(f\" Preview: {result.text[:60]}...\")\n print()\n\n print(\"=== Error Search ===\")\n errors = searcher.search(\"FileNotFoundError\", use_grep=True, top_k=3)\n for result in errors:\n print(f\"Content: {result.text.strip()}\")\n print(\"-\" * 40)\n\nif __name__ == \"__main__\":\n demonstrate_grep_search()\n```\n" }, { "path": "docs/metadata_filtering.md", "content": "# LEANN Metadata Filtering Usage Guide\n\n## Overview\n\nLeann possesses metadata filtering capabilities that allow you to filter search results based on arbitrary metadata fields set during chunking. This feature enables use cases like spoiler-free book search, document filtering by date/type, code search by file type, and potentially much more.\n\n## Basic Usage\n\n### Adding Metadata to Your Documents\n\nWhen building your index, add metadata to each text chunk:\n\n```python\nfrom leann.api import LeannBuilder\n\nbuilder = LeannBuilder(\"hnsw\")\n\n# Add text with metadata\nbuilder.add_text(\n text=\"Chapter 1: Alice falls down the rabbit hole\",\n metadata={\n \"chapter\": 1,\n \"character\": \"Alice\",\n \"themes\": [\"adventure\", \"curiosity\"],\n \"word_count\": 150\n }\n)\n\nbuilder.build_index(\"alice_in_wonderland_index\")\n```\n\n### Searching with Metadata Filters\n\nUse the `metadata_filters` parameter in search calls:\n\n```python\nfrom leann.api import LeannSearcher\n\nsearcher = LeannSearcher(\"alice_in_wonderland_index\")\n\n# Search with filters\nresults = searcher.search(\n query=\"What happens to Alice?\",\n top_k=10,\n metadata_filters={\n \"chapter\": {\"<=\": 5}, # Only chapters 1-5\n \"spoiler_level\": {\"!=\": \"high\"} # No high spoilers\n }\n)\n```\n\n## Filter Syntax\n\n### Basic Structure\n\n```python\nmetadata_filters = {\n \"field_name\": {\"operator\": value},\n \"another_field\": {\"operator\": value}\n}\n```\n\n### Supported Operators\n\n#### Comparison Operators\n- `\"==\"`: Equal to\n- `\"!=\"`: Not equal to\n- `\"<\"`: Less than\n- `\"<=\"`: Less than or equal\n- `\">\"`: Greater than\n- `\">=\"`: Greater than or equal\n\n```python\n# Examples\n{\"chapter\": {\"==\": 1}} # Exactly chapter 1\n{\"page\": {\">\": 100}} # Pages after 100\n{\"rating\": {\">=\": 4.0}} # Rating 4.0 or higher\n{\"word_count\": {\"<\": 500}} # Short passages\n```\n\n#### Membership Operators\n- `\"in\"`: Value is in list\n- `\"not_in\"`: Value is not in list\n\n```python\n# Examples\n{\"character\": {\"in\": [\"Alice\", \"Bob\"]}} # Alice OR Bob\n{\"genre\": {\"not_in\": [\"horror\", \"thriller\"]}} # Exclude genres\n{\"tags\": {\"in\": [\"fiction\", \"adventure\"]}} # Any of these tags\n```\n\n#### String Operators\n- `\"contains\"`: String contains substring\n- `\"starts_with\"`: String starts with prefix\n- `\"ends_with\"`: String ends with suffix\n\n```python\n# Examples\n{\"title\": {\"contains\": \"alice\"}} # Title contains \"alice\"\n{\"filename\": {\"ends_with\": \".py\"}} # Python files\n{\"author\": {\"starts_with\": \"Dr.\"}} # Authors with \"Dr.\" prefix\n```\n\n#### Boolean Operators\n- `\"is_true\"`: Field is truthy\n- `\"is_false\"`: Field is falsy\n\n```python\n# Examples\n{\"is_published\": {\"is_true\": True}} # Published content\n{\"is_draft\": {\"is_false\": False}} # Not drafts\n```\n\n### Multiple Operators on Same Field\n\nYou can apply multiple operators to the same field (AND logic):\n\n```python\nmetadata_filters = {\n \"word_count\": {\n \">=\": 100, # At least 100 words\n \"<=\": 500 # At most 500 words\n }\n}\n```\n\n### Compound Filters\n\nMultiple fields are combined with AND logic:\n\n```python\nmetadata_filters = {\n \"chapter\": {\"<=\": 10}, # Up to chapter 10\n \"character\": {\"==\": \"Alice\"}, # About Alice\n \"spoiler_level\": {\"!=\": \"high\"} # No major spoilers\n}\n```\n\n## Use Case Examples\n\n### 1. Spoiler-Free Book Search\n\n```python\n# Reader has only read up to chapter 5\ndef search_spoiler_free(query, max_chapter):\n return searcher.search(\n query=query,\n metadata_filters={\n \"chapter\": {\"<=\": max_chapter},\n \"spoiler_level\": {\"in\": [\"none\", \"low\"]}\n }\n )\n\nresults = search_spoiler_free(\"What happens to Alice?\", max_chapter=5)\n```\n\n### 2. Document Management by Date\n\n```python\n# Find recent documents\nrecent_docs = searcher.search(\n query=\"project updates\",\n metadata_filters={\n \"date\": {\">=\": \"2024-01-01\"},\n \"document_type\": {\"==\": \"report\"}\n }\n)\n```\n\n### 3. Code Search by File Type\n\n```python\n# Search only Python files\npython_code = searcher.search(\n query=\"authentication function\",\n metadata_filters={\n \"file_extension\": {\"==\": \".py\"},\n \"lines_of_code\": {\"<\": 100}\n }\n)\n```\n\n### 4. Content Filtering by Audience\n\n```python\n# Age-appropriate content\nfamily_content = searcher.search(\n query=\"adventure stories\",\n metadata_filters={\n \"age_rating\": {\"in\": [\"G\", \"PG\"]},\n \"content_warnings\": {\"not_in\": [\"violence\", \"adult_themes\"]}\n }\n)\n```\n\n### 5. Multi-Book Series Management\n\n```python\n# Search across first 3 books only\nearly_series = searcher.search(\n query=\"character development\",\n metadata_filters={\n \"series\": {\"==\": \"Harry Potter\"},\n \"book_number\": {\"<=\": 3}\n }\n)\n```\n\n## Running the Example\n\nYou can see metadata filtering in action with our spoiler-free book RAG example:\n\n```bash\n# Don't forget to set up the environment\nuv venv\nsource .venv/bin/activate\n\n# Set your OpenAI API key (required for embeddings, but you can update the example locally and use ollama instead)\nexport OPENAI_API_KEY=\"your-api-key-here\"\n\n# Run the spoiler-free book RAG example\nuv run examples/spoiler_free_book_rag.py\n```\n\nThis example demonstrates:\n- Building an index with metadata (chapter numbers, characters, themes, locations)\n- Searching with filters to avoid spoilers (e.g., only show results up to chapter 5)\n- Different scenarios for readers at various points in the book\n\nThe example uses Alice's Adventures in Wonderland as sample data and shows how you can search for information without revealing plot points from later chapters.\n\n## Advanced Patterns\n\n### Custom Chunking with metadata\n\n```python\ndef chunk_book_with_metadata(book_text, book_info):\n chunks = []\n\n for chapter_num, chapter_text in parse_chapters(book_text):\n # Extract entities, themes, etc.\n characters = extract_characters(chapter_text)\n themes = classify_themes(chapter_text)\n spoiler_level = assess_spoiler_level(chapter_text, chapter_num)\n\n # Create chunks with rich metadata\n for paragraph in split_paragraphs(chapter_text):\n chunks.append({\n \"text\": paragraph,\n \"metadata\": {\n \"book_title\": book_info[\"title\"],\n \"chapter\": chapter_num,\n \"characters\": characters,\n \"themes\": themes,\n \"spoiler_level\": spoiler_level,\n \"word_count\": len(paragraph.split()),\n \"reading_level\": calculate_reading_level(paragraph)\n }\n })\n\n return chunks\n```\n\n## Performance Considerations\n\n### Efficient Filtering Strategies\n\n1. **Post-search filtering**: Applies filters after vector search, which should be efficient for typical result sets (10-100 results).\n\n2. **Metadata design**: Keep metadata fields simple and avoid deeply nested structures.\n\n### Best Practices\n\n1. **Consistent metadata schema**: Use consistent field names and value types across your documents.\n\n2. **Reasonable metadata size**: Keep metadata reasonably sized to avoid storage overhead.\n\n3. **Type consistency**: Use consistent data types for the same fields (e.g., always integers for chapter numbers).\n\n4. **Index multiple granularities**: Consider chunking at different levels (paragraph, section, chapter) with appropriate metadata.\n\n### Adding Metadata to Existing Indices\n\nTo add metadata filtering to existing indices, you'll need to rebuild them with metadata:\n\n```python\n# Read existing passages and add metadata\ndef add_metadata_to_existing_chunks(chunks):\n for chunk in chunks:\n # Extract or assign metadata based on content\n chunk[\"metadata\"] = extract_metadata(chunk[\"text\"])\n return chunks\n\n# Rebuild index with metadata\nenhanced_chunks = add_metadata_to_existing_chunks(existing_chunks)\nbuilder = LeannBuilder(\"hnsw\")\nfor chunk in enhanced_chunks:\n builder.add_text(chunk[\"text\"], chunk[\"metadata\"])\nbuilder.build_index(\"enhanced_index\")\n```\n" }, { "path": "docs/normalized_embeddings.md", "content": "# Normalized Embeddings Support in LEANN\n\nLEANN now automatically detects normalized embedding models and sets the appropriate distance metric for optimal performance.\n\n## What are Normalized Embeddings?\n\nNormalized embeddings are vectors with L2 norm = 1 (unit vectors). These embeddings are optimized for cosine similarity rather than Maximum Inner Product Search (MIPS).\n\n## Automatic Detection\n\nWhen you create a `LeannBuilder` instance with a normalized embedding model, LEANN will:\n\n1. **Automatically set `distance_metric=\"cosine\"`** if not specified\n2. **Show a warning** if you manually specify a different distance metric\n3. **Provide optimal search performance** with the correct metric\n\n## Supported Normalized Embedding Models\n\n### OpenAI\nAll OpenAI text embedding models are normalized:\n- `text-embedding-ada-002`\n- `text-embedding-3-small`\n- `text-embedding-3-large`\n\n### Voyage AI\nAll Voyage AI embedding models are normalized:\n- `voyage-2`\n- `voyage-3`\n- `voyage-large-2`\n- `voyage-multilingual-2`\n- `voyage-code-2`\n\n### Cohere\nAll Cohere embedding models are normalized:\n- `embed-english-v3.0`\n- `embed-multilingual-v3.0`\n- `embed-english-light-v3.0`\n- `embed-multilingual-light-v3.0`\n\n## Example Usage\n\n```python\nfrom leann.api import LeannBuilder\n\n# Automatic detection - will use cosine distance\nbuilder = LeannBuilder(\n backend_name=\"hnsw\",\n embedding_model=\"text-embedding-3-small\",\n embedding_mode=\"openai\"\n)\n# Warning: Detected normalized embeddings model 'text-embedding-3-small'...\n# Automatically setting distance_metric='cosine'\n\n# Manual override (not recommended)\nbuilder = LeannBuilder(\n backend_name=\"hnsw\",\n embedding_model=\"text-embedding-3-small\",\n embedding_mode=\"openai\",\n distance_metric=\"mips\" # Will show warning\n)\n# Warning: Using 'mips' distance metric with normalized embeddings...\n```\n\n## Non-Normalized Embeddings\n\nModels like `facebook/contriever` and other sentence-transformers models that are not normalized will continue to use MIPS by default, which is optimal for them.\n\n## Why This Matters\n\nUsing the wrong distance metric with normalized embeddings can lead to:\n- **Poor search quality** due to HNSW's early termination with narrow score ranges\n- **Incorrect ranking** of search results\n- **Suboptimal performance** compared to using the correct metric\n\nFor more details on why this happens, see our analysis in the [embedding detection code](../packages/leann-core/src/leann/api.py) which automatically handles normalized embeddings and MIPS distance metric issues.\n" }, { "path": "docs/openclaw-setup.md", "content": "# OpenClaw + LEANN Setup Guide\n\nTwo ways to connect LEANN to your OpenClaw agent: **MCP server** (recommended)\nor **ClawHub skill**.\n\n---\n\n## Option A: MCP Server (Recommended)\n\nOpenClaw natively supports MCP tools. LEANN ships an MCP server that exposes\n`leann_search` and `leann_list` as tools your agent can call directly.\n\n### 1. Install LEANN\n\n```bash\npip install leann-core\n# or\nuv tool install leann-core --with leann\n```\n\n### 2. Build an index on your memory files\n\nUsing Ollama embeddings (recommended if you already run Ollama):\n\n```bash\nleann build openclaw-memory \\\n --docs ~/.openclaw/workspace/MEMORY.md ~/.openclaw/workspace/memory/ \\\n --embedding-mode ollama \\\n --embedding-model nomic-embed-text\n```\n\nOr using local sentence-transformers (no Ollama required):\n\n```bash\nleann build openclaw-memory \\\n --docs ~/.openclaw/workspace/MEMORY.md ~/.openclaw/workspace/memory/ \\\n --embedding-mode sentence-transformers \\\n --embedding-model all-MiniLM-L6-v2\n```\n\nAdd extra directories if you have them:\n\n```bash\nleann build openclaw-memory \\\n --docs ~/.openclaw/workspace/MEMORY.md \\\n ~/.openclaw/workspace/memory/ \\\n ~/Documents/notes/ \\\n --embedding-mode ollama \\\n --embedding-model nomic-embed-text\n```\n\n### 3. Register the MCP server with OpenClaw\n\nAdd to `~/.openclaw/openclaw.json`:\n\n```json5\n{\n // ... your existing config ...\n \"mcpServers\": {\n \"leann\": {\n \"command\": \"leann_mcp\",\n \"args\": [],\n \"env\": {}\n }\n }\n}\n```\n\n### 4. Use it\n\nAsk your agent:\n- \"Search my memories for database decisions\"\n- \"What did we decide about the API design?\"\n- \"Find my notes on deployment\"\n\nThe agent will call `leann_search` via MCP and return structured results.\n\n### 5. Keep the index fresh\n\n```bash\n# Re-run build (idempotent — only processes changed files)\nleann build openclaw-memory \\\n --docs ~/.openclaw/workspace/MEMORY.md ~/.openclaw/workspace/memory/\n\n# Or use watch mode for continuous auto-sync\nleann watch openclaw-memory --interval 30\n```\n\n---\n\n## Option B: ClawHub Skill\n\nIf you prefer the skill-based approach:\n\n```bash\nclawhub install leann-team/leann-memory\n```\n\nOr copy `skills/leann-memory/` from this repo to\n`~/.openclaw/workspace/skills/leann-memory/`.\n\nThe skill tells your agent how to call `leann search` via shell commands.\nSetup steps (install + build index) are the same as above.\n\n---\n\n## Important: Ollama Configuration\n\nIf you use Ollama as your OpenClaw model provider, make sure your\n`~/.openclaw/openclaw.json` uses the **native Ollama API** — not the\nOpenAI-compatible endpoint:\n\n```json5\n{\n \"models\": {\n \"providers\": {\n \"ollama\": {\n \"baseUrl\": \"http://127.0.0.1:11434\", // no /v1 suffix\n \"apiKey\": \"ollama-local\",\n \"api\": \"ollama\" // NOT \"openai-completions\" or \"openai-responses\"\n }\n }\n }\n}\n```\n\nUsing `\"openai-completions\"` or `\"openai-responses\"` silently breaks tool\ncalling — the model outputs tool calls as plain text instead of structured\n`tool_calls`. See [astral-sh/ty#21243](https://github.com/openclaw/openclaw/issues/21243).\n\n---\n\n## Storage Comparison\n\n| Scenario | Default memory-core | LEANN |\n|---|---|---|\n| 1 year daily logs (~12K chunks) | ~23 MB | **~0.7 MB** |\n| + session transcripts (~100K chunks) | ~190 MB | **~6 MB** |\n| + 10 GB indexed documents (~500K chunks) | ~950 MB | **~30 MB** |\n\nAll numbers assume 384-dimensional embeddings (all-MiniLM-L6-v2 or\nnomic-embed-text).\n\n---\n\n## Troubleshooting\n\n**\"leann: command not found\"**\nEnsure LEANN is on your PATH. If installed via `uv tool install`, run\n`uv tool update-shell` and restart your terminal.\n\n**\"Index not found\"**\nRun `leann list` to see available indexes. Build one first with `leann build`.\n\n**Slow first search**\nThe first query loads the embedding model (~90 MB). Subsequent queries reuse the\nwarm daemon and are fast (~0.5s). Use `leann warmup openclaw-memory` to\npre-warm.\n\n**Memory files changed but search results are stale**\nRe-run `leann build openclaw-memory --docs ...` — it detects changes\nautomatically and only re-indexes what changed.\n\n**Agent doesn't use LEANN tools**\nMake sure your Ollama model supports tool calling (e.g. `qwen3:8b` or larger).\nSmaller models like `qwen3:4b` may not reliably invoke tools.\n" }, { "path": "docs/react_agent.md", "content": "# LEANN ReAct Agent Guide\n\n## Overview\n\nThe LEANN ReAct (Reasoning + Acting) Agent enables **multiturn retrieval and reasoning** for complex queries that require multiple search iterations. Unlike the standard `leann ask` command which performs a single search and answer, the ReAct agent can:\n\n- **Reason** about what information is needed\n- **Act** by performing targeted searches\n- **Observe** the results and iterate\n- **Answer** based on all gathered context\n\nThis is particularly useful for questions that require:\n- Multiple pieces of information from different parts of your index\n- Iterative refinement of search queries\n- Complex reasoning that builds on previous findings\n\n## How It Works\n\nThe ReAct agent follows a **Thought → Action → Observation** loop:\n\n1. **Thought**: The agent analyzes the question and determines what information is needed\n2. **Action**: The agent performs a search query based on its reasoning\n3. **Observation**: The agent reviews the search results\n4. **Iteration**: The process repeats until the agent has enough information or reaches the maximum iteration limit\n5. **Final Answer**: The agent synthesizes all gathered information into a comprehensive answer\n\n## Basic Usage\n\n### Command Line\n\n```bash\n# Basic usage\nleann react \"your question\"\n\n# With custom LLM settings\nleann react my-index \"What are the main features discussed?\" \\\n --llm ollama \\\n --model qwen3:8b \\\n --max-iterations 5 \\\n --top-k 5\n```\n\n### Command Options\n\n- `index_name`: Name of the LEANN index to search\n- `query`: The question to research\n- `--llm`: LLM provider (`ollama`, `openai`, `anthropic`, `hf`, `simulated`) - default: `ollama`\n- `--model`: Model name (default: `qwen3:8b`)\n- `--host`: Override Ollama-compatible host (defaults to `LEANN_OLLAMA_HOST` or `OLLAMA_HOST`)\n- `--top-k`: Number of results per search iteration (default: `5`)\n- `--max-iterations`: Maximum number of search iterations (default: `5`)\n- `--api-base`: Base URL for OpenAI-compatible APIs\n- `--api-key`: API key for cloud LLM providers\n\n### Python API\n\n```python\nfrom leann import create_react_agent, LeannSearcher\n\n# Create a searcher\nsearcher = LeannSearcher(index_path=\"path/to/index.leann\")\n\n# Create the ReAct agent\nagent = create_react_agent(\n index_path=\"path/to/index.leann\",\n llm_config={\n \"type\": \"ollama\",\n \"model\": \"qwen3:8b\",\n \"host\": \"http://localhost:11434\" # optional\n },\n max_iterations=5\n)\n\n# Run the agent\nanswer = agent.run(\"What are the main topics covered in the documentation?\", top_k=5)\nprint(answer)\n\n# Access search history\nif agent.search_history:\n print(f\"\\nSearch History ({len(agent.search_history)} iterations):\")\n for entry in agent.search_history:\n print(f\" {entry['iteration']}. {entry['action']} ({entry['results_count']} results)\")\n```\n\n## Example Use Cases\n\n### 1. Multi-faceted Questions\n\n```bash\n# Questions that need information from multiple sources\nleann react docs-index \"What are the differences between HNSW and DiskANN backends, and when should I use each?\"\n```\n\nThe agent will:\n- First search for \"HNSW backend features\"\n- Then search for \"DiskANN backend features\"\n- Compare the results\n- Provide a comprehensive answer\n\n### 2. Iterative Research\n\n```bash\n# Questions requiring multiple search iterations\nleann react codebase-index \"How does the embedding computation work and what optimizations are used?\"\n```\n\nThe agent will:\n- Search for \"embedding computation\"\n- Based on results, search for \"embedding optimizations\"\n- Refine queries based on findings\n- Synthesize the information\n\n### 3. Complex Reasoning\n\n```bash\n# Questions that require building understanding\nleann react research-index \"What are the performance characteristics of different indexing strategies?\"\n```\n\n## Comparison: `leann ask` vs `leann react`\n\n| Feature | `leann ask` | `leann react` |\n|---------|-------------|---------------|\n| **Search iterations** | Single search | Multiple iterations |\n| **Query refinement** | No | Yes, based on observations |\n| **Use case** | Simple Q&A | Complex, multi-faceted questions |\n| **Speed** | Faster | Slower (multiple searches) |\n| **Reasoning** | Direct answer | Iterative reasoning |\n\n### When to Use Each\n\n**Use `leann ask` when:**\n- You have a straightforward question\n- A single search should provide enough context\n- You want a quick answer\n\n**Use `leann react` when:**\n- Your question requires information from multiple sources\n- You need the agent to explore and refine its understanding\n- The answer requires synthesizing multiple pieces of information\n\n## Advanced Configuration\n\n### Custom LLM Providers\n\n```bash\n# Using OpenAI\nleann react my-index \"question\" \\\n --llm openai \\\n --model gpt-4 \\\n --api-base https://api.openai.com/v1 \\\n --api-key $OPENAI_API_KEY\n\n# Using Anthropic\nleann react my-index \"question\" \\\n --llm anthropic \\\n --model claude-3-opus-20240229 \\\n --api-key $ANTHROPIC_API_KEY\n```\n\n### Adjusting Search Parameters\n\n```bash\n# More results per iteration\nleann react my-index \"question\" --top-k 10\n\n# More iterations for complex questions\nleann react my-index \"question\" --max-iterations 10\n```\n\n## Understanding the Output\n\nWhen you run `leann react`, you'll see:\n\n1. **Question**: The original question being researched\n2. **Iteration logs**: Each search action and its results\n3. **Final Answer**: The synthesized answer based on all iterations\n4. **Search History**: Summary of all search iterations performed\n\nExample output:\n\n```\n🤖 Starting ReAct agent with index 'my-index'...\nUsing qwen3:8b (ollama)\n\n🔍 Question: What are the main features of LEANN?\n\n🔍 Action: search(\"LEANN features\")\n[Result 1] (Score: 0.923)\nLEANN is a vector database that saves 97% storage...\n\n🔍 Action: search(\"LEANN storage optimization\")\n[Result 1] (Score: 0.891)\nLEANN uses compact storage and recomputation...\n\n✅ Final Answer:\nLEANN is a vector database with several key features:\n1. 97% storage savings compared to traditional vector databases\n2. Compact storage with recomputation capabilities\n3. Support for multiple backends (HNSW and DiskANN)\n...\n\n📊 Search History (2 iterations):\n 1. search(\"LEANN features\") (5 results)\n 2. search(\"LEANN storage optimization\") (5 results)\n```\n\n## Tips for Best Results\n\n1. **Be specific**: Clear, specific questions work better than vague ones\n2. **Adjust iterations**: Complex questions may need more iterations (increase `--max-iterations`)\n3. **Monitor history**: Check the search history to understand the agent's reasoning\n4. **Use appropriate models**: Larger models generally provide better reasoning, but are slower\n5. **Index quality**: Ensure your index is well-built with relevant content\n\n## Limitations\n\n- **Speed**: Multiple iterations make ReAct slower than single-search queries\n- **Cost**: More LLM calls mean higher costs for cloud providers\n- **Complexity**: Very complex questions may still require human review\n- **Model dependency**: Reasoning quality depends on the LLM's capabilities\n\n## Future Enhancements\n\nThis is the first implementation (1/N) of Deep-Research integration. Future enhancements may include:\n- Web search integration for external information\n- More sophisticated reasoning strategies\n- Parallel search execution\n- Better query optimization\n\n## Related Documentation\n\n- [Basic Usage Guide](../README.md)\n- [CLI Reference](configuration-guide.md)\n- [Embedding Models](normalized_embeddings.md)\n" }, { "path": "docs/roadmap.md", "content": "# LEANN Roadmap\n\nLEANN aims to be a **personal knowledge layer** — not just a storage-efficient vector database, but a unified, always-up-to-date knowledge base that runs entirely on your own machine. It connects your code, images, and personal data (documents, emails, browser history, chats) into a single multimodal search interface.\n\nContributions and feedback are welcome. Join our [Slack](https://join.slack.com/t/leann-e2u9779/shared_invite/zt-3ol2ww9ic-Eg_kB8omwe6xmYVd0epr4Q) to discuss.\n\n---\n\n## Completed\n\n- [x] HNSW backend integration\n- [x] DiskANN backend with MIPS/L2/Cosine support\n- [x] Real-time embedding pipeline\n- [x] Memory-efficient graph pruning\n- [x] IVF backend with incremental add/remove (#231, #89, #141)\n- [x] Merkle tree file-change detection — `leann watch` (#41)\n\n---\n\n## P0 — Core (Q1 2026)\n\n### LEANN MCP — The Best Code Retrieval MCP\n\nThe primary near-term goal: make LEANN the go-to MCP server for code-aware AI assistants. This means **dynamic updates** (your index stays current as you edit code), **rich code context** (AST-aware chunking that understands functions, classes, and modules — not just raw text), and a **dead-simple interface** (one command to build, automatic incremental updates, zero configuration for common setups).\n\n- [x] IVF backend — incremental add/remove without full rebuild (#231, #89, #141)\n- [x] Merkle tree file-change detection — `leann watch` for automatic re-indexing on file changes (#41)\n- [ ] Cold start optimization — faster first-build experience for new users (#166, #177)\n- [ ] Live index updates — push index changes as files are saved, not just on rebuild\n- [ ] Smarter code context — cross-file symbol resolution, call graph awareness, import tracking\n\n### Search Quality\n\n- [ ] Hybrid search — combine dense vector retrieval with sparse keyword matching (BM25) for better recall on exact identifiers and variable names (#233, #90)\n\n### Documentation\n\n- [ ] ReadTheDocs — hosted documentation site (#234)\n- [ ] Benchmarks — recall@k, latency, and storage comparisons across backends\n\n---\n\n## P1 (Q2 2026)\n\n### Multimodal\n\n- [ ] Video retrieval (#160)\n- [ ] CLIP support — image-text cross-modal search (#94)\n- [ ] OCR — extract text from images/scanned documents (#158)\n\n### Platform & Distribution\n\n- [ ] Windows support (#14)\n- [ ] Web UI (#229)\n\n### Applications & Integrations\n\n- [ ] Agent + Deep research (#104)\n- [ ] Local Cursor — local model + local retrieval for code assistance (#47)\n- [ ] LlamaIndex integration (#217)\n- [ ] Obsidian support (#96)\n\n---\n\n## Contributing\n\nIf you're interested in working on any of the items above, please reach out to [@yichuan-w](https://github.com/yichuan-w) or [@andylizf](https://github.com/andylizf). See [CONTRIBUTING.md](CONTRIBUTING.md) for the full contributor workflow.\n" }, { "path": "docs/slack-setup-guide.md", "content": "# Slack Integration Setup Guide\n\nThis guide provides step-by-step instructions for setting up Slack integration with LEANN.\n\n## Overview\n\nLEANN's Slack integration uses MCP (Model Context Protocol) servers to fetch and index your Slack messages for RAG (Retrieval-Augmented Generation). This allows you to search through your Slack conversations using natural language queries.\n\n## Prerequisites\n\n1. **Slack Workspace Access**: You need admin or owner permissions in your Slack workspace to create apps and configure OAuth tokens.\n\n2. **Slack MCP Server**: Install a Slack MCP server (e.g., `slack-mcp-server` via npm)\n\n3. **LEANN**: Ensure you have LEANN installed and working\n\n## Step 1: Create a Slack App\n\n### 1.1 Go to Slack API Dashboard\n\n1. Visit [https://api.slack.com/apps](https://api.slack.com/apps)\n2. Click **\"Create New App\"**\n3. Choose **\"From scratch\"**\n4. Enter your app name (e.g., \"LEANN Slack Integration\")\n5. Select your workspace\n6. Click **\"Create App\"**\n\n### 1.2 Configure App Permissions\n\n#### Token Scopes\n\n1. In your app dashboard, go to **\"OAuth & Permissions\"** in the left sidebar\n2. Scroll down to **\"Scopes\"** section\n3. Under **\"Bot Token Scopes & OAuth Scope\"**, click **\"Add an OAuth Scope\"**\n4. Add the following scopes:\n - `channels:read` - Read public channel information\n - `channels:history` - Read messages in public channels\n - `groups:read` - Read private channel information\n - `groups:history` - Read messages in private channels\n - `im:read` - Read direct message information\n - `im:history` - Read direct messages\n - `mpim:read` - Read group direct message information\n - `mpim:history` - Read group direct messages\n - `users:read` - Read user information\n - `team:read` - Read workspace information\n\n#### App-Level Tokens (Optional)\n\nSome MCP servers may require app-level tokens:\n\n1. Go to **\"Basic Information\"** in the left sidebar\n2. Scroll down to **\"App-Level Tokens\"**\n3. Click **\"Generate Token and Scopes\"**\n4. Enter a name (e.g., \"LEANN Integration\")\n5. Add the `connections:write` scope\n6. Click **\"Generate\"**\n7. Copy the token (starts with `xapp-`)\n\n### 1.3 Install App to Workspace\n\n1. Go to **\"OAuth & Permissions\"** in the left sidebar\n2. Click **\"Install to Workspace\"**\n3. Review the permissions and click **\"Allow\"**\n4. Copy the **\"Bot User OAuth Token\"** (starts with `xoxb-`)\n5. Copy the **\"User OAuth Token\"** (starts with `xoxp-`)\n\n## Step 2: Install Slack MCP Server\n\n### Option A: Using npm (Recommended)\n\n```bash\n# Install globally\nnpm install -g slack-mcp-server\n\n# Or install locally\nnpm install slack-mcp-server\n```\n\n### Option B: Using npx (No installation required)\n\n```bash\n# Use directly without installation\nnpx slack-mcp-server\n```\n\n## Step 3: Install and Configure Ollama (for Real LLM Responses)\n\n### 3.1 Install Ollama\n\n```bash\n# Install Ollama using Homebrew (macOS)\nbrew install ollama\n\n# Or download from https://ollama.ai/\n```\n\n### 3.2 Start Ollama Service\n\n```bash\n# Start Ollama as a service\nbrew services start ollama\n\n# Or start manually\nollama serve\n```\n\n### 3.3 Pull a Model\n\n```bash\n# Pull a lightweight model for testing\nollama pull llama3.2:1b\n\n# Verify the model is available\nollama list\n```\n\n## Step 4: Configure Environment Variables\n\nCreate a `.env` file or set environment variables:\n\n```bash\n# Required: User OAuth Token\nSLACK_OAUTH_TOKEN=xoxp-your-user-oauth-token-here\n\n# Optional: App-Level Token (if your MCP server requires it)\nSLACK_APP_TOKEN=xapp-your-app-token-here\n\n# Optional: Workspace-specific settings\nSLACK_WORKSPACE_ID=T1234567890 # Your workspace ID (optional)\n```\n\n## Step 5: Test the Setup\n\n### 5.1 Test MCP Server Connection\n\n```bash\npython -m apps.slack_rag \\\n --mcp-server \"slack-mcp-server\" \\\n --test-connection \\\n --workspace-name \"Your Workspace Name\"\n```\n\nThis will test the connection and list available tools without indexing any data.\n\n### 5.2 Index a Specific Channel\n\n```bash\npython -m apps.slack_rag \\\n --mcp-server \"slack-mcp-server\" \\\n --workspace-name \"Your Workspace Name\" \\\n --channels general \\\n --query \"What did we discuss about the project?\"\n```\n\n### 5.3 Real RAG Query Examples\n\nThis section demonstrates successful Slack RAG integration queries against the Sky Lab Computing workspace's \"random\" channel. The system successfully retrieves actual conversation messages and performs semantic search with high relevance scores, including finding specific research paper announcements and technical discussions.\n\n### Example 1: Advisor Models Query\n\n**Query:** \"train black-box models to adopt to your personal data\"\n\nThis query demonstrates the system's ability to find specific research announcements about training black-box models for personal data adaptation.\n\n![Advisor Models Query - Command Setup](videos/slack_integration_1.1.png)\n\n![Advisor Models Query - Search Results](videos/slack_integration_1.2.png)\n\n![Advisor Models Query - LLM Response](videos/slack_integration_1.3.png)\n\n### Example 2: Barbarians at the Gate Query\n\n**Query:** \"AI-driven research systems ADRS\"\n\nThis query demonstrates the system's ability to find specific research announcements about AI-driven research systems and algorithm discovery.\n\n![Barbarians Query - Command Setup](videos/slack_integration_2.1.png)\n\n![Barbarians Query - Search Results](videos/slack_integration_2.2.png)\n\n![Barbarians Query - LLM Response](videos/slack_integration_2.3.png)\n\n### Prerequisites\n\n- Bot is installed in the Sky Lab Computing workspace and invited to the target channel (run `/invite @YourBotName` in the channel if needed)\n- Bot token available and exported in the same terminal session\n\n### Commands\n\n1) Set the workspace token for this shell\n\n```bash\nexport SLACK_MCP_XOXP_TOKEN=\"xoxp-***-redacted-***\"\n```\n\n2) Run queries against the \"random\" channel by channel ID (C0GN5BX0F)\n\n**Advisor Models Query:**\n```bash\npython -m apps.slack_rag \\\n --mcp-server \"slack-mcp-server\" \\\n --workspace-name \"Sky Lab Computing\" \\\n --channels C0GN5BX0F \\\n --max-messages-per-channel 100000 \\\n --query \"train black-box models to adopt to your personal data\" \\\n --llm ollama \\\n --llm-model \"llama3.2:1b\" \\\n --llm-host \"http://localhost:11434\" \\\n --no-concatenate-conversations\n```\n\n**Barbarians at the Gate Query:**\n```bash\npython -m apps.slack_rag \\\n --mcp-server \"slack-mcp-server\" \\\n --workspace-name \"Sky Lab Computing\" \\\n --channels C0GN5BX0F \\\n --max-messages-per-channel 100000 \\\n --query \"AI-driven research systems ADRS\" \\\n --llm ollama \\\n --llm-model \"llama3.2:1b\" \\\n --llm-host \"http://localhost:11434\" \\\n --no-concatenate-conversations\n```\n\nThese examples demonstrate the system's ability to find and retrieve specific research announcements and technical discussions from the conversation history, showcasing the power of semantic search in Slack data.\n\n3) Optional: Ask a broader question\n\n```bash\npython test_channel_by_id_or_name.py \\\n --channel-id C0GN5BX0F \\\n --workspace-name \"Sky Lab Computing\" \\\n --query \"What is LEANN about?\"\n```\n\nNotes:\n- If you see `not_in_channel`, invite the bot to the channel and re-run.\n- If you see `channel_not_found`, confirm the channel ID and workspace.\n- Deep search via server-side “search” tools may require additional Slack scopes; the example above performs client-side filtering over retrieved history.\n\n## Common Issues and Solutions\n\n### Issue 1: \"users cache is not ready yet\" Error\n\n**Problem**: You see this warning:\n```\nWARNING - Failed to fetch messages from channel random: Failed to fetch messages: {'code': -32603, 'message': 'users cache is not ready yet, sync process is still running... please wait'}\n```\n\n**Solution**: This is a common timing issue. The LEANN integration now includes automatic retry logic:\n\n1. **Wait and Retry**: The system will automatically retry with exponential backoff (2s, 4s, 8s, etc.)\n2. **Increase Retry Parameters**: If needed, you can customize retry behavior:\n ```bash\n python -m apps.slack_rag \\\n --mcp-server \"slack-mcp-server\" \\\n --max-retries 10 \\\n --retry-delay 3.0 \\\n --channels general \\\n --query \"Your query here\"\n ```\n3. **Keep MCP Server Running**: Start the MCP server separately and keep it running:\n ```bash\n # Terminal 1: Start MCP server\n slack-mcp-server\n\n # Terminal 2: Run LEANN (it will connect to the running server)\n python -m apps.slack_rag --mcp-server \"slack-mcp-server\" --channels general --query \"test\"\n ```\n\n### Issue 2: \"No message fetching tool found\"\n\n**Problem**: The MCP server doesn't have the expected tools.\n\n**Solution**:\n1. Check if your MCP server is properly installed and configured\n2. Verify your Slack tokens are correct\n3. Try a different MCP server implementation\n4. Check the MCP server documentation for required configuration\n\n### Issue 3: Permission Denied Errors\n\n**Problem**: You get permission errors when trying to access channels.\n\n**Solutions**:\n1. **Check Bot Permissions**: Ensure your bot has been added to the channels you want to access\n2. **Verify Token Scopes**: Make sure you have all required scopes configured\n3. **Channel Access**: For private channels, the bot needs to be explicitly invited\n4. **Workspace Permissions**: Ensure your Slack app has the necessary workspace permissions\n\n### Issue 4: Empty Results\n\n**Problem**: No messages are returned even though the channel has messages.\n\n**Solutions**:\n1. **Check Channel Names**: Ensure channel names are correct (without the # symbol)\n2. **Verify Bot Access**: Make sure the bot can access the channels\n3. **Check Date Ranges**: Some MCP servers have limitations on message history\n4. **Increase Message Limits**: Try increasing the message limit:\n ```bash\n python -m apps.slack_rag \\\n --mcp-server \"slack-mcp-server\" \\\n --channels general \\\n --max-messages-per-channel 1000 \\\n --query \"test\"\n ```\n\n## Advanced Configuration\n\n### Custom MCP Server Commands\n\nIf you need to pass additional parameters to your MCP server:\n\n```bash\npython -m apps.slack_rag \\\n --mcp-server \"slack-mcp-server --token-file /path/to/tokens.json\" \\\n --workspace-name \"Your Workspace\" \\\n --channels general \\\n --query \"Your query\"\n```\n\n### Multiple Workspaces\n\nTo work with multiple Slack workspaces, you can:\n\n1. Create separate apps for each workspace\n2. Use different environment variables\n3. Run separate instances with different configurations\n\n### Performance Optimization\n\nFor better performance with large workspaces:\n\n```bash\npython -m apps.slack_rag \\\n --mcp-server \"slack-mcp-server\" \\\n --workspace-name \"Your Workspace\" \\\n --max-messages-per-channel 500 \\\n --no-concatenate-conversations \\\n --query \"Your query\"\n```\n---\n\n## Troubleshooting Checklist\n\n- [ ] Slack app created with proper permissions\n- [ ] Bot token (xoxb-) copied correctly\n- [ ] App-level token (xapp-) created if needed\n- [ ] MCP server installed and accessible\n- [ ] Ollama installed and running (`brew services start ollama`)\n- [ ] Ollama model pulled (`ollama pull llama3.2:1b`)\n- [ ] Environment variables set correctly\n- [ ] Bot invited to relevant channels\n- [ ] Channel names specified without # symbol\n- [ ] Sufficient retry attempts configured\n- [ ] Network connectivity to Slack APIs\n\n## Getting Help\n\nIf you continue to have issues:\n\n1. **Check Logs**: Look for detailed error messages in the console output\n2. **Test MCP Server**: Use `--test-connection` to verify the MCP server is working\n3. **Verify Tokens**: Double-check that your Slack tokens are valid and have the right scopes\n4. **Check Ollama**: Ensure Ollama is running (`ollama serve`) and the model is available (`ollama list`)\n5. **Community Support**: Reach out to the LEANN community for help\n\n## Example Commands\n\n### Basic Usage\n```bash\n# Test connection\npython -m apps.slack_rag --mcp-server \"slack-mcp-server\" --test-connection\n\n# Index specific channels\npython -m apps.slack_rag \\\n --mcp-server \"slack-mcp-server\" \\\n --workspace-name \"My Company\" \\\n --channels general random \\\n --query \"What did we decide about the project timeline?\"\n```\n\n### Advanced Usage\n```bash\n# With custom retry settings\npython -m apps.slack_rag \\\n --mcp-server \"slack-mcp-server\" \\\n --workspace-name \"My Company\" \\\n --channels general \\\n --max-retries 10 \\\n --retry-delay 5.0 \\\n --max-messages-per-channel 2000 \\\n --query \"Show me all decisions made in the last month\"\n```\n" }, { "path": "docs/ultimate_goal.md", "content": "# LEANN — Long-Term Vision\n\n## The Best Personal Data Management Platform\n\nLEANN's ultimate goal is to be the **unified personal knowledge layer** that lives on your machine. Not a cloud service. Not a SaaS product. A local-first system that understands everything you've ever worked on — code, documents, emails, chats, browser history, images — and makes it all instantly searchable and usable.\n\n### 1. Continuous Learning from Your Context\n\nLEANN should get smarter over time by continuously ingesting and indexing your data as you produce it:\n\n- **Always-on indexing**: Watch your filesystem, email, browser, and chat sources. Incrementally update indexes as new data arrives — no manual rebuilds.\n- **Cross-source connections**: Surface relationships across data sources. A Slack conversation about a bug should link to the relevant code change, the related email thread, and the document that describes the feature.\n- **Temporal awareness**: Understand when things happened. \"What was I working on last Tuesday?\" should be a searchable query.\n- **Personalized ranking**: Learn from your search patterns and usage to rank results by relevance to *you*, not just semantic similarity.\n\n### 2. The Best MCP for Code Retrieval\n\nAI coding assistants are only as good as the context they receive. LEANN aims to be the best context provider:\n\n- **AST-aware chunking**: Understand code structure — functions, classes, modules — not just raw text blocks. Retrieve semantically meaningful units.\n- **Dynamic index updates**: The index stays current as you edit. No stale results. No manual rebuilds. Save a file, and the index reflects the change within seconds.\n- **Cross-file understanding**: Resolve symbols across files. When you search for a function, also surface its callers, its tests, and the types it depends on.\n- **Dead-simple interface**: One command to build (`leann build`), automatic incremental updates, MCP server that any AI assistant can plug into with zero configuration.\n- **Repository-scale search**: Handle monorepos and large codebases without breaking a sweat. The storage efficiency (97% reduction vs. traditional vector DBs) makes this feasible on a laptop.\n\n### 3. Multimodal Knowledge Base\n\nText is just the beginning:\n\n- **Image search**: CLIP-based retrieval for screenshots, diagrams, photos. \"Find the architecture diagram from last month.\"\n- **Video retrieval**: Index video content — lectures, meetings, screen recordings — and search by what was said or shown.\n- **OCR pipeline**: Extract and index text from scanned documents, handwritten notes, whiteboard photos.\n- **Unified search**: One query searches across all modalities. The answer might be in a PDF, a screenshot, a code comment, or a Slack message.\n\n### 4. Platform for Personal AI\n\nLooking further ahead, LEANN becomes the memory and retrieval layer for personal AI agents:\n\n- **Agent memory**: AI agents that remember your preferences, past decisions, and project context across sessions.\n- **Deep research**: Agents that can search your entire personal knowledge base to answer complex questions, synthesize information, and generate insights.\n- **Proactive suggestions**: Surface relevant context before you ask for it — \"You discussed this exact problem with a colleague 3 months ago, here's what you decided.\"\n" }, { "path": "examples/__init__.py", "content": "" }, { "path": "examples/basic_demo.py", "content": "\"\"\"\nSimple demo showing basic leann usage\nRun: uv run python examples/basic_demo.py\n\"\"\"\n\nimport argparse\n\nfrom leann import LeannBuilder, LeannChat, LeannSearcher\n\n\ndef main():\n parser = argparse.ArgumentParser(\n description=\"Simple demo of Leann with selectable embedding models.\"\n )\n parser.add_argument(\n \"--embedding_model\",\n type=str,\n default=\"sentence-transformers/all-mpnet-base-v2\",\n help=\"The embedding model to use, e.g., 'sentence-transformers/all-mpnet-base-v2' or 'text-embedding-ada-002'.\",\n )\n args = parser.parse_args()\n\n print(f\"=== Leann Simple Demo with {args.embedding_model} ===\")\n print()\n\n # Sample knowledge base\n chunks = [\n \"Machine learning is a subset of artificial intelligence that enables computers to learn without being explicitly programmed.\",\n \"Deep learning uses neural networks with multiple layers to process data and make decisions.\",\n \"Natural language processing helps computers understand and generate human language.\",\n \"Computer vision enables machines to interpret and understand visual information from images and videos.\",\n \"Reinforcement learning teaches agents to make decisions by receiving rewards or penalties for their actions.\",\n \"Data science combines statistics, programming, and domain expertise to extract insights from data.\",\n \"Big data refers to extremely large datasets that require special tools and techniques to process.\",\n \"Cloud computing provides on-demand access to computing resources over the internet.\",\n ]\n\n print(\"1. Building index (no embeddings stored)...\")\n builder = LeannBuilder(\n embedding_model=args.embedding_model,\n backend_name=\"hnsw\",\n )\n for chunk in chunks:\n builder.add_text(chunk)\n builder.build_index(\"demo_knowledge.leann\")\n print()\n\n print(\"2. Searching with real-time embeddings...\")\n searcher = LeannSearcher(\"demo_knowledge.leann\")\n\n queries = [\n \"What is machine learning?\",\n \"How does neural network work?\",\n \"Tell me about data processing\",\n ]\n\n for query in queries:\n print(f\"Query: {query}\")\n results = searcher.search(query, top_k=2)\n\n for i, result in enumerate(results, 1):\n print(f\" {i}. Score: {result.score:.3f}\")\n print(f\" Text: {result.text[:100]}...\")\n print()\n\n print(\"3. Interactive chat demo:\")\n print(\" (Note: Requires OpenAI API key for real responses)\")\n\n chat = LeannChat(\"demo_knowledge.leann\")\n\n # Demo questions\n demo_questions: list[str] = [\n \"What is the difference between machine learning and deep learning?\",\n \"How is data science related to big data?\",\n ]\n\n for question in demo_questions:\n print(f\" Q: {question}\")\n response = chat.ask(question)\n print(f\" A: {response}\")\n print()\n\n print(\"Demo completed! Try running:\")\n print(\" uv run python apps/document_rag.py\")\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "examples/dynamic_update_no_recompute.py", "content": "\"\"\"Dynamic HNSW update demo without compact storage.\n\nThis script reproduces the minimal scenario we used while debugging on-the-fly\nrecompute:\n\n1. Build a non-compact HNSW index from the first few paragraphs of a text file.\n2. Print the top results with `recompute_embeddings=True`.\n3. Append additional paragraphs with :meth:`LeannBuilder.update_index`.\n4. Run the same query again to show the newly inserted passages.\n\nRun it with ``uv`` (optionally pointing LEANN_HNSW_LOG_PATH at a file to inspect\nZMQ activity)::\n\n LEANN_HNSW_LOG_PATH=embedding_fetch.log \\\n uv run -m examples.dynamic_update_no_recompute \\\n --index-path .leann/examples/leann-demo.leann\n\nBy default the script builds an index from ``data/2501.14312v1 (1).pdf`` and\nthen updates it with LEANN-related material from ``data/2506.08276v1.pdf``.\nIt issues the query \"What's LEANN?\" before and after the update to show how the\nnew passages become immediately searchable. The script uses the\n``sentence-transformers/all-MiniLM-L6-v2`` model with ``is_recompute=True`` so\nFaiss pulls existing vectors on demand via the ZMQ embedding server, while\nfreshly added passages are embedded locally just like the initial build.\n\nTo make storage comparisons easy, the script can also build a matching\n``is_recompute=False`` baseline (enabled by default) and report the index size\ndelta after the update. Disable the baseline run with\n``--skip-compare-no-recompute`` if you only need the recompute flow.\n\"\"\"\n\nimport argparse\nimport json\nfrom collections.abc import Iterable\nfrom pathlib import Path\nfrom typing import Any\n\nfrom leann.api import LeannBuilder, LeannSearcher\nfrom leann.registry import register_project_directory\n\nfrom apps.chunking import create_text_chunks\n\nREPO_ROOT = Path(__file__).resolve().parents[1]\n\nDEFAULT_QUERY = \"What's LEANN?\"\nDEFAULT_INITIAL_FILES = [\n REPO_ROOT / \"data\" / \"2501.14312v1 (1).pdf\",\n REPO_ROOT / \"data\" / \"huawei_pangu.md\",\n REPO_ROOT / \"data\" / \"PrideandPrejudice.txt\",\n]\nDEFAULT_UPDATE_FILES = [REPO_ROOT / \"data\" / \"2506.08276v1.pdf\"]\n\n\ndef load_chunks_from_files(paths: list[Path]) -> list[str]:\n from llama_index.core import SimpleDirectoryReader\n\n documents = []\n for path in paths:\n p = path.expanduser().resolve()\n if not p.exists():\n raise FileNotFoundError(f\"Input path not found: {p}\")\n if p.is_dir():\n reader = SimpleDirectoryReader(str(p), recursive=False)\n documents.extend(reader.load_data(show_progress=True))\n else:\n reader = SimpleDirectoryReader(input_files=[str(p)])\n documents.extend(reader.load_data(show_progress=True))\n\n if not documents:\n return []\n\n chunks = create_text_chunks(\n documents,\n chunk_size=512,\n chunk_overlap=128,\n use_ast_chunking=False,\n )\n return [c for c in chunks if isinstance(c, str) and c.strip()]\n\n\ndef run_search(index_path: Path, query: str, top_k: int, *, recompute_embeddings: bool) -> list:\n searcher = LeannSearcher(str(index_path))\n try:\n return searcher.search(\n query=query,\n top_k=top_k,\n recompute_embeddings=recompute_embeddings,\n batch_size=16,\n )\n finally:\n searcher.cleanup()\n\n\ndef print_results(title: str, results: Iterable) -> None:\n print(f\"\\n=== {title} ===\")\n res_list = list(results)\n print(f\"results count: {len(res_list)}\")\n print(\"passages:\")\n if not res_list:\n print(\" (no passages returned)\")\n for res in res_list:\n snippet = res.text.replace(\"\\n\", \" \")[:120]\n print(f\" - {res.id}: {snippet}... (score={res.score:.4f})\")\n\n\ndef build_initial_index(\n index_path: Path,\n paragraphs: list[str],\n model_name: str,\n embedding_mode: str,\n is_recompute: bool,\n) -> None:\n builder = LeannBuilder(\n backend_name=\"hnsw\",\n embedding_model=model_name,\n embedding_mode=embedding_mode,\n is_compact=False,\n is_recompute=is_recompute,\n )\n for idx, passage in enumerate(paragraphs):\n builder.add_text(passage, metadata={\"id\": str(idx)})\n builder.build_index(str(index_path))\n\n\ndef update_index(\n index_path: Path,\n start_id: int,\n paragraphs: list[str],\n model_name: str,\n embedding_mode: str,\n is_recompute: bool,\n) -> None:\n updater = LeannBuilder(\n backend_name=\"hnsw\",\n embedding_model=model_name,\n embedding_mode=embedding_mode,\n is_compact=False,\n is_recompute=is_recompute,\n )\n for offset, passage in enumerate(paragraphs, start=start_id):\n updater.add_text(passage, metadata={\"id\": str(offset)})\n updater.update_index(str(index_path))\n\n\ndef ensure_index_dir(index_path: Path) -> None:\n index_path.parent.mkdir(parents=True, exist_ok=True)\n\n\ndef cleanup_index_files(index_path: Path) -> None:\n \"\"\"Remove leftover index artifacts for a clean rebuild.\"\"\"\n\n parent = index_path.parent\n if not parent.exists():\n return\n stem = index_path.stem\n for file in parent.glob(f\"{stem}*\"):\n if file.is_file():\n file.unlink()\n\n\ndef index_file_size(index_path: Path) -> int:\n \"\"\"Return the size of the primary .index file for the given index path.\"\"\"\n\n index_file = index_path.parent / f\"{index_path.stem}.index\"\n return index_file.stat().st_size if index_file.exists() else 0\n\n\ndef load_metadata_snapshot(index_path: Path) -> dict[str, Any] | None:\n meta_path = index_path.parent / f\"{index_path.name}.meta.json\"\n if not meta_path.exists():\n return None\n try:\n return json.loads(meta_path.read_text())\n except json.JSONDecodeError:\n return None\n\n\ndef run_workflow(\n *,\n label: str,\n index_path: Path,\n initial_paragraphs: list[str],\n update_paragraphs: list[str],\n model_name: str,\n embedding_mode: str,\n is_recompute: bool,\n query: str,\n top_k: int,\n skip_search: bool,\n) -> dict[str, Any]:\n prefix = f\"[{label}] \" if label else \"\"\n\n ensure_index_dir(index_path)\n cleanup_index_files(index_path)\n\n print(f\"{prefix}Building initial index...\")\n build_initial_index(\n index_path,\n initial_paragraphs,\n model_name,\n embedding_mode,\n is_recompute=is_recompute,\n )\n\n initial_size = index_file_size(index_path)\n if not skip_search:\n before_results = run_search(\n index_path,\n query,\n top_k,\n recompute_embeddings=is_recompute,\n )\n else:\n before_results = None\n\n print(f\"\\n{prefix}Updating index with additional passages...\")\n update_index(\n index_path,\n start_id=len(initial_paragraphs),\n paragraphs=update_paragraphs,\n model_name=model_name,\n embedding_mode=embedding_mode,\n is_recompute=is_recompute,\n )\n\n if not skip_search:\n after_results = run_search(\n index_path,\n query,\n top_k,\n recompute_embeddings=is_recompute,\n )\n else:\n after_results = None\n updated_size = index_file_size(index_path)\n\n return {\n \"initial_size\": initial_size,\n \"updated_size\": updated_size,\n \"delta\": updated_size - initial_size,\n \"before_results\": before_results if not skip_search else None,\n \"after_results\": after_results if not skip_search else None,\n \"metadata\": load_metadata_snapshot(index_path),\n }\n\n\ndef main() -> None:\n parser = argparse.ArgumentParser(description=__doc__)\n parser.add_argument(\n \"--initial-files\",\n type=Path,\n nargs=\"+\",\n default=DEFAULT_INITIAL_FILES,\n help=\"Initial document files (PDF/TXT) used to build the base index\",\n )\n parser.add_argument(\n \"--index-path\",\n type=Path,\n default=Path(\".leann/examples/leann-demo.leann\"),\n help=\"Destination index path (default: .leann/examples/leann-demo.leann)\",\n )\n parser.add_argument(\n \"--initial-count\",\n type=int,\n default=8,\n help=\"Number of chunks to use from the initial documents (default: 8)\",\n )\n parser.add_argument(\n \"--update-files\",\n type=Path,\n nargs=\"*\",\n default=DEFAULT_UPDATE_FILES,\n help=\"Additional documents to add during update (PDF/TXT)\",\n )\n parser.add_argument(\n \"--update-count\",\n type=int,\n default=4,\n help=\"Number of chunks to append from update documents (default: 4)\",\n )\n parser.add_argument(\n \"--update-text\",\n type=str,\n default=(\n \"LEANN (Lightweight Embedding ANN) is an indexing toolkit focused on \"\n \"recompute-aware HNSW graphs, allowing embeddings to be regenerated \"\n \"on demand to keep disk usage minimal.\"\n ),\n help=\"Fallback text to append if --update-files is omitted\",\n )\n parser.add_argument(\n \"--top-k\",\n type=int,\n default=4,\n help=\"Number of results to show for each search (default: 4)\",\n )\n parser.add_argument(\n \"--query\",\n type=str,\n default=DEFAULT_QUERY,\n help=\"Query to run before/after the update\",\n )\n parser.add_argument(\n \"--embedding-model\",\n type=str,\n default=\"sentence-transformers/all-MiniLM-L6-v2\",\n help=\"Embedding model name\",\n )\n parser.add_argument(\n \"--embedding-mode\",\n type=str,\n default=\"sentence-transformers\",\n choices=[\"sentence-transformers\", \"openai\", \"mlx\", \"ollama\"],\n help=\"Embedding backend mode\",\n )\n parser.add_argument(\n \"--compare-no-recompute\",\n dest=\"compare_no_recompute\",\n action=\"store_true\",\n help=\"Also run a baseline with is_recompute=False and report its index growth.\",\n )\n parser.add_argument(\n \"--skip-compare-no-recompute\",\n dest=\"compare_no_recompute\",\n action=\"store_false\",\n help=\"Skip building the no-recompute baseline.\",\n )\n parser.add_argument(\n \"--skip-search\",\n dest=\"skip_search\",\n action=\"store_true\",\n help=\"Skip the search step.\",\n )\n parser.set_defaults(compare_no_recompute=True)\n args = parser.parse_args()\n\n ensure_index_dir(args.index_path)\n register_project_directory(REPO_ROOT)\n\n initial_chunks = load_chunks_from_files(list(args.initial_files))\n if not initial_chunks:\n raise ValueError(\"No text chunks extracted from the initial files.\")\n\n initial = initial_chunks[: args.initial_count]\n if not initial:\n raise ValueError(\"Initial chunk set is empty after applying --initial-count.\")\n\n if args.update_files:\n update_chunks = load_chunks_from_files(list(args.update_files))\n if not update_chunks:\n raise ValueError(\"No text chunks extracted from the update files.\")\n to_add = update_chunks[: args.update_count]\n else:\n if not args.update_text:\n raise ValueError(\"Provide --update-files or --update-text for the update step.\")\n to_add = [args.update_text]\n if not to_add:\n raise ValueError(\"Update chunk set is empty after applying --update-count.\")\n\n recompute_stats = run_workflow(\n label=\"recompute\",\n index_path=args.index_path,\n initial_paragraphs=initial,\n update_paragraphs=to_add,\n model_name=args.embedding_model,\n embedding_mode=args.embedding_mode,\n is_recompute=True,\n query=args.query,\n top_k=args.top_k,\n skip_search=args.skip_search,\n )\n\n if not args.skip_search:\n print_results(\"initial search\", recompute_stats[\"before_results\"])\n if not args.skip_search:\n print_results(\"after update\", recompute_stats[\"after_results\"])\n print(\n f\"\\n[recompute] Index file size change: {recompute_stats['initial_size']} -> {recompute_stats['updated_size']} bytes\"\n f\" (Δ {recompute_stats['delta']})\"\n )\n\n if recompute_stats[\"metadata\"]:\n meta_view = {k: recompute_stats[\"metadata\"].get(k) for k in (\"is_compact\", \"is_pruned\")}\n print(\"[recompute] metadata snapshot:\")\n print(json.dumps(meta_view, indent=2))\n\n if args.compare_no_recompute:\n baseline_path = (\n args.index_path.parent / f\"{args.index_path.stem}-norecompute{args.index_path.suffix}\"\n )\n baseline_stats = run_workflow(\n label=\"no-recompute\",\n index_path=baseline_path,\n initial_paragraphs=initial,\n update_paragraphs=to_add,\n model_name=args.embedding_model,\n embedding_mode=args.embedding_mode,\n is_recompute=False,\n query=args.query,\n top_k=args.top_k,\n skip_search=args.skip_search,\n )\n\n print(\n f\"\\n[no-recompute] Index file size change: {baseline_stats['initial_size']} -> {baseline_stats['updated_size']} bytes\"\n f\" (Δ {baseline_stats['delta']})\"\n )\n\n after_texts = (\n [res.text for res in recompute_stats[\"after_results\"]] if not args.skip_search else None\n )\n baseline_after_texts = (\n [res.text for res in baseline_stats[\"after_results\"]] if not args.skip_search else None\n )\n if after_texts == baseline_after_texts:\n print(\n \"[no-recompute] Search results match recompute baseline; see above for the shared output.\"\n )\n else:\n print(\"[no-recompute] WARNING: search results differ from recompute baseline.\")\n\n if baseline_stats[\"metadata\"]:\n meta_view = {k: baseline_stats[\"metadata\"].get(k) for k in (\"is_compact\", \"is_pruned\")}\n print(\"[no-recompute] metadata snapshot:\")\n print(json.dumps(meta_view, indent=2))\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "examples/grep_search_example.py", "content": "\"\"\"\nGrep Search Example\n\nShows how to use grep-based text search instead of semantic search.\nUseful when you need exact text matches rather than meaning-based results.\n\"\"\"\n\nfrom leann import LeannSearcher\n\n# Load your index\nsearcher = LeannSearcher(\"my-documents.leann\")\n\n# Regular semantic search\nprint(\"=== Semantic Search ===\")\nresults = searcher.search(\"machine learning algorithms\", top_k=3)\nfor result in results:\n print(f\"Score: {result.score:.3f}\")\n print(f\"Text: {result.text[:80]}...\")\n print()\n\n# Grep-based search for exact text matches\nprint(\"=== Grep Search ===\")\nresults = searcher.search(\"def train_model\", top_k=3, use_grep=True)\nfor result in results:\n print(f\"Score: {result.score}\")\n print(f\"Text: {result.text[:80]}...\")\n print()\n\n# Find specific error messages\nerror_results = searcher.search(\"FileNotFoundError\", use_grep=True)\nprint(f\"Found {len(error_results)} files mentioning FileNotFoundError\")\n\n# Search for function definitions\nfunc_results = searcher.search(\"class SearchResult\", use_grep=True, top_k=5)\nprint(f\"Found {len(func_results)} class definitions\")\n" }, { "path": "examples/mcp_integration_demo.py", "content": "#!/usr/bin/env python3\n\"\"\"\nMCP Integration Examples for LEANN\n\nThis script demonstrates how to use LEANN with different MCP servers for\nRAG on various platforms like Slack and Twitter.\n\nExamples:\n1. Slack message RAG via MCP\n2. Twitter bookmark RAG via MCP\n3. Testing MCP server connections\n\"\"\"\n\nimport asyncio\nimport sys\nfrom pathlib import Path\n\n# Add the parent directory to the path so we can import from apps\nsys.path.append(str(Path(__file__).parent.parent))\n\n\nasync def demo_slack_mcp():\n \"\"\"Demonstrate Slack MCP integration.\"\"\"\n print(\"=\" * 60)\n print(\"🔥 Slack MCP RAG Demo\")\n print(\"=\" * 60)\n\n print(\"\\n1. Testing Slack MCP server connection...\")\n\n # This would typically use a real MCP server command\n # For demo purposes, we show what the command would look like\n # slack_app = SlackMCPRAG() # Would be used for actual testing\n\n # Simulate command line arguments for testing\n class MockArgs:\n mcp_server = \"slack-mcp-server\" # This would be the actual MCP server command\n workspace_name = \"my-workspace\"\n channels = [\"general\", \"random\", \"dev-team\"]\n no_concatenate_conversations = False\n max_messages_per_channel = 50\n test_connection = True\n\n print(f\"MCP Server Command: {MockArgs.mcp_server}\")\n print(f\"Workspace: {MockArgs.workspace_name}\")\n print(f\"Channels: {', '.join(MockArgs.channels)}\")\n\n # In a real scenario, you would run:\n # success = await slack_app.test_mcp_connection(MockArgs)\n\n print(\"\\n📝 Example usage:\")\n print(\"python -m apps.slack_rag \\\\\")\n print(\" --mcp-server 'slack-mcp-server' \\\\\")\n print(\" --workspace-name 'my-team' \\\\\")\n print(\" --channels general dev-team \\\\\")\n print(\" --test-connection\")\n\n print(\"\\n🔍 After indexing, you could query:\")\n print(\"- 'What did the team discuss about the project deadline?'\")\n print(\"- 'Find messages about the new feature launch'\")\n print(\"- 'Show me conversations about budget planning'\")\n\n\nasync def demo_twitter_mcp():\n \"\"\"Demonstrate Twitter MCP integration.\"\"\"\n print(\"\\n\" + \"=\" * 60)\n print(\"🐦 Twitter MCP RAG Demo\")\n print(\"=\" * 60)\n\n print(\"\\n1. Testing Twitter MCP server connection...\")\n\n # twitter_app = TwitterMCPRAG() # Would be used for actual testing\n\n class MockArgs:\n mcp_server = \"twitter-mcp-server\"\n username = None # Fetch all bookmarks\n max_bookmarks = 500\n no_tweet_content = False\n no_metadata = False\n test_connection = True\n\n print(f\"MCP Server Command: {MockArgs.mcp_server}\")\n print(f\"Max Bookmarks: {MockArgs.max_bookmarks}\")\n print(f\"Include Content: {not MockArgs.no_tweet_content}\")\n print(f\"Include Metadata: {not MockArgs.no_metadata}\")\n\n print(\"\\n📝 Example usage:\")\n print(\"python -m apps.twitter_rag \\\\\")\n print(\" --mcp-server 'twitter-mcp-server' \\\\\")\n print(\" --max-bookmarks 1000 \\\\\")\n print(\" --test-connection\")\n\n print(\"\\n🔍 After indexing, you could query:\")\n print(\"- 'What AI articles did I bookmark last month?'\")\n print(\"- 'Find tweets about machine learning techniques'\")\n print(\"- 'Show me bookmarked threads about startup advice'\")\n\n\nasync def show_mcp_server_setup():\n \"\"\"Show how to set up MCP servers.\"\"\"\n print(\"\\n\" + \"=\" * 60)\n print(\"⚙️ MCP Server Setup Guide\")\n print(\"=\" * 60)\n\n print(\"\\n🔧 Setting up Slack MCP Server:\")\n print(\"1. Install a Slack MCP server (example commands):\")\n print(\" npm install -g slack-mcp-server\")\n print(\" # OR\")\n print(\" pip install slack-mcp-server\")\n\n print(\"\\n2. Configure Slack credentials:\")\n print(\" export SLACK_BOT_TOKEN='xoxb-your-bot-token'\")\n print(\" export SLACK_APP_TOKEN='xapp-your-app-token'\")\n\n print(\"\\n3. Test the server:\")\n print(\" slack-mcp-server --help\")\n\n print(\"\\n🔧 Setting up Twitter MCP Server:\")\n print(\"1. Install a Twitter MCP server:\")\n print(\" npm install -g twitter-mcp-server\")\n print(\" # OR\")\n print(\" pip install twitter-mcp-server\")\n\n print(\"\\n2. Configure Twitter API credentials:\")\n print(\" export TWITTER_API_KEY='your-api-key'\")\n print(\" export TWITTER_API_SECRET='your-api-secret'\")\n print(\" export TWITTER_ACCESS_TOKEN='your-access-token'\")\n print(\" export TWITTER_ACCESS_TOKEN_SECRET='your-access-token-secret'\")\n\n print(\"\\n3. Test the server:\")\n print(\" twitter-mcp-server --help\")\n\n\nasync def show_integration_benefits():\n \"\"\"Show the benefits of MCP integration.\"\"\"\n print(\"\\n\" + \"=\" * 60)\n print(\"🌟 Benefits of MCP Integration\")\n print(\"=\" * 60)\n\n benefits = [\n (\"🔄 Live Data Access\", \"Fetch real-time data from platforms without manual exports\"),\n (\"🔌 Standardized Protocol\", \"Use any MCP-compatible server with minimal code changes\"),\n (\"🚀 Easy Extension\", \"Add new platforms by implementing MCP readers\"),\n (\"🔒 Secure Access\", \"MCP servers handle authentication and API management\"),\n (\"📊 Rich Metadata\", \"Access full platform metadata (timestamps, engagement, etc.)\"),\n (\"⚡ Efficient Processing\", \"Stream data directly into LEANN without intermediate files\"),\n ]\n\n for title, description in benefits:\n print(f\"\\n{title}\")\n print(f\" {description}\")\n\n\nasync def main():\n \"\"\"Main demo function.\"\"\"\n print(\"🎯 LEANN MCP Integration Examples\")\n print(\"This demo shows how to integrate LEANN with MCP servers for various platforms.\")\n\n await demo_slack_mcp()\n await demo_twitter_mcp()\n await show_mcp_server_setup()\n await show_integration_benefits()\n\n print(\"\\n\" + \"=\" * 60)\n print(\"✨ Next Steps\")\n print(\"=\" * 60)\n print(\"1. Install and configure MCP servers for your platforms\")\n print(\"2. Test connections using --test-connection flag\")\n print(\"3. Run indexing to build your RAG knowledge base\")\n print(\"4. Start querying your personal data!\")\n\n print(\"\\n📚 For more information:\")\n print(\"- Check the README for detailed setup instructions\")\n print(\"- Look at the apps/slack_rag.py and apps/twitter_rag.py for implementation details\")\n print(\"- Explore other MCP servers for additional platforms\")\n\n\nif __name__ == \"__main__\":\n asyncio.run(main())\n" }, { "path": "examples/mlx_demo.py", "content": "import os\n\nfrom leann.api import LeannBuilder, LeannChat\n\n# Define the path for our new MLX-based index\nINDEX_PATH = \"./mlx_diskann_index/leann\"\n\nif os.path.exists(INDEX_PATH + \".meta.json\"):\n print(f\"Index already exists at {INDEX_PATH}. Skipping build.\")\nelse:\n print(\"Initializing LeannBuilder with MLX support...\")\n # 1. Configure LeannBuilder to use MLX\n builder = LeannBuilder(\n backend_name=\"hnsw\",\n embedding_model=\"mlx-community/Qwen3-Embedding-0.6B-4bit-DWQ\",\n embedding_mode=\"mlx\",\n )\n\n # 2. Add documents\n print(\"Adding documents...\")\n docs = [\n \"MLX is an array framework for machine learning on Apple silicon.\",\n \"It was designed by Apple's machine learning research team.\",\n \"The mlx-community organization provides pre-trained models in MLX format.\",\n \"It supports operations on multi-dimensional arrays.\",\n \"Leann can now use MLX for its embedding models.\",\n ]\n for doc in docs:\n builder.add_text(doc)\n\n # 3. Build the index\n print(f\"Building the MLX-based index at: {INDEX_PATH}\")\n builder.build_index(INDEX_PATH)\n print(\"\\nSuccessfully built the index with MLX embeddings!\")\n print(f\"Check the metadata file: {INDEX_PATH}.meta.json\")\n\n\nchat = LeannChat(index_path=INDEX_PATH)\n# add query\nquery = \"MLX is an array framework for machine learning on Apple silicon.\"\nprint(f\"Query: {query}\")\nresponse = chat.ask(query, top_k=3, recompute_beighbor_embeddings=True, complexity=3, beam_width=1)\nprint(f\"Response: {response}\")\n" }, { "path": "examples/spoiler_free_book_rag.py", "content": "#!/usr/bin/env python3\n\"\"\"\nSpoiler-Free Book RAG Example using LEANN Metadata Filtering\n\nThis example demonstrates how to use LEANN's metadata filtering to create\na spoiler-free book RAG system where users can search for information\nup to a specific chapter they've read.\n\nUsage:\n python spoiler_free_book_rag.py\n\"\"\"\n\nimport os\nimport sys\nfrom typing import Any, Optional\n\n# Add LEANN to path (adjust path as needed)\nsys.path.insert(0, os.path.join(os.path.dirname(__file__), \"../packages/leann-core/src\"))\n\nfrom leann.api import LeannBuilder, LeannSearcher\n\n\ndef chunk_book_with_metadata(book_title: str = \"Sample Book\") -> list[dict[str, Any]]:\n \"\"\"\n Create sample book chunks with metadata for demonstration.\n\n In a real implementation, this would parse actual book files (epub, txt, etc.)\n and extract chapter boundaries, character mentions, etc.\n\n Args:\n book_title: Title of the book\n\n Returns:\n List of chunk dictionaries with text and metadata\n \"\"\"\n # Sample book chunks with metadata\n # In practice, you'd use proper text processing libraries\n\n sample_chunks = [\n {\n \"text\": \"Alice was beginning to get very tired of sitting by her sister on the bank, and of having nothing to do.\",\n \"metadata\": {\n \"book\": book_title,\n \"chapter\": 1,\n \"page\": 1,\n \"characters\": [\"Alice\", \"Sister\"],\n \"themes\": [\"boredom\", \"curiosity\"],\n \"location\": \"riverbank\",\n },\n },\n {\n \"text\": \"So she was considering in her own mind (as well as she could, for the hot day made her feel very sleepy and stupid), whether the pleasure of making a daisy-chain would be worth the trouble of getting up and picking the daisies, when suddenly a White Rabbit with pink eyes ran close by her.\",\n \"metadata\": {\n \"book\": book_title,\n \"chapter\": 1,\n \"page\": 2,\n \"characters\": [\"Alice\", \"White Rabbit\"],\n \"themes\": [\"decision\", \"surprise\", \"magic\"],\n \"location\": \"riverbank\",\n },\n },\n {\n \"text\": \"Alice found herself falling down a very deep well. Either the well was very deep, or she fell very slowly, for she had plenty of time as she fell to look about her and to wonder what was going to happen next.\",\n \"metadata\": {\n \"book\": book_title,\n \"chapter\": 2,\n \"page\": 15,\n \"characters\": [\"Alice\"],\n \"themes\": [\"falling\", \"wonder\", \"transformation\"],\n \"location\": \"rabbit hole\",\n },\n },\n {\n \"text\": \"Alice meets the Cheshire Cat, who tells her that everyone in Wonderland is mad, including Alice herself.\",\n \"metadata\": {\n \"book\": book_title,\n \"chapter\": 6,\n \"page\": 85,\n \"characters\": [\"Alice\", \"Cheshire Cat\"],\n \"themes\": [\"madness\", \"philosophy\", \"identity\"],\n \"location\": \"Duchess's house\",\n },\n },\n {\n \"text\": \"At the Queen's croquet ground, Alice witnesses the absurd trial that reveals the arbitrary nature of Wonderland's justice system.\",\n \"metadata\": {\n \"book\": book_title,\n \"chapter\": 8,\n \"page\": 120,\n \"characters\": [\"Alice\", \"Queen of Hearts\", \"King of Hearts\"],\n \"themes\": [\"justice\", \"absurdity\", \"authority\"],\n \"location\": \"Queen's court\",\n },\n },\n {\n \"text\": \"Alice realizes that Wonderland was all a dream, even the Rabbit, as she wakes up on the riverbank next to her sister.\",\n \"metadata\": {\n \"book\": book_title,\n \"chapter\": 12,\n \"page\": 180,\n \"characters\": [\"Alice\", \"Sister\", \"Rabbit\"],\n \"themes\": [\"revelation\", \"reality\", \"growth\"],\n \"location\": \"riverbank\",\n },\n },\n ]\n\n return sample_chunks\n\n\ndef build_spoiler_free_index(book_chunks: list[dict[str, Any]], index_name: str) -> str:\n \"\"\"\n Build a LEANN index with book chunks that include spoiler metadata.\n\n Args:\n book_chunks: List of book chunks with metadata\n index_name: Name for the index\n\n Returns:\n Path to the built index\n \"\"\"\n print(f\"📚 Building spoiler-free book index: {index_name}\")\n\n # Initialize LEANN builder\n builder = LeannBuilder(\n backend_name=\"hnsw\", embedding_model=\"text-embedding-3-small\", embedding_mode=\"openai\"\n )\n\n # Add each chunk with its metadata\n for chunk in book_chunks:\n builder.add_text(text=chunk[\"text\"], metadata=chunk[\"metadata\"])\n\n # Build the index\n index_path = f\"{index_name}_book_index\"\n builder.build_index(index_path)\n\n print(f\"✅ Index built successfully: {index_path}\")\n return index_path\n\n\ndef spoiler_free_search(\n index_path: str,\n query: str,\n max_chapter: int,\n character_filter: Optional[list[str]] = None,\n) -> list[dict[str, Any]]:\n \"\"\"\n Perform a spoiler-free search on the book index.\n\n Args:\n index_path: Path to the LEANN index\n query: Search query\n max_chapter: Maximum chapter number to include\n character_filter: Optional list of characters to focus on\n\n Returns:\n List of search results safe for the reader\n \"\"\"\n print(f\"🔍 Searching: '{query}' (up to chapter {max_chapter})\")\n\n searcher = LeannSearcher(index_path)\n\n metadata_filters = {\"chapter\": {\"<=\": max_chapter}}\n\n if character_filter:\n metadata_filters[\"characters\"] = {\"contains\": character_filter[0]}\n\n results = searcher.search(query=query, top_k=10, metadata_filters=metadata_filters)\n\n return results\n\n\ndef demo_spoiler_free_rag():\n \"\"\"\n Demonstrate the spoiler-free book RAG system.\n \"\"\"\n print(\"🎭 Spoiler-Free Book RAG Demo\")\n print(\"=\" * 40)\n\n # Step 1: Prepare book data\n book_title = \"Alice's Adventures in Wonderland\"\n book_chunks = chunk_book_with_metadata(book_title)\n\n print(f\"📖 Loaded {len(book_chunks)} chunks from '{book_title}'\")\n\n # Step 2: Build the index (in practice, this would be done once)\n try:\n index_path = build_spoiler_free_index(book_chunks, \"alice_wonderland\")\n except Exception as e:\n print(f\"❌ Failed to build index (likely missing dependencies): {e}\")\n print(\n \"💡 This demo shows the filtering logic - actual indexing requires LEANN dependencies\"\n )\n return\n\n # Step 3: Demonstrate various spoiler-free searches\n search_scenarios = [\n {\n \"description\": \"Reader who has only read Chapter 1\",\n \"query\": \"What can you tell me about the rabbit?\",\n \"max_chapter\": 1,\n },\n {\n \"description\": \"Reader who has read up to Chapter 5\",\n \"query\": \"Tell me about Alice's adventures\",\n \"max_chapter\": 5,\n },\n {\n \"description\": \"Reader who has read most of the book\",\n \"query\": \"What does the Cheshire Cat represent?\",\n \"max_chapter\": 10,\n },\n {\n \"description\": \"Reader who has read the whole book\",\n \"query\": \"What can you tell me about the rabbit?\",\n \"max_chapter\": 12,\n },\n ]\n\n for scenario in search_scenarios:\n print(f\"\\n📚 Scenario: {scenario['description']}\")\n print(f\" Query: {scenario['query']}\")\n\n try:\n results = spoiler_free_search(\n index_path=index_path,\n query=scenario[\"query\"],\n max_chapter=scenario[\"max_chapter\"],\n )\n\n print(f\" 📄 Found {len(results)} results:\")\n for i, result in enumerate(results[:3], 1): # Show top 3\n chapter = result.metadata.get(\"chapter\", \"?\")\n location = result.metadata.get(\"location\", \"?\")\n print(f\" {i}. Chapter {chapter} ({location}): {result.text[:80]}...\")\n\n except Exception as e:\n print(f\" ❌ Search failed: {e}\")\n\n\nif __name__ == \"__main__\":\n print(\"📚 LEANN Spoiler-Free Book RAG Example\")\n print(\"=====================================\")\n\n try:\n demo_spoiler_free_rag()\n except ImportError as e:\n print(f\"❌ Cannot run demo due to missing dependencies: {e}\")\n except Exception as e:\n print(f\"❌ Error running demo: {e}\")\n" }, { "path": "llms.txt", "content": "# llms.txt — LEANN MCP and Agent Integration\nproduct: LEANN\nhomepage: https://github.com/yichuan-w/LEANN\ncontact: https://github.com/yichuan-w/LEANN/issues\n\n# Installation\ninstall: uv tool install leann-core --with leann\n\n# MCP Server Entry Point\nmcp.server: leann_mcp\nmcp.protocol_version: 2024-11-05\n\n# Tools\nmcp.tools: leann_list, leann_search\n\nmcp.tool.leann_list.description: List available LEANN indexes\nmcp.tool.leann_list.input: {}\n\nmcp.tool.leann_search.description: Semantic search across a named LEANN index\nmcp.tool.leann_search.input.index_name: string, required\nmcp.tool.leann_search.input.query: string, required\nmcp.tool.leann_search.input.top_k: integer, optional, default=5, min=1, max=20\nmcp.tool.leann_search.input.complexity: integer, optional, default=32, min=16, max=128\n\n# Notes\nnote: Build indexes with `leann build --docs ` before searching.\nexample.add: claude mcp add --scope user leann-server -- leann_mcp\nexample.verify: claude mcp list | cat\n" }, { "path": "packages/__init__.py", "content": "" }, { "path": "packages/leann/README.md", "content": "# LEANN - The smallest vector index in the world\n\nLEANN is a revolutionary vector database that democratizes personal AI. Transform your laptop into a powerful RAG system that can index and search through millions of documents while using **97% less storage** than traditional solutions **without accuracy loss**.\n\n## Installation\n\n```bash\n# Default installation (includes both HNSW and DiskANN backends)\nuv pip install leann\n\n# CPU-only install (Linux)\nuv pip install \\\n --default-index https://download.pytorch.org/whl/cpu \\\n --index https://pypi.org/simple \\\n --index-strategy first-index \\\n \"leann[cpu]\"\n```\n\n## Quick Start\n\n```python\nfrom leann import LeannBuilder, LeannSearcher, LeannChat\nfrom pathlib import Path\nINDEX_PATH = str(Path(\"./\").resolve() / \"demo.leann\")\n\n# Build an index (choose backend: \"hnsw\" or \"diskann\")\nbuilder = LeannBuilder(backend_name=\"hnsw\") # or \"diskann\" for large-scale deployments\nbuilder.add_text(\"LEANN saves 97% storage compared to traditional vector databases.\")\nbuilder.add_text(\"Tung Tung Tung Sahur called—they need their banana‑crocodile hybrid back\")\nbuilder.build_index(INDEX_PATH)\n\n# Search\nsearcher = LeannSearcher(INDEX_PATH)\nresults = searcher.search(\"fantastical AI-generated creatures\", top_k=1)\n\n# Chat with your data\nchat = LeannChat(INDEX_PATH, llm_config={\"type\": \"hf\", \"model\": \"Qwen/Qwen3-0.6B\"})\nresponse = chat.ask(\"How much storage does LEANN save?\", top_k=1)\n```\n\n## License\n\nMIT License\n" }, { "path": "packages/leann/__init__.py", "content": "\"\"\"\nLEANN - Low-storage Embedding Approximation for Neural Networks\n\nA revolutionary vector database that democratizes personal AI.\n\"\"\"\n\n__version__ = \"0.1.0\"\n\n# Re-export main API from leann-core\nfrom leann_core import LeannBuilder, LeannChat, LeannSearcher\n\n__all__ = [\"LeannBuilder\", \"LeannChat\", \"LeannSearcher\"]\n" }, { "path": "packages/leann/pyproject.toml", "content": "[build-system]\nrequires = [\"setuptools>=61.0\"]\nbuild-backend = \"setuptools.build_meta\"\n\n[project]\nname = \"leann\"\nversion = \"0.3.7\"\ndescription = \"LEANN - The smallest vector index in the world. RAG Everything with LEANN!\"\nreadme = \"README.md\"\nrequires-python = \">=3.10\"\nlicense = { text = \"MIT\" }\nauthors = [\n { name = \"LEANN Team\" }\n]\nkeywords = [\"vector-database\", \"rag\", \"embeddings\", \"search\", \"ai\"]\nclassifiers = [\n \"Development Status :: 4 - Beta\",\n \"Intended Audience :: Developers\",\n \"License :: OSI Approved :: MIT License\",\n \"Programming Language :: Python :: 3\",\n \"Programming Language :: Python :: 3.10\",\n \"Programming Language :: Python :: 3.11\",\n \"Programming Language :: Python :: 3.12\",\n \"Programming Language :: Python :: 3.13\",\n]\n\n# Default installation: core + hnsw + diskann\ndependencies = [\n \"leann-core>=0.1.0\",\n \"leann-backend-hnsw>=0.1.0\",\n \"leann-backend-diskann>=0.1.0\",\n]\n\n[project.optional-dependencies]\ncpu = [\n \"leann-core[cpu]>=0.1.0\",\n]\n\n[project.urls]\nRepository = \"https://github.com/yichuan-w/LEANN\"\nIssues = \"https://github.com/yichuan-w/LEANN/issues\"\n" }, { "path": "packages/leann-backend-diskann/__init__.py", "content": "# This file makes the directory a Python package\n" }, { "path": "packages/leann-backend-diskann/leann_backend_diskann/__init__.py", "content": "import os\nfrom pathlib import Path\n\n_DLL_DIR_HANDLES: list[object] = []\n\n\ndef _configure_windows_dll_search_path() -> None:\n \"\"\"Register vcpkg DLL directories so Windows can resolve native dependencies.\n\n Python 3.8+ no longer searches PATH for DLL dependencies of native\n extensions (see https://github.com/numpy/numpy/wiki/windows-dll-notes).\n The standard workaround is ``os.add_dll_directory``.\n\n This only matters for **CI builds and source installs** where C++ deps\n (OpenBLAS, ZeroMQ, protobuf, …) live in a vcpkg tree. Pre-built wheels\n shipped via PyPI bundle all required DLLs inside the wheel (via\n delvewheel), so end-users installing with ``pip install`` are unaffected.\n \"\"\"\n if os.name != \"nt\" or not hasattr(os, \"add_dll_directory\"):\n return\n\n candidate_dirs: list[Path] = []\n env_roots = [os.getenv(\"VCPKG_INSTALLATION_ROOT\"), os.getenv(\"VCPKG_ROOT\")]\n for root in env_roots:\n if not root:\n continue\n root_path = Path(root)\n candidate_dirs.extend(\n [\n root_path / \"installed\" / \"x64-windows\" / \"bin\",\n root_path / \"installed\" / \"x64-windows\" / \"debug\" / \"bin\",\n root_path / \"installed\" / \"x64-windows\" / \"tools\" / \"protobuf\",\n ]\n )\n\n for path_entry in os.environ.get(\"PATH\", \"\").split(\";\"):\n entry = path_entry.strip()\n if not entry:\n continue\n if \"vcpkg\" in entry.lower():\n candidate_dirs.append(Path(entry))\n\n seen: set[str] = set()\n for dll_dir in candidate_dirs:\n resolved = str(dll_dir).lower()\n if resolved in seen or not dll_dir.exists():\n continue\n seen.add(resolved)\n try:\n _DLL_DIR_HANDLES.append(os.add_dll_directory(str(dll_dir)))\n os.environ[\"PATH\"] = f\"{dll_dir};{os.environ.get('PATH', '')}\"\n except OSError:\n continue\n\n\n_configure_windows_dll_search_path()\n\nfrom . import diskann_backend as diskann_backend # noqa: E402\nfrom . import graph_partition # noqa: E402\n\n# Export main classes and functions\nfrom .graph_partition import GraphPartitioner, partition_graph # noqa: E402\n\n__all__ = [\"GraphPartitioner\", \"diskann_backend\", \"graph_partition\", \"partition_graph\"]\n" }, { "path": "packages/leann-backend-diskann/leann_backend_diskann/diskann_backend.py", "content": "import contextlib\nimport logging\nimport os\nimport struct\nimport sys\nfrom pathlib import Path\nfrom typing import Any, Literal, Optional\n\nimport numpy as np\nimport psutil\nfrom leann.interface import (\n LeannBackendBuilderInterface,\n LeannBackendFactoryInterface,\n LeannBackendSearcherInterface,\n)\nfrom leann.registry import register_backend\nfrom leann.searcher_base import BaseSearcher\n\nlogger = logging.getLogger(__name__)\n\n\n@contextlib.contextmanager\ndef suppress_cpp_output_if_needed():\n \"\"\"Suppress C++ stdout/stderr based on LEANN_LOG_LEVEL\"\"\"\n # In CI we avoid fiddling with low-level file descriptors to prevent aborts\n if os.getenv(\"CI\") == \"true\":\n yield\n return\n\n log_level = os.getenv(\"LEANN_LOG_LEVEL\", \"WARNING\").upper()\n\n # Only suppress if log level is WARNING or higher (ERROR, CRITICAL)\n should_suppress = log_level in [\"WARNING\", \"ERROR\", \"CRITICAL\"]\n\n if not should_suppress:\n # Don't suppress, just yield\n yield\n return\n\n # Save original file descriptors\n stdout_fd = sys.stdout.fileno()\n stderr_fd = sys.stderr.fileno()\n\n # Save original stdout/stderr\n stdout_dup = os.dup(stdout_fd)\n stderr_dup = os.dup(stderr_fd)\n\n try:\n # Redirect to /dev/null\n devnull = os.open(os.devnull, os.O_WRONLY)\n os.dup2(devnull, stdout_fd)\n os.dup2(devnull, stderr_fd)\n os.close(devnull)\n\n yield\n\n finally:\n # Restore original file descriptors\n os.dup2(stdout_dup, stdout_fd)\n os.dup2(stderr_dup, stderr_fd)\n os.close(stdout_dup)\n os.close(stderr_dup)\n\n\ndef _get_diskann_metrics():\n from . import _diskannpy as diskannpy # type: ignore\n\n return {\n \"mips\": diskannpy.Metric.INNER_PRODUCT,\n \"l2\": diskannpy.Metric.L2,\n \"cosine\": diskannpy.Metric.COSINE,\n }\n\n\n@contextlib.contextmanager\ndef chdir(path):\n original_dir = os.getcwd()\n os.chdir(path)\n try:\n yield\n finally:\n os.chdir(original_dir)\n\n\ndef _write_vectors_to_bin(data: np.ndarray, file_path: Path):\n num_vectors, dim = data.shape\n with open(file_path, \"wb\") as f:\n f.write(struct.pack(\"I\", num_vectors))\n f.write(struct.pack(\"I\", dim))\n f.write(data.tobytes())\n\n\ndef _calculate_smart_memory_config(data: np.ndarray) -> tuple[float, float]:\n \"\"\"\n Calculate smart memory configuration for DiskANN based on data size and system specs.\n\n Args:\n data: The embedding data array\n\n Returns:\n tuple: (search_memory_maximum, build_memory_maximum) in GB\n \"\"\"\n num_vectors, dim = data.shape\n\n # Calculate embedding storage size\n embedding_size_bytes = num_vectors * dim * 4 # float32 = 4 bytes\n embedding_size_gb = embedding_size_bytes / (1024**3)\n\n # search_memory_maximum: 1/10 of embedding size for optimal PQ compression\n # This controls Product Quantization size - smaller means more compression\n search_memory_gb = max(0.1, embedding_size_gb / 10) # At least 100MB\n\n # build_memory_maximum: Based on available system RAM for sharding control\n # This controls how much memory DiskANN uses during index construction\n available_memory_gb = psutil.virtual_memory().available / (1024**3)\n total_memory_gb = psutil.virtual_memory().total / (1024**3)\n\n # Use 50% of available memory, but at least 2GB and at most 75% of total\n build_memory_gb = max(2.0, min(available_memory_gb * 0.5, total_memory_gb * 0.75))\n\n logger.info(\n f\"Smart memory config - Data: {embedding_size_gb:.2f}GB, \"\n f\"Search mem: {search_memory_gb:.2f}GB (PQ control), \"\n f\"Build mem: {build_memory_gb:.2f}GB (sharding control)\"\n )\n\n return search_memory_gb, build_memory_gb\n\n\n@register_backend(\"diskann\")\nclass DiskannBackend(LeannBackendFactoryInterface):\n @staticmethod\n def builder(**kwargs) -> LeannBackendBuilderInterface:\n return DiskannBuilder(**kwargs)\n\n @staticmethod\n def searcher(index_path: str, **kwargs) -> LeannBackendSearcherInterface:\n return DiskannSearcher(index_path, **kwargs)\n\n\nclass DiskannBuilder(LeannBackendBuilderInterface):\n def __init__(self, **kwargs):\n self.build_params = kwargs\n\n def _safe_cleanup_after_partition(self, index_dir: Path, index_prefix: str):\n \"\"\"\n Safely cleanup files after partition.\n In partition mode, C++ doesn't read _disk.index content,\n so we can delete it if all derived files exist.\n \"\"\"\n disk_index_file = index_dir / f\"{index_prefix}_disk.index\"\n beam_search_file = index_dir / f\"{index_prefix}_disk_beam_search.index\"\n\n # Required files that C++ partition mode needs\n # Note: C++ generates these with _disk.index suffix\n disk_suffix = \"_disk.index\"\n required_files = [\n f\"{index_prefix}{disk_suffix}_medoids.bin\", # Critical: assert fails if missing\n # Note: _centroids.bin is not created in single-shot build - C++ handles this automatically\n f\"{index_prefix}_pq_pivots.bin\", # PQ table\n f\"{index_prefix}_pq_compressed.bin\", # PQ compressed vectors\n ]\n\n # Check if all required files exist\n missing_files = []\n for filename in required_files:\n file_path = index_dir / filename\n if not file_path.exists():\n missing_files.append(filename)\n\n if missing_files:\n logger.warning(\n f\"Cannot safely delete _disk.index - missing required files: {missing_files}\"\n )\n logger.info(\"Keeping all original files for safety\")\n return\n\n # Calculate space savings\n space_saved = 0\n files_to_delete = []\n\n if disk_index_file.exists():\n space_saved += disk_index_file.stat().st_size\n files_to_delete.append(disk_index_file)\n\n if beam_search_file.exists():\n space_saved += beam_search_file.stat().st_size\n files_to_delete.append(beam_search_file)\n\n # Safe to delete!\n for file_to_delete in files_to_delete:\n try:\n os.remove(file_to_delete)\n logger.info(f\"✅ Safely deleted: {file_to_delete.name}\")\n except Exception as e:\n logger.warning(f\"Failed to delete {file_to_delete.name}: {e}\")\n\n if space_saved > 0:\n space_saved_mb = space_saved / (1024 * 1024)\n logger.info(f\"💾 Space saved: {space_saved_mb:.1f} MB\")\n\n # Show what files are kept\n logger.info(\"📁 Kept essential files for partition mode:\")\n for filename in required_files:\n file_path = index_dir / filename\n if file_path.exists():\n size_mb = file_path.stat().st_size / (1024 * 1024)\n logger.info(f\" - {filename} ({size_mb:.1f} MB)\")\n\n def build(self, data: np.ndarray, ids: list[str], index_path: str, **kwargs):\n path = Path(index_path)\n index_dir = path.parent\n index_prefix = path.stem\n index_dir.mkdir(parents=True, exist_ok=True)\n\n if data.dtype != np.float32:\n logger.warning(f\"Converting data to float32, shape: {data.shape}\")\n data = data.astype(np.float32)\n\n data_filename = f\"{index_prefix}_data.bin\"\n _write_vectors_to_bin(data, index_dir / data_filename)\n\n build_kwargs = {**self.build_params, **kwargs}\n\n # Extract is_recompute from nested backend_kwargs if needed\n is_recompute = build_kwargs.get(\"is_recompute\", False)\n if not is_recompute and \"backend_kwargs\" in build_kwargs:\n is_recompute = build_kwargs[\"backend_kwargs\"].get(\"is_recompute\", False)\n\n # Flatten all backend_kwargs parameters to top level for compatibility\n if \"backend_kwargs\" in build_kwargs:\n nested_params = build_kwargs.pop(\"backend_kwargs\")\n build_kwargs.update(nested_params)\n\n metric_enum = _get_diskann_metrics().get(\n build_kwargs.get(\"distance_metric\", \"mips\").lower()\n )\n if metric_enum is None:\n raise ValueError(\n f\"Unsupported distance_metric '{build_kwargs.get('distance_metric', 'unknown')}'.\"\n )\n\n # Calculate smart memory configuration if not explicitly provided\n if (\n \"search_memory_maximum\" not in build_kwargs\n or \"build_memory_maximum\" not in build_kwargs\n ):\n smart_search_mem, smart_build_mem = _calculate_smart_memory_config(data)\n else:\n smart_search_mem = build_kwargs.get(\"search_memory_maximum\", 4.0)\n smart_build_mem = build_kwargs.get(\"build_memory_maximum\", 8.0)\n\n try:\n from . import _diskannpy as diskannpy # type: ignore\n\n with chdir(index_dir):\n diskannpy.build_disk_float_index(\n metric_enum,\n data_filename,\n index_prefix,\n build_kwargs.get(\"complexity\", 64),\n build_kwargs.get(\"graph_degree\", 32),\n build_kwargs.get(\"search_memory_maximum\", smart_search_mem),\n build_kwargs.get(\"build_memory_maximum\", smart_build_mem),\n build_kwargs.get(\"num_threads\", 8),\n build_kwargs.get(\"pq_disk_bytes\", 0),\n \"\",\n )\n\n # Auto-partition if is_recompute is enabled\n if build_kwargs.get(\"is_recompute\", False):\n logger.info(\"is_recompute=True, starting automatic graph partitioning...\")\n from .graph_partition import partition_graph\n\n # Partition the index using absolute paths\n # Convert to absolute paths to avoid issues with working directory changes\n absolute_index_dir = Path(index_dir).resolve()\n absolute_index_prefix_path = str(absolute_index_dir / index_prefix)\n disk_graph_path, partition_bin_path = partition_graph(\n index_prefix_path=absolute_index_prefix_path,\n output_dir=str(absolute_index_dir),\n partition_prefix=index_prefix,\n )\n\n # Safe cleanup: In partition mode, C++ doesn't read _disk.index content\n # but still needs the derived files (_medoids.bin, _centroids.bin, etc.)\n self._safe_cleanup_after_partition(index_dir, index_prefix)\n\n logger.info(\"✅ Graph partitioning completed successfully!\")\n logger.info(f\" - Disk graph: {disk_graph_path}\")\n logger.info(f\" - Partition file: {partition_bin_path}\")\n\n finally:\n temp_data_file = index_dir / data_filename\n if temp_data_file.exists():\n os.remove(temp_data_file)\n logger.debug(f\"Cleaned up temporary data file: {temp_data_file}\")\n\n\nclass DiskannSearcher(BaseSearcher):\n def __init__(self, index_path: str, **kwargs):\n super().__init__(\n index_path,\n backend_module_name=\"leann_backend_diskann.diskann_embedding_server\",\n **kwargs,\n )\n\n # Initialize DiskANN index with suppressed C++ output based on log level\n with suppress_cpp_output_if_needed():\n from . import _diskannpy as diskannpy # type: ignore\n\n distance_metric = kwargs.get(\"distance_metric\", \"mips\").lower()\n metric_enum = _get_diskann_metrics().get(distance_metric)\n if metric_enum is None:\n raise ValueError(f\"Unsupported distance_metric '{distance_metric}'.\")\n\n self.num_threads = kwargs.get(\"num_threads\", 8)\n\n # For DiskANN, we need to reinitialize the index when zmq_port changes\n # Store the initialization parameters for later use\n # Note: C++ load method expects the BASE path (without _disk.index suffix)\n # C++ internally constructs: index_prefix + \"_disk.index\"\n index_name = self.index_path.stem # \"simple_test.leann\" -> \"simple_test\"\n diskann_index_prefix = str(self.index_dir / index_name) # /path/to/simple_test\n full_index_prefix = diskann_index_prefix # /path/to/simple_test (base path)\n\n # Auto-detect partition files and set partition_prefix\n partition_graph_file = self.index_dir / f\"{index_name}_disk_graph.index\"\n partition_bin_file = self.index_dir / f\"{index_name}_partition.bin\"\n\n partition_prefix = \"\"\n if partition_graph_file.exists() and partition_bin_file.exists():\n # C++ expects full path prefix, not just filename\n partition_prefix = str(self.index_dir / index_name) # /path/to/simple_test\n logger.info(\n f\"✅ Detected partition files, using partition_prefix='{partition_prefix}'\"\n )\n else:\n logger.debug(\"No partition files detected, using standard index files\")\n\n self._init_params = {\n \"metric_enum\": metric_enum,\n \"full_index_prefix\": full_index_prefix,\n \"num_threads\": self.num_threads,\n \"num_nodes_to_cache\": kwargs.get(\"num_nodes_to_cache\", 0),\n # 1 -> initialize cache using sample_data; 2 -> ready cache without init; others disable cache\n \"cache_mechanism\": kwargs.get(\"cache_mechanism\", 1),\n \"pq_prefix\": \"\",\n \"partition_prefix\": partition_prefix,\n }\n\n # Log partition configuration for debugging\n if partition_prefix:\n logger.info(\n f\"✅ Detected partition files, using partition_prefix='{partition_prefix}'\"\n )\n self._diskannpy = diskannpy\n self._current_zmq_port = None\n self._index = None\n logger.debug(\"DiskANN searcher initialized (index will be loaded on first search)\")\n\n def close(self):\n \"\"\"Release the C++ index object and its file handles.\n\n On Windows, open memory-mapped files prevent temp directory cleanup.\n Call this (or use LeannSearcher as a context manager) before deleting\n the index directory.\n \"\"\"\n self._index = None\n self._current_zmq_port = None\n import gc\n\n gc.collect()\n\n def _ensure_index_loaded(self, zmq_port: int):\n \"\"\"Ensure the index is loaded with the correct zmq_port.\"\"\"\n if self._index is None or self._current_zmq_port != zmq_port:\n # Need to (re)load the index with the correct zmq_port\n with suppress_cpp_output_if_needed():\n if self._index is not None:\n logger.debug(f\"Reloading DiskANN index with new zmq_port: {zmq_port}\")\n else:\n logger.debug(f\"Loading DiskANN index with zmq_port: {zmq_port}\")\n\n self._index = self._diskannpy.StaticDiskFloatIndex(\n self._init_params[\"metric_enum\"],\n self._init_params[\"full_index_prefix\"],\n self._init_params[\"num_threads\"],\n self._init_params[\"num_nodes_to_cache\"],\n self._init_params[\"cache_mechanism\"],\n zmq_port,\n self._init_params[\"pq_prefix\"],\n self._init_params[\"partition_prefix\"],\n )\n self._current_zmq_port = zmq_port\n\n def search(\n self,\n query: np.ndarray,\n top_k: int,\n complexity: int = 64,\n beam_width: int = 1,\n prune_ratio: float = 0.0,\n recompute_embeddings: bool = False,\n pruning_strategy: Literal[\"global\", \"local\", \"proportional\"] = \"global\",\n zmq_port: Optional[int] = None,\n batch_recompute: bool = False,\n dedup_node_dis: bool = False,\n **kwargs,\n ) -> dict[str, Any]:\n \"\"\"\n Search for nearest neighbors using DiskANN index.\n\n Args:\n query: Query vectors (B, D) where B is batch size, D is dimension\n top_k: Number of nearest neighbors to return\n complexity: Search complexity/candidate list size, higher = more accurate but slower\n beam_width: Number of parallel IO requests per iteration\n prune_ratio: Ratio of neighbors to prune via approximate distance (0.0-1.0)\n recompute_embeddings: Whether to fetch fresh embeddings from server\n pruning_strategy: PQ candidate selection strategy:\n - \"global\": Use global pruning strategy (default)\n - \"local\": Use local pruning strategy\n - \"proportional\": Not supported in DiskANN, falls back to global\n zmq_port: ZMQ port for embedding server communication. Must be provided if recompute_embeddings is True.\n batch_recompute: Whether to batch neighbor recomputation (DiskANN-specific)\n dedup_node_dis: Whether to cache and reuse distance computations (DiskANN-specific)\n **kwargs: Additional DiskANN-specific parameters (for legacy compatibility)\n\n Returns:\n Dict with 'labels' (list of lists) and 'distances' (ndarray)\n \"\"\"\n # Handle zmq_port compatibility: Ensure index is loaded with correct port\n if recompute_embeddings:\n if zmq_port is None:\n raise ValueError(\"zmq_port must be provided if recompute_embeddings is True\")\n self._ensure_index_loaded(zmq_port)\n else:\n # If not recomputing, we still need an index, use a default port\n if self._index is None:\n self._ensure_index_loaded(6666) # Default port when not recomputing\n\n # DiskANN doesn't support \"proportional\" strategy\n if pruning_strategy == \"proportional\":\n raise NotImplementedError(\n \"DiskANN backend does not support 'proportional' pruning strategy. Use 'global' or 'local' instead.\"\n )\n\n if query.dtype != np.float32:\n query = query.astype(np.float32)\n\n # Map pruning_strategy to DiskANN's global_pruning parameter\n if pruning_strategy == \"local\":\n use_global_pruning = False\n else: # \"global\"\n use_global_pruning = True\n\n # Strategy:\n # - Traversal always uses PQ distances\n # - If recompute_embeddings=True, do a single final rerank via deferred fetch\n # (fetch embeddings for the final candidate set only)\n # - Do not recompute neighbor distances along the path\n use_deferred_fetch = True if recompute_embeddings else False\n recompute_neighors = False # Expected typo. For backward compatibility.\n\n with suppress_cpp_output_if_needed():\n labels, distances = self._index.batch_search(\n query,\n query.shape[0],\n top_k,\n complexity,\n beam_width,\n self.num_threads,\n use_deferred_fetch,\n kwargs.get(\"skip_search_reorder\", False),\n recompute_neighors,\n dedup_node_dis,\n prune_ratio,\n batch_recompute,\n use_global_pruning,\n )\n\n string_labels = [[str(int_label) for int_label in batch_labels] for batch_labels in labels]\n\n return {\"labels\": string_labels, \"distances\": distances}\n" }, { "path": "packages/leann-backend-diskann/leann_backend_diskann/diskann_embedding_server.py", "content": "\"\"\"\nDiskANN-specific embedding server\n\"\"\"\n\nimport argparse\nimport json\nimport logging\nimport os\nimport sys\nimport threading\nimport time\nfrom pathlib import Path\nfrom typing import Any, Optional\n\nimport numpy as np\nimport zmq\n\n# Set up logging based on environment variable\nLOG_LEVEL = os.getenv(\"LEANN_LOG_LEVEL\", \"WARNING\").upper()\nlogger = logging.getLogger(__name__)\n\n# Force set logger level (don't rely on basicConfig in subprocess)\nlog_level = getattr(logging, LOG_LEVEL, logging.WARNING)\nlogger.setLevel(log_level)\n\n# Ensure we have a handler if none exists\nif not logger.handlers:\n handler = logging.StreamHandler()\n formatter = logging.Formatter(\"%(asctime)s - %(levelname)s - %(message)s\")\n handler.setFormatter(formatter)\n logger.addHandler(handler)\n logger.propagate = False\n\n\n_RAW_PROVIDER_OPTIONS = os.getenv(\"LEANN_EMBEDDING_OPTIONS\")\ntry:\n PROVIDER_OPTIONS: dict[str, Any] = (\n json.loads(_RAW_PROVIDER_OPTIONS) if _RAW_PROVIDER_OPTIONS else {}\n )\nexcept json.JSONDecodeError:\n logger.warning(\"Failed to parse LEANN_EMBEDDING_OPTIONS; ignoring provider options\")\n PROVIDER_OPTIONS = {}\n\n\ndef create_diskann_embedding_server(\n passages_file: Optional[str] = None,\n zmq_port: int = 5555,\n model_name: str = \"sentence-transformers/all-mpnet-base-v2\",\n embedding_mode: str = \"sentence-transformers\",\n distance_metric: str = \"l2\",\n enable_warmup: bool = False,\n daemon_ttl: int = 0,\n):\n \"\"\"\n Create and start a ZMQ-based embedding server for DiskANN backend.\n Uses ROUTER socket and protobuf communication as required by DiskANN C++ implementation.\n \"\"\"\n logger.info(f\"Starting DiskANN server on port {zmq_port} with model {model_name}\")\n logger.info(f\"Using embedding mode: {embedding_mode}\")\n\n # Add leann-core to path for unified embedding computation\n current_dir = Path(__file__).parent\n leann_core_path = current_dir.parent.parent / \"leann-core\" / \"src\"\n sys.path.insert(0, str(leann_core_path))\n\n try:\n from leann.api import PassageManager\n from leann.embedding_compute import compute_embeddings\n\n logger.info(\"Successfully imported unified embedding computation module\")\n except ImportError as e:\n logger.error(f\"Failed to import embedding computation module: {e}\")\n return\n finally:\n sys.path.pop(0)\n\n if enable_warmup:\n try:\n logger.info(\"Starting warmup embedding request...\")\n _ = compute_embeddings(\n [\"__LEANN_WARMUP__\"],\n model_name,\n mode=embedding_mode,\n provider_options=PROVIDER_OPTIONS,\n )\n logger.info(\"Warmup complete.\")\n except Exception as exc:\n logger.warning(f\"Warmup failed (continuing): {exc}\")\n\n # Check port availability\n import socket\n\n def check_port(port):\n with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:\n return s.connect_ex((\"localhost\", port)) == 0\n\n if check_port(zmq_port):\n logger.error(f\"Port {zmq_port} is already in use\")\n return\n\n # Only support metadata file, fail fast for everything else\n if not passages_file or not passages_file.endswith(\".meta.json\"):\n raise ValueError(\"Only metadata files (.meta.json) are supported\")\n\n # Load metadata to get passage sources\n with open(passages_file) as f:\n meta = json.load(f)\n\n logger.info(f\"Loading PassageManager with metadata_file_path: {passages_file}\")\n passages = PassageManager(meta[\"passage_sources\"], metadata_file_path=passages_file)\n logger.info(f\"Loaded PassageManager with {len(passages)} passages from metadata\")\n\n # Import protobuf after ensuring the path is correct\n try:\n from . import embedding_pb2\n except ImportError as e:\n logger.error(f\"Failed to import protobuf module: {e}\")\n return\n\n def zmq_server_thread():\n \"\"\"ZMQ server thread using REP socket for universal compatibility\"\"\"\n context = zmq.Context()\n socket = context.socket(\n zmq.REP\n ) # REP socket for both BaseSearcher and DiskANN C++ REQ clients\n socket.bind(f\"tcp://*:{zmq_port}\")\n logger.info(f\"DiskANN ZMQ REP server listening on port {zmq_port}\")\n\n socket.setsockopt(zmq.RCVTIMEO, 1000)\n socket.setsockopt(zmq.SNDTIMEO, 1000)\n socket.setsockopt(zmq.LINGER, 0)\n\n while True:\n try:\n # REP socket receives single-part messages\n message = socket.recv()\n\n # Check for empty messages - REP socket requires response to every request\n if len(message) == 0:\n logger.debug(\"Received empty message, sending empty response\")\n socket.send(b\"\") # REP socket must respond to every request\n continue\n\n logger.debug(f\"Received ZMQ request of size {len(message)} bytes\")\n logger.debug(f\"Message preview: {message[:50]}\") # Show first 50 bytes\n\n e2e_start = time.time()\n\n # Try protobuf first (for DiskANN C++ node_ids requests - primary use case)\n texts = []\n node_ids = []\n is_text_request = False\n\n try:\n req_proto = embedding_pb2.NodeEmbeddingRequest()\n req_proto.ParseFromString(message)\n node_ids = list(req_proto.node_ids)\n\n if not node_ids:\n raise RuntimeError(\n f\"PROTOBUF: Received empty node_ids! Message size: {len(message)}\"\n )\n\n logger.info(\n f\"✅ PROTOBUF: Node ID request for {len(node_ids)} node embeddings: {node_ids[:10]}\"\n )\n except Exception as protobuf_error:\n logger.debug(f\"Protobuf parsing failed: {protobuf_error}\")\n # Fallback to msgpack (for BaseSearcher direct text requests)\n try:\n import msgpack\n\n request = msgpack.unpackb(message)\n # For BaseSearcher compatibility, request is a list of texts directly\n if isinstance(request, list) and all(\n isinstance(item, str) for item in request\n ):\n texts = request\n is_text_request = True\n logger.info(f\"✅ MSGPACK: Direct text request for {len(texts)} texts\")\n else:\n raise ValueError(\"Not a valid msgpack text request\")\n except Exception as msgpack_error:\n raise RuntimeError(\n f\"Both protobuf and msgpack parsing failed! Protobuf: {protobuf_error}, Msgpack: {msgpack_error}\"\n )\n\n # Look up texts by node IDs (only if not direct text request)\n if not is_text_request:\n for nid in node_ids:\n try:\n passage_data = passages.get_passage(str(nid))\n txt = passage_data[\"text\"]\n if not txt:\n raise RuntimeError(f\"FATAL: Empty text for passage ID {nid}\")\n texts.append(txt)\n except KeyError as e:\n logger.error(f\"Passage ID {nid} not found: {e}\")\n raise e\n except Exception as e:\n logger.error(f\"Exception looking up passage ID {nid}: {e}\")\n raise\n\n # Debug logging\n logger.debug(f\"Processing {len(texts)} texts\")\n logger.debug(f\"Text lengths: {[len(t) for t in texts[:5]]}\") # Show first 5\n\n # Process embeddings using unified computation\n embeddings = compute_embeddings(\n texts,\n model_name,\n mode=embedding_mode,\n provider_options=PROVIDER_OPTIONS,\n )\n logger.info(\n f\"Computed embeddings for {len(texts)} texts, shape: {embeddings.shape}\"\n )\n\n # Prepare response based on request type\n if is_text_request:\n # For BaseSearcher compatibility: return msgpack format\n import msgpack\n\n response_data = msgpack.packb(embeddings.tolist())\n else:\n # For DiskANN C++ compatibility: return protobuf format\n resp_proto = embedding_pb2.NodeEmbeddingResponse()\n hidden_contiguous = np.ascontiguousarray(embeddings, dtype=np.float32)\n\n # Serialize embeddings data\n resp_proto.embeddings_data = hidden_contiguous.tobytes()\n resp_proto.dimensions.append(hidden_contiguous.shape[0])\n resp_proto.dimensions.append(hidden_contiguous.shape[1])\n\n response_data = resp_proto.SerializeToString()\n\n # Send response back to the client\n socket.send(response_data)\n\n e2e_end = time.time()\n logger.info(f\"⏱️ ZMQ E2E time: {e2e_end - e2e_start:.6f}s\")\n\n except zmq.Again:\n logger.debug(\"ZMQ socket timeout, continuing to listen\")\n continue\n except Exception as e:\n logger.error(f\"Error in ZMQ server loop: {e}\")\n import traceback\n\n traceback.print_exc()\n raise\n\n def zmq_server_thread_with_shutdown(shutdown_event):\n \"\"\"ZMQ server thread that respects shutdown signal.\n\n This creates its own REP socket, binds to zmq_port, and periodically\n checks shutdown_event using recv timeouts to exit cleanly.\n \"\"\"\n logger.info(\"DiskANN ZMQ server thread started with shutdown support\")\n\n context = zmq.Context()\n rep_socket = context.socket(zmq.REP)\n rep_socket.bind(f\"tcp://*:{zmq_port}\")\n logger.info(f\"DiskANN ZMQ REP server listening on port {zmq_port}\")\n\n # Set receive timeout so we can check shutdown_event periodically\n rep_socket.setsockopt(zmq.RCVTIMEO, 1000) # 1 second timeout\n rep_socket.setsockopt(zmq.SNDTIMEO, 1000)\n rep_socket.setsockopt(zmq.LINGER, 0)\n\n try:\n while not shutdown_event.is_set():\n try:\n e2e_start = time.time()\n # REP socket receives single-part messages\n message = rep_socket.recv()\n last_activity[0] = time.time()\n\n # Check for empty messages - REP socket requires response to every request\n if not message:\n logger.warning(\"Received empty message, sending empty response\")\n rep_socket.send(b\"\")\n continue\n\n # Try protobuf first (same logic as original)\n texts = []\n is_text_request = False\n\n try:\n req_proto = embedding_pb2.NodeEmbeddingRequest()\n req_proto.ParseFromString(message)\n node_ids = list(req_proto.node_ids)\n\n # Look up texts by node IDs\n for nid in node_ids:\n try:\n passage_data = passages.get_passage(str(nid))\n txt = passage_data[\"text\"]\n if not txt:\n raise RuntimeError(f\"FATAL: Empty text for passage ID {nid}\")\n texts.append(txt)\n except KeyError:\n raise RuntimeError(f\"FATAL: Passage with ID {nid} not found\")\n\n logger.info(f\"ZMQ received protobuf request for {len(node_ids)} node IDs\")\n except Exception:\n # Fallback to msgpack for text requests\n try:\n import msgpack\n\n request = msgpack.unpackb(message)\n if isinstance(request, list) and all(\n isinstance(item, str) for item in request\n ):\n texts = request\n is_text_request = True\n logger.info(\n f\"ZMQ received msgpack text request for {len(texts)} texts\"\n )\n else:\n raise ValueError(\"Not a valid msgpack text request\")\n except Exception:\n logger.error(\"Both protobuf and msgpack parsing failed!\")\n # Send error response\n resp_proto = embedding_pb2.NodeEmbeddingResponse()\n rep_socket.send(resp_proto.SerializeToString())\n continue\n\n # Process the request\n embeddings = compute_embeddings(\n texts,\n model_name,\n mode=embedding_mode,\n provider_options=PROVIDER_OPTIONS,\n )\n logger.info(f\"Computed embeddings shape: {embeddings.shape}\")\n\n # Validation\n if np.isnan(embeddings).any() or np.isinf(embeddings).any():\n logger.error(\"NaN or Inf detected in embeddings!\")\n # Send error response\n if is_text_request:\n import msgpack\n\n response_data = msgpack.packb([])\n else:\n resp_proto = embedding_pb2.NodeEmbeddingResponse()\n response_data = resp_proto.SerializeToString()\n rep_socket.send(response_data)\n continue\n\n # Prepare response based on request type\n if is_text_request:\n # For direct text requests, return msgpack\n import msgpack\n\n response_data = msgpack.packb(embeddings.tolist())\n else:\n # For protobuf requests, return protobuf\n resp_proto = embedding_pb2.NodeEmbeddingResponse()\n hidden_contiguous = np.ascontiguousarray(embeddings, dtype=np.float32)\n\n resp_proto.embeddings_data = hidden_contiguous.tobytes()\n resp_proto.dimensions.append(hidden_contiguous.shape[0])\n resp_proto.dimensions.append(hidden_contiguous.shape[1])\n\n response_data = resp_proto.SerializeToString()\n\n # Send response back to the client\n rep_socket.send(response_data)\n\n e2e_end = time.time()\n logger.info(f\"⏱️ ZMQ E2E time: {e2e_end - e2e_start:.6f}s\")\n\n except zmq.Again:\n # Timeout - check shutdown_event and continue\n continue\n except Exception as e:\n if not shutdown_event.is_set():\n logger.error(f\"Error in ZMQ server loop: {e}\")\n try:\n # Send error response for REP socket\n resp_proto = embedding_pb2.NodeEmbeddingResponse()\n rep_socket.send(resp_proto.SerializeToString())\n except Exception:\n pass\n else:\n logger.info(\"Shutdown in progress, ignoring ZMQ error\")\n break\n finally:\n try:\n rep_socket.close(0)\n except Exception:\n pass\n try:\n context.term()\n except Exception:\n pass\n\n logger.info(\"DiskANN ZMQ server thread exiting gracefully\")\n\n # Add shutdown coordination\n shutdown_event = threading.Event()\n last_activity = [time.time()]\n\n def shutdown_zmq_server():\n \"\"\"Gracefully shutdown ZMQ server.\"\"\"\n logger.info(\"Initiating graceful shutdown...\")\n shutdown_event.set()\n\n if zmq_thread.is_alive():\n logger.info(\"Waiting for ZMQ thread to finish...\")\n zmq_thread.join(timeout=5)\n if zmq_thread.is_alive():\n logger.warning(\"ZMQ thread did not finish in time\")\n\n # Clean up ZMQ resources\n try:\n # Note: socket and context are cleaned up by thread exit\n logger.info(\"ZMQ resources cleaned up\")\n except Exception as e:\n logger.warning(f\"Error cleaning ZMQ resources: {e}\")\n\n # Clean up other resources\n try:\n import gc\n\n gc.collect()\n logger.info(\"Additional resources cleaned up\")\n except Exception as e:\n logger.warning(f\"Error cleaning additional resources: {e}\")\n\n logger.info(\"Graceful shutdown completed\")\n sys.exit(0)\n\n # Register signal handlers within this function scope\n import signal\n\n def signal_handler(sig, frame):\n logger.info(f\"Received signal {sig}, shutting down gracefully...\")\n shutdown_zmq_server()\n\n signal.signal(signal.SIGTERM, signal_handler)\n signal.signal(signal.SIGINT, signal_handler)\n\n # Start ZMQ thread (NOT daemon!)\n zmq_thread = threading.Thread(\n target=lambda: zmq_server_thread_with_shutdown(shutdown_event),\n daemon=False, # Not daemon - we want to wait for it\n )\n zmq_thread.start()\n logger.info(f\"Started DiskANN ZMQ server thread on port {zmq_port}\")\n\n # Keep the main thread alive\n try:\n while not shutdown_event.is_set():\n if daemon_ttl > 0 and (time.time() - last_activity[0]) >= daemon_ttl:\n logger.info(\n f\"No requests for {daemon_ttl} seconds, shutting down daemon on port {zmq_port}\"\n )\n shutdown_zmq_server()\n return\n time.sleep(0.1) # Check shutdown more frequently\n except KeyboardInterrupt:\n logger.info(\"DiskANN Server shutting down...\")\n shutdown_zmq_server()\n return\n\n # If we reach here, shutdown was triggered by signal\n logger.info(\"Main loop exited, process should be shutting down\")\n\n\nif __name__ == \"__main__\":\n import sys\n\n # Signal handlers are now registered within create_diskann_embedding_server\n\n parser = argparse.ArgumentParser(description=\"DiskANN Embedding service\")\n parser.add_argument(\"--zmq-port\", type=int, default=5555, help=\"ZMQ port to run on\")\n parser.add_argument(\n \"--passages-file\",\n type=str,\n help=\"Metadata JSON file containing passage sources\",\n )\n parser.add_argument(\n \"--model-name\",\n type=str,\n default=\"sentence-transformers/all-mpnet-base-v2\",\n help=\"Embedding model name\",\n )\n parser.add_argument(\n \"--embedding-mode\",\n type=str,\n default=\"sentence-transformers\",\n choices=[\"sentence-transformers\", \"openai\", \"mlx\", \"ollama\"],\n help=\"Embedding backend mode\",\n )\n parser.add_argument(\n \"--distance-metric\",\n type=str,\n default=\"l2\",\n choices=[\"l2\", \"mips\", \"cosine\"],\n help=\"Distance metric for similarity computation\",\n )\n parser.add_argument(\n \"--enable-warmup\",\n action=\"store_true\",\n help=\"Preload model by running one warmup embedding at startup\",\n )\n parser.add_argument(\n \"--daemon-mode\",\n action=\"store_true\",\n help=\"Run as daemon mode (enables idle TTL checks)\",\n )\n parser.add_argument(\n \"--daemon-ttl\",\n type=int,\n default=0,\n help=\"Idle TTL in seconds for daemon mode; 0 disables auto-exit\",\n )\n\n args = parser.parse_args()\n\n # Create and start the DiskANN embedding server\n create_diskann_embedding_server(\n passages_file=args.passages_file,\n zmq_port=args.zmq_port,\n model_name=args.model_name,\n embedding_mode=args.embedding_mode,\n distance_metric=args.distance_metric,\n enable_warmup=args.enable_warmup,\n daemon_ttl=args.daemon_ttl if args.daemon_mode else 0,\n )\n" }, { "path": "packages/leann-backend-diskann/leann_backend_diskann/embedding_pb2.py", "content": "# Generated by the protocol buffer compiler. DO NOT EDIT!\n# source: embedding.proto\n# ruff: noqa\n\"\"\"Generated protocol buffer code.\"\"\"\n\nfrom google.protobuf import descriptor as _descriptor\nfrom google.protobuf import descriptor_pool as _descriptor_pool\nfrom google.protobuf import symbol_database as _symbol_database\nfrom google.protobuf.internal import builder as _builder\n\n# @@protoc_insertion_point(imports)\n\n_sym_db = _symbol_database.Default()\n\n\nDESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(\n b'\\n\\x0f\\x65mbedding.proto\\x12\\x0eprotoembedding\"(\\n\\x14NodeEmbeddingRequest\\x12\\x10\\n\\x08node_ids\\x18\\x01 \\x03(\\r\"Y\\n\\x15NodeEmbeddingResponse\\x12\\x17\\n\\x0f\\x65mbeddings_data\\x18\\x01 \\x01(\\x0c\\x12\\x12\\n\\ndimensions\\x18\\x02 \\x03(\\x05\\x12\\x13\\n\\x0bmissing_ids\\x18\\x03 \\x03(\\rb\\x06proto3'\n)\n\n_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())\n_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, \"embedding_pb2\", globals())\nif not _descriptor._USE_C_DESCRIPTORS:\n DESCRIPTOR._options = None\n _NODEEMBEDDINGREQUEST._serialized_start = 35\n _NODEEMBEDDINGREQUEST._serialized_end = 75\n _NODEEMBEDDINGRESPONSE._serialized_start = 77\n _NODEEMBEDDINGRESPONSE._serialized_end = 166\n# @@protoc_insertion_point(module_scope)\n" }, { "path": "packages/leann-backend-diskann/leann_backend_diskann/graph_partition.py", "content": "#!/usr/bin/env python3\n\"\"\"\nGraph Partition Module for LEANN DiskANN Backend\n\nThis module provides Python bindings for the graph partition functionality\nof DiskANN, allowing users to partition disk-based indices for better\nperformance.\n\"\"\"\n\nimport os\nimport shutil\nimport subprocess\nimport tempfile\nfrom pathlib import Path\nfrom typing import Optional\n\n\nclass GraphPartitioner:\n \"\"\"\n A Python interface for DiskANN's graph partition functionality.\n\n This class provides methods to partition disk-based indices for improved\n search performance and memory efficiency.\n \"\"\"\n\n def __init__(self, build_type: str = \"release\"):\n \"\"\"\n Initialize the GraphPartitioner.\n\n Args:\n build_type: Build type for the executables (\"debug\" or \"release\")\n \"\"\"\n self.build_type = build_type\n self._ensure_executables()\n\n def _get_executable_path(self, name: str) -> str:\n \"\"\"Get the path to a graph partition executable.\"\"\"\n # Get the directory where this Python module is located\n module_dir = Path(__file__).parent\n # Navigate to the graph_partition directory\n graph_partition_dir = module_dir.parent / \"third_party\" / \"DiskANN\" / \"graph_partition\"\n executable_path = graph_partition_dir / \"build\" / self.build_type / \"graph_partition\" / name\n\n if not executable_path.exists():\n raise FileNotFoundError(f\"Executable {name} not found at {executable_path}\")\n\n return str(executable_path)\n\n def _ensure_executables(self):\n \"\"\"Ensure that the required executables are built.\"\"\"\n try:\n self._get_executable_path(\"partitioner\")\n self._get_executable_path(\"index_relayout\")\n except FileNotFoundError:\n # Try to build the executables automatically\n print(\"Executables not found, attempting to build them...\")\n self._build_executables()\n\n def _build_executables(self):\n \"\"\"Build the required executables.\"\"\"\n graph_partition_dir = (\n Path(__file__).parent.parent / \"third_party\" / \"DiskANN\" / \"graph_partition\"\n )\n original_dir = os.getcwd()\n\n try:\n os.chdir(graph_partition_dir)\n\n # Clean any existing build\n if (graph_partition_dir / \"build\").exists():\n shutil.rmtree(graph_partition_dir / \"build\")\n\n # Run the build script\n cmd = [\"./build.sh\", self.build_type, \"split_graph\", \"/tmp/dummy\"]\n subprocess.run(cmd, capture_output=True, text=True, cwd=graph_partition_dir)\n\n # Check if executables were created\n partitioner_path = self._get_executable_path(\"partitioner\")\n relayout_path = self._get_executable_path(\"index_relayout\")\n\n print(f\"✅ Built partitioner: {partitioner_path}\")\n print(f\"✅ Built index_relayout: {relayout_path}\")\n\n except Exception as e:\n raise RuntimeError(f\"Failed to build executables: {e}\")\n finally:\n os.chdir(original_dir)\n\n def partition_graph(\n self,\n index_prefix_path: str,\n output_dir: Optional[str] = None,\n partition_prefix: Optional[str] = None,\n **kwargs,\n ) -> tuple[str, str]:\n \"\"\"\n Partition a disk-based index for improved performance.\n\n Args:\n index_prefix_path: Path to the index prefix (e.g., \"/path/to/index\")\n output_dir: Output directory for results (defaults to parent of index_prefix_path)\n partition_prefix: Prefix for output files (defaults to basename of index_prefix_path)\n **kwargs: Additional parameters for graph partitioning:\n - gp_times: Number of LDG partition iterations (default: 10)\n - lock_nums: Number of lock nodes (default: 10)\n - cut: Cut adjacency list degree (default: 100)\n - scale_factor: Scale factor (default: 1)\n - data_type: Data type (default: \"float\")\n - thread_nums: Number of threads (default: 10)\n\n Returns:\n Tuple of (disk_graph_index_path, partition_bin_path)\n\n Raises:\n RuntimeError: If the partitioning process fails\n \"\"\"\n # Set default parameters\n params = {\n \"gp_times\": 10,\n \"lock_nums\": 10,\n \"cut\": 100,\n \"scale_factor\": 1,\n \"data_type\": \"float\",\n \"thread_nums\": 10,\n **kwargs,\n }\n\n # Determine output directory\n if output_dir is None:\n output_dir = str(Path(index_prefix_path).parent)\n\n # Create output directory if it doesn't exist\n Path(output_dir).mkdir(parents=True, exist_ok=True)\n\n # Determine partition prefix\n if partition_prefix is None:\n partition_prefix = Path(index_prefix_path).name\n\n # Get executable paths\n partitioner_path = self._get_executable_path(\"partitioner\")\n relayout_path = self._get_executable_path(\"index_relayout\")\n\n # Create temporary directory for processing\n with tempfile.TemporaryDirectory() as temp_dir:\n # Change to the graph_partition directory for temporary files\n graph_partition_dir = (\n Path(__file__).parent.parent / \"third_party\" / \"DiskANN\" / \"graph_partition\"\n )\n original_dir = os.getcwd()\n\n try:\n os.chdir(graph_partition_dir)\n\n # Create temporary data directory\n temp_data_dir = Path(temp_dir) / \"data\"\n temp_data_dir.mkdir(parents=True, exist_ok=True)\n\n # Set up paths for temporary files\n graph_path = temp_data_dir / \"starling\" / \"_M_R_L_B\" / \"GRAPH\"\n graph_gp_path = (\n graph_path\n / f\"GP_TIMES_{params['gp_times']}_LOCK_{params['lock_nums']}_GP_USE_FREQ0_CUT{params['cut']}_SCALE{params['scale_factor']}\"\n )\n graph_gp_path.mkdir(parents=True, exist_ok=True)\n\n # Find input index file\n old_index_file = f\"{index_prefix_path}_disk_beam_search.index\"\n if not os.path.exists(old_index_file):\n old_index_file = f\"{index_prefix_path}_disk.index\"\n\n if not os.path.exists(old_index_file):\n raise RuntimeError(f\"Index file not found: {old_index_file}\")\n\n # Run partitioner\n gp_file_path = graph_gp_path / \"_part.bin\"\n partitioner_cmd = [\n partitioner_path,\n \"--index_file\",\n old_index_file,\n \"--data_type\",\n params[\"data_type\"],\n \"--gp_file\",\n str(gp_file_path),\n \"-T\",\n str(params[\"thread_nums\"]),\n \"--ldg_times\",\n str(params[\"gp_times\"]),\n \"--scale\",\n str(params[\"scale_factor\"]),\n \"--mode\",\n \"1\",\n ]\n\n print(f\"Running partitioner: {' '.join(partitioner_cmd)}\")\n result = subprocess.run(\n partitioner_cmd, capture_output=True, text=True, cwd=graph_partition_dir\n )\n\n if result.returncode != 0:\n raise RuntimeError(\n f\"Partitioner failed with return code {result.returncode}.\\n\"\n f\"stdout: {result.stdout}\\n\"\n f\"stderr: {result.stderr}\"\n )\n\n # Run relayout\n part_tmp_index = graph_gp_path / \"_part_tmp.index\"\n relayout_cmd = [\n relayout_path,\n old_index_file,\n str(gp_file_path),\n params[\"data_type\"],\n \"1\",\n ]\n\n print(f\"Running relayout: {' '.join(relayout_cmd)}\")\n result = subprocess.run(\n relayout_cmd, capture_output=True, text=True, cwd=graph_partition_dir\n )\n\n if result.returncode != 0:\n raise RuntimeError(\n f\"Relayout failed with return code {result.returncode}.\\n\"\n f\"stdout: {result.stdout}\\n\"\n f\"stderr: {result.stderr}\"\n )\n\n # Copy results to output directory\n disk_graph_path = Path(output_dir) / f\"{partition_prefix}_disk_graph.index\"\n partition_bin_path = Path(output_dir) / f\"{partition_prefix}_partition.bin\"\n\n shutil.copy2(part_tmp_index, disk_graph_path)\n shutil.copy2(gp_file_path, partition_bin_path)\n\n print(f\"Results copied to: {output_dir}\")\n return str(disk_graph_path), str(partition_bin_path)\n\n finally:\n os.chdir(original_dir)\n\n def get_partition_info(self, partition_bin_path: str) -> dict:\n \"\"\"\n Get information about a partition file.\n\n Args:\n partition_bin_path: Path to the partition binary file\n\n Returns:\n Dictionary containing partition information\n \"\"\"\n if not os.path.exists(partition_bin_path):\n raise FileNotFoundError(f\"Partition file not found: {partition_bin_path}\")\n\n # For now, return basic file information\n # In the future, this could parse the binary file for detailed info\n stat = os.stat(partition_bin_path)\n return {\n \"file_size\": stat.st_size,\n \"file_path\": partition_bin_path,\n \"modified_time\": stat.st_mtime,\n }\n\n\ndef partition_graph(\n index_prefix_path: str,\n output_dir: Optional[str] = None,\n partition_prefix: Optional[str] = None,\n build_type: str = \"release\",\n **kwargs,\n) -> tuple[str, str]:\n \"\"\"\n Convenience function to partition a graph index.\n\n Args:\n index_prefix_path: Path to the index prefix\n output_dir: Output directory (defaults to parent of index_prefix_path)\n partition_prefix: Prefix for output files (defaults to basename of index_prefix_path)\n build_type: Build type for executables (\"debug\" or \"release\")\n **kwargs: Additional parameters for graph partitioning\n\n Returns:\n Tuple of (disk_graph_index_path, partition_bin_path)\n \"\"\"\n partitioner = GraphPartitioner(build_type=build_type)\n return partitioner.partition_graph(index_prefix_path, output_dir, partition_prefix, **kwargs)\n\n\n# Example usage:\nif __name__ == \"__main__\":\n # Example: partition an index\n try:\n disk_graph_path, partition_bin_path = partition_graph(\n \"/path/to/your/index_prefix\", gp_times=10, lock_nums=10, cut=100\n )\n print(\"Partitioning completed successfully!\")\n print(f\"Disk graph index: {disk_graph_path}\")\n print(f\"Partition binary: {partition_bin_path}\")\n except Exception as e:\n print(f\"Partitioning failed: {e}\")\n" }, { "path": "packages/leann-backend-diskann/pyproject.toml", "content": "[build-system]\nrequires = [\"scikit-build-core>=0.10\", \"pybind11>=2.12.0\", \"numpy\", \"cmake>=3.30\"]\nbuild-backend = \"scikit_build_core.build\"\n\n[project]\nname = \"leann-backend-diskann\"\nversion = \"0.3.7\"\ndependencies = [\"leann-core==0.3.7\", \"numpy\", \"protobuf>=3.19.0\"]\n\n[tool.scikit-build]\n# Key: simplified CMake path\ncmake.source-dir = \"third_party/DiskANN\"\n# Key: Python package in root directory, paths match exactly\nwheel.packages = [\"leann_backend_diskann\"]\n# Use default redirect mode\neditable.mode = \"redirect\"\ncmake.build-type = \"Release\"\nbuild.verbose = true\n# Let CMake find packages via Homebrew prefix\ncmake.define = {CMAKE_PREFIX_PATH = {env = \"CMAKE_PREFIX_PATH\"}, OpenMP_ROOT = {env = \"OpenMP_ROOT\"}}\n" }, { "path": "packages/leann-backend-diskann/third_party/embedding.pb.cc", "content": "// Generated by the protocol buffer compiler. DO NOT EDIT!\n// source: embedding.proto\n\n#include \"embedding.pb.h\"\n\n#include \n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n// @@protoc_insertion_point(includes)\n#include \nnamespace protoembedding {\nclass NodeEmbeddingRequestDefaultTypeInternal {\n public:\n ::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed _instance;\n} _NodeEmbeddingRequest_default_instance_;\nclass NodeEmbeddingResponseDefaultTypeInternal {\n public:\n ::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed _instance;\n} _NodeEmbeddingResponse_default_instance_;\n} // namespace protoembedding\nstatic void InitDefaultsscc_info_NodeEmbeddingRequest_embedding_2eproto() {\n GOOGLE_PROTOBUF_VERIFY_VERSION;\n\n {\n void* ptr = &::protoembedding::_NodeEmbeddingRequest_default_instance_;\n new (ptr) ::protoembedding::NodeEmbeddingRequest();\n ::PROTOBUF_NAMESPACE_ID::internal::OnShutdownDestroyMessage(ptr);\n }\n ::protoembedding::NodeEmbeddingRequest::InitAsDefaultInstance();\n}\n\n::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_NodeEmbeddingRequest_embedding_2eproto =\n {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 0, 0, InitDefaultsscc_info_NodeEmbeddingRequest_embedding_2eproto}, {}};\n\nstatic void InitDefaultsscc_info_NodeEmbeddingResponse_embedding_2eproto() {\n GOOGLE_PROTOBUF_VERIFY_VERSION;\n\n {\n void* ptr = &::protoembedding::_NodeEmbeddingResponse_default_instance_;\n new (ptr) ::protoembedding::NodeEmbeddingResponse();\n ::PROTOBUF_NAMESPACE_ID::internal::OnShutdownDestroyMessage(ptr);\n }\n ::protoembedding::NodeEmbeddingResponse::InitAsDefaultInstance();\n}\n\n::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_NodeEmbeddingResponse_embedding_2eproto =\n {{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 0, 0, InitDefaultsscc_info_NodeEmbeddingResponse_embedding_2eproto}, {}};\n\nstatic ::PROTOBUF_NAMESPACE_ID::Metadata file_level_metadata_embedding_2eproto[2];\nstatic constexpr ::PROTOBUF_NAMESPACE_ID::EnumDescriptor const** file_level_enum_descriptors_embedding_2eproto = nullptr;\nstatic constexpr ::PROTOBUF_NAMESPACE_ID::ServiceDescriptor const** file_level_service_descriptors_embedding_2eproto = nullptr;\n\nconst ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_embedding_2eproto::offsets[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = {\n ~0u, // no _has_bits_\n PROTOBUF_FIELD_OFFSET(::protoembedding::NodeEmbeddingRequest, _internal_metadata_),\n ~0u, // no _extensions_\n ~0u, // no _oneof_case_\n ~0u, // no _weak_field_map_\n PROTOBUF_FIELD_OFFSET(::protoembedding::NodeEmbeddingRequest, node_ids_),\n ~0u, // no _has_bits_\n PROTOBUF_FIELD_OFFSET(::protoembedding::NodeEmbeddingResponse, _internal_metadata_),\n ~0u, // no _extensions_\n ~0u, // no _oneof_case_\n ~0u, // no _weak_field_map_\n PROTOBUF_FIELD_OFFSET(::protoembedding::NodeEmbeddingResponse, embeddings_data_),\n PROTOBUF_FIELD_OFFSET(::protoembedding::NodeEmbeddingResponse, dimensions_),\n PROTOBUF_FIELD_OFFSET(::protoembedding::NodeEmbeddingResponse, missing_ids_),\n};\nstatic const ::PROTOBUF_NAMESPACE_ID::internal::MigrationSchema schemas[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = {\n { 0, -1, sizeof(::protoembedding::NodeEmbeddingRequest)},\n { 6, -1, sizeof(::protoembedding::NodeEmbeddingResponse)},\n};\n\nstatic ::PROTOBUF_NAMESPACE_ID::Message const * const file_default_instances[] = {\n reinterpret_cast(&::protoembedding::_NodeEmbeddingRequest_default_instance_),\n reinterpret_cast(&::protoembedding::_NodeEmbeddingResponse_default_instance_),\n};\n\nconst char descriptor_table_protodef_embedding_2eproto[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) =\n \"\\n\\017embedding.proto\\022\\016protoembedding\\\"(\\n\\024Nod\"\n \"eEmbeddingRequest\\022\\020\\n\\010node_ids\\030\\001 \\003(\\r\\\"Y\\n\\025N\"\n \"odeEmbeddingResponse\\022\\027\\n\\017embeddings_data\\030\"\n \"\\001 \\001(\\014\\022\\022\\n\\ndimensions\\030\\002 \\003(\\005\\022\\023\\n\\013missing_ids\"\n \"\\030\\003 \\003(\\rb\\006proto3\"\n ;\nstatic const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable*const descriptor_table_embedding_2eproto_deps[1] = {\n};\nstatic ::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase*const descriptor_table_embedding_2eproto_sccs[2] = {\n &scc_info_NodeEmbeddingRequest_embedding_2eproto.base,\n &scc_info_NodeEmbeddingResponse_embedding_2eproto.base,\n};\nstatic ::PROTOBUF_NAMESPACE_ID::internal::once_flag descriptor_table_embedding_2eproto_once;\nconst ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable descriptor_table_embedding_2eproto = {\n false, false, descriptor_table_protodef_embedding_2eproto, \"embedding.proto\", 174,\n &descriptor_table_embedding_2eproto_once, descriptor_table_embedding_2eproto_sccs, descriptor_table_embedding_2eproto_deps, 2, 0,\n schemas, file_default_instances, TableStruct_embedding_2eproto::offsets,\n file_level_metadata_embedding_2eproto, 2, file_level_enum_descriptors_embedding_2eproto, file_level_service_descriptors_embedding_2eproto,\n};\n\n// Force running AddDescriptors() at dynamic initialization time.\nstatic bool dynamic_init_dummy_embedding_2eproto = (static_cast(::PROTOBUF_NAMESPACE_ID::internal::AddDescriptors(&descriptor_table_embedding_2eproto)), true);\nnamespace protoembedding {\n\n// ===================================================================\n\nvoid NodeEmbeddingRequest::InitAsDefaultInstance() {\n}\nclass NodeEmbeddingRequest::_Internal {\n public:\n};\n\nNodeEmbeddingRequest::NodeEmbeddingRequest(::PROTOBUF_NAMESPACE_ID::Arena* arena)\n : ::PROTOBUF_NAMESPACE_ID::Message(arena),\n node_ids_(arena) {\n SharedCtor();\n RegisterArenaDtor(arena);\n // @@protoc_insertion_point(arena_constructor:protoembedding.NodeEmbeddingRequest)\n}\nNodeEmbeddingRequest::NodeEmbeddingRequest(const NodeEmbeddingRequest& from)\n : ::PROTOBUF_NAMESPACE_ID::Message(),\n node_ids_(from.node_ids_) {\n _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_);\n // @@protoc_insertion_point(copy_constructor:protoembedding.NodeEmbeddingRequest)\n}\n\nvoid NodeEmbeddingRequest::SharedCtor() {\n}\n\nNodeEmbeddingRequest::~NodeEmbeddingRequest() {\n // @@protoc_insertion_point(destructor:protoembedding.NodeEmbeddingRequest)\n SharedDtor();\n _internal_metadata_.Delete<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>();\n}\n\nvoid NodeEmbeddingRequest::SharedDtor() {\n GOOGLE_DCHECK(GetArena() == nullptr);\n}\n\nvoid NodeEmbeddingRequest::ArenaDtor(void* object) {\n NodeEmbeddingRequest* _this = reinterpret_cast< NodeEmbeddingRequest* >(object);\n (void)_this;\n}\nvoid NodeEmbeddingRequest::RegisterArenaDtor(::PROTOBUF_NAMESPACE_ID::Arena*) {\n}\nvoid NodeEmbeddingRequest::SetCachedSize(int size) const {\n _cached_size_.Set(size);\n}\nconst NodeEmbeddingRequest& NodeEmbeddingRequest::default_instance() {\n ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&::scc_info_NodeEmbeddingRequest_embedding_2eproto.base);\n return *internal_default_instance();\n}\n\n\nvoid NodeEmbeddingRequest::Clear() {\n// @@protoc_insertion_point(message_clear_start:protoembedding.NodeEmbeddingRequest)\n ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0;\n // Prevent compiler warnings about cached_has_bits being unused\n (void) cached_has_bits;\n\n node_ids_.Clear();\n _internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>();\n}\n\nconst char* NodeEmbeddingRequest::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) {\n#define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure\n ::PROTOBUF_NAMESPACE_ID::Arena* arena = GetArena(); (void)arena;\n while (!ctx->Done(&ptr)) {\n ::PROTOBUF_NAMESPACE_ID::uint32 tag;\n ptr = ::PROTOBUF_NAMESPACE_ID::internal::ReadTag(ptr, &tag);\n CHK_(ptr);\n switch (tag >> 3) {\n // repeated uint32 node_ids = 1;\n case 1:\n if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 10)) {\n ptr = ::PROTOBUF_NAMESPACE_ID::internal::PackedUInt32Parser(_internal_mutable_node_ids(), ptr, ctx);\n CHK_(ptr);\n } else if (static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 8) {\n _internal_add_node_ids(::PROTOBUF_NAMESPACE_ID::internal::ReadVarint32(&ptr));\n CHK_(ptr);\n } else goto handle_unusual;\n continue;\n default: {\n handle_unusual:\n if ((tag & 7) == 4 || tag == 0) {\n ctx->SetLastTag(tag);\n goto success;\n }\n ptr = UnknownFieldParse(tag,\n _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(),\n ptr, ctx);\n CHK_(ptr != nullptr);\n continue;\n }\n } // switch\n } // while\nsuccess:\n return ptr;\nfailure:\n ptr = nullptr;\n goto success;\n#undef CHK_\n}\n\n::PROTOBUF_NAMESPACE_ID::uint8* NodeEmbeddingRequest::_InternalSerialize(\n ::PROTOBUF_NAMESPACE_ID::uint8* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const {\n // @@protoc_insertion_point(serialize_to_array_start:protoembedding.NodeEmbeddingRequest)\n ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0;\n (void) cached_has_bits;\n\n // repeated uint32 node_ids = 1;\n {\n int byte_size = _node_ids_cached_byte_size_.load(std::memory_order_relaxed);\n if (byte_size > 0) {\n target = stream->WriteUInt32Packed(\n 1, _internal_node_ids(), byte_size, target);\n }\n }\n\n if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) {\n target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::InternalSerializeUnknownFieldsToArray(\n _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance), target, stream);\n }\n // @@protoc_insertion_point(serialize_to_array_end:protoembedding.NodeEmbeddingRequest)\n return target;\n}\n\nsize_t NodeEmbeddingRequest::ByteSizeLong() const {\n// @@protoc_insertion_point(message_byte_size_start:protoembedding.NodeEmbeddingRequest)\n size_t total_size = 0;\n\n ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0;\n // Prevent compiler warnings about cached_has_bits being unused\n (void) cached_has_bits;\n\n // repeated uint32 node_ids = 1;\n {\n size_t data_size = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::\n UInt32Size(this->node_ids_);\n if (data_size > 0) {\n total_size += 1 +\n ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size(\n static_cast<::PROTOBUF_NAMESPACE_ID::int32>(data_size));\n }\n int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(data_size);\n _node_ids_cached_byte_size_.store(cached_size,\n std::memory_order_relaxed);\n total_size += data_size;\n }\n\n if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) {\n return ::PROTOBUF_NAMESPACE_ID::internal::ComputeUnknownFieldsSize(\n _internal_metadata_, total_size, &_cached_size_);\n }\n int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size);\n SetCachedSize(cached_size);\n return total_size;\n}\n\nvoid NodeEmbeddingRequest::MergeFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) {\n// @@protoc_insertion_point(generalized_merge_from_start:protoembedding.NodeEmbeddingRequest)\n GOOGLE_DCHECK_NE(&from, this);\n const NodeEmbeddingRequest* source =\n ::PROTOBUF_NAMESPACE_ID::DynamicCastToGenerated(\n &from);\n if (source == nullptr) {\n // @@protoc_insertion_point(generalized_merge_from_cast_fail:protoembedding.NodeEmbeddingRequest)\n ::PROTOBUF_NAMESPACE_ID::internal::ReflectionOps::Merge(from, this);\n } else {\n // @@protoc_insertion_point(generalized_merge_from_cast_success:protoembedding.NodeEmbeddingRequest)\n MergeFrom(*source);\n }\n}\n\nvoid NodeEmbeddingRequest::MergeFrom(const NodeEmbeddingRequest& from) {\n// @@protoc_insertion_point(class_specific_merge_from_start:protoembedding.NodeEmbeddingRequest)\n GOOGLE_DCHECK_NE(&from, this);\n _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_);\n ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0;\n (void) cached_has_bits;\n\n node_ids_.MergeFrom(from.node_ids_);\n}\n\nvoid NodeEmbeddingRequest::CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) {\n// @@protoc_insertion_point(generalized_copy_from_start:protoembedding.NodeEmbeddingRequest)\n if (&from == this) return;\n Clear();\n MergeFrom(from);\n}\n\nvoid NodeEmbeddingRequest::CopyFrom(const NodeEmbeddingRequest& from) {\n// @@protoc_insertion_point(class_specific_copy_from_start:protoembedding.NodeEmbeddingRequest)\n if (&from == this) return;\n Clear();\n MergeFrom(from);\n}\n\nbool NodeEmbeddingRequest::IsInitialized() const {\n return true;\n}\n\nvoid NodeEmbeddingRequest::InternalSwap(NodeEmbeddingRequest* other) {\n using std::swap;\n _internal_metadata_.Swap<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(&other->_internal_metadata_);\n node_ids_.InternalSwap(&other->node_ids_);\n}\n\n::PROTOBUF_NAMESPACE_ID::Metadata NodeEmbeddingRequest::GetMetadata() const {\n return GetMetadataStatic();\n}\n\n\n// ===================================================================\n\nvoid NodeEmbeddingResponse::InitAsDefaultInstance() {\n}\nclass NodeEmbeddingResponse::_Internal {\n public:\n};\n\nNodeEmbeddingResponse::NodeEmbeddingResponse(::PROTOBUF_NAMESPACE_ID::Arena* arena)\n : ::PROTOBUF_NAMESPACE_ID::Message(arena),\n dimensions_(arena),\n missing_ids_(arena) {\n SharedCtor();\n RegisterArenaDtor(arena);\n // @@protoc_insertion_point(arena_constructor:protoembedding.NodeEmbeddingResponse)\n}\nNodeEmbeddingResponse::NodeEmbeddingResponse(const NodeEmbeddingResponse& from)\n : ::PROTOBUF_NAMESPACE_ID::Message(),\n dimensions_(from.dimensions_),\n missing_ids_(from.missing_ids_) {\n _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_);\n embeddings_data_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());\n if (!from._internal_embeddings_data().empty()) {\n embeddings_data_.Set(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from._internal_embeddings_data(),\n GetArena());\n }\n // @@protoc_insertion_point(copy_constructor:protoembedding.NodeEmbeddingResponse)\n}\n\nvoid NodeEmbeddingResponse::SharedCtor() {\n ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&scc_info_NodeEmbeddingResponse_embedding_2eproto.base);\n embeddings_data_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());\n}\n\nNodeEmbeddingResponse::~NodeEmbeddingResponse() {\n // @@protoc_insertion_point(destructor:protoembedding.NodeEmbeddingResponse)\n SharedDtor();\n _internal_metadata_.Delete<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>();\n}\n\nvoid NodeEmbeddingResponse::SharedDtor() {\n GOOGLE_DCHECK(GetArena() == nullptr);\n embeddings_data_.DestroyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());\n}\n\nvoid NodeEmbeddingResponse::ArenaDtor(void* object) {\n NodeEmbeddingResponse* _this = reinterpret_cast< NodeEmbeddingResponse* >(object);\n (void)_this;\n}\nvoid NodeEmbeddingResponse::RegisterArenaDtor(::PROTOBUF_NAMESPACE_ID::Arena*) {\n}\nvoid NodeEmbeddingResponse::SetCachedSize(int size) const {\n _cached_size_.Set(size);\n}\nconst NodeEmbeddingResponse& NodeEmbeddingResponse::default_instance() {\n ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&::scc_info_NodeEmbeddingResponse_embedding_2eproto.base);\n return *internal_default_instance();\n}\n\n\nvoid NodeEmbeddingResponse::Clear() {\n// @@protoc_insertion_point(message_clear_start:protoembedding.NodeEmbeddingResponse)\n ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0;\n // Prevent compiler warnings about cached_has_bits being unused\n (void) cached_has_bits;\n\n dimensions_.Clear();\n missing_ids_.Clear();\n embeddings_data_.ClearToEmpty(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena());\n _internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>();\n}\n\nconst char* NodeEmbeddingResponse::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) {\n#define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure\n ::PROTOBUF_NAMESPACE_ID::Arena* arena = GetArena(); (void)arena;\n while (!ctx->Done(&ptr)) {\n ::PROTOBUF_NAMESPACE_ID::uint32 tag;\n ptr = ::PROTOBUF_NAMESPACE_ID::internal::ReadTag(ptr, &tag);\n CHK_(ptr);\n switch (tag >> 3) {\n // bytes embeddings_data = 1;\n case 1:\n if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 10)) {\n auto str = _internal_mutable_embeddings_data();\n ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParser(str, ptr, ctx);\n CHK_(ptr);\n } else goto handle_unusual;\n continue;\n // repeated int32 dimensions = 2;\n case 2:\n if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 18)) {\n ptr = ::PROTOBUF_NAMESPACE_ID::internal::PackedInt32Parser(_internal_mutable_dimensions(), ptr, ctx);\n CHK_(ptr);\n } else if (static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 16) {\n _internal_add_dimensions(::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr));\n CHK_(ptr);\n } else goto handle_unusual;\n continue;\n // repeated uint32 missing_ids = 3;\n case 3:\n if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 26)) {\n ptr = ::PROTOBUF_NAMESPACE_ID::internal::PackedUInt32Parser(_internal_mutable_missing_ids(), ptr, ctx);\n CHK_(ptr);\n } else if (static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 24) {\n _internal_add_missing_ids(::PROTOBUF_NAMESPACE_ID::internal::ReadVarint32(&ptr));\n CHK_(ptr);\n } else goto handle_unusual;\n continue;\n default: {\n handle_unusual:\n if ((tag & 7) == 4 || tag == 0) {\n ctx->SetLastTag(tag);\n goto success;\n }\n ptr = UnknownFieldParse(tag,\n _internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(),\n ptr, ctx);\n CHK_(ptr != nullptr);\n continue;\n }\n } // switch\n } // while\nsuccess:\n return ptr;\nfailure:\n ptr = nullptr;\n goto success;\n#undef CHK_\n}\n\n::PROTOBUF_NAMESPACE_ID::uint8* NodeEmbeddingResponse::_InternalSerialize(\n ::PROTOBUF_NAMESPACE_ID::uint8* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const {\n // @@protoc_insertion_point(serialize_to_array_start:protoembedding.NodeEmbeddingResponse)\n ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0;\n (void) cached_has_bits;\n\n // bytes embeddings_data = 1;\n if (this->embeddings_data().size() > 0) {\n target = stream->WriteBytesMaybeAliased(\n 1, this->_internal_embeddings_data(), target);\n }\n\n // repeated int32 dimensions = 2;\n {\n int byte_size = _dimensions_cached_byte_size_.load(std::memory_order_relaxed);\n if (byte_size > 0) {\n target = stream->WriteInt32Packed(\n 2, _internal_dimensions(), byte_size, target);\n }\n }\n\n // repeated uint32 missing_ids = 3;\n {\n int byte_size = _missing_ids_cached_byte_size_.load(std::memory_order_relaxed);\n if (byte_size > 0) {\n target = stream->WriteUInt32Packed(\n 3, _internal_missing_ids(), byte_size, target);\n }\n }\n\n if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) {\n target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::InternalSerializeUnknownFieldsToArray(\n _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance), target, stream);\n }\n // @@protoc_insertion_point(serialize_to_array_end:protoembedding.NodeEmbeddingResponse)\n return target;\n}\n\nsize_t NodeEmbeddingResponse::ByteSizeLong() const {\n// @@protoc_insertion_point(message_byte_size_start:protoembedding.NodeEmbeddingResponse)\n size_t total_size = 0;\n\n ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0;\n // Prevent compiler warnings about cached_has_bits being unused\n (void) cached_has_bits;\n\n // repeated int32 dimensions = 2;\n {\n size_t data_size = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::\n Int32Size(this->dimensions_);\n if (data_size > 0) {\n total_size += 1 +\n ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size(\n static_cast<::PROTOBUF_NAMESPACE_ID::int32>(data_size));\n }\n int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(data_size);\n _dimensions_cached_byte_size_.store(cached_size,\n std::memory_order_relaxed);\n total_size += data_size;\n }\n\n // repeated uint32 missing_ids = 3;\n {\n size_t data_size = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::\n UInt32Size(this->missing_ids_);\n if (data_size > 0) {\n total_size += 1 +\n ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size(\n static_cast<::PROTOBUF_NAMESPACE_ID::int32>(data_size));\n }\n int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(data_size);\n _missing_ids_cached_byte_size_.store(cached_size,\n std::memory_order_relaxed);\n total_size += data_size;\n }\n\n // bytes embeddings_data = 1;\n if (this->embeddings_data().size() > 0) {\n total_size += 1 +\n ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::BytesSize(\n this->_internal_embeddings_data());\n }\n\n if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) {\n return ::PROTOBUF_NAMESPACE_ID::internal::ComputeUnknownFieldsSize(\n _internal_metadata_, total_size, &_cached_size_);\n }\n int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size);\n SetCachedSize(cached_size);\n return total_size;\n}\n\nvoid NodeEmbeddingResponse::MergeFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) {\n// @@protoc_insertion_point(generalized_merge_from_start:protoembedding.NodeEmbeddingResponse)\n GOOGLE_DCHECK_NE(&from, this);\n const NodeEmbeddingResponse* source =\n ::PROTOBUF_NAMESPACE_ID::DynamicCastToGenerated(\n &from);\n if (source == nullptr) {\n // @@protoc_insertion_point(generalized_merge_from_cast_fail:protoembedding.NodeEmbeddingResponse)\n ::PROTOBUF_NAMESPACE_ID::internal::ReflectionOps::Merge(from, this);\n } else {\n // @@protoc_insertion_point(generalized_merge_from_cast_success:protoembedding.NodeEmbeddingResponse)\n MergeFrom(*source);\n }\n}\n\nvoid NodeEmbeddingResponse::MergeFrom(const NodeEmbeddingResponse& from) {\n// @@protoc_insertion_point(class_specific_merge_from_start:protoembedding.NodeEmbeddingResponse)\n GOOGLE_DCHECK_NE(&from, this);\n _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_);\n ::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0;\n (void) cached_has_bits;\n\n dimensions_.MergeFrom(from.dimensions_);\n missing_ids_.MergeFrom(from.missing_ids_);\n if (from.embeddings_data().size() > 0) {\n _internal_set_embeddings_data(from._internal_embeddings_data());\n }\n}\n\nvoid NodeEmbeddingResponse::CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) {\n// @@protoc_insertion_point(generalized_copy_from_start:protoembedding.NodeEmbeddingResponse)\n if (&from == this) return;\n Clear();\n MergeFrom(from);\n}\n\nvoid NodeEmbeddingResponse::CopyFrom(const NodeEmbeddingResponse& from) {\n// @@protoc_insertion_point(class_specific_copy_from_start:protoembedding.NodeEmbeddingResponse)\n if (&from == this) return;\n Clear();\n MergeFrom(from);\n}\n\nbool NodeEmbeddingResponse::IsInitialized() const {\n return true;\n}\n\nvoid NodeEmbeddingResponse::InternalSwap(NodeEmbeddingResponse* other) {\n using std::swap;\n _internal_metadata_.Swap<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(&other->_internal_metadata_);\n dimensions_.InternalSwap(&other->dimensions_);\n missing_ids_.InternalSwap(&other->missing_ids_);\n embeddings_data_.Swap(&other->embeddings_data_, &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArena());\n}\n\n::PROTOBUF_NAMESPACE_ID::Metadata NodeEmbeddingResponse::GetMetadata() const {\n return GetMetadataStatic();\n}\n\n\n// @@protoc_insertion_point(namespace_scope)\n} // namespace protoembedding\nPROTOBUF_NAMESPACE_OPEN\ntemplate<> PROTOBUF_NOINLINE ::protoembedding::NodeEmbeddingRequest* Arena::CreateMaybeMessage< ::protoembedding::NodeEmbeddingRequest >(Arena* arena) {\n return Arena::CreateMessageInternal< ::protoembedding::NodeEmbeddingRequest >(arena);\n}\ntemplate<> PROTOBUF_NOINLINE ::protoembedding::NodeEmbeddingResponse* Arena::CreateMaybeMessage< ::protoembedding::NodeEmbeddingResponse >(Arena* arena) {\n return Arena::CreateMessageInternal< ::protoembedding::NodeEmbeddingResponse >(arena);\n}\nPROTOBUF_NAMESPACE_CLOSE\n\n// @@protoc_insertion_point(global_scope)\n#include \n" }, { "path": "packages/leann-backend-diskann/third_party/embedding.proto", "content": "syntax = \"proto3\";\n\npackage protoembedding;\n\nmessage NodeEmbeddingRequest {\n repeated uint32 node_ids = 1;\n}\n\nmessage NodeEmbeddingResponse {\n bytes embeddings_data = 1; // All embedded binary datas\n repeated int32 dimensions = 2; // Shape [batch_size, embedding_dim]\n repeated uint32 missing_ids = 3; // Missing node ids\n}\n" }, { "path": "packages/leann-backend-hnsw/CMakeLists.txt", "content": "cmake_minimum_required(VERSION 3.24)\nproject(leann_backend_hnsw_wrapper)\nset(CMAKE_C_COMPILER_WORKS 1)\nset(CMAKE_CXX_COMPILER_WORKS 1)\n\n# Set OpenMP path for macOS\nif(APPLE)\n # Detect Homebrew installation path (Apple Silicon vs Intel)\n if(EXISTS \"/opt/homebrew/opt/libomp\")\n set(HOMEBREW_PREFIX \"/opt/homebrew\")\n elseif(EXISTS \"/usr/local/opt/libomp\")\n set(HOMEBREW_PREFIX \"/usr/local\")\n else()\n message(FATAL_ERROR \"Could not find libomp installation. Please install with: brew install libomp\")\n endif()\n\n set(OpenMP_C_FLAGS \"-Xpreprocessor -fopenmp -I${HOMEBREW_PREFIX}/opt/libomp/include\")\n set(OpenMP_CXX_FLAGS \"-Xpreprocessor -fopenmp -I${HOMEBREW_PREFIX}/opt/libomp/include\")\n set(OpenMP_C_LIB_NAMES \"omp\")\n set(OpenMP_CXX_LIB_NAMES \"omp\")\n set(OpenMP_omp_LIBRARY \"${HOMEBREW_PREFIX}/opt/libomp/lib/libomp.dylib\")\n\n # Force use of system libc++ to avoid version mismatch\n set(CMAKE_CXX_FLAGS \"${CMAKE_CXX_FLAGS} -stdlib=libc++\")\n set(CMAKE_EXE_LINKER_FLAGS \"${CMAKE_EXE_LINKER_FLAGS} -stdlib=libc++\")\n set(CMAKE_SHARED_LINKER_FLAGS \"${CMAKE_SHARED_LINKER_FLAGS} -stdlib=libc++\")\n\n # Set minimum macOS version for better compatibility\n set(CMAKE_OSX_DEPLOYMENT_TARGET \"11.0\" CACHE STRING \"Minimum macOS version\")\nendif()\n\n# Find ZMQ via pkg-config on all platforms.\n# Windows CI installs pkgconfiglite + zeromq via vcpkg and exports PKG_CONFIG_PATH.\nfind_package(PkgConfig REQUIRED)\n\n# On ARM64 macOS, ensure pkg-config finds ARM64 Homebrew packages first\nif(APPLE AND CMAKE_SYSTEM_PROCESSOR MATCHES \"aarch64|arm64\")\n set(ENV{PKG_CONFIG_PATH} \"/opt/homebrew/lib/pkgconfig:/opt/homebrew/share/pkgconfig:$ENV{PKG_CONFIG_PATH}\")\nendif()\n\npkg_check_modules(ZMQ REQUIRED IMPORTED_TARGET libzmq)\n\n# This creates PkgConfig::ZMQ target automatically with correct properties\nif(TARGET PkgConfig::ZMQ)\n message(STATUS \"Found and configured ZMQ target: PkgConfig::ZMQ\")\nelse()\n message(FATAL_ERROR \"pkg_check_modules did not create IMPORTED target for ZMQ.\")\nendif()\n\n# Add cppzmq headers\ninclude_directories(SYSTEM third_party/cppzmq)\n\n# Configure msgpack-c - disable boost dependency\nset(MSGPACK_USE_BOOST OFF CACHE BOOL \"\" FORCE)\nadd_compile_definitions(MSGPACK_NO_BOOST)\ninclude_directories(third_party/msgpack-c/include)\n\n# Faiss configuration - streamlined build\nset(FAISS_ENABLE_PYTHON ON CACHE BOOL \"\" FORCE)\nset(FAISS_ENABLE_GPU OFF CACHE BOOL \"\" FORCE)\nset(FAISS_ENABLE_EXTRAS OFF CACHE BOOL \"\" FORCE)\nset(BUILD_TESTING OFF CACHE BOOL \"\" FORCE)\nset(FAISS_ENABLE_C_API OFF CACHE BOOL \"\" FORCE)\nset(FAISS_OPT_LEVEL \"generic\" CACHE STRING \"\" FORCE)\n\n# Disable x86-specific SIMD optimizations (important for ARM64 compatibility)\nset(FAISS_ENABLE_AVX2 OFF CACHE BOOL \"\" FORCE)\nset(FAISS_ENABLE_AVX512 OFF CACHE BOOL \"\" FORCE)\nset(FAISS_ENABLE_SSE4_1 OFF CACHE BOOL \"\" FORCE)\n\n# ARM64-specific configuration\nif(CMAKE_SYSTEM_PROCESSOR MATCHES \"aarch64|arm64\")\n message(STATUS \"Configuring Faiss for ARM64 architecture\")\n\n if(CMAKE_SYSTEM_NAME STREQUAL \"Linux\")\n # Use SVE optimization level for ARM64 Linux (as seen in Faiss conda build)\n set(FAISS_OPT_LEVEL \"sve\" CACHE STRING \"\" FORCE)\n message(STATUS \"Setting FAISS_OPT_LEVEL to 'sve' for ARM64 Linux\")\n else()\n # Use generic optimization for other ARM64 platforms (like macOS)\n set(FAISS_OPT_LEVEL \"generic\" CACHE STRING \"\" FORCE)\n message(STATUS \"Setting FAISS_OPT_LEVEL to 'generic' for ARM64 ${CMAKE_SYSTEM_NAME}\")\n endif()\n\n # ARM64 compatibility: Faiss submodule has been modified to fix x86 header inclusion\n message(STATUS \"Using ARM64-compatible Faiss submodule\")\nendif()\n\n# Additional optimization options from INSTALL.md\nset(CMAKE_BUILD_TYPE \"Release\" CACHE STRING \"\" FORCE)\nset(BUILD_SHARED_LIBS OFF CACHE BOOL \"\" FORCE) # Static library is faster to build\n\n# Avoid building demos and benchmarks\nset(BUILD_DEMOS OFF CACHE BOOL \"\" FORCE)\nset(BUILD_BENCHS OFF CACHE BOOL \"\" FORCE)\n\n# NEW: Tell Faiss to only build the generic version\nset(FAISS_BUILD_GENERIC ON CACHE BOOL \"\" FORCE)\nset(FAISS_BUILD_AVX2 OFF CACHE BOOL \"\" FORCE)\nset(FAISS_BUILD_AVX512 OFF CACHE BOOL \"\" FORCE)\n\n# IMPORTANT: Disable building AVX versions to speed up compilation\nset(FAISS_BUILD_AVX_VERSIONS OFF CACHE BOOL \"\" FORCE)\n\nadd_subdirectory(third_party/faiss)\n" }, { "path": "packages/leann-backend-hnsw/leann_backend_hnsw/__init__.py", "content": "import os\nfrom pathlib import Path\n\n_DLL_DIR_HANDLES: list[object] = []\n\n\ndef _configure_windows_dll_search_path() -> None:\n \"\"\"Register vcpkg DLL directories so Windows can resolve native dependencies.\n\n Python 3.8+ no longer searches PATH for DLL dependencies of native\n extensions (see https://github.com/numpy/numpy/wiki/windows-dll-notes).\n The standard workaround is ``os.add_dll_directory``.\n\n This only matters for **CI builds and source installs** where C++ deps\n (OpenBLAS, ZeroMQ, protobuf, …) live in a vcpkg tree. Pre-built wheels\n shipped via PyPI bundle all required DLLs inside the wheel (via\n delvewheel), so end-users installing with ``pip install`` are unaffected.\n \"\"\"\n if os.name != \"nt\" or not hasattr(os, \"add_dll_directory\"):\n return\n\n candidate_dirs: list[Path] = []\n env_roots = [os.getenv(\"VCPKG_INSTALLATION_ROOT\"), os.getenv(\"VCPKG_ROOT\")]\n for root in env_roots:\n if not root:\n continue\n root_path = Path(root)\n candidate_dirs.extend(\n [\n root_path / \"installed\" / \"x64-windows\" / \"bin\",\n root_path / \"installed\" / \"x64-windows\" / \"debug\" / \"bin\",\n root_path / \"installed\" / \"x64-windows\" / \"tools\" / \"protobuf\",\n ]\n )\n\n for path_entry in os.environ.get(\"PATH\", \"\").split(\";\"):\n entry = path_entry.strip()\n if not entry:\n continue\n if \"vcpkg\" in entry.lower():\n candidate_dirs.append(Path(entry))\n\n seen: set[str] = set()\n for dll_dir in candidate_dirs:\n resolved = str(dll_dir).lower()\n if resolved in seen or not dll_dir.exists():\n continue\n seen.add(resolved)\n try:\n _DLL_DIR_HANDLES.append(os.add_dll_directory(str(dll_dir)))\n os.environ[\"PATH\"] = f\"{dll_dir};{os.environ.get('PATH', '')}\"\n except OSError:\n continue\n\n\n_configure_windows_dll_search_path()\n\nfrom . import hnsw_backend as hnsw_backend # noqa: E402\n" }, { "path": "packages/leann-backend-hnsw/leann_backend_hnsw/convert_to_csr.py", "content": "import argparse\nimport gc # Import garbage collector interface\nimport logging\nimport os\nimport struct\nimport sys\nimport time\nfrom dataclasses import dataclass\nfrom typing import Any, Optional\n\nimport numpy as np\n\n# Set up logging to avoid print buffer issues\nlogger = logging.getLogger(__name__)\nLOG_LEVEL = os.getenv(\"LEANN_LOG_LEVEL\", \"WARNING\").upper()\nlog_level = getattr(logging, LOG_LEVEL, logging.WARNING)\nlogger.setLevel(log_level)\n\n# --- FourCCs (add more if needed) ---\nINDEX_HNSW_FLAT_FOURCC = int.from_bytes(b\"IHNf\", \"little\")\n# Add other HNSW fourccs if you expect different storage types inside HNSW\n# INDEX_HNSW_PQ_FOURCC = int.from_bytes(b'IHNp', 'little')\n# INDEX_HNSW_SQ_FOURCC = int.from_bytes(b'IHNs', 'little')\n# INDEX_HNSW_CAGRA_FOURCC = int.from_bytes(b'IHNc', 'little') # Example\n\nEXPECTED_HNSW_FOURCCS = {INDEX_HNSW_FLAT_FOURCC} # Modify if needed\nNULL_INDEX_FOURCC = int.from_bytes(b\"null\", \"little\")\n\n# --- Helper functions for reading/writing binary data ---\n\n\ndef read_struct(f, fmt):\n \"\"\"Reads data according to the struct format.\"\"\"\n size = struct.calcsize(fmt)\n data = f.read(size)\n if len(data) != size:\n raise EOFError(\n f\"File ended unexpectedly reading struct fmt '{fmt}'. Expected {size} bytes, got {len(data)}.\"\n )\n return struct.unpack(fmt, data)[0]\n\n\ndef read_vector_raw(f, element_fmt_char):\n \"\"\"Reads a vector (size followed by data), returns count and raw bytes.\"\"\"\n count = -1 # Initialize count\n total_bytes = -1 # Initialize total_bytes\n try:\n count = read_struct(f, \" max_reasonable_count or count < 0:\n raise MemoryError(\n f\"Vector count {count} seems unreasonably large, possibly due to file corruption or incorrect format read.\"\n )\n\n total_bytes = count * element_size\n # --- FIX for MemoryError: Check for huge byte size before allocation ---\n max_reasonable_bytes = 50 * (1024**3) # ~50 GB limit\n if total_bytes > max_reasonable_bytes or total_bytes < 0: # Check for overflow\n raise MemoryError(\n f\"Attempting to read {total_bytes} bytes ({count} elements * {element_size} bytes/element), which exceeds the safety limit. File might be corrupted or format mismatch.\"\n )\n\n data_bytes = f.read(total_bytes)\n\n if len(data_bytes) != total_bytes:\n raise EOFError(\n f\"File ended unexpectedly reading vector data. Expected {total_bytes} bytes, got {len(data_bytes)}.\"\n )\n return count, data_bytes\n except (MemoryError, OverflowError) as e:\n # Add context to the error message\n print(\n f\"\\nError during raw vector read (element_fmt='{element_fmt_char}', count={count}, total_bytes={total_bytes}): {e}\",\n file=sys.stderr,\n )\n raise e # Re-raise the original error type\n\n\ndef read_numpy_vector(f, np_dtype, struct_fmt_char):\n \"\"\"Reads a vector into a NumPy array.\"\"\"\n count = -1 # Initialize count for robust error handling\n print(\n f\" Reading vector (dtype={np_dtype}, fmt='{struct_fmt_char}')... \",\n end=\"\",\n flush=True,\n )\n try:\n count, data_bytes = read_vector_raw(f, struct_fmt_char)\n print(f\"Count={count}, Bytes={len(data_bytes)}\")\n if count > 0 and len(data_bytes) > 0:\n arr = np.frombuffer(data_bytes, dtype=np_dtype)\n if arr.size != count:\n raise ValueError(\n f\"Inconsistent array size after reading. Expected {count}, got {arr.size}\"\n )\n return arr\n elif count == 0:\n return np.array([], dtype=np_dtype)\n else:\n raise ValueError(\"Read zero bytes but count > 0.\")\n except MemoryError as e:\n # Now count should be defined (or -1 if error was in read_struct)\n print(\n f\"\\nMemoryError creating NumPy array (dtype={np_dtype}, count={count}). {e}\",\n file=sys.stderr,\n )\n raise e\n except Exception as e: # Catch other potential errors like ValueError\n print(\n f\"\\nError reading numpy vector (dtype={np_dtype}, fmt='{struct_fmt_char}', count={count}): {e}\",\n file=sys.stderr,\n )\n raise e\n\n\ndef write_numpy_vector(f, arr, struct_fmt_char):\n \"\"\"Writes a NumPy array as a vector (size followed by data).\"\"\"\n count = arr.size\n f.write(struct.pack(\" 0 else 0\n\n\ndef write_compact_format(\n f_out,\n original_hnsw_data,\n assign_probas_np,\n cum_nneighbor_per_level_np,\n levels_np,\n compact_level_ptr,\n compact_node_offsets_np,\n compact_neighbors_data,\n storage_fourcc,\n storage_data,\n):\n \"\"\"Write HNSW data in compact format following C++ read order exactly.\"\"\"\n # Write IndexHNSW Header\n f_out.write(struct.pack(\" 1:\n f_out.write(struct.pack(\" HNSWComponents:\n original_hnsw_data: dict[str, Any] = {}\n\n hnsw_index_fourcc = read_struct(f, \" 1:\n original_hnsw_data[\"metric_arg\"] = read_struct(f, \" HNSWComponents:\n with open(path, \"rb\") as f:\n return _read_hnsw_structure(f)\n\n\ndef write_original_format(\n f_out,\n original_hnsw_data,\n assign_probas_np,\n cum_nneighbor_per_level_np,\n levels_np,\n offsets_np,\n neighbors_np,\n storage_fourcc,\n storage_data,\n):\n \"\"\"Write non-compact HNSW data in original FAISS order.\"\"\"\n\n f_out.write(struct.pack(\" 1:\n f_out.write(struct.pack(\" bool:\n \"\"\"Rewrite an HNSW index while dropping the embedded storage section.\"\"\"\n\n start_time = time.time()\n try:\n with open(input_filename, \"rb\") as f_in, open(output_filename, \"wb\") as f_out:\n original_hnsw_data: dict[str, Any] = {}\n\n hnsw_index_fourcc = read_struct(f_in, \" 1:\n original_hnsw_data[\"metric_arg\"] = read_struct(f_in, \" {output_filename}\")\n start_time = time.time()\n original_hnsw_data = {}\n neighbors_np = None # Initialize to allow check in finally block\n try:\n with open(input_filename, \"rb\") as f_in, open(output_filename, \"wb\") as f_out:\n # --- Read IndexHNSW FourCC and Header ---\n print(f\"[{time.time() - start_time:.2f}s] Reading Index HNSW header...\")\n # ... (Keep the header reading logic as before) ...\n hnsw_index_fourcc = read_struct(f_in, \" 1:\n original_hnsw_data[\"metric_arg\"] = read_struct(f_in, \" 0 else 0\n if neighbors_np.size != expected_neighbors_size:\n print(\n f\"Warning: neighbors vector size mismatch. Expected {expected_neighbors_size} based on offsets, got {neighbors_np.size}.\"\n )\n gc.collect()\n\n original_hnsw_data[\"entry_point\"] = read_struct(f_in, \" 0 and i % (ntotal // 100 or 1) == 0: # Log progress roughly every 1%\n progress = (i / ntotal) * 100\n elapsed = time.time() - start_time\n print(\n f\"\\r[{elapsed:.2f}s] Converting node {i}/{ntotal} ({progress:.1f}%)...\",\n end=\"\",\n )\n\n node_max_level = levels_np[i] - 1\n if node_max_level < -1:\n node_max_level = -1\n\n node_ptr_start_index = current_level_ptr_idx\n compact_node_offsets_np[i] = node_ptr_start_index\n\n original_offset_start = offsets_np[i]\n num_pointers_expected = (node_max_level + 1) + 1\n\n for level in range(node_max_level + 1):\n compact_level_ptr.append(current_data_idx)\n\n begin_orig_np = original_offset_start + get_cum_neighbors(\n cum_nneighbor_per_level_np, level\n )\n end_orig_np = original_offset_start + get_cum_neighbors(\n cum_nneighbor_per_level_np, level + 1\n )\n\n begin_orig = int(begin_orig_np)\n end_orig = int(end_orig_np)\n\n neighbors_len = len(neighbors_np) # Cache length\n begin_orig = min(max(0, begin_orig), neighbors_len)\n end_orig = min(max(begin_orig, end_orig), neighbors_len)\n\n if begin_orig < end_orig:\n # Slicing creates a copy, could be memory intensive for large M\n # Consider iterating if memory becomes an issue here\n level_neighbors_slice = neighbors_np[begin_orig:end_orig]\n valid_neighbors_mask = level_neighbors_slice >= 0\n num_valid = np.count_nonzero(valid_neighbors_mask)\n\n if num_valid > 0:\n # Append valid neighbors\n compact_neighbors_data.extend(\n level_neighbors_slice[valid_neighbors_mask]\n )\n current_data_idx += num_valid\n total_valid_neighbors_counted += num_valid\n\n compact_level_ptr.append(current_data_idx)\n current_level_ptr_idx += num_pointers_expected\n\n compact_node_offsets_np[ntotal] = current_level_ptr_idx\n print(\n f\"\\r[{time.time() - start_time:.2f}s] Conversion loop finished. \"\n ) # Clear progress line\n\n # --- Validation Checks ---\n print(f\"[{time.time() - start_time:.2f}s] Running validation checks...\")\n valid_check_passed = True\n # Check 1: Total valid neighbors count\n print(\" Checking total valid neighbor count...\")\n expected_valid_count = np.sum(neighbors_np >= 0)\n if total_valid_neighbors_counted != len(compact_neighbors_data):\n print(\n f\"Error: Mismatch between counted valid neighbors ({total_valid_neighbors_counted}) and final compact_data size ({len(compact_neighbors_data)})!\",\n file=sys.stderr,\n )\n valid_check_passed = False\n if expected_valid_count != len(compact_neighbors_data):\n print(\n f\"Error: Mismatch between NumPy count of valid neighbors ({expected_valid_count}) and final compact_data size ({len(compact_neighbors_data)})!\",\n file=sys.stderr,\n )\n valid_check_passed = False\n else:\n print(f\" OK: Total valid neighbors = {len(compact_neighbors_data)}\")\n\n # Check 2: Final pointer indices consistency\n print(\" Checking final pointer indices...\")\n if compact_node_offsets_np[ntotal] != len(compact_level_ptr):\n print(\n f\"Error: Final node offset ({compact_node_offsets_np[ntotal]}) doesn't match level_ptr size ({len(compact_level_ptr)})!\",\n file=sys.stderr,\n )\n valid_check_passed = False\n if (\n len(compact_level_ptr) > 0 and compact_level_ptr[-1] != len(compact_neighbors_data)\n ) or (len(compact_level_ptr) == 0 and len(compact_neighbors_data) != 0):\n last_ptr = compact_level_ptr[-1] if len(compact_level_ptr) > 0 else -1\n print(\n f\"Error: Last level pointer ({last_ptr}) doesn't match compact_data size ({len(compact_neighbors_data)})!\",\n file=sys.stderr,\n )\n valid_check_passed = False\n else:\n print(\" OK: Final pointers match data size.\")\n\n if not valid_check_passed:\n print(\n \"Error: Validation checks failed. Output file might be incorrect.\",\n file=sys.stderr,\n )\n # Optional: Exit here if validation fails\n # return False\n\n # --- Explicitly delete large intermediate arrays ---\n print(\n f\"[{time.time() - start_time:.2f}s] Deleting original neighbors and offsets arrays...\"\n )\n del neighbors_np\n del offsets_np\n gc.collect()\n\n print(\n f\" CSR Stats: |data|={len(compact_neighbors_data)}, |level_ptr|={len(compact_level_ptr)}\"\n )\n\n # --- Write CSR HNSW graph data using unified function ---\n print(\n f\"[{time.time() - start_time:.2f}s] Writing CSR HNSW graph data in FAISS-compatible order...\"\n )\n\n # Determine storage fourcc and data based on prune_embeddings\n if prune_embeddings:\n print(\" Pruning embeddings: Writing NULL storage marker.\")\n output_storage_fourcc = NULL_INDEX_FOURCC\n storage_data = b\"\"\n else:\n # Keep embeddings - read and preserve original storage data\n if storage_fourcc and storage_fourcc != NULL_INDEX_FOURCC:\n print(\" Preserving embeddings: Reading original storage data...\")\n storage_data = f_in.read() # Read remaining storage data\n output_storage_fourcc = storage_fourcc\n print(f\" Read {len(storage_data)} bytes of storage data\")\n else:\n print(\" No embeddings found in original file (NULL storage)\")\n output_storage_fourcc = NULL_INDEX_FOURCC\n storage_data = b\"\"\n\n # Use the unified write function\n write_compact_format(\n f_out,\n original_hnsw_data,\n assign_probas_np,\n cum_nneighbor_per_level_np,\n levels_np,\n compact_level_ptr,\n compact_node_offsets_np,\n compact_neighbors_data,\n output_storage_fourcc,\n storage_data,\n )\n\n # Clean up memory\n del assign_probas_np, cum_nneighbor_per_level_np, levels_np\n del compact_neighbors_data, compact_level_ptr, compact_node_offsets_np\n gc.collect()\n\n end_time = time.time()\n print(f\"[{end_time - start_time:.2f}s] Conversion complete.\")\n return True\n\n except FileNotFoundError:\n print(f\"Error: Input file not found: {input_filename}\", file=sys.stderr)\n return False\n except MemoryError as e:\n print(\n f\"\\nFatal MemoryError during conversion: {e}. Insufficient RAM.\",\n file=sys.stderr,\n )\n # Clean up potentially partially written output file?\n try:\n os.remove(output_filename)\n except OSError:\n pass\n return False\n except EOFError as e:\n print(\n f\"Error: Reached end of file unexpectedly reading {input_filename}. {e}\",\n file=sys.stderr,\n )\n try:\n os.remove(output_filename)\n except OSError:\n pass\n return False\n except Exception as e:\n print(f\"An unexpected error occurred during conversion: {e}\", file=sys.stderr)\n import traceback\n\n traceback.print_exc()\n try:\n os.remove(output_filename)\n except OSError:\n pass\n return False\n # Ensure neighbors_np is deleted even if an error occurs after its allocation\n finally:\n try:\n if \"neighbors_np\" in locals() and neighbors_np is not None:\n del neighbors_np\n gc.collect()\n except NameError:\n pass\n\n\ndef prune_hnsw_embeddings_inplace(index_filename: str) -> bool:\n \"\"\"Convenience wrapper to prune embeddings in-place.\"\"\"\n\n temp_path = f\"{index_filename}.prune.tmp\"\n success = prune_hnsw_embeddings(index_filename, temp_path)\n if success:\n try:\n os.replace(temp_path, index_filename)\n except Exception as exc: # pragma: no cover - defensive\n logger.error(f\"Failed to replace original index with pruned version: {exc}\")\n try:\n os.remove(temp_path)\n except OSError:\n pass\n return False\n else:\n try:\n os.remove(temp_path)\n except OSError:\n pass\n return success\n\n\n# --- Script Execution ---\nif __name__ == \"__main__\":\n parser = argparse.ArgumentParser(\n description=\"Convert a Faiss IndexHNSWFlat file to a CSR-based HNSW graph file.\"\n )\n parser.add_argument(\"input_index_file\", help=\"Path to the input IndexHNSWFlat file\")\n parser.add_argument(\n \"output_csr_graph_file\", help=\"Path to write the output CSR HNSW graph file\"\n )\n parser.add_argument(\n \"--prune-embeddings\",\n action=\"store_true\",\n default=True,\n help=\"Prune embedding storage (write NULL storage marker)\",\n )\n parser.add_argument(\n \"--keep-embeddings\",\n action=\"store_true\",\n help=\"Keep embedding storage (overrides --prune-embeddings)\",\n )\n\n args = parser.parse_args()\n\n if not os.path.exists(args.input_index_file):\n print(f\"Error: Input file not found: {args.input_index_file}\", file=sys.stderr)\n sys.exit(1)\n\n if os.path.abspath(args.input_index_file) == os.path.abspath(args.output_csr_graph_file):\n print(\"Error: Input and output filenames cannot be the same.\", file=sys.stderr)\n sys.exit(1)\n\n prune_embeddings = args.prune_embeddings and not args.keep_embeddings\n success = convert_hnsw_graph_to_csr(\n args.input_index_file, args.output_csr_graph_file, prune_embeddings\n )\n if not success:\n sys.exit(1)\n" }, { "path": "packages/leann-backend-hnsw/leann_backend_hnsw/hnsw_backend.py", "content": "import logging\nimport os\nimport shutil\nimport time\nfrom pathlib import Path\nfrom typing import Any, Literal, Optional\n\nimport numpy as np\nfrom leann.interface import (\n LeannBackendBuilderInterface,\n LeannBackendFactoryInterface,\n LeannBackendSearcherInterface,\n)\nfrom leann.registry import register_backend\nfrom leann.searcher_base import BaseSearcher\n\nfrom .convert_to_csr import convert_hnsw_graph_to_csr, prune_hnsw_embeddings_inplace\n\nlogger = logging.getLogger(__name__)\n\n\ndef get_metric_map():\n from . import faiss # type: ignore\n\n return {\n \"mips\": faiss.METRIC_INNER_PRODUCT,\n \"l2\": faiss.METRIC_L2,\n \"cosine\": faiss.METRIC_INNER_PRODUCT,\n }\n\n\ndef normalize_l2(data: np.ndarray) -> np.ndarray:\n norms = np.linalg.norm(data, axis=1, keepdims=True)\n norms[norms == 0] = 1 # Avoid division by zero\n return data / norms\n\n\n@register_backend(\"hnsw\")\nclass HNSWBackend(LeannBackendFactoryInterface):\n @staticmethod\n def builder(**kwargs) -> LeannBackendBuilderInterface:\n return HNSWBuilder(**kwargs)\n\n @staticmethod\n def searcher(index_path: str, **kwargs) -> LeannBackendSearcherInterface:\n return HNSWSearcher(index_path, **kwargs)\n\n\nclass HNSWBuilder(LeannBackendBuilderInterface):\n def __init__(self, **kwargs):\n self.build_params = kwargs.copy()\n self.is_compact = self.build_params.setdefault(\"is_compact\", True)\n self.is_recompute = self.build_params.setdefault(\"is_recompute\", True)\n self.M = self.build_params.setdefault(\"M\", 32)\n self.efConstruction = self.build_params.setdefault(\"efConstruction\", 200)\n self.distance_metric = self.build_params.setdefault(\"distance_metric\", \"mips\")\n self.dimensions = self.build_params.get(\"dimensions\")\n if not self.is_recompute and self.is_compact:\n # Auto-correct: non-recompute requires non-compact storage for HNSW\n logger.warning(\n \"is_recompute=False requires non-compact HNSW. Forcing is_compact=False.\"\n )\n self.is_compact = False\n self.build_params[\"is_compact\"] = False\n\n def build(self, data: np.ndarray, ids: list[str], index_path: str, **kwargs):\n from . import faiss # type: ignore\n\n path = Path(index_path)\n index_dir = path.parent\n index_prefix = path.stem\n index_dir.mkdir(parents=True, exist_ok=True)\n\n if data.dtype != np.float32:\n logger.warning(f\"Converting data to float32, shape: {data.shape}\")\n data = data.astype(np.float32)\n\n metric_enum = get_metric_map().get(self.distance_metric.lower())\n if metric_enum is None:\n raise ValueError(f\"Unsupported distance_metric '{self.distance_metric}'.\")\n\n dim = self.dimensions or data.shape[1]\n index = faiss.IndexHNSWFlat(dim, self.M, metric_enum)\n index.hnsw.efConstruction = self.efConstruction\n\n if self.distance_metric.lower() == \"cosine\":\n data = normalize_l2(data)\n\n index.add(data.shape[0], faiss.swig_ptr(data))\n index_file = index_dir / f\"{index_prefix}.index\"\n faiss.write_index(index, str(index_file))\n\n # Persist ID map so searcher can map FAISS integer labels back to passage IDs\n try:\n idmap_file = index_dir / f\"{index_prefix}.ids.txt\"\n with open(idmap_file, \"w\", encoding=\"utf-8\") as f:\n for id_str in ids:\n f.write(str(id_str) + \"\\n\")\n except Exception as e:\n logger.warning(f\"Failed to write ID map: {e}\")\n\n if self.is_compact:\n self._convert_to_csr(index_file)\n elif self.is_recompute:\n prune_hnsw_embeddings_inplace(str(index_file))\n\n def _convert_to_csr(self, index_file: Path):\n \"\"\"Convert built index to CSR format\"\"\"\n mode_str = \"CSR-pruned\" if self.is_recompute else \"CSR-standard\"\n logger.info(f\"INFO: Converting HNSW index to {mode_str} format...\")\n\n csr_temp_file = index_file.with_suffix(\".csr.tmp\")\n\n success = convert_hnsw_graph_to_csr(\n str(index_file), str(csr_temp_file), prune_embeddings=self.is_recompute\n )\n\n if success:\n logger.info(\"✅ CSR conversion successful.\")\n # index_file_old = index_file.with_suffix(\".old\")\n # shutil.move(str(index_file), str(index_file_old))\n shutil.move(str(csr_temp_file), str(index_file))\n logger.info(f\"INFO: Replaced original index with {mode_str} version at '{index_file}'\")\n else:\n # Clean up and fail fast\n if csr_temp_file.exists():\n os.remove(csr_temp_file)\n raise RuntimeError(\"CSR conversion failed - cannot proceed with compact format\")\n\n\nclass HNSWSearcher(BaseSearcher):\n def __init__(self, index_path: str, **kwargs):\n super().__init__(\n index_path,\n backend_module_name=\"leann_backend_hnsw.hnsw_embedding_server\",\n **kwargs,\n )\n from . import faiss # type: ignore\n\n self.distance_metric = (\n self.meta.get(\"backend_kwargs\", {}).get(\"distance_metric\", \"mips\").lower()\n )\n metric_enum = get_metric_map().get(self.distance_metric)\n if metric_enum is None:\n raise ValueError(f\"Unsupported distance_metric '{self.distance_metric}'.\")\n\n backend_meta_kwargs = self.meta.get(\"backend_kwargs\", {})\n self.is_compact = self.meta.get(\"is_compact\", backend_meta_kwargs.get(\"is_compact\", True))\n default_pruned = backend_meta_kwargs.get(\"is_recompute\", self.is_compact)\n self.is_pruned = bool(self.meta.get(\"is_pruned\", default_pruned))\n\n index_file = self.index_dir / f\"{self.index_path.stem}.index\"\n if not index_file.exists():\n raise FileNotFoundError(f\"HNSW index file not found at {index_file}\")\n\n hnsw_config = faiss.HNSWIndexConfig()\n hnsw_config.is_compact = self.is_compact\n hnsw_config.is_recompute = (\n self.is_pruned\n ) # In C++ code, it's called is_recompute, but it's only for loading IIUC.\n\n self._index = faiss.read_index(str(index_file), faiss.IO_FLAG_MMAP, hnsw_config)\n\n # Load ID map if available\n self._id_map: list[str] = []\n try:\n idmap_file = self.index_dir / f\"{self.index_path.stem}.ids.txt\"\n if idmap_file.exists():\n with open(idmap_file, encoding=\"utf-8\") as f:\n self._id_map = [line.rstrip(\"\\n\") for line in f]\n except Exception as e:\n logger.warning(f\"Failed to load ID map: {e}\")\n\n def search(\n self,\n query: np.ndarray,\n top_k: int,\n zmq_port: Optional[int] = None,\n complexity: int = 64,\n beam_width: int = 1,\n prune_ratio: float = 0.0,\n recompute_embeddings: bool = True,\n pruning_strategy: Literal[\"global\", \"local\", \"proportional\"] = \"global\",\n batch_size: int = 0,\n **kwargs,\n ) -> dict[str, Any]:\n \"\"\"\n Search for nearest neighbors using HNSW index.\n\n Args:\n query: Query vectors (B, D) where B is batch size, D is dimension\n top_k: Number of nearest neighbors to return\n complexity: Search complexity/efSearch, higher = more accurate but slower\n beam_width: Number of parallel search paths/beam_size\n prune_ratio: Ratio of neighbors to prune via PQ (0.0-1.0)\n recompute_embeddings: Whether to fetch fresh embeddings from server\n pruning_strategy: PQ candidate selection strategy:\n - \"global\": Use global PQ queue size for selection (default)\n - \"local\": Local pruning, sort and select best candidates\n - \"proportional\": Base selection on new neighbor count ratio\n zmq_port: ZMQ port for embedding server communication. Must be provided if recompute_embeddings is True.\n batch_size: Neighbor processing batch size, 0=disabled (HNSW-specific)\n **kwargs: Additional HNSW-specific parameters (for legacy compatibility)\n\n Returns:\n Dict with 'labels' (list of lists) and 'distances' (ndarray)\n \"\"\"\n from . import faiss # type: ignore\n\n if not recompute_embeddings and self.is_pruned:\n raise RuntimeError(\n \"Recompute is required for pruned/compact HNSW index. \"\n \"Re-run search with --recompute, or rebuild with --no-recompute and --no-compact.\"\n )\n if recompute_embeddings:\n if zmq_port is None:\n raise ValueError(\"zmq_port must be provided if recompute_embeddings is True\")\n if hasattr(self._index, \"set_zmq_port\"):\n self._index.set_zmq_port(zmq_port)\n\n if query.dtype != np.float32:\n query = query.astype(np.float32)\n if self.distance_metric == \"cosine\":\n query = normalize_l2(query)\n\n params = faiss.SearchParametersHNSW()\n if zmq_port is not None:\n params.zmq_port = zmq_port # C++ code won't use this if recompute_embeddings is False\n params.efSearch = complexity\n params.beam_size = beam_width\n\n # For OpenAI embeddings with cosine distance, disable relative distance check\n # This prevents early termination when all scores are in a narrow range\n embedding_model = self.meta.get(\"embedding_model\", \"\").lower()\n if self.distance_metric == \"cosine\" and any(\n openai_model in embedding_model for openai_model in [\"text-embedding\", \"openai\"]\n ):\n params.check_relative_distance = False\n else:\n params.check_relative_distance = True\n\n # PQ pruning: direct mapping to HNSW's pq_pruning_ratio\n params.pq_pruning_ratio = prune_ratio\n\n # Map pruning_strategy to HNSW parameters\n if pruning_strategy == \"local\":\n params.local_prune = True\n params.send_neigh_times_ratio = 0.0\n elif pruning_strategy == \"proportional\":\n params.local_prune = False\n params.send_neigh_times_ratio = 1.0 # Any value > 1e-6 triggers proportional mode\n else: # \"global\"\n params.local_prune = False\n params.send_neigh_times_ratio = 0.0\n\n # HNSW-specific batch processing parameter\n params.batch_size = batch_size\n\n batch_size_query = query.shape[0]\n distances = np.empty((batch_size_query, top_k), dtype=np.float32)\n labels = np.empty((batch_size_query, top_k), dtype=np.int64)\n\n search_time = time.time()\n self._index.search(\n query.shape[0],\n faiss.swig_ptr(query),\n top_k,\n faiss.swig_ptr(distances),\n faiss.swig_ptr(labels),\n params,\n )\n search_time = time.time() - search_time\n logger.info(f\" Search time in HNSWSearcher.search() backend: {search_time} seconds\")\n if self._id_map:\n\n def map_label(x: int) -> str:\n if 0 <= x < len(self._id_map):\n return self._id_map[x]\n return str(x)\n\n string_labels = [\n [map_label(int(label)) for label in batch_labels] for batch_labels in labels\n ]\n else:\n string_labels = [\n [str(int_label) for int_label in batch_labels] for batch_labels in labels\n ]\n\n return {\"labels\": string_labels, \"distances\": distances}\n" }, { "path": "packages/leann-backend-hnsw/leann_backend_hnsw/hnsw_embedding_server.py", "content": "\"\"\"\nHNSW-specific embedding server\n\"\"\"\n\nimport argparse\nimport json\nimport logging\nimport os\nimport sys\nimport threading\nimport time\nfrom pathlib import Path\nfrom typing import Any, Optional\n\nimport msgpack\nimport numpy as np\nimport zmq\n\n# Set up logging based on environment variable\nLOG_LEVEL = os.getenv(\"LEANN_LOG_LEVEL\", \"WARNING\").upper()\nlogger = logging.getLogger(__name__)\n\n# Force set logger level (don't rely on basicConfig in subprocess)\nlog_level = getattr(logging, LOG_LEVEL, logging.WARNING)\nlogger.setLevel(log_level)\n\n# Ensure we have handlers if none exist\nif not logger.handlers:\n stream_handler = logging.StreamHandler()\n formatter = logging.Formatter(\"%(asctime)s - %(levelname)s - %(message)s\")\n stream_handler.setFormatter(formatter)\n logger.addHandler(stream_handler)\n\nlog_path = os.getenv(\"LEANN_HNSW_LOG_PATH\")\nif log_path:\n try:\n file_handler = logging.FileHandler(log_path, mode=\"a\", encoding=\"utf-8\")\n file_formatter = logging.Formatter(\n \"%(asctime)s - %(levelname)s - [pid=%(process)d] %(message)s\"\n )\n file_handler.setFormatter(file_formatter)\n logger.addHandler(file_handler)\n except Exception as exc: # pragma: no cover - best effort logging\n logger.warning(f\"Failed to attach file handler for log path {log_path}: {exc}\")\n\nlogger.propagate = False\n\n_RAW_PROVIDER_OPTIONS = os.getenv(\"LEANN_EMBEDDING_OPTIONS\")\ntry:\n PROVIDER_OPTIONS: dict[str, Any] = (\n json.loads(_RAW_PROVIDER_OPTIONS) if _RAW_PROVIDER_OPTIONS else {}\n )\nexcept json.JSONDecodeError:\n logger.warning(\"Failed to parse LEANN_EMBEDDING_OPTIONS; ignoring provider options\")\n PROVIDER_OPTIONS = {}\n\n\ndef create_hnsw_embedding_server(\n passages_file: Optional[str] = None,\n zmq_port: int = 5555,\n model_name: str = \"sentence-transformers/all-mpnet-base-v2\",\n distance_metric: str = \"mips\",\n embedding_mode: str = \"sentence-transformers\",\n enable_warmup: bool = False,\n daemon_ttl: int = 0,\n):\n \"\"\"\n Create and start a ZMQ-based embedding server for HNSW backend.\n Simplified version using unified embedding computation module.\n \"\"\"\n logger.info(f\"Starting HNSW server on port {zmq_port} with model {model_name}\")\n logger.info(f\"Using embedding mode: {embedding_mode}\")\n\n # Add leann-core to path for unified embedding computation\n current_dir = Path(__file__).parent\n leann_core_path = current_dir.parent.parent / \"leann-core\" / \"src\"\n sys.path.insert(0, str(leann_core_path))\n\n try:\n from leann.api import PassageManager\n from leann.embedding_compute import compute_embeddings\n\n logger.info(\"Successfully imported unified embedding computation module\")\n except ImportError as e:\n logger.error(f\"Failed to import embedding computation module: {e}\")\n return\n finally:\n sys.path.pop(0)\n\n if enable_warmup:\n try:\n logger.info(\"Starting warmup embedding request...\")\n _ = compute_embeddings(\n [\"__LEANN_WARMUP__\"],\n model_name,\n mode=embedding_mode,\n provider_options=PROVIDER_OPTIONS,\n )\n logger.info(\"Warmup complete.\")\n except Exception as exc:\n logger.warning(f\"Warmup failed (continuing): {exc}\")\n\n # Check port availability\n import socket\n\n def check_port(port):\n with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:\n return s.connect_ex((\"localhost\", port)) == 0\n\n if check_port(zmq_port):\n logger.error(f\"Port {zmq_port} is already in use\")\n return\n\n # Only support metadata file, fail fast for everything else\n if not passages_file or not passages_file.endswith(\".meta.json\"):\n raise ValueError(\"Only metadata files (.meta.json) are supported\")\n\n # Load metadata to get passage sources\n with open(passages_file) as f:\n meta = json.load(f)\n\n # Let PassageManager handle path resolution uniformly. It supports fallback order:\n # 1) path/index_path; 2) *_relative; 3) standard siblings next to meta\n passages = PassageManager(meta[\"passage_sources\"], metadata_file_path=passages_file)\n # Dimension from metadata for shaping responses\n try:\n embedding_dim: int = int(meta.get(\"dimensions\", 0))\n except Exception:\n embedding_dim = 0\n logger.info(f\"Loaded PassageManager with {len(passages)} passages from metadata\")\n\n # Attempt to load ID map (maps FAISS integer labels -> passage IDs)\n id_map: list[str] = []\n try:\n meta_path = Path(passages_file)\n base = meta_path.name\n if base.endswith(\".meta.json\"):\n base = base[: -len(\".meta.json\")] # e.g., laion_index.leann\n if base.endswith(\".leann\"):\n base = base[: -len(\".leann\")] # e.g., laion_index\n idmap_file = meta_path.parent / f\"{base}.ids.txt\"\n if idmap_file.exists():\n with open(idmap_file, encoding=\"utf-8\") as f:\n id_map = [line.rstrip(\"\\n\") for line in f]\n logger.info(f\"Loaded ID map with {len(id_map)} entries from {idmap_file}\")\n else:\n logger.warning(f\"ID map file not found at {idmap_file}; will use raw labels\")\n except Exception as e:\n logger.warning(f\"Failed to load ID map: {e}\")\n\n def _map_node_id(nid) -> str:\n try:\n if id_map is not None and len(id_map) > 0 and isinstance(nid, (int, np.integer)):\n idx = int(nid)\n if 0 <= idx < len(id_map):\n return id_map[idx]\n except Exception:\n pass\n return str(nid)\n\n def zmq_server_thread_with_shutdown(shutdown_event):\n \"\"\"ZMQ server thread that respects shutdown signal.\n\n Creates its own REP socket bound to zmq_port and polls with timeouts\n to allow graceful shutdown.\n \"\"\"\n logger.info(\"ZMQ server thread started with shutdown support\")\n\n context = zmq.Context()\n rep_socket = context.socket(zmq.REP)\n rep_socket.bind(f\"tcp://*:{zmq_port}\")\n logger.info(f\"HNSW ZMQ REP server listening on port {zmq_port}\")\n rep_socket.setsockopt(zmq.RCVTIMEO, 1000)\n rep_socket.setsockopt(zmq.SNDTIMEO, 1000)\n rep_socket.setsockopt(zmq.LINGER, 0)\n\n last_request_type = \"unknown\"\n last_request_length = 0\n\n def _build_safe_fallback():\n if last_request_type == \"distance\":\n large_distance = 1e9\n fallback_len = max(0, int(last_request_length))\n return [[large_distance] * fallback_len]\n if last_request_type == \"embedding\":\n bsz = max(0, int(last_request_length))\n dim = max(0, int(embedding_dim))\n if dim > 0:\n return [[bsz, dim], [0.0] * (bsz * dim)]\n return [[0, 0], []]\n if last_request_type == \"text\":\n return []\n return [[0, int(embedding_dim) if embedding_dim > 0 else 0], []]\n\n def _handle_text_embedding(request: list[str]) -> None:\n nonlocal last_request_type, last_request_length\n\n e2e_start = time.time()\n last_request_type = \"text\"\n last_request_length = len(request)\n embeddings = compute_embeddings(\n request,\n model_name,\n mode=embedding_mode,\n provider_options=PROVIDER_OPTIONS,\n )\n rep_socket.send(msgpack.packb(embeddings.tolist()))\n e2e_end = time.time()\n logger.info(f\"⏱️ Direct text embedding E2E time: {e2e_end - e2e_start:.6f}s\")\n\n def _handle_distance_request(request: list[Any]) -> None:\n nonlocal last_request_type, last_request_length\n\n e2e_start = time.time()\n node_ids = request[0]\n if len(node_ids) == 1 and isinstance(node_ids[0], list):\n node_ids = node_ids[0]\n query_vector = np.array(request[1], dtype=np.float32)\n last_request_type = \"distance\"\n last_request_length = len(node_ids)\n\n logger.debug(\"Distance calculation request received\")\n logger.debug(f\" Node IDs: {node_ids}\")\n logger.debug(f\" Query vector dim: {len(query_vector)}\")\n\n texts: list[str] = []\n found_indices: list[int] = []\n for idx, nid in enumerate(node_ids):\n try:\n passage_id = _map_node_id(nid)\n passage_data = passages.get_passage(passage_id)\n txt = passage_data.get(\"text\", \"\")\n if isinstance(txt, str) and len(txt) > 0:\n texts.append(txt)\n found_indices.append(idx)\n else:\n logger.error(f\"Empty text for passage ID {passage_id}\")\n except KeyError:\n logger.error(f\"Passage ID {nid} not found\")\n except Exception as exc:\n logger.error(f\"Exception looking up passage ID {nid}: {exc}\")\n\n large_distance = 1e9\n response_distances = [large_distance] * len(node_ids)\n\n if texts:\n try:\n embeddings = compute_embeddings(\n texts,\n model_name,\n mode=embedding_mode,\n provider_options=PROVIDER_OPTIONS,\n )\n logger.info(\n f\"Computed embeddings for {len(texts)} texts, shape: {embeddings.shape}\"\n )\n if distance_metric == \"l2\":\n partial = np.sum(\n np.square(embeddings - query_vector.reshape(1, -1)), axis=1\n )\n else:\n partial = -np.dot(embeddings, query_vector)\n\n for pos, dval in zip(found_indices, partial.flatten().tolist()):\n response_distances[pos] = float(dval)\n except Exception as exc:\n logger.error(f\"Distance computation error, using sentinels: {exc}\")\n\n rep_socket.send(msgpack.packb([response_distances], use_single_float=True))\n e2e_end = time.time()\n logger.info(f\"⏱️ Distance calculation E2E time: {e2e_end - e2e_start:.6f}s\")\n\n def _handle_embedding_by_id(request: Any) -> None:\n nonlocal last_request_type, last_request_length\n\n if isinstance(request, list) and len(request) == 1 and isinstance(request[0], list):\n node_ids = request[0]\n elif isinstance(request, list):\n node_ids = request\n else:\n node_ids = []\n\n e2e_start = time.time()\n last_request_type = \"embedding\"\n last_request_length = len(node_ids)\n logger.info(f\"ZMQ received {len(node_ids)} node IDs for embedding fetch\")\n\n if embedding_dim <= 0:\n dims = [0, 0]\n flat_data: list[float] = []\n else:\n dims = [len(node_ids), embedding_dim]\n flat_data = [0.0] * (dims[0] * dims[1])\n\n texts: list[str] = []\n found_indices: list[int] = []\n for idx, nid in enumerate(node_ids):\n try:\n passage_id = _map_node_id(nid)\n passage_data = passages.get_passage(passage_id)\n txt = passage_data.get(\"text\", \"\")\n if isinstance(txt, str) and len(txt) > 0:\n texts.append(txt)\n found_indices.append(idx)\n else:\n logger.error(f\"Empty text for passage ID {passage_id}\")\n except KeyError:\n logger.error(f\"Passage with ID {nid} not found\")\n except Exception as exc:\n logger.error(f\"Exception looking up passage ID {nid}: {exc}\")\n\n if texts:\n try:\n embeddings = compute_embeddings(\n texts,\n model_name,\n mode=embedding_mode,\n provider_options=PROVIDER_OPTIONS,\n )\n logger.info(\n f\"Computed embeddings for {len(texts)} texts, shape: {embeddings.shape}\"\n )\n\n if np.isnan(embeddings).any() or np.isinf(embeddings).any():\n logger.error(\n f\"NaN or Inf detected in embeddings! Requested IDs: {node_ids[:5]}...\"\n )\n dims = [0, embedding_dim]\n flat_data = []\n else:\n emb_f32 = np.ascontiguousarray(embeddings, dtype=np.float32)\n flat = emb_f32.flatten().tolist()\n for j, pos in enumerate(found_indices):\n start = pos * embedding_dim\n end = start + embedding_dim\n if end <= len(flat_data):\n flat_data[start:end] = flat[\n j * embedding_dim : (j + 1) * embedding_dim\n ]\n except Exception as exc:\n logger.error(f\"Embedding computation error, returning zeros: {exc}\")\n\n response_payload = [dims, flat_data]\n rep_socket.send(msgpack.packb(response_payload, use_single_float=True))\n e2e_end = time.time()\n logger.info(f\"⏱️ Fallback Embed by Id E2E time: {e2e_end - e2e_start:.6f}s\")\n\n try:\n while not shutdown_event.is_set():\n try:\n logger.debug(\"🔍 Waiting for ZMQ message...\")\n request_bytes = rep_socket.recv()\n last_activity[0] = time.time()\n except zmq.Again:\n continue\n\n try:\n request = msgpack.unpackb(request_bytes)\n except Exception as exc:\n if shutdown_event.is_set():\n logger.info(\"Shutdown in progress, ignoring ZMQ error\")\n break\n logger.error(f\"Error unpacking ZMQ message: {exc}\")\n try:\n safe = _build_safe_fallback()\n rep_socket.send(msgpack.packb(safe, use_single_float=True))\n except Exception:\n pass\n continue\n\n try:\n # Model query\n if (\n isinstance(request, list)\n and len(request) == 1\n and request[0] == \"__QUERY_MODEL__\"\n ):\n rep_socket.send(msgpack.packb([model_name]))\n # Direct text embedding\n elif (\n isinstance(request, list)\n and request\n and all(isinstance(item, str) for item in request)\n ):\n _handle_text_embedding(request)\n # Distance calculation: [[ids], [query_vector]]\n elif (\n isinstance(request, list)\n and len(request) == 2\n and isinstance(request[0], list)\n and isinstance(request[1], list)\n ):\n _handle_distance_request(request)\n # Embedding-by-id fallback\n else:\n _handle_embedding_by_id(request)\n except Exception as exc:\n if shutdown_event.is_set():\n logger.info(\"Shutdown in progress, ignoring ZMQ error\")\n break\n logger.error(f\"Error in ZMQ server loop: {exc}\")\n try:\n safe = _build_safe_fallback()\n rep_socket.send(msgpack.packb(safe, use_single_float=True))\n except Exception:\n pass\n finally:\n try:\n rep_socket.close(0)\n except Exception:\n pass\n try:\n context.term()\n except Exception:\n pass\n\n logger.info(\"ZMQ server thread exiting gracefully\")\n\n # Add shutdown coordination\n shutdown_event = threading.Event()\n last_activity = [time.time()]\n\n def shutdown_zmq_server():\n \"\"\"Gracefully shutdown ZMQ server.\"\"\"\n logger.info(\"Initiating graceful shutdown...\")\n shutdown_event.set()\n\n if zmq_thread.is_alive():\n logger.info(\"Waiting for ZMQ thread to finish...\")\n zmq_thread.join(timeout=5)\n if zmq_thread.is_alive():\n logger.warning(\"ZMQ thread did not finish in time\")\n\n # Clean up ZMQ resources\n try:\n # Note: socket and context are cleaned up by thread exit\n logger.info(\"ZMQ resources cleaned up\")\n except Exception as e:\n logger.warning(f\"Error cleaning ZMQ resources: {e}\")\n\n # Clean up other resources\n try:\n import gc\n\n gc.collect()\n logger.info(\"Additional resources cleaned up\")\n except Exception as e:\n logger.warning(f\"Error cleaning additional resources: {e}\")\n\n logger.info(\"Graceful shutdown completed\")\n sys.exit(0)\n\n # Register signal handlers within this function scope\n import signal\n\n def signal_handler(sig, frame):\n logger.info(f\"Received signal {sig}, shutting down gracefully...\")\n shutdown_zmq_server()\n\n signal.signal(signal.SIGTERM, signal_handler)\n signal.signal(signal.SIGINT, signal_handler)\n\n # Pass shutdown_event to ZMQ thread\n zmq_thread = threading.Thread(\n target=lambda: zmq_server_thread_with_shutdown(shutdown_event),\n daemon=False, # Not daemon - we want to wait for it\n )\n zmq_thread.start()\n logger.info(f\"Started HNSW ZMQ server thread on port {zmq_port}\")\n\n # Keep the main thread alive\n try:\n while not shutdown_event.is_set():\n if daemon_ttl > 0 and (time.time() - last_activity[0]) >= daemon_ttl:\n logger.info(\n f\"No requests for {daemon_ttl} seconds, shutting down daemon on port {zmq_port}\"\n )\n shutdown_zmq_server()\n return\n time.sleep(0.1) # Check shutdown more frequently\n except KeyboardInterrupt:\n logger.info(\"HNSW Server shutting down...\")\n shutdown_zmq_server()\n return\n\n # If we reach here, shutdown was triggered by signal\n logger.info(\"Main loop exited, process should be shutting down\")\n\n\nif __name__ == \"__main__\":\n import sys\n\n # Signal handlers are now registered within create_hnsw_embedding_server\n\n parser = argparse.ArgumentParser(description=\"HNSW Embedding service\")\n parser.add_argument(\"--zmq-port\", type=int, default=5555, help=\"ZMQ port to run on\")\n parser.add_argument(\n \"--passages-file\",\n type=str,\n help=\"JSON file containing passage ID to text mapping\",\n )\n parser.add_argument(\n \"--model-name\",\n type=str,\n default=\"sentence-transformers/all-mpnet-base-v2\",\n help=\"Embedding model name\",\n )\n parser.add_argument(\n \"--distance-metric\", type=str, default=\"mips\", help=\"Distance metric to use\"\n )\n parser.add_argument(\n \"--embedding-mode\",\n type=str,\n default=\"sentence-transformers\",\n choices=[\"sentence-transformers\", \"openai\", \"mlx\", \"ollama\"],\n help=\"Embedding backend mode\",\n )\n parser.add_argument(\n \"--enable-warmup\",\n action=\"store_true\",\n help=\"Preload model by running one warmup embedding at startup\",\n )\n parser.add_argument(\n \"--daemon-mode\",\n action=\"store_true\",\n help=\"Run as daemon mode (enables idle TTL checks)\",\n )\n parser.add_argument(\n \"--daemon-ttl\",\n type=int,\n default=0,\n help=\"Idle TTL in seconds for daemon mode; 0 disables auto-exit\",\n )\n\n args = parser.parse_args()\n\n # Create and start the HNSW embedding server\n create_hnsw_embedding_server(\n passages_file=args.passages_file,\n zmq_port=args.zmq_port,\n model_name=args.model_name,\n distance_metric=args.distance_metric,\n embedding_mode=args.embedding_mode,\n enable_warmup=args.enable_warmup,\n daemon_ttl=args.daemon_ttl if args.daemon_mode else 0,\n )\n" }, { "path": "packages/leann-backend-hnsw/pyproject.toml", "content": "# packages/leann-backend-hnsw/pyproject.toml\n\n[build-system]\nrequires = [\"scikit-build-core>=0.10\", \"numpy\", \"swig\"]\nbuild-backend = \"scikit_build_core.build\"\n\n[project]\nname = \"leann-backend-hnsw\"\nversion = \"0.3.7\"\ndescription = \"Custom-built HNSW (Faiss) backend for the Leann toolkit.\"\ndependencies = [\n \"leann-core==0.3.7\",\n \"numpy\",\n \"pyzmq>=23.0.0\",\n \"msgpack>=1.0.0\",\n]\n\n[tool.scikit-build]\nwheel.packages = [\"leann_backend_hnsw\"]\neditable.mode = \"redirect\"\ncmake.build-type = \"Release\"\nbuild.verbose = true\n\n# CMake definitions to optimize compilation and find Homebrew packages\n[tool.scikit-build.cmake.define]\nCMAKE_PREFIX_PATH = {env = \"CMAKE_PREFIX_PATH\"}\nOpenMP_ROOT = {env = \"OpenMP_ROOT\"}\n" }, { "path": "packages/leann-backend-ivf/README.md", "content": "# LEANN IVF Backend\n\nFAISS **IndexIVFFlat** backend for LEANN with **add** and **delete** APIs for incremental updates (#231, #89, #141).\n\n## Install\n\n```bash\nuv sync --package leann-backend-ivf\n# or\npip install leann-backend-ivf\n```\n\nRequires `faiss-cpu`. For query embedding during search, the IVF searcher uses the same embedding server as HNSW; install `leann-backend-hnsw` if you use recompute/query embedding.\n\n## Build\n\n```bash\nleann build my-index --docs ./src --backend-name ivf\n```\n\nOptional backend kwargs (e.g. via API): `nlist` (default 100), `distance_metric` (\"l2\" or \"cosine\"/\"mips\").\n\n## Add / Delete API\n\nFollows the same idea as HNSW’s update path; use these for incremental workflows (e.g. after Merkle tree / file-change API):\n\n- **`add_vectors(index_path, embeddings, passage_ids)`**\n Appends vectors to an existing IVF index. `embeddings`: `(N, D)` float32; `passage_ids`: list of N passage id strings (must not already exist).\n\n- **`remove_ids(index_path, passage_ids)`**\n Removes vectors by passage id. Returns the number of vectors removed. Use after detecting changed chunk ids: delete old, then re-insert new.\n\nExample:\n\n```python\nfrom leann_backend_ivf import add_vectors, remove_ids\n\n# After file-change API says chunk id \"abc123\" changed:\nremove_ids(\"/path/to/index.leann\", [\"abc123\"])\n# Re-embed and add new chunk with same or new id\nadd_vectors(\"/path/to/index.leann\", new_embeddings, [\"abc123\"])\n```\n\n## Core integration (#89, #141)\n\n`LeannBuilder.update_index(index_path, remove_passage_ids=None)` in leann-core supports IVF: it calls `add_vectors` for appends and, when `remove_passage_ids` is set, calls `remove_ids` first (e.g. from a file-change / Merkle API). Use for incremental build or reindex: detect changed chunk ids → `update_index(path, remove_passage_ids=changed_ids)` then add chunks for the new content.\n\n## Notes\n\n- IVF stores vectors in flat lists per cluster; no compact/recompute mode like HNSW. Good for frequent add/delete.\n- ID mapping is stored in `ivf_id_map.json` next to the index; passage ids are stable across add/delete.\n- Search uses `nprobe` (default from `complexity`, capped by `nlist`). Tune `nlist` at build time for speed/recall.\n" }, { "path": "packages/leann-backend-ivf/leann_backend_ivf/__init__.py", "content": "\"\"\"LEANN IVF backend: FAISS IndexIVFFlat with add/delete APIs for incremental updates.\"\"\"\n\nfrom .ivf_backend import (\n IVFBackend,\n IVFBuilder,\n IVFSearcher,\n add_vectors,\n remove_ids,\n)\n\n__all__ = [\n \"IVFBackend\",\n \"IVFBuilder\",\n \"IVFSearcher\",\n \"add_vectors\",\n \"remove_ids\",\n]\n" }, { "path": "packages/leann-backend-ivf/leann_backend_ivf/ivf_backend.py", "content": "\"\"\"\nIVF backend: FAISS IndexIVFFlat with DirectMap.Hashtable for add/remove by passage id.\n\nUses IndexIVFFlat + DirectMap.Hashtable (no IndexIDMap2) so remove_ids works correctly.\nProvides add_vectors() and remove_ids() for incremental updates.\n\"\"\"\n\nimport json\nimport logging\nfrom pathlib import Path\nfrom typing import Any, Optional\n\nimport numpy as np\n\ntry:\n import faiss\nexcept ImportError:\n faiss = None # type: ignore[assignment]\n\nfrom leann.interface import (\n LeannBackendBuilderInterface,\n LeannBackendFactoryInterface,\n LeannBackendSearcherInterface,\n)\nfrom leann.registry import register_backend\nfrom leann.searcher_base import BaseSearcher\n\nlogger = logging.getLogger(__name__)\n\nID_MAP_FILENAME = \"ivf_id_map.json\"\n\n\ndef _check_faiss():\n if faiss is None:\n raise ImportError(\n \"faiss-cpu is required for IVF backend. Install with: pip install faiss-cpu\"\n )\n\n\ndef _get_metric_map():\n _check_faiss()\n return {\n \"mips\": faiss.METRIC_INNER_PRODUCT,\n \"l2\": faiss.METRIC_L2,\n \"cosine\": faiss.METRIC_INNER_PRODUCT,\n }\n\n\ndef _normalize_l2(data: np.ndarray) -> np.ndarray:\n norms = np.linalg.norm(data, axis=1, keepdims=True)\n norms[norms == 0] = 1\n return data / norms\n\n\ndef _load_id_map(index_dir: Path, index_prefix: str) -> tuple[dict[int, str], dict[str, int], int]:\n \"\"\"Load id_map.json. Returns (id_to_passage, passage_to_id, next_id).\"\"\"\n path = index_dir / f\"{index_prefix}.{ID_MAP_FILENAME}\"\n if not path.exists():\n return {}, {}, 0\n with open(path, encoding=\"utf-8\") as f:\n data = json.load(f)\n id_to_passage = {int(k): v for k, v in data.get(\"id_to_passage\", {}).items()}\n passage_to_id = data.get(\"passage_to_id\", {})\n next_id = int(data.get(\"next_id\", 0))\n return id_to_passage, passage_to_id, next_id\n\n\ndef _save_id_map(\n index_dir: Path, index_prefix: str, id_to_passage: dict[int, str], next_id: int\n) -> None:\n passage_to_id = {v: k for k, v in id_to_passage.items()}\n path = index_dir / f\"{index_prefix}.{ID_MAP_FILENAME}\"\n with open(path, \"w\", encoding=\"utf-8\") as f:\n json.dump(\n {\n \"id_to_passage\": {str(k): v for k, v in id_to_passage.items()},\n \"passage_to_id\": passage_to_id,\n \"next_id\": next_id,\n },\n f,\n indent=2,\n )\n\n\n@register_backend(\"ivf\")\nclass IVFBackend(LeannBackendFactoryInterface):\n @staticmethod\n def builder(**kwargs) -> LeannBackendBuilderInterface:\n return IVFBuilder(**kwargs)\n\n @staticmethod\n def searcher(index_path: str, **kwargs) -> LeannBackendSearcherInterface:\n return IVFSearcher(index_path, **kwargs)\n\n\nclass IVFBuilder(LeannBackendBuilderInterface):\n def __init__(self, **kwargs):\n _check_faiss()\n self.build_params = kwargs.copy()\n self.nlist = self.build_params.setdefault(\"nlist\", 100)\n self.distance_metric = self.build_params.setdefault(\"distance_metric\", \"l2\")\n self.dimensions = self.build_params.get(\"dimensions\")\n\n def build(self, data: np.ndarray, ids: list[str], index_path: str, **kwargs) -> None:\n _check_faiss()\n path = Path(index_path)\n index_dir = path.parent\n index_prefix = path.stem\n index_dir.mkdir(parents=True, exist_ok=True)\n\n if data.dtype != np.float32:\n data = data.astype(np.float32)\n data = np.ascontiguousarray(data)\n dim = self.dimensions or data.shape[1]\n n = data.shape[0]\n metric_enum = _get_metric_map().get(self.distance_metric.lower())\n if metric_enum is None:\n raise ValueError(f\"Unsupported distance_metric '{self.distance_metric}'.\")\n\n if self.distance_metric.lower() == \"cosine\":\n data = _normalize_l2(data)\n\n quantizer = (\n faiss.IndexFlatL2(dim) if metric_enum == faiss.METRIC_L2 else faiss.IndexFlatIP(dim)\n )\n ivf = faiss.IndexIVFFlat(quantizer, dim, self.nlist, metric_enum)\n ivf.train(data)\n ivf.set_direct_map_type(faiss.DirectMap.Hashtable)\n faiss_ids = np.arange(n, dtype=np.int64)\n ivf.add_with_ids(data, faiss_ids)\n\n index_file = index_dir / f\"{index_prefix}.index\"\n faiss.write_index(ivf, str(index_file))\n\n id_to_passage = dict(enumerate(ids))\n _save_id_map(index_dir, index_prefix, id_to_passage, next_id=n)\n\n\nclass IVFSearcher(BaseSearcher):\n def __init__(self, index_path: str, **kwargs):\n # Use HNSW embedding server for query embedding if available (same as other backends)\n super().__init__(\n index_path,\n backend_module_name=\"leann_backend_hnsw.hnsw_embedding_server\",\n **kwargs,\n )\n _check_faiss()\n self.distance_metric = (\n self.meta.get(\"backend_kwargs\", {}).get(\"distance_metric\", \"l2\").lower()\n )\n index_prefix = self.index_path.stem\n index_file = self.index_dir / f\"{index_prefix}.index\"\n if not index_file.exists():\n raise FileNotFoundError(f\"IVF index file not found at {index_file}\")\n\n self._index = faiss.read_index(str(index_file))\n self._id_to_passage: dict[int, str] = {}\n id_to_passage, _, _ = _load_id_map(self.index_dir, index_prefix)\n self._id_to_passage = id_to_passage\n\n def search(\n self,\n query: np.ndarray,\n top_k: int,\n complexity: int = 64,\n nprobe: Optional[int] = None,\n **kwargs,\n ) -> dict[str, Any]:\n _check_faiss()\n if query.dtype != np.float32:\n query = query.astype(np.float32)\n if self.distance_metric == \"cosine\":\n query = _normalize_l2(query)\n ivf_index = faiss.extract_index_ivf(self._index)\n nprobe = nprobe or min(complexity, ivf_index.nlist)\n ivf_index.nprobe = nprobe\n distances, label_rows = self._index.search(query, top_k)\n\n def map_label(x: int) -> str:\n return self._id_to_passage.get(int(x), str(x))\n\n string_labels = [[map_label(int(lab)) for lab in row] for row in label_rows]\n return {\"labels\": string_labels, \"distances\": distances}\n\n def compute_query_embedding(\n self,\n query: str,\n use_server_if_available: bool = True,\n zmq_port: Optional[int] = None,\n query_template: Optional[str] = None,\n ) -> np.ndarray:\n return super().compute_query_embedding(\n query,\n use_server_if_available=use_server_if_available,\n zmq_port=zmq_port,\n query_template=query_template,\n )\n\n\ndef add_vectors(index_path: str, embeddings: np.ndarray, passage_ids: list[str]) -> None:\n \"\"\"\n Append vectors to an existing IVF index (same role as HNSW update_index add path).\n\n Args:\n index_path: Path to the .leann index (e.g. .../documents.leann).\n embeddings: (N, D) float32 array.\n passage_ids: List of N passage id strings (must not already exist in index).\n \"\"\"\n _check_faiss()\n path = Path(index_path)\n index_dir = path.parent\n index_prefix = path.stem\n index_file = index_dir / f\"{index_prefix}.index\"\n if not index_file.exists():\n raise FileNotFoundError(f\"IVF index not found: {index_file}\")\n\n embeddings = np.ascontiguousarray(embeddings.astype(np.float32))\n id_to_passage, passage_to_id, next_id = _load_id_map(index_dir, index_prefix)\n for pid in passage_ids:\n if pid in passage_to_id:\n raise ValueError(f\"Passage id '{pid}' already exists in index.\")\n\n n = embeddings.shape[0]\n if n != len(passage_ids):\n raise ValueError(\"embeddings.shape[0] must equal len(passage_ids).\")\n\n index = faiss.read_index(str(index_file))\n new_ids = np.arange(next_id, next_id + n, dtype=np.int64)\n index.add_with_ids(embeddings, new_ids)\n faiss.write_index(index, str(index_file))\n\n for i, pid in enumerate(passage_ids):\n id_to_passage[next_id + i] = pid\n _save_id_map(index_dir, index_prefix, id_to_passage, next_id=next_id + n)\n logger.info(\"IVF add_vectors: appended %d vectors, next_id=%d\", n, next_id + n)\n\n\ndef remove_ids(index_path: str, passage_ids: list[str]) -> int:\n \"\"\"\n Remove vectors by passage id (for incremental update: delete changed chunks before re-insert).\n\n Args:\n index_path: Path to the .leann index.\n passage_ids: List of passage id strings to remove.\n\n Returns:\n Number of vectors actually removed.\n \"\"\"\n _check_faiss()\n path = Path(index_path)\n index_dir = path.parent\n index_prefix = path.stem\n index_file = index_dir / f\"{index_prefix}.index\"\n if not index_file.exists():\n raise FileNotFoundError(f\"IVF index not found: {index_file}\")\n\n id_to_passage, passage_to_id, next_id = _load_id_map(index_dir, index_prefix)\n to_remove_int: list[int] = []\n for pid in passage_ids:\n if pid in passage_to_id:\n to_remove_int.append(passage_to_id[pid])\n if not to_remove_int:\n return 0\n\n index = faiss.read_index(str(index_file))\n ntotal_before = index.ntotal\n sel = np.array(to_remove_int, dtype=np.int64)\n nremoved = index.remove_ids(sel)\n faiss.write_index(index, str(index_file))\n\n for pid in passage_ids:\n if pid in passage_to_id:\n i = passage_to_id[pid]\n id_to_passage.pop(i, None)\n # passage_to_id is rebuilt from id_to_passage when we save; we don't decrease next_id so new adds get new ids\n _save_id_map(index_dir, index_prefix, id_to_passage, next_id=next_id)\n logger.info(\n \"IVF remove_ids: ntotal %d -> %d, removed %d vectors (requested %d, found %d in id_map)\",\n ntotal_before,\n index.ntotal,\n nremoved,\n len(passage_ids),\n len(to_remove_int),\n )\n if nremoved != len(to_remove_int):\n logger.warning(\n \"IVF remove_ids: FAISS removed %d but expected %d. Possible index inconsistency.\",\n nremoved,\n len(to_remove_int),\n )\n return nremoved\n" }, { "path": "packages/leann-backend-ivf/pyproject.toml", "content": "[project]\nname = \"leann-backend-ivf\"\nversion = \"0.3.6\"\ndescription = \"FAISS IVF backend for LEANN with add/delete support for incremental updates.\"\nreadme = \"README.md\"\nrequires-python = \">=3.10\"\ndependencies = [\n \"leann-core>=0.3.6\",\n \"numpy>=1.20.0\",\n \"faiss-cpu>=1.7.4\",\n]\n\n[project.optional-dependencies]\n# Optional: use HNSW embedding server for query embedding (same as other backends)\nquery-server = [\"leann-backend-hnsw>=0.3.6\", \"pyzmq>=23.0.0\", \"msgpack>=1.0.0\"]\n\n[tool.setuptools.packages.find]\nwhere = [\".\"]\ninclude = [\"leann_backend_ivf*\"]\n" }, { "path": "packages/leann-core/pyproject.toml", "content": "[build-system]\nrequires = [\"setuptools>=61.0\"]\nbuild-backend = \"setuptools.build_meta\"\n\n[project]\nname = \"leann-core\"\nversion = \"0.3.7\"\ndescription = \"Core API and plugin system for LEANN\"\nreadme = \"README.md\"\nrequires-python = \">=3.10\"\nlicense = { text = \"MIT\" }\n\n# All required dependencies included\ndependencies = [\n \"numpy>=1.20.0,<2; python_version < '3.13'\",\n \"numpy>=2,<3; python_version >= '3.13'\",\n \"tqdm>=4.60.0\",\n \"psutil>=5.8.0\",\n \"pyzmq>=23.0.0\",\n \"msgpack>=1.0.0\",\n \"torch>=2.0.0\",\n \"sentence-transformers>=3.0.0\",\n \"llama-index-core>=0.12.0\",\n \"llama-index-readers-file>=0.4.0\", # Essential for document reading\n \"llama-index-embeddings-huggingface>=0.5.5\", # For embeddings\n \"python-dotenv>=1.0.0\",\n \"openai>=1.0.0\",\n \"huggingface-hub>=0.20.0\",\n # Keep Py3.9 compatible below 4.46; allow newer for Py>=3.10 (e.g., ColQwen/ColPali).\n \"transformers>=4.30.0,<4.46; python_version < '3.10'\",\n \"transformers>=4.53.1,<4.58; python_version >= '3.10'\",\n \"requests>=2.25.0\",\n \"accelerate>=0.20.0\",\n \"PyPDF2>=3.0.0\",\n \"pymupdf>=1.23.0\",\n \"pdfplumber>=0.10.0\",\n \"nbconvert>=7.0.0\", # For .ipynb file support\n \"gitignore-parser>=0.1.12\", # For proper .gitignore handling\n \"mlx>=0.26.3; sys_platform == 'darwin' and platform_machine == 'arm64'\",\n \"mlx-lm>=0.26.0; sys_platform == 'darwin' and platform_machine == 'arm64'\",\n]\n\n[project.optional-dependencies]\ncpu = [\n \"torch==2.2.2; platform_system == 'Linux' and python_version < '3.13'\",\n]\ncolab = [\n \"torch>=2.0.0,<3.0.0\", # Limit torch version to avoid conflicts\n \"transformers>=4.30.0,<4.46; python_version < '3.10'\", # Py3.9 compatibility\n \"transformers>=4.53.1,<4.58; python_version >= '3.10'\",\n \"accelerate>=0.20.0,<1.0.0\", # Limit accelerate version\n]\nserver = [\n \"fastapi>=0.115.0\",\n \"pydantic>=2.0.0\",\n \"uvicorn[standard]>=0.34.0\",\n]\n\n[project.scripts]\nleann = \"leann.cli:main\"\nleann_mcp = \"leann.mcp:main\"\n\n[tool.setuptools.packages.find]\nwhere = [\"src\"]\n" }, { "path": "packages/leann-core/src/leann/__init__.py", "content": "# packages/leann-core/src/leann/__init__.py\nimport os\nimport platform\n\n# ruff: noqa: E402 (env vars must be set before importing the rest of the package)\n\n# Fix OpenMP/FAISS threading defaults for common platforms\nsystem = platform.system()\n\nif system == \"Darwin\":\n # macOS ARM64: prevent runaway threading and duplicate lib issues\n os.environ[\"OMP_NUM_THREADS\"] = \"1\"\n os.environ[\"MKL_NUM_THREADS\"] = \"1\"\n os.environ[\"KMP_DUPLICATE_LIB_OK\"] = \"TRUE\"\n os.environ[\"KMP_BLOCKTIME\"] = \"0\"\n # Additional fixes for PyTorch/sentence-transformers on macOS ARM64 only in CI\n if os.environ.get(\"CI\") == \"true\":\n os.environ[\"PYTORCH_ENABLE_MPS_FALLBACK\"] = \"0\"\n os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\nelif system == \"Linux\":\n # Linux CPU-only: default to single-thread to avoid FAISS/ZMQ hangs (issue #208)\n os.environ.setdefault(\"OMP_NUM_THREADS\", \"1\")\n os.environ.setdefault(\"MKL_NUM_THREADS\", \"1\")\n os.environ.setdefault(\"FAISS_NUM_THREADS\", \"1\")\n os.environ.setdefault(\"OMP_WAIT_POLICY\", \"PASSIVE\")\n\nfrom .api import LeannBuilder, LeannChat, LeannSearcher\nfrom .react_agent import ReActAgent, create_react_agent\nfrom .registry import BACKEND_REGISTRY, autodiscover_backends\n\nautodiscover_backends()\n\n__all__ = [\n \"BACKEND_REGISTRY\",\n \"LeannBuilder\",\n \"LeannChat\",\n \"LeannSearcher\",\n \"ReActAgent\",\n \"create_react_agent\",\n]\n" }, { "path": "packages/leann-core/src/leann/api.py", "content": "\"\"\"\nThis file contains the core API for the LEANN project, now definitively updated\nwith the correct, original embedding logic from the user's reference code.\n\"\"\"\n\nimport json\nimport logging\nimport os\nimport pickle\nimport re\nimport subprocess\nimport time\nimport warnings\nfrom collections import Counter, defaultdict\nfrom dataclasses import dataclass, field\nfrom pathlib import Path\nfrom typing import Any, Literal, Optional, Union\n\nimport numpy as np\nfrom leann_backend_hnsw.convert_to_csr import prune_hnsw_embeddings_inplace\n\nfrom leann.interactive_utils import create_api_session\nfrom leann.interface import LeannBackendSearcherInterface\n\nfrom .chat import get_llm\nfrom .embedding_server_manager import EmbeddingServerManager\nfrom .interface import LeannBackendFactoryInterface\nfrom .metadata_filter import MetadataFilterEngine\nfrom .registry import BACKEND_REGISTRY\n\nlogger = logging.getLogger(__name__)\n\n\ndef get_registered_backends() -> list[str]:\n \"\"\"Get list of registered backend names.\"\"\"\n return list(BACKEND_REGISTRY.keys())\n\n\ndef compute_embeddings(\n chunks: list[str],\n model_name: str,\n mode: str = \"sentence-transformers\",\n use_server: bool = True,\n port: Optional[int] = None,\n is_build=False,\n provider_options: Optional[dict[str, Any]] = None,\n) -> np.ndarray:\n \"\"\"\n Computes embeddings using different backends.\n\n Args:\n chunks: List of text chunks to embed\n model_name: Name of the embedding model\n mode: Embedding backend mode. Options:\n - \"sentence-transformers\": Use sentence-transformers library (default)\n - \"mlx\": Use MLX backend for Apple Silicon\n - \"openai\": Use OpenAI embedding API\n - \"gemini\": Use Google Gemini embedding API\n use_server: Whether to use embedding server (True for search, False for build)\n\n Returns:\n numpy array of embeddings\n \"\"\"\n if use_server:\n # Use embedding server (for search/query)\n if port is None:\n raise ValueError(\"port is required when use_server is True\")\n return compute_embeddings_via_server(chunks, model_name, port=port)\n else:\n # Use direct computation (for build_index)\n from .embedding_compute import (\n compute_embeddings as compute_embeddings_direct,\n )\n\n return compute_embeddings_direct(\n chunks,\n model_name,\n mode=mode,\n is_build=is_build,\n provider_options=provider_options,\n )\n\n\ndef compute_embeddings_via_server(chunks: list[str], model_name: str, port: int) -> np.ndarray:\n \"\"\"Computes embeddings using sentence-transformers.\n\n Args:\n chunks: List of text chunks to embed\n model_name: Name of the sentence transformer model\n \"\"\"\n logger.info(\n f\"Computing embeddings for {len(chunks)} chunks using SentenceTransformer model '{model_name}' (via embedding server)...\"\n )\n import msgpack\n import numpy as np\n import zmq\n\n # Connect to embedding server\n context = zmq.Context()\n socket = context.socket(zmq.REQ)\n socket.connect(f\"tcp://localhost:{port}\")\n\n # Send chunks to server for embedding computation\n request = chunks\n socket.send(msgpack.packb(request))\n\n # Receive embeddings from server\n response = socket.recv()\n embeddings_list = msgpack.unpackb(response)\n\n # Convert back to numpy array\n embeddings = np.array(embeddings_list, dtype=np.float32)\n\n socket.close()\n context.term()\n\n return embeddings\n\n\n@dataclass\nclass SearchResult:\n id: str\n score: float\n text: str\n metadata: dict[str, Any] = field(default_factory=dict)\n\n\nclass PassageManager:\n def __init__(\n self, passage_sources: list[dict[str, Any]], metadata_file_path: Optional[str] = None\n ):\n self.offset_maps: dict[str, dict[str, int]] = {}\n self.passage_files: dict[str, str] = {}\n # Avoid materializing a single gigantic global map to reduce memory\n # footprint on very large corpora (e.g., 60M+ passages). Instead, keep\n # per-shard maps and do a lightweight per-shard lookup on demand.\n self._total_count: int = 0\n self.filter_engine = MetadataFilterEngine() # Initialize filter engine\n\n # Derive index base name for standard sibling fallbacks, e.g., .passages.*\n index_name_base = None\n if metadata_file_path:\n meta_name = Path(metadata_file_path).name\n if meta_name.endswith(\".meta.json\"):\n index_name_base = meta_name[: -len(\".meta.json\")]\n\n for source in passage_sources:\n assert source[\"type\"] == \"jsonl\", \"only jsonl is supported\"\n passage_file = source.get(\"path\", \"\")\n index_file = source.get(\"index_path\", \"\") # .idx file\n\n # Fix path resolution - relative paths should be relative to metadata file directory\n def _resolve_candidates(\n primary: str,\n relative_key: str,\n default_name: Optional[str],\n source_dict: dict[str, Any],\n ) -> list[Path]:\n \"\"\"\n Build an ordered list of candidate paths. For relative paths specified in\n metadata, prefer resolution relative to the metadata file directory first,\n then fall back to CWD-based resolution, and finally to conventional\n sibling defaults (e.g., .passages.idx / .jsonl).\n \"\"\"\n candidates: list[Path] = []\n # 1) Primary path\n if primary:\n p = Path(primary)\n if p.is_absolute():\n candidates.append(p)\n else:\n # Prefer metadata-relative resolution for relative paths\n if metadata_file_path:\n candidates.append(Path(metadata_file_path).parent / p)\n # Also consider CWD-relative as a fallback for legacy layouts\n candidates.append(Path.cwd() / p)\n # 2) metadata-relative explicit relative key (if present)\n if metadata_file_path and source_dict.get(relative_key):\n candidates.append(Path(metadata_file_path).parent / source_dict[relative_key])\n # 3) metadata-relative standard sibling filename\n if metadata_file_path and default_name:\n candidates.append(Path(metadata_file_path).parent / default_name)\n return candidates\n\n # Build candidate lists and pick first existing; otherwise keep last candidate for error message\n idx_default = f\"{index_name_base}.passages.idx\" if index_name_base else None\n idx_candidates = _resolve_candidates(\n index_file, \"index_path_relative\", idx_default, source\n )\n pas_default = f\"{index_name_base}.passages.jsonl\" if index_name_base else None\n pas_candidates = _resolve_candidates(passage_file, \"path_relative\", pas_default, source)\n\n def _pick_existing(cands: list[Path]) -> str:\n for c in cands:\n if c.exists():\n return str(c.resolve())\n # Fallback to last candidate (best guess) even if not exists; will error below\n return str(cands[-1].resolve()) if cands else \"\"\n\n index_file = _pick_existing(idx_candidates)\n passage_file = _pick_existing(pas_candidates)\n\n if not Path(index_file).exists():\n raise FileNotFoundError(f\"Passage index file not found: {index_file}\")\n\n with open(index_file, \"rb\") as f:\n offset_map: dict[str, int] = pickle.load(f)\n self.offset_maps[passage_file] = offset_map\n self.passage_files[passage_file] = passage_file\n self._total_count += len(offset_map)\n\n def get_passage(self, passage_id: str) -> dict[str, Any]:\n # Fast path: check each shard map (there are typically few shards).\n # This avoids building a massive combined dict while keeping lookups\n # bounded by the number of shards.\n for passage_file, offset_map in self.offset_maps.items():\n try:\n offset = offset_map[passage_id]\n with open(passage_file, encoding=\"utf-8\") as f:\n f.seek(offset)\n return json.loads(f.readline())\n except KeyError:\n continue\n raise KeyError(f\"Passage ID not found: {passage_id}\")\n\n def filter_search_results(\n self,\n search_results: list[SearchResult],\n metadata_filters: Optional[dict[str, dict[str, Union[str, int, float, bool, list]]]],\n ) -> list[SearchResult]:\n \"\"\"\n Apply metadata filters to search results.\n\n Args:\n search_results: List of SearchResult objects\n metadata_filters: Filter specifications to apply\n\n Returns:\n Filtered list of SearchResult objects\n \"\"\"\n if not metadata_filters:\n return search_results\n\n logger.debug(f\"Applying metadata filters to {len(search_results)} results\")\n\n # Convert SearchResult objects to dictionaries for the filter engine\n result_dicts = []\n for result in search_results:\n result_dicts.append(\n {\n \"id\": result.id,\n \"score\": result.score,\n \"text\": result.text,\n \"metadata\": result.metadata,\n }\n )\n\n # Apply filters using the filter engine\n filtered_dicts = self.filter_engine.apply_filters(result_dicts, metadata_filters)\n\n # Convert back to SearchResult objects\n filtered_results = []\n for result_dict in filtered_dicts:\n filtered_results.append(\n SearchResult(\n id=result_dict[\"id\"],\n score=result_dict[\"score\"],\n text=result_dict[\"text\"],\n metadata=result_dict[\"metadata\"],\n )\n )\n\n logger.debug(f\"Filtered results: {len(filtered_results)} remaining\")\n return filtered_results\n\n def __len__(self) -> int:\n return self._total_count\n\n\nclass BM25Scorer:\n def __init__(self, k1: float = 1.2, b: float = 0.75):\n self.k1 = k1\n self.b = b\n self.doc_freqs = None # How many docs contain each term (DF)\n self.doc_lengths = {} # How long each doc is (in words)\n self.word_counts = {} # How many times each word appears in each doc (TF)\n self.avg_doc_length = None\n self.corpus_size = None\n self.idlist = set() # List of all document IDs for easier searching\n\n def _tokenize(self, text: str) -> list[str]:\n return re.sub(r\"[^\\w\\s]\", \"\", text).lower().split()\n\n def fit(self, documents: list[dict[str, Any]]):\n \"\"\"\n Build BM25 statistics from a document corpus.\n Must be called before scoring.\n \"\"\"\n self.corpus_size = len(documents)\n self.doc_lengths = {}\n self.word_counts = {}\n self.idlist = set()\n doc_freqs = defaultdict(int)\n\n for doc_data in documents:\n doc_id = doc_data[\"id\"]\n words = self._tokenize(doc_data[\"text\"])\n doc_length = len(words)\n self.doc_lengths[doc_id] = doc_length\n\n unique_words = set(words)\n for word in unique_words:\n doc_freqs[word] += 1\n self.word_counts[doc_id] = dict(Counter(words))\n self.idlist.add(doc_id)\n\n self.doc_freqs = dict(doc_freqs)\n self.avg_doc_length = sum(self.doc_lengths.values()) / len(self.doc_lengths)\n\n def score(self, query_words: list[str], document_id: str) -> float:\n if (\n self.doc_freqs is None\n or self.doc_lengths == {}\n or self.word_counts == {}\n or self.avg_doc_length is None\n or self.corpus_size is None\n ):\n raise ValueError(\"BM25 model not fitted. Call fit() before scoring.\")\n\n passage_words = self.word_counts[document_id]\n passage_length = sum(passage_words.values())\n score = 0.0\n for word in query_words:\n if word not in self.doc_freqs:\n continue\n word_freq = passage_words[word] if word in passage_words else 0\n idf = np.log(\n (self.corpus_size - self.doc_freqs[word] + 0.5) / (self.doc_freqs[word] + 0.5) + 1\n )\n tf = (word_freq * (self.k1 + 1)) / (\n word_freq + self.k1 * (1 - self.b + self.b * (passage_length / self.avg_doc_length))\n )\n score += idf * tf\n return score\n\n def search(self, query: str, top_k: int = 5) -> list[SearchResult]:\n query_words = self._tokenize(query)\n scores = {doc_id: self.score(query_words, doc_id) for doc_id in self.idlist}\n sorted_scores = sorted(scores.items(), key=lambda x: x[1], reverse=True)\n return [\n SearchResult(id=doc_id, score=score, text=\"\", metadata={})\n for doc_id, score in sorted_scores[:top_k]\n ]\n\n\nclass LeannBuilder:\n def __init__(\n self,\n backend_name: str,\n embedding_model: str = \"facebook/contriever\",\n dimensions: Optional[int] = None,\n embedding_mode: str = \"sentence-transformers\",\n embedding_options: Optional[dict[str, Any]] = None,\n **backend_kwargs,\n ):\n self.backend_name = backend_name\n # Normalize incompatible combinations early (for consistent metadata)\n if backend_name == \"hnsw\":\n is_recompute = backend_kwargs.get(\"is_recompute\", True)\n is_compact = backend_kwargs.get(\"is_compact\", True)\n if is_recompute is False and is_compact is True:\n warnings.warn(\n \"HNSW with is_recompute=False requires non-compact storage. Forcing is_compact=False.\",\n UserWarning,\n stacklevel=2,\n )\n backend_kwargs[\"is_compact\"] = False\n\n backend_factory: Optional[LeannBackendFactoryInterface] = BACKEND_REGISTRY.get(backend_name)\n if backend_factory is None:\n raise ValueError(f\"Backend '{backend_name}' not found or not registered.\")\n self.backend_factory = backend_factory\n self.embedding_model = embedding_model\n self.dimensions = dimensions\n self.embedding_mode = embedding_mode\n self.embedding_options = embedding_options or {}\n\n # Check if we need to use cosine distance for normalized embeddings\n normalized_embeddings_models = {\n # OpenAI models\n (\"openai\", \"text-embedding-ada-002\"),\n (\"openai\", \"text-embedding-3-small\"),\n (\"openai\", \"text-embedding-3-large\"),\n # Voyage AI models\n (\"voyage\", \"voyage-2\"),\n (\"voyage\", \"voyage-3\"),\n (\"voyage\", \"voyage-large-2\"),\n (\"voyage\", \"voyage-multilingual-2\"),\n (\"voyage\", \"voyage-code-2\"),\n # Cohere models\n (\"cohere\", \"embed-english-v3.0\"),\n (\"cohere\", \"embed-multilingual-v3.0\"),\n (\"cohere\", \"embed-english-light-v3.0\"),\n (\"cohere\", \"embed-multilingual-light-v3.0\"),\n }\n\n # Also check for patterns in model names\n is_normalized = False\n current_model_lower = embedding_model.lower()\n current_mode_lower = embedding_mode.lower()\n\n # Check exact matches\n for mode, model in normalized_embeddings_models:\n if (current_mode_lower == mode and current_model_lower == model) or (\n mode in current_mode_lower and model in current_model_lower\n ):\n is_normalized = True\n break\n\n # Check patterns\n if not is_normalized:\n # OpenAI patterns\n if \"openai\" in current_mode_lower or \"openai\" in current_model_lower:\n if any(\n pattern in current_model_lower\n for pattern in [\"text-embedding\", \"ada\", \"3-small\", \"3-large\"]\n ):\n is_normalized = True\n # Voyage patterns\n elif \"voyage\" in current_mode_lower or \"voyage\" in current_model_lower:\n is_normalized = True\n # Cohere patterns\n elif \"cohere\" in current_mode_lower or \"cohere\" in current_model_lower:\n if \"embed\" in current_model_lower:\n is_normalized = True\n\n # Handle distance metric\n if is_normalized and \"distance_metric\" not in backend_kwargs:\n backend_kwargs[\"distance_metric\"] = \"cosine\"\n warnings.warn(\n f\"Detected normalized embeddings model '{embedding_model}' with mode '{embedding_mode}'. \"\n f\"Automatically setting distance_metric='cosine' for optimal performance. \"\n f\"Normalized embeddings (L2 norm = 1) should use cosine similarity instead of MIPS.\",\n UserWarning,\n stacklevel=2,\n )\n elif is_normalized and backend_kwargs.get(\"distance_metric\", \"\").lower() != \"cosine\":\n current_metric = backend_kwargs.get(\"distance_metric\", \"mips\")\n warnings.warn(\n f\"Warning: Using '{current_metric}' distance metric with normalized embeddings model \"\n f\"'{embedding_model}' may lead to suboptimal search results. \"\n f\"Consider using 'cosine' distance metric for better performance.\",\n UserWarning,\n stacklevel=2,\n )\n\n self.backend_kwargs = backend_kwargs\n self.chunks: list[dict[str, Any]] = []\n\n def add_text(self, text: str, metadata: Optional[dict[str, Any]] = None):\n if metadata is None:\n metadata = {}\n passage_id = metadata.get(\"id\", str(len(self.chunks)))\n chunk_data = {\"id\": passage_id, \"text\": text, \"metadata\": metadata}\n self.chunks.append(chunk_data)\n\n def build_index(self, index_path: str):\n if not self.chunks:\n raise ValueError(\"No chunks added.\")\n\n # Filter out invalid/empty text chunks early to keep passage and embedding counts aligned\n valid_chunks: list[dict[str, Any]] = []\n skipped = 0\n for chunk in self.chunks:\n text = chunk.get(\"text\", \"\")\n if isinstance(text, str) and text.strip():\n valid_chunks.append(chunk)\n else:\n skipped += 1\n if skipped > 0:\n print(\n f\"Warning: Skipping {skipped} empty/invalid text chunk(s). Processing {len(valid_chunks)} valid chunks\"\n )\n self.chunks = valid_chunks\n if not self.chunks:\n raise ValueError(\"All provided chunks are empty or invalid. Nothing to index.\")\n if self.dimensions is None:\n self.dimensions = len(\n compute_embeddings(\n [\"dummy\"],\n self.embedding_model,\n self.embedding_mode,\n use_server=False,\n provider_options=self.embedding_options,\n )[0]\n )\n path = Path(index_path)\n index_dir = path.parent\n index_name = path.name\n index_dir.mkdir(parents=True, exist_ok=True)\n passages_file = index_dir / f\"{index_name}.passages.jsonl\"\n offset_file = index_dir / f\"{index_name}.passages.idx\"\n offset_map = {}\n with open(passages_file, \"w\", encoding=\"utf-8\") as f:\n try:\n from tqdm import tqdm\n\n chunk_iterator = tqdm(self.chunks, desc=\"Writing passages\", unit=\"chunk\")\n except ImportError:\n chunk_iterator = self.chunks\n\n for chunk in chunk_iterator:\n offset = f.tell()\n json.dump(\n {\n \"id\": chunk[\"id\"],\n \"text\": chunk[\"text\"],\n \"metadata\": chunk[\"metadata\"],\n },\n f,\n ensure_ascii=False,\n )\n f.write(\"\\n\")\n offset_map[chunk[\"id\"]] = offset\n with open(offset_file, \"wb\") as f:\n pickle.dump(offset_map, f)\n texts_to_embed = [c[\"text\"] for c in self.chunks]\n embeddings = compute_embeddings(\n texts_to_embed,\n self.embedding_model,\n self.embedding_mode,\n use_server=False,\n is_build=True,\n provider_options=self.embedding_options,\n )\n string_ids = [chunk[\"id\"] for chunk in self.chunks]\n # Persist ID map alongside index so backends that return integer labels can remap to passage IDs\n try:\n idmap_file = (\n index_dir\n / f\"{index_name[: -len('.leann')] if index_name.endswith('.leann') else index_name}.ids.txt\"\n )\n with open(idmap_file, \"w\", encoding=\"utf-8\") as f:\n for sid in string_ids:\n f.write(str(sid) + \"\\n\")\n except Exception:\n pass\n current_backend_kwargs = {**self.backend_kwargs, \"dimensions\": self.dimensions}\n builder_instance = self.backend_factory.builder(**current_backend_kwargs)\n builder_instance.build(embeddings, string_ids, index_path, **current_backend_kwargs)\n leann_meta_path = index_dir / f\"{index_name}.meta.json\"\n meta_data = {\n \"version\": \"1.0\",\n \"backend_name\": self.backend_name,\n \"embedding_model\": self.embedding_model,\n \"dimensions\": self.dimensions,\n \"backend_kwargs\": self.backend_kwargs,\n \"embedding_mode\": self.embedding_mode,\n \"passage_sources\": [\n {\n \"type\": \"jsonl\",\n # Preserve existing relative file names (backward-compatible)\n \"path\": passages_file.name,\n \"index_path\": offset_file.name,\n # Add optional redundant relative keys for remote build portability (non-breaking)\n \"path_relative\": passages_file.name,\n \"index_path_relative\": offset_file.name,\n }\n ],\n }\n\n if self.embedding_options:\n meta_data[\"embedding_options\"] = self.embedding_options\n\n # Add storage status flags for HNSW backend\n if self.backend_name == \"hnsw\":\n is_compact = self.backend_kwargs.get(\"is_compact\", True)\n is_recompute = self.backend_kwargs.get(\"is_recompute\", True)\n meta_data[\"is_compact\"] = is_compact\n meta_data[\"is_pruned\"] = bool(is_recompute)\n with open(leann_meta_path, \"w\", encoding=\"utf-8\") as f:\n json.dump(meta_data, f, indent=2)\n\n def build_index_from_embeddings(self, index_path: str, embeddings_file: str):\n \"\"\"\n Build an index from pre-computed embeddings stored in a pickle file.\n\n Args:\n index_path: Path where the index will be saved\n embeddings_file: Path to pickle file containing (ids, embeddings) tuple\n \"\"\"\n # Load pre-computed embeddings\n with open(embeddings_file, \"rb\") as f:\n data = pickle.load(f)\n\n if not isinstance(data, tuple) or len(data) != 2:\n raise ValueError(\n f\"Invalid embeddings file format. Expected tuple with 2 elements, got {type(data)}\"\n )\n\n ids, embeddings = data\n\n if not isinstance(embeddings, np.ndarray):\n raise ValueError(f\"Expected embeddings to be numpy array, got {type(embeddings)}\")\n\n if len(ids) != embeddings.shape[0]:\n raise ValueError(\n f\"Mismatch between number of IDs ({len(ids)}) and embeddings ({embeddings.shape[0]})\"\n )\n\n # Validate/set dimensions\n embedding_dim = embeddings.shape[1]\n if self.dimensions is None:\n self.dimensions = embedding_dim\n elif self.dimensions != embedding_dim:\n raise ValueError(f\"Dimension mismatch: expected {self.dimensions}, got {embedding_dim}\")\n\n logger.info(\n f\"Building index from precomputed embeddings: {len(ids)} items, {embedding_dim} dimensions\"\n )\n\n # Ensure we have text data for each embedding\n if len(self.chunks) != len(ids):\n # If no text chunks provided, create placeholder text entries\n if not self.chunks:\n logger.info(\"No text chunks provided, creating placeholder entries...\")\n for id_val in ids:\n self.add_text(\n f\"Document {id_val}\",\n metadata={\"id\": str(id_val), \"from_embeddings\": True},\n )\n else:\n raise ValueError(\n f\"Number of text chunks ({len(self.chunks)}) doesn't match number of embeddings ({len(ids)})\"\n )\n\n # Build file structure\n path = Path(index_path)\n index_dir = path.parent\n index_name = path.name\n index_dir.mkdir(parents=True, exist_ok=True)\n passages_file = index_dir / f\"{index_name}.passages.jsonl\"\n offset_file = index_dir / f\"{index_name}.passages.idx\"\n\n # Write passages and create offset map\n offset_map = {}\n with open(passages_file, \"w\", encoding=\"utf-8\") as f:\n for chunk in self.chunks:\n offset = f.tell()\n json.dump(\n {\n \"id\": chunk[\"id\"],\n \"text\": chunk[\"text\"],\n \"metadata\": chunk[\"metadata\"],\n },\n f,\n ensure_ascii=False,\n )\n f.write(\"\\n\")\n offset_map[chunk[\"id\"]] = offset\n\n with open(offset_file, \"wb\") as f:\n pickle.dump(offset_map, f)\n\n # Build the vector index using precomputed embeddings\n string_ids = [str(id_val) for id_val in ids]\n # Persist ID map (order == embeddings order)\n try:\n idmap_file = (\n index_dir\n / f\"{index_name[: -len('.leann')] if index_name.endswith('.leann') else index_name}.ids.txt\"\n )\n with open(idmap_file, \"w\", encoding=\"utf-8\") as f:\n for sid in string_ids:\n f.write(str(sid) + \"\\n\")\n except Exception:\n pass\n current_backend_kwargs = {**self.backend_kwargs, \"dimensions\": self.dimensions}\n builder_instance = self.backend_factory.builder(**current_backend_kwargs)\n builder_instance.build(embeddings, string_ids, index_path)\n\n # Create metadata file\n leann_meta_path = index_dir / f\"{index_name}.meta.json\"\n meta_data = {\n \"version\": \"1.0\",\n \"backend_name\": self.backend_name,\n \"embedding_model\": self.embedding_model,\n \"dimensions\": self.dimensions,\n \"backend_kwargs\": self.backend_kwargs,\n \"embedding_mode\": self.embedding_mode,\n \"passage_sources\": [\n {\n \"type\": \"jsonl\",\n # Preserve existing relative file names (backward-compatible)\n \"path\": passages_file.name,\n \"index_path\": offset_file.name,\n # Add optional redundant relative keys for remote build portability (non-breaking)\n \"path_relative\": passages_file.name,\n \"index_path_relative\": offset_file.name,\n }\n ],\n \"built_from_precomputed_embeddings\": True,\n \"embeddings_source\": str(embeddings_file),\n }\n\n if self.embedding_options:\n meta_data[\"embedding_options\"] = self.embedding_options\n\n # Add storage status flags for HNSW backend\n if self.backend_name == \"hnsw\":\n is_compact = self.backend_kwargs.get(\"is_compact\", True)\n is_recompute = self.backend_kwargs.get(\"is_recompute\", True)\n meta_data[\"is_compact\"] = is_compact\n meta_data[\"is_pruned\"] = bool(is_recompute)\n\n with open(leann_meta_path, \"w\", encoding=\"utf-8\") as f:\n json.dump(meta_data, f, indent=2)\n\n logger.info(f\"Index built successfully from precomputed embeddings: {index_path}\")\n\n @staticmethod\n def _compact_passages(\n passages_file: Path, offset_file: Path, offset_map: dict[str, int]\n ) -> None:\n \"\"\"Rewrite passages.jsonl keeping only entries referenced by offset_map.\"\"\"\n live_entries: list[str] = []\n for _pid, offset in sorted(offset_map.items(), key=lambda x: x[1]):\n with open(passages_file, encoding=\"utf-8\") as f:\n f.seek(offset)\n live_entries.append(f.readline())\n\n tmp_file = passages_file.with_suffix(\".jsonl.tmp\")\n new_offset_map: dict[str, int] = {}\n with open(tmp_file, \"w\", encoding=\"utf-8\") as f:\n for line in live_entries:\n data = json.loads(line)\n new_offset_map[data[\"id\"]] = f.tell()\n f.write(line if line.endswith(\"\\n\") else line + \"\\n\")\n\n tmp_file.replace(passages_file)\n offset_map.clear()\n offset_map.update(new_offset_map)\n with open(offset_file, \"wb\") as f:\n pickle.dump(offset_map, f)\n\n def update_index(self, index_path: str, remove_passage_ids: Optional[list[str]] = None) -> None:\n \"\"\"Append new passages and vectors to an existing index (HNSW or IVF).\n For IVF, optional remove_passage_ids removes those ids first (e.g. from file-change API).\n \"\"\"\n if not self.chunks and not remove_passage_ids:\n raise ValueError(\"No new chunks or passage ids to remove provided for update.\")\n\n path = Path(index_path)\n index_dir = path.parent\n index_name = path.name\n index_prefix = path.stem\n\n meta_path = index_dir / f\"{index_name}.meta.json\"\n passages_file = index_dir / f\"{index_name}.passages.jsonl\"\n offset_file = index_dir / f\"{index_name}.passages.idx\"\n index_file = index_dir / f\"{index_prefix}.index\"\n\n if not meta_path.exists() or not passages_file.exists() or not offset_file.exists():\n raise FileNotFoundError(\"Index metadata or passage files are missing; cannot update.\")\n if not index_file.exists():\n raise FileNotFoundError(f\"Index file not found: {index_file}\")\n\n with open(meta_path, encoding=\"utf-8\") as f:\n meta = json.load(f)\n backend_name = meta.get(\"backend_name\")\n if backend_name != self.backend_name:\n raise ValueError(\n f\"Index was built with backend '{backend_name}', cannot update with '{self.backend_name}'.\"\n )\n\n with open(offset_file, \"rb\") as f:\n offset_map: dict[str, int] = pickle.load(f)\n existing_ids = set(offset_map.keys())\n\n # IVF: optional delete (for reindex / file-change: remove then re-insert)\n if remove_passage_ids and backend_name == \"ivf\":\n try:\n from leann_backend_ivf import remove_ids as ivf_remove_ids\n\n nremoved = ivf_remove_ids(str(path), remove_passage_ids)\n if nremoved < len(remove_passage_ids):\n logger.warning(\n \"IVF update_index: removed %d of %d requested passage IDs \"\n \"(some may have been stale).\",\n nremoved,\n len(remove_passage_ids),\n )\n except ImportError:\n raise RuntimeError(\n \"IVF backend required for remove_ids. Install leann-backend-ivf.\"\n )\n for pid in remove_passage_ids:\n offset_map.pop(pid, None)\n existing_ids -= set(remove_passage_ids)\n\n # Compact passages.jsonl: rewrite keeping only entries in offset_map\n self._compact_passages(passages_file, offset_file, offset_map)\n\n if not self.chunks:\n meta[\"total_passages\"] = len(offset_map)\n with open(meta_path, \"w\", encoding=\"utf-8\") as f:\n json.dump(meta, f, indent=2)\n self.chunks.clear()\n return\n\n meta_backend_kwargs = meta.get(\"backend_kwargs\", {})\n if backend_name == \"hnsw\":\n index_is_compact = meta.get(\"is_compact\", meta_backend_kwargs.get(\"is_compact\", True))\n if index_is_compact:\n raise ValueError(\n \"Compact HNSW indices do not support in-place updates. Rebuild required.\"\n )\n\n distance_metric = meta_backend_kwargs.get(\n \"distance_metric\", self.backend_kwargs.get(\"distance_metric\", \"mips\")\n ).lower()\n needs_recompute = bool(\n meta.get(\"is_pruned\")\n or meta_backend_kwargs.get(\"is_recompute\")\n or self.backend_kwargs.get(\"is_recompute\")\n )\n\n valid_chunks: list[dict[str, Any]] = []\n for chunk in self.chunks:\n text = chunk.get(\"text\", \"\")\n if not isinstance(text, str) or not text.strip():\n continue\n metadata = chunk.setdefault(\"metadata\", {})\n passage_id = chunk.get(\"id\") or metadata.get(\"id\")\n if passage_id and passage_id in existing_ids:\n raise ValueError(f\"Passage ID '{passage_id}' already exists in the index.\")\n valid_chunks.append(chunk)\n\n if not valid_chunks:\n # Remove-only or file emptied: we may have already removed ids, just update meta\n meta[\"total_passages\"] = len(offset_map)\n with open(meta_path, \"w\", encoding=\"utf-8\") as f:\n json.dump(meta, f, indent=2)\n self.chunks.clear()\n return\n\n texts_to_embed = [chunk[\"text\"] for chunk in valid_chunks]\n embeddings = compute_embeddings(\n texts_to_embed,\n self.embedding_model,\n self.embedding_mode,\n use_server=False,\n is_build=True,\n provider_options=self.embedding_options,\n )\n\n embedding_dim = embeddings.shape[1]\n expected_dim = meta.get(\"dimensions\")\n if expected_dim is not None and expected_dim != embedding_dim:\n raise ValueError(\n f\"Dimension mismatch during update: existing index uses {expected_dim}, got {embedding_dim}.\"\n )\n\n embeddings = np.ascontiguousarray(embeddings, dtype=np.float32)\n if distance_metric == \"cosine\":\n norms = np.linalg.norm(embeddings, axis=1, keepdims=True)\n norms[norms == 0] = 1\n embeddings = embeddings / norms\n\n # IVF: add_vectors then append passages/offset (no ZMQ/server)\n if backend_name == \"ivf\":\n for i, chunk in enumerate(valid_chunks):\n pid = chunk.get(\"id\") or chunk.get(\"metadata\", {}).get(\"id\")\n if not pid:\n pid = str(len(offset_map) + i)\n chunk.setdefault(\"metadata\", {})[\"id\"] = pid\n chunk[\"id\"] = pid\n passage_ids = [c[\"id\"] for c in valid_chunks]\n try:\n from leann_backend_ivf import add_vectors as ivf_add_vectors\n\n ivf_add_vectors(str(path), embeddings, passage_ids)\n except ImportError:\n raise RuntimeError(\"IVF backend required. Install leann-backend-ivf.\")\n rollback_passages_size = passages_file.stat().st_size if passages_file.exists() else 0\n offset_map_backup = offset_map.copy()\n try:\n with open(passages_file, \"a\", encoding=\"utf-8\") as f:\n for chunk in valid_chunks:\n off = f.tell()\n json.dump(\n {\n \"id\": chunk[\"id\"],\n \"text\": chunk[\"text\"],\n \"metadata\": chunk.get(\"metadata\", {}),\n },\n f,\n ensure_ascii=False,\n )\n f.write(\"\\n\")\n offset_map[chunk[\"id\"]] = off\n with open(offset_file, \"wb\") as f:\n pickle.dump(offset_map, f)\n meta[\"total_passages\"] = len(offset_map)\n with open(meta_path, \"w\", encoding=\"utf-8\") as f:\n json.dump(meta, f, indent=2)\n logger.info(\n \"Appended %d passages to IVF index '%s'. Total: %d\",\n len(valid_chunks),\n index_path,\n len(offset_map),\n )\n except Exception:\n if passages_file.exists():\n with open(passages_file, \"rb+\") as f:\n f.truncate(rollback_passages_size)\n offset_map = offset_map_backup\n with open(offset_file, \"wb\") as f:\n pickle.dump(offset_map, f)\n raise\n self.chunks.clear()\n return\n\n # HNSW path below\n from leann_backend_hnsw import faiss\n\n index = faiss.read_index(str(index_file))\n if hasattr(index, \"is_recompute\"):\n index.is_recompute = needs_recompute\n print(f\"index.is_recompute: {index.is_recompute}\")\n if getattr(index, \"storage\", None) is None:\n if index.metric_type == faiss.METRIC_INNER_PRODUCT:\n storage_index = faiss.IndexFlatIP(index.d)\n else:\n storage_index = faiss.IndexFlatL2(index.d)\n index.storage = storage_index\n index.own_fields = True\n # Faiss expects storage.ntotal to reflect the existing graph's\n # population (even if the vectors themselves were pruned from disk\n # for recompute mode). When we attach a fresh IndexFlat here its\n # ntotal starts at zero, which later causes IndexHNSW::add to\n # believe new \"preset\" levels were provided and trips the\n # `n0 + n == levels.size()` assertion. Seed the temporary storage\n # with the current ntotal so Faiss maintains the proper offset for\n # incoming vectors.\n try:\n storage_index.ntotal = index.ntotal\n except AttributeError:\n # Older Faiss builds may not expose ntotal as a writable\n # attribute; in that case we fall back to the default behaviour.\n pass\n if index.d != embedding_dim:\n raise ValueError(\n f\"Existing index dimension ({index.d}) does not match new embeddings ({embedding_dim}).\"\n )\n\n passage_meta_mode = meta.get(\"embedding_mode\", self.embedding_mode)\n passage_provider_options = meta.get(\"embedding_options\", self.embedding_options)\n\n base_id = index.ntotal\n for offset, chunk in enumerate(valid_chunks):\n new_id = str(base_id + offset)\n chunk.setdefault(\"metadata\", {})[\"id\"] = new_id\n chunk[\"id\"] = new_id\n\n # Append passages/offsets before we attempt index.add so the ZMQ server\n # can resolve newly assigned IDs during recompute. Keep rollback hooks\n # so we can restore files if the update fails mid-way.\n rollback_passages_size = passages_file.stat().st_size if passages_file.exists() else 0\n offset_map_backup = offset_map.copy()\n\n try:\n with open(passages_file, \"a\", encoding=\"utf-8\") as f:\n for chunk in valid_chunks:\n offset = f.tell()\n json.dump(\n {\n \"id\": chunk[\"id\"],\n \"text\": chunk[\"text\"],\n \"metadata\": chunk.get(\"metadata\", {}),\n },\n f,\n ensure_ascii=False,\n )\n f.write(\"\\n\")\n offset_map[chunk[\"id\"]] = offset\n\n with open(offset_file, \"wb\") as f:\n pickle.dump(offset_map, f)\n\n server_manager: Optional[EmbeddingServerManager] = None\n server_started = False\n requested_zmq_port = int(os.getenv(\"LEANN_UPDATE_ZMQ_PORT\", \"5557\"))\n\n try:\n if needs_recompute:\n server_manager = EmbeddingServerManager(\n backend_module_name=\"leann_backend_hnsw.hnsw_embedding_server\"\n )\n server_started, actual_port = server_manager.start_server(\n port=requested_zmq_port,\n model_name=self.embedding_model,\n embedding_mode=passage_meta_mode,\n passages_file=str(meta_path),\n distance_metric=distance_metric,\n use_daemon=False,\n enable_warmup=False,\n provider_options=passage_provider_options,\n )\n if not server_started:\n raise RuntimeError(\n \"Failed to start HNSW embedding server for recompute update.\"\n )\n if actual_port != requested_zmq_port:\n logger.warning(\n \"Embedding server started on port %s instead of requested %s. \"\n \"Using reassigned port.\",\n actual_port,\n requested_zmq_port,\n )\n if hasattr(index.hnsw, \"set_zmq_port\"):\n index.hnsw.set_zmq_port(actual_port)\n elif hasattr(index, \"set_zmq_port\"):\n index.set_zmq_port(actual_port)\n\n if needs_recompute:\n for i in range(embeddings.shape[0]):\n print(f\"add {i} embeddings\")\n index.add(1, faiss.swig_ptr(embeddings[i : i + 1]))\n else:\n index.add(embeddings.shape[0], faiss.swig_ptr(embeddings))\n faiss.write_index(index, str(index_file))\n finally:\n if server_started and server_manager is not None:\n server_manager.stop_server()\n\n except Exception:\n # Roll back appended passages/offset map to keep files consistent.\n if passages_file.exists():\n with open(passages_file, \"rb+\") as f:\n f.truncate(rollback_passages_size)\n offset_map = offset_map_backup\n with open(offset_file, \"wb\") as f:\n pickle.dump(offset_map, f)\n raise\n\n meta[\"total_passages\"] = len(offset_map)\n with open(meta_path, \"w\", encoding=\"utf-8\") as f:\n json.dump(meta, f, indent=2)\n\n logger.info(\n \"Appended %d passages to index '%s'. New total: %d\",\n len(valid_chunks),\n index_path,\n len(offset_map),\n )\n\n self.chunks.clear()\n\n if needs_recompute:\n prune_hnsw_embeddings_inplace(str(index_file))\n\n\nclass LeannSearcher:\n def __init__(\n self,\n index_path: str,\n enable_warmup: bool = True,\n recompute_embeddings: bool = True,\n use_daemon: bool = True,\n daemon_ttl_seconds: int = 900,\n **backend_kwargs,\n ):\n # Fix path resolution for Colab and other environments\n if not Path(index_path).is_absolute():\n index_path = str(Path(index_path).resolve())\n\n self.meta_path_str = f\"{index_path}.meta.json\"\n if not Path(self.meta_path_str).exists():\n parent_dir = Path(index_path).parent\n print(\n f\"Leann metadata file not found at {self.meta_path_str}, and you may need to rm -rf {parent_dir}\"\n )\n # highlight in red the filenotfound error\n raise FileNotFoundError(\n f\"Leann metadata file not found at {self.meta_path_str}, \\033[91m you may need to rm -rf {parent_dir}\\033[0m\"\n )\n with open(self.meta_path_str, encoding=\"utf-8\") as f:\n self.meta_data = json.load(f)\n backend_name = self.meta_data[\"backend_name\"]\n self.embedding_model = self.meta_data[\"embedding_model\"]\n # Support both old and new format\n self.embedding_mode = self.meta_data.get(\"embedding_mode\", \"sentence-transformers\")\n self.embedding_options = self.meta_data.get(\"embedding_options\", {})\n # Delegate portability handling to PassageManager\n self.passage_manager = PassageManager(\n self.meta_data.get(\"passage_sources\", []), metadata_file_path=self.meta_path_str\n )\n # Preserve backend name for conditional parameter forwarding\n self.backend_name = backend_name\n backend_factory = BACKEND_REGISTRY.get(backend_name)\n if backend_factory is None:\n raise ValueError(f\"Backend '{backend_name}' not found.\")\n\n # Global recompute flag for this searcher (explicit knob, default True)\n self.recompute_embeddings: bool = bool(recompute_embeddings)\n\n # Warmup flag: keep using the existing enable_warmup parameter,\n # but default it to True so cold-start happens earlier.\n self._warmup: bool = bool(enable_warmup)\n self._use_daemon: bool = bool(use_daemon)\n self._daemon_ttl_seconds: int = int(daemon_ttl_seconds)\n\n final_kwargs = {**self.meta_data.get(\"backend_kwargs\", {}), **backend_kwargs}\n final_kwargs[\"enable_warmup\"] = self._warmup\n final_kwargs[\"use_daemon\"] = self._use_daemon\n final_kwargs[\"daemon_ttl_seconds\"] = self._daemon_ttl_seconds\n if self.embedding_options:\n final_kwargs.setdefault(\"embedding_options\", self.embedding_options)\n self.backend_impl: LeannBackendSearcherInterface = backend_factory.searcher(\n index_path, **final_kwargs\n )\n self.bm25_scorer: Optional[BM25Scorer] = None\n\n # Optional one-shot warmup at construction time to hide cold-start latency.\n if self._warmup:\n self.warmup()\n\n def warmup(self) -> None:\n \"\"\"Warm up embedding path so first user query is faster.\"\"\"\n try:\n _ = self.backend_impl.compute_query_embedding(\n \"__LEANN_WARMUP__\",\n use_server_if_available=self.recompute_embeddings,\n )\n except Exception as exc:\n logger.warning(f\"Warmup embedding failed (ignored): {exc}\")\n\n def search(\n self,\n query: str,\n top_k: int = 5,\n complexity: int = 64,\n beam_width: int = 1,\n prune_ratio: float = 0.0,\n recompute_embeddings: Optional[bool] = None,\n pruning_strategy: Literal[\"global\", \"local\", \"proportional\"] = \"global\",\n expected_zmq_port: int = 5557,\n metadata_filters: Optional[dict[str, dict[str, Union[str, int, float, bool, list]]]] = None,\n batch_size: int = 0,\n use_grep: bool = False,\n gemma: float = 1.0,\n provider_options: Optional[dict[str, Any]] = None,\n **kwargs,\n ) -> list[SearchResult]:\n \"\"\"\n Search for nearest neighbors with optional metadata filtering.\n\n Args:\n query: Text query to search for\n top_k: Number of nearest neighbors to return\n complexity: Search complexity/candidate list size, higher = more accurate but slower\n beam_width: Number of parallel search paths/IO requests per iteration\n prune_ratio: Ratio of neighbors to prune via approximate distance (0.0-1.0)\n recompute_embeddings: (Deprecated) Per-call override for recompute mode.\n Configure this at LeannSearcher(..., recompute_embeddings=...) instead.\n pruning_strategy: Candidate selection strategy - \"global\" (default), \"local\", or \"proportional\"\n expected_zmq_port: ZMQ port for embedding server communication\n metadata_filters: Optional filters to apply to search results based on metadata.\n Format: {\"field_name\": {\"operator\": value}}\n Supported operators:\n - Comparison: \"==\", \"!=\", \"<\", \"<=\", \">\", \">=\"\n - Membership: \"in\", \"not_in\"\n - String: \"contains\", \"starts_with\", \"ends_with\"\n Example: {\"chapter\": {\"<=\": 5}, \"tags\": {\"in\": [\"fiction\", \"drama\"]}}\n gemma: Weight of vector search results in hybrid search (0.0-1.0), 1 = pure vector search, 0 = pure keyword search\n **kwargs: Backend-specific parameters\n\n Returns:\n List of SearchResult objects with text, metadata, and similarity scores\n \"\"\"\n # Handle grep search\n if use_grep:\n return self._grep_search(query, top_k)\n\n logger.info(\"🔍 LeannSearcher.search() called:\")\n logger.info(f\" Query: '{query}'\")\n logger.info(f\" Top_k: {top_k}\")\n logger.info(f\" Metadata filters: {metadata_filters}\")\n logger.info(f\" Additional kwargs: {kwargs}\")\n\n # Smart top_k detection and adjustment\n # Use PassageManager length (sum of shard sizes) to avoid\n # depending on a massive combined map\n total_docs = len(self.passage_manager)\n original_top_k = top_k\n if top_k > total_docs:\n top_k = total_docs\n logger.warning(\n f\" ⚠️ Requested top_k ({original_top_k}) exceeds total documents ({total_docs})\"\n )\n logger.warning(f\" ✅ Auto-adjusted top_k to {top_k} to match available documents\")\n\n # Handle pure keyword search\n if gemma == 0.0:\n start_time = time.time()\n bm25_results = self._bm25_search(query, top_k)\n # Convert BM25 results to the expected format\n results = {\n \"labels\": [[r.id for r in bm25_results]],\n \"distances\": [[r.score for r in bm25_results]],\n }\n else:\n # Perform vector search\n zmq_port = None\n\n # Resolve effective recompute flag for this search.\n if recompute_embeddings is not None:\n logger.warning(\n \"LeannSearcher.search(..., recompute_embeddings=...) is deprecated and \"\n \"will be removed in a future version. Configure recompute at \"\n \"LeannSearcher(..., recompute_embeddings=...) instead.\"\n )\n effective_recompute = bool(recompute_embeddings)\n else:\n effective_recompute = self.recompute_embeddings\n\n start_time = time.time()\n if effective_recompute:\n zmq_port = self.backend_impl._ensure_server_running(\n self.meta_path_str,\n port=expected_zmq_port,\n enable_warmup=self._warmup,\n use_daemon=self._use_daemon,\n daemon_ttl_seconds=self._daemon_ttl_seconds,\n **kwargs,\n )\n del expected_zmq_port\n zmq_time = time.time() - start_time\n logger.info(f\" Launching server time: {zmq_time} seconds\")\n\n start_time = time.time()\n\n # Extract query template from stored embedding_options with fallback chain:\n # 1. Check provider_options override (highest priority)\n # 2. Check query_prompt_template (new format)\n # 3. Check prompt_template (old format for backward compat)\n # 4. None (no template)\n query_template = None\n if provider_options and \"prompt_template\" in provider_options:\n query_template = provider_options[\"prompt_template\"]\n elif \"query_prompt_template\" in self.embedding_options:\n query_template = self.embedding_options[\"query_prompt_template\"]\n elif \"prompt_template\" in self.embedding_options:\n query_template = self.embedding_options[\"prompt_template\"]\n\n query_embedding = self.backend_impl.compute_query_embedding(\n query,\n use_server_if_available=effective_recompute,\n zmq_port=zmq_port,\n query_template=query_template,\n )\n logger.info(f\" Generated embedding shape: {query_embedding.shape}\")\n embedding_time = time.time() - start_time\n logger.info(f\" Embedding time: {embedding_time} seconds\")\n\n start_time = time.time()\n backend_search_kwargs: dict[str, Any] = {\n \"complexity\": complexity,\n \"beam_width\": beam_width,\n \"prune_ratio\": prune_ratio,\n \"recompute_embeddings\": effective_recompute,\n \"pruning_strategy\": pruning_strategy,\n \"zmq_port\": zmq_port,\n }\n # Only HNSW supports batching; forward conditionally\n if self.backend_name == \"hnsw\":\n backend_search_kwargs[\"batch_size\"] = batch_size\n\n # Merge any extra kwargs last\n backend_search_kwargs.update(kwargs)\n\n results = self.backend_impl.search(\n query_embedding,\n top_k,\n **backend_search_kwargs,\n )\n\n # Handle hybrid search\n if 0.0 < gemma < 1.0:\n logger.info(f\" 🌟 Hybrid search enabled with gemma={gemma}\")\n BM25_WEIGHT = 1.0 - gemma\n bm25_results = self._bm25_search(query, top_k)\n hybrid_scores: dict[str, float] = {}\n # Add vector search scores (weighted by gemma)\n if \"labels\" in results and \"distances\" in results:\n for doc_id, score in zip(results[\"labels\"][0], results[\"distances\"][0]):\n hybrid_scores[doc_id] = gemma * score\n # Add BM25 scores (weighted by BM25_WEIGHT)\n for bm25_result in bm25_results:\n doc_id = bm25_result.id\n if doc_id in hybrid_scores:\n hybrid_scores[doc_id] += BM25_WEIGHT * bm25_result.score\n else:\n hybrid_scores[doc_id] = BM25_WEIGHT * bm25_result.score\n\n sorted_hybrid = sorted(hybrid_scores.items(), key=lambda x: x[1], reverse=True)[:top_k]\n results[\"labels\"] = [[doc_id for doc_id, _ in sorted_hybrid]]\n results[\"distances\"] = [[score for _, score in sorted_hybrid]]\n\n logger.info(\n f\" Combined {len(hybrid_scores)} unique documents from vector and BM25 search\"\n )\n\n search_time = time.time() - start_time\n logger.info(f\" Search time in search() LEANN searcher: {search_time} seconds\")\n logger.info(f\" Backend returned: labels={len(results.get('labels', [[]])[0])} results\")\n\n enriched_results = []\n if \"labels\" in results and \"distances\" in results:\n logger.info(f\" Processing {len(results['labels'][0])} passage IDs:\")\n # Python 3.9 does not support zip(strict=...); lengths are expected to match\n for i, (string_id, dist) in enumerate(\n zip(results[\"labels\"][0], results[\"distances\"][0])\n ):\n try:\n passage_data = self.passage_manager.get_passage(string_id)\n enriched_results.append(\n SearchResult(\n id=string_id,\n score=float(dist),\n text=passage_data[\"text\"],\n metadata=passage_data.get(\"metadata\", {}),\n )\n )\n\n # Color codes for better logging\n GREEN = \"\\033[92m\"\n BLUE = \"\\033[94m\"\n YELLOW = \"\\033[93m\"\n RESET = \"\\033[0m\"\n\n # Truncate text for display (first 100 chars)\n display_text = passage_data[\"text\"]\n logger.info(\n f\" {GREEN}✓{RESET} {BLUE}[{i + 1:2d}]{RESET} {YELLOW}ID:{RESET} '{string_id}' {YELLOW}Score:{RESET} {dist:.4f} {YELLOW}Text:{RESET} {display_text}\"\n )\n except KeyError:\n RED = \"\\033[91m\"\n RESET = \"\\033[0m\"\n logger.error(\n f\" {RED}✗{RESET} [{i + 1:2d}] ID: '{string_id}' -> {RED}ERROR: Passage not found!{RESET}\"\n )\n\n # Apply metadata filters if specified\n if metadata_filters:\n logger.info(f\" 🔍 Applying metadata filters: {metadata_filters}\")\n enriched_results = self.passage_manager.filter_search_results(\n enriched_results, metadata_filters\n )\n\n # Define color codes outside the loop for final message\n GREEN = \"\\033[92m\"\n RESET = \"\\033[0m\"\n logger.info(f\" {GREEN}✓ Final enriched results: {len(enriched_results)} passages{RESET}\")\n return enriched_results\n\n def _init_bm25(self) -> None:\n \"\"\"Initialize BM25 scorer\"\"\"\n self.bm25_scorer = BM25Scorer()\n # Load all the files directly\n passages = []\n for passage_file in self.passage_manager.passage_files.values():\n with open(passage_file, encoding=\"utf-8\") as f:\n for line in f:\n if line.strip():\n data = json.loads(line)\n passages.append(data)\n self.bm25_scorer.fit(passages)\n\n def _bm25_search(self, query: str, top_k: int = 5) -> list[SearchResult]:\n \"\"\"Perform BM25 search on raw passages\"\"\"\n if self.bm25_scorer is None:\n self._init_bm25()\n logger.info(\" BM25 scorer initialized\")\n scorer = self.bm25_scorer\n if scorer is None:\n raise RuntimeError(\"BM25 scorer failed to initialize\")\n return scorer.search(query, top_k)\n\n def _find_jsonl_file(self) -> Optional[str]:\n \"\"\"Find the .jsonl file containing raw passages for grep search\"\"\"\n index_path = Path(self.meta_path_str).parent\n potential_files = [\n index_path / \"documents.leann.passages.jsonl\",\n index_path.parent / \"documents.leann.passages.jsonl\",\n ]\n\n for file_path in potential_files:\n if file_path.exists():\n return str(file_path)\n return None\n\n def _grep_search(self, query: str, top_k: int = 5) -> list[SearchResult]:\n \"\"\"Perform grep-based search on raw passages\"\"\"\n jsonl_file = self._find_jsonl_file()\n if not jsonl_file:\n raise FileNotFoundError(\"No .jsonl passages file found for grep search\")\n\n try:\n cmd = [\"grep\", \"-i\", \"-n\", query, jsonl_file]\n result = subprocess.run(cmd, capture_output=True, text=True, check=False)\n\n if result.returncode == 1:\n return []\n elif result.returncode != 0:\n raise RuntimeError(f\"Grep failed: {result.stderr}\")\n\n matches = []\n for line in result.stdout.strip().split(\"\\n\"):\n if not line:\n continue\n parts = line.split(\":\", 1)\n if len(parts) != 2:\n continue\n\n try:\n data = json.loads(parts[1])\n text = data.get(\"text\", \"\")\n score = text.lower().count(query.lower())\n\n matches.append(\n SearchResult(\n id=data.get(\"id\", parts[0]),\n text=text,\n metadata=data.get(\"metadata\", {}),\n score=float(score),\n )\n )\n except json.JSONDecodeError:\n continue\n\n matches.sort(key=lambda x: x.score, reverse=True)\n return matches[:top_k]\n\n except FileNotFoundError:\n raise RuntimeError(\n \"grep command not found. Please install grep or use semantic search.\"\n )\n\n def _python_regex_search(self, query: str, top_k: int = 5) -> list[SearchResult]:\n \"\"\"Fallback regex search\"\"\"\n jsonl_file = self._find_jsonl_file()\n if not jsonl_file:\n raise FileNotFoundError(\"No .jsonl file found\")\n\n pattern = re.compile(re.escape(query), re.IGNORECASE)\n matches = []\n\n with open(jsonl_file, encoding=\"utf-8\") as f:\n for line_num, line in enumerate(f, 1):\n if pattern.search(line):\n try:\n data = json.loads(line.strip())\n matches.append(\n SearchResult(\n id=data.get(\"id\", str(line_num)),\n text=data.get(\"text\", \"\"),\n metadata=data.get(\"metadata\", {}),\n score=float(len(pattern.findall(data.get(\"text\", \"\")))),\n )\n )\n except json.JSONDecodeError:\n continue\n\n matches.sort(key=lambda x: x.score, reverse=True)\n return matches[:top_k]\n\n def cleanup(self):\n \"\"\"Explicitly cleanup embedding server and backend index resources.\n This method should be called after you're done using the searcher,\n especially in test environments or batch processing scenarios.\n On Windows, this releases file handles held by native backends\n (e.g., DiskANN memory-mapped index files).\n \"\"\"\n backend = getattr(self.backend_impl, \"embedding_server_manager\", None)\n if backend is not None:\n backend.stop_server()\n close_fn = getattr(self.backend_impl, \"close\", None)\n if close_fn is not None:\n close_fn()\n\n # Enable automatic cleanup patterns\n def __enter__(self):\n return self\n\n def __exit__(self, exc_type, exc, tb):\n try:\n self.cleanup()\n except Exception:\n pass\n\n def __del__(self):\n try:\n self.cleanup()\n except Exception:\n # Avoid noisy errors during interpreter shutdown\n pass\n\n\nclass LeannChat:\n def __init__(\n self,\n index_path: str,\n llm_config: Optional[dict[str, Any]] = None,\n enable_warmup: bool = False,\n searcher: Optional[LeannSearcher] = None,\n **kwargs,\n ):\n if searcher is None:\n self.searcher = LeannSearcher(index_path, enable_warmup=enable_warmup, **kwargs)\n self._owns_searcher = True\n else:\n self.searcher = searcher\n self._owns_searcher = False\n self.llm = get_llm(llm_config)\n\n def ask(\n self,\n question: str,\n top_k: int = 5,\n complexity: int = 64,\n beam_width: int = 1,\n prune_ratio: float = 0.0,\n recompute_embeddings: bool = True,\n pruning_strategy: Literal[\"global\", \"local\", \"proportional\"] = \"global\",\n llm_kwargs: Optional[dict[str, Any]] = None,\n expected_zmq_port: int = 5557,\n metadata_filters: Optional[dict[str, dict[str, Union[str, int, float, bool, list]]]] = None,\n batch_size: int = 0,\n use_grep: bool = False,\n gemma: float = 1.0,\n **search_kwargs,\n ):\n if llm_kwargs is None:\n llm_kwargs = {}\n search_time = time.time()\n results = self.searcher.search(\n question,\n top_k=top_k,\n complexity=complexity,\n beam_width=beam_width,\n prune_ratio=prune_ratio,\n recompute_embeddings=recompute_embeddings,\n pruning_strategy=pruning_strategy,\n expected_zmq_port=expected_zmq_port,\n metadata_filters=metadata_filters,\n use_grep=use_grep,\n gemma=gemma,\n batch_size=batch_size,\n **search_kwargs,\n )\n search_time = time.time() - search_time\n logger.info(f\" Search time: {search_time} seconds\")\n context = \"\\n\\n\".join([r.text for r in results])\n prompt = (\n \"Here is some retrieved context that might help answer your question:\\n\\n\"\n f\"{context}\\n\\n\"\n f\"Question: {question}\\n\\n\"\n \"Please provide the best answer you can based on this context and your knowledge.\"\n )\n\n logger.info(\"The context provided to the LLM is:\")\n logger.info(f\"{'Relevance':<10} | {'Chunk id':<10} | {'Content':<60} | {'Source':<80}\")\n logger.info(\"-\" * 150)\n for r in results:\n chunk_relevance = f\"{r.score:.3f}\"\n chunk_id = r.id\n chunk_content = r.text[:60]\n chunk_source = r.metadata.get(\"source\", \"\")[:80]\n logger.info(\n f\"{chunk_relevance:<10} | {chunk_id:<10} | {chunk_content:<60} | {chunk_source:<80}\"\n )\n ask_time = time.time()\n ans = self.llm.ask(prompt, **llm_kwargs)\n ask_time = time.time() - ask_time\n logger.info(f\" Ask time: {ask_time} seconds\")\n return ans\n\n def start_interactive(self):\n \"\"\"Start interactive chat session.\"\"\"\n session = create_api_session()\n\n def handle_query(user_input: str):\n response = self.ask(user_input)\n print(f\"Leann: {response}\")\n\n session.run_interactive_loop(handle_query)\n\n def cleanup(self):\n \"\"\"Explicitly cleanup embedding server resources.\n\n This method should be called after you're done using the chat interface,\n especially in test environments or batch processing scenarios.\n \"\"\"\n # Only stop the embedding server if this LeannChat instance created the searcher.\n # When a shared searcher is passed in, avoid shutting down the server to enable reuse.\n if getattr(self, \"_owns_searcher\", False) and hasattr(self.searcher, \"cleanup\"):\n self.searcher.cleanup()\n\n # Enable automatic cleanup patterns\n def __enter__(self):\n return self\n\n def __exit__(self, exc_type, exc, tb):\n try:\n self.cleanup()\n except Exception:\n pass\n\n def __del__(self):\n try:\n self.cleanup()\n except Exception:\n pass\n" }, { "path": "packages/leann-core/src/leann/chat.py", "content": "#!/usr/bin/env python3\n\"\"\"\nThis file contains the chat generation logic for the LEANN project,\nsupporting different backends like Ollama, Hugging Face Transformers, and a simulation mode.\n\"\"\"\n\nimport difflib\nimport logging\nimport os\nfrom abc import ABC, abstractmethod\nfrom typing import Any, Optional, cast\n\nimport torch\n\nfrom .settings import (\n resolve_anthropic_api_key,\n resolve_anthropic_base_url,\n resolve_minimax_api_key,\n resolve_minimax_base_url,\n resolve_ollama_host,\n resolve_openai_api_key,\n resolve_openai_base_url,\n)\n\n# Configure logging\nlogging.basicConfig(level=logging.INFO)\nlogger = logging.getLogger(__name__)\n\n\ndef check_ollama_models(host: str) -> list[str]:\n \"\"\"Check available Ollama models and return a list\"\"\"\n try:\n import requests\n\n response = requests.get(f\"{host}/api/tags\", timeout=5)\n if response.status_code == 200:\n data = response.json()\n return [model[\"name\"] for model in data.get(\"models\", [])]\n return []\n except Exception:\n return []\n\n\ndef check_ollama_model_exists_remotely(model_name: str) -> tuple[bool, list[str]]:\n \"\"\"Check if a model exists in Ollama's remote library and return available tags\n\n Returns:\n (model_exists, available_tags): bool and list of matching tags\n \"\"\"\n try:\n import re\n\n import requests\n\n # Split model name and tag\n if \":\" in model_name:\n base_model, requested_tag = model_name.split(\":\", 1)\n else:\n base_model, requested_tag = model_name, None\n\n # First check if base model exists in library\n library_response = requests.get(\"https://ollama.com/library\", timeout=8)\n if library_response.status_code != 200:\n return True, [] # Assume exists if can't check\n\n # Extract model names from library page\n models_in_library = re.findall(r'href=\"/library/([^\"]+)\"', library_response.text)\n\n if base_model not in models_in_library:\n return False, [] # Base model doesn't exist\n\n # If base model exists, get available tags\n tags_response = requests.get(f\"https://ollama.com/library/{base_model}/tags\", timeout=8)\n if tags_response.status_code != 200:\n return True, [] # Base model exists but can't get tags\n\n # Extract tags for this model - be more specific to avoid HTML artifacts\n tag_pattern = rf\"{re.escape(base_model)}:[a-zA-Z0-9\\.\\-_]+\"\n raw_tags = re.findall(tag_pattern, tags_response.text)\n\n # Clean up tags - remove HTML artifacts and duplicates\n available_tags = []\n seen = set()\n for tag in raw_tags:\n # Skip if it looks like HTML (contains < or >)\n if \"<\" in tag or \">\" in tag:\n continue\n if tag not in seen:\n seen.add(tag)\n available_tags.append(tag)\n\n # Check if exact model exists\n if requested_tag is None:\n # User just requested base model, suggest tags\n return True, available_tags[:10] # Return up to 10 tags\n else:\n exact_match = model_name in available_tags\n return exact_match, available_tags[:10]\n\n except Exception:\n pass\n\n # If scraping fails, assume model might exist (don't block user)\n return True, []\n\n\ndef search_ollama_models_fuzzy(query: str, available_models: list[str]) -> list[str]:\n \"\"\"Use intelligent fuzzy search for Ollama models\"\"\"\n if not available_models:\n return []\n\n query_lower = query.lower()\n suggestions = []\n\n # 1. Exact matches first\n exact_matches = [m for m in available_models if query_lower == m.lower()]\n suggestions.extend(exact_matches)\n\n # 2. Starts with query\n starts_with = [\n m for m in available_models if m.lower().startswith(query_lower) and m not in suggestions\n ]\n suggestions.extend(starts_with)\n\n # 3. Contains query\n contains = [m for m in available_models if query_lower in m.lower() and m not in suggestions]\n suggestions.extend(contains)\n\n # 4. Base model name matching (remove version numbers)\n def get_base_name(model_name: str) -> str:\n \"\"\"Extract base name without version (e.g., 'llama3:8b' -> 'llama3')\"\"\"\n return model_name.split(\":\")[0].split(\"-\")[0]\n\n query_base = get_base_name(query_lower)\n base_matches = [\n m\n for m in available_models\n if get_base_name(m.lower()) == query_base and m not in suggestions\n ]\n suggestions.extend(base_matches)\n\n # 5. Family/variant matching\n model_families = {\n \"llama\": [\"llama2\", \"llama3\", \"alpaca\", \"vicuna\", \"codellama\"],\n \"qwen\": [\"qwen\", \"qwen2\", \"qwen3\"],\n \"gemma\": [\"gemma\", \"gemma2\"],\n \"phi\": [\"phi\", \"phi2\", \"phi3\"],\n \"mistral\": [\"mistral\", \"mixtral\", \"openhermes\"],\n \"dolphin\": [\"dolphin\", \"openchat\"],\n \"deepseek\": [\"deepseek\", \"deepseek-coder\"],\n }\n\n query_family = None\n for family, variants in model_families.items():\n if any(variant in query_lower for variant in variants):\n query_family = family\n break\n\n if query_family:\n family_variants = model_families[query_family]\n family_matches = [\n m\n for m in available_models\n if any(variant in m.lower() for variant in family_variants) and m not in suggestions\n ]\n suggestions.extend(family_matches)\n\n # 6. Use difflib for remaining fuzzy matches\n remaining_models = [m for m in available_models if m not in suggestions]\n difflib_matches = difflib.get_close_matches(query_lower, remaining_models, n=3, cutoff=0.4)\n suggestions.extend(difflib_matches)\n\n return suggestions[:8] # Return top 8 suggestions\n\n\n# Remove this function entirely - we don't need external API calls for Ollama\n\n\n# Remove this too - no need for fallback\n\n\ndef suggest_similar_models(invalid_model: str, available_models: list[str]) -> list[str]:\n \"\"\"Use difflib to find similar model names\"\"\"\n if not available_models:\n return []\n\n # Get close matches using fuzzy matching\n suggestions = difflib.get_close_matches(invalid_model, available_models, n=3, cutoff=0.3)\n return suggestions\n\n\ndef check_hf_model_exists(model_name: str) -> bool:\n \"\"\"Quick check if HuggingFace model exists without downloading\"\"\"\n try:\n from huggingface_hub import model_info\n\n model_info(model_name)\n return True\n except Exception:\n return False\n\n\ndef get_popular_hf_models() -> list[str]:\n \"\"\"Return a list of popular HuggingFace models for suggestions\"\"\"\n try:\n from huggingface_hub import list_models\n\n # Get popular text-generation models, sorted by downloads\n models = list_models(\n filter=\"text-generation\",\n sort=\"downloads\",\n direction=-1,\n limit=20, # Get top 20 most downloaded\n )\n\n # Extract model names and filter for chat/conversation models\n model_names = []\n chat_keywords = [\"chat\", \"instruct\", \"dialog\", \"conversation\", \"assistant\"]\n\n for model in models:\n model_name = model.id if hasattr(model, \"id\") else str(model)\n # Prioritize models with chat-related keywords\n if any(keyword in model_name.lower() for keyword in chat_keywords):\n model_names.append(model_name)\n elif len(model_names) < 10: # Fill up with other popular models\n model_names.append(model_name)\n\n return model_names[:10] if model_names else _get_fallback_hf_models()\n\n except Exception:\n # Fallback to static list if API call fails\n return _get_fallback_hf_models()\n\n\ndef _get_fallback_hf_models() -> list[str]:\n \"\"\"Fallback list of popular HuggingFace models\"\"\"\n return [\n \"microsoft/DialoGPT-medium\",\n \"microsoft/DialoGPT-large\",\n \"facebook/blenderbot-400M-distill\",\n \"microsoft/phi-2\",\n \"deepseek-ai/deepseek-llm-7b-chat\",\n \"microsoft/DialoGPT-small\",\n \"facebook/blenderbot_small-90M\",\n \"microsoft/phi-1_5\",\n \"facebook/opt-350m\",\n \"EleutherAI/gpt-neo-1.3B\",\n ]\n\n\ndef search_hf_models_fuzzy(query: str, limit: int = 10) -> list[str]:\n \"\"\"Use HuggingFace Hub's native fuzzy search for model suggestions\"\"\"\n try:\n from huggingface_hub import list_models\n\n # HF Hub's search is already fuzzy! It handles typos and partial matches\n models = list_models(\n search=query,\n filter=\"text-generation\",\n sort=\"downloads\",\n direction=-1,\n limit=limit,\n )\n\n model_names = [model.id if hasattr(model, \"id\") else str(model) for model in models]\n\n # If direct search doesn't return enough results, try some variations\n if len(model_names) < 3:\n # Try searching for partial matches or common variations\n variations = []\n\n # Extract base name (e.g., \"gpt3\" from \"gpt-3.5\")\n base_query = query.lower().replace(\"-\", \"\").replace(\".\", \"\").replace(\"_\", \"\")\n if base_query != query.lower():\n variations.append(base_query)\n\n # Try common model name patterns\n if \"gpt\" in query.lower():\n variations.extend([\"gpt2\", \"gpt-neo\", \"gpt-j\", \"dialoGPT\"])\n elif \"llama\" in query.lower():\n variations.extend([\"llama2\", \"alpaca\", \"vicuna\"])\n elif \"bert\" in query.lower():\n variations.extend([\"roberta\", \"distilbert\", \"albert\"])\n\n # Search with variations\n for var in variations[:2]: # Limit to 2 variations to avoid too many API calls\n try:\n var_models = list_models(\n search=var,\n filter=\"text-generation\",\n sort=\"downloads\",\n direction=-1,\n limit=3,\n )\n var_names = [\n model.id if hasattr(model, \"id\") else str(model) for model in var_models\n ]\n model_names.extend(var_names)\n except Exception:\n continue\n\n # Remove duplicates while preserving order\n seen = set()\n unique_models = []\n for model in model_names:\n if model not in seen:\n seen.add(model)\n unique_models.append(model)\n\n return unique_models[:limit]\n\n except Exception:\n # If search fails, return empty list\n return []\n\n\ndef search_hf_models(query: str, limit: int = 10) -> list[str]:\n \"\"\"Simple search for HuggingFace models based on query (kept for backward compatibility)\"\"\"\n return search_hf_models_fuzzy(query, limit)\n\n\ndef validate_model_and_suggest(\n model_name: str, llm_type: str, host: Optional[str] = None\n) -> Optional[str]:\n \"\"\"Validate model name and provide suggestions if invalid\"\"\"\n if llm_type == \"ollama\":\n resolved_host = resolve_ollama_host(host)\n available_models = check_ollama_models(resolved_host)\n if available_models and model_name not in available_models:\n error_msg = f\"Model '{model_name}' not found in your local Ollama installation.\"\n\n # Check if the model exists remotely and get available tags\n model_exists_remotely, available_tags = check_ollama_model_exists_remotely(model_name)\n\n if model_exists_remotely and model_name in available_tags:\n # Exact model exists remotely - suggest pulling it\n error_msg += \"\\n\\nTo install the requested model:\\n\"\n error_msg += f\" ollama pull {model_name}\\n\"\n\n # Show local alternatives\n suggestions = search_ollama_models_fuzzy(model_name, available_models)\n if suggestions:\n error_msg += \"\\nOr use one of these similar installed models:\\n\"\n for i, suggestion in enumerate(suggestions, 1):\n error_msg += f\" {i}. {suggestion}\\n\"\n\n elif model_exists_remotely and available_tags:\n # Base model exists but requested tag doesn't - suggest correct tags\n base_model = model_name.split(\":\")[0]\n requested_tag = model_name.split(\":\", 1)[1] if \":\" in model_name else None\n\n error_msg += (\n f\"\\n\\nModel '{base_model}' exists, but tag '{requested_tag}' is not available.\"\n )\n error_msg += f\"\\n\\nAvailable {base_model} models you can install:\\n\"\n for i, tag in enumerate(available_tags[:8], 1):\n error_msg += f\" {i}. ollama pull {tag}\\n\"\n if len(available_tags) > 8:\n error_msg += f\" ... and {len(available_tags) - 8} more variants\\n\"\n\n # Also show local alternatives\n suggestions = search_ollama_models_fuzzy(model_name, available_models)\n if suggestions:\n error_msg += \"\\nOr use one of these similar installed models:\\n\"\n for i, suggestion in enumerate(suggestions, 1):\n error_msg += f\" {i}. {suggestion}\\n\"\n\n else:\n # Model doesn't exist remotely - show fuzzy suggestions\n suggestions = search_ollama_models_fuzzy(model_name, available_models)\n error_msg += f\"\\n\\nModel '{model_name}' was not found in Ollama's library.\"\n\n if suggestions:\n error_msg += (\n \"\\n\\nDid you mean one of these installed models?\\n\"\n + \"\\nTry to use ollama pull to install the model you need\\n\"\n )\n\n for i, suggestion in enumerate(suggestions, 1):\n error_msg += f\" {i}. {suggestion}\\n\"\n else:\n error_msg += \"\\n\\nYour installed models:\\n\"\n for i, model in enumerate(available_models[:8], 1):\n error_msg += f\" {i}. {model}\\n\"\n if len(available_models) > 8:\n error_msg += f\" ... and {len(available_models) - 8} more\\n\"\n\n error_msg += \"\\n\\nCommands:\"\n error_msg += \"\\n ollama list # List installed models\"\n if model_exists_remotely and available_tags:\n if model_name in available_tags:\n error_msg += f\"\\n ollama pull {model_name} # Install requested model\"\n else:\n error_msg += (\n f\"\\n ollama pull {available_tags[0]} # Install recommended variant\"\n )\n error_msg += \"\\n https://ollama.com/library # Browse available models\"\n return error_msg\n\n elif llm_type == \"hf\":\n # For HF models, we can do a quick existence check\n if not check_hf_model_exists(model_name):\n # Use HF Hub's native fuzzy search directly\n search_suggestions = search_hf_models_fuzzy(model_name, limit=8)\n\n error_msg = f\"Model '{model_name}' not found on HuggingFace Hub.\"\n if search_suggestions:\n error_msg += \"\\n\\nDid you mean one of these?\\n\"\n for i, suggestion in enumerate(search_suggestions, 1):\n error_msg += f\" {i}. {suggestion}\\n\"\n else:\n # Fallback to popular models if search returns nothing\n popular_models = get_popular_hf_models()\n error_msg += \"\\n\\nPopular chat models:\\n\"\n for i, model in enumerate(popular_models[:5], 1):\n error_msg += f\" {i}. {model}\\n\"\n\n error_msg += f\"\\nSearch more: https://huggingface.co/models?search={model_name}&pipeline_tag=text-generation\"\n return error_msg\n\n return None # Model is valid or we can't check\n\n\nclass LLMInterface(ABC):\n \"\"\"Abstract base class for a generic Language Model (LLM) interface.\"\"\"\n\n @abstractmethod\n def ask(self, prompt: str, **kwargs) -> str:\n \"\"\"\n Additional keyword arguments (kwargs) for advanced search customization. Example usage:\n chat.ask(\n \"What is ANN?\",\n top_k=10,\n complexity=64,\n beam_width=8,\n skip_search_reorder=True,\n recompute_beighbor_embeddings=True,\n dedup_node_dis=True,\n prune_ratio=0.1,\n batch_recompute=True,\n global_pruning=True\n )\n\n Supported kwargs:\n - complexity (int): Search complexity parameter (default: 32)\n - beam_width (int): Beam width for search (default: 4)\n - skip_search_reorder (bool): Skip search reorder step (default: False)\n - recompute_beighbor_embeddings (bool): Enable ZMQ embedding server for neighbor recomputation (default: False)\n - dedup_node_dis (bool): Deduplicate nodes by distance (default: False)\n - prune_ratio (float): Pruning ratio for search (default: 0.0)\n - batch_recompute (bool): Enable batch recomputation (default: False)\n - global_pruning (bool): Enable global pruning (default: False)\n \"\"\"\n\n # \"\"\"\n # Sends a prompt to the LLM and returns the generated text.\n\n # Args:\n # prompt: The input prompt for the LLM.\n # **kwargs: Additional keyword arguments for the LLM backend.\n\n # Returns:\n # The response string from the LLM.\n # \"\"\"\n pass\n\n\nclass OllamaChat(LLMInterface):\n \"\"\"LLM interface for Ollama models.\"\"\"\n\n def __init__(self, model: str = \"llama3:8b\", host: Optional[str] = None):\n self.model = model\n self.host = resolve_ollama_host(host)\n logger.info(f\"Initializing OllamaChat with model='{model}' and host='{self.host}'\")\n try:\n import requests\n\n # Check if the Ollama server is responsive\n if self.host:\n requests.get(self.host)\n\n # Pre-check model availability with helpful suggestions\n model_error = validate_model_and_suggest(model, \"ollama\", self.host)\n if model_error:\n raise ValueError(model_error)\n\n except ImportError:\n raise ImportError(\n \"The 'requests' library is required for Ollama. Please install it with 'pip install requests'.\"\n )\n except requests.exceptions.ConnectionError:\n logger.error(\n f\"Could not connect to Ollama at {self.host}. Please ensure Ollama is running.\"\n )\n raise ConnectionError(\n f\"Could not connect to Ollama at {self.host}. Please ensure Ollama is running.\"\n )\n\n def ask(self, prompt: str, **kwargs) -> str:\n import json\n\n import requests\n\n full_url = f\"{self.host}/api/generate\"\n\n # Handle thinking budget for reasoning models\n options = kwargs.copy()\n thinking_budget = kwargs.get(\"thinking_budget\")\n if thinking_budget:\n # Remove thinking_budget from options as it's not a standard Ollama option\n options.pop(\"thinking_budget\", None)\n # Only apply reasoning parameters to models that support it\n reasoning_supported_models = [\n \"gpt-oss:20b\",\n \"gpt-oss:120b\",\n \"deepseek-r1\",\n \"deepseek-coder\",\n ]\n\n if thinking_budget in [\"low\", \"medium\", \"high\"]:\n if any(model in self.model.lower() for model in reasoning_supported_models):\n options[\"reasoning\"] = {\"effort\": thinking_budget, \"exclude\": False}\n logger.info(f\"Applied reasoning effort={thinking_budget} to model {self.model}\")\n else:\n logger.warning(\n f\"Thinking budget '{thinking_budget}' requested but model '{self.model}' may not support reasoning parameters. Proceeding without reasoning.\"\n )\n\n payload = {\n \"model\": self.model,\n \"prompt\": prompt,\n \"stream\": False, # Keep it simple for now\n \"options\": options,\n }\n logger.debug(f\"Sending request to Ollama: {payload}\")\n try:\n logger.info(\"Sending request to Ollama and waiting for response...\")\n response = requests.post(full_url, data=json.dumps(payload))\n response.raise_for_status()\n\n # The response from Ollama can be a stream of JSON objects, handle this\n response_parts = response.text.strip().split(\"\\n\")\n full_response = \"\"\n for part in response_parts:\n if part:\n json_part = json.loads(part)\n full_response += json_part.get(\"response\", \"\")\n if json_part.get(\"done\"):\n break\n return full_response\n except requests.exceptions.RequestException as e:\n logger.error(f\"Error communicating with Ollama: {e}\")\n return f\"Error: Could not get a response from Ollama. Details: {e}\"\n\n\nclass HFChat(LLMInterface):\n \"\"\"LLM interface for local Hugging Face Transformers models with proper chat templates.\n\n Args:\n model_name (str): Name of the Hugging Face model to load.\n trust_remote_code (bool): Whether to allow execution of code from the model repository.\n Defaults to False for security. Only enable for trusted models as this can pose\n a security risk if the model repository is compromised.\n \"\"\"\n\n def __init__(\n self, model_name: str = \"deepseek-ai/deepseek-llm-7b-chat\", trust_remote_code: bool = False\n ):\n logger.info(f\"Initializing HFChat with model='{model_name}'\")\n\n # Security warning when trust_remote_code is enabled\n if trust_remote_code:\n logger.warning(\n \"SECURITY WARNING: trust_remote_code=True allows execution of arbitrary code from the model repository. \"\n \"Only enable this for models from trusted sources. This creates a potential security risk if the model \"\n \"repository is compromised.\"\n )\n\n self.trust_remote_code = trust_remote_code\n\n # Pre-check model availability with helpful suggestions\n model_error = validate_model_and_suggest(model_name, \"hf\")\n if model_error:\n raise ValueError(model_error)\n\n try:\n import torch\n from transformers import AutoModelForCausalLM, AutoTokenizer\n except ImportError:\n raise ImportError(\n \"The 'transformers' and 'torch' libraries are required for Hugging Face models. Please install them with 'pip install transformers torch'.\"\n )\n\n # Auto-detect device (check environment variable first)\n env_device = os.getenv(\"LEANN_LLM_DEVICE\")\n if env_device:\n self.device = env_device\n logger.info(f\"Using device from LEANN_LLM_DEVICE: {self.device}\")\n elif torch.cuda.is_available():\n self.device = \"cuda\"\n logger.info(\"CUDA is available. Using GPU.\")\n elif hasattr(torch.backends, \"mps\") and torch.backends.mps.is_available():\n self.device = \"mps\"\n logger.info(\"MPS is available. Using Apple Silicon GPU.\")\n else:\n self.device = \"cpu\"\n logger.info(\"No GPU detected. Using CPU.\")\n\n # Load tokenizer and model with timeout protection\n try:\n import signal\n\n def timeout_handler(signum, frame):\n raise TimeoutError(\"Model download/loading timed out\")\n\n # Set timeout for model loading (60 seconds)\n old_handler = signal.signal(signal.SIGALRM, timeout_handler)\n signal.alarm(60)\n\n try:\n logger.info(f\"Loading tokenizer for {model_name}...\")\n self.tokenizer = AutoTokenizer.from_pretrained(\n model_name, trust_remote_code=self.trust_remote_code\n )\n\n logger.info(f\"Loading model {model_name}...\")\n # Determine device_map based on device setting\n if self.device == \"cpu\":\n device_map = None\n elif self.device.startswith(\"cuda:\"):\n # Specific GPU requested, use it exclusively\n device_map = {\"\": self.device}\n else:\n # Auto mode: let HuggingFace distribute across available GPUs\n device_map = \"auto\"\n\n self.model = AutoModelForCausalLM.from_pretrained(\n model_name,\n torch_dtype=torch.float16 if self.device != \"cpu\" else torch.float32,\n device_map=device_map,\n trust_remote_code=self.trust_remote_code,\n )\n logger.info(f\"Successfully loaded {model_name}\")\n finally:\n signal.alarm(0) # Cancel the alarm\n signal.signal(signal.SIGALRM, old_handler) # Restore old handler\n\n except TimeoutError:\n logger.error(f\"Model loading timed out for {model_name}\")\n raise RuntimeError(\n f\"Model loading timed out for {model_name}. Please check your internet connection or try a smaller model.\"\n )\n except Exception as e:\n logger.error(f\"Failed to load model {model_name}: {e}\")\n raise\n\n # Move model to device if not using device_map\n if self.device != \"cpu\" and \"device_map\" not in str(self.model):\n self.model = self.model.to(self.device)\n\n # Set pad token if not present\n if self.tokenizer.pad_token is None:\n self.tokenizer.pad_token = self.tokenizer.eos_token\n\n def ask(self, prompt: str, **kwargs) -> str:\n print(\"kwargs in HF: \", kwargs)\n # Check if this is a Qwen model and add /no_think by default\n is_qwen_model = \"qwen\" in self.model.config._name_or_path.lower()\n\n # For Qwen models, automatically add /no_think to the prompt\n if is_qwen_model and \"/no_think\" not in prompt and \"/think\" not in prompt:\n prompt = prompt + \" /no_think\"\n\n # Prepare chat template\n messages = [{\"role\": \"user\", \"content\": prompt}]\n\n # Apply chat template if available\n if hasattr(self.tokenizer, \"apply_chat_template\"):\n try:\n formatted_prompt = self.tokenizer.apply_chat_template(\n messages, tokenize=False, add_generation_prompt=True\n )\n except Exception as e:\n logger.warning(f\"Chat template failed, using raw prompt: {e}\")\n formatted_prompt = prompt\n else:\n # Fallback for models without chat template\n formatted_prompt = prompt\n\n # Tokenize input\n inputs = self.tokenizer(\n formatted_prompt,\n return_tensors=\"pt\",\n padding=True,\n truncation=True,\n max_length=2048,\n )\n\n # Move inputs to device\n if self.device != \"cpu\":\n inputs = {k: v.to(self.device) for k, v in inputs.items()}\n\n # Set generation parameters\n generation_config = {\n \"max_new_tokens\": kwargs.get(\"max_tokens\", kwargs.get(\"max_new_tokens\", 512)),\n \"temperature\": kwargs.get(\"temperature\", 0.7),\n \"top_p\": kwargs.get(\"top_p\", 0.9),\n \"do_sample\": kwargs.get(\"temperature\", 0.7) > 0,\n \"pad_token_id\": self.tokenizer.eos_token_id,\n \"eos_token_id\": self.tokenizer.eos_token_id,\n }\n\n # Handle temperature=0 for greedy decoding\n if generation_config[\"temperature\"] == 0.0:\n generation_config[\"do_sample\"] = False\n generation_config.pop(\"temperature\")\n\n logger.info(f\"Generating with HuggingFace model, config: {generation_config}\")\n\n # Generate\n with torch.no_grad():\n outputs = self.model.generate(**inputs, **generation_config)\n\n # Decode response\n generated_tokens = outputs[0][inputs[\"input_ids\"].shape[1] :]\n response = self.tokenizer.decode(generated_tokens, skip_special_tokens=True)\n\n return response.strip()\n\n\nclass GeminiChat(LLMInterface):\n \"\"\"LLM interface for Google Gemini models.\"\"\"\n\n def __init__(self, model: str = \"gemini-2.5-flash\", api_key: Optional[str] = None):\n self.model = model\n self.api_key = api_key or os.getenv(\"GEMINI_API_KEY\")\n\n if not self.api_key:\n raise ValueError(\n \"Gemini API key is required. Set GEMINI_API_KEY environment variable or pass api_key parameter.\"\n )\n\n logger.info(f\"Initializing Gemini Chat with model='{model}'\")\n\n try:\n import google.genai as genai\n\n self.client = genai.Client(api_key=self.api_key)\n except ImportError:\n raise ImportError(\n \"The 'google-genai' library is required for Gemini models. Please install it with 'uv pip install google-genai'.\"\n )\n\n def ask(self, prompt: str, **kwargs) -> str:\n logger.info(f\"Sending request to Gemini with model {self.model}\")\n\n try:\n from google.genai.types import GenerateContentConfig\n\n generation_config = GenerateContentConfig(\n temperature=kwargs.get(\"temperature\", 0.7),\n max_output_tokens=kwargs.get(\"max_tokens\", 1000),\n )\n\n # Handle top_p parameter\n if \"top_p\" in kwargs:\n generation_config.top_p = kwargs[\"top_p\"]\n\n response = self.client.models.generate_content(\n model=self.model,\n contents=prompt,\n config=generation_config,\n )\n # Handle potential None response text\n response_text = response.text\n if response_text is None:\n logger.warning(\"Gemini returned None response text\")\n return \"\"\n return response_text.strip()\n except Exception as e:\n logger.error(f\"Error communicating with Gemini: {e}\")\n return f\"Error: Could not get a response from Gemini. Details: {e}\"\n\n\nclass OpenAIChat(LLMInterface):\n \"\"\"LLM interface for OpenAI models.\"\"\"\n\n def __init__(\n self,\n model: str = \"gpt-4o\",\n api_key: Optional[str] = None,\n base_url: Optional[str] = None,\n ):\n self.model = model\n self.base_url = resolve_openai_base_url(base_url)\n self.api_key = resolve_openai_api_key(api_key)\n\n if not self.api_key:\n raise ValueError(\n \"OpenAI API key is required. Set OPENAI_API_KEY environment variable or pass api_key parameter.\"\n )\n\n logger.info(\n \"Initializing OpenAI Chat with model='%s' and base_url='%s'\",\n model,\n self.base_url,\n )\n\n try:\n import openai\n\n self.client = openai.OpenAI(api_key=self.api_key, base_url=self.base_url)\n except ImportError:\n raise ImportError(\n \"The 'openai' library is required for OpenAI models. Please install it with 'pip install openai'.\"\n )\n\n def ask(self, prompt: str, **kwargs) -> str:\n # Default parameters for OpenAI\n params = {\n \"model\": self.model,\n \"messages\": [{\"role\": \"user\", \"content\": prompt}],\n \"temperature\": kwargs.get(\"temperature\", 0.7),\n }\n\n # Handle max_tokens vs max_completion_tokens based on model\n max_tokens = kwargs.get(\"max_tokens\", 1000)\n if \"o3\" in self.model or \"o4\" in self.model or \"o1\" in self.model:\n # o-series models use max_completion_tokens\n params[\"max_completion_tokens\"] = max_tokens\n params[\"temperature\"] = 1.0\n else:\n # Other models use max_tokens\n params[\"max_tokens\"] = max_tokens\n\n # Handle thinking budget for reasoning models\n thinking_budget = kwargs.get(\"thinking_budget\")\n if thinking_budget and thinking_budget in [\"low\", \"medium\", \"high\"]:\n # Check if this is an o-series model (partial match for model names)\n o_series_models = [\"o3\", \"o3-mini\", \"o4-mini\", \"o1\", \"o3-pro\", \"o3-deep-research\"]\n if any(model in self.model for model in o_series_models):\n # Use the correct OpenAI reasoning parameter format\n params[\"reasoning_effort\"] = thinking_budget\n logger.info(f\"Applied reasoning_effort={thinking_budget} to model {self.model}\")\n else:\n logger.warning(\n f\"Thinking budget '{thinking_budget}' requested but model '{self.model}' may not support reasoning parameters. Proceeding without reasoning.\"\n )\n\n # Add other kwargs (excluding thinking_budget as it's handled above)\n for k, v in kwargs.items():\n if k not in [\"max_tokens\", \"temperature\", \"thinking_budget\"]:\n params[k] = v\n\n logger.info(f\"Sending request to OpenAI with model {self.model}\")\n\n try:\n response = cast(Any, self.client.chat.completions).create(**params)\n print(\n f\"Total tokens = {response.usage.total_tokens}, prompt tokens = {response.usage.prompt_tokens}, completion tokens = {response.usage.completion_tokens}\"\n )\n if response.choices[0].finish_reason == \"length\":\n print(\"The query is exceeding the maximum allowed number of tokens\")\n return response.choices[0].message.content.strip()\n except Exception as e:\n logger.error(f\"Error communicating with OpenAI: {e}\")\n return f\"Error: Could not get a response from OpenAI. Details: {e}\"\n\n\nclass AnthropicChat(LLMInterface):\n \"\"\"LLM interface for Anthropic Claude models.\"\"\"\n\n def __init__(\n self,\n model: str = \"claude-haiku-4-5\",\n api_key: Optional[str] = None,\n base_url: Optional[str] = None,\n ):\n self.model = model\n self.base_url = resolve_anthropic_base_url(base_url)\n self.api_key = resolve_anthropic_api_key(api_key)\n\n if not self.api_key:\n raise ValueError(\n \"Anthropic API key is required. Set ANTHROPIC_API_KEY environment variable or pass api_key parameter.\"\n )\n\n logger.info(\n \"Initializing Anthropic Chat with model='%s' and base_url='%s'\",\n model,\n self.base_url,\n )\n\n try:\n import anthropic\n\n # Allow custom Anthropic-compatible endpoints via base_url\n self.client = anthropic.Anthropic(\n api_key=self.api_key,\n base_url=self.base_url,\n )\n except ImportError:\n raise ImportError(\n \"The 'anthropic' library is required for Anthropic models. Please install it with 'pip install anthropic'.\"\n )\n\n def ask(self, prompt: str, **kwargs) -> str:\n logger.info(f\"Sending request to Anthropic with model {self.model}\")\n\n try:\n # Anthropic API parameters\n params = {\n \"model\": self.model,\n \"max_tokens\": kwargs.get(\"max_tokens\", 1000),\n \"messages\": [{\"role\": \"user\", \"content\": prompt}],\n }\n\n # Add optional parameters\n if \"temperature\" in kwargs:\n params[\"temperature\"] = kwargs[\"temperature\"]\n if \"top_p\" in kwargs:\n params[\"top_p\"] = kwargs[\"top_p\"]\n\n response = self.client.messages.create(**params)\n\n # Extract text from response\n response_text = response.content[0].text\n\n # Log token usage\n print(\n f\"Total tokens = {response.usage.input_tokens + response.usage.output_tokens}, \"\n f\"input tokens = {response.usage.input_tokens}, \"\n f\"output tokens = {response.usage.output_tokens}\"\n )\n\n if response.stop_reason == \"max_tokens\":\n print(\"The query is exceeding the maximum allowed number of tokens\")\n\n return response_text.strip()\n except Exception as e:\n logger.error(f\"Error communicating with Anthropic: {e}\")\n return f\"Error: Could not get a response from Anthropic. Details: {e}\"\n\n\nclass MiniMaxChat(LLMInterface):\n \"\"\"LLM interface for MiniMax models via the OpenAI-compatible API.\n\n Supported models:\n - MiniMax-M2.5 (default): Peak Performance. Ultimate Value.\n - MiniMax-M2.5-highspeed: Same performance, faster and more agile.\n\n Both models support a 204,800-token context window.\n \"\"\"\n\n def __init__(\n self,\n model: str = \"MiniMax-M2.5\",\n api_key: Optional[str] = None,\n base_url: Optional[str] = None,\n ):\n self.model = model\n self.base_url = resolve_minimax_base_url(base_url)\n self.api_key = resolve_minimax_api_key(api_key)\n\n if not self.api_key:\n raise ValueError(\n \"MiniMax API key is required. Set MINIMAX_API_KEY environment variable or pass api_key parameter.\"\n )\n\n logger.info(\n \"Initializing MiniMax Chat with model='%s' and base_url='%s'\",\n model,\n self.base_url,\n )\n\n try:\n import openai\n\n self.client = openai.OpenAI(api_key=self.api_key, base_url=self.base_url)\n except ImportError:\n raise ImportError(\n \"The 'openai' library is required for MiniMax models. Please install it with 'pip install openai'.\"\n )\n\n def ask(self, prompt: str, **kwargs) -> str:\n # Default parameters for MiniMax (OpenAI-compatible)\n params = {\n \"model\": self.model,\n \"messages\": [{\"role\": \"user\", \"content\": prompt}],\n \"temperature\": kwargs.get(\"temperature\", 0.7),\n \"max_tokens\": kwargs.get(\"max_tokens\", 1000),\n }\n\n # Add optional parameters\n if \"top_p\" in kwargs:\n params[\"top_p\"] = kwargs[\"top_p\"]\n\n logger.info(f\"Sending request to MiniMax with model {self.model}\")\n\n try:\n response = cast(Any, self.client.chat.completions).create(**params)\n print(\n f\"Total tokens = {response.usage.total_tokens}, prompt tokens = {response.usage.prompt_tokens}, completion tokens = {response.usage.completion_tokens}\"\n )\n if response.choices[0].finish_reason == \"length\":\n print(\"The query is exceeding the maximum allowed number of tokens\")\n return response.choices[0].message.content.strip()\n except Exception as e:\n logger.error(f\"Error communicating with MiniMax: {e}\")\n return f\"Error: Could not get a response from MiniMax. Details: {e}\"\n\n\nclass SimulatedChat(LLMInterface):\n \"\"\"A simple simulated chat for testing and development.\"\"\"\n\n def ask(self, prompt: str, **kwargs) -> str:\n logger.info(\"Simulating LLM call...\")\n print(\"Prompt sent to LLM (simulation):\", prompt[:500] + \"...\")\n return \"This is a simulated answer from the LLM based on the retrieved context.\"\n\n\ndef get_llm(llm_config: Optional[dict[str, Any]] = None) -> LLMInterface:\n \"\"\"\n Factory function to get an LLM interface based on configuration.\n\n Args:\n llm_config: A dictionary specifying the LLM type and its parameters.\n Example: {\"type\": \"ollama\", \"model\": \"llama3\"}\n {\"type\": \"hf\", \"model\": \"distilgpt2\"}\n None (for simulation mode)\n\n Returns:\n An instance of an LLMInterface subclass.\n \"\"\"\n if llm_config is None:\n llm_config = {\n \"type\": \"openai\",\n \"model\": \"gpt-4o\",\n \"api_key\": os.getenv(\"OPENAI_API_KEY\"),\n }\n\n llm_type = llm_config.get(\"type\", \"openai\")\n model = llm_config.get(\"model\")\n\n logger.info(f\"Attempting to create LLM of type='{llm_type}' with model='{model}'\")\n\n if llm_type == \"ollama\":\n return OllamaChat(\n model=model or \"llama3:8b\",\n host=llm_config.get(\"host\"),\n )\n elif llm_type == \"hf\":\n return HFChat(\n model_name=model or \"deepseek-ai/deepseek-llm-7b-chat\",\n trust_remote_code=llm_config.get(\"trust_remote_code\", False),\n )\n elif llm_type == \"openai\":\n return OpenAIChat(\n model=model or \"gpt-4o\",\n api_key=llm_config.get(\"api_key\"),\n base_url=llm_config.get(\"base_url\"),\n )\n elif llm_type == \"gemini\":\n return GeminiChat(model=model or \"gemini-2.5-flash\", api_key=llm_config.get(\"api_key\"))\n elif llm_type == \"anthropic\":\n return AnthropicChat(\n model=model or \"claude-3-5-sonnet-20241022\",\n api_key=llm_config.get(\"api_key\"),\n base_url=llm_config.get(\"base_url\"),\n )\n elif llm_type == \"minimax\":\n return MiniMaxChat(\n model=model or \"MiniMax-M2.5\",\n api_key=llm_config.get(\"api_key\"),\n base_url=llm_config.get(\"base_url\"),\n )\n elif llm_type == \"simulated\":\n return SimulatedChat()\n else:\n raise ValueError(f\"Unknown LLM type: '{llm_type}'\")\n" }, { "path": "packages/leann-core/src/leann/chunking_utils.py", "content": "\"\"\"\nEnhanced chunking utilities with AST-aware code chunking support.\nPackaged within leann-core so installed wheels can import it reliably.\n\"\"\"\n\nimport logging\nfrom pathlib import Path\nfrom typing import Any, Optional\n\nfrom llama_index.core.node_parser import SentenceSplitter\n\nlogger = logging.getLogger(__name__)\n\n# Flag to ensure AST token warning only shown once per session\n_ast_token_warning_shown = False\n\n\ndef estimate_token_count(text: str) -> int:\n \"\"\"\n Estimate token count for a text string.\n Uses conservative estimation: ~4 characters per token for natural text,\n ~1.2 tokens per character for code (worse tokenization).\n\n Args:\n text: Input text to estimate tokens for\n\n Returns:\n Estimated token count\n \"\"\"\n try:\n import tiktoken\n\n encoder = tiktoken.get_encoding(\"cl100k_base\")\n return len(encoder.encode(text))\n except ImportError:\n # Fallback: Conservative character-based estimation\n # Assume worst case for code: 1.2 tokens per character\n return int(len(text) * 1.2)\n\n\ndef calculate_safe_chunk_size(\n model_token_limit: int,\n overlap_tokens: int,\n chunking_mode: str = \"traditional\",\n safety_factor: float = 0.9,\n) -> int:\n \"\"\"\n Calculate safe chunk size accounting for overlap and safety margin.\n\n Args:\n model_token_limit: Maximum tokens supported by embedding model\n overlap_tokens: Overlap size (tokens for traditional, chars for AST)\n chunking_mode: \"traditional\" (tokens) or \"ast\" (characters)\n safety_factor: Safety margin (0.9 = 10% safety margin)\n\n Returns:\n Safe chunk size: tokens for traditional, characters for AST\n \"\"\"\n safe_limit = int(model_token_limit * safety_factor)\n\n if chunking_mode == \"traditional\":\n # Traditional chunking uses tokens\n # Max chunk = chunk_size + overlap, so chunk_size = limit - overlap\n return max(1, safe_limit - overlap_tokens)\n else: # AST chunking\n # AST uses characters, need to convert\n # Conservative estimate: 1.2 tokens per char for code\n overlap_chars = int(overlap_tokens * 3) # ~3 chars per token for code\n safe_chars = int(safe_limit / 1.2)\n return max(1, safe_chars - overlap_chars)\n\n\ndef validate_chunk_token_limits(chunks: list[str], max_tokens: int = 512) -> tuple[list[str], int]:\n \"\"\"\n Validate that chunks don't exceed token limits and truncate if necessary.\n\n Args:\n chunks: List of text chunks to validate\n max_tokens: Maximum tokens allowed per chunk\n\n Returns:\n Tuple of (validated_chunks, num_truncated)\n \"\"\"\n validated_chunks = []\n num_truncated = 0\n\n for i, chunk in enumerate(chunks):\n estimated_tokens = estimate_token_count(chunk)\n\n if estimated_tokens > max_tokens:\n # Truncate chunk to fit token limit\n try:\n import tiktoken\n\n encoder = tiktoken.get_encoding(\"cl100k_base\")\n tokens = encoder.encode(chunk)\n if len(tokens) > max_tokens:\n truncated_tokens = tokens[:max_tokens]\n truncated_chunk = encoder.decode(truncated_tokens)\n validated_chunks.append(truncated_chunk)\n num_truncated += 1\n logger.warning(\n f\"Truncated chunk {i} from {len(tokens)} to {max_tokens} tokens \"\n f\"(from {len(chunk)} to {len(truncated_chunk)} characters)\"\n )\n else:\n validated_chunks.append(chunk)\n except ImportError:\n # Fallback: Conservative character truncation\n char_limit = int(max_tokens / 1.2) # Conservative for code\n if len(chunk) > char_limit:\n truncated_chunk = chunk[:char_limit]\n validated_chunks.append(truncated_chunk)\n num_truncated += 1\n logger.warning(\n f\"Truncated chunk {i} from {len(chunk)} to {char_limit} characters \"\n f\"(conservative estimate for {max_tokens} tokens)\"\n )\n else:\n validated_chunks.append(chunk)\n else:\n validated_chunks.append(chunk)\n\n if num_truncated > 0:\n logger.warning(f\"Truncated {num_truncated}/{len(chunks)} chunks to fit token limits\")\n\n return validated_chunks, num_truncated\n\n\n# Code file extensions supported by astchunk\nCODE_EXTENSIONS = {\n \".py\": \"python\",\n \".java\": \"java\",\n \".cs\": \"csharp\",\n \".ts\": \"typescript\",\n \".tsx\": \"typescript\",\n \".js\": \"typescript\",\n \".jsx\": \"typescript\",\n}\n\n\ndef detect_code_files(documents, code_extensions=None) -> tuple[list, list]:\n \"\"\"Separate documents into code files and regular text files.\"\"\"\n if code_extensions is None:\n code_extensions = CODE_EXTENSIONS\n\n code_docs = []\n text_docs = []\n\n for doc in documents:\n file_path = doc.metadata.get(\"file_path\", \"\") or doc.metadata.get(\"file_name\", \"\")\n if file_path:\n file_ext = Path(file_path).suffix.lower()\n if file_ext in code_extensions:\n doc.metadata[\"language\"] = code_extensions[file_ext]\n doc.metadata[\"is_code\"] = True\n code_docs.append(doc)\n else:\n doc.metadata[\"is_code\"] = False\n text_docs.append(doc)\n else:\n doc.metadata[\"is_code\"] = False\n text_docs.append(doc)\n\n logger.info(f\"Detected {len(code_docs)} code files and {len(text_docs)} text files\")\n return code_docs, text_docs\n\n\ndef get_language_from_extension(file_path: str) -> Optional[str]:\n \"\"\"Return language string from a filename/extension using CODE_EXTENSIONS.\"\"\"\n ext = Path(file_path).suffix.lower()\n return CODE_EXTENSIONS.get(ext)\n\n\ndef create_ast_chunks(\n documents,\n max_chunk_size: int = 512,\n chunk_overlap: int = 64,\n metadata_template: str = \"default\",\n) -> list[dict[str, Any]]:\n \"\"\"Create AST-aware chunks from code documents using astchunk.\n\n Falls back to traditional chunking if astchunk is unavailable.\n\n Returns:\n List of dicts with {\"text\": str, \"metadata\": dict}\n \"\"\"\n try:\n from astchunk import ASTChunkBuilder # optional dependency\n except ImportError as e:\n logger.error(f\"astchunk not available: {e}\")\n logger.info(\"Falling back to traditional chunking for code files\")\n return _traditional_chunks_as_dicts(documents, max_chunk_size, chunk_overlap)\n\n all_chunks = []\n for doc in documents:\n language = doc.metadata.get(\"language\")\n if not language:\n logger.warning(\"No language detected; falling back to traditional chunking\")\n all_chunks.extend(_traditional_chunks_as_dicts([doc], max_chunk_size, chunk_overlap))\n continue\n\n try:\n # Warn once if AST chunk size + overlap might exceed common token limits\n # Note: Actual truncation happens at embedding time with dynamic model limits\n global _ast_token_warning_shown\n estimated_max_tokens = int(\n (max_chunk_size + chunk_overlap) * 1.2\n ) # Conservative estimate\n if estimated_max_tokens > 512 and not _ast_token_warning_shown:\n logger.warning(\n f\"AST chunk size ({max_chunk_size}) + overlap ({chunk_overlap}) = {max_chunk_size + chunk_overlap} chars \"\n f\"may exceed 512 token limit (~{estimated_max_tokens} tokens estimated). \"\n f\"Consider reducing --ast-chunk-size to {int(400 / 1.2)} or --ast-chunk-overlap to {int(50 / 1.2)}. \"\n f\"Note: Chunks will be auto-truncated at embedding time based on your model's actual token limit.\"\n )\n _ast_token_warning_shown = True\n\n configs = {\n \"max_chunk_size\": max_chunk_size,\n \"language\": language,\n \"metadata_template\": metadata_template,\n \"chunk_overlap\": chunk_overlap if chunk_overlap > 0 else 0,\n }\n\n repo_metadata = {\n \"file_path\": doc.metadata.get(\"file_path\", \"\"),\n \"file_name\": doc.metadata.get(\"file_name\", \"\"),\n \"source\": doc.metadata.get(\"source\", \"\"),\n \"creation_date\": doc.metadata.get(\"creation_date\", \"\"),\n \"last_modified_date\": doc.metadata.get(\"last_modified_date\", \"\"),\n }\n configs[\"repo_level_metadata\"] = repo_metadata\n\n chunk_builder = ASTChunkBuilder(**configs)\n code_content = doc.get_content()\n if not code_content or not code_content.strip():\n logger.warning(\"Empty code content, skipping\")\n continue\n\n chunks = chunk_builder.chunkify(code_content)\n for chunk in chunks:\n chunk_text: str | None = None\n astchunk_metadata: dict[str, Any] = {}\n\n if hasattr(chunk, \"text\"):\n chunk_text = str(chunk.text) if chunk.text else None\n elif isinstance(chunk, str):\n chunk_text = chunk\n elif isinstance(chunk, dict):\n # Handle astchunk format: {\"content\": \"...\", \"metadata\": {...}}\n if \"content\" in chunk:\n chunk_text = chunk[\"content\"]\n astchunk_metadata = chunk.get(\"metadata\", {})\n elif \"text\" in chunk:\n chunk_text = chunk[\"text\"]\n else:\n chunk_text = str(chunk) # Last resort\n else:\n chunk_text = str(chunk)\n\n if chunk_text and chunk_text.strip():\n # Extract document-level metadata\n doc_metadata = {\n \"file_path\": doc.metadata.get(\"file_path\", \"\"),\n \"file_name\": doc.metadata.get(\"file_name\", \"\"),\n \"source\": doc.metadata.get(\"source\", \"\"),\n }\n if \"creation_date\" in doc.metadata:\n doc_metadata[\"creation_date\"] = doc.metadata[\"creation_date\"]\n if \"last_modified_date\" in doc.metadata:\n doc_metadata[\"last_modified_date\"] = doc.metadata[\"last_modified_date\"]\n\n # Merge document metadata + astchunk metadata\n combined_metadata = {**doc_metadata, **astchunk_metadata}\n\n all_chunks.append({\"text\": chunk_text.strip(), \"metadata\": combined_metadata})\n\n logger.info(\n f\"Created {len(chunks)} AST chunks from {language} file: {doc.metadata.get('file_name', 'unknown')}\"\n )\n except Exception as e:\n logger.warning(f\"AST chunking failed for {language} file: {e}\")\n logger.info(\"Falling back to traditional chunking\")\n all_chunks.extend(_traditional_chunks_as_dicts([doc], max_chunk_size, chunk_overlap))\n\n return all_chunks\n\n\ndef create_traditional_chunks(\n documents, chunk_size: int = 256, chunk_overlap: int = 128\n) -> list[dict[str, Any]]:\n \"\"\"Create traditional text chunks using LlamaIndex SentenceSplitter.\n\n Returns:\n List of dicts with {\"text\": str, \"metadata\": dict}\n \"\"\"\n if chunk_size <= 0:\n logger.warning(f\"Invalid chunk_size={chunk_size}, using default value of 256\")\n chunk_size = 256\n if chunk_overlap < 0:\n chunk_overlap = 0\n if chunk_overlap >= chunk_size:\n chunk_overlap = chunk_size // 2\n\n node_parser = SentenceSplitter(\n chunk_size=chunk_size,\n chunk_overlap=chunk_overlap,\n separator=\" \",\n paragraph_separator=\"\\n\\n\",\n )\n\n result = []\n for doc in documents:\n # Extract document-level metadata\n doc_metadata = {\n \"file_path\": doc.metadata.get(\"file_path\", \"\"),\n \"file_name\": doc.metadata.get(\"file_name\", \"\"),\n \"source\": doc.metadata.get(\"source\", \"\"),\n }\n if \"creation_date\" in doc.metadata:\n doc_metadata[\"creation_date\"] = doc.metadata[\"creation_date\"]\n if \"last_modified_date\" in doc.metadata:\n doc_metadata[\"last_modified_date\"] = doc.metadata[\"last_modified_date\"]\n\n try:\n nodes = node_parser.get_nodes_from_documents([doc])\n if nodes:\n for node in nodes:\n result.append({\"text\": node.get_content(), \"metadata\": doc_metadata})\n except Exception as e:\n logger.error(f\"Traditional chunking failed for document: {e}\")\n content = doc.get_content()\n if content and content.strip():\n result.append({\"text\": content.strip(), \"metadata\": doc_metadata})\n\n return result\n\n\ndef _traditional_chunks_as_dicts(\n documents, chunk_size: int = 256, chunk_overlap: int = 128\n) -> list[dict[str, Any]]:\n \"\"\"Helper: Traditional chunking that returns dict format for consistency.\n\n This is now just an alias for create_traditional_chunks for backwards compatibility.\n \"\"\"\n return create_traditional_chunks(documents, chunk_size, chunk_overlap)\n\n\ndef create_text_chunks(\n documents,\n chunk_size: int = 256,\n chunk_overlap: int = 128,\n use_ast_chunking: bool = False,\n ast_chunk_size: int = 512,\n ast_chunk_overlap: int = 64,\n code_file_extensions: Optional[list[str]] = None,\n ast_fallback_traditional: bool = True,\n) -> list[dict[str, Any]]:\n \"\"\"Create text chunks from documents with optional AST support for code files.\n\n Returns:\n List of dicts with {\"text\": str, \"metadata\": dict}\n \"\"\"\n if not documents:\n logger.warning(\"No documents provided for chunking\")\n return []\n\n local_code_extensions = CODE_EXTENSIONS.copy()\n if code_file_extensions:\n ext_mapping = {\n \".py\": \"python\",\n \".java\": \"java\",\n \".cs\": \"c_sharp\",\n \".ts\": \"typescript\",\n \".tsx\": \"typescript\",\n }\n for ext in code_file_extensions:\n if ext.lower() not in local_code_extensions:\n if ext.lower() in ext_mapping:\n local_code_extensions[ext.lower()] = ext_mapping[ext.lower()]\n else:\n logger.warning(f\"Unsupported extension {ext}, will use traditional chunking\")\n\n all_chunks = []\n if use_ast_chunking:\n code_docs, text_docs = detect_code_files(documents, local_code_extensions)\n if code_docs:\n try:\n ast_chunks = create_ast_chunks(\n code_docs, max_chunk_size=ast_chunk_size, chunk_overlap=ast_chunk_overlap\n )\n # Prepend line numbers to code chunks for navigation\n for chunk in ast_chunks:\n start_line = chunk.get(\"metadata\", {}).get(\"start_line_no\")\n if start_line is not None:\n lines = chunk[\"text\"].split(\"\\n\")\n end_line = start_line + len(lines) - 1\n w = len(str(end_line))\n chunk[\"text\"] = \"\\n\".join(\n f\"{start_line + i:>{w}}|{line}\" for i, line in enumerate(lines)\n )\n all_chunks.extend(ast_chunks)\n except Exception as e:\n logger.error(f\"AST chunking failed: {e}\")\n if ast_fallback_traditional:\n all_chunks.extend(\n _traditional_chunks_as_dicts(code_docs, chunk_size, chunk_overlap)\n )\n else:\n raise\n if text_docs:\n all_chunks.extend(_traditional_chunks_as_dicts(text_docs, chunk_size, chunk_overlap))\n else:\n all_chunks = _traditional_chunks_as_dicts(documents, chunk_size, chunk_overlap)\n\n logger.info(f\"Total chunks created: {len(all_chunks)}\")\n\n # Note: Token truncation is now handled at embedding time with dynamic model limits\n # See get_model_token_limit() and truncate_to_token_limit() in embedding_compute.py\n return all_chunks\n" }, { "path": "packages/leann-core/src/leann/cli.py", "content": "import argparse\nimport asyncio\nimport contextlib\nimport hashlib\nimport io\nimport json\nimport os\nimport pickle\nimport sys\nimport time\nimport uuid\nfrom pathlib import Path\nfrom typing import Any, Optional, Union\n\nfrom llama_index.core import SimpleDirectoryReader\nfrom llama_index.core.node_parser import SentenceSplitter\nfrom tqdm import tqdm\n\nfrom .api import LeannBuilder, LeannChat, LeannSearcher\nfrom .embedding_server_manager import EmbeddingServerManager\nfrom .interactive_utils import create_cli_session\nfrom .registry import register_project_directory\nfrom .settings import (\n resolve_anthropic_base_url,\n resolve_minimax_api_key,\n resolve_minimax_base_url,\n resolve_ollama_host,\n resolve_openai_api_key,\n resolve_openai_base_url,\n)\nfrom .sync import FileSynchronizer\n\n\ndef _normalize_path(path: str) -> str:\n \"\"\"Return absolute path string for consistent keys.\"\"\"\n if not path:\n return path\n return str(Path(path).resolve())\n\n\n@contextlib.contextmanager\ndef suppress_cpp_output(suppress: bool = True):\n \"\"\"Context manager to suppress C++ stdout/stderr output from FAISS/HNSW\n while preserving Python print() output.\n\n C++ native code writes directly to OS file descriptors (fd 1 / fd 2).\n Python print() goes through sys.stdout / sys.stderr, which are Python\n file objects. We redirect the OS fds to /dev/null (silencing C++) but\n point sys.stdout / sys.stderr at copies of the *original* fds so that\n Python output still reaches the terminal.\n \"\"\"\n if not suppress:\n yield\n return\n\n # 1. Duplicate the original OS file descriptors\n saved_stdout_fd = os.dup(1)\n saved_stderr_fd = os.dup(2)\n\n # 2. Build Python file objects that write to the saved (real) fds.\n # closefd=False so closing these wrappers won't close the duped fds.\n py_stdout = io.TextIOWrapper(\n io.FileIO(saved_stdout_fd, mode=\"w\", closefd=False), encoding=sys.stdout.encoding or \"utf-8\"\n )\n py_stderr = io.TextIOWrapper(\n io.FileIO(saved_stderr_fd, mode=\"w\", closefd=False), encoding=sys.stderr.encoding or \"utf-8\"\n )\n\n old_sys_stdout = sys.stdout\n old_sys_stderr = sys.stderr\n\n try:\n # 3. Redirect OS-level fds to /dev/null → silences C++ output\n devnull = os.open(os.devnull, os.O_WRONLY)\n os.dup2(devnull, 1)\n os.dup2(devnull, 2)\n os.close(devnull)\n\n # 4. Point Python's sys.stdout/stderr at the real terminal\n sys.stdout = py_stdout\n sys.stderr = py_stderr\n\n yield\n finally:\n # 5. Restore everything\n # Flush wrappers first (they still need the saved fds to be open)\n py_stdout.flush()\n py_stderr.flush()\n\n sys.stdout = old_sys_stdout\n sys.stderr = old_sys_stderr\n\n os.dup2(saved_stdout_fd, 1)\n os.dup2(saved_stderr_fd, 2)\n os.close(saved_stdout_fd)\n os.close(saved_stderr_fd)\n\n\ndef extract_pdf_text_with_pymupdf(file_path: str) -> str | None:\n \"\"\"Extract text from PDF using PyMuPDF for better quality.\"\"\"\n try:\n import fitz # PyMuPDF\n\n doc = fitz.open(file_path)\n text = \"\"\n for page in doc:\n text += page.get_text()\n doc.close()\n return text\n except ImportError:\n # Fallback to default reader\n return None\n\n\ndef extract_pdf_text_with_pdfplumber(file_path: str) -> str | None:\n \"\"\"Extract text from PDF using pdfplumber for better quality.\"\"\"\n try:\n import pdfplumber\n\n text = \"\"\n with pdfplumber.open(file_path) as pdf:\n for page in pdf.pages:\n text += page.extract_text() or \"\"\n return text\n except ImportError:\n # Fallback to default reader\n return None\n\n\nclass LeannCLI:\n def __init__(self):\n # Always use project-local .leann directory (like .git)\n self.indexes_dir = Path.cwd() / \".leann\" / \"indexes\"\n self.indexes_dir.mkdir(parents=True, exist_ok=True)\n\n # Default parser for documents\n self.node_parser = SentenceSplitter(\n chunk_size=256, chunk_overlap=128, separator=\" \", paragraph_separator=\"\\n\\n\"\n )\n\n # Code-optimized parser\n self.code_parser = SentenceSplitter(\n chunk_size=512, # Larger chunks for code context\n chunk_overlap=50, # Less overlap to preserve function boundaries\n separator=\"\\n\", # Split by lines for code\n paragraph_separator=\"\\n\\n\", # Preserve logical code blocks\n )\n\n def get_index_path(self, index_name: str) -> str:\n index_dir = self.indexes_dir / index_name\n return str(index_dir / \"documents.leann\")\n\n def index_exists(self, index_name: str) -> bool:\n index_dir = self.indexes_dir / index_name\n meta_file = index_dir / \"documents.leann.meta.json\"\n return meta_file.exists()\n\n def create_parser(self) -> argparse.ArgumentParser:\n parser = argparse.ArgumentParser(\n prog=\"leann\",\n description=\"The smallest vector index in the world. RAG Everything with LEANN!\",\n formatter_class=argparse.RawDescriptionHelpFormatter,\n epilog=\"\"\"\nExamples:\n leann build my-docs --docs ./documents # Build index from directory\n leann build my-code --docs ./src ./tests ./config # Build index from multiple directories\n leann build my-files --docs ./file1.py ./file2.txt ./docs/ # Build index from files and directories\n leann build my-mixed --docs ./readme.md ./src/ ./config.json # Build index from mixed files/dirs\n leann build my-ppts --docs ./ --file-types .pptx,.pdf # Index only PowerPoint and PDF files\n leann search my-docs \"query\" # Search in my-docs index\n leann ask my-docs \"question\" # Ask my-docs index\n leann react my-docs \"complex question\" # Use ReAct agent for multiturn retrieval\n leann list # List all stored indexes\n leann remove my-docs # Remove an index (local first, then global)\n \"\"\",\n )\n\n # Global verbosity options\n verbosity_group = parser.add_mutually_exclusive_group()\n verbosity_group.add_argument(\n \"-v\",\n \"--verbose\",\n action=\"store_true\",\n help=\"Show detailed output including C++ backend logs from FAISS/HNSW\",\n )\n verbosity_group.add_argument(\n \"-q\",\n \"--quiet\",\n action=\"store_true\",\n help=\"Suppress all non-essential output (default behavior)\",\n )\n\n subparsers = parser.add_subparsers(dest=\"command\", help=\"Available commands\")\n\n # Build command\n build_parser = subparsers.add_parser(\"build\", help=\"Build document index\")\n build_parser.add_argument(\n \"index_name\", nargs=\"?\", help=\"Index name (default: current directory name)\"\n )\n build_parser.add_argument(\n \"--docs\",\n type=str,\n nargs=\"+\",\n default=[\".\"],\n help=\"Documents directories and/or files (default: current directory)\",\n )\n build_parser.add_argument(\n \"--backend-name\",\n type=str,\n default=\"hnsw\",\n choices=[\"hnsw\", \"diskann\", \"ivf\"],\n help=\"Backend to use (default: hnsw)\",\n )\n build_parser.add_argument(\n \"--embedding-model\",\n type=str,\n default=\"facebook/contriever\",\n help=\"Embedding model (default: facebook/contriever)\",\n )\n build_parser.add_argument(\n \"--embedding-mode\",\n type=str,\n default=\"sentence-transformers\",\n choices=[\"sentence-transformers\", \"openai\", \"mlx\", \"ollama\"],\n help=\"Embedding backend mode (default: sentence-transformers)\",\n )\n build_parser.add_argument(\n \"--embedding-host\",\n type=str,\n default=None,\n help=\"Override Ollama-compatible embedding host\",\n )\n build_parser.add_argument(\n \"--embedding-api-base\",\n type=str,\n default=None,\n help=\"Base URL for OpenAI-compatible embedding services\",\n )\n build_parser.add_argument(\n \"--embedding-api-key\",\n type=str,\n default=None,\n help=\"API key for embedding service (defaults to OPENAI_API_KEY)\",\n )\n build_parser.add_argument(\n \"--embedding-prompt-template\",\n type=str,\n default=None,\n help=\"Prompt template to prepend to all texts for embedding (e.g., 'query: ' for search)\",\n )\n build_parser.add_argument(\n \"--query-prompt-template\",\n type=str,\n default=None,\n help=\"Prompt template for queries (different from build template for task-specific models)\",\n )\n build_parser.add_argument(\n \"--force\",\n \"-f\",\n action=\"store_true\",\n help=\"Force full rebuild of existing index (without this, build does incremental update: add new files only)\",\n )\n build_parser.add_argument(\n \"--graph-degree\", type=int, default=32, help=\"Graph degree (default: 32)\"\n )\n build_parser.add_argument(\n \"--complexity\", type=int, default=64, help=\"Build complexity (default: 64)\"\n )\n build_parser.add_argument(\"--num-threads\", type=int, default=1)\n build_parser.add_argument(\n \"--compact\",\n action=argparse.BooleanOptionalAction,\n default=False,\n help=\"Use compact (CSR) graph storage. Compact indices are read-only and cannot be updated incrementally. Default: false (allows incremental updates while still pruning embeddings for 97%% compression).\",\n )\n build_parser.add_argument(\n \"--recompute\",\n action=argparse.BooleanOptionalAction,\n default=True,\n help=\"Enable recomputation (default: true)\",\n )\n build_parser.add_argument(\n \"--file-types\",\n type=str,\n help=\"Comma-separated list of file extensions to include (e.g., '.txt,.pdf,.pptx'). If not specified, uses default supported types.\",\n )\n build_parser.add_argument(\n \"--include-hidden\",\n action=argparse.BooleanOptionalAction,\n default=False,\n help=\"Include hidden files and directories (paths starting with '.') during indexing (default: false)\",\n )\n build_parser.add_argument(\n \"--doc-chunk-size\",\n type=int,\n default=256,\n help=\"Document chunk size in TOKENS (default: 256). Final chunks may be larger due to overlap. For 512 token models: recommended 350 tokens (350 + 128 overlap = 478 max)\",\n )\n build_parser.add_argument(\n \"--doc-chunk-overlap\",\n type=int,\n default=128,\n help=\"Document chunk overlap in TOKENS (default: 128). Added to chunk size, not included in it\",\n )\n build_parser.add_argument(\n \"--code-chunk-size\",\n type=int,\n default=512,\n help=\"Code chunk size in TOKENS (default: 512). Final chunks may be larger due to overlap. For 512 token models: recommended 400 tokens (400 + 50 overlap = 450 max)\",\n )\n build_parser.add_argument(\n \"--code-chunk-overlap\",\n type=int,\n default=50,\n help=\"Code chunk overlap in TOKENS (default: 50). Added to chunk size, not included in it\",\n )\n build_parser.add_argument(\n \"--use-ast-chunking\",\n action=\"store_true\",\n help=\"Enable AST-aware chunking for code files (requires astchunk)\",\n )\n build_parser.add_argument(\n \"--ast-chunk-size\",\n type=int,\n default=300,\n help=\"AST chunk size in CHARACTERS (non-whitespace) (default: 300). Final chunks may be larger due to overlap and expansion. For 512 token models: recommended 300 chars (300 + 64 overlap ~= 480 tokens)\",\n )\n build_parser.add_argument(\n \"--ast-chunk-overlap\",\n type=int,\n default=64,\n help=\"AST chunk overlap in CHARACTERS (default: 64). Added to chunk size, not included in it. ~1.2 tokens per character for code\",\n )\n build_parser.add_argument(\n \"--ast-fallback-traditional\",\n action=\"store_true\",\n default=True,\n help=\"Fall back to traditional chunking if AST chunking fails (default: True)\",\n )\n\n # Watch command\n watch_parser = subparsers.add_parser(\n \"watch\",\n help=\"Monitor source files and auto-rebuild index when changes are detected\",\n )\n watch_parser.add_argument(\"index_name\", help=\"Index name\")\n watch_parser.add_argument(\n \"--interval\",\n type=int,\n default=30,\n help=\"Poll interval in seconds (default: 30)\",\n )\n watch_parser.add_argument(\n \"--once\",\n action=\"store_true\",\n help=\"Check once for changes and exit (do not loop)\",\n )\n watch_parser.add_argument(\n \"--dry-run\",\n action=\"store_true\",\n help=\"Report changes without rebuilding (original watch behavior)\",\n )\n\n # Search command\n search_parser = subparsers.add_parser(\"search\", help=\"Search documents\")\n search_parser.add_argument(\"index_name\", help=\"Index name\")\n search_parser.add_argument(\"query\", help=\"Search query\")\n search_parser.add_argument(\n \"--top-k\", type=int, default=5, help=\"Number of results (default: 5)\"\n )\n search_parser.add_argument(\n \"--complexity\", type=int, default=64, help=\"Search complexity (default: 64)\"\n )\n search_parser.add_argument(\"--beam-width\", type=int, default=1)\n search_parser.add_argument(\"--prune-ratio\", type=float, default=0.0)\n search_parser.add_argument(\n \"--recompute\",\n dest=\"recompute_embeddings\",\n action=argparse.BooleanOptionalAction,\n default=True,\n help=\"Enable/disable embedding recomputation (default: enabled). Should not do a `no-recompute` search in a `recompute` build.\",\n )\n search_parser.add_argument(\n \"--pruning-strategy\",\n choices=[\"global\", \"local\", \"proportional\"],\n default=\"global\",\n help=\"Pruning strategy (default: global)\",\n )\n search_parser.add_argument(\n \"--json\",\n action=\"store_true\",\n help=\"Output results as JSON array (machine-readable)\",\n )\n search_parser.add_argument(\n \"--non-interactive\",\n action=\"store_true\",\n help=\"Non-interactive mode: automatically select index without prompting\",\n )\n search_parser.add_argument(\n \"--show-metadata\",\n action=\"store_true\",\n help=\"Display file paths and metadata in search results\",\n )\n search_parser.add_argument(\n \"--embedding-prompt-template\",\n type=str,\n default=None,\n help=\"Prompt template to prepend to query for embedding (e.g., 'query: ' for search)\",\n )\n search_parser.add_argument(\n \"--daemon\",\n dest=\"use_daemon\",\n action=argparse.BooleanOptionalAction,\n default=True,\n help=\"Enable/disable cross-process daemon reuse for embedding server (default: enabled)\",\n )\n search_parser.add_argument(\n \"--daemon-ttl\",\n type=int,\n default=900,\n help=\"Daemon idle TTL in seconds (default: 900, 0 = never expire)\",\n )\n search_parser.add_argument(\n \"--warmup\",\n dest=\"enable_warmup\",\n action=argparse.BooleanOptionalAction,\n default=True,\n help=\"Enable/disable warmup when starting embedding server (default: enabled)\",\n )\n\n # Warmup command\n warmup_parser = subparsers.add_parser(\"warmup\", help=\"Warm up an index embedding server\")\n warmup_parser.add_argument(\"index_name\", help=\"Index name\")\n warmup_parser.add_argument(\n \"--daemon\",\n dest=\"use_daemon\",\n action=argparse.BooleanOptionalAction,\n default=True,\n help=\"Enable/disable daemon mode for warmup (default: enabled)\",\n )\n warmup_parser.add_argument(\n \"--daemon-ttl\",\n type=int,\n default=900,\n help=\"Daemon idle TTL in seconds (default: 900, 0 = never expire)\",\n )\n warmup_parser.add_argument(\n \"--warmup\",\n dest=\"enable_warmup\",\n action=argparse.BooleanOptionalAction,\n default=True,\n help=\"Enable/disable warmup request itself (default: enabled)\",\n )\n\n # Daemon command\n daemon_parser = subparsers.add_parser(\"daemon\", help=\"Manage embedding daemons\")\n daemon_subparsers = daemon_parser.add_subparsers(dest=\"daemon_command\")\n\n daemon_start = daemon_subparsers.add_parser(\"start\", help=\"Start daemon for an index\")\n daemon_start.add_argument(\"index_name\", help=\"Index name\")\n daemon_start.add_argument(\n \"--daemon-ttl\",\n type=int,\n default=900,\n help=\"Daemon idle TTL in seconds (default: 900, 0 = never expire)\",\n )\n daemon_start.add_argument(\n \"--warmup\",\n dest=\"enable_warmup\",\n action=argparse.BooleanOptionalAction,\n default=True,\n help=\"Enable/disable startup warmup (default: enabled)\",\n )\n\n daemon_stop = daemon_subparsers.add_parser(\"stop\", help=\"Stop daemon(s)\")\n daemon_stop.add_argument(\"index_name\", nargs=\"?\", help=\"Index name to stop\")\n daemon_stop.add_argument(\n \"--all\",\n action=\"store_true\",\n help=\"Stop all LEANN embedding daemons\",\n )\n\n daemon_status = daemon_subparsers.add_parser(\"status\", help=\"Show daemon status\")\n daemon_status.add_argument(\"index_name\", nargs=\"?\", help=\"Optional index name filter\")\n\n # Ask command\n ask_parser = subparsers.add_parser(\"ask\", help=\"Ask questions\")\n ask_parser.add_argument(\"index_name\", help=\"Index name\")\n ask_parser.add_argument(\n \"query\",\n nargs=\"?\",\n help=\"Question to ask (omit for prompt or when using --interactive)\",\n )\n ask_parser.add_argument(\n \"--llm\",\n type=str,\n default=\"ollama\",\n choices=[\"simulated\", \"ollama\", \"hf\", \"openai\", \"anthropic\", \"minimax\"],\n help=\"LLM provider (default: ollama)\",\n )\n ask_parser.add_argument(\n \"--model\", type=str, default=\"qwen3:8b\", help=\"Model name (default: qwen3:8b)\"\n )\n ask_parser.add_argument(\n \"--host\",\n type=str,\n default=None,\n help=\"Override Ollama-compatible host (defaults to LEANN_OLLAMA_HOST/OLLAMA_HOST)\",\n )\n ask_parser.add_argument(\n \"--interactive\", \"-i\", action=\"store_true\", help=\"Interactive chat mode\"\n )\n ask_parser.add_argument(\n \"--top-k\", type=int, default=20, help=\"Retrieval count (default: 20)\"\n )\n ask_parser.add_argument(\"--complexity\", type=int, default=32)\n ask_parser.add_argument(\"--beam-width\", type=int, default=1)\n ask_parser.add_argument(\"--prune-ratio\", type=float, default=0.0)\n ask_parser.add_argument(\n \"--recompute\",\n dest=\"recompute_embeddings\",\n action=argparse.BooleanOptionalAction,\n default=True,\n help=\"Enable/disable embedding recomputation during ask (default: enabled)\",\n )\n ask_parser.add_argument(\n \"--pruning-strategy\",\n choices=[\"global\", \"local\", \"proportional\"],\n default=\"global\",\n )\n ask_parser.add_argument(\n \"--thinking-budget\",\n type=str,\n choices=[\"low\", \"medium\", \"high\"],\n default=None,\n help=\"Thinking budget for reasoning models (low/medium/high). Supported by GPT-Oss:20b and other reasoning models.\",\n )\n ask_parser.add_argument(\n \"--api-base\",\n type=str,\n default=None,\n help=\"Base URL for OpenAI-compatible APIs (e.g., http://localhost:10000/v1)\",\n )\n ask_parser.add_argument(\n \"--api-key\",\n type=str,\n default=None,\n help=\"API key for cloud LLM providers (OpenAI, Anthropic)\",\n )\n\n # React command (multiturn retrieval agent)\n react_parser = subparsers.add_parser(\n \"react\", help=\"Use ReAct agent for multiturn retrieval and reasoning\"\n )\n react_parser.add_argument(\"index_name\", help=\"Index name\")\n react_parser.add_argument(\"query\", help=\"Question to research\")\n react_parser.add_argument(\n \"--llm\",\n type=str,\n default=\"ollama\",\n choices=[\"simulated\", \"ollama\", \"hf\", \"openai\", \"anthropic\", \"minimax\"],\n help=\"LLM provider (default: ollama)\",\n )\n react_parser.add_argument(\n \"--model\", type=str, default=\"qwen3:8b\", help=\"Model name (default: qwen3:8b)\"\n )\n react_parser.add_argument(\n \"--host\",\n type=str,\n default=None,\n help=\"Override Ollama-compatible host (defaults to LEANN_OLLAMA_HOST/OLLAMA_HOST)\",\n )\n react_parser.add_argument(\n \"--top-k\", type=int, default=5, help=\"Number of results per search (default: 5)\"\n )\n react_parser.add_argument(\n \"--max-iterations\",\n type=int,\n default=5,\n help=\"Maximum number of search iterations (default: 5)\",\n )\n react_parser.add_argument(\n \"--api-base\",\n type=str,\n default=None,\n help=\"Base URL for OpenAI-compatible APIs (e.g., http://localhost:10000/v1)\",\n )\n react_parser.add_argument(\n \"--api-key\",\n type=str,\n default=None,\n help=\"API key for cloud LLM providers (OpenAI, Anthropic)\",\n )\n\n # List command\n subparsers.add_parser(\"list\", help=\"List all indexes\")\n\n # Remove command\n remove_parser = subparsers.add_parser(\"remove\", help=\"Remove an index\")\n remove_parser.add_argument(\"index_name\", help=\"Index name to remove\")\n remove_parser.add_argument(\n \"--force\", \"-f\", action=\"store_true\", help=\"Force removal without confirmation\"\n )\n\n # Serve command (HTTP API server)\n serve_parser = subparsers.add_parser(\n \"serve\", help=\"Start HTTP API server for LEANN vector DB\"\n )\n serve_parser.add_argument(\n \"--host\", type=str, default=None, help=\"Host to bind to (default: 0.0.0.0)\"\n )\n serve_parser.add_argument(\n \"--port\", type=int, default=None, help=\"Port to bind to (default: 8000)\"\n )\n\n return parser\n\n def register_project_dir(self):\n \"\"\"Register current project directory in global registry\"\"\"\n register_project_directory()\n\n def _build_gitignore_parser(self, docs_dir: str):\n \"\"\"Build gitignore parser using gitignore-parser library.\"\"\"\n from gitignore_parser import parse_gitignore\n\n # Try to parse the root .gitignore\n gitignore_path = Path(docs_dir) / \".gitignore\"\n\n if gitignore_path.exists():\n try:\n # gitignore-parser automatically handles all subdirectory .gitignore files!\n matches = parse_gitignore(str(gitignore_path))\n print(f\"📋 Loaded .gitignore from {docs_dir} (includes all subdirectories)\")\n return matches\n except Exception as e:\n print(f\"Warning: Could not parse .gitignore: {e}\")\n else:\n print(\"📋 No .gitignore found\")\n\n # Fallback: basic pattern matching for essential files\n essential_patterns = {\".git\", \".DS_Store\", \"__pycache__\", \"node_modules\", \".venv\", \"venv\"}\n\n def basic_matches(file_path):\n path_parts = Path(file_path).parts\n return any(part in essential_patterns for part in path_parts)\n\n return basic_matches\n\n def _should_exclude_file(self, file_path: Path, gitignore_matches) -> bool:\n \"\"\"Check if a file should be excluded using gitignore parser.\n\n Always match against absolute, posix-style paths for consistency with\n gitignore_parser expectations.\n \"\"\"\n try:\n absolute_path = file_path.resolve()\n except Exception:\n absolute_path = Path(str(file_path))\n return gitignore_matches(absolute_path.as_posix())\n\n def _is_git_submodule(self, path: Path) -> bool:\n \"\"\"Check if a path is a git submodule.\"\"\"\n try:\n # Find the git repo root\n current_dir = Path.cwd()\n while current_dir != current_dir.parent:\n if (current_dir / \".git\").exists():\n gitmodules_path = current_dir / \".gitmodules\"\n if gitmodules_path.exists():\n # Read .gitmodules to check if this path is a submodule\n gitmodules_content = gitmodules_path.read_text()\n # Convert path to relative to git root\n try:\n relative_path = path.resolve().relative_to(current_dir)\n # Check if this path appears in .gitmodules\n return f\"path = {relative_path}\" in gitmodules_content\n except ValueError:\n # Path is not under git root\n return False\n break\n current_dir = current_dir.parent\n return False\n except Exception:\n # If anything goes wrong, assume it's not a submodule\n return False\n\n def list_indexes(self):\n # Get all project directories with .leann\n global_registry = Path.home() / \".leann\" / \"projects.json\"\n all_projects = []\n\n if global_registry.exists():\n try:\n import json\n\n with open(global_registry) as f:\n all_projects = json.load(f)\n except Exception:\n pass\n\n # Filter to only existing directories with .leann\n valid_projects = []\n for project_dir in all_projects:\n project_path = Path(project_dir)\n if project_path.exists() and (project_path / \".leann\" / \"indexes\").exists():\n valid_projects.append(project_path)\n\n # Add current project if it has .leann but not in registry\n current_path = Path.cwd()\n if (current_path / \".leann\" / \"indexes\").exists() and current_path not in valid_projects:\n valid_projects.append(current_path)\n\n # Separate current and other projects\n other_projects = []\n\n for project_path in valid_projects:\n if project_path != current_path:\n other_projects.append(project_path)\n\n print(\"📚 LEANN Indexes\")\n print(\"=\" * 50)\n\n total_indexes = 0\n current_indexes_count = 0\n\n # Show current project first (most important)\n print(\"\\n🏠 Current Project\")\n print(f\" {current_path}\")\n print(\" \" + \"─\" * 45)\n\n current_indexes = self._discover_indexes_in_project(\n current_path, exclude_dirs=other_projects\n )\n if current_indexes:\n for idx in current_indexes:\n total_indexes += 1\n current_indexes_count += 1\n type_icon = \"📁\" if idx[\"type\"] == \"cli\" else \"📄\"\n print(f\" {current_indexes_count}. {type_icon} {idx['name']} {idx['status']}\")\n if idx[\"size_mb\"] > 0:\n print(f\" 📦 Size: {idx['size_mb']:.1f} MB\")\n else:\n print(\" 📭 No indexes in current project\")\n\n # Show other projects (reference information)\n if other_projects:\n print(\"\\n\\n🗂️ Other Projects\")\n print(\" \" + \"─\" * 45)\n\n for project_path in other_projects:\n project_indexes = self._discover_indexes_in_project(project_path)\n if not project_indexes:\n continue\n\n print(f\"\\n 📂 {project_path.name}\")\n print(f\" {project_path}\")\n\n for idx in project_indexes:\n total_indexes += 1\n type_icon = \"📁\" if idx[\"type\"] == \"cli\" else \"📄\"\n print(f\" • {type_icon} {idx['name']} {idx['status']}\")\n if idx[\"size_mb\"] > 0:\n print(f\" 📦 {idx['size_mb']:.1f} MB\")\n\n # Summary and usage info\n print(\"\\n\" + \"=\" * 50)\n if total_indexes == 0:\n print(\"💡 Get started:\")\n print(\" leann build my-docs --docs ./documents\")\n else:\n # Count only projects that have at least one discoverable index\n projects_count = 0\n for p in valid_projects:\n if p == current_path:\n discovered = self._discover_indexes_in_project(p, exclude_dirs=other_projects)\n else:\n discovered = self._discover_indexes_in_project(p)\n if len(discovered) > 0:\n projects_count += 1\n print(f\"📊 Total: {total_indexes} indexes across {projects_count} projects\")\n\n if current_indexes_count > 0:\n print(\"\\n💫 Quick start (current project):\")\n # Get first index from current project for example\n current_indexes_dir = current_path / \".leann\" / \"indexes\"\n if current_indexes_dir.exists():\n current_index_dirs = [d for d in current_indexes_dir.iterdir() if d.is_dir()]\n if current_index_dirs:\n example_name = current_index_dirs[0].name\n print(f' leann search {example_name} \"your query\"')\n print(f\" leann ask {example_name} --interactive\")\n else:\n print(\"\\n💡 Create your first index:\")\n print(\" leann build my-docs --docs ./documents\")\n\n def _discover_indexes_in_project(\n self, project_path: Path, exclude_dirs: Optional[list[Path]] = None\n ):\n \"\"\"Discover all indexes in a project directory (both CLI and apps formats)\n\n exclude_dirs: when provided, skip any APP-format index files that are\n located under these directories. This prevents duplicates when the\n current project is a parent directory of other registered projects.\n \"\"\"\n indexes = []\n exclude_dirs = exclude_dirs or []\n # normalize to resolved paths once for comparison\n try:\n exclude_dirs_resolved = [p.resolve() for p in exclude_dirs]\n except Exception:\n exclude_dirs_resolved = exclude_dirs\n\n # 1. CLI format: .leann/indexes/index_name/\n cli_indexes_dir = project_path / \".leann\" / \"indexes\"\n if cli_indexes_dir.exists():\n for index_dir in cli_indexes_dir.iterdir():\n if index_dir.is_dir():\n meta_file = index_dir / \"documents.leann.meta.json\"\n status = \"✅\" if meta_file.exists() else \"❌\"\n\n size_mb = 0\n if meta_file.exists():\n try:\n size_mb = sum(\n f.stat().st_size for f in index_dir.iterdir() if f.is_file()\n ) / (1024 * 1024)\n except (OSError, PermissionError):\n pass\n\n indexes.append(\n {\n \"name\": index_dir.name,\n \"type\": \"cli\",\n \"status\": status,\n \"size_mb\": size_mb,\n \"path\": index_dir,\n }\n )\n\n # 2. Apps format: *.leann.meta.json files anywhere in the project\n cli_indexes_dir = project_path / \".leann\" / \"indexes\"\n for meta_file in project_path.rglob(\"*.leann.meta.json\"):\n if meta_file.is_file():\n # Skip CLI-built indexes (which store meta under .leann/indexes//)\n try:\n if cli_indexes_dir.exists() and cli_indexes_dir in meta_file.parents:\n continue\n except Exception:\n pass\n # Skip meta files that live under excluded directories\n try:\n meta_parent_resolved = meta_file.parent.resolve()\n if any(\n meta_parent_resolved.is_relative_to(ex_dir)\n for ex_dir in exclude_dirs_resolved\n ):\n continue\n except Exception:\n # best effort; if resolve or comparison fails, do not exclude\n pass\n # Use the parent directory name as the app index display name\n display_name = meta_file.parent.name\n # Extract file base used to store files\n file_base = meta_file.name.replace(\".leann.meta.json\", \"\")\n\n # Apps indexes are considered complete if the .leann.meta.json file exists\n status = \"✅\"\n\n # Calculate total size of all related files (use file base)\n size_mb = 0\n try:\n index_dir = meta_file.parent\n for related_file in index_dir.glob(f\"{file_base}.leann*\"):\n size_mb += related_file.stat().st_size / (1024 * 1024)\n except (OSError, PermissionError):\n pass\n\n indexes.append(\n {\n \"name\": display_name,\n \"type\": \"app\",\n \"status\": status,\n \"size_mb\": size_mb,\n \"path\": meta_file,\n }\n )\n\n return indexes\n\n def remove_index(self, index_name: str, force: bool = False):\n \"\"\"Safely remove an index - always show all matches for transparency\"\"\"\n\n # Always do a comprehensive search for safety\n print(f\"🔍 Searching for all indexes named '{index_name}'...\")\n all_matches = self._find_all_matching_indexes(index_name)\n\n if not all_matches:\n print(f\"❌ Index '{index_name}' not found in any project.\")\n return False\n\n if len(all_matches) == 1:\n return self._remove_single_match(all_matches[0], index_name, force)\n else:\n return self._remove_from_multiple_matches(all_matches, index_name, force)\n\n def _find_all_matching_indexes(self, index_name: str):\n \"\"\"Find all indexes with the given name across all projects\"\"\"\n matches = []\n\n # Get all registered projects\n global_registry = Path.home() / \".leann\" / \"projects.json\"\n all_projects = []\n\n if global_registry.exists():\n try:\n import json\n\n with open(global_registry) as f:\n all_projects = json.load(f)\n except Exception:\n pass\n\n # Always include current project\n current_path = Path.cwd()\n if str(current_path) not in all_projects:\n all_projects.append(str(current_path))\n\n # Search across all projects\n for project_dir in all_projects:\n project_path = Path(project_dir)\n if not project_path.exists():\n continue\n\n # 1) CLI-format index under .leann/indexes/\n index_dir = project_path / \".leann\" / \"indexes\" / index_name\n if index_dir.exists():\n is_current = project_path == current_path\n matches.append(\n {\n \"project_path\": project_path,\n \"index_dir\": index_dir,\n \"is_current\": is_current,\n \"kind\": \"cli\",\n }\n )\n\n # 2) App-format indexes\n # We support two ways of addressing apps:\n # a) by the file base (e.g., `pdf_documents`)\n # b) by the parent directory name (e.g., `new_txt`)\n seen_app_meta = set()\n\n # 2a) by file base\n for meta_file in project_path.rglob(f\"{index_name}.leann.meta.json\"):\n if meta_file.is_file():\n # Skip CLI-built indexes' meta under .leann/indexes\n try:\n cli_indexes_dir = project_path / \".leann\" / \"indexes\"\n if cli_indexes_dir.exists() and cli_indexes_dir in meta_file.parents:\n continue\n except Exception:\n pass\n is_current = project_path == current_path\n key = (str(project_path), str(meta_file))\n if key in seen_app_meta:\n continue\n seen_app_meta.add(key)\n matches.append(\n {\n \"project_path\": project_path,\n \"files_dir\": meta_file.parent,\n \"meta_file\": meta_file,\n \"is_current\": is_current,\n \"kind\": \"app\",\n \"display_name\": meta_file.parent.name,\n \"file_base\": meta_file.name.replace(\".leann.meta.json\", \"\"),\n }\n )\n\n # 2b) by parent directory name\n for meta_file in project_path.rglob(\"*.leann.meta.json\"):\n if meta_file.is_file() and meta_file.parent.name == index_name:\n # Skip CLI-built indexes' meta under .leann/indexes\n try:\n cli_indexes_dir = project_path / \".leann\" / \"indexes\"\n if cli_indexes_dir.exists() and cli_indexes_dir in meta_file.parents:\n continue\n except Exception:\n pass\n is_current = project_path == current_path\n key = (str(project_path), str(meta_file))\n if key in seen_app_meta:\n continue\n seen_app_meta.add(key)\n matches.append(\n {\n \"project_path\": project_path,\n \"files_dir\": meta_file.parent,\n \"meta_file\": meta_file,\n \"is_current\": is_current,\n \"kind\": \"app\",\n \"display_name\": meta_file.parent.name,\n \"file_base\": meta_file.name.replace(\".leann.meta.json\", \"\"),\n }\n )\n\n # Sort: current project first, then by project name\n matches.sort(key=lambda x: (not x[\"is_current\"], x[\"project_path\"].name))\n return matches\n\n def _remove_single_match(self, match, index_name: str, force: bool):\n \"\"\"Handle removal when only one match is found\"\"\"\n project_path = match[\"project_path\"]\n is_current = match[\"is_current\"]\n kind = match.get(\"kind\", \"cli\")\n\n if is_current:\n location_info = \"current project\"\n emoji = \"🏠\"\n else:\n location_info = f\"other project '{project_path.name}'\"\n emoji = \"📂\"\n\n print(f\"✅ Found 1 index named '{index_name}':\")\n print(f\" {emoji} Location: {location_info}\")\n if kind == \"cli\":\n print(f\" 📍 Path: {project_path / '.leann' / 'indexes' / index_name}\")\n else:\n print(f\" 📍 Meta: {match['meta_file']}\")\n\n if not force:\n if not is_current:\n print(\"\\n⚠️ CROSS-PROJECT REMOVAL!\")\n print(\" This will delete the index from another project.\")\n\n response = input(f\" ❓ Confirm removal from {location_info}? (y/N): \").strip().lower()\n if response not in [\"y\", \"yes\"]:\n print(\" ❌ Removal cancelled.\")\n return False\n\n if kind == \"cli\":\n return self._delete_index_directory(\n match[\"index_dir\"],\n index_name,\n project_path if not is_current else None,\n is_app=False,\n )\n else:\n return self._delete_index_directory(\n match[\"files_dir\"],\n match.get(\"display_name\", index_name),\n project_path if not is_current else None,\n is_app=True,\n meta_file=match.get(\"meta_file\"),\n app_file_base=match.get(\"file_base\"),\n )\n\n def _remove_from_multiple_matches(self, matches, index_name: str, force: bool):\n \"\"\"Handle removal when multiple matches are found\"\"\"\n\n print(f\"⚠️ Found {len(matches)} indexes named '{index_name}':\")\n print(\" \" + \"─\" * 50)\n\n for i, match in enumerate(matches, 1):\n project_path = match[\"project_path\"]\n is_current = match[\"is_current\"]\n kind = match.get(\"kind\", \"cli\")\n\n if is_current:\n print(f\" {i}. 🏠 Current project ({'CLI' if kind == 'cli' else 'APP'})\")\n else:\n print(f\" {i}. 📂 {project_path.name} ({'CLI' if kind == 'cli' else 'APP'})\")\n\n # Show path details\n if kind == \"cli\":\n print(f\" 📍 {project_path / '.leann' / 'indexes' / index_name}\")\n else:\n print(f\" 📍 {match['meta_file']}\")\n\n # Show size info\n try:\n if kind == \"cli\":\n size_mb = sum(\n f.stat().st_size for f in match[\"index_dir\"].iterdir() if f.is_file()\n ) / (1024 * 1024)\n else:\n file_base = match.get(\"file_base\")\n size_mb = 0.0\n if file_base:\n size_mb = sum(\n f.stat().st_size\n for f in match[\"files_dir\"].glob(f\"{file_base}.leann*\")\n if f.is_file()\n ) / (1024 * 1024)\n print(f\" 📦 Size: {size_mb:.1f} MB\")\n except (OSError, PermissionError):\n pass\n\n print(\" \" + \"─\" * 50)\n\n if force:\n print(\" ❌ Multiple matches found, but --force specified.\")\n print(\" Please run without --force to choose which one to remove.\")\n return False\n\n try:\n choice = input(\n f\" ❓ Which one to remove? (1-{len(matches)}, or 'c' to cancel): \"\n ).strip()\n if choice.lower() == \"c\":\n print(\" ❌ Removal cancelled.\")\n return False\n\n choice_idx = int(choice) - 1\n if 0 <= choice_idx < len(matches):\n selected_match = matches[choice_idx]\n project_path = selected_match[\"project_path\"]\n is_current = selected_match[\"is_current\"]\n kind = selected_match.get(\"kind\", \"cli\")\n\n location = \"current project\" if is_current else f\"'{project_path.name}' project\"\n print(f\" 🎯 Selected: Remove from {location}\")\n\n # Final confirmation for safety\n confirm = input(\n f\" ❓ FINAL CONFIRMATION - Type '{index_name}' to proceed: \"\n ).strip()\n if confirm != index_name:\n print(\" ❌ Confirmation failed. Removal cancelled.\")\n return False\n\n if kind == \"cli\":\n return self._delete_index_directory(\n selected_match[\"index_dir\"],\n index_name,\n project_path if not is_current else None,\n is_app=False,\n )\n else:\n return self._delete_index_directory(\n selected_match[\"files_dir\"],\n selected_match.get(\"display_name\", index_name),\n project_path if not is_current else None,\n is_app=True,\n meta_file=selected_match.get(\"meta_file\"),\n app_file_base=selected_match.get(\"file_base\"),\n )\n else:\n print(\" ❌ Invalid choice. Removal cancelled.\")\n return False\n\n except (ValueError, KeyboardInterrupt):\n print(\"\\n ❌ Invalid input. Removal cancelled.\")\n return False\n\n def _delete_index_directory(\n self,\n index_dir: Path,\n index_display_name: str,\n project_path: Optional[Path] = None,\n is_app: bool = False,\n meta_file: Optional[Path] = None,\n app_file_base: Optional[str] = None,\n ):\n \"\"\"Delete a CLI index directory or APP index files safely.\"\"\"\n try:\n if is_app:\n removed = 0\n errors = 0\n # Delete only files that belong to this app index (based on file base)\n pattern_base = app_file_base or \"\"\n for f in index_dir.glob(f\"{pattern_base}.leann*\"):\n try:\n f.unlink()\n removed += 1\n except Exception:\n errors += 1\n # Best-effort: also remove the meta file if specified and still exists\n if meta_file and meta_file.exists():\n try:\n meta_file.unlink()\n removed += 1\n except Exception:\n errors += 1\n\n if removed > 0 and errors == 0:\n if project_path:\n print(\n f\"✅ App index '{index_display_name}' removed from {project_path.name}\"\n )\n else:\n print(f\"✅ App index '{index_display_name}' removed successfully\")\n return True\n elif removed > 0 and errors > 0:\n print(\n f\"⚠️ App index '{index_display_name}' partially removed (some files couldn't be deleted)\"\n )\n return True\n else:\n print(\n f\"❌ No files found to remove for app index '{index_display_name}' in {index_dir}\"\n )\n return False\n else:\n import shutil\n\n shutil.rmtree(index_dir)\n\n if project_path:\n print(f\"✅ Index '{index_display_name}' removed from {project_path.name}\")\n else:\n print(f\"✅ Index '{index_display_name}' removed successfully\")\n return True\n except Exception as e:\n print(f\"❌ Error removing index '{index_display_name}': {e}\")\n return False\n\n def load_documents(\n self,\n docs_paths: Union[str, list],\n custom_file_types: Union[str, None] = None,\n include_hidden: bool = False,\n args: Optional[dict[str, Any]] = None,\n ):\n # Handle both single path (string) and multiple paths (list) for backward compatibility\n if isinstance(docs_paths, str):\n docs_paths = [docs_paths]\n\n # Separate files and directories\n files = []\n directories = []\n for path in docs_paths:\n path_obj = Path(path)\n if path_obj.is_file():\n files.append(str(path_obj))\n elif path_obj.is_dir():\n # Check if this is a git submodule - if so, skip it\n if self._is_git_submodule(path_obj):\n print(f\"⚠️ Skipping git submodule: {path}\")\n continue\n directories.append(str(path_obj))\n else:\n print(f\"⚠️ Warning: Path '{path}' does not exist, skipping...\")\n continue\n\n # Print summary of what we're processing\n total_items = len(files) + len(directories)\n items_desc = []\n if files:\n items_desc.append(f\"{len(files)} file{'s' if len(files) > 1 else ''}\")\n if directories:\n items_desc.append(\n f\"{len(directories)} director{'ies' if len(directories) > 1 else 'y'}\"\n )\n\n print(f\"Loading documents from {' and '.join(items_desc)} ({total_items} total):\")\n if files:\n print(f\" 📄 Files: {', '.join([Path(f).name for f in files])}\")\n if directories:\n print(f\" 📁 Directories: {', '.join(directories)}\")\n\n if custom_file_types:\n print(f\"Using custom file types: {custom_file_types}\")\n\n all_documents = []\n\n # Helper to detect hidden path components\n def _path_has_hidden_segment(p: Path) -> bool:\n return any(part.startswith(\".\") and part not in [\".\", \"..\"] for part in p.parts)\n\n # First, process individual files if any\n if files:\n print(f\"\\n🔄 Processing {len(files)} individual file{'s' if len(files) > 1 else ''}...\")\n\n # Load individual files using SimpleDirectoryReader with input_files\n # Note: We skip gitignore filtering for explicitly specified files\n try:\n # Group files by their parent directory for efficient loading\n from collections import defaultdict\n\n files_by_dir = defaultdict(list)\n for file_path in files:\n file_path_obj = Path(file_path)\n if not include_hidden and _path_has_hidden_segment(file_path_obj):\n print(f\" ⚠️ Skipping hidden file: {file_path}\")\n continue\n parent_dir = str(file_path_obj.parent)\n files_by_dir[parent_dir].append(str(file_path_obj))\n\n # Load files from each parent directory\n for parent_dir, file_list in files_by_dir.items():\n print(\n f\" Loading {len(file_list)} file{'s' if len(file_list) > 1 else ''} from {parent_dir}\"\n )\n try:\n file_docs = SimpleDirectoryReader(\n parent_dir,\n input_files=file_list,\n # exclude_hidden only affects directory scans; input_files are explicit\n filename_as_id=True,\n ).load_data()\n for doc in file_docs:\n if not doc.metadata.get(\"source\"):\n doc.metadata[\"source\"] = doc.metadata.get(\"file_path\", \"\")\n all_documents.extend(file_docs)\n print(\n f\" ✅ Loaded {len(file_docs)} document{'s' if len(file_docs) > 1 else ''}\"\n )\n except Exception as e:\n print(f\" ❌ Warning: Could not load files from {parent_dir}: {e}\")\n\n except Exception as e:\n print(f\"❌ Error processing individual files: {e}\")\n\n # Define file extensions to process\n if custom_file_types:\n # Parse custom file types from comma-separated string\n code_extensions = [ext.strip() for ext in custom_file_types.split(\",\") if ext.strip()]\n # Ensure extensions start with a dot\n code_extensions = [ext if ext.startswith(\".\") else f\".{ext}\" for ext in code_extensions]\n else:\n # Use default supported file types\n code_extensions = [\n # Original document types\n \".txt\",\n \".md\",\n \".docx\",\n \".pptx\",\n # Code files for Claude Code integration\n \".py\",\n \".js\",\n \".ts\",\n \".jsx\",\n \".tsx\",\n \".java\",\n \".cpp\",\n \".c\",\n \".h\",\n \".hpp\",\n \".cs\",\n \".go\",\n \".rs\",\n \".rb\",\n \".php\",\n \".swift\",\n \".kt\",\n \".scala\",\n \".r\",\n \".sql\",\n \".sh\",\n \".bash\",\n \".zsh\",\n \".fish\",\n \".ps1\",\n \".bat\",\n # Config and markup files\n \".json\",\n \".yaml\",\n \".yml\",\n \".xml\",\n \".toml\",\n \".ini\",\n \".cfg\",\n \".conf\",\n \".html\",\n \".css\",\n \".scss\",\n \".less\",\n \".vue\",\n \".svelte\",\n # Data science\n \".ipynb\",\n \".R\",\n \".py\",\n \".jl\",\n ]\n\n # Process each directory\n if directories:\n print(\n f\"\\n🔄 Processing {len(directories)} director{'ies' if len(directories) > 1 else 'y'}...\"\n )\n\n for docs_dir in directories:\n print(f\"Processing directory: {docs_dir}\")\n # Build gitignore parser for each directory\n gitignore_matches = self._build_gitignore_parser(docs_dir)\n\n # Try to use better PDF parsers first, but only if PDFs are requested\n documents = []\n # Use resolved absolute paths to avoid mismatches (symlinks, relative vs absolute)\n docs_path = Path(docs_dir).resolve()\n\n # Check if we should process PDFs\n should_process_pdfs = custom_file_types is None or \".pdf\" in custom_file_types\n\n if should_process_pdfs:\n for file_path in docs_path.rglob(\"*.pdf\"):\n # Check if file matches any exclude pattern\n try:\n # Ensure both paths are resolved before computing relativity\n file_path_resolved = file_path.resolve()\n # Determine directory scope using the non-resolved path to avoid\n # misclassifying symlinked entries as outside the docs directory\n relative_path = file_path.relative_to(docs_path)\n if not include_hidden and _path_has_hidden_segment(relative_path):\n continue\n # Use absolute path for gitignore matching\n if self._should_exclude_file(file_path_resolved, gitignore_matches):\n continue\n except ValueError:\n # Skip files that can't be made relative to docs_path\n print(f\"⚠️ Skipping file outside directory scope: {file_path}\")\n continue\n\n print(f\"Processing PDF: {file_path}\")\n\n # Try PyMuPDF first (best quality)\n text = extract_pdf_text_with_pymupdf(str(file_path))\n if text is None:\n # Try pdfplumber\n text = extract_pdf_text_with_pdfplumber(str(file_path))\n\n if text:\n # Create a simple document structure\n from llama_index.core import Document\n\n doc = Document(text=text, metadata={\"source\": str(file_path)})\n documents.append(doc)\n else:\n # Fallback to default reader\n print(f\"Using default reader for {file_path}\")\n try:\n default_docs = SimpleDirectoryReader(\n str(file_path.parent),\n exclude_hidden=not include_hidden,\n filename_as_id=True,\n required_exts=[file_path.suffix],\n ).load_data()\n documents.extend(default_docs)\n except Exception as e:\n print(f\"Warning: Could not process {file_path}: {e}\")\n\n # Load other file types with default reader\n # Exclude PDFs from code_extensions if they were already processed separately\n other_file_extensions = code_extensions\n if should_process_pdfs and \".pdf\" in code_extensions:\n other_file_extensions = [ext for ext in code_extensions if ext != \".pdf\"]\n\n try:\n # Create a custom file filter function using our PathSpec\n def file_filter(\n file_path: str, docs_dir=docs_dir, gitignore_matches=gitignore_matches\n ) -> bool:\n \"\"\"Return True if file should be included (not excluded)\"\"\"\n try:\n docs_path_obj = Path(docs_dir).resolve()\n file_path_obj = Path(file_path).resolve()\n # Use absolute path for gitignore matching\n _ = file_path_obj.relative_to(docs_path_obj) # validate scope\n return not self._should_exclude_file(file_path_obj, gitignore_matches)\n except (ValueError, OSError):\n return True # Include files that can't be processed\n\n # Only load other file types if there are extensions to process\n if other_file_extensions:\n other_docs = SimpleDirectoryReader(\n docs_dir,\n recursive=True,\n encoding=\"utf-8\",\n required_exts=other_file_extensions,\n file_extractor={}, # Use default extractors\n exclude_hidden=not include_hidden,\n filename_as_id=True,\n ).load_data(show_progress=True)\n else:\n other_docs = []\n\n # Filter documents after loading based on gitignore rules\n filtered_docs = []\n for doc in other_docs:\n file_path = doc.metadata.get(\"file_path\", \"\")\n if file_filter(file_path):\n doc.metadata[\"source\"] = file_path\n filtered_docs.append(doc)\n\n documents.extend(filtered_docs)\n except ValueError as e:\n if \"No files found\" in str(e):\n print(f\"No additional files found for other supported types in {docs_dir}.\")\n else:\n raise e\n\n all_documents.extend(documents)\n print(f\"Loaded {len(documents)} documents from {docs_dir}\")\n\n documents = all_documents\n\n all_texts = []\n\n # Define code file extensions for intelligent chunking\n code_file_exts = {\n \".py\",\n \".js\",\n \".ts\",\n \".jsx\",\n \".tsx\",\n \".java\",\n \".cpp\",\n \".c\",\n \".h\",\n \".hpp\",\n \".cs\",\n \".go\",\n \".rs\",\n \".rb\",\n \".php\",\n \".swift\",\n \".kt\",\n \".scala\",\n \".r\",\n \".sql\",\n \".sh\",\n \".bash\",\n \".zsh\",\n \".fish\",\n \".ps1\",\n \".bat\",\n \".json\",\n \".yaml\",\n \".yml\",\n \".xml\",\n \".toml\",\n \".ini\",\n \".cfg\",\n \".conf\",\n \".html\",\n \".css\",\n \".scss\",\n \".less\",\n \".vue\",\n \".svelte\",\n \".ipynb\",\n \".R\",\n \".jl\",\n }\n\n print(\"start chunking documents\")\n\n # Check if AST chunking is requested\n use_ast = getattr(args, \"use_ast_chunking\", False)\n\n if use_ast:\n print(\"🧠 Using AST-aware chunking for code files\")\n try:\n # Import enhanced chunking utilities from packaged module\n from .chunking_utils import create_text_chunks\n\n # Use enhanced chunking with AST support\n chunk_texts = create_text_chunks(\n documents,\n chunk_size=self.node_parser.chunk_size,\n chunk_overlap=self.node_parser.chunk_overlap,\n use_ast_chunking=True,\n ast_chunk_size=getattr(args, \"ast_chunk_size\", 768),\n ast_chunk_overlap=getattr(args, \"ast_chunk_overlap\", 96),\n code_file_extensions=None, # Use defaults\n ast_fallback_traditional=getattr(args, \"ast_fallback_traditional\", True),\n )\n\n # create_text_chunks now returns list[dict] with metadata preserved\n all_texts.extend(chunk_texts)\n\n except ImportError as e:\n print(\n f\"⚠️ AST chunking utilities not available in package ({e}), falling back to traditional chunking\"\n )\n use_ast = False\n\n if not use_ast:\n # Use traditional chunking logic\n for doc in tqdm(documents, desc=\"Chunking documents\", unit=\"doc\"):\n # Check if this is a code file based on source path\n source_path = doc.metadata.get(\"source\", \"\")\n file_path = doc.metadata.get(\"file_path\", \"\")\n is_code_file = any(\n (source_path or file_path).endswith(ext) for ext in code_file_exts\n )\n\n # For code files, prepend line numbers so chunks carry them\n if is_code_file:\n from llama_index.core.schema import MediaResource\n\n original_text = doc.get_content()\n lines = original_text.split(\"\\n\")\n width = len(str(len(lines)))\n numbered = \"\\n\".join(f\"{i + 1:>{width}}|{line}\" for i, line in enumerate(lines))\n doc.text_resource = MediaResource(text=numbered)\n\n # Extract metadata to preserve with chunks\n chunk_metadata = {\n \"file_path\": file_path or source_path,\n \"file_name\": doc.metadata.get(\"file_name\", \"\"),\n \"source\": source_path,\n }\n\n # Add optional metadata if available\n if \"creation_date\" in doc.metadata:\n chunk_metadata[\"creation_date\"] = doc.metadata[\"creation_date\"]\n if \"last_modified_date\" in doc.metadata:\n chunk_metadata[\"last_modified_date\"] = doc.metadata[\"last_modified_date\"]\n\n # Use appropriate parser based on file type\n parser = self.code_parser if is_code_file else self.node_parser\n nodes = parser.get_nodes_from_documents([doc])\n\n for node in nodes:\n text = node.get_content()\n # For code chunks, trim a partial first line left by overlap\n # (a valid line starts with digits followed by '|')\n if is_code_file and text and not text[0].isdigit():\n first_nl = text.find(\"\\n\")\n if first_nl != -1:\n text = text[first_nl + 1 :]\n all_texts.append({\"text\": text, \"metadata\": chunk_metadata.copy()})\n\n print(f\"Loaded {len(documents)} documents, {len(all_texts)} chunks\")\n return all_texts\n\n def _parse_file_types(self, custom_file_types: Optional[str]) -> Optional[list[str]]:\n if not custom_file_types:\n return None\n extensions = [ext.strip() for ext in custom_file_types.split(\",\") if ext.strip()]\n return [ext if ext.startswith(\".\") else f\".{ext}\" for ext in extensions]\n\n def _sync_ignore_patterns(self, include_hidden: bool) -> Optional[list[str]]:\n if include_hidden:\n return None\n return [\"**/.*\"]\n\n def _build_embedding_options(self, args) -> dict[str, Any]:\n \"\"\"Build embedding provider options dict from CLI args.\"\"\"\n opts: dict[str, Any] = {}\n if args.embedding_mode == \"ollama\":\n opts[\"host\"] = resolve_ollama_host(args.embedding_host)\n elif args.embedding_mode == \"openai\":\n opts[\"base_url\"] = resolve_openai_base_url(args.embedding_api_base)\n resolved_key = resolve_openai_api_key(args.embedding_api_key)\n if resolved_key:\n opts[\"api_key\"] = resolved_key\n if args.query_prompt_template:\n if args.embedding_prompt_template:\n opts[\"build_prompt_template\"] = args.embedding_prompt_template\n opts[\"query_prompt_template\"] = args.query_prompt_template\n elif args.embedding_prompt_template:\n opts[\"prompt_template\"] = args.embedding_prompt_template\n return opts\n\n def _resolve_sync_roots(self, docs_paths: list[str]) -> list[str]:\n roots: set[str] = set()\n for path in docs_paths:\n path_obj = Path(path).resolve()\n if path_obj.is_dir():\n roots.add(str(path_obj))\n elif path_obj.is_file():\n roots.add(str(path_obj.parent))\n return sorted(roots)\n\n def _create_synchronizers(\n self,\n index_dir: Path,\n roots: list[str],\n include_extensions: Optional[list[str]] = None,\n ignore_patterns: Optional[list[str]] = None,\n ) -> list[FileSynchronizer]:\n \"\"\"Create FileSynchronizers with snapshots stored in the index dir. Shared by build and watch.\"\"\"\n synchronizers: list[FileSynchronizer] = []\n for root in roots:\n tag = hashlib.sha256(root.encode()).hexdigest()[:12]\n snapshot_path = str(index_dir / f\"sync_{tag}.pickle\")\n try:\n fs = FileSynchronizer(\n root_dir=root,\n ignore_patterns=ignore_patterns,\n include_extensions=include_extensions,\n snapshot_path=snapshot_path,\n )\n synchronizers.append(fs)\n except Exception as exc:\n print(f\"Warning: Failed to init synchronizer for {root}: {exc}\")\n return synchronizers\n\n def _build_synchronizers(\n self,\n docs_paths: list[str],\n index_dir: Path,\n file_types: Optional[str] = None,\n include_hidden: bool = False,\n ) -> list[FileSynchronizer]:\n \"\"\"Create FileSynchronizers for build from docs_paths.\"\"\"\n roots = self._resolve_sync_roots(docs_paths)\n include_extensions = self._parse_file_types(file_types)\n ignore_patterns = self._sync_ignore_patterns(include_hidden)\n return self._create_synchronizers(index_dir, roots, include_extensions, ignore_patterns)\n\n def _detect_build_changes(\n self,\n synchronizers: list[FileSynchronizer],\n ) -> tuple[set[str], set[str], set[str]]:\n \"\"\"Detect added/removed/modified files across all source roots using content hashes.\"\"\"\n all_added: set[str] = set()\n all_removed: set[str] = set()\n all_modified: set[str] = set()\n for fs in synchronizers:\n added, removed, modified = fs.detect_changes()\n all_added.update(added)\n all_removed.update(removed)\n all_modified.update(modified)\n return all_added, all_removed, all_modified\n\n def _commit_synchronizers(self, synchronizers: list[FileSynchronizer]) -> None:\n \"\"\"Persist all synchronizer snapshots after a successful build.\"\"\"\n for fs in synchronizers:\n fs.commit()\n\n @staticmethod\n def _assign_chunk_ids(chunks: list[dict]) -> None:\n \"\"\"Assign stable IDs to chunks based on their file path and position.\"\"\"\n from collections import defaultdict\n\n by_path: dict[str, list] = defaultdict(list)\n for c in chunks:\n p = c.get(\"metadata\", {}).get(\"file_path\") or c.get(\"metadata\", {}).get(\"source\") or \"\"\n by_path[_normalize_path(p)].append(c)\n for path_key, path_chunks in by_path.items():\n for idx, c in enumerate(path_chunks):\n sid = hashlib.sha256(f\"{path_key}:{idx}\".encode()).hexdigest()[:16]\n c.setdefault(\"metadata\", {})[\"id\"] = sid\n c[\"id\"] = sid\n\n @staticmethod\n def _assign_unique_chunk_ids(chunks: list[dict]) -> None:\n \"\"\"Assign unique IDs for incremental (avoids collision when path lookup misses some old ids).\"\"\"\n for c in chunks:\n sid = uuid.uuid4().hex[:16]\n c.setdefault(\"metadata\", {})[\"id\"] = sid\n c[\"id\"] = sid\n\n def _chunks_for_paths(self, all_texts: list[dict], paths: set[str]) -> list[dict]:\n \"\"\"Filter chunks belonging to the given file paths.\"\"\"\n return [\n c\n for c in all_texts\n if _normalize_path(\n c.get(\"metadata\", {}).get(\"file_path\") or c.get(\"metadata\", {}).get(\"source\") or \"\"\n )\n in paths\n ]\n\n def _make_incremental_builder(self, args) -> \"LeannBuilder\":\n return LeannBuilder(\n backend_name=args.backend_name,\n embedding_model=args.embedding_model,\n embedding_mode=args.embedding_mode,\n embedding_options=self._build_embedding_options(args) or None,\n graph_degree=args.graph_degree,\n complexity=args.complexity,\n is_compact=args.compact,\n is_recompute=args.recompute,\n num_threads=args.num_threads,\n )\n\n def _incremental_add_only(\n self,\n index_path: str,\n all_texts: list[dict],\n args,\n new_paths: set[str],\n ) -> bool:\n \"\"\"Add-only incremental update (works for HNSW and IVF).\"\"\"\n new_chunks = self._chunks_for_paths(all_texts, new_paths)\n if not new_chunks:\n return False\n self._assign_chunk_ids(new_chunks)\n builder = self._make_incremental_builder(args)\n for chunk in new_chunks:\n builder.add_text(chunk[\"text\"], metadata=chunk[\"metadata\"])\n print(\n f\"Incremental update: adding {len(new_chunks)} chunks from {len(new_paths)} new file(s)...\"\n )\n builder.update_index(index_path)\n print(f\"Index updated at {index_path}\")\n return True\n\n def _incremental_ivf_remove_only(\n self, index_path: str, index_dir: Path, removed_paths: set[str], args\n ) -> bool:\n \"\"\"IVF remove-only fast path: remove chunk IDs without loading or chunking documents.\"\"\"\n passages_file = index_dir / \"documents.leann.passages.jsonl\"\n if not passages_file.exists():\n return False\n offset_file = index_dir / \"documents.leann.passages.idx\"\n live_ids: set[str] | None = None\n if offset_file.exists():\n with open(offset_file, \"rb\") as f:\n live_ids = set(pickle.load(f).keys())\n chunk_ids_by_file = self._load_chunk_ids_by_file(passages_file, live_ids=live_ids)\n roots = self._load_sync_roots(index_dir)\n ids_to_remove: list[str] = []\n seen_ids: set[str] = set()\n for p in removed_paths:\n for key in self._path_lookup_keys(p, roots):\n for pid in chunk_ids_by_file.get(key, []):\n if pid not in seen_ids:\n seen_ids.add(pid)\n ids_to_remove.append(pid)\n if not ids_to_remove:\n return False\n print(\n f\"Incremental IVF update (-{len(removed_paths)} removed): removing {len(ids_to_remove)} old chunks...\"\n )\n builder = self._make_incremental_builder(args)\n builder.update_index(index_path, remove_passage_ids=ids_to_remove)\n print(f\"Index updated at {index_path}\")\n return True\n\n def _path_lookup_keys(self, path: str, roots: list[str]) -> list[str]:\n \"\"\"Return path variations for lookup (sync paths may be relative to roots).\"\"\"\n keys = [path, _normalize_path(path)]\n for root in roots:\n candidate = str((Path(root) / path).resolve())\n if candidate not in keys:\n keys.append(candidate)\n return keys\n\n def _incremental_ivf_update(\n self,\n index_path: str,\n index_dir: Path,\n all_texts: list[dict],\n args,\n new_paths: set[str],\n removed_paths: set[str],\n modified_paths: set[str],\n sync_roots: list[str],\n ) -> bool:\n \"\"\"IVF incremental update: remove old chunks for modified/removed files, add new chunks.\"\"\"\n passages_file = index_dir / \"documents.leann.passages.jsonl\"\n offset_file = index_dir / \"documents.leann.passages.idx\"\n live_ids: set[str] | None = None\n if offset_file.exists():\n with open(offset_file, \"rb\") as f:\n live_ids = set(pickle.load(f).keys())\n chunk_ids_by_file = (\n self._load_chunk_ids_by_file(passages_file, live_ids=live_ids)\n if passages_file.exists()\n else {}\n )\n\n # Collect old chunk IDs to remove (modified + removed files)\n # Try all path variations: same file can have different path formats in passages\n ids_to_remove: list[str] = []\n seen_ids: set[str] = set()\n for p in modified_paths | removed_paths:\n for key in self._path_lookup_keys(p, sync_roots):\n for pid in chunk_ids_by_file.get(key, []):\n if pid not in seen_ids:\n seen_ids.add(pid)\n ids_to_remove.append(pid)\n\n # Collect new chunks to add (modified + new files)\n # Build path set for matching: chunks may have paths in different formats\n changed_paths = new_paths | modified_paths\n path_set: set[str] = set()\n for p in changed_paths:\n path_set.update(self._path_lookup_keys(p, sync_roots))\n new_chunks = self._chunks_for_paths(all_texts, path_set)\n # Use unique IDs: passages can have mixed path formats so we may miss some ids_to_remove\n self._assign_unique_chunk_ids(new_chunks)\n\n if not ids_to_remove and not new_chunks:\n return False\n\n builder = self._make_incremental_builder(args)\n for chunk in new_chunks:\n builder.add_text(chunk[\"text\"], metadata=chunk[\"metadata\"])\n\n parts = []\n if ids_to_remove:\n parts.append(f\"removing {len(ids_to_remove)} old chunks\")\n if new_chunks:\n parts.append(f\"adding {len(new_chunks)} new chunks\")\n file_parts = []\n if new_paths:\n file_parts.append(f\"+{len(new_paths)} added\")\n if modified_paths:\n file_parts.append(f\"~{len(modified_paths)} modified\")\n if removed_paths:\n file_parts.append(f\"-{len(removed_paths)} removed\")\n print(f\"Incremental IVF update ({', '.join(file_parts)}): {', '.join(parts)}...\")\n\n builder.update_index(\n index_path, remove_passage_ids=ids_to_remove if ids_to_remove else None\n )\n print(f\"Index updated at {index_path}\")\n return True\n\n @staticmethod\n def _log_rebuild_reason(\n meta: dict, args, new_paths: set, removed_paths: set, modified_paths: set\n ) -> None:\n \"\"\"Print a human-readable explanation of why incremental update is not possible.\"\"\"\n if removed_paths or modified_paths:\n reasons = []\n if removed_paths:\n reasons.append(f\"{len(removed_paths)} file(s) removed\")\n if modified_paths:\n reasons.append(f\"{len(modified_paths)} file(s) modified\")\n print(\n f\"Incremental update not possible ({', '.join(reasons)}); falling back to full rebuild.\"\n )\n return\n\n blockers = []\n if meta.get(\"backend_name\") not in (\"hnsw\", \"ivf\"):\n blockers.append(\n f\"backend '{meta.get('backend_name')}' does not support incremental updates\"\n )\n if meta.get(\"is_compact\", meta.get(\"backend_kwargs\", {}).get(\"is_compact\", True)):\n blockers.append(\"index is compact (read-only); rebuild with --no-compact to enable\")\n if meta.get(\"embedding_model\") != args.embedding_model:\n blockers.append(\n f\"embedding model changed ('{meta.get('embedding_model')}' -> '{args.embedding_model}')\"\n )\n if meta.get(\"embedding_mode\") != args.embedding_mode:\n blockers.append(\n f\"embedding mode changed ('{meta.get('embedding_mode')}' -> '{args.embedding_mode}')\"\n )\n if blockers:\n print(\n f\"Incremental update not possible: {'; '.join(blockers)}. Falling back to full rebuild.\"\n )\n else:\n changes = []\n if new_paths:\n changes.append(f\"+{len(new_paths)} added\")\n summary = \", \".join(changes) if changes else \"unknown reason\"\n print(f\"Full rebuild starting ({summary})...\")\n\n def _write_sync_config(\n self,\n index_dir: Path,\n roots: list[str],\n include_extensions: Optional[list[str]],\n ignore_patterns: Optional[list[str]],\n ) -> None:\n sync_config_path = index_dir / \"sync_roots.json\"\n config = {\n \"roots\": roots,\n \"include_extensions\": include_extensions,\n \"ignore_patterns\": ignore_patterns,\n }\n with open(sync_config_path, \"w\", encoding=\"utf-8\") as f:\n json.dump(config, f, indent=2)\n\n def _load_sync_roots(self, index_dir: Path) -> list[str]:\n \"\"\"Load sync roots from index dir (for path resolution in incremental updates).\"\"\"\n sync_config_path = index_dir / \"sync_roots.json\"\n if not sync_config_path.exists():\n return []\n try:\n with open(sync_config_path, encoding=\"utf-8\") as f:\n config = json.load(f)\n return config.get(\"roots\") or []\n except (json.JSONDecodeError, OSError):\n return []\n\n def _resolve_index_for_watch(self, index_name: str) -> Optional[dict[str, Path]]:\n if self.index_exists(index_name):\n index_dir = self.indexes_dir / index_name\n passages_file = index_dir / \"documents.leann.passages.jsonl\"\n return {\"index_dir\": index_dir, \"passages_file\": passages_file}\n\n all_matches = self._find_all_matching_indexes(index_name)\n if not all_matches:\n print(\n f\"Index '{index_name}' not found. Use 'leann build {index_name} --docs [ ...]' to create it.\"\n )\n return None\n\n if len(all_matches) == 1:\n match = all_matches[0]\n else:\n current_matches = [m for m in all_matches if m.get(\"is_current\")]\n match = current_matches[0] if current_matches else all_matches[0]\n location_desc = (\n \"current project\"\n if match.get(\"is_current\")\n else f\"project '{match['project_path'].name}'\"\n )\n print(\n f\"Found {len(all_matches)} indexes named '{index_name}', using index from {location_desc}\"\n )\n\n if match.get(\"kind\") == \"cli\":\n index_dir = match[\"index_dir\"]\n passages_file = index_dir / \"documents.leann.passages.jsonl\"\n else:\n index_dir = match[\"meta_file\"].parent\n file_base = match[\"file_base\"]\n passages_file = index_dir / f\"{file_base}.passages.jsonl\"\n\n return {\"index_dir\": index_dir, \"passages_file\": passages_file}\n\n def _load_chunk_ids_by_file(\n self, passages_file: Path, live_ids: set[str] | None = None\n ) -> dict[str, list[str]]:\n \"\"\"Load chunk IDs grouped by file path from passages.jsonl.\n\n If *live_ids* is provided, skip entries whose ID is not in the set\n (filters out stale entries left by prior incremental updates).\n \"\"\"\n chunk_ids_by_file: dict[str, list[str]] = {}\n with open(passages_file, encoding=\"utf-8\") as f:\n for line in f:\n line = line.strip()\n if not line:\n continue\n try:\n data = json.loads(line)\n except json.JSONDecodeError:\n continue\n metadata = data.get(\"metadata\") or {}\n file_path = metadata.get(\"file_path\") or metadata.get(\"source\")\n if not file_path:\n continue\n chunk_id = data.get(\"id\")\n if chunk_id is None:\n continue\n if live_ids is not None and str(chunk_id) not in live_ids:\n continue\n normalized_path = str(Path(file_path).resolve())\n chunk_ids_by_file.setdefault(normalized_path, []).append(str(chunk_id))\n if file_path != normalized_path:\n chunk_ids_by_file.setdefault(file_path, []).append(str(chunk_id))\n return chunk_ids_by_file\n\n async def build_index(self, args):\n docs_paths = args.docs\n # Use current directory name if index_name not provided\n if args.index_name:\n index_name = args.index_name\n else:\n index_name = Path.cwd().name\n print(f\"Using current directory name as index: '{index_name}'\")\n\n index_dir = self.indexes_dir / index_name\n index_path = self.get_index_path(index_name)\n\n # Display all paths being indexed with file/directory distinction\n files = [p for p in docs_paths if Path(p).is_file()]\n directories = [p for p in docs_paths if Path(p).is_dir()]\n\n print(f\"📂 Indexing {len(docs_paths)} path{'s' if len(docs_paths) > 1 else ''}:\")\n if files:\n print(f\" 📄 Files ({len(files)}):\")\n for i, file_path in enumerate(files, 1):\n print(f\" {i}. {Path(file_path).resolve()}\")\n if directories:\n print(f\" 📁 Directories ({len(directories)}):\")\n for i, dir_path in enumerate(directories, 1):\n print(f\" {i}. {Path(dir_path).resolve()}\")\n\n # Configure chunking based on CLI args before loading documents\n # Guard against invalid configurations\n doc_chunk_size = max(1, int(args.doc_chunk_size))\n doc_chunk_overlap = max(0, int(args.doc_chunk_overlap))\n if doc_chunk_overlap >= doc_chunk_size:\n print(\n f\"⚠️ Adjusting doc chunk overlap from {doc_chunk_overlap} to {doc_chunk_size - 1} (must be < chunk size)\"\n )\n doc_chunk_overlap = doc_chunk_size - 1\n\n code_chunk_size = max(1, int(args.code_chunk_size))\n code_chunk_overlap = max(0, int(args.code_chunk_overlap))\n if code_chunk_overlap >= code_chunk_size:\n print(\n f\"⚠️ Adjusting code chunk overlap from {code_chunk_overlap} to {code_chunk_size - 1} (must be < chunk size)\"\n )\n code_chunk_overlap = code_chunk_size - 1\n\n self.node_parser = SentenceSplitter(\n chunk_size=doc_chunk_size,\n chunk_overlap=doc_chunk_overlap,\n separator=\" \",\n paragraph_separator=\"\\n\\n\",\n )\n self.code_parser = SentenceSplitter(\n chunk_size=code_chunk_size,\n chunk_overlap=code_chunk_overlap,\n separator=\"\\n\",\n paragraph_separator=\"\\n\\n\",\n )\n\n # Detect changes first so we can skip load_documents for remove-only\n index_dir.mkdir(parents=True, exist_ok=True)\n synchronizers = self._build_synchronizers(\n docs_paths, index_dir, file_types=args.file_types, include_hidden=args.include_hidden\n )\n\n if index_dir.exists() and not args.force and synchronizers:\n meta_path = index_dir / \"documents.leann.meta.json\"\n new_paths, removed_paths, modified_paths = self._detect_build_changes(synchronizers)\n\n if not new_paths and not removed_paths and not modified_paths:\n print(\"Index up to date.\")\n return\n\n # Show change summary (add / modify / delete) before build\n change_parts = []\n if new_paths:\n change_parts.append(f\"+{len(new_paths)} added\")\n if modified_paths:\n change_parts.append(f\"~{len(modified_paths)} modified\")\n if removed_paths:\n change_parts.append(f\"-{len(removed_paths)} removed\")\n if change_parts:\n print(f\"Changes: {', '.join(change_parts)}\")\n\n if meta_path.exists():\n with open(meta_path, encoding=\"utf-8\") as f:\n meta = json.load(f)\n\n backend_name = meta.get(\"backend_name\")\n is_compact = meta.get(\n \"is_compact\", meta.get(\"backend_kwargs\", {}).get(\"is_compact\", True)\n )\n same_embedding = (\n meta.get(\"embedding_model\") == args.embedding_model\n and meta.get(\"embedding_mode\") == args.embedding_mode\n )\n\n # IVF supports remove+add, so it can handle modified and removed files incrementally\n can_ivf_update = backend_name == \"ivf\" and not is_compact and same_embedding\n # HNSW only supports add (no remove), so it needs add-only changes\n can_add_only = (\n not removed_paths\n and not modified_paths\n and backend_name in (\"hnsw\", \"ivf\")\n and not is_compact\n and same_embedding\n )\n\n # Remove-only fast path: no load/chunk, just remove IDs from index\n if can_ivf_update and removed_paths and not new_paths and not modified_paths:\n result = self._incremental_ivf_remove_only(\n index_path, index_dir, removed_paths, args\n )\n if result:\n self._commit_synchronizers(synchronizers)\n self._write_sync_config(\n index_dir,\n self._resolve_sync_roots(docs_paths),\n self._parse_file_types(args.file_types),\n self._sync_ignore_patterns(args.include_hidden),\n )\n self.register_project_dir()\n return\n\n # Load only changed files (no need to load/chunk the entire corpus)\n # Resolve paths relative to sync roots (sync returns paths relative to each root)\n roots = self._resolve_sync_roots(docs_paths)\n paths_to_load = new_paths | modified_paths\n resolved_paths: list[str] = []\n for p in paths_to_load:\n path_obj = Path(p)\n if path_obj.is_absolute() and path_obj.exists():\n resolved_paths.append(p)\n else:\n for root in roots:\n candidate = Path(root) / p\n if candidate.exists():\n resolved_paths.append(str(candidate.resolve()))\n break\n else:\n resolved_paths.append(p) # fallback: pass as-is\n all_texts = self.load_documents(\n resolved_paths,\n args.file_types,\n include_hidden=args.include_hidden,\n args=args,\n )\n # Proceed even when all_texts is empty (e.g. file emptied): we still need to remove old chunks\n if not all_texts and not (can_ivf_update and (modified_paths or removed_paths)):\n print(\"No documents found\")\n return\n\n if can_ivf_update and (new_paths or modified_paths or removed_paths):\n result = self._incremental_ivf_update(\n index_path,\n index_dir,\n all_texts,\n args,\n new_paths,\n removed_paths,\n modified_paths,\n roots,\n )\n if result:\n self._commit_synchronizers(synchronizers)\n self._write_sync_config(\n index_dir,\n self._resolve_sync_roots(docs_paths),\n self._parse_file_types(args.file_types),\n self._sync_ignore_patterns(args.include_hidden),\n )\n self.register_project_dir()\n return\n\n elif can_add_only and new_paths:\n result = self._incremental_add_only(\n index_path,\n all_texts,\n args,\n new_paths,\n )\n if result:\n self._commit_synchronizers(synchronizers)\n self._write_sync_config(\n index_dir,\n self._resolve_sync_roots(docs_paths),\n self._parse_file_types(args.file_types),\n self._sync_ignore_patterns(args.include_hidden),\n )\n self.register_project_dir()\n return\n\n else:\n self._log_rebuild_reason(meta, args, new_paths, removed_paths, modified_paths)\n\n # Full rebuild: load documents if not already loaded (first build or force)\n try:\n _ = all_texts\n except NameError:\n all_texts = self.load_documents(\n docs_paths, args.file_types, include_hidden=args.include_hidden, args=args\n )\n if not all_texts:\n print(\"No documents found\")\n return\n\n print(f\"Building index '{index_name}' with {args.backend_name} backend...\")\n\n builder = LeannBuilder(\n backend_name=args.backend_name,\n embedding_model=args.embedding_model,\n embedding_mode=args.embedding_mode,\n embedding_options=self._build_embedding_options(args) or None,\n graph_degree=args.graph_degree,\n complexity=args.complexity,\n is_compact=args.compact,\n is_recompute=args.recompute,\n num_threads=args.num_threads,\n )\n\n for chunk in all_texts:\n builder.add_text(chunk[\"text\"], metadata=chunk[\"metadata\"])\n\n builder.build_index(index_path)\n for fs in synchronizers:\n fs.create_snapshot()\n self._write_sync_config(\n index_dir,\n self._resolve_sync_roots(docs_paths),\n self._parse_file_types(args.file_types),\n self._sync_ignore_patterns(args.include_hidden),\n )\n print(f\"Index built at {index_path}\")\n self.register_project_dir()\n\n def _watch_check_changes(self, index_name: str) -> tuple[set[str], set[str], set[str]]:\n \"\"\"Check for file changes using the same snapshots as build (index_dir).\"\"\"\n resolved = self._resolve_index_for_watch(index_name)\n if not resolved:\n return set(), set(), set()\n\n index_dir = resolved[\"index_dir\"]\n sync_config_path = index_dir / \"sync_roots.json\"\n if not sync_config_path.exists():\n return set(), set(), set()\n\n with open(sync_config_path, encoding=\"utf-8\") as f:\n config = json.load(f)\n\n roots = config.get(\"roots\") or []\n if not roots:\n return set(), set(), set()\n\n synchronizers = self._create_synchronizers(\n index_dir,\n roots,\n include_extensions=config.get(\"include_extensions\"),\n ignore_patterns=config.get(\"ignore_patterns\"),\n )\n return self._detect_build_changes(synchronizers)\n\n def _watch_report_changes(\n self,\n index_name: str,\n added: set[str],\n removed: set[str],\n modified: set[str],\n ) -> None:\n \"\"\"Print a summary of detected file changes.\"\"\"\n resolved = self._resolve_index_for_watch(index_name)\n passages_file = resolved[\"passages_file\"] if resolved else None\n\n chunk_ids_by_file: dict[str, list[str]] = {}\n if passages_file and passages_file.exists():\n chunk_ids_by_file = self._load_chunk_ids_by_file(passages_file)\n\n print(\"\\n=== Changes detected ===\")\n for label, paths in (\n (\"added\", sorted(added)),\n (\"removed\", sorted(removed)),\n (\"modified\", sorted(modified)),\n ):\n if not paths:\n continue\n print(f\"\\n{label} ({len(paths)}):\")\n for file_path in paths:\n normalized_path = str(Path(file_path).resolve())\n chunk_ids = chunk_ids_by_file.get(normalized_path) or chunk_ids_by_file.get(\n file_path, []\n )\n chunk_display = \", \".join(chunk_ids) if chunk_ids else \"(not in index)\"\n print(f\" - {file_path}\")\n print(f\" chunks: {chunk_display}\")\n\n async def _watch_trigger_build(self, index_name: str) -> None:\n \"\"\"Trigger an idempotent build for the given index, reusing its stored config.\"\"\"\n resolved = self._resolve_index_for_watch(index_name)\n if not resolved:\n return\n index_dir = resolved[\"index_dir\"]\n sync_config_path = index_dir / \"sync_roots.json\"\n if not sync_config_path.exists():\n return\n with open(sync_config_path, encoding=\"utf-8\") as f:\n config = json.load(f)\n roots = config.get(\"roots\") or []\n if not roots:\n return\n\n meta_path = index_dir / \"documents.leann.meta.json\"\n if not meta_path.exists():\n print(f\"Index metadata missing for '{index_name}', cannot rebuild.\")\n return\n with open(meta_path, encoding=\"utf-8\") as f:\n meta = json.load(f)\n\n parser = self.create_parser()\n build_args_list = [\n \"build\",\n index_name,\n \"--docs\",\n *roots,\n \"--backend-name\",\n meta.get(\"backend_name\", \"hnsw\"),\n \"--embedding-model\",\n meta.get(\"embedding_model\", \"all-MiniLM-L6-v2\"),\n \"--embedding-mode\",\n meta.get(\"embedding_mode\", \"sentence-transformers\"),\n ]\n bkw = meta.get(\"backend_kwargs\", {})\n if not bkw.get(\"is_compact\", False):\n build_args_list.append(\"--no-compact\")\n if bkw.get(\"is_recompute\", True):\n build_args_list.append(\"--recompute\")\n\n build_args = parser.parse_args(build_args_list)\n await self.build_index(build_args)\n\n async def watch_index(self, args):\n index_name = args.index_name\n resolved = self._resolve_index_for_watch(index_name)\n if not resolved:\n return\n\n index_dir = resolved[\"index_dir\"]\n sync_config_path = index_dir / \"sync_roots.json\"\n if not sync_config_path.exists():\n print(\n f\"Sync config not found for index '{index_name}'. \"\n f\"Run 'leann build {index_name} --docs ' first.\"\n )\n return\n\n dry_run = getattr(args, \"dry_run\", False)\n once = getattr(args, \"once\", False)\n interval = getattr(args, \"interval\", 5)\n\n if once:\n added, removed, modified = self._watch_check_changes(index_name)\n if not added and not removed and not modified:\n print(\"No changes detected.\")\n return\n self._watch_report_changes(index_name, added, removed, modified)\n if not dry_run:\n await self._watch_trigger_build(index_name)\n return\n\n print(f\"Watching index '{index_name}' (interval={interval}s, ctrl-c to stop)...\")\n try:\n while True:\n added, removed, modified = self._watch_check_changes(index_name)\n if added or removed or modified:\n self._watch_report_changes(index_name, added, removed, modified)\n if not dry_run:\n await self._watch_trigger_build(index_name)\n await asyncio.sleep(interval)\n except KeyboardInterrupt:\n print(\"\\nWatch stopped.\")\n\n def _resolve_index_path(\n self,\n index_name: str,\n *,\n non_interactive: bool = True,\n purpose: str = \"use\",\n quiet: bool = False,\n ) -> Optional[str]:\n \"\"\"Resolve index path from current project or registered projects.\"\"\"\n _print = (lambda *a, **kw: print(*a, file=sys.stderr, **kw)) if quiet else print\n\n if self.index_exists(index_name):\n return self.get_index_path(index_name)\n\n all_matches = self._find_all_matching_indexes(index_name)\n if not all_matches:\n _print(\n f\"Index '{index_name}' not found. Use 'leann build {index_name} --docs [ ...]' to create it.\"\n )\n return None\n\n def _match_to_path(match: dict[str, Any]) -> str:\n if match[\"kind\"] == \"cli\":\n return str(match[\"index_dir\"] / \"documents.leann\")\n meta_file = match[\"meta_file\"]\n file_base = match[\"file_base\"]\n return str(meta_file.parent / f\"{file_base}.leann\")\n\n if len(all_matches) == 1:\n match = all_matches[0]\n project_info = (\n \"current project\"\n if match[\"is_current\"]\n else f\"project '{match['project_path'].name}'\"\n )\n _print(f\"Using index '{index_name}' from {project_info}\")\n return _match_to_path(match)\n\n if non_interactive:\n current_matches = [m for m in all_matches if m[\"is_current\"]]\n match = current_matches[0] if current_matches else all_matches[0]\n location_desc = (\n \"current project\"\n if match[\"is_current\"]\n else f\"project '{match['project_path'].name}'\"\n )\n _print(\n f\"Found {len(all_matches)} indexes named '{index_name}', using index from {location_desc}\"\n )\n return _match_to_path(match)\n\n print(f\"Found {len(all_matches)} indexes named '{index_name}':\")\n for i, match in enumerate(all_matches, 1):\n project_path = match[\"project_path\"]\n is_current = match[\"is_current\"]\n kind = match.get(\"kind\", \"cli\")\n if is_current:\n print(f\" {i}. 🏠 Current project ({'CLI' if kind == 'cli' else 'APP'})\")\n else:\n print(f\" {i}. 📂 {project_path.name} ({'CLI' if kind == 'cli' else 'APP'})\")\n\n try:\n choice = input(f\"Which index to {purpose}? (1-{len(all_matches)}): \").strip()\n choice_idx = int(choice) - 1\n if 0 <= choice_idx < len(all_matches):\n match = all_matches[choice_idx]\n project_info = (\n \"current project\"\n if match[\"is_current\"]\n else f\"project '{match['project_path'].name}'\"\n )\n print(f\"Using index '{index_name}' from {project_info}\")\n return _match_to_path(match)\n print(\"Invalid choice. Aborting.\")\n return None\n except (ValueError, KeyboardInterrupt):\n print(\"Invalid input. Aborting.\")\n return None\n\n async def search_documents(self, args):\n index_name = args.index_name\n query = args.query\n json_mode = getattr(args, \"json\", False)\n\n index_path = self._resolve_index_path(\n index_name,\n non_interactive=args.non_interactive,\n purpose=\"search\",\n quiet=json_mode,\n )\n if not index_path:\n return\n\n # Build provider_options for runtime override\n provider_options = {}\n if args.embedding_prompt_template:\n provider_options[\"prompt_template\"] = args.embedding_prompt_template\n\n if json_mode:\n sys.stdout.flush()\n saved_fd = os.dup(1)\n devnull = os.open(os.devnull, os.O_WRONLY)\n os.dup2(devnull, 1)\n os.close(devnull)\n\n try:\n searcher = LeannSearcher(\n index_path=index_path,\n enable_warmup=args.enable_warmup,\n use_daemon=args.use_daemon,\n daemon_ttl_seconds=args.daemon_ttl,\n )\n results = searcher.search(\n query,\n top_k=args.top_k,\n complexity=args.complexity,\n beam_width=args.beam_width,\n prune_ratio=args.prune_ratio,\n recompute_embeddings=args.recompute_embeddings,\n pruning_strategy=args.pruning_strategy,\n provider_options=provider_options if provider_options else None,\n )\n finally:\n if json_mode:\n import ctypes\n\n libc = ctypes.CDLL(None)\n libc.fflush(None)\n os.dup2(saved_fd, 1)\n os.close(saved_fd)\n\n if json_mode:\n json_results = [\n {\n \"id\": r.id,\n \"score\": r.score,\n \"text\": r.text,\n \"metadata\": r.metadata,\n }\n for r in results\n ]\n print(json.dumps(json_results, ensure_ascii=False, indent=2))\n return\n\n print(f\"Search results for '{query}' (top {len(results)}):\")\n for i, result in enumerate(results, 1):\n print(f\"{i}. Score: {result.score:.3f}\")\n\n if args.show_metadata and result.metadata:\n file_path = result.metadata.get(\"file_path\", \"\")\n if file_path:\n print(f\" File: {file_path}\")\n\n file_name = result.metadata.get(\"file_name\", \"\")\n if file_name and file_name != file_path:\n print(f\" Name: {file_name}\")\n\n if \"creation_date\" in result.metadata:\n print(f\" Created: {result.metadata['creation_date']}\")\n if \"last_modified_date\" in result.metadata:\n print(f\" Modified: {result.metadata['last_modified_date']}\")\n\n print(f\" {result.text}\")\n print(f\" Source: {result.metadata.get('source', '')}\")\n print()\n\n async def warmup_index(self, args):\n index_path = self._resolve_index_path(\n args.index_name,\n non_interactive=True,\n purpose=\"warm up\",\n )\n if not index_path:\n return\n\n searcher = LeannSearcher(\n index_path=index_path,\n recompute_embeddings=True,\n enable_warmup=args.enable_warmup,\n use_daemon=args.use_daemon,\n daemon_ttl_seconds=args.daemon_ttl,\n )\n if args.enable_warmup:\n searcher.warmup()\n print(\n f\"Warmed index '{args.index_name}' (daemon={'on' if args.use_daemon else 'off'}, ttl={args.daemon_ttl}s)\"\n )\n\n async def daemon_command(self, args):\n if not args.daemon_command:\n print(\"Please specify one of: start, stop, status\")\n return\n\n if args.daemon_command == \"status\":\n records = EmbeddingServerManager.list_daemons()\n if args.index_name:\n index_path = self._resolve_index_path(\n args.index_name,\n non_interactive=True,\n purpose=\"check daemon status for\",\n )\n if not index_path:\n return\n meta_path = str(Path(f\"{index_path}.meta.json\").resolve())\n records = [\n r\n for r in records\n if r.get(\"config_signature\", {}).get(\"passages_file\") == meta_path\n ]\n\n if not records:\n print(\"No active embedding daemons.\")\n return\n\n print(f\"Active embedding daemons: {len(records)}\")\n for record in records:\n cfg = record.get(\"config_signature\", {})\n print(\n f\"- pid={record.get('pid')} port={record.get('port')} backend={record.get('backend_module_name')} model={cfg.get('model_name')}\"\n )\n return\n\n if args.daemon_command == \"start\":\n index_path = self._resolve_index_path(\n args.index_name,\n non_interactive=True,\n purpose=\"start daemon for\",\n )\n if not index_path:\n return\n\n searcher = LeannSearcher(\n index_path=index_path,\n recompute_embeddings=True,\n enable_warmup=args.enable_warmup,\n use_daemon=True,\n daemon_ttl_seconds=args.daemon_ttl,\n )\n searcher.warmup()\n print(\n f\"Daemon started for '{args.index_name}' (ttl={args.daemon_ttl}s, warmup={'on' if args.enable_warmup else 'off'})\"\n )\n return\n\n if args.daemon_command == \"stop\":\n if args.all:\n stopped = EmbeddingServerManager.stop_daemons()\n print(f\"Stopped {stopped} daemon(s).\")\n return\n\n if not args.index_name:\n print(\"Provide an index name or pass --all.\")\n return\n\n index_path = self._resolve_index_path(\n args.index_name,\n non_interactive=True,\n purpose=\"stop daemon for\",\n )\n if not index_path:\n return\n meta_path = str(Path(f\"{index_path}.meta.json\").resolve())\n stopped = EmbeddingServerManager.stop_daemons(passages_file=meta_path)\n print(f\"Stopped {stopped} daemon(s) for index '{args.index_name}'.\")\n\n async def ask_questions(self, args):\n index_name = args.index_name\n index_path = self.get_index_path(index_name)\n\n if not self.index_exists(index_name):\n print(\n f\"Index '{index_name}' not found. Use 'leann build {index_name} --docs [ ...]' to create it.\"\n )\n return\n\n print(f\"Starting chat with index '{index_name}'...\")\n print(f\"Using {args.model} ({args.llm})\")\n\n llm_config = {\"type\": args.llm, \"model\": args.model}\n if args.llm == \"ollama\":\n llm_config[\"host\"] = resolve_ollama_host(args.host)\n elif args.llm == \"openai\":\n llm_config[\"base_url\"] = resolve_openai_base_url(args.api_base)\n resolved_api_key = resolve_openai_api_key(args.api_key)\n if resolved_api_key:\n llm_config[\"api_key\"] = resolved_api_key\n elif args.llm == \"anthropic\":\n # For Anthropic, pass base_url and API key if provided\n if args.api_base:\n llm_config[\"base_url\"] = resolve_anthropic_base_url(args.api_base)\n if args.api_key:\n llm_config[\"api_key\"] = args.api_key\n elif args.llm == \"minimax\":\n llm_config[\"base_url\"] = resolve_minimax_base_url(args.api_base)\n resolved_api_key = resolve_minimax_api_key(args.api_key)\n if resolved_api_key:\n llm_config[\"api_key\"] = resolved_api_key\n\n chat = LeannChat(index_path=index_path, llm_config=llm_config)\n\n llm_kwargs: dict[str, Any] = {}\n if args.thinking_budget:\n llm_kwargs[\"thinking_budget\"] = args.thinking_budget\n\n def _ask_once(prompt: str) -> None:\n query_start_time = time.time()\n response = chat.ask(\n prompt,\n top_k=args.top_k,\n complexity=args.complexity,\n beam_width=args.beam_width,\n prune_ratio=args.prune_ratio,\n recompute_embeddings=args.recompute_embeddings,\n pruning_strategy=args.pruning_strategy,\n llm_kwargs=llm_kwargs,\n )\n query_completion_time = time.time() - query_start_time\n print(f\"LEANN: {response}\")\n print(f\"The query took {query_completion_time:.3f} seconds to finish\")\n\n initial_query = (args.query or \"\").strip()\n\n if args.interactive:\n # Create interactive session\n session = create_cli_session(index_name)\n\n if initial_query:\n _ask_once(initial_query)\n\n session.run_interactive_loop(_ask_once)\n else:\n query = initial_query or input(\"Enter your question: \").strip()\n if not query:\n print(\"No question provided. Exiting.\")\n return\n\n _ask_once(query)\n\n async def react_agent(self, args):\n \"\"\"Run ReAct agent for multiturn retrieval.\"\"\"\n index_name = args.index_name\n query = args.query\n\n # Find the index (similar to search_documents)\n index_path = self.get_index_path(index_name)\n if self.index_exists(index_name):\n pass\n else:\n all_matches = self._find_all_matching_indexes(index_name)\n if not all_matches:\n print(\n f\"Index '{index_name}' not found. Use 'leann build {index_name} --docs [ ...]' to create it.\"\n )\n return\n elif len(all_matches) == 1:\n match = all_matches[0]\n if match[\"kind\"] == \"cli\":\n index_path = str(match[\"index_dir\"] / \"documents.leann\")\n else:\n meta_file = match[\"meta_file\"]\n file_base = match[\"file_base\"]\n index_path = str(meta_file.parent / f\"{file_base}.leann\")\n else:\n # Multiple matches - use first one for now\n match = all_matches[0]\n if match[\"kind\"] == \"cli\":\n index_path = str(match[\"index_dir\"] / \"documents.leann\")\n else:\n meta_file = match[\"meta_file\"]\n file_base = match[\"file_base\"]\n index_path = str(meta_file.parent / f\"{file_base}.leann\")\n print(f\"Found {len(all_matches)} indexes named '{index_name}', using first match\")\n\n print(f\"🤖 Starting ReAct agent with index '{index_name}'...\")\n print(f\"Using {args.model} ({args.llm})\")\n\n llm_config = {\"type\": args.llm, \"model\": args.model}\n if args.llm == \"ollama\":\n llm_config[\"host\"] = resolve_ollama_host(args.host)\n elif args.llm == \"openai\":\n llm_config[\"base_url\"] = resolve_openai_base_url(args.api_base)\n resolved_api_key = resolve_openai_api_key(args.api_key)\n if resolved_api_key:\n llm_config[\"api_key\"] = resolved_api_key\n elif args.llm == \"anthropic\":\n if args.api_base:\n llm_config[\"base_url\"] = resolve_anthropic_base_url(args.api_base)\n if args.api_key:\n llm_config[\"api_key\"] = args.api_key\n elif args.llm == \"minimax\":\n llm_config[\"base_url\"] = resolve_minimax_base_url(args.api_base)\n resolved_api_key = resolve_minimax_api_key(args.api_key)\n if resolved_api_key:\n llm_config[\"api_key\"] = resolved_api_key\n\n from .react_agent import create_react_agent\n\n agent = create_react_agent(\n index_path=index_path,\n llm_config=llm_config,\n max_iterations=args.max_iterations,\n )\n\n print(f\"\\n🔍 Question: {query}\\n\")\n answer = agent.run(query, top_k=args.top_k)\n print(f\"\\n✅ Final Answer:\\n{answer}\\n\")\n\n if agent.search_history:\n print(f\"\\n📊 Search History ({len(agent.search_history)} iterations):\")\n for entry in agent.search_history:\n print(\n f\" {entry['iteration']}. {entry['action']} ({entry['results_count']} results)\"\n )\n\n async def serve_api(self, args):\n \"\"\"Start the HTTP API server.\"\"\"\n import os\n\n try:\n from .server import main as server_main\n\n # Override host/port if provided via CLI args\n if args.host:\n os.environ[\"LEANN_SERVER_HOST\"] = args.host\n if args.port:\n os.environ[\"LEANN_SERVER_PORT\"] = str(args.port)\n\n # Run the server (this is blocking, so we don't await it)\n # The server_main function handles uvicorn.run which blocks\n server_main()\n except ImportError as e:\n print(\n \"❌ HTTP server dependencies not installed.\\n\"\n \"Install them with:\\n\"\n \" uv pip install 'leann-core[server]'\\n\"\n \"or:\\n\"\n \" uv pip install 'fastapi>=0.115' 'pydantic>=2' 'uvicorn[standard]'\\n\"\n )\n raise SystemExit(1) from e\n except Exception as e:\n print(f\"❌ Error starting server: {e}\")\n raise SystemExit(1) from e\n\n async def run(self, args=None):\n parser = self.create_parser()\n\n if args is None:\n args = parser.parse_args()\n\n if not args.command:\n parser.print_help()\n return\n\n # Determine whether to suppress C++ output\n # Default is to suppress (quiet mode), unless --verbose is specified\n suppress = not getattr(args, \"verbose\", False)\n\n if args.command == \"list\":\n self.list_indexes()\n elif args.command == \"remove\":\n self.remove_index(args.index_name, args.force)\n elif args.command == \"build\":\n with suppress_cpp_output(suppress):\n await self.build_index(args)\n elif args.command == \"watch\":\n await self.watch_index(args)\n elif args.command == \"search\":\n with suppress_cpp_output(suppress):\n await self.search_documents(args)\n elif args.command == \"warmup\":\n with suppress_cpp_output(suppress):\n await self.warmup_index(args)\n elif args.command == \"daemon\":\n await self.daemon_command(args)\n elif args.command == \"ask\":\n with suppress_cpp_output(suppress):\n await self.ask_questions(args)\n elif args.command == \"react\":\n with suppress_cpp_output(suppress):\n await self.react_agent(args)\n elif args.command == \"serve\":\n await self.serve_api(args)\n else:\n parser.print_help()\n\n\ndef main():\n import logging\n\n import dotenv\n\n dotenv.load_dotenv()\n\n # Set clean logging for CLI usage\n logging.getLogger().setLevel(logging.WARNING) # Only show warnings and errors\n\n cli = LeannCLI()\n asyncio.run(cli.run())\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "packages/leann-core/src/leann/embedding_compute.py", "content": "\"\"\"\nUnified embedding computation module\nConsolidates all embedding computation logic using SentenceTransformer\nPreserves all optimization parameters to ensure performance\n\"\"\"\n\nimport json\nimport logging\nimport os\nimport subprocess\nimport time\nfrom typing import Any, Optional, Protocol, cast\n\nimport numpy as np\nimport tiktoken\nimport torch\n\nfrom .settings import resolve_ollama_host, resolve_openai_api_key, resolve_openai_base_url\n\n# Set up logger with proper level\nlogger = logging.getLogger(__name__)\nLOG_LEVEL = os.getenv(\"LEANN_LOG_LEVEL\", \"WARNING\").upper()\nlog_level = getattr(logging, LOG_LEVEL, logging.WARNING)\nlogger.setLevel(log_level)\n\n\nclass _SentenceTransformerLike(Protocol):\n def eval(self) -> Any: ...\n def parameters(self) -> Any: ...\n def encode(self, *args: Any, **kwargs: Any) -> Any: ...\n def half(self) -> Any: ...\n\n\n# Token limit registry for embedding models\n# Used as fallback when dynamic discovery fails (e.g., LM Studio, OpenAI)\n# Ollama models use dynamic discovery via /api/show\nEMBEDDING_MODEL_LIMITS = {\n # Nomic models (common across servers)\n \"nomic-embed-text\": 2048, # Corrected from 512 - verified via /api/show\n \"nomic-embed-text-v1.5\": 2048,\n \"nomic-embed-text-v2\": 512,\n # Other embedding models\n \"mxbai-embed-large\": 512,\n \"all-minilm\": 512,\n \"bge-m3\": 8192,\n \"snowflake-arctic-embed\": 512,\n # OpenAI models\n \"text-embedding-3-small\": 8192,\n \"text-embedding-3-large\": 8192,\n \"text-embedding-ada-002\": 8192,\n}\n\n# Runtime cache for dynamically discovered token limits\n# Key: (model_name, base_url), Value: token_limit\n# Prevents repeated SDK/API calls for the same model\n_token_limit_cache: dict[tuple[str, str], int] = {}\n\n\ndef get_model_token_limit(\n model_name: str,\n base_url: Optional[str] = None,\n default: int = 2048,\n) -> int:\n \"\"\"\n Get token limit for a given embedding model.\n Uses hybrid approach: dynamic discovery for Ollama, registry fallback for others.\n Caches discovered limits to prevent repeated API/SDK calls.\n\n Args:\n model_name: Name of the embedding model\n base_url: Base URL of the embedding server (for dynamic discovery)\n default: Default token limit if model not found\n\n Returns:\n Token limit for the model in tokens\n \"\"\"\n # Check cache first to avoid repeated SDK/API calls\n cache_key = (model_name, base_url or \"\")\n if cache_key in _token_limit_cache:\n cached_limit = _token_limit_cache[cache_key]\n logger.debug(f\"Using cached token limit for {model_name}: {cached_limit}\")\n return cached_limit\n\n # Try Ollama dynamic discovery if base_url provided\n if base_url:\n # Detect Ollama servers by port or \"ollama\" in URL\n if \"11434\" in base_url or \"ollama\" in base_url.lower():\n limit = _query_ollama_context_limit(model_name, base_url)\n if limit:\n _token_limit_cache[cache_key] = limit\n return limit\n\n # Try LM Studio SDK discovery\n if \"1234\" in base_url or \"lmstudio\" in base_url.lower() or \"lm.studio\" in base_url.lower():\n # Convert HTTP to WebSocket URL\n ws_url = base_url.replace(\"https://\", \"wss://\").replace(\"http://\", \"ws://\")\n # Remove /v1 suffix if present\n if ws_url.endswith(\"/v1\"):\n ws_url = ws_url[:-3]\n\n limit = _query_lmstudio_context_limit(model_name, ws_url)\n if limit:\n _token_limit_cache[cache_key] = limit\n return limit\n\n # Fallback to known model registry with version handling (from PR #154)\n # Handle versioned model names (e.g., \"nomic-embed-text:latest\" -> \"nomic-embed-text\")\n base_model_name = model_name.split(\":\")[0]\n\n # Check exact match first\n if model_name in EMBEDDING_MODEL_LIMITS:\n limit = EMBEDDING_MODEL_LIMITS[model_name]\n _token_limit_cache[cache_key] = limit\n return limit\n\n # Check base name match\n if base_model_name in EMBEDDING_MODEL_LIMITS:\n limit = EMBEDDING_MODEL_LIMITS[base_model_name]\n _token_limit_cache[cache_key] = limit\n return limit\n\n # Check partial matches for common patterns\n for known_model, registry_limit in EMBEDDING_MODEL_LIMITS.items():\n if known_model in base_model_name or base_model_name in known_model:\n _token_limit_cache[cache_key] = registry_limit\n return registry_limit\n\n # Default fallback\n logger.warning(f\"Unknown model '{model_name}', using default {default} token limit\")\n _token_limit_cache[cache_key] = default\n return default\n\n\ndef truncate_to_token_limit(texts: list[str], token_limit: int) -> list[str]:\n \"\"\"\n Truncate texts to fit within token limit using tiktoken.\n\n Args:\n texts: List of text strings to truncate\n token_limit: Maximum number of tokens allowed\n\n Returns:\n List of truncated texts (same length as input)\n \"\"\"\n if not texts:\n return []\n\n # Use tiktoken with cl100k_base encoding\n enc = tiktoken.get_encoding(\"cl100k_base\")\n\n truncated_texts = []\n truncation_count = 0\n total_tokens_removed = 0\n max_original_length = 0\n\n for i, text in enumerate(texts):\n tokens = enc.encode(text)\n original_length = len(tokens)\n\n if original_length <= token_limit:\n # Text is within limit, keep as is\n truncated_texts.append(text)\n else:\n # Truncate to token_limit\n truncated_tokens = tokens[:token_limit]\n truncated_text = enc.decode(truncated_tokens)\n truncated_texts.append(truncated_text)\n\n # Track truncation statistics\n truncation_count += 1\n tokens_removed = original_length - token_limit\n total_tokens_removed += tokens_removed\n max_original_length = max(max_original_length, original_length)\n\n # Log individual truncation at WARNING level (first few only)\n if truncation_count <= 3:\n logger.warning(\n f\"Text {i + 1} truncated: {original_length} → {token_limit} tokens \"\n f\"({tokens_removed} tokens removed)\"\n )\n elif truncation_count == 4:\n logger.warning(\"Further truncation warnings suppressed...\")\n\n # Log summary at INFO level\n if truncation_count > 0:\n logger.warning(\n f\"Truncation summary: {truncation_count}/{len(texts)} texts truncated \"\n f\"(removed {total_tokens_removed} tokens total, longest was {max_original_length} tokens)\"\n )\n else:\n logger.debug(\n f\"No truncation needed - all {len(texts)} texts within {token_limit} token limit\"\n )\n\n return truncated_texts\n\n\ndef _query_ollama_context_limit(model_name: str, base_url: str) -> Optional[int]:\n \"\"\"\n Query Ollama /api/show for model context limit.\n\n Args:\n model_name: Name of the Ollama model\n base_url: Base URL of the Ollama server\n\n Returns:\n Context limit in tokens if found, None otherwise\n \"\"\"\n try:\n import requests\n\n response = requests.post(\n f\"{base_url}/api/show\",\n json={\"name\": model_name},\n timeout=5,\n )\n if response.status_code == 200:\n data = response.json()\n if \"model_info\" in data:\n # Look for *.context_length in model_info\n for key, value in data[\"model_info\"].items():\n if \"context_length\" in key and isinstance(value, int):\n logger.info(f\"Detected {model_name} context limit: {value} tokens\")\n return value\n except Exception as e:\n logger.debug(f\"Failed to query Ollama context limit: {e}\")\n\n return None\n\n\ndef _query_lmstudio_context_limit(model_name: str, base_url: str) -> Optional[int]:\n \"\"\"\n Query LM Studio SDK for model context length via Node.js subprocess.\n\n Args:\n model_name: Name of the LM Studio model\n base_url: Base URL of the LM Studio server (WebSocket format, e.g., \"ws://localhost:1234\")\n\n Returns:\n Context limit in tokens if found, None otherwise\n \"\"\"\n # Inline JavaScript using @lmstudio/sdk\n # Note: Load model temporarily for metadata, then unload to respect JIT auto-evict\n js_code = f\"\"\"\n const {{ LMStudioClient }} = require('@lmstudio/sdk');\n (async () => {{\n try {{\n const client = new LMStudioClient({{ baseUrl: '{base_url}' }});\n const model = await client.embedding.load('{model_name}', {{ verbose: false }});\n const contextLength = await model.getContextLength();\n await model.unload(); // Unload immediately to respect JIT auto-evict settings\n console.log(JSON.stringify({{ contextLength, identifier: '{model_name}' }}));\n }} catch (error) {{\n console.error(JSON.stringify({{ error: error.message }}));\n process.exit(1);\n }}\n }})();\n \"\"\"\n\n try:\n # Set NODE_PATH to include global modules for @lmstudio/sdk resolution\n env = os.environ.copy()\n\n # Try to get npm global root (works with nvm, brew node, etc.)\n try:\n npm_root = subprocess.run(\n [\"npm\", \"root\", \"-g\"],\n capture_output=True,\n text=True,\n timeout=5,\n )\n if npm_root.returncode == 0:\n global_modules = npm_root.stdout.strip()\n # Append to existing NODE_PATH if present\n existing_node_path = env.get(\"NODE_PATH\", \"\")\n env[\"NODE_PATH\"] = (\n f\"{global_modules}:{existing_node_path}\"\n if existing_node_path\n else global_modules\n )\n except Exception:\n # If npm not available, continue with existing NODE_PATH\n pass\n\n result = subprocess.run(\n [\"node\", \"-e\", js_code],\n capture_output=True,\n text=True,\n timeout=10,\n env=env,\n )\n\n if result.returncode != 0:\n logger.debug(f\"LM Studio SDK error: {result.stderr}\")\n return None\n\n data = json.loads(result.stdout)\n context_length = data.get(\"contextLength\")\n\n if context_length and context_length > 0:\n logger.info(f\"LM Studio SDK detected {model_name} context length: {context_length}\")\n return context_length\n\n except FileNotFoundError:\n logger.debug(\"Node.js not found - install Node.js for LM Studio SDK features\")\n except subprocess.TimeoutExpired:\n logger.debug(\"LM Studio SDK query timeout\")\n except json.JSONDecodeError:\n logger.debug(\"LM Studio SDK returned invalid JSON\")\n except Exception as e:\n logger.debug(f\"LM Studio SDK query failed: {e}\")\n\n return None\n\n\n# Global model cache to avoid repeated loading\n_model_cache: dict[str, Any] = {}\n\n\ndef compute_embeddings(\n texts: list[str],\n model_name: str,\n mode: str = \"sentence-transformers\",\n is_build: bool = False,\n batch_size: int = 32,\n adaptive_optimization: bool = True,\n manual_tokenize: bool = False,\n max_length: int = 512,\n provider_options: Optional[dict[str, Any]] = None,\n) -> np.ndarray:\n \"\"\"\n Unified embedding computation entry point\n\n Args:\n texts: List of texts to compute embeddings for\n model_name: Model name\n mode: Computation mode ('sentence-transformers', 'openai', 'mlx', 'ollama')\n is_build: Whether this is a build operation (shows progress bar)\n batch_size: Batch size for processing\n adaptive_optimization: Whether to use adaptive optimization based on batch size\n\n Returns:\n Normalized embeddings array, shape: (len(texts), embedding_dim)\n \"\"\"\n provider_options = provider_options or {}\n wrapper_start_time = time.time()\n logger.debug(\n f\"[compute_embeddings] entry: mode={mode}, model='{model_name}', text_count={len(texts)}\"\n )\n\n # Allow batch_size override from provider_options (disables adaptive_optimization)\n if \"batch_size\" in provider_options:\n batch_size = provider_options[\"batch_size\"]\n adaptive_optimization = False # User-specified batch_size takes precedence\n\n if mode == \"sentence-transformers\":\n inner_start_time = time.time()\n result = compute_embeddings_sentence_transformers(\n texts,\n model_name,\n is_build=is_build,\n batch_size=batch_size,\n adaptive_optimization=adaptive_optimization,\n manual_tokenize=manual_tokenize,\n max_length=max_length,\n )\n inner_end_time = time.time()\n wrapper_end_time = time.time()\n logger.debug(\n \"[compute_embeddings] sentence-transformers timings: \"\n f\"inner={inner_end_time - inner_start_time:.6f}s, \"\n f\"wrapper_total={wrapper_end_time - wrapper_start_time:.6f}s\"\n )\n return result\n elif mode == \"openai\":\n return compute_embeddings_openai(\n texts,\n model_name,\n base_url=provider_options.get(\"base_url\"),\n api_key=provider_options.get(\"api_key\"),\n provider_options=provider_options,\n )\n elif mode == \"mlx\":\n return compute_embeddings_mlx(texts, model_name)\n elif mode == \"ollama\":\n return compute_embeddings_ollama(\n texts,\n model_name,\n is_build=is_build,\n host=provider_options.get(\"host\"),\n provider_options=provider_options,\n )\n elif mode == \"gemini\":\n return compute_embeddings_gemini(texts, model_name, is_build=is_build)\n else:\n raise ValueError(f\"Unsupported embedding mode: {mode}\")\n\n\ndef compute_embeddings_sentence_transformers(\n texts: list[str],\n model_name: str,\n use_fp16: bool = True,\n device: str = \"auto\",\n batch_size: int = 32,\n is_build: bool = False,\n adaptive_optimization: bool = True,\n manual_tokenize: bool = False,\n max_length: int = 512,\n) -> np.ndarray:\n \"\"\"\n Compute embeddings using SentenceTransformer with model caching and adaptive optimization\n\n Args:\n texts: List of texts to compute embeddings for\n model_name: Model name\n use_fp16: Whether to use FP16 precision\n device: Device to use ('auto', 'cuda', 'mps', 'cpu')\n batch_size: Batch size for processing\n is_build: Whether this is a build operation (shows progress bar)\n adaptive_optimization: Whether to use adaptive optimization based on batch size\n \"\"\"\n outer_start_time = time.time()\n # Handle empty input\n if not texts:\n raise ValueError(\"Cannot compute embeddings for empty text list\")\n logger.info(\n f\"Computing embeddings for {len(texts)} texts using SentenceTransformer, model: '{model_name}'\"\n )\n\n # Auto-detect device\n if device == \"auto\":\n # Check environment variable first\n env_device = os.getenv(\"LEANN_EMBEDDING_DEVICE\")\n if env_device:\n device = env_device\n logger.info(f\"Using device from LEANN_EMBEDDING_DEVICE: {device}\")\n elif torch.cuda.is_available():\n device = \"cuda\"\n elif hasattr(torch.backends, \"mps\") and torch.backends.mps.is_available():\n device = \"mps\"\n else:\n device = \"cpu\"\n\n # Apply optimizations based on benchmark results\n if adaptive_optimization:\n # Use optimal batch_size constants for different devices based on benchmark results\n if device == \"mps\":\n batch_size = 128 # MPS optimal batch size from benchmark\n if model_name == \"Qwen/Qwen3-Embedding-0.6B\":\n batch_size = 32\n elif device == \"cuda\":\n batch_size = 256 # CUDA optimal batch size\n # Keep original batch_size for CPU\n\n # Create cache key\n cache_key = f\"sentence_transformers_{model_name}_{device}_{use_fp16}_optimized\"\n\n pre_model_init_end_time = time.time()\n logger.debug(\n \"compute_embeddings_sentence_transformers pre-model-init time \"\n f\"(device/batch selection etc.): {pre_model_init_end_time - outer_start_time:.6f}s\"\n )\n\n # Check if model is already cached\n start_time = time.time()\n if cache_key in _model_cache:\n logger.info(f\"Using cached optimized model: {model_name}\")\n model = cast(_SentenceTransformerLike, _model_cache[cache_key])\n else:\n logger.info(f\"Loading and caching optimized SentenceTransformer model: {model_name}\")\n from sentence_transformers import SentenceTransformer\n\n logger.info(f\"Using device: {device}\")\n\n # Apply hardware optimizations\n if device == \"cuda\":\n # TODO: Haven't tested this yet\n torch.backends.cuda.matmul.allow_tf32 = True\n torch.backends.cudnn.allow_tf32 = True\n torch.backends.cudnn.benchmark = True\n torch.backends.cudnn.deterministic = False\n torch.cuda.set_per_process_memory_fraction(0.9)\n elif device == \"mps\":\n try:\n if hasattr(torch.mps, \"set_per_process_memory_fraction\"):\n torch.mps.set_per_process_memory_fraction(0.9)\n except AttributeError:\n logger.warning(\"Some MPS optimizations not available in this PyTorch version\")\n elif device == \"cpu\":\n # TODO: Haven't tested this yet\n torch.set_num_threads(min(8, os.cpu_count() or 4))\n try:\n torch.backends.mkldnn.enabled = True\n except AttributeError:\n pass\n\n # Prepare optimized model and tokenizer parameters\n model_kwargs = {\n \"torch_dtype\": torch.float16 if use_fp16 else torch.float32,\n \"low_cpu_mem_usage\": True,\n \"_fast_init\": True,\n \"attn_implementation\": \"eager\", # Use eager attention for speed\n }\n\n tokenizer_kwargs = {\n \"use_fast\": True,\n \"padding\": True,\n \"truncation\": True,\n }\n\n try:\n # Try loading with advanced parameters first (newer versions)\n local_model_kwargs = model_kwargs.copy()\n local_tokenizer_kwargs = tokenizer_kwargs.copy()\n local_model_kwargs[\"local_files_only\"] = True\n local_tokenizer_kwargs[\"local_files_only\"] = True\n\n model = SentenceTransformer(\n model_name,\n device=device,\n model_kwargs=local_model_kwargs,\n tokenizer_kwargs=local_tokenizer_kwargs,\n local_files_only=True,\n )\n logger.info(\"Model loaded successfully! (local + optimized)\")\n except TypeError as e:\n if \"model_kwargs\" in str(e) or \"tokenizer_kwargs\" in str(e):\n logger.warning(\n f\"Advanced parameters not supported ({e}), using basic initialization...\"\n )\n # Fallback to basic initialization for older versions\n try:\n model = SentenceTransformer(\n model_name,\n device=device,\n local_files_only=True,\n )\n logger.info(\"Model loaded successfully! (local + basic)\")\n except Exception as e2:\n logger.warning(f\"Local loading failed ({e2}), trying network download...\")\n model = SentenceTransformer(\n model_name,\n device=device,\n local_files_only=False,\n )\n logger.info(\"Model loaded successfully! (network + basic)\")\n else:\n raise\n except Exception as e:\n logger.warning(f\"Local loading failed ({e}), trying network download...\")\n # Fallback to network loading with advanced parameters\n try:\n network_model_kwargs = model_kwargs.copy()\n network_tokenizer_kwargs = tokenizer_kwargs.copy()\n network_model_kwargs[\"local_files_only\"] = False\n network_tokenizer_kwargs[\"local_files_only\"] = False\n\n model = SentenceTransformer(\n model_name,\n device=device,\n model_kwargs=network_model_kwargs,\n tokenizer_kwargs=network_tokenizer_kwargs,\n local_files_only=False,\n )\n logger.info(\"Model loaded successfully! (network + optimized)\")\n except TypeError as e2:\n if \"model_kwargs\" in str(e2) or \"tokenizer_kwargs\" in str(e2):\n logger.warning(\n f\"Advanced parameters not supported ({e2}), using basic network loading...\"\n )\n model = SentenceTransformer(\n model_name,\n device=device,\n local_files_only=False,\n )\n logger.info(\"Model loaded successfully! (network + basic)\")\n else:\n raise\n\n # Apply additional optimizations based on mode\n if use_fp16 and device in [\"cuda\", \"mps\"]:\n try:\n model = model.half()\n logger.info(f\"Applied FP16 precision: {model_name}\")\n except Exception as e:\n logger.warning(f\"FP16 optimization failed: {e}\")\n\n # Apply torch.compile optimization\n if device in [\"cuda\", \"mps\"]:\n try:\n model = torch.compile(model, mode=\"reduce-overhead\", dynamic=True)\n logger.info(f\"Applied torch.compile optimization: {model_name}\")\n except Exception as e:\n logger.warning(f\"torch.compile optimization failed: {e}\")\n\n model = cast(_SentenceTransformerLike, model)\n\n # Set model to eval mode and disable gradients for inference\n model.eval()\n for param in model.parameters():\n param.requires_grad_(False)\n\n # Cache the model\n _model_cache[cache_key] = model\n logger.info(f\"Model cached: {cache_key}\")\n\n end_time = time.time()\n\n # Compute embeddings with optimized inference mode\n logger.info(\n f\"Starting embedding computation... (batch_size: {batch_size}, manual_tokenize={manual_tokenize})\"\n )\n logger.info(f\"start sentence transformers {model} takes {end_time - start_time}\")\n\n start_time = time.time()\n if not manual_tokenize:\n # Use SentenceTransformer's optimized encode path (default)\n with torch.inference_mode():\n embeddings = model.encode(\n texts,\n batch_size=batch_size,\n show_progress_bar=is_build, # Don't show progress bar in server environment\n convert_to_numpy=True,\n normalize_embeddings=False,\n device=device,\n )\n # Synchronize if CUDA to measure accurate wall time\n try:\n if torch.cuda.is_available():\n torch.cuda.synchronize()\n except Exception:\n pass\n else:\n # Manual tokenization + forward pass using HF AutoTokenizer/AutoModel.\n # This path is reserved for an aggressively optimized FP pipeline\n # (no quantization), mainly for experimentation.\n try:\n from transformers import AutoModel, AutoTokenizer\n except Exception as e:\n raise ImportError(f\"transformers is required for manual_tokenize=True: {e}\")\n\n tok_cache_key = f\"hf_tokenizer_{model_name}\"\n mdl_cache_key = f\"hf_model_{model_name}_{device}_{use_fp16}_fp\"\n\n if tok_cache_key in _model_cache and mdl_cache_key in _model_cache:\n hf_tokenizer = _model_cache[tok_cache_key]\n hf_model = _model_cache[mdl_cache_key]\n logger.info(\"Using cached HF tokenizer/model for manual FP path\")\n else:\n logger.info(\"Loading HF tokenizer/model for manual FP path\")\n hf_tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=True)\n\n torch_dtype = torch.float16 if (use_fp16 and device == \"cuda\") else torch.float32\n hf_model = AutoModel.from_pretrained(\n model_name,\n torch_dtype=torch_dtype,\n )\n hf_model.to(device)\n\n hf_model.eval()\n # Optional compile on supported devices\n if device in [\"cuda\", \"mps\"]:\n try:\n hf_model = torch.compile(hf_model, mode=\"reduce-overhead\", dynamic=True)\n logger.info(\n f\"Applied torch.compile to HF model for {model_name} \"\n f\"(device={device}, dtype={torch_dtype})\"\n )\n except Exception as exc:\n logger.warning(f\"torch.compile optimization failed: {exc}\")\n\n _model_cache[tok_cache_key] = hf_tokenizer\n _model_cache[mdl_cache_key] = hf_model\n\n all_embeddings: list[np.ndarray] = []\n # Progress bar when building or for large inputs\n show_progress = is_build or len(texts) > 32\n try:\n if show_progress:\n from tqdm import tqdm\n\n batch_iter = tqdm(\n range(0, len(texts), batch_size),\n desc=\"Embedding (manual)\",\n unit=\"batch\",\n )\n else:\n batch_iter = range(0, len(texts), batch_size)\n except Exception:\n batch_iter = range(0, len(texts), batch_size)\n\n start_time_manual = time.time()\n with torch.inference_mode():\n for start_index in batch_iter:\n end_index = min(start_index + batch_size, len(texts))\n batch_texts = texts[start_index:end_index]\n inputs = hf_tokenizer(\n batch_texts,\n padding=True,\n truncation=True,\n max_length=max_length,\n return_tensors=\"pt\",\n )\n inputs = {k: v.to(device) for k, v in inputs.items()}\n outputs = hf_model(**inputs)\n last_hidden_state = outputs.last_hidden_state # (B, L, H)\n attention_mask = inputs.get(\"attention_mask\")\n if attention_mask is None:\n pooled = last_hidden_state.mean(dim=1)\n else:\n mask = attention_mask.unsqueeze(-1).to(last_hidden_state.dtype)\n masked = last_hidden_state * mask\n lengths = mask.sum(dim=1).clamp(min=1)\n pooled = masked.sum(dim=1) / lengths\n batch_embeddings = pooled.detach().to(\"cpu\").float().numpy()\n all_embeddings.append(batch_embeddings)\n\n embeddings = np.vstack(all_embeddings).astype(np.float32, copy=False)\n try:\n if torch.cuda.is_available():\n torch.cuda.synchronize()\n except Exception:\n pass\n end_time = time.time()\n logger.info(f\"Manual tokenize time taken: {end_time - start_time_manual} seconds\")\n end_time = time.time()\n logger.info(f\"Generated {len(embeddings)} embeddings, dimension: {embeddings.shape[1]}\")\n logger.info(f\"Time taken: {end_time - start_time} seconds\")\n\n # Validate results\n if np.isnan(embeddings).any() or np.isinf(embeddings).any():\n raise RuntimeError(f\"Detected NaN or Inf values in embeddings, model: {model_name}\")\n\n outer_end_time = time.time()\n logger.debug(\n \"compute_embeddings_sentence_transformers total time \"\n f\"(function entry -> return): {outer_end_time - outer_start_time:.6f}s\"\n )\n\n return embeddings\n\n\ndef compute_embeddings_openai(\n texts: list[str],\n model_name: str,\n base_url: Optional[str] = None,\n api_key: Optional[str] = None,\n provider_options: Optional[dict[str, Any]] = None,\n) -> np.ndarray:\n # TODO: @yichuan-w add progress bar only in build mode\n \"\"\"Compute embeddings using OpenAI API\"\"\"\n try:\n import openai\n except ImportError as e:\n raise ImportError(f\"OpenAI package not installed: {e}\")\n\n # Validate input list\n if not texts:\n raise ValueError(\"Cannot compute embeddings for empty text list\")\n # Extra validation: abort early if any item is empty/whitespace\n invalid_count = sum(1 for t in texts if not isinstance(t, str) or not t.strip())\n if invalid_count > 0:\n raise ValueError(\n f\"Found {invalid_count} empty/invalid text(s) in input. Upstream should filter before calling OpenAI.\"\n )\n\n # Extract base_url and api_key from provider_options if not provided directly\n provider_options = provider_options or {}\n effective_base_url = base_url or provider_options.get(\"base_url\")\n effective_api_key = api_key or provider_options.get(\"api_key\")\n\n resolved_base_url = resolve_openai_base_url(effective_base_url)\n resolved_api_key = resolve_openai_api_key(effective_api_key)\n\n if not resolved_api_key:\n raise RuntimeError(\"OPENAI_API_KEY environment variable not set\")\n\n # Create OpenAI client\n client = openai.OpenAI(api_key=resolved_api_key, base_url=resolved_base_url)\n\n logger.info(\n f\"Computing embeddings for {len(texts)} texts using OpenAI API, model: '{model_name}'\"\n )\n\n # Apply prompt template if provided\n # Priority: build_prompt_template (new format) > prompt_template (old format)\n prompt_template = provider_options.get(\"build_prompt_template\") or provider_options.get(\n \"prompt_template\"\n )\n\n if prompt_template:\n logger.warning(f\"Applying prompt template: '{prompt_template}'\")\n texts = [f\"{prompt_template}{text}\" for text in texts]\n\n # Query token limit and apply truncation\n token_limit = get_model_token_limit(model_name, base_url=effective_base_url)\n logger.info(f\"Using token limit: {token_limit} for model '{model_name}'\")\n texts = truncate_to_token_limit(texts, token_limit)\n\n # OpenAI has limits on batch size and input length\n max_batch_size = 800 # Conservative batch size because the token limit is 300K\n all_embeddings = []\n # get the avg len of texts\n avg_len = sum(len(text) for text in texts) / len(texts)\n # if avg len is less than 1000, use the max batch size\n if avg_len > 300:\n max_batch_size = 500\n\n # Gemini's OpenAI-compatible endpoint hard-limits embedding batches to 100 inputs per request.\n # If we exceed this, the API returns:\n # \"BatchEmbedContentsRequest.requests: at most 100 requests can be in one batch\"\n if \"generativelanguage.googleapis.com\" in (resolved_base_url or \"\"):\n max_batch_size = min(max_batch_size, 100)\n logger.info(\n \"Detected Gemini OpenAI-compatible base_url; capping embedding batch_size to %d.\",\n max_batch_size,\n )\n\n # if avg len is less than 1000, use the max batch size\n\n try:\n from tqdm import tqdm\n\n total_batches = (len(texts) + max_batch_size - 1) // max_batch_size\n batch_range = range(0, len(texts), max_batch_size)\n batch_iterator = tqdm(\n batch_range, desc=\"Computing embeddings\", unit=\"batch\", total=total_batches\n )\n except ImportError:\n # Fallback when tqdm is not available\n batch_iterator = range(0, len(texts), max_batch_size)\n\n for i in batch_iterator:\n batch_texts = texts[i : i + max_batch_size]\n\n try:\n response = client.embeddings.create(model=model_name, input=batch_texts)\n batch_embeddings = [embedding.embedding for embedding in response.data]\n\n # Verify we got the expected number of embeddings\n if len(batch_embeddings) != len(batch_texts):\n logger.warning(\n f\"Expected {len(batch_texts)} embeddings but got {len(batch_embeddings)}\"\n )\n\n # Only take the number of embeddings that match the batch size\n all_embeddings.extend(batch_embeddings[: len(batch_texts)])\n except Exception as e:\n logger.error(f\"Batch {i} failed: {e}\")\n raise\n\n embeddings = np.array(all_embeddings, dtype=np.float32)\n logger.info(f\"Generated {len(embeddings)} embeddings, dimension: {embeddings.shape[1]}\")\n return embeddings\n\n\ndef compute_embeddings_mlx(chunks: list[str], model_name: str, batch_size: int = 16) -> np.ndarray:\n # TODO: @yichuan-w add progress bar only in build mode\n \"\"\"Computes embeddings using an MLX model.\"\"\"\n try:\n import mlx.core as mx\n from mlx_lm.utils import load\n except ImportError as e:\n raise RuntimeError(\n \"MLX or related libraries not available. Install with: uv pip install mlx mlx-lm\"\n ) from e\n\n logger.info(\n f\"Computing embeddings for {len(chunks)} chunks using MLX model '{model_name}' with batch_size={batch_size}...\"\n )\n\n # Cache MLX model and tokenizer\n cache_key = f\"mlx_{model_name}\"\n if cache_key in _model_cache:\n logger.info(f\"Using cached MLX model: {model_name}\")\n model, tokenizer = _model_cache[cache_key]\n else:\n logger.info(f\"Loading and caching MLX model: {model_name}\")\n model, tokenizer = load(model_name)\n _model_cache[cache_key] = (model, tokenizer)\n logger.info(f\"MLX model cached: {cache_key}\")\n\n # Process chunks in batches with progress bar\n all_embeddings = []\n\n try:\n from tqdm import tqdm\n\n batch_iterator = tqdm(\n range(0, len(chunks), batch_size), desc=\"Computing embeddings\", unit=\"batch\"\n )\n except ImportError:\n batch_iterator = range(0, len(chunks), batch_size)\n\n for i in batch_iterator:\n batch_chunks = chunks[i : i + batch_size]\n\n # Tokenize all chunks in the batch\n batch_token_ids = []\n for chunk in batch_chunks:\n token_ids = tokenizer.encode(chunk)\n batch_token_ids.append(token_ids)\n\n # Pad sequences to the same length for batch processing\n max_length = max(len(ids) for ids in batch_token_ids)\n padded_token_ids = []\n for token_ids in batch_token_ids:\n # Pad with tokenizer.pad_token_id or 0\n padded = token_ids + [0] * (max_length - len(token_ids))\n padded_token_ids.append(padded)\n\n # Convert to MLX array with batch dimension\n input_ids = mx.array(padded_token_ids)\n\n # Get embeddings for the batch\n embeddings = model(input_ids)\n\n # Mean pooling for each sequence in the batch\n pooled = embeddings.mean(axis=1) # Shape: (batch_size, hidden_size)\n\n # Convert batch embeddings to numpy\n for j in range(len(batch_chunks)):\n pooled_list = pooled[j].tolist() # Convert to list\n pooled_numpy = np.array(pooled_list, dtype=np.float32)\n all_embeddings.append(pooled_numpy)\n\n # Stack numpy arrays\n return np.stack(all_embeddings)\n\n\ndef compute_embeddings_ollama(\n texts: list[str],\n model_name: str,\n is_build: bool = False,\n host: Optional[str] = None,\n provider_options: Optional[dict[str, Any]] = None,\n) -> np.ndarray:\n \"\"\"\n Compute embeddings using Ollama API with true batch processing.\n\n Uses the /api/embed endpoint which supports batch inputs.\n Batch size: 32 for MPS/CPU, 128 for CUDA to optimize performance.\n\n Args:\n texts: List of texts to compute embeddings for\n model_name: Ollama model name (e.g., \"nomic-embed-text\", \"mxbai-embed-large\")\n is_build: Whether this is a build operation (shows progress bar)\n host: Ollama host URL (defaults to environment or http://localhost:11434)\n provider_options: Optional provider-specific options (e.g., prompt_template)\n\n Returns:\n Normalized embeddings array, shape: (len(texts), embedding_dim)\n \"\"\"\n try:\n import requests\n except ImportError:\n raise ImportError(\n \"The 'requests' library is required for Ollama embeddings. Install with: uv pip install requests\"\n )\n\n if not texts:\n raise ValueError(\"Cannot compute embeddings for empty text list\")\n\n resolved_host = resolve_ollama_host(host)\n\n logger.info(\n f\"Computing embeddings for {len(texts)} texts using Ollama API, model: '{model_name}', host: '{resolved_host}'\"\n )\n\n # Check if Ollama is running\n try:\n response = requests.get(f\"{resolved_host}/api/version\", timeout=5)\n response.raise_for_status()\n except requests.exceptions.ConnectionError:\n error_msg = (\n f\"❌ Could not connect to Ollama at {resolved_host}.\\n\\n\"\n \"Please ensure Ollama is running:\\n\"\n \" • macOS/Linux: ollama serve\\n\"\n \" • Windows: Make sure Ollama is running in the system tray\\n\\n\"\n \"Installation: https://ollama.com/download\"\n )\n raise RuntimeError(error_msg)\n except Exception as e:\n raise RuntimeError(f\"Unexpected error connecting to Ollama: {e}\")\n\n # Check if model exists and provide helpful suggestions\n try:\n response = requests.get(f\"{resolved_host}/api/tags\", timeout=5)\n response.raise_for_status()\n models = response.json()\n model_names = [model[\"name\"] for model in models.get(\"models\", [])]\n\n # Filter for embedding models (models that support embeddings)\n embedding_models = []\n suggested_embedding_models = [\n \"nomic-embed-text\",\n \"mxbai-embed-large\",\n \"bge-m3\",\n \"all-minilm\",\n \"snowflake-arctic-embed\",\n ]\n\n for model in model_names:\n # Check if it's an embedding model (by name patterns or known models)\n base_name = model.split(\":\")[0]\n if any(emb in base_name for emb in [\"embed\", \"bge\", \"minilm\", \"e5\"]):\n embedding_models.append(model)\n\n # Check if model exists (handle versioned names) and resolve to full name\n resolved_model_name = None\n for name in model_names:\n # Exact match\n if model_name == name:\n resolved_model_name = name\n break\n # Match without version tag (use the versioned name)\n elif model_name == name.split(\":\")[0]:\n resolved_model_name = name\n break\n\n if not resolved_model_name:\n error_msg = f\"❌ Model '{model_name}' not found in local Ollama.\\n\\n\"\n\n # Suggest pulling the model\n error_msg += \"📦 To install this embedding model:\\n\"\n error_msg += f\" ollama pull {model_name}\\n\\n\"\n\n # Show available embedding models\n if embedding_models:\n error_msg += \"✅ Available embedding models:\\n\"\n for model in embedding_models[:5]:\n error_msg += f\" • {model}\\n\"\n if len(embedding_models) > 5:\n error_msg += f\" ... and {len(embedding_models) - 5} more\\n\"\n else:\n error_msg += \"💡 Popular embedding models to install:\\n\"\n for model in suggested_embedding_models[:3]:\n error_msg += f\" • ollama pull {model}\\n\"\n\n error_msg += \"\\n📚 Browse more: https://ollama.com/library\"\n raise ValueError(error_msg)\n\n # Use the resolved model name for all subsequent operations\n if resolved_model_name != model_name:\n logger.info(f\"Resolved model name '{model_name}' to '{resolved_model_name}'\")\n model_name = resolved_model_name\n\n # Verify the model supports embeddings by testing it with /api/embed\n try:\n test_response = requests.post(\n f\"{resolved_host}/api/embed\",\n json={\"model\": model_name, \"input\": \"test\"},\n timeout=10,\n )\n if test_response.status_code != 200:\n error_msg = (\n f\"⚠️ Model '{model_name}' exists but may not support embeddings.\\n\\n\"\n f\"Please use an embedding model like:\\n\"\n )\n for model in suggested_embedding_models[:3]:\n error_msg += f\" • {model}\\n\"\n raise ValueError(error_msg)\n except requests.exceptions.RequestException:\n # If test fails, continue anyway - model might still work\n pass\n\n except requests.exceptions.RequestException as e:\n logger.warning(f\"Could not verify model existence: {e}\")\n\n # Determine batch size based on device availability\n # Check for CUDA/MPS availability using torch if available\n batch_size = 32 # Default for MPS/CPU\n try:\n import torch\n\n if torch.cuda.is_available():\n batch_size = 128 # CUDA gets larger batch size\n elif hasattr(torch.backends, \"mps\") and torch.backends.mps.is_available():\n batch_size = 32 # MPS gets smaller batch size\n except ImportError:\n # If torch is not available, use conservative batch size\n batch_size = 32\n\n logger.info(f\"Using batch size: {batch_size} for true batch processing\")\n\n # Apply prompt template if provided\n provider_options = provider_options or {}\n # Priority: build_prompt_template (new format) > prompt_template (old format)\n prompt_template = provider_options.get(\"build_prompt_template\") or provider_options.get(\n \"prompt_template\"\n )\n\n if prompt_template:\n logger.warning(f\"Applying prompt template: '{prompt_template}'\")\n texts = [f\"{prompt_template}{text}\" for text in texts]\n\n # Get model token limit and apply truncation before batching\n token_limit = get_model_token_limit(model_name, base_url=resolved_host)\n logger.info(f\"Model '{model_name}' token limit: {token_limit}\")\n\n # Apply truncation to all texts before batch processing\n # Function logs truncation details internally\n texts = truncate_to_token_limit(texts, token_limit)\n\n def get_batch_embeddings(batch_texts):\n \"\"\"Get embeddings for a batch of texts using /api/embed endpoint.\"\"\"\n max_retries = 3\n retry_count = 0\n\n # Texts are already truncated to token limit by the outer function\n while retry_count < max_retries:\n try:\n # Use /api/embed endpoint with \"input\" parameter for batch processing\n response = requests.post(\n f\"{resolved_host}/api/embed\",\n json={\"model\": model_name, \"input\": batch_texts},\n timeout=60, # Increased timeout for batch processing\n )\n response.raise_for_status()\n\n result = response.json()\n batch_embeddings = result.get(\"embeddings\")\n\n if batch_embeddings is None:\n raise ValueError(\"No embeddings returned from API\")\n\n if not isinstance(batch_embeddings, list):\n raise ValueError(f\"Invalid embeddings format: {type(batch_embeddings)}\")\n\n if len(batch_embeddings) != len(batch_texts):\n raise ValueError(\n f\"Mismatch: requested {len(batch_texts)} embeddings, got {len(batch_embeddings)}\"\n )\n\n return batch_embeddings, []\n\n except requests.exceptions.Timeout:\n retry_count += 1\n if retry_count >= max_retries:\n logger.warning(f\"Timeout for batch after {max_retries} retries\")\n return None, list(range(len(batch_texts)))\n\n except Exception as e:\n retry_count += 1\n if retry_count >= max_retries:\n # Enhanced error detection for token limit violations\n error_msg = str(e).lower()\n if \"token\" in error_msg and (\n \"limit\" in error_msg or \"exceed\" in error_msg or \"length\" in error_msg\n ):\n logger.error(\n f\"Token limit exceeded for batch. Error: {e}. \"\n f\"Consider reducing chunk sizes or check token truncation.\"\n )\n else:\n logger.error(f\"Failed to get embeddings for batch: {e}\")\n return None, list(range(len(batch_texts)))\n\n return None, list(range(len(batch_texts)))\n\n # Process texts in batches\n all_embeddings = []\n all_failed_indices = []\n\n # Setup progress bar if needed\n show_progress = is_build or len(texts) > 10\n try:\n if show_progress:\n from tqdm import tqdm\n except ImportError:\n show_progress = False\n\n # Process batches\n num_batches = (len(texts) + batch_size - 1) // batch_size\n\n if show_progress:\n batch_iterator = tqdm(range(num_batches), desc=\"Computing Ollama embeddings (batched)\")\n else:\n batch_iterator = range(num_batches)\n\n for batch_idx in batch_iterator:\n start_idx = batch_idx * batch_size\n end_idx = min(start_idx + batch_size, len(texts))\n batch_texts = texts[start_idx:end_idx]\n\n batch_embeddings, batch_failed = get_batch_embeddings(batch_texts)\n\n if batch_embeddings is not None:\n all_embeddings.extend(batch_embeddings)\n else:\n # Entire batch failed, add None placeholders\n all_embeddings.extend([None] * len(batch_texts))\n # Adjust failed indices to global indices\n global_failed = [start_idx + idx for idx in batch_failed]\n all_failed_indices.extend(global_failed)\n\n # Handle failed embeddings\n if all_failed_indices:\n if len(all_failed_indices) == len(texts):\n raise RuntimeError(\"Failed to compute any embeddings\")\n\n logger.warning(\n f\"Failed to compute embeddings for {len(all_failed_indices)}/{len(texts)} texts\"\n )\n\n # Use zero embeddings as fallback for failed ones\n valid_embedding = next((e for e in all_embeddings if e is not None), None)\n if valid_embedding:\n embedding_dim = len(valid_embedding)\n for i, embedding in enumerate(all_embeddings):\n if embedding is None:\n all_embeddings[i] = [0.0] * embedding_dim\n\n # Remove None values\n all_embeddings = [e for e in all_embeddings if e is not None]\n\n if not all_embeddings:\n raise RuntimeError(\"No valid embeddings were computed\")\n\n # Validate embedding dimensions\n expected_dim = len(all_embeddings[0])\n inconsistent_dims = []\n for i, embedding in enumerate(all_embeddings):\n if len(embedding) != expected_dim:\n inconsistent_dims.append((i, len(embedding)))\n\n if inconsistent_dims:\n error_msg = f\"Ollama returned inconsistent embedding dimensions. Expected {expected_dim}, but got:\\n\"\n for idx, dim in inconsistent_dims[:10]: # Show first 10 inconsistent ones\n error_msg += f\" - Text {idx}: {dim} dimensions\\n\"\n if len(inconsistent_dims) > 10:\n error_msg += f\" ... and {len(inconsistent_dims) - 10} more\\n\"\n error_msg += f\"\\nThis is likely an Ollama API bug with model '{model_name}'. Please try:\\n\"\n error_msg += \"1. Restart Ollama service: 'ollama serve'\\n\"\n error_msg += f\"2. Re-pull the model: 'ollama pull {model_name}'\\n\"\n error_msg += (\n \"3. Use sentence-transformers instead: --embedding-mode sentence-transformers\\n\"\n )\n error_msg += \"4. Report this issue to Ollama: https://github.com/ollama/ollama/issues\"\n raise ValueError(error_msg)\n\n # Convert to numpy array and normalize\n embeddings = np.array(all_embeddings, dtype=np.float32)\n\n # Normalize embeddings (L2 normalization)\n norms = np.linalg.norm(embeddings, axis=1, keepdims=True)\n embeddings = embeddings / (norms + 1e-8) # Add small epsilon to avoid division by zero\n\n logger.info(f\"Generated {len(embeddings)} embeddings, dimension: {embeddings.shape[1]}\")\n\n return embeddings\n\n\ndef compute_embeddings_gemini(\n texts: list[str], model_name: str = \"text-embedding-004\", is_build: bool = False\n) -> np.ndarray:\n \"\"\"\n Compute embeddings using Google Gemini API.\n\n Args:\n texts: List of texts to compute embeddings for\n model_name: Gemini model name (default: \"text-embedding-004\")\n is_build: Whether this is a build operation (shows progress bar)\n\n Returns:\n Embeddings array, shape: (len(texts), embedding_dim)\n \"\"\"\n try:\n import os\n\n import google.genai as genai\n except ImportError as e:\n raise ImportError(f\"Google GenAI package not installed: {e}\")\n\n api_key = os.getenv(\"GEMINI_API_KEY\")\n if not api_key:\n raise RuntimeError(\"GEMINI_API_KEY environment variable not set\")\n\n # Cache Gemini client\n cache_key = \"gemini_client\"\n if cache_key in _model_cache:\n client = _model_cache[cache_key]\n else:\n client = genai.Client(api_key=api_key)\n _model_cache[cache_key] = client\n logger.info(\"Gemini client cached\")\n\n logger.info(\n f\"Computing embeddings for {len(texts)} texts using Gemini API, model: '{model_name}'\"\n )\n\n # Gemini supports batch embedding\n max_batch_size = 100 # Conservative batch size for Gemini\n all_embeddings = []\n\n try:\n from tqdm import tqdm\n\n total_batches = (len(texts) + max_batch_size - 1) // max_batch_size\n batch_range = range(0, len(texts), max_batch_size)\n batch_iterator = tqdm(\n batch_range, desc=\"Computing embeddings\", unit=\"batch\", total=total_batches\n )\n except ImportError:\n # Fallback when tqdm is not available\n batch_iterator = range(0, len(texts), max_batch_size)\n\n for i in batch_iterator:\n batch_texts = texts[i : i + max_batch_size]\n\n try:\n # Use the embed_content method from the new Google GenAI SDK\n response = client.models.embed_content(\n model=model_name,\n contents=batch_texts,\n config=genai.types.EmbedContentConfig(\n task_type=\"RETRIEVAL_DOCUMENT\" # For document embedding\n ),\n )\n\n # Extract embeddings from response\n for embedding_data in response.embeddings:\n all_embeddings.append(embedding_data.values)\n except Exception as e:\n logger.error(f\"Batch {i} failed: {e}\")\n raise\n\n embeddings = np.array(all_embeddings, dtype=np.float32)\n logger.info(f\"Generated {len(embeddings)} embeddings, dimension: {embeddings.shape[1]}\")\n\n return embeddings\n" }, { "path": "packages/leann-core/src/leann/embedding_server_manager.py", "content": "import atexit\nimport contextlib\nimport hashlib\nimport json\nimport logging\nimport os\nimport socket\nimport subprocess\nimport sys\nimport threading\nimport time\nfrom pathlib import Path\nfrom typing import Any, Optional\n\nfrom .settings import encode_provider_options\n\n# Lightweight, self-contained server manager with no cross-process inspection\n\n# Set up logging based on environment variable\nLOG_LEVEL = os.getenv(\"LEANN_LOG_LEVEL\", \"WARNING\").upper()\nlogging.basicConfig(\n level=getattr(logging, LOG_LEVEL, logging.INFO),\n format=\"%(levelname)s - %(name)s - %(message)s\",\n)\nlogger = logging.getLogger(__name__)\n_LOCK_STALE_SECONDS = 600\n_FLOCK_TIMEOUT_SECONDS = 300\n_REGISTRY_LOCKS_GUARD = threading.Lock()\n_REGISTRY_LOCKS: dict[str, threading.Lock] = {}\n\n\ndef _flock_acquire(lock_file) -> None: # type: ignore[type-arg]\n \"\"\"Acquire an exclusive file lock for cross-process synchronisation.\n\n Uses ``fcntl.flock`` on POSIX and ``msvcrt.locking`` on Windows. Both are\n auto-released when the file descriptor is closed or the owning process\n exits, so a holder crash does not permanently block other waiters.\n \"\"\"\n try:\n import fcntl\n\n fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX)\n return\n except ImportError:\n pass\n except OSError:\n return\n\n if sys.platform != \"win32\":\n return\n import msvcrt\n\n # msvcrt.locking operates on byte ranges; ensure the file has content.\n lock_file.seek(0, 2)\n if lock_file.tell() == 0:\n lock_file.write(\"\\n\")\n lock_file.flush()\n lock_file.seek(0)\n\n deadline = time.monotonic() + _FLOCK_TIMEOUT_SECONDS\n while time.monotonic() < deadline:\n try:\n msvcrt.locking(lock_file.fileno(), msvcrt.LK_NBLCK, 1)\n return\n except OSError:\n time.sleep(0.5)\n logger.warning(\n \"Cross-process file lock timed out after %ds; proceeding without lock\",\n _FLOCK_TIMEOUT_SECONDS,\n )\n\n\ndef _flock_release(lock_file) -> None: # type: ignore[type-arg]\n \"\"\"Release the file lock acquired by :func:`_flock_acquire`.\"\"\"\n try:\n import fcntl\n\n fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN)\n return\n except ImportError:\n pass\n except OSError:\n return\n\n if sys.platform != \"win32\":\n return\n try:\n import msvcrt\n\n lock_file.seek(0)\n msvcrt.locking(lock_file.fileno(), msvcrt.LK_UNLCK, 1)\n except (ImportError, OSError):\n pass\n\n\ndef _is_colab_environment() -> bool:\n \"\"\"Check if we're running in Google Colab environment.\"\"\"\n return \"COLAB_GPU\" in os.environ or \"COLAB_TPU\" in os.environ\n\n\ndef _get_available_port(start_port: int = 5557) -> int:\n \"\"\"Get an available port starting from start_port.\"\"\"\n port = start_port\n while port < start_port + 100: # Try up to 100 ports\n try:\n with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:\n s.bind((\"localhost\", port))\n return port\n except OSError:\n port += 1\n raise RuntimeError(f\"No available ports found in range {start_port}-{start_port + 100}\")\n\n\ndef _check_port(port: int) -> bool:\n \"\"\"Check if a port is in use\"\"\"\n with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:\n return s.connect_ex((\"localhost\", port)) == 0\n\n\ndef _pid_is_alive(pid: int) -> bool:\n \"\"\"Best-effort liveness check for a process id.\"\"\"\n if pid <= 0:\n return False\n try:\n os.kill(pid, 0)\n return True\n except OSError:\n return False\n\n\n# Note: All cross-process scanning helpers removed for simplicity\n\n\ndef _safe_resolve(path: Path) -> str:\n \"\"\"Resolve paths safely even if the target does not yet exist.\"\"\"\n try:\n return str(path.resolve(strict=False))\n except Exception:\n return str(path)\n\n\ndef _safe_stat_signature(path: Path) -> dict:\n \"\"\"Return a lightweight signature describing the current state of a path.\"\"\"\n signature: dict[str, object] = {\"path\": _safe_resolve(path)}\n try:\n stat = path.stat()\n except FileNotFoundError:\n signature[\"missing\"] = True\n except Exception as exc: # pragma: no cover - unexpected filesystem errors\n signature[\"error\"] = str(exc)\n else:\n signature[\"mtime_ns\"] = stat.st_mtime_ns\n signature[\"size\"] = stat.st_size\n return signature\n\n\ndef _build_passages_signature(passages_file: Optional[str]) -> Optional[dict]:\n \"\"\"Collect modification signatures for metadata and referenced passage files.\"\"\"\n if not passages_file:\n return None\n\n meta_path = Path(passages_file)\n signature: dict[str, object] = {\"meta\": _safe_stat_signature(meta_path)}\n\n try:\n with meta_path.open(encoding=\"utf-8\") as fh:\n meta = json.load(fh)\n except FileNotFoundError:\n signature[\"meta_missing\"] = True\n signature[\"sources\"] = []\n return signature\n except json.JSONDecodeError as exc:\n signature[\"meta_error\"] = f\"json_error:{exc}\"\n signature[\"sources\"] = []\n return signature\n except Exception as exc: # pragma: no cover - unexpected errors\n signature[\"meta_error\"] = str(exc)\n signature[\"sources\"] = []\n return signature\n\n base_dir = meta_path.parent\n seen_paths: set[str] = set()\n source_signatures: list[dict[str, object]] = []\n\n for source in meta.get(\"passage_sources\", []):\n for key, kind in (\n (\"path\", \"passages\"),\n (\"path_relative\", \"passages\"),\n (\"index_path\", \"index\"),\n (\"index_path_relative\", \"index\"),\n ):\n raw_path = source.get(key)\n if not raw_path:\n continue\n candidate = Path(raw_path)\n if not candidate.is_absolute():\n candidate = base_dir / candidate\n resolved = _safe_resolve(candidate)\n if resolved in seen_paths:\n continue\n seen_paths.add(resolved)\n sig = _safe_stat_signature(candidate)\n sig[\"kind\"] = kind\n source_signatures.append(sig)\n\n signature[\"sources\"] = source_signatures\n return signature\n\n\n# Note: All cross-process scanning helpers removed for simplicity\n\n\nclass EmbeddingServerManager:\n \"\"\"\n A simplified manager for embedding server processes that avoids complex update mechanisms.\n \"\"\"\n\n def __init__(self, backend_module_name: str):\n \"\"\"\n Initializes the manager for a specific backend.\n\n Args:\n backend_module_name (str): The full module name of the backend's server script.\n e.g., \"leann_backend_diskann.embedding_server\"\n \"\"\"\n self.backend_module_name = backend_module_name\n self.server_process: Optional[subprocess.Popen] = None\n self.server_port: Optional[int] = None\n # Track last-started config for in-process reuse only\n self._server_config: Optional[dict] = None\n self._daemon_mode = False\n self._registry_path: Optional[Path] = None\n self._atexit_registered = False\n # Also register a weakref finalizer to ensure cleanup when manager is GC'ed\n try:\n import weakref\n\n self._finalizer = weakref.finalize(self, self._finalize_process)\n except Exception:\n self._finalizer = None\n\n def start_server(\n self,\n port: int,\n model_name: str,\n embedding_mode: str = \"sentence-transformers\",\n **kwargs,\n ) -> tuple[bool, int]:\n \"\"\"Start the embedding server.\"\"\"\n # passages_file may be present in kwargs for server CLI, but we don't need it here\n provider_options = kwargs.pop(\"provider_options\", None)\n passages_file = kwargs.get(\"passages_file\", \"\")\n distance_metric = kwargs.get(\"distance_metric\", \"\")\n use_daemon = bool(kwargs.get(\"use_daemon\", True))\n daemon_ttl_seconds = int(kwargs.get(\"daemon_ttl_seconds\", 900))\n\n config_signature = self._build_config_signature(\n model_name=model_name,\n embedding_mode=embedding_mode,\n provider_options=provider_options,\n passages_file=passages_file,\n distance_metric=distance_metric,\n )\n\n # If this manager already has a live server, just reuse it\n if (\n self.server_process\n and self.server_process.poll() is None\n and self.server_port\n and self._server_config == config_signature\n ):\n logger.info(\"Reusing in-process server\")\n return True, self.server_port\n\n # Configuration changed, stop existing ephemeral server before starting a new one\n if self.server_process and self.server_process.poll() is None and not self._daemon_mode:\n logger.info(\"Existing server configuration differs; restarting embedding server\")\n self.stop_server()\n\n # Reuse an already-running daemon from registry if possible.\n if use_daemon and not _is_colab_environment():\n with self._registry_lock(config_signature):\n adopted = self._adopt_registered_server(config_signature)\n if adopted is not None:\n self.server_process = None\n self.server_port = adopted\n self._server_config = config_signature\n self._daemon_mode = True\n return True, adopted\n\n # For Colab environment, use a different strategy\n if _is_colab_environment():\n logger.info(\"Detected Colab environment, using alternative startup strategy\")\n return self._start_server_colab(\n port,\n model_name,\n embedding_mode,\n config_signature=config_signature,\n provider_options=provider_options,\n **kwargs,\n )\n\n # Always pick a fresh available port\n try:\n actual_port = _get_available_port(port)\n except RuntimeError:\n logger.error(\"No available ports found\")\n return False, port\n\n # Start a new server (guarded by lock in daemon mode to avoid race).\n if use_daemon and not _is_colab_environment():\n with self._registry_lock(config_signature):\n adopted = self._adopt_registered_server(config_signature)\n if adopted is not None:\n self.server_process = None\n self.server_port = adopted\n self._server_config = config_signature\n self._daemon_mode = True\n return True, adopted\n started, actual_port = self._start_new_server(\n actual_port,\n model_name,\n embedding_mode,\n provider_options=provider_options,\n config_signature=config_signature,\n **kwargs,\n )\n if started:\n self._daemon_mode = True\n self._registry_path = self._write_registry_record(\n port=actual_port,\n config_signature=config_signature,\n daemon_ttl_seconds=daemon_ttl_seconds,\n )\n return started, actual_port\n\n started, actual_port = self._start_new_server(\n actual_port,\n model_name,\n embedding_mode,\n provider_options=provider_options,\n config_signature=config_signature,\n **kwargs,\n )\n if started:\n self._daemon_mode = False\n return started, actual_port\n\n def _build_config_signature(\n self,\n *,\n model_name: str,\n embedding_mode: str,\n provider_options: Optional[dict],\n passages_file: Optional[str],\n distance_metric: Optional[str],\n ) -> dict:\n \"\"\"Create a signature describing the current server configuration.\"\"\"\n passages_path = \"\"\n if passages_file:\n passages_path = _safe_resolve(Path(passages_file))\n return {\n \"model_name\": model_name,\n \"passages_file\": passages_path,\n \"embedding_mode\": embedding_mode,\n \"distance_metric\": distance_metric or \"\",\n \"provider_options\": provider_options or {},\n \"passages_signature\": _build_passages_signature(passages_file),\n }\n\n def _start_server_colab(\n self,\n port: int,\n model_name: str,\n embedding_mode: str = \"sentence-transformers\",\n *,\n config_signature: Optional[dict] = None,\n provider_options: Optional[dict] = None,\n **kwargs,\n ) -> tuple[bool, int]:\n \"\"\"Start server with Colab-specific configuration.\"\"\"\n # Try to find an available port\n try:\n actual_port = _get_available_port(port)\n except RuntimeError:\n logger.error(\"No available ports found\")\n return False, port\n\n logger.info(f\"Starting server on port {actual_port} for Colab environment\")\n\n # Use a simpler startup strategy for Colab\n command = self._build_server_command(actual_port, model_name, embedding_mode, **kwargs)\n\n try:\n # In Colab, we'll use a more direct approach\n self._launch_server_process_colab(\n command,\n actual_port,\n provider_options=provider_options,\n config_signature=config_signature,\n )\n started, ready_port = self._wait_for_server_ready_colab(actual_port)\n if started:\n self._server_config = config_signature or {\n \"model_name\": model_name,\n \"passages_file\": kwargs.get(\"passages_file\", \"\"),\n \"embedding_mode\": embedding_mode,\n \"provider_options\": provider_options or {},\n }\n return started, ready_port\n except Exception as e:\n logger.error(f\"Failed to start embedding server in Colab: {e}\")\n return False, actual_port\n\n # Note: No compatibility check needed; manager is per-searcher and configs are stable per instance\n\n def _start_new_server(\n self,\n port: int,\n model_name: str,\n embedding_mode: str,\n provider_options: Optional[dict] = None,\n config_signature: Optional[dict] = None,\n **kwargs,\n ) -> tuple[bool, int]:\n \"\"\"Start a new embedding server on the given port.\"\"\"\n logger.info(f\"Starting embedding server on port {port}...\")\n\n command = self._build_server_command(port, model_name, embedding_mode, **kwargs)\n\n try:\n self._launch_server_process(\n command,\n port,\n provider_options=provider_options,\n config_signature=config_signature,\n )\n started, ready_port = self._wait_for_server_ready(port)\n if started:\n self._server_config = config_signature or {\n \"model_name\": model_name,\n \"passages_file\": kwargs.get(\"passages_file\", \"\"),\n \"embedding_mode\": embedding_mode,\n \"provider_options\": provider_options or {},\n }\n return started, ready_port\n except Exception as e:\n logger.error(f\"Failed to start embedding server: {e}\")\n return False, port\n\n def _build_server_command(\n self, port: int, model_name: str, embedding_mode: str, **kwargs\n ) -> list:\n \"\"\"Build the command to start the embedding server.\"\"\"\n command = [\n sys.executable,\n \"-m\",\n self.backend_module_name,\n \"--zmq-port\",\n str(port),\n \"--model-name\",\n model_name,\n ]\n\n if kwargs.get(\"passages_file\"):\n # Convert to absolute path to ensure subprocess can find the file\n passages_file = Path(kwargs[\"passages_file\"]).resolve()\n command.extend([\"--passages-file\", str(passages_file)])\n if embedding_mode != \"sentence-transformers\":\n command.extend([\"--embedding-mode\", embedding_mode])\n if kwargs.get(\"distance_metric\"):\n command.extend([\"--distance-metric\", kwargs[\"distance_metric\"]])\n if kwargs.get(\"enable_warmup\", True):\n command.append(\"--enable-warmup\")\n if kwargs.get(\"use_daemon\", True):\n command.append(\"--daemon-mode\")\n ttl = int(kwargs.get(\"daemon_ttl_seconds\", 900))\n command.extend([\"--daemon-ttl\", str(ttl)])\n\n return command\n\n def _launch_server_process(\n self,\n command: list,\n port: int,\n *,\n provider_options: Optional[dict] = None,\n config_signature: Optional[dict] = None,\n ) -> None:\n \"\"\"Launch the server process.\"\"\"\n project_root = Path(__file__).parent.parent.parent.parent.parent\n logger.info(f\"Command: {' '.join(command)}\")\n\n # In CI environment, redirect stdout to avoid buffer deadlock but keep stderr for debugging\n # Embedding servers use many print statements that can fill stdout buffers\n is_ci = os.environ.get(\"CI\") == \"true\"\n if is_ci:\n stdout_target = subprocess.DEVNULL\n stderr_target = None # Keep stderr for error debugging in CI\n logger.info(\n \"CI environment detected, redirecting embedding server stdout to DEVNULL, keeping stderr\"\n )\n else:\n stdout_target = None # Direct to console for visible logs\n stderr_target = None # Direct to console for visible logs\n\n # Start embedding server subprocess\n logger.info(f\"Starting server process with command: {' '.join(command)}\")\n env = os.environ.copy()\n encoded_options = encode_provider_options(provider_options)\n if encoded_options:\n env[\"LEANN_EMBEDDING_OPTIONS\"] = encoded_options\n\n self.server_process = subprocess.Popen(\n command,\n cwd=project_root,\n stdout=stdout_target,\n stderr=stderr_target,\n env=env,\n )\n self.server_port = port\n # Record config for in-process reuse (best effort; refined later when ready)\n if config_signature is not None:\n self._server_config = config_signature\n else: # Fallback for unexpected code paths\n try:\n self._server_config = {\n \"model_name\": command[command.index(\"--model-name\") + 1]\n if \"--model-name\" in command\n else \"\",\n \"passages_file\": command[command.index(\"--passages-file\") + 1]\n if \"--passages-file\" in command\n else \"\",\n \"embedding_mode\": command[command.index(\"--embedding-mode\") + 1]\n if \"--embedding-mode\" in command\n else \"sentence-transformers\",\n \"provider_options\": provider_options or {},\n }\n except Exception:\n self._server_config = {\n \"model_name\": \"\",\n \"passages_file\": \"\",\n \"embedding_mode\": \"sentence-transformers\",\n \"provider_options\": provider_options or {},\n }\n logger.info(f\"Server process started with PID: {self.server_process.pid}\")\n\n # Register atexit callback only when we actually start a process\n if not self._atexit_registered:\n # Always attempt best-effort finalize at interpreter exit\n atexit.register(self._finalize_process)\n self._atexit_registered = True\n # Touch finalizer so it knows there is a live process\n finalizer = getattr(self, \"_finalizer\", None)\n if finalizer is not None and getattr(finalizer, \"alive\", False) is False:\n try:\n import weakref\n\n self._finalizer = weakref.finalize(self, self._finalize_process)\n except Exception:\n pass\n\n def _wait_for_server_ready(self, port: int) -> tuple[bool, int]:\n \"\"\"Wait for the server to be ready.\"\"\"\n max_wait, wait_interval = 120, 0.5\n for _ in range(int(max_wait / wait_interval)):\n if _check_port(port):\n logger.info(\"Embedding server is ready!\")\n return True, port\n\n if self.server_process and self.server_process.poll() is not None:\n logger.error(\"Server terminated during startup.\")\n return False, port\n\n time.sleep(wait_interval)\n\n logger.error(f\"Server failed to start within {max_wait} seconds.\")\n self.stop_server()\n return False, port\n\n def stop_server(self):\n \"\"\"Stops the embedding server process if it's running.\"\"\"\n if self._daemon_mode:\n # Daemon is intentionally process-global: this manager detaches by default.\n self.server_process = None\n self.server_port = None\n self._server_config = None\n self._daemon_mode = False\n self._registry_path = None\n return\n\n if not self.server_process:\n return\n\n if self.server_process and self.server_process.poll() is not None:\n # Process already terminated\n self.server_process = None\n self.server_port = None\n self._server_config = None\n return\n\n logger.info(\n f\"Terminating server process (PID: {self.server_process.pid}) for backend {self.backend_module_name}...\"\n )\n\n # Use simple termination first; if the server installed signal handlers,\n # it will exit cleanly. Otherwise escalate to kill after a short wait.\n try:\n self.server_process.terminate()\n except Exception:\n pass\n\n try:\n self.server_process.wait(timeout=5) # Give more time for graceful shutdown\n logger.info(f\"Server process {self.server_process.pid} terminated gracefully.\")\n except subprocess.TimeoutExpired:\n logger.warning(\n f\"Server process {self.server_process.pid} did not terminate within 5 seconds, force killing...\"\n )\n try:\n self.server_process.kill()\n except Exception:\n pass\n try:\n self.server_process.wait(timeout=2)\n logger.info(f\"Server process {self.server_process.pid} killed successfully.\")\n except subprocess.TimeoutExpired:\n logger.error(\n f\"Failed to kill server process {self.server_process.pid} - it may be hung\"\n )\n\n # Clean up process resources with timeout to avoid CI hang\n try:\n # Use shorter timeout in CI environments\n is_ci = os.environ.get(\"CI\") == \"true\"\n timeout = 3 if is_ci else 10\n self.server_process.wait(timeout=timeout)\n logger.info(f\"Server process {self.server_process.pid} cleanup completed\")\n except subprocess.TimeoutExpired:\n logger.warning(f\"Process cleanup timeout after {timeout}s, proceeding anyway\")\n except Exception as e:\n logger.warning(f\"Error during process cleanup: {e}\")\n finally:\n self.server_process = None\n self.server_port = None\n self._server_config = None\n\n def _finalize_process(self) -> None:\n \"\"\"Best-effort cleanup used by weakref.finalize/atexit.\"\"\"\n try:\n self.stop_server()\n except Exception:\n pass\n\n def _adopt_existing_server(self, *args, **kwargs) -> None:\n # Legacy no-op retained for compatibility.\n return\n\n def _launch_server_process_colab(\n self,\n command: list,\n port: int,\n *,\n provider_options: Optional[dict] = None,\n config_signature: Optional[dict] = None,\n ) -> None:\n \"\"\"Launch the server process with Colab-specific settings.\"\"\"\n logger.info(f\"Colab Command: {' '.join(command)}\")\n\n # In Colab, we need to be more careful about process management\n env = os.environ.copy()\n encoded_options = encode_provider_options(provider_options)\n if encoded_options:\n env[\"LEANN_EMBEDDING_OPTIONS\"] = encoded_options\n\n self.server_process = subprocess.Popen(\n command,\n stdout=subprocess.PIPE,\n stderr=subprocess.PIPE,\n text=True,\n env=env,\n )\n self.server_port = port\n logger.info(f\"Colab server process started with PID: {self.server_process.pid}\")\n\n # Register atexit callback (unified)\n if not self._atexit_registered:\n atexit.register(self._finalize_process)\n self._atexit_registered = True\n # Record config for in-process reuse is best-effort in Colab mode\n if config_signature is not None:\n self._server_config = config_signature\n else:\n self._server_config = {\n \"model_name\": \"\",\n \"passages_file\": \"\",\n \"embedding_mode\": \"sentence-transformers\",\n \"provider_options\": provider_options or {},\n }\n\n def _wait_for_server_ready_colab(self, port: int) -> tuple[bool, int]:\n \"\"\"Wait for the server to be ready with Colab-specific timeout.\"\"\"\n max_wait, wait_interval = 30, 0.5 # Shorter timeout for Colab\n\n for _ in range(int(max_wait / wait_interval)):\n if _check_port(port):\n logger.info(\"Colab embedding server is ready!\")\n return True, port\n\n if self.server_process and self.server_process.poll() is not None:\n # Check for error output\n stdout, stderr = self.server_process.communicate()\n logger.error(\"Colab server terminated during startup.\")\n logger.error(f\"stdout: {stdout}\")\n logger.error(f\"stderr: {stderr}\")\n return False, port\n\n time.sleep(wait_interval)\n\n logger.error(f\"Colab server failed to start within {max_wait} seconds.\")\n self.stop_server()\n return False, port\n\n @staticmethod\n def _registry_dir() -> Path:\n return Path.home() / \".leann\" / \"servers\"\n\n @contextlib.contextmanager\n def _registry_lock(self, config_signature: dict[str, Any]):\n \"\"\"Best-effort lock around the daemon check-then-start critical section.\n\n Two layers protect the registry from concurrent mutation:\n\n 1. **threading.Lock** (per registry key, module-global) — serialises\n threads inside the same process. POSIX ``fcntl.flock`` is\n process-granularity and does *not* block concurrent threads within\n a single process, so this layer is required on every platform.\n 2. **File lock** (``fcntl`` on POSIX, ``msvcrt`` on Windows) —\n serialises separate OS processes.\n \"\"\"\n lock_file = None\n lock_info_path: Optional[Path] = None\n thread_lock: Optional[threading.Lock] = None\n try:\n lock_path = self._registry_dir()\n lock_path.mkdir(parents=True, exist_ok=True)\n lock_key = self._registry_key(config_signature)\n\n with _REGISTRY_LOCKS_GUARD:\n thread_lock = _REGISTRY_LOCKS.setdefault(lock_key, threading.Lock())\n thread_lock.acquire()\n\n lock_file = (lock_path / f\"{lock_key}.lock\").open(\"a+\")\n lock_info_path = lock_path / f\"{lock_key}.lockinfo.json\"\n self._recover_stale_lock_info(lock_info_path)\n _flock_acquire(lock_file)\n self._write_lock_info(lock_info_path)\n yield\n finally:\n if lock_info_path is not None:\n lock_info_path.unlink(missing_ok=True)\n if lock_file is not None:\n _flock_release(lock_file)\n lock_file.close()\n if thread_lock is not None and thread_lock.locked():\n thread_lock.release()\n\n def _recover_stale_lock_info(self, lock_info_path: Path) -> None:\n if not lock_info_path.exists():\n return\n try:\n info = json.loads(lock_info_path.read_text(encoding=\"utf-8\"))\n pid = int(info.get(\"pid\") or 0)\n ts = float(info.get(\"ts\") or 0)\n except Exception:\n lock_info_path.unlink(missing_ok=True)\n return\n\n age = (time.time() - ts) if ts else (_LOCK_STALE_SECONDS + 1)\n if age > _LOCK_STALE_SECONDS or (pid > 0 and not _pid_is_alive(pid)):\n lock_info_path.unlink(missing_ok=True)\n\n def _write_lock_info(self, lock_info_path: Path) -> None:\n payload = {\"pid\": os.getpid(), \"ts\": time.time()}\n tmp_path = lock_info_path.with_suffix(\".tmp\")\n tmp_path.write_text(json.dumps(payload), encoding=\"utf-8\")\n os.replace(tmp_path, lock_info_path)\n\n def _registry_key(self, config_signature: dict[str, Any]) -> str:\n payload = {\n \"backend_module_name\": self.backend_module_name,\n \"config_signature\": config_signature,\n }\n encoded = json.dumps(payload, sort_keys=True, ensure_ascii=True).encode(\"utf-8\")\n return hashlib.sha256(encoded).hexdigest()\n\n def _write_registry_record(\n self,\n *,\n port: int,\n config_signature: dict[str, Any],\n daemon_ttl_seconds: int,\n ) -> Path:\n registry_dir = self._registry_dir()\n registry_dir.mkdir(parents=True, exist_ok=True)\n registry_path = registry_dir / f\"{self._registry_key(config_signature)}.json\"\n record = {\n \"pid\": self.server_process.pid if self.server_process else None,\n \"port\": port,\n \"backend_module_name\": self.backend_module_name,\n \"daemon_ttl_seconds\": int(daemon_ttl_seconds),\n \"created_at\": time.time(),\n \"config_signature\": config_signature,\n }\n tmp_path = registry_path.with_suffix(\".json.tmp\")\n tmp_path.write_text(json.dumps(record, indent=2), encoding=\"utf-8\")\n os.replace(tmp_path, registry_path)\n return registry_path\n\n def _adopt_registered_server(self, config_signature: dict[str, Any]) -> Optional[int]:\n registry_dir = self._registry_dir()\n if not registry_dir.exists():\n return None\n\n target = registry_dir / f\"{self._registry_key(config_signature)}.json\"\n if not target.exists():\n return None\n\n try:\n record = json.loads(target.read_text(encoding=\"utf-8\"))\n except Exception:\n target.unlink(missing_ok=True)\n return None\n\n if record.get(\"backend_module_name\") != self.backend_module_name:\n target.unlink(missing_ok=True)\n return None\n if record.get(\"config_signature\") != config_signature:\n target.unlink(missing_ok=True)\n return None\n\n pid = int(record.get(\"pid\") or 0)\n port = int(record.get(\"port\") or 0)\n if not _pid_is_alive(pid) or not _check_port(port):\n target.unlink(missing_ok=True)\n return None\n\n self._registry_path = target\n logger.info(\"Reusing daemonized embedding server on port %s\", port)\n return port\n\n @classmethod\n def list_daemons(cls) -> list[dict[str, Any]]:\n registry_dir = cls._registry_dir()\n if not registry_dir.exists():\n return []\n\n records: list[dict[str, Any]] = []\n for record_path in sorted(registry_dir.glob(\"*.json\")):\n try:\n record = json.loads(record_path.read_text(encoding=\"utf-8\"))\n except Exception:\n record_path.unlink(missing_ok=True)\n continue\n\n pid = int(record.get(\"pid\") or 0)\n port = int(record.get(\"port\") or 0)\n alive = _pid_is_alive(pid) and _check_port(port)\n if not alive:\n record_path.unlink(missing_ok=True)\n continue\n\n record[\"record_path\"] = str(record_path)\n records.append(record)\n return records\n\n @classmethod\n def stop_daemons(\n cls,\n *,\n backend_module_name: Optional[str] = None,\n passages_file: Optional[str] = None,\n ) -> int:\n resolved_passages_file = _safe_resolve(Path(passages_file)) if passages_file else None\n stopped = 0\n for record in cls.list_daemons():\n if backend_module_name and record.get(\"backend_module_name\") != backend_module_name:\n continue\n\n record_passages = (\n record.get(\"config_signature\", {}).get(\"passages_file\")\n if isinstance(record.get(\"config_signature\"), dict)\n else None\n )\n if resolved_passages_file and record_passages != resolved_passages_file:\n continue\n\n pid = int(record.get(\"pid\") or 0)\n try:\n os.kill(pid, 15)\n stopped += 1\n except OSError:\n pass\n\n record_path = record.get(\"record_path\")\n if record_path:\n Path(record_path).unlink(missing_ok=True)\n return stopped\n" }, { "path": "packages/leann-core/src/leann/interactive_utils.py", "content": "\"\"\"\nInteractive session utilities for LEANN applications.\n\nProvides shared readline functionality and command handling across\nCLI, API, and RAG example interactive modes.\n\"\"\"\n\nimport atexit\nimport os\nfrom pathlib import Path\nfrom types import ModuleType\nfrom typing import Callable, Optional\n\n# Try to import readline with fallback for Windows\nHAS_READLINE = False\nreadline: ModuleType | None = None\ntry:\n import readline\n\n HAS_READLINE = True\nexcept ImportError:\n # Windows doesn't have readline by default\n pass\n\n\nclass InteractiveSession:\n \"\"\"Manages interactive session with optional readline support and common commands.\"\"\"\n\n def __init__(\n self,\n history_name: str,\n prompt: str = \"You: \",\n welcome_message: str = \"\",\n ):\n \"\"\"\n Initialize interactive session with optional readline support.\n\n Args:\n history_name: Name for history file (e.g., \"cli\", \"api_chat\")\n (ignored if readline not available)\n prompt: Input prompt to display\n welcome_message: Message to show when starting session\n\n Note:\n On systems without readline (e.g., Windows), falls back to basic input()\n with limited functionality (no history, no line editing).\n \"\"\"\n self.history_name = history_name\n self.prompt = prompt\n self.welcome_message = welcome_message\n self._setup_complete = False\n\n def setup_readline(self):\n \"\"\"Setup readline with history support (if available).\"\"\"\n if self._setup_complete:\n return\n\n if not HAS_READLINE:\n # Readline not available (likely Windows), skip setup\n self._setup_complete = True\n return\n rl = readline\n if rl is None:\n self._setup_complete = True\n return\n\n # History file setup\n history_dir = Path.home() / \".leann\" / \"history\"\n history_dir.mkdir(parents=True, exist_ok=True)\n history_file = history_dir / f\"{self.history_name}.history\"\n\n # Load history if exists\n try:\n rl.read_history_file(str(history_file))\n rl.set_history_length(1000)\n except FileNotFoundError:\n pass\n\n # Save history on exit\n atexit.register(rl.write_history_file, str(history_file))\n\n # Optional: Enable vi editing mode (commented out by default)\n # readline.parse_and_bind(\"set editing-mode vi\")\n\n self._setup_complete = True\n\n def _show_help(self):\n \"\"\"Show available commands.\"\"\"\n print(\"Commands:\")\n print(\" quit/exit/q - Exit the chat\")\n print(\" help - Show this help message\")\n print(\" clear - Clear screen\")\n print(\" history - Show command history\")\n\n def _show_history(self):\n \"\"\"Show command history.\"\"\"\n if not HAS_READLINE:\n print(\" History not available (readline not supported on this system)\")\n return\n rl = readline\n if rl is None:\n print(\" History not available (readline not supported on this system)\")\n return\n\n history_length = rl.get_current_history_length()\n if history_length == 0:\n print(\" No history available\")\n return\n\n for i in range(history_length):\n item = rl.get_history_item(i + 1)\n if item:\n print(f\" {i + 1}: {item}\")\n\n def get_user_input(self) -> Optional[str]:\n \"\"\"\n Get user input with readline support.\n\n Returns:\n User input string, or None if EOF (Ctrl+D)\n \"\"\"\n try:\n return input(self.prompt).strip()\n except KeyboardInterrupt:\n print(\"\\n(Use 'quit' to exit)\")\n return \"\" # Return empty string to continue\n except EOFError:\n print(\"\\nGoodbye!\")\n return None\n\n def run_interactive_loop(self, handler_func: Callable[[str], None]):\n \"\"\"\n Run the interactive loop with a custom handler function.\n\n Args:\n handler_func: Function to handle user input that's not a built-in command\n Should accept a string and handle the user's query\n \"\"\"\n self.setup_readline()\n\n if self.welcome_message:\n print(self.welcome_message)\n\n while True:\n user_input = self.get_user_input()\n\n if user_input is None: # EOF (Ctrl+D)\n break\n\n if not user_input: # Empty input or KeyboardInterrupt\n continue\n\n # Handle built-in commands\n command = user_input.lower()\n if command in [\"quit\", \"exit\", \"q\"]:\n print(\"Goodbye!\")\n break\n elif command == \"help\":\n self._show_help()\n elif command == \"clear\":\n os.system(\"clear\" if os.name != \"nt\" else \"cls\")\n elif command == \"history\":\n self._show_history()\n else:\n # Regular user input - pass to handler\n try:\n handler_func(user_input)\n except Exception as e:\n print(f\"Error: {e}\")\n\n\ndef create_cli_session(index_name: str) -> InteractiveSession:\n \"\"\"Create an interactive session for CLI usage.\"\"\"\n return InteractiveSession(\n history_name=index_name,\n prompt=\"\\nYou: \",\n welcome_message=\"LEANN Assistant ready! Type 'quit' to exit, 'help' for commands\\n\"\n + \"=\" * 40,\n )\n\n\ndef create_api_session() -> InteractiveSession:\n \"\"\"Create an interactive session for API chat.\"\"\"\n return InteractiveSession(\n history_name=\"api_chat\",\n prompt=\"You: \",\n welcome_message=\"Leann Chat started (type 'quit' to exit, 'help' for commands)\\n\"\n + \"=\" * 40,\n )\n\n\ndef create_rag_session(app_name: str, data_description: str) -> InteractiveSession:\n \"\"\"Create an interactive session for RAG examples.\"\"\"\n return InteractiveSession(\n history_name=f\"{app_name}_rag\",\n prompt=\"You: \",\n welcome_message=f\"[Interactive Mode] Chat with your {data_description} data!\\nType 'quit' or 'exit' to stop, 'help' for commands.\\n\"\n + \"=\" * 40,\n )\n" }, { "path": "packages/leann-core/src/leann/interface.py", "content": "from abc import ABC, abstractmethod\nfrom typing import Any, Literal, Optional\n\nimport numpy as np\n\n\nclass LeannBackendBuilderInterface(ABC):\n \"\"\"Backend interface for building indexes\"\"\"\n\n @abstractmethod\n def build(self, data: np.ndarray, ids: list[str], index_path: str, **kwargs) -> None:\n \"\"\"Build index\n\n Args:\n data: Vector data (N, D)\n ids: List of string IDs for each vector\n index_path: Path to save index\n **kwargs: Backend-specific build parameters\n \"\"\"\n pass\n\n\nclass LeannBackendSearcherInterface(ABC):\n \"\"\"Backend interface for searching\"\"\"\n\n @abstractmethod\n def __init__(self, index_path: str, **kwargs):\n \"\"\"Initialize searcher\n\n Args:\n index_path: Path to index file\n **kwargs: Backend-specific loading parameters\n \"\"\"\n pass\n\n @abstractmethod\n def _ensure_server_running(\n self, passages_source_file: str, port: Optional[int], **kwargs\n ) -> int:\n \"\"\"Ensure server is running\"\"\"\n pass\n\n @abstractmethod\n def search(\n self,\n query: np.ndarray,\n top_k: int,\n complexity: int = 64,\n beam_width: int = 1,\n prune_ratio: float = 0.0,\n recompute_embeddings: bool = False,\n pruning_strategy: Literal[\"global\", \"local\", \"proportional\"] = \"global\",\n zmq_port: Optional[int] = None,\n **kwargs,\n ) -> dict[str, Any]:\n \"\"\"Search for nearest neighbors\n\n Args:\n query: Query vectors (B, D) where B is batch size, D is dimension\n top_k: Number of nearest neighbors to return\n complexity: Search complexity/candidate list size, higher = more accurate but slower\n beam_width: Number of parallel search paths/IO requests per iteration\n prune_ratio: Ratio of neighbors to prune via approximate distance (0.0-1.0)\n recompute_embeddings: Whether to fetch fresh embeddings from server vs use stored PQ codes\n pruning_strategy: PQ candidate selection strategy - \"global\" (default), \"local\", or \"proportional\"\n zmq_port: ZMQ port for embedding server communication. Must be provided if recompute_embeddings is True.\n **kwargs: Backend-specific parameters\n\n Returns:\n {\"labels\": [...], \"distances\": [...]}\n \"\"\"\n pass\n\n @abstractmethod\n def compute_query_embedding(\n self,\n query: str,\n use_server_if_available: bool = True,\n zmq_port: Optional[int] = None,\n query_template: Optional[str] = None,\n ) -> np.ndarray:\n \"\"\"Compute embedding for a query string\n\n Args:\n query: The query string to embed\n zmq_port: ZMQ port for embedding server\n use_server_if_available: Whether to try using embedding server first\n query_template: Optional prompt template to prepend to query\n\n Returns:\n Query embedding as numpy array with shape (1, D)\n \"\"\"\n pass\n\n\nclass LeannBackendFactoryInterface(ABC):\n \"\"\"Backend factory interface\"\"\"\n\n @staticmethod\n @abstractmethod\n def builder(**kwargs) -> LeannBackendBuilderInterface:\n \"\"\"Create Builder instance\"\"\"\n pass\n\n @staticmethod\n @abstractmethod\n def searcher(index_path: str, **kwargs) -> LeannBackendSearcherInterface:\n \"\"\"Create Searcher instance\"\"\"\n pass\n" }, { "path": "packages/leann-core/src/leann/mcp.py", "content": "#!/usr/bin/env python3\n\nimport json\nimport subprocess\nimport sys\n\n\ndef handle_request(request):\n if request.get(\"method\") == \"initialize\":\n return {\n \"jsonrpc\": \"2.0\",\n \"id\": request.get(\"id\"),\n \"result\": {\n \"capabilities\": {\"tools\": {}},\n \"protocolVersion\": \"2024-11-05\",\n \"serverInfo\": {\"name\": \"leann-mcp\", \"version\": \"1.0.0\"},\n },\n }\n\n elif request.get(\"method\") == \"tools/list\":\n return {\n \"jsonrpc\": \"2.0\",\n \"id\": request.get(\"id\"),\n \"result\": {\n \"tools\": [\n {\n \"name\": \"leann_search\",\n \"description\": \"\"\"🔍 Search code using natural language - like having a coding assistant who knows your entire codebase!\n\n🎯 **Perfect for**:\n- \"How does authentication work?\" → finds auth-related code\n- \"Error handling patterns\" → locates try-catch blocks and error logic\n- \"Database connection setup\" → finds DB initialization code\n- \"API endpoint definitions\" → locates route handlers\n- \"Configuration management\" → finds config files and usage\n\n💡 **Pro tip**: Use this before making any changes to understand existing patterns and conventions.\"\"\",\n \"inputSchema\": {\n \"type\": \"object\",\n \"properties\": {\n \"index_name\": {\n \"type\": \"string\",\n \"description\": \"Name of the LEANN index to search. Use 'leann_list' first to see available indexes.\",\n },\n \"query\": {\n \"type\": \"string\",\n \"description\": \"Search query - can be natural language (e.g., 'how to handle errors') or technical terms (e.g., 'async function definition')\",\n },\n \"top_k\": {\n \"type\": \"integer\",\n \"default\": 5,\n \"minimum\": 1,\n \"maximum\": 20,\n \"description\": \"Number of search results to return. Use 5-10 for focused results, 15-20 for comprehensive exploration.\",\n },\n \"complexity\": {\n \"type\": \"integer\",\n \"default\": 32,\n \"minimum\": 16,\n \"maximum\": 128,\n \"description\": \"Search complexity level. Use 16-32 for fast searches (recommended), 64+ for higher precision when needed.\",\n },\n \"show_metadata\": {\n \"type\": \"boolean\",\n \"default\": False,\n \"description\": \"Include file paths and metadata in search results. Useful for understanding which files contain the results.\",\n },\n },\n \"required\": [\"index_name\", \"query\"],\n },\n },\n {\n \"name\": \"leann_list\",\n \"description\": \"📋 Show all your indexed codebases - your personal code library! Use this to see what's available for search.\",\n \"inputSchema\": {\"type\": \"object\", \"properties\": {}},\n },\n ]\n },\n }\n\n elif request.get(\"method\") == \"tools/call\":\n tool_name = request[\"params\"][\"name\"]\n args = request[\"params\"].get(\"arguments\", {})\n\n try:\n if tool_name == \"leann_search\":\n # Validate required parameters\n if not args.get(\"index_name\") or not args.get(\"query\"):\n return {\n \"jsonrpc\": \"2.0\",\n \"id\": request.get(\"id\"),\n \"result\": {\n \"content\": [\n {\n \"type\": \"text\",\n \"text\": \"Error: Both index_name and query are required\",\n }\n ]\n },\n }\n\n cmd = [\n \"leann\",\n \"search\",\n args[\"index_name\"],\n args[\"query\"],\n f\"--top-k={args.get('top_k', 5)}\",\n f\"--complexity={args.get('complexity', 32)}\",\n \"--non-interactive\",\n \"--json\",\n ]\n if args.get(\"show_metadata\", False):\n cmd.append(\"--show-metadata\")\n result = subprocess.run(cmd, capture_output=True, text=True)\n\n elif tool_name == \"leann_list\":\n result = subprocess.run([\"leann\", \"list\"], capture_output=True, text=True)\n\n return {\n \"jsonrpc\": \"2.0\",\n \"id\": request.get(\"id\"),\n \"result\": {\n \"content\": [\n {\n \"type\": \"text\",\n \"text\": result.stdout\n if result.returncode == 0\n else f\"Error: {result.stderr}\",\n }\n ]\n },\n }\n\n except Exception as e:\n return {\n \"jsonrpc\": \"2.0\",\n \"id\": request.get(\"id\"),\n \"error\": {\"code\": -1, \"message\": str(e)},\n }\n\n\ndef main():\n for line in sys.stdin:\n try:\n request = json.loads(line.strip())\n response = handle_request(request)\n if response:\n print(json.dumps(response))\n sys.stdout.flush()\n except Exception as e:\n error_response = {\n \"jsonrpc\": \"2.0\",\n \"id\": None,\n \"error\": {\"code\": -1, \"message\": str(e)},\n }\n print(json.dumps(error_response))\n sys.stdout.flush()\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "packages/leann-core/src/leann/metadata_filter.py", "content": "\"\"\"\nMetadata filtering engine for LEANN search results.\n\nThis module provides generic metadata filtering capabilities that can be applied\nto search results from any LEANN backend. The filtering supports various\noperators for different data types including numbers, strings, booleans, and lists.\n\"\"\"\n\nimport logging\nfrom typing import Any, Optional, Union\n\nlogger = logging.getLogger(__name__)\n\n# Type alias for filter specifications\nFilterValue = Union[str, int, float, bool, list]\nFilterSpec = dict[str, FilterValue]\nMetadataFilters = dict[str, FilterSpec]\n\n\nclass MetadataFilterEngine:\n \"\"\"\n Engine for evaluating metadata filters against search results.\n\n Supports various operators for filtering based on metadata fields:\n - Comparison: ==, !=, <, <=, >, >=\n - Membership: in, not_in\n - String operations: contains, starts_with, ends_with\n - Boolean operations: is_true, is_false\n \"\"\"\n\n def __init__(self):\n \"\"\"Initialize the filter engine with supported operators.\"\"\"\n self.operators = {\n \"==\": self._equals,\n \"!=\": self._not_equals,\n \"<\": self._less_than,\n \"<=\": self._less_than_or_equal,\n \">\": self._greater_than,\n \">=\": self._greater_than_or_equal,\n \"in\": self._in,\n \"not_in\": self._not_in,\n \"contains\": self._contains,\n \"starts_with\": self._starts_with,\n \"ends_with\": self._ends_with,\n \"is_true\": self._is_true,\n \"is_false\": self._is_false,\n }\n\n def apply_filters(\n self, search_results: list[dict[str, Any]], metadata_filters: Optional[MetadataFilters]\n ) -> list[dict[str, Any]]:\n \"\"\"\n Apply metadata filters to a list of search results.\n\n Args:\n search_results: List of result dictionaries, each containing 'metadata' field\n metadata_filters: Dictionary of filter specifications\n Format: {\"field_name\": {\"operator\": value}}\n\n Returns:\n Filtered list of search results\n \"\"\"\n if not metadata_filters:\n return search_results\n\n logger.debug(f\"Applying filters: {metadata_filters}\")\n logger.debug(f\"Input results count: {len(search_results)}\")\n\n filtered_results = []\n for result in search_results:\n if self._evaluate_filters(result, metadata_filters):\n filtered_results.append(result)\n\n logger.debug(f\"Filtered results count: {len(filtered_results)}\")\n return filtered_results\n\n def _evaluate_filters(self, result: dict[str, Any], filters: MetadataFilters) -> bool:\n \"\"\"\n Evaluate all filters against a single search result.\n\n All filters must pass (AND logic) for the result to be included.\n\n Args:\n result: Full search result dictionary (including metadata, text, etc.)\n filters: Filter specifications to evaluate\n\n Returns:\n True if all filters pass, False otherwise\n \"\"\"\n for field_name, filter_spec in filters.items():\n if not self._evaluate_field_filter(result, field_name, filter_spec):\n return False\n return True\n\n def _evaluate_field_filter(\n self, result: dict[str, Any], field_name: str, filter_spec: FilterSpec\n ) -> bool:\n \"\"\"\n Evaluate a single field filter against a search result.\n\n Args:\n result: Full search result dictionary\n field_name: Name of the field to filter on\n filter_spec: Filter specification for this field\n\n Returns:\n True if the filter passes, False otherwise\n \"\"\"\n # First check top-level fields, then check metadata\n field_value = result.get(field_name)\n if field_value is None:\n # Try to get from metadata if not found at top level\n metadata = result.get(\"metadata\", {})\n field_value = metadata.get(field_name)\n\n # Handle missing fields - they fail all filters except existence checks\n if field_value is None:\n logger.debug(f\"Field '{field_name}' not found in result or metadata\")\n return False\n\n # Evaluate each operator in the filter spec\n for operator, expected_value in filter_spec.items():\n if operator not in self.operators:\n logger.warning(f\"Unsupported operator: {operator}\")\n return False\n\n try:\n if not self.operators[operator](field_value, expected_value):\n logger.debug(\n f\"Filter failed: {field_name} {operator} {expected_value} \"\n f\"(actual: {field_value})\"\n )\n return False\n except Exception as e:\n logger.warning(\n f\"Error evaluating filter {field_name} {operator} {expected_value}: {e}\"\n )\n return False\n\n return True\n\n # Comparison operators\n def _equals(self, field_value: Any, expected_value: Any) -> bool:\n \"\"\"Check if field value equals expected value.\"\"\"\n return field_value == expected_value\n\n def _not_equals(self, field_value: Any, expected_value: Any) -> bool:\n \"\"\"Check if field value does not equal expected value.\"\"\"\n return field_value != expected_value\n\n def _less_than(self, field_value: Any, expected_value: Any) -> bool:\n \"\"\"Check if field value is less than expected value.\"\"\"\n return self._numeric_compare(field_value, expected_value, lambda a, b: a < b)\n\n def _less_than_or_equal(self, field_value: Any, expected_value: Any) -> bool:\n \"\"\"Check if field value is less than or equal to expected value.\"\"\"\n return self._numeric_compare(field_value, expected_value, lambda a, b: a <= b)\n\n def _greater_than(self, field_value: Any, expected_value: Any) -> bool:\n \"\"\"Check if field value is greater than expected value.\"\"\"\n return self._numeric_compare(field_value, expected_value, lambda a, b: a > b)\n\n def _greater_than_or_equal(self, field_value: Any, expected_value: Any) -> bool:\n \"\"\"Check if field value is greater than or equal to expected value.\"\"\"\n return self._numeric_compare(field_value, expected_value, lambda a, b: a >= b)\n\n # Membership operators\n def _in(self, field_value: Any, expected_value: Any) -> bool:\n \"\"\"Check if field value is in the expected list/collection.\"\"\"\n if not isinstance(expected_value, (list, tuple, set)):\n raise ValueError(\"'in' operator requires a list, tuple, or set\")\n return field_value in expected_value\n\n def _not_in(self, field_value: Any, expected_value: Any) -> bool:\n \"\"\"Check if field value is not in the expected list/collection.\"\"\"\n if not isinstance(expected_value, (list, tuple, set)):\n raise ValueError(\"'not_in' operator requires a list, tuple, or set\")\n return field_value not in expected_value\n\n # String operators\n def _contains(self, field_value: Any, expected_value: Any) -> bool:\n \"\"\"Check if field value contains the expected substring.\"\"\"\n field_str = str(field_value)\n expected_str = str(expected_value)\n return expected_str in field_str\n\n def _starts_with(self, field_value: Any, expected_value: Any) -> bool:\n \"\"\"Check if field value starts with the expected prefix.\"\"\"\n field_str = str(field_value)\n expected_str = str(expected_value)\n return field_str.startswith(expected_str)\n\n def _ends_with(self, field_value: Any, expected_value: Any) -> bool:\n \"\"\"Check if field value ends with the expected suffix.\"\"\"\n field_str = str(field_value)\n expected_str = str(expected_value)\n return field_str.endswith(expected_str)\n\n # Boolean operators\n def _is_true(self, field_value: Any, expected_value: Any) -> bool:\n \"\"\"Check if field value is truthy.\"\"\"\n return bool(field_value)\n\n def _is_false(self, field_value: Any, expected_value: Any) -> bool:\n \"\"\"Check if field value is falsy.\"\"\"\n return not bool(field_value)\n\n # Helper methods\n def _numeric_compare(self, field_value: Any, expected_value: Any, compare_func) -> bool:\n \"\"\"\n Helper for numeric comparisons with type coercion.\n\n Args:\n field_value: Value from metadata\n expected_value: Value to compare against\n compare_func: Comparison function to apply\n\n Returns:\n Result of comparison\n \"\"\"\n try:\n # Try to convert both values to numbers for comparison\n if isinstance(field_value, str) and isinstance(expected_value, str):\n # String comparison if both are strings\n return compare_func(field_value, expected_value)\n\n # Numeric comparison - attempt to convert to float\n field_num = (\n float(field_value) if not isinstance(field_value, (int, float)) else field_value\n )\n expected_num = (\n float(expected_value)\n if not isinstance(expected_value, (int, float))\n else expected_value\n )\n\n return compare_func(field_num, expected_num)\n except (ValueError, TypeError):\n # Fall back to string comparison if numeric conversion fails\n return compare_func(str(field_value), str(expected_value))\n" }, { "path": "packages/leann-core/src/leann/react_agent.py", "content": "\"\"\"\nSimple ReAct agent for multiturn retrieval with LEANN.\n\nThis implements a basic ReAct (Reasoning + Acting) agent pattern:\n- Thought: LLM reasons about what to do next\n- Action: Performs a search action\n- Observation: Gets results from search\n- Repeat until final answer\n\nReference: Inspired by mini-swe-agent pattern, kept simple for multiturn retrieval.\n\"\"\"\n\nfrom __future__ import annotations\n\nimport logging\nfrom typing import Any\n\nfrom .api import LeannSearcher, SearchResult\nfrom .chat import LLMInterface, get_llm\n\nlogger = logging.getLogger(__name__)\n\n\nclass ReActAgent:\n \"\"\"\n Simple ReAct agent for multiturn retrieval.\n\n The agent follows this pattern:\n 1. Thought: LLM reasons about what information is needed\n 2. Action: Performs a search query\n 3. Observation: Gets search results\n 4. Repeat until LLM decides it has enough information to answer\n \"\"\"\n\n def __init__(\n self,\n searcher: LeannSearcher,\n llm: LLMInterface | None = None,\n llm_config: dict[str, Any] | None = None,\n max_iterations: int = 5,\n ):\n \"\"\"\n Initialize the ReAct agent.\n\n Args:\n searcher: LeannSearcher instance for performing searches\n llm: LLM interface (if None, will create from llm_config)\n llm_config: Configuration for creating LLM if llm is None\n max_iterations: Maximum number of search iterations (default: 5)\n \"\"\"\n self.searcher = searcher\n if llm is None:\n self.llm = get_llm(llm_config)\n else:\n self.llm = llm\n self.max_iterations = max_iterations\n self.search_history: list[dict[str, Any]] = []\n\n def _format_search_results(self, results: list[SearchResult]) -> str:\n \"\"\"Format search results as a string for the LLM.\"\"\"\n if not results:\n return \"No results found.\"\n formatted = []\n for i, result in enumerate(results, 1):\n formatted.append(f\"[Result {i}] (Score: {result.score:.3f})\\n{result.text[:500]}...\")\n if result.metadata.get(\"source\"):\n formatted[-1] += f\"\\nSource: {result.metadata['source']}\"\n return \"\\n\\n\".join(formatted)\n\n def _create_react_prompt(\n self, question: str, iteration: int, previous_observations: list[str]\n ) -> str:\n \"\"\"Create the ReAct prompt for the LLM.\"\"\"\n prompt = f\"\"\"You are a helpful assistant that answers questions by searching through a knowledge base.\n\nQuestion: {question}\n\nYou can search the knowledge base by using the action: search(\"query\")\n\nPrevious observations:\n\"\"\"\n if previous_observations:\n for i, obs in enumerate(previous_observations, 1):\n prompt += f\"\\nObservation {i}:\\n{obs}\\n\"\n else:\n prompt += \"None yet.\\n\"\n\n prompt += f\"\"\"\nCurrent iteration: {iteration}/{self.max_iterations}\n\nThink step by step:\n1. If you need more information, use search(\"your search query\")\n2. If you have enough information, provide your final answer\n\nFormat your response as:\nThought: [your reasoning]\nAction: search(\"query\") OR Final Answer: [your answer]\n\"\"\"\n\n return prompt\n\n def _parse_llm_response(self, response: str) -> tuple[str, str | None]:\n \"\"\"\n Parse LLM response to extract thought and action.\n\n Returns:\n (thought, action) where action is either a search query string or None if final answer\n \"\"\"\n thought = \"\"\n action = None\n\n # Extract thought\n if \"Thought:\" in response:\n thought_part = response.split(\"Thought:\")[1]\n if \"Action:\" in thought_part:\n thought = thought_part.split(\"Action:\")[0].strip()\n elif \"Final Answer:\" in thought_part:\n thought = thought_part.split(\"Final Answer:\")[0].strip()\n else:\n thought = thought_part.strip()\n else:\n # If no explicit Thought, use everything before Action/Final Answer\n if \"Action:\" in response or \"Final Answer:\" in response:\n thought = response.split(\"Action:\")[0].split(\"Final Answer:\")[0].strip()\n else:\n thought = response.strip()\n\n # Extract action\n if \"Final Answer:\" in response:\n # Agent wants to provide final answer\n action = None\n elif \"Action:\" in response:\n action_part = response.split(\"Action:\")[1].strip()\n # Try to extract search query\n if 'search(\"' in action_part:\n start = action_part.find('search(\"') + 7\n end = action_part.find('\")', start)\n if end != -1:\n action = action_part[start:end]\n else:\n # Try with single quote\n end = action_part.find('\")', start)\n if end != -1:\n action = action_part[start:end]\n elif \"search(\" in action_part:\n # Handle without quotes\n start = action_part.find(\"search(\") + 7\n end = action_part.find(\")\", start)\n if end != -1:\n action = action_part[start:end].strip('\"').strip(\"'\")\n elif \"search(\" in response.lower():\n # Try to extract search query even if format is slightly different\n import re\n\n match = re.search(r'search\\([\"\\']([^\"\\']+)[\"\\']\\)', response, re.IGNORECASE)\n if match:\n action = match.group(1)\n\n return thought, action\n\n def search(self, query: str, top_k: int = 5) -> list[SearchResult]:\n \"\"\"\n Perform a search and return results.\n\n Args:\n query: Search query string\n top_k: Number of results to return\n\n Returns:\n List of SearchResult objects\n \"\"\"\n logger.info(f\"🔍 Searching: {query}\")\n results = self.searcher.search(query, top_k=top_k)\n return results\n\n def run(self, question: str, top_k: int = 5) -> str:\n \"\"\"\n Run the ReAct agent to answer a question.\n\n Args:\n question: The question to answer\n top_k: Number of search results per iteration\n\n Returns:\n Final answer string\n \"\"\"\n logger.info(f\"🤖 Starting ReAct agent for question: {question}\")\n self.search_history = []\n previous_observations: list[str] = []\n all_context: list[str] = []\n\n for iteration in range(1, self.max_iterations + 1):\n logger.info(f\"\\n--- Iteration {iteration}/{self.max_iterations} ---\")\n\n # Create prompt for this iteration\n prompt = self._create_react_prompt(question, iteration, previous_observations)\n\n # Get LLM response\n logger.info(\"💭 Getting LLM reasoning...\")\n response = self.llm.ask(prompt)\n\n # Parse response\n thought, action = self._parse_llm_response(response)\n logger.info(f\"Thought: {thought}\")\n\n if action is None:\n # LLM wants to provide final answer\n if \"Final Answer:\" in response:\n final_answer = response.split(\"Final Answer:\")[1].strip()\n else:\n # Extract answer from response\n final_answer = response.strip()\n # Remove any action markers if present\n if \"Action:\" in final_answer:\n final_answer = final_answer.split(\"Action:\")[0].strip()\n\n logger.info(f\"✅ Final answer: {final_answer}\")\n return final_answer\n\n # Perform search action\n logger.info(f'🔍 Action: search(\"{action}\")')\n results = self.search(action, top_k=top_k)\n\n # Format observation\n observation = self._format_search_results(results)\n previous_observations.append(observation)\n all_context.append(f\"Search: {action}\\n{observation}\")\n\n # Store in history\n self.search_history.append(\n {\n \"iteration\": iteration,\n \"thought\": thought,\n \"action\": action,\n \"results_count\": len(results),\n }\n )\n\n # If no results, might want to stop early\n if not results and iteration >= 2:\n logger.warning(\"No results found, asking LLM for final answer...\")\n final_prompt = f\"\"\"Based on the previous searches, provide your best answer to the question.\n\nQuestion: {question}\n\nPrevious searches and results:\n{chr(10).join(all_context)}\n\nSince no new results were found, provide your final answer based on what you know.\n\"\"\"\n final_answer = self.llm.ask(final_prompt)\n return final_answer.strip()\n\n # Max iterations reached, get final answer\n logger.warning(f\"Reached max iterations ({self.max_iterations}), getting final answer...\")\n final_prompt = f\"\"\"Based on all the searches performed, provide your final answer to the question.\n\nQuestion: {question}\n\nAll search results:\n{chr(10).join(all_context)}\n\nProvide your final answer now.\n\"\"\"\n final_answer = self.llm.ask(final_prompt)\n return final_answer.strip()\n\n\ndef create_react_agent(\n index_path: str,\n llm_config: dict[str, Any] | None = None,\n max_iterations: int = 5,\n **searcher_kwargs,\n) -> ReActAgent:\n \"\"\"\n Convenience function to create a ReActAgent.\n\n Args:\n index_path: Path to LEANN index\n llm_config: LLM configuration dict\n max_iterations: Maximum search iterations\n **searcher_kwargs: Additional kwargs for LeannSearcher\n\n Returns:\n ReActAgent instance\n \"\"\"\n searcher = LeannSearcher(index_path, **searcher_kwargs)\n return ReActAgent(searcher=searcher, llm_config=llm_config, max_iterations=max_iterations)\n" }, { "path": "packages/leann-core/src/leann/registry.py", "content": "# packages/leann-core/src/leann/registry.py\n\nimport importlib\nimport importlib.metadata\nimport json\nimport logging\nfrom pathlib import Path\nfrom typing import TYPE_CHECKING, Optional, Union\n\nif TYPE_CHECKING:\n from leann.interface import LeannBackendFactoryInterface\n\n# Set up logger for this module\nlogger = logging.getLogger(__name__)\n\nBACKEND_REGISTRY: dict[str, \"LeannBackendFactoryInterface\"] = {}\n\n\ndef register_backend(name: str):\n \"\"\"A decorator to register a new backend class.\"\"\"\n\n def decorator(cls):\n logger.debug(f\"Registering backend '{name}'\")\n BACKEND_REGISTRY[name] = cls\n return cls\n\n return decorator\n\n\ndef autodiscover_backends():\n \"\"\"Automatically discovers and imports all 'leann-backend-*' packages.\"\"\"\n # print(\"INFO: Starting backend auto-discovery...\")\n discovered_backends = []\n for dist in importlib.metadata.distributions():\n dist_name = dist.metadata[\"name\"]\n if dist_name is None:\n continue\n if dist_name.startswith(\"leann-backend-\"):\n backend_module_name = dist_name.replace(\"-\", \"_\")\n discovered_backends.append(backend_module_name)\n\n for backend_module_name in sorted(discovered_backends): # sort for deterministic loading\n try:\n importlib.import_module(backend_module_name)\n # Registration message is printed by the decorator\n except ImportError:\n # print(f\"WARN: Could not import backend module '{backend_module_name}': {e}\")\n pass\n # print(\"INFO: Backend auto-discovery finished.\")\n\n\ndef register_project_directory(project_dir: Optional[Union[str, Path]] = None):\n \"\"\"\n Register a project directory in the global LEANN registry.\n\n This allows `leann list` to discover indexes created by apps or other tools.\n\n Args:\n project_dir: Directory to register. If None, uses current working directory.\n \"\"\"\n if project_dir is None:\n project_dir = Path.cwd()\n else:\n project_dir = Path(project_dir)\n\n # Only register directories that have some kind of LEANN content\n # Either .leann/indexes/ (CLI format) or *.leann.meta.json files (apps format)\n has_cli_indexes = (project_dir / \".leann\" / \"indexes\").exists()\n has_app_indexes = any(project_dir.rglob(\"*.leann.meta.json\"))\n\n if not (has_cli_indexes or has_app_indexes):\n # Don't register if there are no LEANN indexes\n return\n\n global_registry = Path.home() / \".leann\" / \"projects.json\"\n global_registry.parent.mkdir(exist_ok=True)\n\n project_str = str(project_dir.resolve())\n\n # Load existing registry\n projects = []\n if global_registry.exists():\n try:\n with open(global_registry) as f:\n projects = json.load(f)\n except Exception:\n logger.debug(\"Could not load existing project registry\")\n projects = []\n\n # Add project if not already present\n if project_str not in projects:\n projects.append(project_str)\n\n # Save updated registry\n try:\n with open(global_registry, \"w\") as f:\n json.dump(projects, f, indent=2)\n logger.debug(f\"Registered project directory: {project_str}\")\n except Exception as e:\n logger.warning(f\"Could not save project registry: {e}\")\n" }, { "path": "packages/leann-core/src/leann/searcher_base.py", "content": "import json\nfrom abc import ABC, abstractmethod\nfrom pathlib import Path\nfrom typing import Any, Literal, Optional\n\nimport numpy as np\n\nfrom .embedding_server_manager import EmbeddingServerManager\nfrom .interface import LeannBackendSearcherInterface\n\n\nclass BaseSearcher(LeannBackendSearcherInterface, ABC):\n \"\"\"\n Abstract base class for Leann searchers, containing common logic for\n loading metadata, managing embedding servers, and handling file paths.\n \"\"\"\n\n def __init__(self, index_path: str, backend_module_name: str, **kwargs):\n \"\"\"\n Initializes the BaseSearcher.\n\n Args:\n index_path: Path to the Leann index file (e.g., '.../my_index.leann').\n backend_module_name: The specific embedding server module to use\n (e.g., 'leann_backend_hnsw.hnsw_embedding_server').\n **kwargs: Additional keyword arguments.\n \"\"\"\n self.index_path = Path(index_path)\n self.index_dir = self.index_path.parent\n self.meta = kwargs.get(\"meta\", self._load_meta())\n\n if not self.meta:\n raise ValueError(\"Searcher requires metadata from .meta.json.\")\n\n self.dimensions = self.meta.get(\"dimensions\")\n if not self.dimensions:\n raise ValueError(\"Dimensions not found in Leann metadata.\")\n\n self.embedding_model = self.meta.get(\"embedding_model\")\n if not self.embedding_model:\n print(\"WARNING: embedding_model not found in meta.json. Recompute will fail.\")\n\n self.embedding_mode = self.meta.get(\"embedding_mode\", \"sentence-transformers\")\n self.embedding_options = self.meta.get(\"embedding_options\", {})\n self.enable_warmup = bool(kwargs.get(\"enable_warmup\", True))\n self.use_daemon = bool(kwargs.get(\"use_daemon\", True))\n self.daemon_ttl_seconds = int(kwargs.get(\"daemon_ttl_seconds\", 900))\n\n self.embedding_server_manager = EmbeddingServerManager(\n backend_module_name=backend_module_name,\n )\n\n def _load_meta(self) -> dict[str, Any]:\n \"\"\"Loads the metadata file associated with the index.\"\"\"\n # This is the corrected logic for finding the meta file.\n meta_path = self.index_dir / f\"{self.index_path.name}.meta.json\"\n if not meta_path.exists():\n raise FileNotFoundError(f\"Leann metadata file not found at {meta_path}\")\n with open(meta_path, encoding=\"utf-8\") as f:\n return json.load(f)\n\n def _ensure_server_running(\n self, passages_source_file: str, port: Optional[int], **kwargs\n ) -> int:\n \"\"\"\n Ensures the embedding server is running if recompute is needed.\n This is a helper for subclasses.\n \"\"\"\n if not self.embedding_model:\n raise ValueError(\"Cannot use recompute mode without 'embedding_model' in meta.json.\")\n\n # Get distance_metric from meta if not provided in kwargs\n distance_metric = (\n kwargs.get(\"distance_metric\")\n or self.meta.get(\"backend_kwargs\", {}).get(\"distance_metric\")\n or \"mips\"\n )\n\n # Filter out ALL prompt templates from provider_options during search\n # Templates are applied in compute_query_embedding (line 109-110) BEFORE server call\n # The server should never apply templates during search to avoid double-templating\n search_provider_options = {\n k: v\n for k, v in self.embedding_options.items()\n if k not in (\"build_prompt_template\", \"query_prompt_template\", \"prompt_template\")\n }\n\n server_started, actual_port = self.embedding_server_manager.start_server(\n port=port if port is not None else 5557,\n model_name=self.embedding_model,\n embedding_mode=self.embedding_mode,\n passages_file=passages_source_file,\n distance_metric=distance_metric,\n enable_warmup=kwargs.get(\"enable_warmup\", self.enable_warmup),\n use_daemon=kwargs.get(\"use_daemon\", self.use_daemon),\n daemon_ttl_seconds=kwargs.get(\"daemon_ttl_seconds\", self.daemon_ttl_seconds),\n provider_options=search_provider_options,\n )\n if not server_started:\n raise RuntimeError(f\"Failed to start embedding server on port {actual_port}\")\n\n return actual_port\n\n def compute_query_embedding(\n self,\n query: str,\n use_server_if_available: bool = True,\n zmq_port: Optional[int] = None,\n query_template: Optional[str] = None,\n ) -> np.ndarray:\n \"\"\"\n Compute embedding for a query string.\n\n Args:\n query: The query string to embed\n zmq_port: ZMQ port for embedding server\n use_server_if_available: Whether to try using embedding server first\n query_template: Optional prompt template to prepend to query\n\n Returns:\n Query embedding as numpy array\n \"\"\"\n # Apply query template BEFORE any computation path\n # This ensures template is applied consistently for both server and fallback paths\n if query_template:\n query = f\"{query_template}{query}\"\n\n # Try to use embedding server if available and requested\n if use_server_if_available:\n try:\n # TODO: Maybe we can directly use this port here?\n # For this internal method, it's ok to assume that the server is running\n # on that port?\n\n # Ensure we have a server with passages_file for compatibility\n passages_source_file = self.index_dir / f\"{self.index_path.name}.meta.json\"\n # Convert to absolute path to ensure server can find it\n zmq_port = self._ensure_server_running(\n str(passages_source_file.resolve()),\n zmq_port,\n enable_warmup=self.enable_warmup,\n use_daemon=self.use_daemon,\n daemon_ttl_seconds=self.daemon_ttl_seconds,\n )\n\n return self._compute_embedding_via_server([query], zmq_port)[\n 0:1\n ] # Return (1, D) shape\n except Exception as e:\n print(f\"⚠️ Embedding server failed: {e}\")\n print(\"⏭️ Falling back to direct model loading...\")\n\n # Fallback to direct computation\n from .embedding_compute import compute_embeddings\n\n embedding_mode = self.meta.get(\"embedding_mode\", \"sentence-transformers\")\n return compute_embeddings(\n [query],\n self.embedding_model,\n embedding_mode,\n provider_options=self.embedding_options,\n )\n\n def _compute_embedding_via_server(self, chunks: list, zmq_port: int) -> np.ndarray:\n \"\"\"Compute embeddings using the ZMQ embedding server.\"\"\"\n import msgpack\n import zmq\n\n try:\n context = zmq.Context()\n socket = context.socket(zmq.REQ)\n socket.setsockopt(zmq.RCVTIMEO, 30000) # 30 second timeout\n socket.connect(f\"tcp://localhost:{zmq_port}\")\n\n # Send embedding request\n request = chunks\n request_bytes = msgpack.packb(request)\n socket.send(request_bytes)\n\n # Wait for response\n response_bytes = socket.recv()\n response = msgpack.unpackb(response_bytes)\n\n socket.close()\n context.term()\n\n # Convert response to numpy array\n if isinstance(response, list) and len(response) > 0:\n return np.array(response, dtype=np.float32)\n else:\n raise RuntimeError(\"Invalid response from embedding server\")\n\n except Exception as e:\n raise RuntimeError(f\"Failed to compute embeddings via server: {e}\")\n\n @abstractmethod\n def search(\n self,\n query: np.ndarray,\n top_k: int,\n complexity: int = 64,\n beam_width: int = 1,\n prune_ratio: float = 0.0,\n recompute_embeddings: bool = False,\n pruning_strategy: Literal[\"global\", \"local\", \"proportional\"] = \"global\",\n zmq_port: Optional[int] = None,\n **kwargs,\n ) -> dict[str, Any]:\n \"\"\"\n Search for the top_k nearest neighbors of the query vector.\n\n Args:\n query: Query vectors (B, D) where B is batch size, D is dimension\n top_k: Number of nearest neighbors to return\n complexity: Search complexity/candidate list size, higher = more accurate but slower\n beam_width: Number of parallel search paths/IO requests per iteration\n prune_ratio: Ratio of neighbors to prune via approximate distance (0.0-1.0)\n recompute_embeddings: Whether to fetch fresh embeddings from server vs use stored PQ codes\n pruning_strategy: PQ candidate selection strategy - \"global\" (default), \"local\", or \"proportional\"\n zmq_port: ZMQ port for embedding server communication. Must be provided if recompute_embeddings is True.\n **kwargs: Backend-specific parameters (e.g., batch_size, dedup_node_dis, etc.)\n\n Returns:\n Dict with 'labels' (list of lists) and 'distances' (ndarray)\n \"\"\"\n pass\n\n def __del__(self):\n \"\"\"Ensures the embedding server is stopped when the searcher is destroyed.\"\"\"\n if hasattr(self, \"embedding_server_manager\"):\n self.embedding_server_manager.stop_server()\n" }, { "path": "packages/leann-core/src/leann/server.py", "content": "\"\"\"\nMinimal HTTP API server for LEANN.\n\nThis exposes LEANN indexes over HTTP so clients can:\n- List available indexes\n- Run semantic search against an index\n\nThe design intentionally keeps dependencies optional:\n- FastAPI + pydantic are imported lazily inside `create_app()`\n- uvicorn is imported lazily inside `main()`\n\nThis way, core LEANN usage is unaffected unless you actually run the server.\n\"\"\"\n\nfrom __future__ import annotations\n\nimport os\nfrom pathlib import Path\nfrom typing import Any\n\nfrom .api import LeannSearcher\nfrom .cli import LeannCLI\n\n\ndef _ensure_fastapi():\n \"\"\"Lazy import FastAPI and Pydantic, with a clear error if missing.\"\"\"\n try:\n from fastapi import FastAPI, HTTPException\n from pydantic import BaseModel\n except ImportError as e: # pragma: no cover - dependency error path\n raise RuntimeError(\n \"FastAPI and pydantic are required for the LEANN HTTP server.\\n\"\n \"Install them with:\\n\\n\"\n \" uv pip install 'fastapi>=0.115' 'pydantic>=2' 'uvicorn[standard]'\\n\"\n ) from e\n\n return FastAPI, HTTPException, BaseModel\n\n\ndef _resolve_index_path(index_name: str) -> str:\n \"\"\"\n Resolve an index path for the HTTP server.\n\n For now we use the same convention as the CLI:\n - Look in the current project's `.leann/indexes//documents.leann`.\n\n This keeps behavior predictable when running `leann serve` from a project root.\n \"\"\"\n cli = LeannCLI()\n index_path = cli.get_index_path(index_name)\n if not cli.index_exists(index_name):\n raise FileNotFoundError(\n f\"Index '{index_name}' not found in current project. \"\n f\"Build it with: leann build {index_name} --docs ./your_docs\"\n )\n return index_path\n\n\ndef _list_current_project_indexes() -> list[dict[str, Any]]:\n \"\"\"\n Return machine-readable index metadata for the current project.\n\n This mirrors `LeannCLI.list_indexes()` but only for the current project\n and without printing to stdout.\n \"\"\"\n cli = LeannCLI()\n current_path = Path.cwd()\n indexes: list[dict[str, Any]] = []\n\n for idx in cli._discover_indexes_in_project(current_path):\n # `idx` includes keys like: name, type (cli/app), status, size_mb\n indexes.append(\n {\n \"name\": idx.get(\"name\", \"\"),\n \"type\": idx.get(\"type\", \"cli\"),\n \"status\": idx.get(\"status\", \"\"),\n \"size_mb\": idx.get(\"size_mb\", 0.0),\n \"project_path\": str(current_path),\n }\n )\n\n return indexes\n\n\ndef create_app():\n \"\"\"\n Create and return a FastAPI application exposing LEANN as a simple vector DB.\n\n Endpoints:\n - GET /health -> basic health check\n - GET /indexes -> list indexes in current project\n - POST /indexes/{name}/search -> semantic search\n \"\"\"\n\n FastAPI, HTTPException, BaseModel = _ensure_fastapi()\n\n class SearchRequest(BaseModel):\n query: str\n top_k: int = 5\n complexity: int = 64\n beam_width: int = 1\n prune_ratio: float = 0.0\n recompute_embeddings: bool = True\n pruning_strategy: str = \"global\"\n use_grep: bool = False\n\n class SearchResultModel(BaseModel):\n id: str\n score: float\n text: str\n metadata: dict[str, Any]\n\n app = FastAPI(\n title=\"LEANN Vector DB Server\",\n description=(\n \"HTTP API for querying LEANN indexes.\\n\\n\"\n \"This is a minimal first version focused on search. \"\n \"Run it from a project root where `.leann/indexes` exists.\"\n ),\n version=\"0.1.0\",\n )\n\n @app.get(\"/health\")\n async def health() -> dict[str, str]:\n return {\"status\": \"ok\"}\n\n @app.get(\"/indexes\")\n async def list_indexes() -> list[dict[str, Any]]:\n \"\"\"\n List indexes for the current project (the working directory where the server runs).\n \"\"\"\n return _list_current_project_indexes()\n\n @app.post(\"/indexes/{index_name}/search\", response_model=list[SearchResultModel])\n async def search_index(index_name: str, body: SearchRequest):\n \"\"\"\n Run semantic search against an existing LEANN index.\n \"\"\"\n try:\n index_path = _resolve_index_path(index_name)\n except FileNotFoundError as e:\n raise HTTPException(status_code=404, detail=str(e))\n\n searcher = LeannSearcher(index_path=index_path)\n results = searcher.search(\n query=body.query,\n top_k=body.top_k,\n complexity=body.complexity,\n beam_width=body.beam_width,\n prune_ratio=body.prune_ratio,\n recompute_embeddings=body.recompute_embeddings,\n pruning_strategy=body.pruning_strategy, # type: ignore[arg-type]\n use_grep=body.use_grep,\n )\n\n # Normalize into JSON-serializable structures\n return [\n SearchResultModel(\n id=r.id,\n score=float(r.score),\n text=r.text,\n metadata=dict(r.metadata or {}),\n )\n for r in results\n ]\n\n return app\n\n\ndef main() -> None:\n \"\"\"\n Entrypoint to run the HTTP server with uvicorn.\n\n Example:\n uv run python -m leann.server\n # or, after wiring a CLI command:\n leann serve\n \"\"\"\n try:\n import uvicorn\n except ImportError as e: # pragma: no cover - dependency error path\n raise RuntimeError(\n \"uvicorn is required to run the LEANN HTTP server.\\n\"\n \"Install it with:\\n\\n\"\n \" uv pip install 'uvicorn[standard]'\\n\"\n ) from e\n\n app = create_app()\n host = os.getenv(\"LEANN_SERVER_HOST\", \"0.0.0.0\")\n port = int(os.getenv(\"LEANN_SERVER_PORT\", \"8000\"))\n\n uvicorn.run(app, host=host, port=port)\n\n\nif __name__ == \"__main__\": # pragma: no cover\n main()\n" }, { "path": "packages/leann-core/src/leann/settings.py", "content": "\"\"\"Runtime configuration helpers for LEANN.\"\"\"\n\nfrom __future__ import annotations\n\nimport json\nimport os\nfrom typing import Any\n\n# Default fallbacks to preserve current behaviour while keeping them in one place.\n_DEFAULT_OLLAMA_HOST = \"http://localhost:11434\"\n_DEFAULT_OPENAI_BASE_URL = \"https://api.openai.com/v1\"\n_DEFAULT_ANTHROPIC_BASE_URL = \"https://api.anthropic.com\"\n_DEFAULT_MINIMAX_BASE_URL = \"https://api.minimax.io/v1\"\n\n\ndef _clean_url(value: str) -> str:\n \"\"\"Normalize URL strings by stripping trailing slashes.\"\"\"\n\n return value.rstrip(\"/\") if value else value\n\n\ndef resolve_ollama_host(explicit: str | None = None) -> str:\n \"\"\"Resolve the Ollama-compatible endpoint to use.\"\"\"\n\n candidates = (\n explicit,\n os.getenv(\"LEANN_LOCAL_LLM_HOST\"),\n os.getenv(\"LEANN_OLLAMA_HOST\"),\n os.getenv(\"OLLAMA_HOST\"),\n os.getenv(\"LOCAL_LLM_ENDPOINT\"),\n )\n\n for candidate in candidates:\n if candidate:\n return _clean_url(candidate)\n\n return _clean_url(_DEFAULT_OLLAMA_HOST)\n\n\ndef resolve_openai_base_url(explicit: str | None = None) -> str:\n \"\"\"Resolve the base URL for OpenAI-compatible services.\"\"\"\n\n candidates = (\n explicit,\n os.getenv(\"LEANN_OPENAI_BASE_URL\"),\n os.getenv(\"OPENAI_BASE_URL\"),\n os.getenv(\"LOCAL_OPENAI_BASE_URL\"),\n )\n\n for candidate in candidates:\n if candidate:\n return _clean_url(candidate)\n\n return _clean_url(_DEFAULT_OPENAI_BASE_URL)\n\n\ndef resolve_anthropic_base_url(explicit: str | None = None) -> str:\n \"\"\"Resolve the base URL for Anthropic-compatible services.\"\"\"\n\n candidates = (\n explicit,\n os.getenv(\"LEANN_ANTHROPIC_BASE_URL\"),\n os.getenv(\"ANTHROPIC_BASE_URL\"),\n os.getenv(\"LOCAL_ANTHROPIC_BASE_URL\"),\n )\n\n for candidate in candidates:\n if candidate:\n return _clean_url(candidate)\n\n return _clean_url(_DEFAULT_ANTHROPIC_BASE_URL)\n\n\ndef resolve_openai_api_key(explicit: str | None = None) -> str | None:\n \"\"\"Resolve the API key for OpenAI-compatible services.\"\"\"\n\n if explicit:\n return explicit\n\n return os.getenv(\"OPENAI_API_KEY\")\n\n\ndef resolve_anthropic_api_key(explicit: str | None = None) -> str | None:\n \"\"\"Resolve the API key for Anthropic services.\"\"\"\n\n if explicit:\n return explicit\n\n return os.getenv(\"ANTHROPIC_API_KEY\")\n\n\ndef resolve_minimax_base_url(explicit: str | None = None) -> str:\n \"\"\"Resolve the base URL for MiniMax-compatible services.\"\"\"\n\n candidates = (\n explicit,\n os.getenv(\"LEANN_MINIMAX_BASE_URL\"),\n os.getenv(\"MINIMAX_BASE_URL\"),\n )\n\n for candidate in candidates:\n if candidate:\n return _clean_url(candidate)\n\n return _clean_url(_DEFAULT_MINIMAX_BASE_URL)\n\n\ndef resolve_minimax_api_key(explicit: str | None = None) -> str | None:\n \"\"\"Resolve the API key for MiniMax services.\"\"\"\n\n if explicit:\n return explicit\n\n return os.getenv(\"MINIMAX_API_KEY\")\n\n\ndef encode_provider_options(options: dict[str, Any] | None) -> str | None:\n \"\"\"Serialize provider options for child processes.\"\"\"\n\n if not options:\n return None\n\n try:\n return json.dumps(options)\n except (TypeError, ValueError):\n # Fall back to empty payload if serialization fails\n return None\n" }, { "path": "packages/leann-core/src/leann/sync.py", "content": "import logging\nimport os\nimport pickle\nfrom dataclasses import dataclass, field\nfrom hashlib import sha256\nfrom typing import Optional\n\nfrom llama_index.core import SimpleDirectoryReader\n\nlogger = logging.getLogger(__name__)\n\n\ndef hash_data(data: str | bytes):\n if isinstance(data, str):\n data = data.encode()\n return sha256(data).hexdigest()\n\n\n@dataclass\nclass MerkleTreeNode:\n ## TODO: this merkle tree only has two layer, need to improve if we want to scale to large codebase\n hash: str\n data: str\n children: dict[str, \"MerkleTreeNode\"] = field(default_factory=dict)\n parent_id: str | None = None\n\n\nclass MerkleTree:\n def __init__(self):\n self.nodes: dict[str, MerkleTreeNode] = {}\n self.root: MerkleTreeNode | None = None\n\n def add_node(self, data: str, parent_id=None, hash: Optional[str] = None):\n hash = hash_data(data) if hash is None else hash\n\n node = MerkleTreeNode(hash=hash, data=data, parent_id=parent_id)\n self.nodes[hash] = node\n\n if parent_id is None:\n self.root = node\n else:\n self.nodes[parent_id].children[hash] = node\n\n return hash\n\n def compare_with(self, other: \"MerkleTree\"):\n \"\"\"\n Simple comparison of two flat trees. Check the individual file hashes\n only if the root has changed, otherwise return no changes.\n \"\"\"\n assert self.root is not None and other.root is not None\n\n if self.root.hash == other.root.hash:\n return [], [], []\n\n old_files = self.root.children\n new_files = other.root.children\n\n all_nodes = new_files.keys() | old_files.keys()\n\n added, removed, modified = [], [], []\n for path in all_nodes:\n if path in new_files and path in old_files:\n if new_files[path].data != old_files[path].data:\n modified.append(path)\n elif path in new_files and path not in old_files:\n added.append(path)\n else:\n removed.append(path)\n\n return added, removed, modified\n\n\nclass FileSynchronizer:\n def __init__(\n self,\n root_dir: str,\n ignore_patterns: Optional[list] = None,\n include_extensions: Optional[list] = None,\n auto_load=True,\n snapshot_path: Optional[str] = None,\n ):\n if not os.path.isdir(root_dir):\n raise ValueError(\"This is not a valid directory\")\n self.root_dir = root_dir\n self.ignore_patterns = ignore_patterns\n self.include_extensions = include_extensions\n self._custom_snapshot_path = snapshot_path\n self._pending_tree: Optional[MerkleTree] = None\n self.tree: Optional[MerkleTree] = None\n if auto_load:\n self.load_snapshot()\n\n def generate_file_hashes(self):\n file_hashes = {}\n try:\n reader = SimpleDirectoryReader(\n self.root_dir,\n recursive=True,\n exclude=self.ignore_patterns,\n required_exts=self.include_extensions,\n exclude_empty=False,\n )\n except ValueError:\n # Empty directory — no files to hash\n return file_hashes\n # print('reader.iter_data() length', len(list(reader.iter_data())))\n\n for file in reader.iter_data():\n if len(file) > 1:\n # print('file length is greater than 1', file)\n continue # SimpleDirectoryReader can load more than 1 documents for weird file types e.g. PDFs\n file = file[0]\n try:\n file_hash = hash_data(file.text)\n file_hashes[file.metadata[\"file_path\"]] = file_hash\n except Exception:\n logger.error(f\"Cannot hash file {file.metadata['file_path']}\")\n continue\n\n return file_hashes\n\n def build_merkle_tree(self, file_hashes):\n \"\"\"\n Build a flat merkle tree suitable for quick checking of file changes.\n \"\"\"\n tree = MerkleTree()\n\n sorted_paths = sorted(file_hashes)\n root_data = \"\".join(path + file_hashes[path] for path in sorted_paths)\n\n root_id = tree.add_node(root_data)\n\n for path in sorted_paths:\n tree.add_node(file_hashes[path], parent_id=root_id, hash=path)\n\n return tree\n\n def detect_changes(self) -> tuple[list[str], list[str], list[str]]:\n \"\"\"Detect changes without persisting. Call commit() after successful processing.\"\"\"\n file_hashes = self.generate_file_hashes()\n new_tree = self.build_merkle_tree(file_hashes)\n self._pending_tree = new_tree\n\n if self.tree is None:\n return list(file_hashes.keys()), [], []\n\n return self.tree.compare_with(new_tree)\n\n def commit(self):\n \"\"\"Persist the pending snapshot after successful processing.\"\"\"\n if self._pending_tree is not None:\n self.tree = self._pending_tree\n self._pending_tree = None\n self.save_snapshot()\n\n def create_snapshot(self):\n \"\"\"Build and persist a snapshot from the current file state (for initial / forced builds).\"\"\"\n file_hashes = self.generate_file_hashes()\n self.tree = self.build_merkle_tree(file_hashes)\n self.save_snapshot()\n\n def check_for_changes(self) -> tuple[list[str], list[str], list[str]]:\n \"\"\"Detect and auto-commit changes (convenience wrapper).\"\"\"\n changes = self.detect_changes()\n self.commit()\n return changes\n\n @property\n def snapshot_path(self):\n if self._custom_snapshot_path:\n return self._custom_snapshot_path\n return f\"{self.root_dir}.sync_context.pickle\"\n\n def save_snapshot(self):\n assert self.tree is not None\n\n with open(self.snapshot_path, \"wb\") as f:\n pickle.dump(self.tree, f)\n\n def load_snapshot(self):\n try:\n with open(self.snapshot_path, \"rb\") as f:\n self.tree = pickle.load(f)\n except FileNotFoundError:\n self.tree = None\n" }, { "path": "packages/leann-mcp/README.md", "content": "# 🔥 LEANN Claude Code Integration\n\nTransform your development workflow with intelligent code assistance using LEANN's semantic search directly in Claude Code.\n\nFor agent-facing discovery details, see `llms.txt` in the repository root.\n\n**Backend work (IVF, incremental build, reindex):** See [BACKEND.md](./BACKEND.md) for how the MCP server relates to backends and for notes on #231, #89, and #141.\n\n## Prerequisites\n\nInstall LEANN globally for MCP integration (with default backend):\n\n```bash\nuv tool install leann-core --with leann\n```\nThis installs the `leann` CLI into an isolated tool environment and includes both backends so `leann build` works out-of-the-box.\n\n## 🚀 Quick Setup\n\nAdd the LEANN MCP server to Claude Code. Choose the scope based on how widely you want it available. Below is the command to install it globally; if you prefer a local install, skip this step:\n\n```bash\n# Global (recommended): available in all projects for your user\nclaude mcp add --scope user leann-server -- leann_mcp\n```\n\n- `leann-server`: the display name of the MCP server in Claude Code (you can change it).\n- `leann_mcp`: the Python entry point installed with LEANN that starts the MCP server.\n\nVerify it is registered globally:\n\n```bash\nclaude mcp list | cat\n```\n\n## 🛠️ Available Tools\n\nOnce connected, you'll have access to these powerful semantic search tools in Claude Code:\n\n- **`leann_list`** - List all available indexes across your projects\n- **`leann_search`** - Perform semantic searches across code and documents\n\n\n## 🎯 Quick Start Example\n\n```bash\n# Add locally if you did not add it globally (current folder only; default if --scope is omitted)\nclaude mcp add leann-server -- leann_mcp\n\n# Build an index for your project (change to your actual path)\n# See the advanced examples below for more ways to configure indexing\n# Set the index name (replace 'my-project' with your own)\nleann build my-project --docs $(git ls-files)\n\n# Start Claude Code\nclaude\n```\n**Performance tip**: For maximum speed when storage space is not a concern, add the `--no-recompute` flag to your build command. This materializes all tensors and stores them on disk, avoiding recomputation on subsequent builds:\n\n```bash\nleann build my-project --docs $(git ls-files) --no-recompute\n```\n\n## 🚀 Advanced Usage Examples to build the index\n\n### Index Entire Git Repository\n```bash\n# Index all tracked files in your Git repository.\n# Note: submodules are currently skipped; we can add them back if needed.\nleann build my-repo --docs $(git ls-files) --embedding-mode sentence-transformers --embedding-model all-MiniLM-L6-v2 --backend hnsw\n\n# Index only tracked Python files from Git.\nleann build my-python-code --docs $(git ls-files \"*.py\") --embedding-mode sentence-transformers --embedding-model all-MiniLM-L6-v2 --backend hnsw\n\n# If you encounter empty requests caused by empty files (e.g., __init__.py), exclude zero-byte files. Thanks @ww2283 for pointing [that](https://github.com/yichuan-w/LEANN/issues/48) out\nleann build leann-prospec-lig --docs $(find ./src -name \"*.py\" -not -empty) --embedding-mode openai --embedding-model text-embedding-3-small\n```\n\n### Multiple Directories and Files\n```bash\n# Index multiple directories\nleann build my-codebase --docs ./src ./tests ./docs ./config --embedding-mode sentence-transformers --embedding-model all-MiniLM-L6-v2 --backend hnsw\n\n# Mix files and directories\nleann build my-project --docs ./README.md ./src/ ./package.json ./docs/ --embedding-mode sentence-transformers --embedding-model all-MiniLM-L6-v2 --backend hnsw\n\n# Specific files only\nleann build my-configs --docs ./tsconfig.json ./package.json ./webpack.config.js --embedding-mode sentence-transformers --embedding-model all-MiniLM-L6-v2 --backend hnsw\n```\n\n### Advanced Git Integration\n```bash\n# Index recently modified files\nleann build recent-changes --docs $(git diff --name-only HEAD~10..HEAD) --embedding-mode sentence-transformers --embedding-model all-MiniLM-L6-v2 --backend hnsw\n\n# Index files matching pattern\nleann build frontend --docs $(git ls-files \"*.tsx\" \"*.ts\" \"*.jsx\" \"*.js\") --embedding-mode sentence-transformers --embedding-model all-MiniLM-L6-v2 --backend hnsw\n\n# Index documentation and config files\nleann build docs-and-configs --docs $(git ls-files \"*.md\" \"*.yml\" \"*.yaml\" \"*.json\" \"*.toml\") --embedding-mode sentence-transformers --embedding-model all-MiniLM-L6-v2 --backend hnsw\n```\n\n\n## **Try this in Claude Code:**\n```\nHelp me understand this codebase. List available indexes and search for authentication patterns.\n```\n\n

\n \"LEANN\n

\n\nIf you see a prompt asking whether to proceed with LEANN, you can now use it in your chat!\n\n## 🧠 How It Works\n\nThe integration consists of three key components working seamlessly together:\n\n- **`leann`** - Core CLI tool for indexing and searching (installed globally via `uv tool install`)\n- **`leann_mcp`** - MCP server that wraps `leann` commands for Claude Code integration\n- **Claude Code** - Calls `leann_mcp`, which executes `leann` commands and returns intelligent results\n\n## 📁 File Support\n\nLEANN understands **30+ file types** including:\n- **Programming**: Python, JavaScript, TypeScript, Java, Go, Rust, C++, C#\n- **Data**: SQL, YAML, JSON, CSV, XML\n- **Documentation**: Markdown, TXT, PDF\n- **And many more!**\n\n## 💾 Storage & Organization\n\n- **Project indexes**: Stored in `.leann/` directory (just like `.git`)\n- **Global registry**: Project tracking at `~/.leann/projects.json`\n- **Multi-project support**: Switch between different codebases seamlessly\n- **Portable**: Transfer indexes between machines with minimal overhead\n\n## 🗑️ Uninstalling\n\nTo remove the LEANN MCP server from Claude Code:\n\n```bash\nclaude mcp remove leann-server\n```\nTo remove LEANN\n```\nuv pip uninstall leann leann-backend-hnsw leann-core\n```\n\nTo globally remove LEANN (for version update)\n```\nuv tool list | cat\nuv tool uninstall leann-core\ncommand -v leann || echo \"leann gone\"\ncommand -v leann_mcp || echo \"leann_mcp gone\"\n```\n" }, { "path": "packages/wechat-exporter/__init__.py", "content": "__all__ = []\n" }, { "path": "packages/wechat-exporter/main.py", "content": "import json\nimport sqlite3\nimport xml.etree.ElementTree as ElementTree\nfrom pathlib import Path\nfrom typing import Annotated\n\nimport requests\nimport typer\nfrom tqdm import tqdm\n\napp = typer.Typer()\n\n\ndef get_safe_path(s: str) -> str:\n \"\"\"\n Remove invalid characters to sanitize a path.\n :param s: str to sanitize\n :returns: sanitized str\n \"\"\"\n ban_chars = \"\\\\ / : * ? \\\" ' < > | $ \\r \\n\".replace(\" \", \"\")\n for i in ban_chars:\n s = s.replace(i, \"\")\n return s\n\n\ndef process_history(history: str):\n if history.startswith(\"\"):\n try:\n root = ElementTree.fromstring(history)\n title = root.find(\".//title\").text if root.find(\".//title\") is not None else None\n quoted = (\n root.find(\".//refermsg/content\").text\n if root.find(\".//refermsg/content\") is not None\n else None\n )\n if title and quoted:\n return {\"title\": title, \"quoted\": process_history(quoted)}\n if title:\n return title\n except Exception:\n return history\n return history\n\n\ndef get_message(history: dict | str):\n if isinstance(history, dict):\n if \"title\" in history:\n return history[\"title\"]\n else:\n return history\n\n\ndef export_chathistory(user_id: str):\n res = requests.get(\n \"http://localhost:48065/wechat/chatlog\",\n params={\"userId\": user_id, \"count\": 100000},\n ).json()\n for i in range(len(res[\"chatLogs\"])):\n res[\"chatLogs\"][i][\"content\"] = process_history(res[\"chatLogs\"][i][\"content\"])\n res[\"chatLogs\"][i][\"message\"] = get_message(res[\"chatLogs\"][i][\"content\"])\n return res[\"chatLogs\"]\n\n\n@app.command()\ndef export_all(dest: Annotated[Path, typer.Argument(help=\"Destination path to export to.\")]):\n \"\"\"\n Export all users' chat history to json files.\n \"\"\"\n if not dest.is_dir():\n if not dest.exists():\n inp = typer.prompt(\"Destination path does not exist, create it? (y/n)\")\n if inp.lower() == \"y\":\n dest.mkdir(parents=True)\n else:\n typer.echo(\"Aborted.\", err=True)\n return\n else:\n typer.echo(\"Destination path is not a directory!\", err=True)\n return\n all_users = requests.get(\"http://localhost:48065/wechat/allcontacts\").json()\n\n exported_count = 0\n for user in tqdm(all_users):\n try:\n usr_chatlog = export_chathistory(user[\"arg\"])\n\n # Only write file if there are messages\n if len(usr_chatlog) > 0:\n out_path = dest / get_safe_path((user[\"title\"] or \"\") + \"-\" + user[\"arg\"] + \".json\")\n with open(out_path, \"w\", encoding=\"utf-8\") as f:\n json.dump(usr_chatlog, f, ensure_ascii=False, indent=2)\n exported_count += 1\n except Exception as e:\n print(f\"Error exporting {user.get('title', 'Unknown')}: {e}\")\n continue\n\n print(f\"Exported {exported_count} users' chat history to {dest} in json.\")\n\n\n@app.command()\ndef export_sqlite(\n dest: Annotated[Path, typer.Argument(help=\"Destination path to export to.\")] = Path(\n \"chatlog.db\"\n ),\n):\n \"\"\"\n Export all users' chat history to a sqlite database.\n \"\"\"\n connection = sqlite3.connect(dest)\n cursor = connection.cursor()\n cursor.execute(\n \"CREATE TABLE IF NOT EXISTS chatlog (id INTEGER PRIMARY KEY AUTOINCREMENT, with_id TEXT, from_user TEXT, to_user TEXT, message TEXT, timest DATETIME, auxiliary TEXT)\"\n )\n cursor.execute(\"CREATE INDEX IF NOT EXISTS chatlog_with_id_index ON chatlog (with_id)\")\n cursor.execute(\"CREATE TABLE iF NOT EXISTS users (id TEXT PRIMARY KEY, name TEXT)\")\n\n all_users = requests.get(\"http://localhost:48065/wechat/allcontacts\").json()\n for user in tqdm(all_users):\n cursor.execute(\n \"INSERT OR IGNORE INTO users (id, name) VALUES (?, ?)\",\n (user[\"arg\"], user[\"title\"]),\n )\n usr_chatlog = export_chathistory(user[\"arg\"])\n for msg in usr_chatlog:\n cursor.execute(\n \"INSERT INTO chatlog (with_id, from_user, to_user, message, timest, auxiliary) VALUES (?, ?, ?, ?, ?, ?)\",\n (\n user[\"arg\"],\n msg[\"fromUser\"],\n msg[\"toUser\"],\n msg[\"message\"],\n msg[\"createTime\"],\n str(msg[\"content\"]),\n ),\n )\n connection.commit()\n\n\ndef main():\n app()\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "pyproject.toml", "content": "[build-system]\nrequires = [\"setuptools>=61.0\", \"cmake>=3.24\"]\nbuild-backend = \"setuptools.build_meta\"\n\n[project]\nname = \"leann-workspace\"\nversion = \"0.1.0\"\nrequires-python = \">=3.10\"\n\ndependencies = [\n \"leann-core\",\n \"leann-backend-hnsw\",\n \"typer>=0.12.3\",\n \"numpy>=1.26.0\",\n \"torch\",\n \"tqdm\",\n \"datasets>=2.15.0\",\n \"evaluate\",\n \"colorama\",\n \"boto3\",\n \"protobuf==4.25.3\",\n \"sglang\",\n \"ollama\",\n \"requests>=2.25.0\",\n \"sentence-transformers>=3.0.0\",\n # Keep Py3.9 compatible below 4.46; allow newer for Py>=3.10 (e.g., ColQwen/ColPali).\n \"transformers<4.46; python_version < '3.10'\",\n \"transformers>=4.53.1,<4.58; python_version >= '3.10'\",\n \"openai>=1.0.0\",\n # PDF parsing dependencies - essential for document processing\n \"PyPDF2>=3.0.0\",\n \"pdfplumber>=0.11.0\",\n \"pymupdf>=1.26.0\",\n \"pypdfium2>=4.30.0\",\n # LlamaIndex core and readers - updated versions\n \"llama-index>=0.12.44\",\n \"llama-index-readers-file>=0.4.0\", # Essential for PDF parsing\n # \"llama-index-readers-docling\", # Requires Python >= 3.10\n # \"llama-index-node-parser-docling\", # Requires Python >= 3.10\n \"llama-index-vector-stores-faiss>=0.4.0\",\n \"llama-index-embeddings-huggingface>=0.5.5\",\n # Other dependencies\n \"ipykernel==6.29.5\",\n \"msgpack>=1.1.1\",\n \"mlx>=0.26.3; sys_platform == 'darwin' and platform_machine == 'arm64'\",\n \"mlx-lm>=0.26.0; sys_platform == 'darwin' and platform_machine == 'arm64'\",\n \"psutil>=5.8.0\",\n \"pybind11>=3.0.0\",\n \"pathspec>=0.12.1\",\n \"nbconvert>=7.16.6\",\n \"gitignore-parser>=0.1.12\",\n # AST-aware code chunking dependencies\n \"astchunk>=0.1.0\",\n \"tree-sitter>=0.20.0\",\n \"tree-sitter-python>=0.20.0\",\n \"tree-sitter-java>=0.20.0\",\n \"tree-sitter-c-sharp>=0.20.0\",\n \"tree-sitter-typescript>=0.20.0\",\n \"torchvision>=0.23.0\",\n \"einops\",\n \"seaborn\",\n]\n\n[project.optional-dependencies]\ndiskann = [\n \"leann-backend-diskann\",\n]\n\n# Add a new optional dependency group for document processing\ndocuments = [\n \"beautifulsoup4>=4.13.0\", # For HTML parsing\n \"python-docx>=0.8.11\", # For Word documents (creating/editing)\n \"docx2txt>=0.9\", # For Word documents (text extraction)\n \"openpyxl>=3.1.0\", # For Excel files\n \"pandas>=2.2.0\", # For data processing\n]\n\n[tool.setuptools]\npy-modules = []\npackages = [\"wechat_exporter\"]\npackage-dir = { \"wechat_exporter\" = \"packages/wechat-exporter\" }\n\n[project.scripts]\nwechat-exporter = \"wechat_exporter.main:main\"\n\n\n[tool.uv.sources]\nleann-core = { path = \"packages/leann-core\", editable = true }\nleann-backend-diskann = { path = \"packages/leann-backend-diskann\", editable = true }\nleann-backend-hnsw = { path = \"packages/leann-backend-hnsw\", editable = true }\nastchunk = { path = \"packages/astchunk-leann\", editable = true }\n\n[dependency-groups]\n# Minimal lint toolchain for CI and local hooks\nlint = [\n \"pre-commit>=3.5.0\",\n \"ruff==0.12.7\", # Fixed version to ensure consistent formatting across all environments\n]\n\n# Test toolchain (no heavy project runtime deps)\ntest = [\n \"pytest>=9.0\",\n \"pytest-cov>=4.0\",\n \"pytest-xdist>=3.0\",\n \"pytest-timeout>=2.0\",\n \"python-dotenv>=1.0.0\",\n]\n\n# dependencies by apps/ should list here\ndev = [\n \"matplotlib\",\n \"huggingface-hub>=0.20.0\",\n]\n\n[tool.ruff]\ntarget-version = \"py39\"\nline-length = 100\nextend-exclude = [\n \"third_party\",\n \"apps/multimodal/vision-based-pdf-multi-vector/multi-vector-leann-paper-example.py\",\n \"apps/multimodal/vision-based-pdf-multi-vector/multi-vector-leann-similarity-map.py\"\n]\n\n\n[tool.ruff.lint]\nselect = [\n \"E\", # pycodestyle errors\n \"W\", # pycodestyle warnings\n \"F\", # pyflakes\n \"I\", # isort\n \"B\", # flake8-bugbear\n \"C4\", # flake8-comprehensions\n \"UP\", # pyupgrade\n \"N\", # pep8-naming\n \"RUF\", # ruff-specific rules\n]\nignore = [\n \"E501\", # line too long (handled by formatter)\n \"B008\", # do not perform function calls in argument defaults\n \"B904\", # raise without from\n \"N812\", # lowercase imported as non-lowercase\n \"N806\", # variable in function should be lowercase\n \"RUF012\", # mutable class attributes should be annotated with typing.ClassVar\n]\n\n[tool.ruff.format]\nquote-style = \"double\"\nindent-style = \"space\"\nskip-magic-trailing-comma = false\nline-ending = \"auto\"\n\n[tool.lychee]\naccept = [\"200\", \"403\", \"429\", \"503\"]\ntimeout = 20\nmax_retries = 2\nexclude = [\"localhost\", \"127.0.0.1\", \"example.com\"]\nexclude_path = [\".git/\", \".venv/\", \"__pycache__/\", \"third_party/\"]\nscheme = [\"https\", \"http\"]\n\n[tool.ty]\n# Type checking with ty (Astral's fast Python type checker)\n# ty is 10-100x faster than mypy. See: https://docs.astral.sh/ty/\n\n[tool.ty.environment]\npython-version = \"3.11\"\nextra-paths = [\"apps\", \"packages/leann-core/src\"]\n\n[tool.ty.rules]\n# Disable some noisy rules that have many false positives\npossibly-missing-attribute = \"ignore\"\nunresolved-import = \"ignore\" # Many optional dependencies\n\n[tool.pytest.ini_options]\ntestpaths = [\"tests\"]\npython_files = [\"test_*.py\"]\npython_classes = [\"Test*\"]\npython_functions = [\"test_*\"]\nmarkers = [\n \"slow: marks tests as slow (deselect with '-m \\\"not slow\\\"')\",\n \"openai: marks tests that require OpenAI API key\",\n \"integration: marks tests that require live services (Ollama, LM Studio, etc.)\",\n]\ntimeout = 300 # Reduced from 600s (10min) to 300s (5min) for CI safety\naddopts = [\n \"-v\",\n \"--tb=short\",\n \"--strict-markers\",\n \"--disable-warnings\",\n]\nenv = [\n \"HF_HUB_DISABLE_SYMLINKS=1\",\n \"TOKENIZERS_PARALLELISM=false\",\n]\n" }, { "path": "scripts/hf_upload.py", "content": "#!/usr/bin/env python3\n\"\"\"\nUpload local evaluation data to Hugging Face Hub, excluding diskann_rpj_wiki.\n\nDefaults:\n- repo_id: LEANN-RAG/leann-rag-evaluation-data (dataset)\n- folder_path: benchmarks/data\n- ignore_patterns: diskann_rpj_wiki/** and .cache/**\n\nRequires authentication via `huggingface-cli login` or HF_TOKEN env var.\n\"\"\"\n\nfrom __future__ import annotations\n\nimport argparse\nimport os\n\ntry:\n from huggingface_hub import HfApi\nexcept Exception as e:\n raise SystemExit(\n \"huggingface_hub is required. Install with: pip install huggingface_hub hf_transfer\"\n ) from e\n\n\ndef _enable_transfer_accel_if_available() -> None:\n \"\"\"Best-effort enabling of accelerated transfers across hub versions.\n\n Tries the public util if present; otherwise, falls back to env flag when\n hf_transfer is installed. Silently no-ops if unavailable.\n \"\"\"\n try:\n # Newer huggingface_hub exposes this under utils\n from huggingface_hub.utils import hf_hub_enable_hf_transfer # type: ignore\n\n hf_hub_enable_hf_transfer()\n return\n except Exception:\n pass\n\n try:\n # If hf_transfer is installed, set env flag recognized by the hub\n import hf_transfer # noqa: F401\n\n os.environ.setdefault(\"HF_HUB_ENABLE_HF_TRANSFER\", \"1\")\n except Exception:\n # Acceleration not available; proceed without it\n pass\n\n\ndef parse_args() -> argparse.Namespace:\n p = argparse.ArgumentParser(description=\"Upload local data to HF, excluding diskann_rpj_wiki\")\n p.add_argument(\n \"--repo-id\",\n default=\"LEANN-RAG/leann-rag-evaluation-data\",\n help=\"Target dataset repo id (namespace/name)\",\n )\n p.add_argument(\n \"--folder-path\",\n default=\"benchmarks/data\",\n help=\"Local folder to upload (default: benchmarks/data)\",\n )\n p.add_argument(\n \"--ignore\",\n default=[\"diskann_rpj_wiki/**\", \".cache/**\"],\n nargs=\"+\",\n help=\"Glob patterns to ignore (space-separated)\",\n )\n p.add_argument(\n \"--allow\",\n default=[\"**\"],\n nargs=\"+\",\n help=\"Glob patterns to allow (space-separated). Defaults to everything.\",\n )\n p.add_argument(\n \"--message\",\n default=\"sync local data (exclude diskann_rpj_wiki)\",\n help=\"Commit message\",\n )\n p.add_argument(\n \"--no-transfer-accel\",\n action=\"store_true\",\n help=\"Disable hf_transfer accelerated uploads\",\n )\n return p.parse_args()\n\n\ndef main() -> None:\n args = parse_args()\n\n if not args.no_transfer_accel:\n _enable_transfer_accel_if_available()\n\n if not os.path.isdir(args.folder_path):\n raise SystemExit(f\"Folder not found: {args.folder_path}\")\n\n print(\"Uploading to Hugging Face Hub:\")\n print(f\" repo_id: {args.repo_id}\")\n print(\" repo_type: dataset\")\n print(f\" folder_path: {args.folder_path}\")\n print(f\" allow_patterns: {args.allow}\")\n print(f\" ignore_patterns:{args.ignore}\")\n\n api = HfApi()\n\n # Perform upload. This skips unchanged files by content hash.\n api.upload_folder(\n repo_id=args.repo_id,\n repo_type=\"dataset\",\n folder_path=args.folder_path,\n path_in_repo=\".\",\n allow_patterns=args.allow,\n ignore_patterns=args.ignore,\n commit_message=args.message,\n )\n\n print(\"Upload completed (unchanged files were skipped by the Hub).\")\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "skills/leann-memory/README.md", "content": "# LEANN Memory Search for OpenClaw\n\n97% storage-compressed semantic memory search with free local embeddings.\n\n## Why\n\nEvery OpenClaw memory solution stores full embedding vectors. On a 256 GB Mac\nMini, heavy users accumulate 500 MB - 6 GB+ of embedding indexes over time.\nLEANN compresses this to ~2% of the original size through graph-based selective\nrecomputation, while using high-quality local embeddings (zero API cost).\n\n| Feature | Default memory | LEANN |\n|---|---|---|\n| Storage (50K chunks) | ~75 MB | **~2 MB** |\n| Embedding cost | Remote API ($) | **$0 (local)** |\n| Scale | ~100K chunks | **60M+ passages** |\n\n## Install\n\n```bash\n# Install LEANN\npip install leann-core\n\n# Install the skill\nclawhub install leann-team/leann-memory\n\n# Or manually: copy this directory to ~/.openclaw/workspace/skills/leann-memory/\n```\n\n## Quick Start\n\n```bash\n# Build index on your memory files\nleann build openclaw-memory \\\n --docs ~/.openclaw/workspace/MEMORY.md ~/.openclaw/workspace/memory/ \\\n --embedding-model all-MiniLM-L6-v2\n\n# Test a search\nleann search openclaw-memory \"what did we decide about the database\" --json\n```\n\nThen ask your OpenClaw agent: \"search my memories for database decisions\"\n\n## Auto-Sync\n\nKeep the index updated as new memories are added:\n\n```bash\n# One-shot check and rebuild\nleann watch openclaw-memory --once\n\n# Continuous monitoring (runs in background)\nleann watch openclaw-memory --interval 30\n```\n\n## How It Works\n\nLEANN stores a pruned neighbor graph instead of full embedding vectors. During\nsearch, embeddings are recomputed on-demand via a local daemon. OpenClaw's async\n\"sleep time compute\" model makes recomputation latency invisible to users.\n\n## Links\n\n- [LEANN Repository](https://github.com/yichuan-w/LEANN)\n- [Integration Plan](../../docs/openclaw-integration-plan.md)\n" }, { "path": "skills/leann-memory/claw.json", "content": "{\n \"name\": \"leann-memory\",\n \"version\": \"1.0.0\",\n \"description\": \"97% storage-compressed semantic memory search with free local embeddings. Drop-in enhancement for OpenClaw memory — same search quality, 1/30th the disk space, zero API cost.\",\n \"author\": \"leann-team\",\n \"license\": \"MIT\",\n \"permissions\": [\"shell\"],\n \"entry\": \"instructions.md\",\n \"tags\": [\n \"memory\",\n \"search\",\n \"vector-database\",\n \"embeddings\",\n \"local\",\n \"compression\",\n \"rag\",\n \"semantic-search\"\n ],\n \"models\": [\"claude-*\", \"gpt-*\", \"gemini-*\", \"qwen-*\", \"llama-*\"],\n \"minOpenClawVersion\": \"0.8.0\"\n}\n" }, { "path": "skills/leann-memory/instructions.md", "content": "# LEANN Memory Search\n\nYou have access to LEANN, a high-performance semantic search engine with 97%\nstorage compression. Use it to search the user's memories, notes, documents,\nand knowledge bases with higher quality than the default memory search.\n\n## When to Use\n\n- User asks to search memories, notes, or knowledge bases\n- User wants to recall past decisions, conversations, or facts\n- User says \"what did we decide about X\", \"find my notes on Y\", \"recall\", \"remember\"\n- User asks about something that might be in their indexed documents\n\n## Prerequisites Check\n\nBefore first use, verify LEANN is installed:\n\n```bash\nwhich leann\n```\n\nIf not installed, run:\n\n```bash\npip install leann-core\n```\n\n## First-Time Setup\n\nIf no LEANN index exists for OpenClaw memory, build one:\n\n```bash\nleann build openclaw-memory \\\n --docs ~/.openclaw/workspace/MEMORY.md ~/.openclaw/workspace/memory/ \\\n --embedding-model all-MiniLM-L6-v2 \\\n --embedding-mode sentence-transformers\n```\n\nThis creates a compressed index (~2 MB for 50K chunks vs ~75 MB uncompressed).\nThe index auto-detects changes on subsequent `leann build` runs.\n\n## Search Workflow\n\n1. Search with the user's query:\n\n```bash\nleann search openclaw-memory \"\" --top-k 5 --json --non-interactive\n```\n\n2. Parse the JSON output — each result has `id`, `score`, `text`, and `metadata`\n3. Present the most relevant results with source attribution\n4. If the user wants more context, increase `--top-k` to 10 or 15\n\n## Keeping the Index Updated\n\nThe index is idempotent — re-running build only processes changed files:\n\n```bash\nleann build openclaw-memory \\\n --docs ~/.openclaw/workspace/MEMORY.md ~/.openclaw/workspace/memory/\n```\n\nFor continuous monitoring, use watch mode:\n\n```bash\nleann watch openclaw-memory\n```\n\n## Output Format\n\nThe `--json` flag returns a JSON array:\n\n```json\n[\n {\n \"id\": \"a1b2c3\",\n \"score\": 0.847,\n \"text\": \"The user decided to use PostgreSQL for the project database...\",\n \"metadata\": {\n \"file_path\": \"/home/user/.openclaw/workspace/memory/2026-02-15.md\",\n \"source\": \"memory/2026-02-15.md\"\n }\n }\n]\n```\n\n## Tips\n\n- Higher `--top-k` values (10-15) give more comprehensive results at minimal cost\n- The search uses local embeddings — zero API calls, zero latency from network\n- Re-run `leann build` periodically or use `leann watch` for auto-sync\n- For extra document directories, add them to the build command:\n `--docs ~/.openclaw/workspace/memory/ ~/Documents/notes/`\n" }, { "path": "sky/leann-build.yaml", "content": "name: leann-build\n\nresources:\n # Choose a GPU for fast embeddings (examples: L4, A10G, A100). CPU also works but is slower.\n accelerators: L4:1\n # Optionally pin a cloud, otherwise SkyPilot will auto-select\n # cloud: aws\n disk_size: 100\n\nenvs:\n # Build parameters (override with: sky launch -c leann-gpu sky/leann-build.yaml -e key=value)\n index_name: my-index\n docs: ./data\n backend: hnsw # hnsw | diskann\n complexity: 64\n graph_degree: 32\n num_threads: 8\n # Embedding selection\n embedding_mode: sentence-transformers # sentence-transformers | openai | mlx | ollama\n embedding_model: facebook/contriever\n # Storage/latency knobs\n recompute: true # true => selective recomputation (recommended)\n compact: true # for HNSW only\n # Optional pass-through\n extra_args: \"\"\n # Rebuild control\n force: true\n\n# Sync local paths to the remote VM. Adjust as needed.\nfile_mounts:\n # Example: mount your local data directory used for building\n ~/leann-data: ${docs}\n\nsetup: |\n set -e\n # Install uv (package manager)\n curl -LsSf https://astral.sh/uv/install.sh | sh\n export PATH=\"$HOME/.local/bin:$PATH\"\n\n # Ensure modern libstdc++ for FAISS (GLIBCXX >= 3.4.30)\n sudo apt-get update -y\n sudo apt-get install -y libstdc++6 libgomp1\n # Also upgrade conda's libstdc++ in base env (Skypilot images include conda)\n if command -v conda >/dev/null 2>&1; then\n conda install -y -n base -c conda-forge libstdcxx-ng\n fi\n\n # Install LEANN CLI and backends into the user environment\n uv pip install --upgrade pip\n uv pip install leann-core leann-backend-hnsw leann-backend-diskann\n\nrun: |\n export PATH=\"$HOME/.local/bin:$PATH\"\n # Derive flags from env\n recompute_flag=\"\"\n if [ \"${recompute}\" = \"false\" ] || [ \"${recompute}\" = \"0\" ]; then\n recompute_flag=\"--no-recompute\"\n fi\n force_flag=\"\"\n if [ \"${force}\" = \"true\" ] || [ \"${force}\" = \"1\" ]; then\n force_flag=\"--force\"\n fi\n\n # Build command\n python -m leann.cli build ${index_name} \\\n --docs ~/leann-data \\\n --backend ${backend} \\\n --complexity ${complexity} \\\n --graph-degree ${graph_degree} \\\n --num-threads ${num_threads} \\\n --embedding-mode ${embedding_mode} \\\n --embedding-model ${embedding_model} \\\n ${recompute_flag} ${force_flag} ${extra_args}\n\n # Print where the index is stored for downstream rsync\n echo \"INDEX_OUT_DIR=~/.leann/indexes/${index_name}\"\n" }, { "path": "tests/README.md", "content": "# LEANN Tests\n\nThis directory contains automated tests for the LEANN project using pytest.\n\n## Test Files\n\n### `test_readme_examples.py`\nTests the examples shown in README.md:\n- The basic example code that users see first (parametrized for both HNSW and DiskANN backends)\n- Import statements work correctly\n- Different backend options (HNSW, DiskANN)\n- Different LLM configuration options (parametrized for both backends)\n- **All main README examples are tested with both HNSW and DiskANN backends using pytest parametrization**\n\n### `test_basic.py`\nBasic functionality tests that verify:\n- All packages can be imported correctly\n- C++ extensions (FAISS, DiskANN) load properly\n- Basic index building and searching works for both HNSW and DiskANN backends\n- Uses parametrized tests to test both backends\n\n### `test_document_rag.py`\nTests the document RAG example functionality:\n- Tests with facebook/contriever embeddings\n- Tests with OpenAI embeddings (if API key is available)\n- Tests error handling with invalid parameters\n- Verifies that normalized embeddings are detected and cosine distance is used\n\n### `test_diskann_partition.py`\nTests DiskANN graph partitioning functionality:\n- Tests DiskANN index building without partitioning (baseline)\n- Tests automatic graph partitioning with `is_recompute=True`\n- Verifies that partition files are created and large files are cleaned up for storage saving\n- Tests search functionality with partitioned indices\n- Validates medoid and max_base_norm file generation and usage\n- Includes performance comparison between DiskANN (with partition) and HNSW\n- **Note**: These tests are skipped in CI due to hardware requirements and computation time\n\n### `test_prompt_template_e2e.py`\nIntegration tests for prompt template feature with live embedding services:\n- Tests prompt template prepending with EmbeddingGemma (OpenAI-compatible API via LM Studio)\n- Tests hybrid token limit discovery (Ollama dynamic detection, registry fallback, default)\n- Tests LM Studio SDK bridge for automatic context length detection (requires Node.js + @lmstudio/sdk)\n- **Note**: These tests require live services (LM Studio, Ollama) and are marked with `@pytest.mark.integration`\n- **Important**: Prompt templates are ONLY for EmbeddingGemma and similar task-specific models, NOT regular embedding models\n\n## Running Tests\n\n### Install test dependencies:\n```bash\n# Using uv dependency groups (tools only)\nuv sync --only-group test\n```\n\n### Run all tests:\n```bash\npytest tests/\n\n# Or with coverage\npytest tests/ --cov=leann --cov-report=html\n\n# Run in parallel (faster)\npytest tests/ -n auto\n```\n\n### Run specific tests:\n```bash\n# Only basic tests\npytest tests/test_basic.py\n\n# Only tests that don't require OpenAI\npytest tests/ -m \"not openai\"\n\n# Skip slow tests\npytest tests/ -m \"not slow\"\n\n# Skip integration tests (that require live services)\npytest tests/ -m \"not integration\"\n\n# Run only integration tests (requires LM Studio or Ollama running)\npytest tests/test_prompt_template_e2e.py -v -s\n\n# Run DiskANN partition tests (requires local machine, not CI)\npytest tests/test_diskann_partition.py\n```\n\n### Run with specific backend:\n```bash\n# Test only HNSW backend\npytest tests/test_basic.py::test_backend_basic[hnsw]\npytest tests/test_readme_examples.py::test_readme_basic_example[hnsw]\n\n# Test only DiskANN backend\npytest tests/test_basic.py::test_backend_basic[diskann]\npytest tests/test_readme_examples.py::test_readme_basic_example[diskann]\n\n# All DiskANN tests (parametrized + specialized partition tests)\npytest tests/ -k diskann\n```\n\n## CI/CD Integration\n\nTests are automatically run in GitHub Actions:\n1. After building wheel packages\n2. On multiple Python versions (3.9 - 3.13)\n3. On both Ubuntu and macOS\n4. Using pytest with appropriate markers and flags\n\n### pytest.ini Configuration\n\nThe `pytest.ini` file configures:\n- Test discovery paths\n- Default timeout (600 seconds)\n- Environment variables (HF_HUB_DISABLE_SYMLINKS, TOKENIZERS_PARALLELISM)\n- Custom markers for slow and OpenAI tests\n- Verbose output with short tracebacks\n\n### Integration Test Prerequisites\n\nIntegration tests (`test_prompt_template_e2e.py`) require live services:\n\n**Required:**\n- LM Studio running at `http://localhost:1234` with EmbeddingGemma model loaded\n\n**Optional:**\n- Ollama running at `http://localhost:11434` for token limit detection tests\n- Node.js + @lmstudio/sdk installed (`npm install -g @lmstudio/sdk`) for SDK bridge tests\n\nTests gracefully skip if services are unavailable.\n\n### Known Issues\n\n- OpenAI tests are automatically skipped if no API key is provided\n- Integration tests require live embedding services and may fail due to proxy settings (set `unset ALL_PROXY all_proxy` if needed)\n" }, { "path": "tests/openclaw/.gitignore", "content": "docker-data/\n" }, { "path": "tests/openclaw/__init__.py", "content": "" }, { "path": "tests/openclaw/conftest.py", "content": "\"\"\"Shared fixtures for OpenClaw integration tests.\"\"\"\n\nimport json\nimport shutil\nfrom pathlib import Path\n\nimport pytest\n\nFIXTURES_DIR = Path(__file__).parent / \"fixtures\"\n\n\n@pytest.fixture\ndef memory_fixtures(tmp_path):\n \"\"\"Copy memory fixture files into a temp directory and return the path.\"\"\"\n dest = tmp_path / \"memory_docs\"\n shutil.copytree(FIXTURES_DIR, dest)\n return dest\n\n\n@pytest.fixture\ndef leann_index_dir(tmp_path):\n \"\"\"Provide a clean temporary directory for LEANN indexes.\"\"\"\n idx_dir = tmp_path / \".leann\" / \"indexes\"\n idx_dir.mkdir(parents=True)\n return idx_dir\n\n\n@pytest.fixture\ndef skill_dir():\n \"\"\"Return the path to the leann-memory skill directory.\"\"\"\n return Path(__file__).parent.parent.parent / \"skills\" / \"leann-memory\"\n\n\n@pytest.fixture\ndef claw_manifest(skill_dir):\n \"\"\"Load and return the parsed claw.json manifest.\"\"\"\n manifest_path = skill_dir / \"claw.json\"\n assert manifest_path.exists(), f\"claw.json not found at {manifest_path}\"\n return json.loads(manifest_path.read_text(encoding=\"utf-8\"))\n" }, { "path": "tests/openclaw/docker-compose.yml", "content": "services:\n openclaw-test:\n image: ghcr.io/phioranex/openclaw-docker:latest\n container_name: openclaw-leann-test\n environment:\n - HOME=/home/node\n - OLLAMA_API_KEY=ollama-local\n volumes:\n - ./docker-data:/home/node/.openclaw\n - ../../:/leann-tree:ro\n ports:\n - \"18790:18789\"\n extra_hosts:\n - \"host.docker.internal:host-gateway\"\n entrypoint: [\"/app/node_modules/.pnpm/node_modules/.bin/openclaw\"]\n command: [\"gateway\", \"run\", \"--port\", \"18789\", \"--bind\", \"lan\", \"--allow-unconfigured\", \"--verbose\"]\n" }, { "path": "tests/openclaw/fixtures/MEMORY.md", "content": "# Core Memory\n\n- Name: Alice Chen\n- Role: Senior backend engineer at Acme Corp\n- Tech stack: Python, PostgreSQL, Docker, Kubernetes\n- Preferred editor: Neovim\n- Communication: prefers Telegram over email\n- Time zone: UTC+8\n" }, { "path": "tests/openclaw/fixtures/memory/2026-02-15.md", "content": "# 2026-02-15 Saturday\n\n- Discussed migrating the user-service from REST to gRPC with Bob.\n- Alice prefers Protocol Buffers v3 syntax for defining gRPC services.\n- Started a proof-of-concept branch `feature/grpc-migration`.\n- Ran benchmarks: gRPC was 3.2x faster than REST for the /users endpoint.\n- Team decided to proceed with migration; targeting Q2 completion.\n" }, { "path": "tests/openclaw/fixtures/memory/2026-02-20.md", "content": "# 2026-02-20 Thursday\n\n- Deployed the v2.1.0 hotfix for the payment gateway timeout issue.\n- Root cause: connection pool exhaustion under high concurrent load.\n- Fix: increased pool size from 10 to 50 and added circuit breaker pattern.\n- Alice reviewed the Kubernetes HPA settings — autoscaling now triggers at 60% CPU.\n- Need to follow up with the SRE team about Prometheus alerting thresholds.\n" }, { "path": "tests/openclaw/fixtures/memory/2026-02-25.md", "content": "# 2026-02-25 Wednesday\n\n- Met with the ML team about integrating the recommendation engine into the product catalog.\n- They use sentence-transformers for embeddings — same library LEANN uses.\n- Discussed vector database options: Pinecone (expensive), Milvus (complex), LEANN (lightweight).\n- Alice suggested using LEANN for the PoC because of its 97% storage compression.\n- Action items: set up a LEANN index on the product descriptions by Friday.\n" }, { "path": "tests/openclaw/run_docker_test.sh", "content": "#!/usr/bin/env bash\n#\n# End-to-end test: spin up an isolated OpenClaw Docker container,\n# install the leann-memory skill, build an index, and search.\n#\n# Usage:\n# tests/openclaw/run_docker_test.sh # run all steps\n# tests/openclaw/run_docker_test.sh --down # tear down only\n#\nset -euo pipefail\n\nSCRIPT_DIR=\"$(cd \"$(dirname \"${BASH_SOURCE[0]}\")\" && pwd)\"\ncd \"$SCRIPT_DIR\"\n\nCOMPOSE=\"docker compose -f docker-compose.yml\"\nCONTAINER=\"openclaw-leann-test\"\nHOST_PORT=18790\n\n# ---------- helpers ----------\n\nlog() { printf '\\033[1;34m==>\\033[0m %s\\n' \"$*\"; }\npass() { printf '\\033[1;32m ✓ \\033[0m %s\\n' \"$*\"; }\nfail() { printf '\\033[1;31m ✗ \\033[0m %s\\n' \"$*\"; exit 1; }\n\ncleanup() {\n log \"Tearing down container …\"\n $COMPOSE down --volumes --remove-orphans 2>/dev/null || true\n rm -rf docker-data\n}\n\nif [[ \"${1:-}\" == \"--down\" ]]; then\n cleanup\n exit 0\nfi\n\ntrap cleanup EXIT\n\n# ---------- 1. Start container ----------\n\nlog \"Starting isolated OpenClaw container on port $HOST_PORT …\"\nrm -rf docker-data\n$COMPOSE up -d --wait 2>&1 || $COMPOSE up -d 2>&1\n\nsleep 3\n\nif docker ps --format '{{.Names}}' | grep -q \"$CONTAINER\"; then\n pass \"Container $CONTAINER is running\"\nelse\n fail \"Container $CONTAINER failed to start\"\nfi\n\n# ---------- 2. Verify gateway ----------\n\nlog \"Checking gateway health …\"\nfor i in $(seq 1 10); do\n if curl -sf \"http://localhost:$HOST_PORT\" >/dev/null 2>&1 || \\\n curl -sf \"http://localhost:$HOST_PORT/health\" >/dev/null 2>&1; then\n pass \"Gateway responding on port $HOST_PORT\"\n break\n fi\n if [ \"$i\" -eq 10 ]; then\n # Gateway may not have an HTTP health endpoint; check if process is running\n if docker exec \"$CONTAINER\" pgrep -f \"gateway\" >/dev/null 2>&1; then\n pass \"Gateway process is running inside container\"\n else\n echo \"--- Container logs ---\"\n docker logs \"$CONTAINER\" --tail 20\n fail \"Gateway is not running\"\n fi\n fi\n sleep 2\ndone\n\n# ---------- 3. Install skill fixtures ----------\n\nlog \"Copying skill and memory fixtures into container …\"\ndocker exec \"$CONTAINER\" mkdir -p /home/node/.openclaw/workspace/memory\n\ndocker cp fixtures/MEMORY.md \"$CONTAINER\":/home/node/.openclaw/workspace/MEMORY.md\nfor f in fixtures/memory/*.md; do\n docker cp \"$f\" \"$CONTAINER\":/home/node/.openclaw/workspace/memory/\ndone\n\ndocker exec \"$CONTAINER\" ls -la /home/node/.openclaw/workspace/ | head -10\ndocker exec \"$CONTAINER\" ls -la /home/node/.openclaw/workspace/memory/\npass \"Memory fixtures installed\"\n\n# ---------- 4. Run fast pytest tests (no model needed) ----------\n\nlog \"Running fast (non-model) tests locally …\"\ncd \"$SCRIPT_DIR/../..\"\nuv run pytest tests/openclaw/test_skill_manifest.py tests/openclaw/test_mcp_protocol.py -v 2>&1\npass \"Fast tests passed\"\n\n# ---------- 5. Run model tests (optional) ----------\n\nif [[ \"${SKIP_SLOW:-}\" != \"1\" ]]; then\n log \"Running slow build-and-search tests …\"\n uv run pytest tests/openclaw/test_build_and_search.py -v --timeout=300 2>&1\n pass \"Build-and-search tests passed\"\nelse\n log \"Skipping slow tests (SKIP_SLOW=1)\"\nfi\n\n# ---------- done ----------\n\necho \"\"\nlog \"All OpenClaw integration tests passed!\"\necho \" Container: $CONTAINER\"\necho \" Gateway: ws://localhost:$HOST_PORT\"\necho \" Cleanup: $0 --down\"\n" }, { "path": "tests/openclaw/test_build_and_search.py", "content": "\"\"\"\nEnd-to-end test: build a LEANN index on OpenClaw-style memory files,\nthen search with --json and verify results.\n\nRequires the embedding model (~90 MB download on first run).\nMarked 'slow' — skip with: pytest -m \"not slow\"\n\"\"\"\n\nimport asyncio\nimport json\nimport os\n\nimport pytest\n\npytestmark = [\n pytest.mark.slow,\n pytest.mark.skipif(os.environ.get(\"CI\") == \"true\", reason=\"Skip in CI — needs model download\"),\n]\n\n\n@pytest.fixture\ndef cli_instance(leann_index_dir):\n \"\"\"Create a LeannCLI instance pointing at a temporary index directory.\"\"\"\n from leann.cli import LeannCLI\n\n cli = LeannCLI()\n cli.indexes_dir = leann_index_dir\n cli.indexes_dir.mkdir(parents=True, exist_ok=True)\n return cli\n\n\nBUILD_ARGV_TEMPLATE = [\n \"build\",\n \"openclaw-memory\",\n \"--docs\",\n \"{docs_dir}\",\n \"--backend-name\",\n \"hnsw\",\n \"--no-compact\",\n \"--embedding-model\",\n \"all-MiniLM-L6-v2\",\n \"--embedding-mode\",\n \"sentence-transformers\",\n]\n\n\ndef _parse_args(cli, argv: list[str]):\n parser = cli.create_parser()\n return parser.parse_args(argv)\n\n\ndef _build_argv(docs_dir: str) -> list[str]:\n return [a.format(docs_dir=docs_dir) for a in BUILD_ARGV_TEMPLATE]\n\n\ndef test_build_memory_index(cli_instance, memory_fixtures):\n \"\"\"Build a non-compact HNSW index on the memory fixtures.\"\"\"\n args = _parse_args(cli_instance, _build_argv(str(memory_fixtures)))\n asyncio.get_event_loop().run_until_complete(cli_instance.build_index(args))\n\n index_dir = cli_instance.indexes_dir / \"openclaw-memory\"\n assert index_dir.exists(), \"Index directory was not created\"\n\n meta_files = list(index_dir.glob(\"*.meta.json\"))\n assert len(meta_files) >= 1, \"Missing meta.json\"\n\n passages_files = list(index_dir.glob(\"*.passages.jsonl\"))\n assert len(passages_files) >= 1, \"Missing passages.jsonl\"\n\n\ndef _build_and_search(cli_instance, memory_fixtures, capsys, query, top_k=3):\n \"\"\"Helper: build index, clear capsys, search, return parsed JSON results.\"\"\"\n build_args = _parse_args(cli_instance, _build_argv(str(memory_fixtures)))\n asyncio.get_event_loop().run_until_complete(cli_instance.build_index(build_args))\n capsys.readouterr() # discard build output\n\n search_args = _parse_args(\n cli_instance,\n [\"search\", \"openclaw-memory\", query, \"--top-k\", str(top_k), \"--json\", \"--non-interactive\"],\n )\n asyncio.get_event_loop().run_until_complete(cli_instance.search_documents(search_args))\n\n captured = capsys.readouterr()\n assert captured.out.strip(), f\"Search produced no stdout (stderr: {captured.err[:200]})\"\n return json.loads(captured.out)\n\n\ndef test_search_returns_json(cli_instance, memory_fixtures, capsys):\n \"\"\"Build then search with --json; output must be valid JSON with results.\"\"\"\n results = _build_and_search(cli_instance, memory_fixtures, capsys, \"gRPC migration\")\n assert isinstance(results, list)\n assert len(results) > 0\n assert all({\"id\", \"score\", \"text\", \"metadata\"} <= set(r.keys()) for r in results)\n\n\ndef test_search_relevance(cli_instance, memory_fixtures, capsys):\n \"\"\"Top result for 'payment gateway' should come from the Feb 20 memory.\"\"\"\n results = _build_and_search(\n cli_instance, memory_fixtures, capsys, \"payment gateway timeout fix\"\n )\n top_text = results[0][\"text\"].lower()\n assert \"payment\" in top_text or \"gateway\" in top_text or \"hotfix\" in top_text\n\n\ndef test_idempotent_rebuild(cli_instance, memory_fixtures, capsys):\n \"\"\"Running build twice should detect no changes on the second run.\"\"\"\n args1 = _parse_args(cli_instance, _build_argv(str(memory_fixtures)))\n asyncio.get_event_loop().run_until_complete(cli_instance.build_index(args1))\n capsys.readouterr() # discard first build output\n\n args2 = _parse_args(cli_instance, _build_argv(str(memory_fixtures)))\n asyncio.get_event_loop().run_until_complete(cli_instance.build_index(args2))\n\n captured = capsys.readouterr()\n assert \"up to date\" in captured.out.lower() or \"no changes\" in captured.out.lower()\n\n\ndef test_incremental_add(cli_instance, memory_fixtures, capsys):\n \"\"\"Adding a new file should trigger incremental update, not full rebuild.\"\"\"\n args1 = _parse_args(cli_instance, _build_argv(str(memory_fixtures)))\n asyncio.get_event_loop().run_until_complete(cli_instance.build_index(args1))\n capsys.readouterr() # discard first build output\n\n new_file = memory_fixtures / \"memory\" / \"2026-02-26.md\"\n new_file.write_text(\n \"# 2026-02-26 Thursday\\n\\n- Tested LEANN integration with OpenClaw.\\n\"\n \"- The semantic search returned highly relevant memory results.\\n\",\n encoding=\"utf-8\",\n )\n\n args2 = _parse_args(cli_instance, _build_argv(str(memory_fixtures)))\n asyncio.get_event_loop().run_until_complete(cli_instance.build_index(args2))\n capsys.readouterr() # discard rebuild output\n\n search_args = _parse_args(\n cli_instance,\n [\n \"search\",\n \"openclaw-memory\",\n \"LEANN OpenClaw integration test\",\n \"--top-k\",\n \"3\",\n \"--json\",\n \"--non-interactive\",\n ],\n )\n asyncio.get_event_loop().run_until_complete(cli_instance.search_documents(search_args))\n\n captured = capsys.readouterr()\n assert captured.out.strip(), f\"Search produced no stdout (stderr: {captured.err[:200]})\"\n results = json.loads(captured.out)\n assert any(\"leann\" in r[\"text\"].lower() or \"openclaw\" in r[\"text\"].lower() for r in results)\n" }, { "path": "tests/openclaw/test_mcp_e2e.py", "content": "\"\"\"\nEnd-to-end test for the MCP server: build a real index, then invoke\nhandle_request(tools/call → leann_search) and verify JSON results come back\nthrough the full MCP → subprocess → leann CLI pipeline.\n\nRequires the embedding model and `leann` on PATH (satisfied by uv run pytest).\nMarked 'slow'.\n\"\"\"\n\nimport asyncio\nimport json\nimport os\nimport shutil\nimport subprocess\nimport sys\nfrom pathlib import Path\n\nimport pytest\n\nsys.path.insert(0, str(Path(__file__).parent.parent.parent / \"packages\" / \"leann-core\" / \"src\"))\n\npytestmark = [\n pytest.mark.slow,\n pytest.mark.skipif(os.environ.get(\"CI\") == \"true\", reason=\"Skip in CI — needs model download\"),\n pytest.mark.skipif(not shutil.which(\"leann\"), reason=\"leann CLI not on PATH\"),\n]\n\n\n@pytest.fixture(scope=\"module\")\ndef mcp_index(tmp_path_factory):\n \"\"\"Build a real LEANN index once for the entire module.\"\"\"\n from leann.cli import LeannCLI\n\n tmp = tmp_path_factory.mktemp(\"mcp_e2e\")\n fixtures = Path(__file__).parent / \"fixtures\"\n docs = tmp / \"docs\"\n shutil.copytree(fixtures, docs)\n\n cli = LeannCLI()\n cli.indexes_dir = tmp / \".leann\" / \"indexes\"\n cli.indexes_dir.mkdir(parents=True)\n\n parser = cli.create_parser()\n args = parser.parse_args(\n [\n \"build\",\n \"openclaw-memory\",\n \"--docs\",\n str(docs),\n \"--backend-name\",\n \"hnsw\",\n \"--no-compact\",\n \"--embedding-model\",\n \"all-MiniLM-L6-v2\",\n \"--embedding-mode\",\n \"sentence-transformers\",\n ]\n )\n asyncio.get_event_loop().run_until_complete(cli.build_index(args))\n\n index_dir = cli.indexes_dir / \"openclaw-memory\"\n assert (index_dir / \"documents.leann.meta.json\").exists(), \"Index build failed\"\n return index_dir\n\n\ndef _project_root(mcp_index):\n \"\"\"The temp dir that contains .leann/indexes/ — used as cwd for subprocess.\n\n mcp_index = tmp/.leann/indexes/openclaw-memory\n We need tmp (3 parents up) so LeannCLI finds .leann/indexes/.\n \"\"\"\n return str(mcp_index.parent.parent.parent)\n\n\ndef test_mcp_search_via_subprocess(mcp_index):\n \"\"\"Invoke `leann search --json` as a subprocess (same as MCP server does).\"\"\"\n leann_bin = shutil.which(\"leann\")\n assert leann_bin is not None, \"leann not found on PATH\"\n\n cmd = [\n leann_bin,\n \"search\",\n \"openclaw-memory\",\n \"gRPC migration benchmarks\",\n \"--top-k=3\",\n \"--complexity=32\",\n \"--non-interactive\",\n \"--json\",\n ]\n cwd = _project_root(mcp_index)\n result = subprocess.run(cmd, capture_output=True, text=True, cwd=cwd)\n assert result.returncode == 0, f\"leann search failed (cwd={cwd}): {result.stderr[:500]}\"\n assert result.stdout.strip(), f\"No output from leann search (stderr: {result.stderr[:500]})\"\n\n results = json.loads(result.stdout)\n assert isinstance(results, list)\n assert len(results) > 0\n assert all(\"text\" in r and \"score\" in r for r in results)\n assert all(isinstance(r[\"score\"], float) for r in results), \"score must be native float\"\n\n\ndef test_mcp_search_relevance(mcp_index):\n \"\"\"Search for 'payment gateway' should return relevant content.\"\"\"\n leann_bin = shutil.which(\"leann\")\n assert leann_bin is not None, \"leann not found on PATH\"\n\n cmd = [\n leann_bin,\n \"search\",\n \"openclaw-memory\",\n \"payment gateway timeout hotfix\",\n \"--top-k=3\",\n \"--complexity=32\",\n \"--non-interactive\",\n \"--json\",\n ]\n cwd = _project_root(mcp_index)\n result = subprocess.run(cmd, capture_output=True, text=True, cwd=cwd)\n assert result.returncode == 0, f\"leann search failed (cwd={cwd}): {result.stderr[:500]}\"\n assert result.stdout.strip(), f\"No output from leann search (stderr: {result.stderr[:500]})\"\n\n results = json.loads(result.stdout)\n top_text = results[0][\"text\"].lower()\n assert any(kw in top_text for kw in [\"payment\", \"gateway\", \"hotfix\", \"timeout\"])\n\n\ndef test_mcp_list(mcp_index):\n \"\"\"leann list should show the openclaw-memory index.\"\"\"\n leann_bin = shutil.which(\"leann\")\n assert leann_bin is not None, \"leann not found on PATH\"\n\n result = subprocess.run(\n [leann_bin, \"list\"],\n capture_output=True,\n text=True,\n cwd=_project_root(mcp_index),\n )\n assert result.returncode == 0, f\"leann list failed: {result.stderr[:300]}\"\n assert \"openclaw-memory\" in result.stdout\n\n\ndef test_mcp_stdio_protocol(mcp_index):\n \"\"\"Spawn the MCP server as a subprocess, send JSON-RPC via stdin, read responses.\"\"\"\n leann_mcp_bin = shutil.which(\"leann_mcp\")\n if not leann_mcp_bin:\n pytest.skip(\"leann_mcp not on PATH\")\n assert leann_mcp_bin is not None\n\n requests = [\n {\"jsonrpc\": \"2.0\", \"id\": 1, \"method\": \"initialize\", \"params\": {}},\n {\"jsonrpc\": \"2.0\", \"id\": 2, \"method\": \"tools/list\", \"params\": {}},\n ]\n stdin_data = \"\\n\".join(json.dumps(r) for r in requests) + \"\\n\"\n\n proc = subprocess.run(\n [leann_mcp_bin],\n input=stdin_data,\n capture_output=True,\n text=True,\n timeout=10,\n )\n\n lines = [line for line in proc.stdout.strip().split(\"\\n\") if line.strip()]\n assert len(lines) >= 2, f\"Expected 2 responses, got {len(lines)}: {proc.stdout[:300]}\"\n\n init_resp = json.loads(lines[0])\n assert init_resp[\"id\"] == 1\n assert init_resp[\"result\"][\"serverInfo\"][\"name\"] == \"leann-mcp\"\n\n list_resp = json.loads(lines[1])\n assert list_resp[\"id\"] == 2\n tool_names = {t[\"name\"] for t in list_resp[\"result\"][\"tools\"]}\n assert \"leann_search\" in tool_names\n assert \"leann_list\" in tool_names\n" }, { "path": "tests/openclaw/test_mcp_protocol.py", "content": "\"\"\"Test the LEANN MCP server JSON-RPC protocol handling.\"\"\"\n\nimport json\nimport sys\nfrom pathlib import Path\n\nsys.path.insert(0, str(Path(__file__).parent.parent.parent / \"packages\" / \"leann-core\" / \"src\"))\nfrom leann.mcp import handle_request\n\n\ndef test_initialize():\n \"\"\"MCP initialize should return server info and capabilities.\"\"\"\n req = {\"jsonrpc\": \"2.0\", \"id\": 1, \"method\": \"initialize\", \"params\": {}}\n resp = handle_request(req)\n assert resp[\"id\"] == 1\n result = resp[\"result\"]\n assert result[\"protocolVersion\"] == \"2024-11-05\"\n assert result[\"serverInfo\"][\"name\"] == \"leann-mcp\"\n assert \"tools\" in result[\"capabilities\"]\n\n\ndef test_tools_list():\n \"\"\"MCP tools/list should expose leann_search and leann_list.\"\"\"\n req = {\"jsonrpc\": \"2.0\", \"id\": 2, \"method\": \"tools/list\", \"params\": {}}\n resp = handle_request(req)\n tools = resp[\"result\"][\"tools\"]\n names = {t[\"name\"] for t in tools}\n assert \"leann_search\" in names\n assert \"leann_list\" in names\n\n\ndef test_tools_list_search_schema():\n \"\"\"leann_search tool must declare index_name and query as required params.\"\"\"\n req = {\"jsonrpc\": \"2.0\", \"id\": 3, \"method\": \"tools/list\", \"params\": {}}\n resp = handle_request(req)\n search_tool = next(t for t in resp[\"result\"][\"tools\"] if t[\"name\"] == \"leann_search\")\n schema = search_tool[\"inputSchema\"]\n assert \"index_name\" in schema[\"properties\"]\n assert \"query\" in schema[\"properties\"]\n assert \"index_name\" in schema[\"required\"]\n assert \"query\" in schema[\"required\"]\n\n\ndef test_search_missing_params():\n \"\"\"leann_search should return an error when required params are missing.\"\"\"\n req = {\n \"jsonrpc\": \"2.0\",\n \"id\": 4,\n \"method\": \"tools/call\",\n \"params\": {\"name\": \"leann_search\", \"arguments\": {}},\n }\n resp = handle_request(req)\n text = resp[\"result\"][\"content\"][0][\"text\"]\n assert \"Error\" in text or \"error\" in text\n\n\ndef test_search_missing_query():\n \"\"\"leann_search should error when query is empty.\"\"\"\n req = {\n \"jsonrpc\": \"2.0\",\n \"id\": 5,\n \"method\": \"tools/call\",\n \"params\": {\"name\": \"leann_search\", \"arguments\": {\"index_name\": \"test\", \"query\": \"\"}},\n }\n resp = handle_request(req)\n text = resp[\"result\"][\"content\"][0][\"text\"]\n assert \"Error\" in text or \"error\" in text\n\n\ndef test_jsonrpc_envelope():\n \"\"\"All responses must follow JSON-RPC 2.0 format.\"\"\"\n req = {\"jsonrpc\": \"2.0\", \"id\": 42, \"method\": \"initialize\", \"params\": {}}\n resp = handle_request(req)\n assert resp[\"jsonrpc\"] == \"2.0\"\n assert resp[\"id\"] == 42\n serialized = json.dumps(resp)\n parsed = json.loads(serialized)\n assert parsed == resp\n" }, { "path": "tests/openclaw/test_openclaw_e2e.py", "content": "\"\"\"End-to-end integration test: real OpenClaw instance → memory formation → LEANN indexing → search.\n\nRequirements:\n - Docker running with the openclaw-leann-test container (see docker-compose.yml)\n - Ollama running on host with a model available (e.g. qwen3:8b)\n - LEANN installed in the current virtualenv\n\nRun:\n uv run pytest tests/openclaw/test_openclaw_e2e.py -m integration -v --timeout=600\n\"\"\"\n\nimport json\nimport shutil\nimport subprocess\nimport time\nfrom pathlib import Path\n\nimport pytest\n\nDOCKER_CONTAINER = \"openclaw-leann-test\"\nOPENCLAW_BIN = \"/app/node_modules/.pnpm/node_modules/.bin/openclaw\"\nWORKSPACE_DIR = Path(__file__).parent / \"docker-data\" / \"workspace\"\n\n\ndef _docker_running() -> bool:\n try:\n result = subprocess.run(\n [\"docker\", \"inspect\", \"--format\", \"{{.State.Running}}\", DOCKER_CONTAINER],\n capture_output=True,\n text=True,\n timeout=5,\n )\n return result.stdout.strip() == \"true\"\n except (FileNotFoundError, subprocess.TimeoutExpired):\n return False\n\n\ndef _openclaw_agent(message: str, timeout: int = 300) -> dict:\n \"\"\"Send a message through OpenClaw's full agent CLI and return parsed JSON.\"\"\"\n result = subprocess.run(\n [\n \"docker\",\n \"exec\",\n DOCKER_CONTAINER,\n OPENCLAW_BIN,\n \"agent\",\n \"--agent\",\n \"main\",\n \"-m\",\n message,\n \"--json\",\n \"--timeout\",\n str(timeout),\n ],\n capture_output=True,\n text=True,\n timeout=timeout + 30,\n )\n assert result.returncode == 0, f\"openclaw agent failed: {result.stderr[:500]}\"\n return json.loads(result.stdout)\n\n\ndef _leann_cmd(\n args: list[str], cwd: str | None = None, timeout: int = 60\n) -> subprocess.CompletedProcess[str]:\n leann_bin = shutil.which(\"leann\")\n assert leann_bin is not None, \"leann not found on PATH\"\n resolved_cwd: str = cwd if cwd is not None else str(Path(__file__).parent.parent.parent)\n return subprocess.run(\n [leann_bin, *args],\n capture_output=True,\n text=True,\n cwd=resolved_cwd,\n timeout=timeout,\n )\n\n\npytestmark = [pytest.mark.integration, pytest.mark.slow]\n\n\n@pytest.fixture(scope=\"module\")\ndef openclaw_ready():\n \"\"\"Ensure OpenClaw Docker container is running.\"\"\"\n if not _docker_running():\n pytest.skip(f\"Docker container '{DOCKER_CONTAINER}' not running\")\n\n\n@pytest.fixture(scope=\"module\")\ndef openclaw_memory(openclaw_ready):\n \"\"\"Send messages to OpenClaw and wait for memory file creation.\n\n Returns the path to the workspace memory directory.\n \"\"\"\n memory_dir = WORKSPACE_DIR / \"memory\"\n memory_md = WORKSPACE_DIR / \"MEMORY.md\"\n\n already_has_memory = memory_dir.exists() and any(memory_dir.glob(\"*.md\"))\n if already_has_memory and memory_md.exists():\n return memory_dir\n\n today = time.strftime(\"%Y-%m-%d\")\n\n _openclaw_agent(\n f\"Save to memory/{today}.md: I am TestUser working on LEANN, \"\n \"a vector database with 97% storage compression via graph-pruned \"\n \"recomputation. Today I fixed numpy float32 JSON serialization \"\n \"and C++ printf stdout pollution in FAISS. Stack: Python 3.11, \"\n \"uv, Cursor IDE, macOS.\",\n )\n\n _openclaw_agent(\n \"Update MEMORY.md with my profile: TestUser, developer of LEANN. \"\n \"LEANN backends: HNSW (FAISS), DiskANN, IVF. \"\n \"OpenClaw integration via MCP — leann_search and leann_list tools. \"\n \"Preferred stack: Python 3.11, uv, Cursor, macOS.\",\n )\n\n assert memory_dir.exists(), \"OpenClaw did not create memory/ directory\"\n md_files = list(memory_dir.glob(\"*.md\"))\n assert len(md_files) > 0, \"No .md files in memory/\"\n\n return memory_dir\n\n\n@pytest.fixture(scope=\"module\")\ndef leann_index(openclaw_memory):\n \"\"\"Build a LEANN index over real OpenClaw memory files.\"\"\"\n index_name = \"openclaw-e2e-test\"\n memory_md = WORKSPACE_DIR / \"MEMORY.md\"\n\n docs_args = [\"--docs\", str(openclaw_memory)]\n if memory_md.exists():\n docs_args.extend([str(memory_md)])\n\n result = _leann_cmd(\n [\n \"build\",\n index_name,\n *docs_args,\n \"--embedding-mode\",\n \"ollama\",\n \"--embedding-model\",\n \"nomic-embed-text\",\n ],\n timeout=120,\n )\n assert result.returncode == 0, f\"leann build failed: {result.stderr[:500]}\"\n\n yield index_name\n\n _leann_cmd([\"remove\", index_name], timeout=10)\n\n\nclass TestOpenClawMemoryFormation:\n \"\"\"Verify OpenClaw actually creates memory files.\"\"\"\n\n def test_memory_dir_exists(self, openclaw_memory):\n assert openclaw_memory.exists()\n\n def test_daily_log_created(self, openclaw_memory):\n md_files = list(openclaw_memory.glob(\"*.md\"))\n assert len(md_files) > 0\n content = md_files[0].read_text(encoding=\"utf-8\")\n assert len(content) > 50, \"Daily log is too short to be real\"\n\n def test_daily_log_content(self, openclaw_memory):\n md_files = list(openclaw_memory.glob(\"*.md\"))\n content = md_files[0].read_text(encoding=\"utf-8\").lower()\n assert any(kw in content for kw in [\"leann\", \"testuser\", \"vector\"]), (\n f\"Daily log missing expected keywords: {content[:200]}\"\n )\n\n def test_memory_md_exists(self, openclaw_memory):\n memory_md = WORKSPACE_DIR / \"MEMORY.md\"\n assert memory_md.exists(), \"MEMORY.md not created by OpenClaw\"\n\n def test_memory_md_content(self, openclaw_memory):\n memory_md = WORKSPACE_DIR / \"MEMORY.md\"\n if not memory_md.exists():\n pytest.skip(\"MEMORY.md not created\")\n content = memory_md.read_text(encoding=\"utf-8\").lower()\n assert any(kw in content for kw in [\"leann\", \"hnsw\", \"mcp\"]), (\n f\"MEMORY.md missing expected keywords: {content[:200]}\"\n )\n\n\nclass TestLeannIndexOnRealMemory:\n \"\"\"Verify LEANN can index and search real OpenClaw memory.\"\"\"\n\n def test_index_built(self, leann_index):\n result = _leann_cmd([\"list\"])\n assert result.returncode == 0\n assert leann_index in result.stdout\n\n def test_search_returns_results(self, leann_index):\n result = _leann_cmd(\n [\n \"search\",\n leann_index,\n \"vector database storage compression\",\n \"--top-k=3\",\n \"--non-interactive\",\n \"--json\",\n ]\n )\n assert result.returncode == 0, f\"search failed: {result.stderr[:300]}\"\n results = json.loads(result.stdout)\n assert isinstance(results, list)\n assert len(results) > 0\n\n def test_search_relevance_bug_fix(self, leann_index):\n result = _leann_cmd(\n [\n \"search\",\n leann_index,\n \"numpy float32 JSON serialization bug\",\n \"--top-k=3\",\n \"--non-interactive\",\n \"--json\",\n ]\n )\n assert result.returncode == 0\n results = json.loads(result.stdout)\n assert len(results) > 0\n top_text = results[0][\"text\"].lower()\n assert any(kw in top_text for kw in [\"numpy\", \"float\", \"json\", \"serializ\"]), (\n f\"Top result not relevant: {top_text[:200]}\"\n )\n\n def test_search_relevance_project_info(self, leann_index):\n result = _leann_cmd(\n [\n \"search\",\n leann_index,\n \"what backends does LEANN support\",\n \"--top-k=3\",\n \"--non-interactive\",\n \"--json\",\n ]\n )\n assert result.returncode == 0\n results = json.loads(result.stdout)\n assert len(results) > 0\n all_text = \" \".join(r[\"text\"].lower() for r in results)\n assert any(kw in all_text for kw in [\"hnsw\", \"diskann\", \"backend\"]), (\n f\"Results don't mention backends: {all_text[:300]}\"\n )\n\n def test_search_score_is_native_float(self, leann_index):\n result = _leann_cmd(\n [\n \"search\",\n leann_index,\n \"LEANN project\",\n \"--top-k=3\",\n \"--non-interactive\",\n \"--json\",\n ]\n )\n assert result.returncode == 0\n results = json.loads(result.stdout)\n assert all(isinstance(r[\"score\"], float) for r in results), (\n \"score must be native float, not numpy.float32\"\n )\n\n def test_search_metadata_has_file_path(self, leann_index):\n result = _leann_cmd(\n [\n \"search\",\n leann_index,\n \"TestUser developer\",\n \"--top-k=1\",\n \"--non-interactive\",\n \"--json\",\n ]\n )\n assert result.returncode == 0\n results = json.loads(result.stdout)\n assert len(results) > 0\n meta = results[0].get(\"metadata\", {})\n assert \"file_name\" in meta\n assert meta[\"file_name\"].endswith(\".md\")\n" }, { "path": "tests/openclaw/test_skill_manifest.py", "content": "\"\"\"Validate the ClawHub skill manifest and instructions.\"\"\"\n\n\ndef test_claw_json_required_fields(claw_manifest):\n \"\"\"claw.json must contain all fields required by ClawHub.\"\"\"\n required = {\"name\", \"version\", \"description\", \"author\", \"license\", \"permissions\", \"entry\"}\n missing = required - claw_manifest.keys()\n assert not missing, f\"Missing required fields: {missing}\"\n\n\ndef test_claw_json_name(claw_manifest):\n assert claw_manifest[\"name\"] == \"leann-memory\"\n\n\ndef test_claw_json_permissions(claw_manifest):\n \"\"\"Skill requires shell permission to invoke the leann CLI.\"\"\"\n assert \"shell\" in claw_manifest[\"permissions\"]\n\n\ndef test_claw_json_entry_exists(skill_dir, claw_manifest):\n \"\"\"The entry file referenced in claw.json must exist.\"\"\"\n entry = skill_dir / claw_manifest[\"entry\"]\n assert entry.exists(), f\"Entry file {entry} does not exist\"\n\n\ndef test_claw_json_tags(claw_manifest):\n \"\"\"Should include relevant tags for discoverability.\"\"\"\n tags = set(claw_manifest.get(\"tags\", []))\n assert \"memory\" in tags\n assert \"search\" in tags\n\n\ndef test_claw_json_models(claw_manifest):\n \"\"\"Should declare compatible model families.\"\"\"\n models = claw_manifest.get(\"models\", [])\n assert len(models) >= 1, \"Must declare at least one compatible model\"\n\n\ndef test_instructions_contains_build_command(skill_dir, claw_manifest):\n \"\"\"instructions.md should tell the agent how to build an index.\"\"\"\n instructions = (skill_dir / claw_manifest[\"entry\"]).read_text(encoding=\"utf-8\")\n assert \"leann build\" in instructions\n\n\ndef test_instructions_contains_search_command(skill_dir, claw_manifest):\n \"\"\"instructions.md should tell the agent how to search.\"\"\"\n instructions = (skill_dir / claw_manifest[\"entry\"]).read_text(encoding=\"utf-8\")\n assert \"leann search\" in instructions\n assert \"--json\" in instructions\n\n\ndef test_instructions_contains_install_check(skill_dir, claw_manifest):\n \"\"\"instructions.md should have a prerequisite check for leann installation.\"\"\"\n instructions = (skill_dir / claw_manifest[\"entry\"]).read_text(encoding=\"utf-8\")\n assert \"which leann\" in instructions or \"leann --version\" in instructions\n\n\ndef test_readme_exists(skill_dir):\n \"\"\"README.md should exist for human-facing documentation.\"\"\"\n readme = skill_dir / \"README.md\"\n assert readme.exists(), \"README.md missing from skill directory\"\n" }, { "path": "tests/support/fake_embedding_server_module.py", "content": "import argparse\nimport signal\nimport socket\nimport time\n\n\ndef main() -> None:\n parser = argparse.ArgumentParser(description=\"Fake embedding server for e2e tests\")\n parser.add_argument(\"--zmq-port\", type=int, required=True)\n parser.add_argument(\"--model-name\", type=str, required=True)\n parser.add_argument(\"--passages-file\", type=str, default=\"\")\n parser.add_argument(\"--distance-metric\", type=str, default=\"\")\n parser.add_argument(\"--embedding-mode\", type=str, default=\"sentence-transformers\")\n parser.add_argument(\"--enable-warmup\", action=\"store_true\")\n parser.add_argument(\"--daemon-mode\", action=\"store_true\")\n parser.add_argument(\"--daemon-ttl\", type=int, default=0)\n args = parser.parse_args()\n\n stop = {\"value\": False}\n last_activity = time.time()\n\n def _handle_signal(signum, frame):\n stop[\"value\"] = True\n\n signal.signal(signal.SIGTERM, _handle_signal)\n signal.signal(signal.SIGINT, _handle_signal)\n\n # Bind and keep accepting connections so _check_port sees the process as alive.\n server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)\n server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)\n server.bind((\"127.0.0.1\", args.zmq_port))\n server.listen(16)\n server.settimeout(0.2)\n\n try:\n while not stop[\"value\"]:\n if args.daemon_mode and args.daemon_ttl > 0:\n if (time.time() - last_activity) >= args.daemon_ttl:\n break\n try:\n conn, _addr = server.accept()\n # liveness probes connect+close without sending payload; do not\n # treat them as activity so TTL expiry can still be observed.\n try:\n conn.settimeout(0.01)\n payload = conn.recv(1)\n if payload:\n last_activity = time.time()\n except Exception:\n pass\n conn.close()\n except socket.timeout:\n pass\n finally:\n server.close()\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "tests/test_astchunk_integration.py", "content": "\"\"\"\nTest suite for astchunk integration with LEANN.\nTests AST-aware chunking functionality, language detection, and fallback mechanisms.\n\"\"\"\n\nimport os\nimport subprocess\nimport sys\nimport tempfile\nfrom pathlib import Path\nfrom unittest.mock import Mock, patch\n\nimport pytest\n\n# Add apps directory to path for imports\nsys.path.insert(0, str(Path(__file__).parent.parent / \"apps\"))\n\nfrom typing import Optional\n\nfrom chunking import (\n create_ast_chunks,\n create_text_chunks,\n create_traditional_chunks,\n detect_code_files,\n get_language_from_extension,\n)\n\n\nclass MockDocument:\n \"\"\"Mock LlamaIndex Document for testing.\"\"\"\n\n def __init__(self, content: str, file_path: str = \"\", metadata: Optional[dict] = None):\n self.content = content\n self.metadata = metadata or {}\n if file_path:\n self.metadata[\"file_path\"] = file_path\n\n def get_content(self) -> str:\n return self.content\n\n\nclass TestCodeFileDetection:\n \"\"\"Test code file detection and language mapping.\"\"\"\n\n def test_detect_code_files_python(self):\n \"\"\"Test detection of Python files.\"\"\"\n docs = [\n MockDocument(\"print('hello')\", \"/path/to/file.py\"),\n MockDocument(\"This is text\", \"/path/to/file.txt\"),\n ]\n\n code_docs, text_docs = detect_code_files(docs)\n\n assert len(code_docs) == 1\n assert len(text_docs) == 1\n assert code_docs[0].metadata[\"language\"] == \"python\"\n assert code_docs[0].metadata[\"is_code\"] is True\n assert text_docs[0].metadata[\"is_code\"] is False\n\n def test_detect_code_files_multiple_languages(self):\n \"\"\"Test detection of multiple programming languages.\"\"\"\n docs = [\n MockDocument(\"def func():\", \"/path/to/script.py\"),\n MockDocument(\"public class Test {}\", \"/path/to/Test.java\"),\n MockDocument(\"interface ITest {}\", \"/path/to/test.ts\"),\n MockDocument(\"using System;\", \"/path/to/Program.cs\"),\n MockDocument(\"Regular text content\", \"/path/to/document.txt\"),\n ]\n\n code_docs, text_docs = detect_code_files(docs)\n\n assert len(code_docs) == 4\n assert len(text_docs) == 1\n\n languages = [doc.metadata[\"language\"] for doc in code_docs]\n assert \"python\" in languages\n assert \"java\" in languages\n assert \"typescript\" in languages\n assert \"csharp\" in languages\n\n def test_detect_code_files_no_file_path(self):\n \"\"\"Test handling of documents without file paths.\"\"\"\n docs = [\n MockDocument(\"some content\"),\n MockDocument(\"other content\", metadata={\"some_key\": \"value\"}),\n ]\n\n code_docs, text_docs = detect_code_files(docs)\n\n assert len(code_docs) == 0\n assert len(text_docs) == 2\n for doc in text_docs:\n assert doc.metadata[\"is_code\"] is False\n\n def test_get_language_from_extension(self):\n \"\"\"Test language detection from file extensions.\"\"\"\n assert get_language_from_extension(\"test.py\") == \"python\"\n assert get_language_from_extension(\"Test.java\") == \"java\"\n assert get_language_from_extension(\"component.tsx\") == \"typescript\"\n assert get_language_from_extension(\"Program.cs\") == \"csharp\"\n assert get_language_from_extension(\"document.txt\") is None\n assert get_language_from_extension(\"\") is None\n\n\nclass TestChunkingFunctions:\n \"\"\"Test various chunking functionality.\"\"\"\n\n def test_create_traditional_chunks(self):\n \"\"\"Test traditional text chunking.\"\"\"\n docs = [\n MockDocument(\n \"This is a test document. It has multiple sentences. We want to test chunking.\"\n )\n ]\n\n chunks = create_traditional_chunks(docs, chunk_size=50, chunk_overlap=10)\n\n assert len(chunks) > 0\n # Traditional chunks now return dict format for consistency\n assert all(isinstance(chunk, dict) for chunk in chunks)\n assert all(\"text\" in chunk and \"metadata\" in chunk for chunk in chunks)\n assert all(len(chunk[\"text\"].strip()) > 0 for chunk in chunks)\n\n def test_create_traditional_chunks_empty_docs(self):\n \"\"\"Test traditional chunking with empty documents.\"\"\"\n chunks = create_traditional_chunks([], chunk_size=50, chunk_overlap=10)\n assert chunks == []\n\n @pytest.mark.skipif(\n os.environ.get(\"CI\") == \"true\",\n reason=\"Skip astchunk tests in CI - dependency may not be available\",\n )\n def test_create_ast_chunks_with_astchunk_available(self):\n \"\"\"Test AST chunking when astchunk is available.\"\"\"\n python_code = '''\ndef hello_world():\n \"\"\"Print hello world message.\"\"\"\n print(\"Hello, World!\")\n\ndef add_numbers(a, b):\n \"\"\"Add two numbers and return the result.\"\"\"\n return a + b\n\nclass Calculator:\n \"\"\"A simple calculator class.\"\"\"\n\n def __init__(self):\n self.history = []\n\n def add(self, a, b):\n result = a + b\n self.history.append(f\"{a} + {b} = {result}\")\n return result\n'''\n\n docs = [MockDocument(python_code, \"/test/calculator.py\", {\"language\": \"python\"})]\n\n try:\n chunks = create_ast_chunks(docs, max_chunk_size=200, chunk_overlap=50)\n\n # Should have multiple chunks due to different functions/classes\n assert len(chunks) > 0\n # R3: Expect dict format with \"text\" and \"metadata\" keys\n assert all(isinstance(chunk, dict) for chunk in chunks), \"All chunks should be dicts\"\n assert all(\"text\" in chunk and \"metadata\" in chunk for chunk in chunks), (\n \"Each chunk should have 'text' and 'metadata' keys\"\n )\n assert all(len(chunk[\"text\"].strip()) > 0 for chunk in chunks), (\n \"Each chunk text should be non-empty\"\n )\n\n # Check metadata is present\n assert all(\"file_path\" in chunk[\"metadata\"] for chunk in chunks), (\n \"Each chunk should have file_path metadata\"\n )\n\n # Check that code structure is somewhat preserved\n combined_content = \" \".join([c[\"text\"] for c in chunks])\n assert \"def hello_world\" in combined_content\n assert \"class Calculator\" in combined_content\n\n except ImportError:\n # astchunk not available, should fall back to traditional chunking\n chunks = create_ast_chunks(docs, max_chunk_size=200, chunk_overlap=50)\n assert len(chunks) > 0 # Should still get chunks from fallback\n\n def test_create_ast_chunks_fallback_to_traditional(self):\n \"\"\"Test AST chunking falls back to traditional when astchunk is not available.\"\"\"\n docs = [MockDocument(\"def test(): pass\", \"/test/script.py\", {\"language\": \"python\"})]\n\n # Mock astchunk import to fail\n with patch(\"chunking.create_ast_chunks\"):\n # First call (actual test) should import astchunk and potentially fail\n # Let's call the actual function to test the import error handling\n chunks = create_ast_chunks(docs)\n\n # Should return some chunks (either from astchunk or fallback)\n assert isinstance(chunks, list)\n\n def test_create_text_chunks_traditional_mode(self):\n \"\"\"Test text chunking in traditional mode.\"\"\"\n docs = [\n MockDocument(\"def test(): pass\", \"/test/script.py\"),\n MockDocument(\"This is regular text.\", \"/test/doc.txt\"),\n ]\n\n chunks = create_text_chunks(docs, use_ast_chunking=False, chunk_size=50, chunk_overlap=10)\n\n assert len(chunks) > 0\n # R3: Traditional chunking should also return dict format for consistency\n assert all(isinstance(chunk, dict) for chunk in chunks), \"All chunks should be dicts\"\n assert all(\"text\" in chunk and \"metadata\" in chunk for chunk in chunks), (\n \"Each chunk should have 'text' and 'metadata' keys\"\n )\n\n def test_create_text_chunks_ast_mode(self):\n \"\"\"Test text chunking in AST mode.\"\"\"\n docs = [\n MockDocument(\"def test(): pass\", \"/test/script.py\"),\n MockDocument(\"This is regular text.\", \"/test/doc.txt\"),\n ]\n\n chunks = create_text_chunks(\n docs,\n use_ast_chunking=True,\n ast_chunk_size=100,\n ast_chunk_overlap=20,\n chunk_size=50,\n chunk_overlap=10,\n )\n\n assert len(chunks) > 0\n # R3: AST mode should also return dict format\n assert all(isinstance(chunk, dict) for chunk in chunks), \"All chunks should be dicts\"\n assert all(\"text\" in chunk and \"metadata\" in chunk for chunk in chunks), (\n \"Each chunk should have 'text' and 'metadata' keys\"\n )\n\n def test_create_text_chunks_custom_extensions(self):\n \"\"\"Test text chunking with custom code file extensions.\"\"\"\n docs = [\n MockDocument(\"function test() {}\", \"/test/script.js\"), # Not in default extensions\n MockDocument(\"Regular text\", \"/test/doc.txt\"),\n ]\n\n # First without custom extensions - should treat .js as text\n chunks_without = create_text_chunks(docs, use_ast_chunking=True, code_file_extensions=None)\n\n # Then with custom extensions - should treat .js as code\n chunks_with = create_text_chunks(\n docs, use_ast_chunking=True, code_file_extensions=[\".js\", \".jsx\"]\n )\n\n # Both should return chunks\n assert len(chunks_without) > 0\n assert len(chunks_with) > 0\n\n\nclass TestIntegrationWithDocumentRAG:\n \"\"\"Integration tests with the document RAG system.\"\"\"\n\n @pytest.fixture\n def temp_code_dir(self):\n \"\"\"Create a temporary directory with sample code files.\"\"\"\n with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as temp_dir:\n temp_path = Path(temp_dir)\n\n # Create sample Python file\n python_file = temp_path / \"example.py\"\n python_file.write_text('''\ndef fibonacci(n):\n \"\"\"Calculate fibonacci number.\"\"\"\n if n <= 1:\n return n\n return fibonacci(n-1) + fibonacci(n-2)\n\nclass MathUtils:\n @staticmethod\n def factorial(n):\n if n <= 1:\n return 1\n return n * MathUtils.factorial(n-1)\n''')\n\n # Create sample text file\n text_file = temp_path / \"readme.txt\"\n text_file.write_text(\"This is a sample text file for testing purposes.\")\n\n yield temp_path\n\n @pytest.mark.skipif(\n os.environ.get(\"CI\") == \"true\",\n reason=\"Skip integration tests in CI to avoid dependency issues\",\n )\n @pytest.mark.timeout(0)\n def test_document_rag_with_ast_chunking(self, temp_code_dir):\n \"\"\"Test document RAG with AST chunking enabled.\"\"\"\n with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as index_dir:\n cmd = [\n sys.executable,\n \"apps/document_rag.py\",\n \"--llm\",\n \"simulated\",\n \"--embedding-model\",\n \"facebook/contriever\",\n \"--embedding-mode\",\n \"sentence-transformers\",\n \"--index-dir\",\n index_dir,\n \"--data-dir\",\n str(temp_code_dir),\n \"--enable-code-chunking\",\n \"--query\",\n \"How does the fibonacci function work?\",\n ]\n\n env = os.environ.copy()\n env[\"HF_HUB_DISABLE_SYMLINKS\"] = \"1\"\n env[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n\n try:\n result = subprocess.run(\n cmd,\n capture_output=True,\n text=True,\n timeout=300, # 5 minutes\n env=env,\n )\n\n # Should succeed even if astchunk is not available (fallback)\n assert result.returncode == 0, f\"Command failed: {result.stderr}\"\n\n output = result.stdout + result.stderr\n assert \"Index saved to\" in output or \"Using existing index\" in output\n\n except subprocess.TimeoutExpired:\n pytest.skip(\"Test timed out - likely due to model download in CI\")\n\n @pytest.mark.skipif(\n os.environ.get(\"CI\") == \"true\",\n reason=\"Skip integration tests in CI to avoid dependency issues\",\n )\n @pytest.mark.timeout(0)\n def test_code_rag_application(self, temp_code_dir):\n \"\"\"Test the specialized code RAG application.\"\"\"\n with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as index_dir:\n cmd = [\n sys.executable,\n \"apps/code_rag.py\",\n \"--llm\",\n \"simulated\",\n \"--embedding-model\",\n \"facebook/contriever\",\n \"--index-dir\",\n index_dir,\n \"--repo-dir\",\n str(temp_code_dir),\n \"--query\",\n \"What classes are defined in this code?\",\n ]\n\n env = os.environ.copy()\n env[\"HF_HUB_DISABLE_SYMLINKS\"] = \"1\"\n env[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n\n try:\n result = subprocess.run(cmd, capture_output=True, text=True, timeout=300, env=env)\n\n # Should succeed\n assert result.returncode == 0, f\"Command failed: {result.stderr}\"\n\n output = result.stdout + result.stderr\n assert \"Using AST-aware chunking\" in output or \"traditional chunking\" in output\n\n except subprocess.TimeoutExpired:\n pytest.skip(\"Test timed out - likely due to model download in CI\")\n\n\nclass TestASTContentExtraction:\n \"\"\"Test AST content extraction bug fix.\n\n These tests verify that astchunk's dict format with 'content' key is handled correctly,\n and that the extraction logic doesn't fall through to stringifying entire dicts.\n \"\"\"\n\n def test_extract_content_from_astchunk_dict(self):\n \"\"\"Test that astchunk dict format with 'content' key is handled correctly.\n\n Bug: Current code checks for chunk[\"text\"] but astchunk returns chunk[\"content\"].\n This causes fallthrough to str(chunk), stringifying the entire dict.\n\n This test will FAIL until the bug is fixed because:\n - Current code will stringify the dict: \"{'content': '...', 'metadata': {...}}\"\n - Fixed code should extract just the content value\n \"\"\"\n # Mock the ASTChunkBuilder class\n mock_builder = Mock()\n\n # Astchunk returns this format\n astchunk_format_chunk = {\n \"content\": \"def hello():\\n print('world')\",\n \"metadata\": {\n \"filepath\": \"test.py\",\n \"line_count\": 2,\n \"start_line_no\": 0,\n \"end_line_no\": 1,\n \"node_count\": 1,\n },\n }\n mock_builder.chunkify.return_value = [astchunk_format_chunk]\n\n # Create mock document\n doc = MockDocument(\n \"def hello():\\n print('world')\", \"/test/test.py\", {\"language\": \"python\"}\n )\n\n # Mock the astchunk module and its ASTChunkBuilder class\n mock_astchunk = Mock()\n mock_astchunk.ASTChunkBuilder = Mock(return_value=mock_builder)\n\n # Patch sys.modules to inject our mock before the import\n with patch.dict(\"sys.modules\", {\"astchunk\": mock_astchunk}):\n # Call create_ast_chunks\n chunks = create_ast_chunks([doc])\n\n # R3: Should return dict format with proper metadata\n assert len(chunks) > 0, \"Should return at least one chunk\"\n\n # R3: Each chunk should be a dict\n chunk = chunks[0]\n assert isinstance(chunk, dict), \"Chunk should be a dict\"\n assert \"text\" in chunk, \"Chunk should have 'text' key\"\n assert \"metadata\" in chunk, \"Chunk should have 'metadata' key\"\n\n chunk_text = chunk[\"text\"]\n\n # CRITICAL: Should NOT contain stringified dict markers in the text field\n # These assertions will FAIL with current buggy code\n assert \"'content':\" not in chunk_text, (\n f\"Chunk text contains stringified dict - extraction failed! Got: {chunk_text[:100]}...\"\n )\n assert \"'metadata':\" not in chunk_text, (\n \"Chunk text contains stringified metadata - extraction failed! \"\n f\"Got: {chunk_text[:100]}...\"\n )\n assert \"{\" not in chunk_text or \"def hello\" in chunk_text.split(\"{\")[0], (\n \"Chunk text appears to be a stringified dict\"\n )\n\n # Should contain actual content\n assert \"def hello()\" in chunk_text, \"Should extract actual code content\"\n assert \"print('world')\" in chunk_text, \"Should extract complete code content\"\n\n # R3: Should preserve astchunk metadata\n assert \"filepath\" in chunk[\"metadata\"] or \"file_path\" in chunk[\"metadata\"], (\n \"Should preserve file path metadata\"\n )\n\n def test_extract_text_key_fallback(self):\n \"\"\"Test that 'text' key still works for backward compatibility.\n\n Some chunks might use 'text' instead of 'content' - ensure backward compatibility.\n This test should PASS even with current code.\n \"\"\"\n mock_builder = Mock()\n\n # Some chunks might use \"text\" key\n text_key_chunk = {\"text\": \"def legacy_function():\\n return True\"}\n mock_builder.chunkify.return_value = [text_key_chunk]\n\n # Create mock document\n doc = MockDocument(\n \"def legacy_function():\\n return True\", \"/test/legacy.py\", {\"language\": \"python\"}\n )\n\n # Mock the astchunk module\n mock_astchunk = Mock()\n mock_astchunk.ASTChunkBuilder = Mock(return_value=mock_builder)\n\n with patch.dict(\"sys.modules\", {\"astchunk\": mock_astchunk}):\n # Call create_ast_chunks\n chunks = create_ast_chunks([doc])\n\n # R3: Should extract text correctly as dict format\n assert len(chunks) > 0\n chunk = chunks[0]\n assert isinstance(chunk, dict), \"Chunk should be a dict\"\n assert \"text\" in chunk, \"Chunk should have 'text' key\"\n\n chunk_text = chunk[\"text\"]\n\n # Should NOT be stringified\n assert \"'text':\" not in chunk_text, \"Should not stringify dict with 'text' key\"\n\n # Should contain actual content\n assert \"def legacy_function()\" in chunk_text\n assert \"return True\" in chunk_text\n\n def test_handles_string_chunks(self):\n \"\"\"Test that plain string chunks still work.\n\n Some chunkers might return plain strings - verify these are preserved.\n This test should PASS with current code.\n \"\"\"\n mock_builder = Mock()\n\n # Plain string chunk\n plain_string_chunk = \"def simple_function():\\n pass\"\n mock_builder.chunkify.return_value = [plain_string_chunk]\n\n # Create mock document\n doc = MockDocument(\n \"def simple_function():\\n pass\", \"/test/simple.py\", {\"language\": \"python\"}\n )\n\n # Mock the astchunk module\n mock_astchunk = Mock()\n mock_astchunk.ASTChunkBuilder = Mock(return_value=mock_builder)\n\n with patch.dict(\"sys.modules\", {\"astchunk\": mock_astchunk}):\n # Call create_ast_chunks\n chunks = create_ast_chunks([doc])\n\n # R3: Should wrap string in dict format\n assert len(chunks) > 0\n chunk = chunks[0]\n assert isinstance(chunk, dict), \"Even string chunks should be wrapped in dict\"\n assert \"text\" in chunk, \"Chunk should have 'text' key\"\n\n chunk_text = chunk[\"text\"]\n\n assert chunk_text == plain_string_chunk.strip(), (\n \"Should preserve plain string chunk content\"\n )\n assert \"def simple_function()\" in chunk_text\n assert \"pass\" in chunk_text\n\n def test_multiple_chunks_with_mixed_formats(self):\n \"\"\"Test handling of multiple chunks with different formats.\n\n Real-world scenario: astchunk might return a mix of formats.\n This test will FAIL if any chunk with 'content' key gets stringified.\n \"\"\"\n mock_builder = Mock()\n\n # Mix of formats\n mixed_chunks = [\n {\"content\": \"def first():\\n return 1\", \"metadata\": {\"line_count\": 2}},\n \"def second():\\n return 2\", # Plain string\n {\"text\": \"def third():\\n return 3\"}, # Old format\n {\"content\": \"class MyClass:\\n pass\", \"metadata\": {\"node_count\": 1}},\n ]\n mock_builder.chunkify.return_value = mixed_chunks\n\n # Create mock document\n code = \"def first():\\n return 1\\n\\ndef second():\\n return 2\\n\\ndef third():\\n return 3\\n\\nclass MyClass:\\n pass\"\n doc = MockDocument(code, \"/test/mixed.py\", {\"language\": \"python\"})\n\n # Mock the astchunk module\n mock_astchunk = Mock()\n mock_astchunk.ASTChunkBuilder = Mock(return_value=mock_builder)\n\n with patch.dict(\"sys.modules\", {\"astchunk\": mock_astchunk}):\n # Call create_ast_chunks\n chunks = create_ast_chunks([doc])\n\n # R3: Should extract all chunks correctly as dicts\n assert len(chunks) == 4, \"Should extract all 4 chunks\"\n\n # Check each chunk\n for i, chunk in enumerate(chunks):\n assert isinstance(chunk, dict), f\"Chunk {i} should be a dict\"\n assert \"text\" in chunk, f\"Chunk {i} should have 'text' key\"\n assert \"metadata\" in chunk, f\"Chunk {i} should have 'metadata' key\"\n\n chunk_text = chunk[\"text\"]\n # None should be stringified dicts\n assert \"'content':\" not in chunk_text, f\"Chunk {i} text is stringified (has 'content':)\"\n assert \"'metadata':\" not in chunk_text, (\n f\"Chunk {i} text is stringified (has 'metadata':)\"\n )\n assert \"'text':\" not in chunk_text, f\"Chunk {i} text is stringified (has 'text':)\"\n\n # Verify actual content is present\n combined = \"\\n\".join([c[\"text\"] for c in chunks])\n assert \"def first()\" in combined\n assert \"def second()\" in combined\n assert \"def third()\" in combined\n assert \"class MyClass:\" in combined\n\n def test_empty_content_value_handling(self):\n \"\"\"Test handling of chunks with empty content values.\n\n Edge case: chunk has 'content' key but value is empty.\n Should skip these chunks, not stringify them.\n \"\"\"\n mock_builder = Mock()\n\n chunks_with_empty = [\n {\"content\": \"\", \"metadata\": {\"line_count\": 0}}, # Empty content\n {\"content\": \" \", \"metadata\": {\"line_count\": 1}}, # Whitespace only\n {\"content\": \"def valid():\\n return True\", \"metadata\": {\"line_count\": 2}}, # Valid\n ]\n mock_builder.chunkify.return_value = chunks_with_empty\n\n doc = MockDocument(\n \"def valid():\\n return True\", \"/test/empty.py\", {\"language\": \"python\"}\n )\n\n # Mock the astchunk module\n mock_astchunk = Mock()\n mock_astchunk.ASTChunkBuilder = Mock(return_value=mock_builder)\n\n with patch.dict(\"sys.modules\", {\"astchunk\": mock_astchunk}):\n chunks = create_ast_chunks([doc])\n\n # R3: Should only have the valid chunk (empty ones filtered out)\n assert len(chunks) == 1, \"Should filter out empty content chunks\"\n\n chunk = chunks[0]\n assert isinstance(chunk, dict), \"Chunk should be a dict\"\n assert \"text\" in chunk, \"Chunk should have 'text' key\"\n assert \"def valid()\" in chunk[\"text\"]\n\n # Should not have stringified the empty dict\n assert \"'content': ''\" not in chunk[\"text\"]\n\n\nclass TestASTMetadataPreservation:\n \"\"\"Test metadata preservation in AST chunk dictionaries.\n\n R3: These tests define the contract for metadata preservation when returning\n chunk dictionaries instead of plain strings. Each chunk dict should have:\n - \"text\": str - the actual chunk content\n - \"metadata\": dict - all metadata from document AND astchunk\n\n These tests will FAIL until G3 implementation changes return type to list[dict].\n \"\"\"\n\n def test_ast_chunks_preserve_file_metadata(self):\n \"\"\"Test that document metadata is preserved in chunk metadata.\n\n This test verifies that all document-level metadata (file_path, file_name,\n creation_date, last_modified_date) is included in each chunk's metadata dict.\n\n This will FAIL because current code returns list[str], not list[dict].\n \"\"\"\n # Create mock document with rich metadata\n python_code = '''\ndef calculate_sum(numbers):\n \"\"\"Calculate sum of numbers.\"\"\"\n return sum(numbers)\n\nclass DataProcessor:\n \"\"\"Process data records.\"\"\"\n\n def process(self, data):\n return [x * 2 for x in data]\n'''\n doc = MockDocument(\n python_code,\n file_path=\"/project/src/utils.py\",\n metadata={\n \"language\": \"python\",\n \"file_path\": \"/project/src/utils.py\",\n \"file_name\": \"utils.py\",\n \"creation_date\": \"2024-01-15T10:30:00\",\n \"last_modified_date\": \"2024-10-31T15:45:00\",\n },\n )\n\n # Mock astchunk to return chunks with metadata\n mock_builder = Mock()\n astchunk_chunks = [\n {\n \"content\": \"def calculate_sum(numbers):\\n return sum(numbers)\",\n \"metadata\": {\n \"filepath\": \"/project/src/utils.py\",\n \"line_count\": 2,\n \"start_line_no\": 1,\n \"end_line_no\": 2,\n \"node_count\": 1,\n },\n },\n {\n \"content\": \"class DataProcessor:\\n def process(self, data):\\n return [x * 2 for x in data]\",\n \"metadata\": {\n \"filepath\": \"/project/src/utils.py\",\n \"line_count\": 3,\n \"start_line_no\": 5,\n \"end_line_no\": 7,\n \"node_count\": 2,\n },\n },\n ]\n mock_builder.chunkify.return_value = astchunk_chunks\n\n mock_astchunk = Mock()\n mock_astchunk.ASTChunkBuilder = Mock(return_value=mock_builder)\n\n with patch.dict(\"sys.modules\", {\"astchunk\": mock_astchunk}):\n chunks = create_ast_chunks([doc])\n\n # CRITICAL: These assertions will FAIL with current list[str] return type\n assert len(chunks) == 2, \"Should return 2 chunks\"\n\n for i, chunk in enumerate(chunks):\n # Structure assertions - WILL FAIL: current code returns strings\n assert isinstance(chunk, dict), f\"Chunk {i} should be dict, got {type(chunk)}\"\n assert \"text\" in chunk, f\"Chunk {i} must have 'text' key\"\n assert \"metadata\" in chunk, f\"Chunk {i} must have 'metadata' key\"\n assert isinstance(chunk[\"metadata\"], dict), f\"Chunk {i} metadata should be dict\"\n\n # Document metadata preservation - WILL FAIL\n metadata = chunk[\"metadata\"]\n assert \"file_path\" in metadata, f\"Chunk {i} should preserve file_path\"\n assert metadata[\"file_path\"] == \"/project/src/utils.py\", (\n f\"Chunk {i} file_path incorrect\"\n )\n\n assert \"file_name\" in metadata, f\"Chunk {i} should preserve file_name\"\n assert metadata[\"file_name\"] == \"utils.py\", f\"Chunk {i} file_name incorrect\"\n\n assert \"creation_date\" in metadata, f\"Chunk {i} should preserve creation_date\"\n assert metadata[\"creation_date\"] == \"2024-01-15T10:30:00\", (\n f\"Chunk {i} creation_date incorrect\"\n )\n\n assert \"last_modified_date\" in metadata, f\"Chunk {i} should preserve last_modified_date\"\n assert metadata[\"last_modified_date\"] == \"2024-10-31T15:45:00\", (\n f\"Chunk {i} last_modified_date incorrect\"\n )\n\n # Verify metadata is consistent across chunks from same document\n assert chunks[0][\"metadata\"][\"file_path\"] == chunks[1][\"metadata\"][\"file_path\"], (\n \"All chunks from same document should have same file_path\"\n )\n\n # Verify text content is present and not stringified\n assert \"def calculate_sum\" in chunks[0][\"text\"]\n assert \"class DataProcessor\" in chunks[1][\"text\"]\n\n def test_ast_chunks_include_astchunk_metadata(self):\n \"\"\"Test that astchunk-specific metadata is merged into chunk metadata.\n\n This test verifies that astchunk's metadata (line_count, start_line_no,\n end_line_no, node_count) is merged with document metadata.\n\n This will FAIL because current code returns list[str], not list[dict].\n \"\"\"\n python_code = '''\ndef function_one():\n \"\"\"First function.\"\"\"\n x = 1\n y = 2\n return x + y\n\ndef function_two():\n \"\"\"Second function.\"\"\"\n return 42\n'''\n doc = MockDocument(\n python_code,\n file_path=\"/test/code.py\",\n metadata={\n \"language\": \"python\",\n \"file_path\": \"/test/code.py\",\n \"file_name\": \"code.py\",\n },\n )\n\n # Mock astchunk with detailed metadata\n mock_builder = Mock()\n astchunk_chunks = [\n {\n \"content\": \"def function_one():\\n x = 1\\n y = 2\\n return x + y\",\n \"metadata\": {\n \"filepath\": \"/test/code.py\",\n \"line_count\": 4,\n \"start_line_no\": 1,\n \"end_line_no\": 4,\n \"node_count\": 5, # function, assignments, return\n },\n },\n {\n \"content\": \"def function_two():\\n return 42\",\n \"metadata\": {\n \"filepath\": \"/test/code.py\",\n \"line_count\": 2,\n \"start_line_no\": 7,\n \"end_line_no\": 8,\n \"node_count\": 2, # function, return\n },\n },\n ]\n mock_builder.chunkify.return_value = astchunk_chunks\n\n mock_astchunk = Mock()\n mock_astchunk.ASTChunkBuilder = Mock(return_value=mock_builder)\n\n with patch.dict(\"sys.modules\", {\"astchunk\": mock_astchunk}):\n chunks = create_ast_chunks([doc])\n\n # CRITICAL: These will FAIL with current list[str] return\n assert len(chunks) == 2\n\n # First chunk - function_one\n chunk1 = chunks[0]\n assert isinstance(chunk1, dict), \"Chunk should be dict\"\n assert \"metadata\" in chunk1\n\n metadata1 = chunk1[\"metadata\"]\n\n # Check astchunk metadata is present\n assert \"line_count\" in metadata1, \"Should include astchunk line_count\"\n assert metadata1[\"line_count\"] == 4, \"line_count should be 4\"\n\n assert \"start_line_no\" in metadata1, \"Should include astchunk start_line_no\"\n assert metadata1[\"start_line_no\"] == 1, \"start_line_no should be 1\"\n\n assert \"end_line_no\" in metadata1, \"Should include astchunk end_line_no\"\n assert metadata1[\"end_line_no\"] == 4, \"end_line_no should be 4\"\n\n assert \"node_count\" in metadata1, \"Should include astchunk node_count\"\n assert metadata1[\"node_count\"] == 5, \"node_count should be 5\"\n\n # Second chunk - function_two\n chunk2 = chunks[1]\n metadata2 = chunk2[\"metadata\"]\n\n assert metadata2[\"line_count\"] == 2, \"line_count should be 2\"\n assert metadata2[\"start_line_no\"] == 7, \"start_line_no should be 7\"\n assert metadata2[\"end_line_no\"] == 8, \"end_line_no should be 8\"\n assert metadata2[\"node_count\"] == 2, \"node_count should be 2\"\n\n # Verify document metadata is ALSO present (merged, not replaced)\n assert metadata1[\"file_path\"] == \"/test/code.py\"\n assert metadata1[\"file_name\"] == \"code.py\"\n assert metadata2[\"file_path\"] == \"/test/code.py\"\n assert metadata2[\"file_name\"] == \"code.py\"\n\n # Verify text content is correct\n assert \"def function_one\" in chunk1[\"text\"]\n assert \"def function_two\" in chunk2[\"text\"]\n\n def test_traditional_chunks_as_dicts_helper(self):\n \"\"\"Test the helper function that wraps traditional chunks as dicts.\n\n This test verifies that when create_traditional_chunks is called,\n its plain string chunks are wrapped into dict format with metadata.\n\n This will FAIL because the helper function _traditional_chunks_as_dicts()\n doesn't exist yet, and create_traditional_chunks returns list[str].\n \"\"\"\n # Create documents with various metadata\n docs = [\n MockDocument(\n \"This is the first paragraph of text. It contains multiple sentences. \"\n \"This should be split into chunks based on size.\",\n file_path=\"/docs/readme.txt\",\n metadata={\n \"file_path\": \"/docs/readme.txt\",\n \"file_name\": \"readme.txt\",\n \"creation_date\": \"2024-01-01\",\n },\n ),\n MockDocument(\n \"Second document with different metadata. It also has content that needs chunking.\",\n file_path=\"/docs/guide.md\",\n metadata={\n \"file_path\": \"/docs/guide.md\",\n \"file_name\": \"guide.md\",\n \"last_modified_date\": \"2024-10-31\",\n },\n ),\n ]\n\n # Call create_traditional_chunks (which should now return list[dict])\n chunks = create_traditional_chunks(docs, chunk_size=50, chunk_overlap=10)\n\n # CRITICAL: Will FAIL - current code returns list[str]\n assert len(chunks) > 0, \"Should return chunks\"\n\n for i, chunk in enumerate(chunks):\n # Structure assertions - WILL FAIL\n assert isinstance(chunk, dict), f\"Chunk {i} should be dict, got {type(chunk)}\"\n assert \"text\" in chunk, f\"Chunk {i} must have 'text' key\"\n assert \"metadata\" in chunk, f\"Chunk {i} must have 'metadata' key\"\n\n # Text should be non-empty\n assert len(chunk[\"text\"].strip()) > 0, f\"Chunk {i} text should be non-empty\"\n\n # Metadata should include document info\n metadata = chunk[\"metadata\"]\n assert \"file_path\" in metadata, f\"Chunk {i} should have file_path in metadata\"\n assert \"file_name\" in metadata, f\"Chunk {i} should have file_name in metadata\"\n\n # Verify metadata tracking works correctly\n # At least one chunk should be from readme.txt\n readme_chunks = [c for c in chunks if \"readme.txt\" in c[\"metadata\"][\"file_name\"]]\n assert len(readme_chunks) > 0, \"Should have chunks from readme.txt\"\n\n # At least one chunk should be from guide.md\n guide_chunks = [c for c in chunks if \"guide.md\" in c[\"metadata\"][\"file_name\"]]\n assert len(guide_chunks) > 0, \"Should have chunks from guide.md\"\n\n # Verify creation_date is preserved for readme chunks\n for chunk in readme_chunks:\n assert chunk[\"metadata\"].get(\"creation_date\") == \"2024-01-01\", (\n \"readme.txt chunks should preserve creation_date\"\n )\n\n # Verify last_modified_date is preserved for guide chunks\n for chunk in guide_chunks:\n assert chunk[\"metadata\"].get(\"last_modified_date\") == \"2024-10-31\", (\n \"guide.md chunks should preserve last_modified_date\"\n )\n\n # Verify text content is present\n all_text = \" \".join([c[\"text\"] for c in chunks])\n assert \"first paragraph\" in all_text\n assert \"Second document\" in all_text\n\n\nclass TestErrorHandling:\n \"\"\"Test error handling and edge cases.\"\"\"\n\n def test_text_chunking_empty_documents(self):\n \"\"\"Test text chunking with empty document list.\"\"\"\n chunks = create_text_chunks([])\n assert chunks == []\n\n def test_text_chunking_invalid_parameters(self):\n \"\"\"Test text chunking with invalid parameters.\"\"\"\n docs = [MockDocument(\"test content\")]\n\n # Should handle negative chunk sizes gracefully\n chunks = create_text_chunks(\n docs, chunk_size=0, chunk_overlap=0, ast_chunk_size=0, ast_chunk_overlap=0\n )\n\n # Should still return some result\n assert isinstance(chunks, list)\n\n def test_create_ast_chunks_no_language(self):\n \"\"\"Test AST chunking with documents missing language metadata.\"\"\"\n docs = [MockDocument(\"def test(): pass\", \"/test/script.py\")] # No language set\n\n chunks = create_ast_chunks(docs)\n\n # Should fall back to traditional chunking\n assert isinstance(chunks, list)\n assert len(chunks) >= 0 # May be empty if fallback also fails\n\n def test_create_ast_chunks_empty_content(self):\n \"\"\"Test AST chunking with empty content.\"\"\"\n docs = [MockDocument(\"\", \"/test/script.py\", {\"language\": \"python\"})]\n\n chunks = create_ast_chunks(docs)\n\n # Should handle empty content gracefully\n assert isinstance(chunks, list)\n\n\nif __name__ == \"__main__\":\n pytest.main([__file__, \"-v\"])\n" }, { "path": "tests/test_basic.py", "content": "\"\"\"\nBasic functionality tests for CI pipeline using pytest.\n\"\"\"\n\nimport os\nimport tempfile\nfrom pathlib import Path\n\nimport pytest\n\n\ndef test_imports():\n \"\"\"Test that all packages can be imported.\"\"\"\n\n # Test C++ extensions\n\n\n@pytest.mark.skipif(\n os.environ.get(\"CI\") == \"true\", reason=\"Skip model tests in CI to avoid MPS memory issues\"\n)\n@pytest.mark.parametrize(\"backend_name\", [\"hnsw\", \"diskann\"])\ndef test_backend_basic(backend_name):\n \"\"\"Test basic functionality for each backend.\"\"\"\n from leann.api import LeannBuilder, LeannSearcher, SearchResult\n\n with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as temp_dir:\n index_path = str(Path(temp_dir) / f\"test.{backend_name}\")\n\n texts = [f\"This is document {i} about topic {i % 5}\" for i in range(100)]\n\n if backend_name == \"hnsw\":\n builder = LeannBuilder(\n backend_name=\"hnsw\",\n embedding_model=\"facebook/contriever\",\n embedding_mode=\"sentence-transformers\",\n M=16,\n efConstruction=200,\n )\n else: # diskann\n builder = LeannBuilder(\n backend_name=\"diskann\",\n embedding_model=\"facebook/contriever\",\n embedding_mode=\"sentence-transformers\",\n num_neighbors=32,\n search_list_size=50,\n )\n\n for text in texts:\n builder.add_text(text)\n\n builder.build_index(index_path)\n\n with LeannSearcher(index_path) as searcher:\n results = searcher.search(\"document about topic 2\", top_k=5)\n\n assert len(results) > 0\n assert isinstance(results[0], SearchResult)\n assert \"topic 2\" in results[0].text or \"document\" in results[0].text\n\n\n@pytest.mark.skipif(\n os.environ.get(\"CI\") == \"true\", reason=\"Skip model tests in CI to avoid MPS memory issues\"\n)\ndef test_large_index():\n \"\"\"Test with larger dataset.\"\"\"\n from leann.api import LeannBuilder, LeannSearcher\n\n with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as temp_dir:\n index_path = str(Path(temp_dir) / \"test_large.hnsw\")\n texts = [f\"Document {i}: {' '.join([f'word{j}' for j in range(50)])}\" for i in range(1000)]\n\n builder = LeannBuilder(\n backend_name=\"hnsw\",\n embedding_model=\"facebook/contriever\",\n embedding_mode=\"sentence-transformers\",\n )\n\n for text in texts:\n builder.add_text(text)\n\n builder.build_index(index_path)\n\n with LeannSearcher(index_path) as searcher:\n results = searcher.search(\"word10 word20\", top_k=10)\n assert len(results) == 10\n" }, { "path": "tests/test_ci_minimal.py", "content": "\"\"\"\nMinimal tests for CI that don't require model loading or significant memory.\n\"\"\"\n\nimport subprocess\nimport sys\n\n\ndef test_package_imports():\n \"\"\"Test that all core packages can be imported.\"\"\"\n # Core package\n\n # Backend packages\n\n # Core modules\n\n assert True # If we get here, imports worked\n\n\ndef test_cli_help():\n \"\"\"Test that CLI example shows help.\"\"\"\n result = subprocess.run(\n [sys.executable, \"apps/document_rag.py\", \"--help\"], capture_output=True, text=True\n )\n\n assert result.returncode == 0\n assert \"usage:\" in result.stdout.lower() or \"usage:\" in result.stderr.lower()\n assert \"--llm\" in result.stdout or \"--llm\" in result.stderr\n\n\ndef test_backend_registration():\n \"\"\"Test that backends are properly registered.\"\"\"\n from leann.api import get_registered_backends\n\n backends = get_registered_backends()\n assert \"hnsw\" in backends\n assert \"diskann\" in backends\n\n\ndef test_version_info():\n \"\"\"Test that packages have version information.\"\"\"\n import leann\n import leann_backend_diskann\n import leann_backend_hnsw\n\n # Check that packages have __version__ or can be imported\n assert hasattr(leann, \"__version__\") or True\n assert hasattr(leann_backend_hnsw, \"__version__\") or True\n assert hasattr(leann_backend_diskann, \"__version__\") or True\n" }, { "path": "tests/test_cli_ask.py", "content": "from leann.cli import LeannCLI\n\n\ndef test_cli_ask_accepts_positional_query(tmp_path, monkeypatch):\n monkeypatch.chdir(tmp_path)\n\n cli = LeannCLI()\n parser = cli.create_parser()\n\n args = parser.parse_args([\"ask\", \"my-docs\", \"Where are prompts configured?\"])\n\n assert args.command == \"ask\"\n assert args.index_name == \"my-docs\"\n assert args.query == \"Where are prompts configured?\"\n" }, { "path": "tests/test_cli_daemon_workflow.py", "content": "import argparse\nimport asyncio\nfrom pathlib import Path\nfrom typing import Any\n\nfrom leann.cli import LeannCLI\n\n\ndef test_search_passes_daemon_flags_to_searcher(monkeypatch):\n cli = LeannCLI()\n monkeypatch.setattr(\n cli,\n \"_resolve_index_path\",\n lambda *args, **kwargs: \"/tmp/demo/documents.leann\",\n )\n\n captured: dict[str, dict[str, Any]] = {\"init\": {}, \"search\": {}}\n\n class DummySearcher:\n def __init__(self, *args, **kwargs):\n captured[\"init\"] = kwargs\n\n def search(self, *args, **kwargs):\n captured[\"search\"] = kwargs\n return []\n\n monkeypatch.setattr(\"leann.cli.LeannSearcher\", DummySearcher)\n\n args = argparse.Namespace(\n index_name=\"demo\",\n query=\"hello\",\n top_k=3,\n complexity=64,\n beam_width=1,\n prune_ratio=0.0,\n recompute_embeddings=True,\n pruning_strategy=\"global\",\n non_interactive=True,\n show_metadata=False,\n embedding_prompt_template=None,\n use_daemon=True,\n daemon_ttl=222,\n enable_warmup=True,\n )\n asyncio.run(cli.search_documents(args))\n\n assert captured[\"init\"][\"enable_warmup\"] is True\n assert captured[\"init\"][\"use_daemon\"] is True\n assert captured[\"init\"][\"daemon_ttl_seconds\"] == 222\n assert captured[\"search\"][\"recompute_embeddings\"] is True\n\n\ndef test_warmup_command_calls_searcher_warmup(monkeypatch):\n cli = LeannCLI()\n monkeypatch.setattr(\n cli,\n \"_resolve_index_path\",\n lambda *args, **kwargs: \"/tmp/demo/documents.leann\",\n )\n\n state: dict[str, int] = {\"warmup_called\": 0}\n\n class DummySearcher:\n def __init__(self, *args, **kwargs):\n pass\n\n def warmup(self):\n state[\"warmup_called\"] += 1\n\n monkeypatch.setattr(\"leann.cli.LeannSearcher\", DummySearcher)\n\n args = argparse.Namespace(\n index_name=\"demo\",\n use_daemon=True,\n daemon_ttl=120,\n enable_warmup=True,\n )\n asyncio.run(cli.warmup_index(args))\n assert state[\"warmup_called\"] == 1\n\n\ndef test_daemon_status_filters_by_index(monkeypatch, capsys):\n cli = LeannCLI()\n monkeypatch.setattr(\n cli,\n \"_resolve_index_path\",\n lambda *args, **kwargs: \"/tmp/demo/documents.leann\",\n )\n meta_path = str(Path(\"/tmp/demo/documents.leann.meta.json\").resolve())\n\n monkeypatch.setattr(\n \"leann.cli.EmbeddingServerManager.list_daemons\",\n lambda: [\n {\n \"pid\": 101,\n \"port\": 5557,\n \"backend_module_name\": \"leann_backend_hnsw.hnsw_embedding_server\",\n \"config_signature\": {\"passages_file\": meta_path, \"model_name\": \"m1\"},\n },\n {\n \"pid\": 202,\n \"port\": 5558,\n \"backend_module_name\": \"leann_backend_hnsw.hnsw_embedding_server\",\n \"config_signature\": {\n \"passages_file\": \"/tmp/other/doc.meta.json\",\n \"model_name\": \"m2\",\n },\n },\n ],\n )\n\n args = argparse.Namespace(daemon_command=\"status\", index_name=\"demo\")\n asyncio.run(cli.daemon_command(args))\n out = capsys.readouterr().out\n assert \"Active embedding daemons: 1\" in out\n assert \"pid=101\" in out\n assert \"pid=202\" not in out\n\n\ndef test_daemon_stop_by_index_calls_stop_daemons(monkeypatch):\n cli = LeannCLI()\n monkeypatch.setattr(\n cli,\n \"_resolve_index_path\",\n lambda *args, **kwargs: \"/tmp/demo/documents.leann\",\n )\n captured: dict[str, dict[str, Any]] = {\"kwargs\": {}}\n\n def fake_stop_daemons(**kwargs):\n captured[\"kwargs\"] = kwargs\n return 1\n\n monkeypatch.setattr(\"leann.cli.EmbeddingServerManager.stop_daemons\", fake_stop_daemons)\n\n args = argparse.Namespace(daemon_command=\"stop\", index_name=\"demo\", all=False)\n asyncio.run(cli.daemon_command(args))\n\n assert captured[\"kwargs\"][\"passages_file\"].endswith(\"documents.leann.meta.json\")\n\n\ndef test_daemon_start_calls_searcher_warmup(monkeypatch):\n cli = LeannCLI()\n monkeypatch.setattr(\n cli,\n \"_resolve_index_path\",\n lambda *args, **kwargs: \"/tmp/demo/documents.leann\",\n )\n\n state: dict[str, Any] = {\"warmup_called\": 0, \"init_kwargs\": {}}\n\n class DummySearcher:\n def __init__(self, *args, **kwargs):\n state[\"init_kwargs\"] = kwargs\n\n def warmup(self):\n state[\"warmup_called\"] += 1\n\n monkeypatch.setattr(\"leann.cli.LeannSearcher\", DummySearcher)\n\n args = argparse.Namespace(\n daemon_command=\"start\",\n index_name=\"demo\",\n daemon_ttl=88,\n enable_warmup=True,\n )\n asyncio.run(cli.daemon_command(args))\n\n assert state[\"warmup_called\"] == 1\n assert state[\"init_kwargs\"][\"use_daemon\"] is True\n assert state[\"init_kwargs\"][\"daemon_ttl_seconds\"] == 88\n\n\ndef test_daemon_status_all_lists_records(monkeypatch, capsys):\n cli = LeannCLI()\n monkeypatch.setattr(\n \"leann.cli.EmbeddingServerManager.list_daemons\",\n lambda: [\n {\n \"pid\": 301,\n \"port\": 6001,\n \"backend_module_name\": \"leann_backend_hnsw.hnsw_embedding_server\",\n \"config_signature\": {\"model_name\": \"m-a\"},\n },\n {\n \"pid\": 302,\n \"port\": 6002,\n \"backend_module_name\": \"leann_backend_diskann.diskann_embedding_server\",\n \"config_signature\": {\"model_name\": \"m-b\"},\n },\n ],\n )\n args = argparse.Namespace(daemon_command=\"status\", index_name=None)\n asyncio.run(cli.daemon_command(args))\n out = capsys.readouterr().out\n assert \"Active embedding daemons: 2\" in out\n assert \"pid=301\" in out\n assert \"pid=302\" in out\n\n\ndef test_daemon_stop_all_calls_manager(monkeypatch):\n cli = LeannCLI()\n captured = {\"called\": False}\n\n def fake_stop_daemons(**kwargs):\n captured[\"called\"] = True\n assert kwargs == {}\n return 2\n\n monkeypatch.setattr(\"leann.cli.EmbeddingServerManager.stop_daemons\", fake_stop_daemons)\n args = argparse.Namespace(daemon_command=\"stop\", index_name=None, all=True)\n asyncio.run(cli.daemon_command(args))\n assert captured[\"called\"] is True\n" }, { "path": "tests/test_cli_prompt_template.py", "content": "\"\"\"\nTests for CLI argument integration of --embedding-prompt-template.\n\nThese tests verify that:\n1. The --embedding-prompt-template flag is properly registered on build and search commands\n2. The template value flows from CLI args to embedding_options dict\n3. The template is passed through to compute_embeddings() function\n4. Default behavior (no flag) is handled correctly\n\"\"\"\n\nfrom unittest.mock import Mock, patch\n\nfrom leann.cli import LeannCLI\n\n\nclass TestCLIPromptTemplateArgument:\n \"\"\"Tests for --embedding-prompt-template on build and search commands.\"\"\"\n\n def test_commands_accept_prompt_template_argument(self):\n \"\"\"Verify that build and search parsers accept --embedding-prompt-template flag.\"\"\"\n cli = LeannCLI()\n parser = cli.create_parser()\n template_value = \"search_query: \"\n\n # Test build command\n build_args = parser.parse_args(\n [\n \"build\",\n \"test-index\",\n \"--docs\",\n \"/tmp/test-docs\",\n \"--embedding-prompt-template\",\n template_value,\n ]\n )\n assert build_args.command == \"build\"\n assert hasattr(build_args, \"embedding_prompt_template\"), (\n \"build command should have embedding_prompt_template attribute\"\n )\n assert build_args.embedding_prompt_template == template_value\n\n # Test search command\n search_args = parser.parse_args(\n [\"search\", \"test-index\", \"my query\", \"--embedding-prompt-template\", template_value]\n )\n assert search_args.command == \"search\"\n assert hasattr(search_args, \"embedding_prompt_template\"), (\n \"search command should have embedding_prompt_template attribute\"\n )\n assert search_args.embedding_prompt_template == template_value\n\n def test_commands_default_to_none(self):\n \"\"\"Verify default value is None when flag not provided (backward compatibility).\"\"\"\n cli = LeannCLI()\n parser = cli.create_parser()\n\n # Test build command default\n build_args = parser.parse_args([\"build\", \"test-index\", \"--docs\", \"/tmp/test-docs\"])\n assert hasattr(build_args, \"embedding_prompt_template\"), (\n \"build command should have embedding_prompt_template attribute\"\n )\n assert build_args.embedding_prompt_template is None, (\n \"Build default value should be None when flag not provided\"\n )\n\n # Test search command default\n search_args = parser.parse_args([\"search\", \"test-index\", \"my query\"])\n assert hasattr(search_args, \"embedding_prompt_template\"), (\n \"search command should have embedding_prompt_template attribute\"\n )\n assert search_args.embedding_prompt_template is None, (\n \"Search default value should be None when flag not provided\"\n )\n\n\nclass TestBuildCommandPromptTemplateArgumentExtras:\n \"\"\"Additional build-specific tests for prompt template argument.\"\"\"\n\n def test_build_command_prompt_template_with_multiword_value(self):\n \"\"\"\n Verify that template values with spaces are handled correctly.\n\n Templates like \"search_document: \" or \"Represent this sentence for searching: \"\n should be accepted as a single string argument.\n \"\"\"\n cli = LeannCLI()\n parser = cli.create_parser()\n\n template = \"Represent this sentence for searching: \"\n args = parser.parse_args(\n [\n \"build\",\n \"test-index\",\n \"--docs\",\n \"/tmp/test-docs\",\n \"--embedding-prompt-template\",\n template,\n ]\n )\n\n assert args.embedding_prompt_template == template\n\n\nclass TestPromptTemplateStoredInEmbeddingOptions:\n \"\"\"Tests for template storage in embedding_options dict.\"\"\"\n\n @patch(\"leann.cli.LeannBuilder\")\n def test_prompt_template_stored_in_embedding_options_on_build(\n self, mock_builder_class, tmp_path\n ):\n \"\"\"\n Verify that when --embedding-prompt-template is provided to build command,\n the value is stored in embedding_options dict passed to LeannBuilder.\n\n This test will fail because the CLI doesn't currently process this argument\n and add it to embedding_options.\n \"\"\"\n # Setup mocks\n mock_builder = Mock()\n mock_builder_class.return_value = mock_builder\n\n # Create CLI and run build command\n cli = LeannCLI()\n\n # Mock load_documents to return a document so builder is created\n cli.load_documents = Mock(return_value=[{\"text\": \"test content\", \"metadata\": {}}]) # type: ignore[assignment]\n\n parser = cli.create_parser()\n\n template = \"search_query: \"\n args = parser.parse_args(\n [\n \"build\",\n \"test-index\",\n \"--docs\",\n str(tmp_path),\n \"--embedding-prompt-template\",\n template,\n \"--force\", # Force rebuild to ensure LeannBuilder is called\n ]\n )\n\n # Run the build command\n import asyncio\n\n asyncio.run(cli.build_index(args))\n\n # Check that LeannBuilder was called with embedding_options containing prompt_template\n call_kwargs = mock_builder_class.call_args.kwargs\n assert \"embedding_options\" in call_kwargs, \"LeannBuilder should receive embedding_options\"\n\n embedding_options = call_kwargs[\"embedding_options\"]\n assert embedding_options is not None, (\n \"embedding_options should not be None when template provided\"\n )\n assert \"prompt_template\" in embedding_options, (\n \"embedding_options should contain 'prompt_template' key\"\n )\n assert embedding_options[\"prompt_template\"] == template, (\n f\"Template should be '{template}', got {embedding_options.get('prompt_template')}\"\n )\n\n @patch(\"leann.cli.LeannBuilder\")\n def test_prompt_template_not_in_options_when_not_provided(self, mock_builder_class, tmp_path):\n \"\"\"\n Verify that when --embedding-prompt-template is NOT provided,\n embedding_options either doesn't have the key or it's None.\n\n This ensures we don't pass empty/None values unnecessarily.\n \"\"\"\n # Setup mocks\n mock_builder = Mock()\n mock_builder_class.return_value = mock_builder\n\n cli = LeannCLI()\n\n # Mock load_documents to return a document so builder is created\n cli.load_documents = Mock(return_value=[{\"text\": \"test content\", \"metadata\": {}}]) # type: ignore[assignment]\n\n parser = cli.create_parser()\n\n args = parser.parse_args(\n [\n \"build\",\n \"test-index\",\n \"--docs\",\n str(tmp_path),\n \"--force\", # Force rebuild to ensure LeannBuilder is called\n ]\n )\n\n import asyncio\n\n asyncio.run(cli.build_index(args))\n\n # Check that if embedding_options is passed, it doesn't have prompt_template\n call_kwargs = mock_builder_class.call_args.kwargs\n if call_kwargs.get(\"embedding_options\"):\n embedding_options = call_kwargs[\"embedding_options\"]\n # Either the key shouldn't exist, or it should be None\n assert (\n \"prompt_template\" not in embedding_options\n or embedding_options[\"prompt_template\"] is None\n ), \"prompt_template should not be set when flag not provided\"\n\n # R1 Tests: Build-time separate template storage\n @patch(\"leann.cli.LeannBuilder\")\n def test_build_stores_separate_templates(self, mock_builder_class, tmp_path):\n \"\"\"\n R1 Test 1: Verify that when both --embedding-prompt-template and\n --query-prompt-template are provided to build command, both values\n are stored separately in embedding_options dict as build_prompt_template\n and query_prompt_template.\n\n This test will fail because:\n 1. CLI doesn't accept --query-prompt-template flag yet\n 2. CLI doesn't store templates as separate build_prompt_template and\n query_prompt_template keys\n\n Expected behavior after implementation:\n - .meta.json contains: {\"embedding_options\": {\n \"build_prompt_template\": \"doc: \",\n \"query_prompt_template\": \"query: \"\n }}\n \"\"\"\n # Setup mocks\n mock_builder = Mock()\n mock_builder_class.return_value = mock_builder\n\n cli = LeannCLI()\n\n # Mock load_documents to return a document so builder is created\n cli.load_documents = Mock(return_value=[{\"text\": \"test content\", \"metadata\": {}}]) # type: ignore[assignment]\n\n parser = cli.create_parser()\n\n build_template = \"doc: \"\n query_template = \"query: \"\n args = parser.parse_args(\n [\n \"build\",\n \"test-index\",\n \"--docs\",\n str(tmp_path),\n \"--embedding-prompt-template\",\n build_template,\n \"--query-prompt-template\",\n query_template,\n \"--force\",\n ]\n )\n\n # Run the build command\n import asyncio\n\n asyncio.run(cli.build_index(args))\n\n # Check that LeannBuilder was called with separate template keys\n call_kwargs = mock_builder_class.call_args.kwargs\n assert \"embedding_options\" in call_kwargs, \"LeannBuilder should receive embedding_options\"\n\n embedding_options = call_kwargs[\"embedding_options\"]\n assert embedding_options is not None, (\n \"embedding_options should not be None when templates provided\"\n )\n\n assert \"build_prompt_template\" in embedding_options, (\n \"embedding_options should contain 'build_prompt_template' key\"\n )\n assert embedding_options[\"build_prompt_template\"] == build_template, (\n f\"build_prompt_template should be '{build_template}'\"\n )\n\n assert \"query_prompt_template\" in embedding_options, (\n \"embedding_options should contain 'query_prompt_template' key\"\n )\n assert embedding_options[\"query_prompt_template\"] == query_template, (\n f\"query_prompt_template should be '{query_template}'\"\n )\n\n # Old key should NOT be present when using new separate template format\n assert \"prompt_template\" not in embedding_options, (\n \"Old 'prompt_template' key should not be present with separate templates\"\n )\n\n @patch(\"leann.cli.LeannBuilder\")\n def test_build_backward_compat_single_template(self, mock_builder_class, tmp_path):\n \"\"\"\n R1 Test 2: Verify backward compatibility - when only\n --embedding-prompt-template is provided (old behavior), it should\n still be stored as 'prompt_template' in embedding_options.\n\n This ensures existing workflows continue to work unchanged.\n\n This test currently passes because it matches existing behavior, but it\n documents the requirement that this behavior must be preserved after\n implementing the separate template feature.\n\n Expected behavior:\n - .meta.json contains: {\"embedding_options\": {\"prompt_template\": \"prompt: \"}}\n - No build_prompt_template or query_prompt_template keys\n \"\"\"\n # Setup mocks\n mock_builder = Mock()\n mock_builder_class.return_value = mock_builder\n\n cli = LeannCLI()\n\n # Mock load_documents to return a document so builder is created\n cli.load_documents = Mock(return_value=[{\"text\": \"test content\", \"metadata\": {}}]) # type: ignore[assignment]\n\n parser = cli.create_parser()\n\n template = \"prompt: \"\n args = parser.parse_args(\n [\n \"build\",\n \"test-index\",\n \"--docs\",\n str(tmp_path),\n \"--embedding-prompt-template\",\n template,\n \"--force\",\n ]\n )\n\n # Run the build command\n import asyncio\n\n asyncio.run(cli.build_index(args))\n\n # Check that LeannBuilder was called with old format\n call_kwargs = mock_builder_class.call_args.kwargs\n assert \"embedding_options\" in call_kwargs, \"LeannBuilder should receive embedding_options\"\n\n embedding_options = call_kwargs[\"embedding_options\"]\n assert embedding_options is not None, (\n \"embedding_options should not be None when template provided\"\n )\n\n assert \"prompt_template\" in embedding_options, (\n \"embedding_options should contain old 'prompt_template' key for backward compat\"\n )\n assert embedding_options[\"prompt_template\"] == template, (\n f\"prompt_template should be '{template}'\"\n )\n\n # New keys should NOT be present in backward compat mode\n assert \"build_prompt_template\" not in embedding_options, (\n \"build_prompt_template should not be present with single template flag\"\n )\n assert \"query_prompt_template\" not in embedding_options, (\n \"query_prompt_template should not be present with single template flag\"\n )\n\n @patch(\"leann.cli.LeannBuilder\")\n def test_build_no_templates(self, mock_builder_class, tmp_path):\n \"\"\"\n R1 Test 3: Verify that when no template flags are provided,\n embedding_options has no prompt template keys.\n\n This ensures clean defaults and no unnecessary keys in .meta.json.\n\n This test currently passes because it matches existing behavior, but it\n documents the requirement that this behavior must be preserved after\n implementing the separate template feature.\n\n Expected behavior:\n - .meta.json has no prompt_template, build_prompt_template, or\n query_prompt_template keys (or embedding_options is empty/None)\n \"\"\"\n # Setup mocks\n mock_builder = Mock()\n mock_builder_class.return_value = mock_builder\n\n cli = LeannCLI()\n\n # Mock load_documents to return a document so builder is created\n cli.load_documents = Mock(return_value=[{\"text\": \"test content\", \"metadata\": {}}]) # type: ignore[assignment]\n\n parser = cli.create_parser()\n\n args = parser.parse_args([\"build\", \"test-index\", \"--docs\", str(tmp_path), \"--force\"])\n\n # Run the build command\n import asyncio\n\n asyncio.run(cli.build_index(args))\n\n # Check that no template keys are present\n call_kwargs = mock_builder_class.call_args.kwargs\n if call_kwargs.get(\"embedding_options\"):\n embedding_options = call_kwargs[\"embedding_options\"]\n\n # None of the template keys should be present\n assert \"prompt_template\" not in embedding_options, (\n \"prompt_template should not be present when no flags provided\"\n )\n assert \"build_prompt_template\" not in embedding_options, (\n \"build_prompt_template should not be present when no flags provided\"\n )\n assert \"query_prompt_template\" not in embedding_options, (\n \"query_prompt_template should not be present when no flags provided\"\n )\n\n\nclass TestPromptTemplateFlowsToComputeEmbeddings:\n \"\"\"Tests for template flowing through to compute_embeddings function.\"\"\"\n\n @patch(\"leann.api.compute_embeddings\")\n def test_prompt_template_flows_to_compute_embeddings_via_provider_options(\n self, mock_compute_embeddings, tmp_path\n ):\n \"\"\"\n Verify that the prompt template flows from CLI args through LeannBuilder\n to compute_embeddings() function via provider_options parameter.\n\n This is an integration test that verifies the complete flow:\n CLI → embedding_options → LeannBuilder → compute_embeddings(provider_options)\n\n This test will fail because:\n 1. CLI doesn't capture the argument yet\n 2. embedding_options doesn't include prompt_template\n 3. LeannBuilder doesn't pass it through to compute_embeddings\n \"\"\"\n # Mock compute_embeddings to return dummy embeddings as numpy array\n import numpy as np\n\n mock_compute_embeddings.return_value = np.array([[0.1, 0.2, 0.3]], dtype=np.float32)\n\n # Use real LeannBuilder (not mocked) to test the actual flow\n cli = LeannCLI()\n\n # Mock load_documents to return a simple document\n cli.load_documents = Mock(return_value=[{\"text\": \"test content\", \"metadata\": {}}]) # type: ignore[assignment]\n\n parser = cli.create_parser()\n\n template = \"search_document: \"\n args = parser.parse_args(\n [\n \"build\",\n \"test-index\",\n \"--docs\",\n str(tmp_path),\n \"--embedding-prompt-template\",\n template,\n \"--backend-name\",\n \"hnsw\", # Use hnsw backend\n \"--force\", # Force rebuild to ensure index is created\n ]\n )\n\n # This should fail because the flow isn't implemented yet\n import asyncio\n\n asyncio.run(cli.build_index(args))\n\n # Verify compute_embeddings was called with provider_options containing prompt_template\n assert mock_compute_embeddings.called, \"compute_embeddings should have been called\"\n\n # Check the call arguments\n call_kwargs = mock_compute_embeddings.call_args.kwargs\n assert \"provider_options\" in call_kwargs, (\n \"compute_embeddings should receive provider_options parameter\"\n )\n\n provider_options = call_kwargs[\"provider_options\"]\n assert provider_options is not None, \"provider_options should not be None\"\n assert \"prompt_template\" in provider_options, (\n \"provider_options should contain prompt_template key\"\n )\n assert provider_options[\"prompt_template\"] == template, (\n f\"Template should be '{template}', got {provider_options.get('prompt_template')}\"\n )\n\n\nclass TestPromptTemplateArgumentHelp:\n \"\"\"Tests for argument help text and documentation.\"\"\"\n\n def test_build_command_prompt_template_has_help_text(self):\n \"\"\"\n Verify that --embedding-prompt-template has descriptive help text.\n\n Good help text is crucial for CLI usability.\n \"\"\"\n cli = LeannCLI()\n parser = cli.create_parser()\n\n # Get the build subparser\n # This is a bit tricky - we need to parse to get the help\n # We'll check that the help includes relevant keywords\n import io\n from contextlib import redirect_stdout\n\n f = io.StringIO()\n try:\n with redirect_stdout(f):\n parser.parse_args([\"build\", \"--help\"])\n except SystemExit:\n pass # --help causes sys.exit(0)\n\n help_text = f.getvalue()\n assert \"--embedding-prompt-template\" in help_text, (\n \"Help text should mention --embedding-prompt-template\"\n )\n # Check for keywords that should be in the help\n help_lower = help_text.lower()\n assert any(keyword in help_lower for keyword in [\"template\", \"prompt\", \"prepend\"]), (\n \"Help text should explain what the prompt template does\"\n )\n\n def test_search_command_prompt_template_has_help_text(self):\n \"\"\"\n Verify that search command also has help text for --embedding-prompt-template.\n \"\"\"\n cli = LeannCLI()\n parser = cli.create_parser()\n\n import io\n from contextlib import redirect_stdout\n\n f = io.StringIO()\n try:\n with redirect_stdout(f):\n parser.parse_args([\"search\", \"--help\"])\n except SystemExit:\n pass # --help causes sys.exit(0)\n\n help_text = f.getvalue()\n assert \"--embedding-prompt-template\" in help_text, (\n \"Search help text should mention --embedding-prompt-template\"\n )\n" }, { "path": "tests/test_cli_verbosity.py", "content": "\"\"\"Tests for CLI verbosity options.\n\nThis module tests the configurable verbosity functionality that allows\nsuppressing C++ output from FAISS/HNSW.\nSee: https://github.com/yichuan-w/LEANN/issues/187\n\"\"\"\n\nimport os\n\nimport pytest\n\n\nclass TestSuppressCppOutput:\n \"\"\"Test the suppress_cpp_output context manager.\"\"\"\n\n def test_suppress_cpp_output_captures_stdout(self):\n \"\"\"C output to stdout should be suppressed when enabled.\"\"\"\n from leann.cli import suppress_cpp_output\n\n with suppress_cpp_output(suppress=True):\n # This goes to fd 1, but is redirected to devnull\n os.write(1, b\"This should be suppressed\\n\")\n\n # If we got here without error, suppression worked\n # The text was written to devnull\n\n def test_suppress_cpp_output_captures_stderr(self):\n \"\"\"C output to stderr should be suppressed when enabled.\"\"\"\n from leann.cli import suppress_cpp_output\n\n with suppress_cpp_output(suppress=True):\n # This goes to fd 2, but is redirected to devnull\n os.write(2, b\"This error should be suppressed\\n\")\n\n def test_suppress_cpp_output_restores_fds(self):\n \"\"\"File descriptors should be restored after context.\"\"\"\n from leann.cli import suppress_cpp_output\n\n # Save original fds\n original_stdout = os.dup(1)\n original_stderr = os.dup(2)\n\n try:\n with suppress_cpp_output(suppress=True):\n pass\n\n # Verify fds are still valid and point to original destinations\n # by checking we can write to them\n os.write(1, b\"\") # Should not raise\n os.write(2, b\"\") # Should not raise\n finally:\n os.close(original_stdout)\n os.close(original_stderr)\n\n def test_suppress_cpp_output_disabled(self):\n \"\"\"When suppress=False, output should not be suppressed.\"\"\"\n from leann.cli import suppress_cpp_output\n\n with suppress_cpp_output(suppress=False):\n # This should work normally\n os.write(1, b\"\")\n os.write(2, b\"\")\n\n\nclass TestCliVerbosityArgs:\n \"\"\"Test CLI argument parsing for verbosity options.\"\"\"\n\n def test_verbose_flag_parsed(self):\n \"\"\"--verbose flag should be parsed correctly.\"\"\"\n from leann.cli import LeannCLI\n\n cli = LeannCLI()\n parser = cli.create_parser()\n\n args = parser.parse_args([\"--verbose\", \"list\"])\n assert args.verbose is True\n assert args.quiet is False\n\n def test_quiet_flag_parsed(self):\n \"\"\"-q flag should be parsed correctly.\"\"\"\n from leann.cli import LeannCLI\n\n cli = LeannCLI()\n parser = cli.create_parser()\n\n args = parser.parse_args([\"-q\", \"list\"])\n assert args.quiet is True\n assert args.verbose is False\n\n def test_verbose_short_flag(self):\n \"\"\"-v should work as shorthand for --verbose.\"\"\"\n from leann.cli import LeannCLI\n\n cli = LeannCLI()\n parser = cli.create_parser()\n\n args = parser.parse_args([\"-v\", \"list\"])\n assert args.verbose is True\n\n def test_verbose_and_quiet_mutually_exclusive(self):\n \"\"\"--verbose and --quiet should be mutually exclusive.\"\"\"\n from leann.cli import LeannCLI\n\n cli = LeannCLI()\n parser = cli.create_parser()\n\n with pytest.raises(SystemExit):\n parser.parse_args([\"--verbose\", \"--quiet\", \"list\"])\n\n def test_default_is_quiet(self):\n \"\"\"Default behavior should be quiet (suppress C++ output).\"\"\"\n from leann.cli import LeannCLI\n\n cli = LeannCLI()\n parser = cli.create_parser()\n\n args = parser.parse_args([\"list\"])\n assert args.verbose is False\n assert args.quiet is False\n # When both are False, we suppress by default\n\n\nclass TestVerbosityIntegration:\n \"\"\"Integration tests for verbosity in commands.\"\"\"\n\n def test_list_command_does_not_suppress(self):\n \"\"\"List command should work without suppression.\"\"\"\n import asyncio\n\n from leann.cli import LeannCLI\n\n cli = LeannCLI()\n parser = cli.create_parser()\n\n # List command should not raise\n args = parser.parse_args([\"list\"])\n asyncio.run(cli.run(args))\n\n def test_verbose_flag_with_list(self):\n \"\"\"Verbose flag should work with list command.\"\"\"\n import asyncio\n\n from leann.cli import LeannCLI\n\n cli = LeannCLI()\n parser = cli.create_parser()\n\n args = parser.parse_args([\"-v\", \"list\"])\n asyncio.run(cli.run(args))\n" }, { "path": "tests/test_cpu_only_install.py", "content": "\"\"\"Packaging metadata checks for CPU-only installs.\"\"\"\n\nfrom pathlib import Path\n\nimport pytest\n\ntry:\n import tomllib\nexcept ModuleNotFoundError: # pragma: no cover - fallback for Python < 3.11\n tomllib = pytest.importorskip(\"tomli\")\n\n\ndef _load_leann_pyproject():\n pyproject_path = Path(__file__).resolve().parents[1] / \"packages\" / \"leann\" / \"pyproject.toml\"\n return tomllib.loads(pyproject_path.read_text())\n\n\ndef _load_leann_core_pyproject():\n pyproject_path = (\n Path(__file__).resolve().parents[1] / \"packages\" / \"leann-core\" / \"pyproject.toml\"\n )\n return tomllib.loads(pyproject_path.read_text())\n\n\ndef test_leann_base_dependencies_include_diskann():\n data = _load_leann_pyproject()\n deps = data[\"project\"].get(\"dependencies\", [])\n\n assert \"leann-core>=0.1.0\" in deps\n assert \"leann-backend-hnsw>=0.1.0\" in deps\n assert \"leann-backend-diskann>=0.1.0\" in deps\n\n\ndef test_leann_core_numpy_pinned_below_2():\n data = _load_leann_core_pyproject()\n deps = data[\"project\"].get(\"dependencies\", [])\n\n assert any(dep.startswith(\"numpy\") and \"<2\" in dep for dep in deps)\n\n\ndef test_leann_core_cpu_extra_pins_cpu_torch():\n data = _load_leann_core_pyproject()\n extras = data[\"project\"].get(\"optional-dependencies\", {})\n\n cpu_deps = extras.get(\"cpu\", [])\n assert cpu_deps, \"cpu extra should be defined\"\n assert any(\n dep.startswith(\"torch\")\n and \"==2.2.2\" in dep\n and \"platform_system == 'Linux'\" in dep\n and \"python_version < '3.13'\" in dep\n for dep in cpu_deps\n )\n\n\ndef test_leann_cpu_extra_defined():\n data = _load_leann_pyproject()\n extras = data[\"project\"].get(\"optional-dependencies\", {})\n\n assert \"cpu\" in extras\n assert \"leann-core[cpu]>=0.1.0\" in extras[\"cpu\"]\n" }, { "path": "tests/test_diskann_partition.py", "content": "\"\"\"\nTest DiskANN graph partitioning functionality.\n\nTests the automatic graph partitioning feature that was implemented to save\nstorage space by partitioning large DiskANN indices and safely deleting\nredundant files while maintaining search functionality.\n\"\"\"\n\nimport os\nimport tempfile\nfrom pathlib import Path\n\nimport pytest\n\n\n@pytest.mark.skipif(\n os.environ.get(\"CI\") == \"true\",\n reason=\"Skip DiskANN partition tests in CI - requires specific hardware and large memory\",\n)\ndef test_diskann_without_partition():\n \"\"\"Test DiskANN index building without partition (baseline).\"\"\"\n from leann.api import LeannBuilder, LeannSearcher\n\n with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as temp_dir:\n index_path = str(Path(temp_dir) / \"test_no_partition.leann\")\n\n texts = [\n f\"Document {i} discusses topic {i % 10} with detailed analysis of subject {i // 10}.\"\n for i in range(500)\n ]\n\n builder = LeannBuilder(\n backend_name=\"diskann\",\n embedding_model=\"facebook/contriever\",\n embedding_mode=\"sentence-transformers\",\n num_neighbors=32,\n search_list_size=50,\n is_recompute=False,\n )\n\n for text in texts:\n builder.add_text(text)\n\n builder.build_index(index_path)\n\n index_dir = Path(index_path).parent\n assert index_dir.exists()\n\n index_prefix = Path(index_path).stem\n required_files = [\n f\"{index_prefix}_disk.index\",\n f\"{index_prefix}_pq_compressed.bin\",\n f\"{index_prefix}_pq_pivots.bin\",\n ]\n\n generated_files = [f.name for f in index_dir.glob(f\"{index_prefix}*\")]\n print(f\"Generated files: {generated_files}\")\n\n for required_file in required_files:\n file_path = index_dir / required_file\n assert file_path.exists(), f\"Required file {required_file} not found\"\n\n partition_files = [f\"{index_prefix}_disk_graph.index\", f\"{index_prefix}_partition.bin\"]\n\n for partition_file in partition_files:\n file_path = index_dir / partition_file\n assert not file_path.exists(), (\n f\"Partition file {partition_file} should not exist in non-partition mode\"\n )\n\n with LeannSearcher(index_path) as searcher:\n results = searcher.search(\"topic 3 analysis\", top_k=3)\n\n assert len(results) > 0\n assert all(\n result.score is not None and result.score != float(\"-inf\") for result in results\n )\n\n\n@pytest.mark.skipif(\n os.environ.get(\"CI\") == \"true\",\n reason=\"Skip DiskANN partition tests in CI - requires specific hardware and large memory\",\n)\ndef test_diskann_with_partition():\n \"\"\"Test DiskANN index building with automatic graph partitioning.\"\"\"\n from leann.api import LeannBuilder\n\n with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as temp_dir:\n index_path = str(Path(temp_dir) / \"test_with_partition.leann\")\n\n texts = [\n f\"Document {i} explores subject {i % 15} with comprehensive coverage of area {i // 15}.\"\n for i in range(500)\n ]\n\n builder = LeannBuilder(\n backend_name=\"diskann\",\n embedding_model=\"facebook/contriever\",\n embedding_mode=\"sentence-transformers\",\n num_neighbors=32,\n search_list_size=50,\n is_recompute=True,\n )\n\n for text in texts:\n builder.add_text(text)\n\n builder.build_index(index_path)\n\n index_dir = Path(index_path).parent\n assert index_dir.exists()\n\n index_prefix = Path(index_path).stem\n partition_files = [\n f\"{index_prefix}_disk_graph.index\",\n f\"{index_prefix}_partition.bin\",\n f\"{index_prefix}_pq_compressed.bin\",\n f\"{index_prefix}_pq_pivots.bin\",\n ]\n\n for partition_file in partition_files:\n file_path = index_dir / partition_file\n assert file_path.exists(), f\"Expected partition file {partition_file} not found\"\n\n large_files = [f\"{index_prefix}_disk.index\", f\"{index_prefix}_disk_beam_search.index\"]\n\n for large_file in large_files:\n file_path = index_dir / large_file\n assert not file_path.exists(), (\n f\"Large file {large_file} should have been deleted for storage saving\"\n )\n\n required_files = [\n f\"{index_prefix}_disk.index_medoids.bin\",\n f\"{index_prefix}_disk.index_max_base_norm.bin\",\n ]\n\n for req_file in required_files:\n file_path = index_dir / req_file\n assert file_path.exists(), (\n f\"Required auxiliary file {req_file} missing for partition mode\"\n )\n\n\n@pytest.mark.skipif(\n os.environ.get(\"CI\") == \"true\",\n reason=\"Skip DiskANN partition tests in CI - requires specific hardware and large memory\",\n)\ndef test_diskann_partition_search_functionality():\n \"\"\"Test that search works correctly with partitioned indices.\"\"\"\n from leann.api import LeannBuilder, LeannSearcher\n\n with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as temp_dir:\n index_path = str(Path(temp_dir) / \"test_partition_search.leann\")\n\n texts = [\n \"LEANN is a storage-efficient approximate nearest neighbor search system.\",\n \"Graph partitioning helps reduce memory usage in large scale vector search.\",\n \"DiskANN provides high-performance disk-based approximate nearest neighbor search.\",\n \"Vector embeddings enable semantic search over unstructured text data.\",\n \"Approximate nearest neighbor algorithms trade accuracy for speed and storage.\",\n ] * 100\n\n builder = LeannBuilder(\n backend_name=\"diskann\",\n embedding_model=\"facebook/contriever\",\n embedding_mode=\"sentence-transformers\",\n is_recompute=True,\n )\n\n for text in texts:\n builder.add_text(text)\n\n builder.build_index(index_path)\n\n with LeannSearcher(index_path) as searcher:\n test_queries = [\n (\"vector search algorithms\", 5),\n (\"LEANN storage efficiency\", 3),\n (\"graph partitioning memory\", 4),\n (\"approximate nearest neighbor\", 7),\n ]\n\n for query, top_k in test_queries:\n results = searcher.search(query, top_k=top_k)\n\n assert len(results) == top_k, f\"Expected {top_k} results for query '{query}'\"\n assert all(result.score is not None for result in results), (\n \"All results should have scores\"\n )\n assert all(result.score != float(\"-inf\") for result in results), (\n \"No result should have -inf score\"\n )\n assert all(result.text is not None for result in results), (\n \"All results should have text\"\n )\n\n scores = [result.score for result in results]\n assert scores == sorted(scores, reverse=True), (\n \"Results should be sorted by score descending\"\n )\n\n\n@pytest.mark.skipif(\n os.environ.get(\"CI\") == \"true\",\n reason=\"Skip DiskANN partition tests in CI - requires specific hardware and large memory\",\n)\ndef test_diskann_medoid_and_norm_files():\n \"\"\"Test that medoid and max_base_norm files are correctly generated and used.\"\"\"\n import struct\n\n from leann.api import LeannBuilder, LeannSearcher\n\n with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as temp_dir:\n index_path = str(Path(temp_dir) / \"test_medoid_norm.leann\")\n\n texts = [f\"Test document {i} with content about subject {i % 10}.\" for i in range(200)]\n\n builder = LeannBuilder(\n backend_name=\"diskann\",\n embedding_model=\"facebook/contriever\",\n embedding_mode=\"sentence-transformers\",\n is_recompute=True,\n )\n\n for text in texts:\n builder.add_text(text)\n\n builder.build_index(index_path)\n\n index_dir = Path(index_path).parent\n index_prefix = Path(index_path).stem\n\n medoids_file = index_dir / f\"{index_prefix}_disk.index_medoids.bin\"\n assert medoids_file.exists(), \"Medoids file should be generated\"\n\n with open(medoids_file, \"rb\") as f:\n nshards = struct.unpack(\"= 0, \"Medoid ID should be valid (not hardcoded 0)\"\n\n norm_file = index_dir / f\"{index_prefix}_disk.index_max_base_norm.bin\"\n assert norm_file.exists(), \"Max base norm file should be generated\"\n\n with open(norm_file, \"rb\") as f:\n npts = struct.unpack(\" 0, \"Norm value should be positive\"\n assert norm_val != float(\"inf\"), \"Norm value should be finite\"\n\n with LeannSearcher(index_path) as searcher:\n results = searcher.search(\"test subject\", top_k=3)\n\n assert len(results) > 0\n assert all(result.score != float(\"-inf\") for result in results), (\n \"Scores should not be -inf when norm file is correct\"\n )\n\n\n@pytest.mark.skipif(\n os.environ.get(\"CI\") == \"true\",\n reason=\"Skip performance comparison in CI - requires significant compute time\",\n)\ndef test_diskann_vs_hnsw_performance():\n \"\"\"Compare DiskANN (with partition) vs HNSW performance.\"\"\"\n import time\n\n from leann.api import LeannBuilder, LeannSearcher\n\n with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as temp_dir:\n texts = [\n f\"Performance test document {i} covering topic {i % 20} in detail.\" for i in range(1000)\n ]\n query = \"performance topic test\"\n\n diskann_path = str(Path(temp_dir) / \"perf_diskann.leann\")\n diskann_builder = LeannBuilder(\n backend_name=\"diskann\",\n embedding_model=\"facebook/contriever\",\n embedding_mode=\"sentence-transformers\",\n is_recompute=True,\n )\n\n for text in texts:\n diskann_builder.add_text(text)\n\n start_time = time.time()\n diskann_builder.build_index(diskann_path)\n\n hnsw_path = str(Path(temp_dir) / \"perf_hnsw.leann\")\n hnsw_builder = LeannBuilder(\n backend_name=\"hnsw\",\n embedding_model=\"facebook/contriever\",\n embedding_mode=\"sentence-transformers\",\n is_recompute=True,\n )\n\n for text in texts:\n hnsw_builder.add_text(text)\n\n start_time = time.time()\n hnsw_builder.build_index(hnsw_path)\n\n with (\n LeannSearcher(diskann_path) as diskann_searcher,\n LeannSearcher(hnsw_path) as hnsw_searcher,\n ):\n diskann_searcher.search(query, top_k=5)\n hnsw_searcher.search(query, top_k=5)\n\n start_time = time.time()\n diskann_results = diskann_searcher.search(query, top_k=10)\n diskann_search_time = time.time() - start_time\n\n start_time = time.time()\n hnsw_results = hnsw_searcher.search(query, top_k=10)\n hnsw_search_time = time.time() - start_time\n\n assert len(diskann_results) == 10\n assert len(hnsw_results) == 10\n assert all(r.score != float(\"-inf\") for r in diskann_results)\n assert all(r.score != float(\"-inf\") for r in hnsw_results)\n\n if hnsw_search_time > 0:\n speed_ratio = hnsw_search_time / diskann_search_time\n print(f\"DiskANN search time: {diskann_search_time:.4f}s\")\n print(f\"HNSW search time: {hnsw_search_time:.4f}s\")\n print(f\"DiskANN is {speed_ratio:.2f}x faster than HNSW\")\n" }, { "path": "tests/test_document_rag.py", "content": "\"\"\"\nTest document_rag functionality using pytest.\n\"\"\"\n\nimport os\nimport subprocess\nimport sys\nimport tempfile\nfrom pathlib import Path\n\nimport pytest\n\n\n@pytest.fixture\ndef test_data_dir():\n \"\"\"Return the path to test data directory.\"\"\"\n return Path(\"data\")\n\n\n@pytest.mark.skipif(\n os.environ.get(\"CI\") == \"true\", reason=\"Skip model tests in CI to avoid MPS memory issues\"\n)\ndef test_document_rag_simulated(test_data_dir):\n \"\"\"Test document_rag with simulated LLM.\"\"\"\n with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as temp_dir:\n # Use a subdirectory that doesn't exist yet to force index creation\n index_dir = Path(temp_dir) / \"test_index\"\n cmd = [\n sys.executable,\n \"apps/document_rag.py\",\n \"--llm\",\n \"simulated\",\n \"--embedding-model\",\n \"facebook/contriever\",\n \"--embedding-mode\",\n \"sentence-transformers\",\n \"--index-dir\",\n str(index_dir),\n \"--data-dir\",\n str(test_data_dir),\n \"--query\",\n \"What is Pride and Prejudice about?\",\n ]\n\n env = os.environ.copy()\n env[\"HF_HUB_DISABLE_SYMLINKS\"] = \"1\"\n env[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n\n result = subprocess.run(cmd, capture_output=True, text=True, timeout=600, env=env)\n\n # Check return code\n assert result.returncode == 0, f\"Command failed: {result.stderr}\"\n\n # Verify output\n output = result.stdout + result.stderr\n assert \"Index saved to\" in output or \"Using existing index\" in output\n assert \"This is a simulated answer\" in output\n\n\n@pytest.mark.skipif(\n os.environ.get(\"CI\") == \"true\",\n reason=\"Skip AST chunking tests in CI to avoid dependency issues\",\n)\ndef test_document_rag_with_ast_chunking(test_data_dir):\n \"\"\"Test document_rag with AST-aware chunking enabled.\"\"\"\n with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as temp_dir:\n # Use a subdirectory that doesn't exist yet to force index creation\n index_dir = Path(temp_dir) / \"test_ast_index\"\n cmd = [\n sys.executable,\n \"apps/document_rag.py\",\n \"--llm\",\n \"simulated\",\n \"--embedding-model\",\n \"facebook/contriever\",\n \"--embedding-mode\",\n \"sentence-transformers\",\n \"--index-dir\",\n str(index_dir),\n \"--data-dir\",\n str(test_data_dir),\n \"--enable-code-chunking\", # Enable AST chunking\n \"--query\",\n \"What is Pride and Prejudice about?\",\n ]\n\n env = os.environ.copy()\n env[\"HF_HUB_DISABLE_SYMLINKS\"] = \"1\"\n env[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n\n result = subprocess.run(cmd, capture_output=True, text=True, timeout=600, env=env)\n\n # Check return code\n assert result.returncode == 0, f\"Command failed: {result.stderr}\"\n\n # Verify output\n output = result.stdout + result.stderr\n assert \"Index saved to\" in output or \"Using existing index\" in output\n assert \"This is a simulated answer\" in output\n\n # Should mention AST chunking if code files are present\n # (might not be relevant for the test data, but command should succeed)\n\n\n@pytest.mark.skipif(not os.environ.get(\"OPENAI_API_KEY\"), reason=\"OpenAI API key not available\")\n@pytest.mark.skipif(\n os.environ.get(\"CI\") == \"true\", reason=\"Skip OpenAI tests in CI to avoid API costs\"\n)\ndef test_document_rag_openai(test_data_dir):\n \"\"\"Test document_rag with OpenAI embeddings.\"\"\"\n with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as temp_dir:\n # Use a subdirectory that doesn't exist yet to force index creation\n index_dir = Path(temp_dir) / \"test_index_openai\"\n cmd = [\n sys.executable,\n \"apps/document_rag.py\",\n \"--llm\",\n \"simulated\", # Use simulated LLM to avoid GPT-4 costs\n \"--embedding-model\",\n \"text-embedding-3-small\",\n \"--embedding-mode\",\n \"openai\",\n \"--index-dir\",\n str(index_dir),\n \"--data-dir\",\n str(test_data_dir),\n \"--query\",\n \"What is Pride and Prejudice about?\",\n ]\n\n env = os.environ.copy()\n env[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n\n result = subprocess.run(cmd, capture_output=True, text=True, timeout=600, env=env)\n\n assert result.returncode == 0, f\"Command failed: {result.stderr}\"\n\n # Verify cosine distance was used\n output = result.stdout + result.stderr\n assert any(\n msg in output\n for msg in [\n \"distance_metric='cosine'\",\n \"Automatically setting distance_metric='cosine'\",\n \"Using cosine distance\",\n ]\n )\n\n\ndef test_document_rag_error_handling(test_data_dir):\n \"\"\"Test document_rag with invalid parameters.\"\"\"\n with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as temp_dir:\n cmd = [\n sys.executable,\n \"apps/document_rag.py\",\n \"--llm\",\n \"invalid_llm_type\",\n \"--index-dir\",\n temp_dir,\n \"--data-dir\",\n str(test_data_dir),\n ]\n\n result = subprocess.run(cmd, capture_output=True, text=True, timeout=60)\n\n # Should fail with invalid LLM type\n assert result.returncode != 0\n assert \"invalid choice\" in result.stderr or \"invalid_llm_type\" in result.stderr\n" }, { "path": "tests/test_embedding_prompt_template.py", "content": "\"\"\"Unit tests for prompt template prepending in OpenAI embeddings.\n\nThis test suite defines the contract for prompt template functionality that allows\nusers to prepend a consistent prompt to all embedding inputs. These tests verify:\n\n1. Template prepending to all input texts before embedding computation\n2. Graceful handling of None/missing provider_options\n3. Empty string template behavior (no-op)\n4. Logging of template application for observability\n5. Template application before token truncation\n\nAll tests are written in Red Phase - they should FAIL initially because the\nimplementation does not exist yet.\n\"\"\"\n\nfrom unittest.mock import MagicMock, Mock, patch\n\nimport numpy as np\nimport pytest\nfrom leann.embedding_compute import compute_embeddings_openai\n\n\nclass TestPromptTemplatePrepending:\n \"\"\"Tests for prompt template prepending in compute_embeddings_openai.\"\"\"\n\n @pytest.fixture\n def mock_openai_client(self):\n \"\"\"Create mock OpenAI client that captures input texts.\"\"\"\n mock_client = MagicMock()\n\n # Mock the embeddings.create response\n mock_response = Mock()\n mock_response.data = [\n Mock(embedding=[0.1, 0.2, 0.3]),\n Mock(embedding=[0.4, 0.5, 0.6]),\n ]\n mock_client.embeddings.create.return_value = mock_response\n\n return mock_client\n\n @pytest.fixture\n def mock_openai_module(self, mock_openai_client, monkeypatch):\n \"\"\"Mock the openai module to return our mock client.\"\"\"\n # Mock the API key environment variable\n monkeypatch.setenv(\"OPENAI_API_KEY\", \"fake-test-key-for-mocking\")\n\n # openai is imported inside the function, so we need to patch it there\n with patch(\"openai.OpenAI\", return_value=mock_openai_client) as mock_openai:\n yield mock_openai\n\n def test_prompt_template_prepended_to_all_texts(self, mock_openai_module, mock_openai_client):\n \"\"\"Verify template is prepended to all input texts.\n\n When provider_options contains \"prompt_template\", that template should\n be prepended to every text in the input list before sending to OpenAI API.\n\n This is the core functionality: the template acts as a consistent prefix\n that provides context or instruction for the embedding model.\n \"\"\"\n texts = [\"First document\", \"Second document\"]\n template = \"search_document: \"\n provider_options = {\"prompt_template\": template}\n\n # Call compute_embeddings_openai with provider_options\n result = compute_embeddings_openai(\n texts=texts,\n model_name=\"text-embedding-3-small\",\n provider_options=provider_options,\n )\n\n # Verify embeddings.create was called with templated texts\n mock_openai_client.embeddings.create.assert_called_once()\n call_args = mock_openai_client.embeddings.create.call_args\n\n # Extract the input texts sent to API\n sent_texts = call_args.kwargs[\"input\"]\n\n # Verify template was prepended to all texts\n assert len(sent_texts) == 2, \"Should send same number of texts\"\n assert sent_texts[0] == \"search_document: First document\", (\n \"Template should be prepended to first text\"\n )\n assert sent_texts[1] == \"search_document: Second document\", (\n \"Template should be prepended to second text\"\n )\n\n # Verify result is valid embeddings array\n assert isinstance(result, np.ndarray)\n assert result.shape == (2, 3), \"Should return correct shape\"\n\n def test_template_not_applied_when_missing_or_empty(\n self, mock_openai_module, mock_openai_client\n ):\n \"\"\"Verify template not applied when provider_options is None, missing key, or empty string.\n\n This consolidated test covers three scenarios where templates should NOT be applied:\n 1. provider_options is None (default behavior)\n 2. provider_options exists but missing 'prompt_template' key\n 3. prompt_template is explicitly set to empty string \"\"\n\n In all cases, texts should be sent to the API unchanged.\n \"\"\"\n # Scenario 1: None provider_options\n texts = [\"Original text one\", \"Original text two\"]\n result = compute_embeddings_openai(\n texts=texts,\n model_name=\"text-embedding-3-small\",\n provider_options=None,\n )\n call_args = mock_openai_client.embeddings.create.call_args\n sent_texts = call_args.kwargs[\"input\"]\n assert sent_texts[0] == \"Original text one\", (\n \"Text should be unchanged with None provider_options\"\n )\n assert sent_texts[1] == \"Original text two\"\n assert isinstance(result, np.ndarray)\n assert result.shape == (2, 3)\n\n # Reset mock for next scenario\n mock_openai_client.reset_mock()\n mock_response = Mock()\n mock_response.data = [\n Mock(embedding=[0.1, 0.2, 0.3]),\n Mock(embedding=[0.4, 0.5, 0.6]),\n ]\n mock_openai_client.embeddings.create.return_value = mock_response\n\n # Scenario 2: Missing 'prompt_template' key\n texts = [\"Text without template\", \"Another text\"]\n provider_options = {\"base_url\": \"https://api.openai.com/v1\"}\n result = compute_embeddings_openai(\n texts=texts,\n model_name=\"text-embedding-3-small\",\n provider_options=provider_options,\n )\n call_args = mock_openai_client.embeddings.create.call_args\n sent_texts = call_args.kwargs[\"input\"]\n assert sent_texts[0] == \"Text without template\", \"Text should be unchanged with missing key\"\n assert sent_texts[1] == \"Another text\"\n assert isinstance(result, np.ndarray)\n\n # Reset mock for next scenario\n mock_openai_client.reset_mock()\n mock_openai_client.embeddings.create.return_value = mock_response\n\n # Scenario 3: Empty string template\n texts = [\"Text one\", \"Text two\"]\n provider_options = {\"prompt_template\": \"\"}\n result = compute_embeddings_openai(\n texts=texts,\n model_name=\"text-embedding-3-small\",\n provider_options=provider_options,\n )\n call_args = mock_openai_client.embeddings.create.call_args\n sent_texts = call_args.kwargs[\"input\"]\n assert sent_texts[0] == \"Text one\", \"Empty template should not modify text\"\n assert sent_texts[1] == \"Text two\"\n assert isinstance(result, np.ndarray)\n\n def test_prompt_template_with_multiple_batches(self, mock_openai_module, mock_openai_client):\n \"\"\"Verify template is prepended in all batches when texts exceed batch size.\n\n OpenAI API has batch size limits. When input texts are split into\n multiple batches, the template should be prepended to texts in every batch.\n\n This ensures consistency across all API calls.\n \"\"\"\n # Create many texts that will be split into multiple batches\n texts = [f\"Document {i}\" for i in range(1000)]\n template = \"passage: \"\n provider_options = {\"prompt_template\": template}\n\n # Mock multiple batch responses\n mock_response = Mock()\n mock_response.data = [Mock(embedding=[0.1, 0.2, 0.3]) for _ in range(1000)]\n mock_openai_client.embeddings.create.return_value = mock_response\n\n result = compute_embeddings_openai(\n texts=texts,\n model_name=\"text-embedding-3-small\",\n provider_options=provider_options,\n )\n\n # Verify embeddings.create was called multiple times (batching)\n assert mock_openai_client.embeddings.create.call_count >= 2, (\n \"Should make multiple API calls for large text list\"\n )\n\n # Verify template was prepended in ALL batches\n for call in mock_openai_client.embeddings.create.call_args_list:\n sent_texts = call.kwargs[\"input\"]\n for text in sent_texts:\n assert text.startswith(template), (\n f\"All texts in all batches should start with template. Got: {text}\"\n )\n\n # Verify result shape\n assert result.shape[0] == 1000, \"Should return embeddings for all texts\"\n\n def test_gemini_openai_compat_caps_batch_size_to_100(\n self, mock_openai_module, mock_openai_client\n ):\n texts = [f\"Doc {i}\" for i in range(250)]\n provider_options = {\"base_url\": \"https://generativelanguage.googleapis.com/v1beta/openai\"}\n\n mock_response = Mock()\n mock_response.data = [Mock(embedding=[0.1, 0.2, 0.3]) for _ in range(250)]\n mock_openai_client.embeddings.create.return_value = mock_response\n\n result = compute_embeddings_openai(\n texts=texts,\n model_name=\"gemini-embedding-001\",\n provider_options=provider_options,\n )\n\n # Should chunk into <=100 inputs per request.\n assert mock_openai_client.embeddings.create.call_count == 3\n batch_sizes = [\n len(call.kwargs[\"input\"])\n for call in mock_openai_client.embeddings.create.call_args_list\n ]\n assert batch_sizes == [100, 100, 50]\n assert result.shape[0] == 250\n\n def test_prompt_template_with_special_characters(self, mock_openai_module, mock_openai_client):\n \"\"\"Verify template with special characters is handled correctly.\n\n Templates may contain special characters, Unicode, newlines, etc.\n These should all be prepended correctly without encoding issues.\n \"\"\"\n texts = [\"Document content\"]\n # Template with various special characters\n template = \"🔍 Search query [EN]: \"\n provider_options = {\"prompt_template\": template}\n\n result = compute_embeddings_openai(\n texts=texts,\n model_name=\"text-embedding-3-small\",\n provider_options=provider_options,\n )\n\n # Verify special characters in template were preserved\n call_args = mock_openai_client.embeddings.create.call_args\n sent_texts = call_args.kwargs[\"input\"]\n\n assert sent_texts[0] == \"🔍 Search query [EN]: Document content\", (\n \"Special characters in template should be preserved\"\n )\n\n assert isinstance(result, np.ndarray)\n\n def test_prompt_template_integration_with_existing_validation(\n self, mock_openai_module, mock_openai_client\n ):\n \"\"\"Verify template works with existing input validation.\n\n compute_embeddings_openai has validation for empty texts and whitespace.\n Template prepending should happen AFTER validation, so validation errors\n are thrown based on original texts, not templated texts.\n\n This ensures users get clear error messages about their input.\n \"\"\"\n # Empty text should still raise ValueError even with template\n texts = [\"\"]\n provider_options = {\"prompt_template\": \"prefix: \"}\n\n with pytest.raises(ValueError, match=\"empty/invalid\"):\n compute_embeddings_openai(\n texts=texts,\n model_name=\"text-embedding-3-small\",\n provider_options=provider_options,\n )\n\n def test_prompt_template_with_api_key_and_base_url(\n self, mock_openai_module, mock_openai_client\n ):\n \"\"\"Verify template works alongside other provider_options.\n\n provider_options may contain multiple settings: prompt_template,\n base_url, api_key. All should work together correctly.\n \"\"\"\n texts = [\"Test document\"]\n provider_options = {\n \"prompt_template\": \"embed: \",\n \"base_url\": \"https://custom.api.com/v1\",\n \"api_key\": \"test-key-123\",\n }\n\n result = compute_embeddings_openai(\n texts=texts,\n model_name=\"text-embedding-3-small\",\n provider_options=provider_options,\n )\n\n # Verify template was applied\n call_args = mock_openai_client.embeddings.create.call_args\n sent_texts = call_args.kwargs[\"input\"]\n assert sent_texts[0] == \"embed: Test document\"\n\n # Verify OpenAI client was created with correct base_url\n mock_openai_module.assert_called()\n client_init_kwargs = mock_openai_module.call_args.kwargs\n assert client_init_kwargs[\"base_url\"] == \"https://custom.api.com/v1\"\n assert client_init_kwargs[\"api_key\"] == \"test-key-123\"\n\n assert isinstance(result, np.ndarray)\n" }, { "path": "tests/test_embedding_server_cli_flags.py", "content": "import os\nimport subprocess\nimport sys\nfrom pathlib import Path\n\n\ndef _run_help(module_name: str) -> str:\n root = Path(__file__).resolve().parents[1]\n extra_paths = [\n str(root / \"packages\" / \"leann-core\" / \"src\"),\n str(root / \"packages\" / \"leann-backend-hnsw\"),\n str(root / \"packages\" / \"leann-backend-diskann\"),\n ]\n env = os.environ.copy()\n existing = env.get(\"PYTHONPATH\", \"\")\n env[\"PYTHONPATH\"] = os.pathsep.join(extra_paths + ([existing] if existing else []))\n\n proc = subprocess.run(\n [sys.executable, \"-m\", module_name, \"--help\"],\n check=True,\n text=True,\n capture_output=True,\n env=env,\n )\n return proc.stdout\n\n\ndef test_hnsw_server_help_has_daemon_and_warmup_flags():\n out = _run_help(\"leann_backend_hnsw.hnsw_embedding_server\")\n assert \"--enable-warmup\" in out\n assert \"--daemon-mode\" in out\n assert \"--daemon-ttl\" in out\n\n\ndef test_diskann_server_help_has_daemon_and_warmup_flags():\n out = _run_help(\"leann_backend_diskann.diskann_embedding_server\")\n assert \"--enable-warmup\" in out\n assert \"--daemon-mode\" in out\n assert \"--daemon-ttl\" in out\n" }, { "path": "tests/test_embedding_server_manager.py", "content": "import json\nimport os\nimport threading\nimport time\nfrom typing import Any, cast\n\nimport pytest\nfrom leann.embedding_server_manager import EmbeddingServerManager\n\n\nclass DummyProcess:\n def __init__(self, pid=12345):\n self.pid = pid\n self._terminated = False\n\n def poll(self):\n return 0 if self._terminated else None\n\n def terminate(self):\n self._terminated = True\n\n def kill(self):\n self._terminated = True\n\n def wait(self, timeout=None):\n self._terminated = True\n return 0\n\n\n@pytest.fixture\ndef embedding_manager(monkeypatch):\n manager = EmbeddingServerManager(\"leann_backend_hnsw.hnsw_embedding_server\")\n\n def fake_get_available_port(start_port):\n return start_port\n\n monkeypatch.setattr(\n \"leann.embedding_server_manager._get_available_port\",\n fake_get_available_port,\n )\n\n start_calls = []\n\n def fake_start_new_server(self, port, model_name, embedding_mode, **kwargs):\n config_signature = kwargs.get(\"config_signature\")\n start_calls.append(config_signature)\n self.server_process = DummyProcess()\n self.server_port = port\n self._server_config = config_signature\n return True, port\n\n monkeypatch.setattr(\n EmbeddingServerManager,\n \"_start_new_server\",\n fake_start_new_server,\n )\n\n # Ensure stop_server doesn't try to operate on real subprocesses\n def fake_stop_server(self):\n self.server_process = None\n self.server_port = None\n self._server_config = None\n\n monkeypatch.setattr(EmbeddingServerManager, \"stop_server\", fake_stop_server)\n\n return manager, start_calls\n\n\ndef _write_meta(meta_path, passages_name, index_name, total):\n meta_path.write_text(\n json.dumps(\n {\n \"backend_name\": \"hnsw\",\n \"embedding_model\": \"test-model\",\n \"embedding_mode\": \"sentence-transformers\",\n \"dimensions\": 3,\n \"backend_kwargs\": {},\n \"passage_sources\": [\n {\n \"type\": \"jsonl\",\n \"path\": passages_name,\n \"index_path\": index_name,\n }\n ],\n \"total_passages\": total,\n }\n ),\n encoding=\"utf-8\",\n )\n\n\ndef test_server_restarts_when_metadata_changes(tmp_path, embedding_manager):\n manager, start_calls = embedding_manager\n\n meta_path = tmp_path / \"example.meta.json\"\n passages_path = tmp_path / \"example.passages.jsonl\"\n index_path = tmp_path / \"example.passages.idx\"\n\n passages_path.write_text(\"first\\n\", encoding=\"utf-8\")\n index_path.write_bytes(b\"index\")\n _write_meta(meta_path, passages_path.name, index_path.name, total=1)\n\n # Initial start populates signature\n ok, port = manager.start_server(\n port=6000,\n model_name=\"test-model\",\n passages_file=str(meta_path),\n use_daemon=False,\n )\n assert ok\n assert port == 6000\n assert len(start_calls) == 1\n\n initial_signature = start_calls[0][\"passages_signature\"]\n\n # No metadata change => reuse existing server\n ok, port_again = manager.start_server(\n port=6000,\n model_name=\"test-model\",\n passages_file=str(meta_path),\n use_daemon=False,\n )\n assert ok\n assert port_again == 6000\n assert len(start_calls) == 1\n\n # Modify passage data and metadata to force signature change\n time.sleep(0.01) # Ensure filesystem timestamps move forward\n passages_path.write_text(\"second\\n\", encoding=\"utf-8\")\n _write_meta(meta_path, passages_path.name, index_path.name, total=2)\n\n ok, port_third = manager.start_server(\n port=6000,\n model_name=\"test-model\",\n passages_file=str(meta_path),\n use_daemon=False,\n )\n assert ok\n assert port_third == 6000\n assert len(start_calls) == 2\n\n updated_signature = start_calls[1][\"passages_signature\"]\n assert updated_signature != initial_signature\n\n\ndef test_list_daemons_ignores_stale_records(tmp_path, monkeypatch):\n registry_dir = tmp_path / \"servers\"\n registry_dir.mkdir()\n monkeypatch.setattr(EmbeddingServerManager, \"_registry_dir\", staticmethod(lambda: registry_dir))\n\n stale = registry_dir / \"stale.json\"\n stale.write_text(\n json.dumps(\n {\n \"pid\": 999999,\n \"port\": 65531,\n \"backend_module_name\": \"leann_backend_hnsw.hnsw_embedding_server\",\n \"config_signature\": {\"passages_file\": \"/tmp/a.meta.json\"},\n }\n ),\n encoding=\"utf-8\",\n )\n\n records = EmbeddingServerManager.list_daemons()\n assert records == []\n assert not stale.exists()\n\n\ndef test_stop_daemons_filters_by_backend_and_passages(tmp_path, monkeypatch):\n registry_dir = tmp_path / \"servers\"\n registry_dir.mkdir()\n monkeypatch.setattr(EmbeddingServerManager, \"_registry_dir\", staticmethod(lambda: registry_dir))\n\n meta_path = (tmp_path / \"x.meta.json\").resolve()\n record = registry_dir / \"daemon.json\"\n record.write_text(\n json.dumps(\n {\n \"pid\": 12345,\n \"port\": 6001,\n \"backend_module_name\": \"leann_backend_hnsw.hnsw_embedding_server\",\n \"config_signature\": {\"passages_file\": str(meta_path)},\n }\n ),\n encoding=\"utf-8\",\n )\n\n monkeypatch.setattr(\n EmbeddingServerManager,\n \"list_daemons\",\n classmethod(\n lambda cls: [\n {\n \"pid\": 12345,\n \"port\": 6001,\n \"backend_module_name\": \"leann_backend_hnsw.hnsw_embedding_server\",\n \"config_signature\": {\"passages_file\": str(meta_path)},\n \"record_path\": str(record),\n }\n ]\n ),\n )\n\n killed: list[tuple[int, int]] = []\n\n def fake_kill(pid: int, sig: int):\n killed.append((pid, sig))\n\n monkeypatch.setattr(os, \"kill\", fake_kill)\n\n stopped = EmbeddingServerManager.stop_daemons(\n backend_module_name=\"leann_backend_hnsw.hnsw_embedding_server\",\n passages_file=str(meta_path),\n )\n assert stopped == 1\n assert killed == [(12345, 15)]\n assert not record.exists()\n\n\ndef test_daemon_registry_reuse_across_manager_instances(tmp_path, monkeypatch):\n \"\"\"Second manager should adopt the daemon started by first manager.\"\"\"\n registry_dir = tmp_path / \"servers\"\n registry_dir.mkdir()\n monkeypatch.setattr(EmbeddingServerManager, \"_registry_dir\", staticmethod(lambda: registry_dir))\n monkeypatch.setattr(\n \"leann.embedding_server_manager._get_available_port\",\n lambda start_port: start_port,\n )\n monkeypatch.setattr(\"leann.embedding_server_manager._check_port\", lambda port: True)\n monkeypatch.setattr(\"leann.embedding_server_manager._pid_is_alive\", lambda pid: pid == 22222)\n\n starts = []\n\n def fake_start_new_server(self, port, model_name, embedding_mode, **kwargs):\n starts.append((port, model_name, embedding_mode))\n self.server_process = DummyProcess(pid=22222)\n self.server_port = port\n self._server_config = kwargs.get(\"config_signature\")\n return True, port\n\n monkeypatch.setattr(EmbeddingServerManager, \"_start_new_server\", fake_start_new_server)\n\n manager1 = EmbeddingServerManager(\"leann_backend_hnsw.hnsw_embedding_server\")\n ok1, port1 = manager1.start_server(\n port=6011,\n model_name=\"test-model\",\n use_daemon=True,\n daemon_ttl_seconds=120,\n )\n assert ok1 and port1 == 6011\n assert len(starts) == 1\n\n manager2 = EmbeddingServerManager(\"leann_backend_hnsw.hnsw_embedding_server\")\n ok2, port2 = manager2.start_server(\n port=6011,\n model_name=\"test-model\",\n use_daemon=True,\n daemon_ttl_seconds=120,\n )\n assert ok2 and port2 == 6011\n # No second process spawn: adopted from registry.\n assert len(starts) == 1\n assert manager2.server_process is None\n\n\ndef test_stale_registry_falls_back_to_fresh_start(tmp_path, monkeypatch):\n \"\"\"If registry points to dead daemon, manager should start a new process.\"\"\"\n registry_dir = tmp_path / \"servers\"\n registry_dir.mkdir()\n monkeypatch.setattr(EmbeddingServerManager, \"_registry_dir\", staticmethod(lambda: registry_dir))\n monkeypatch.setattr(\n \"leann.embedding_server_manager._get_available_port\",\n lambda start_port: start_port,\n )\n\n starts = []\n\n def fake_start_new_server(self, port, model_name, embedding_mode, **kwargs):\n starts.append(port)\n self.server_process = DummyProcess(pid=33333)\n self.server_port = port\n self._server_config = kwargs.get(\"config_signature\")\n return True, port\n\n monkeypatch.setattr(EmbeddingServerManager, \"_start_new_server\", fake_start_new_server)\n\n manager = EmbeddingServerManager(\"leann_backend_hnsw.hnsw_embedding_server\")\n signature = manager._build_config_signature(\n model_name=\"test-model\",\n embedding_mode=\"sentence-transformers\",\n provider_options=None,\n passages_file=None,\n distance_metric=None,\n )\n stale_file = registry_dir / f\"{manager._registry_key(signature)}.json\"\n stale_file.write_text(\n json.dumps(\n {\n \"pid\": 999999,\n \"port\": 6012,\n \"backend_module_name\": \"leann_backend_hnsw.hnsw_embedding_server\",\n \"config_signature\": signature,\n }\n ),\n encoding=\"utf-8\",\n )\n\n monkeypatch.setattr(\"leann.embedding_server_manager._pid_is_alive\", lambda pid: False)\n monkeypatch.setattr(\"leann.embedding_server_manager._check_port\", lambda port: False)\n\n ok, port = manager.start_server(\n port=6012,\n model_name=\"test-model\",\n use_daemon=True,\n )\n assert ok and port == 6012\n assert starts == [6012]\n assert stale_file.exists()\n refreshed = json.loads(stale_file.read_text(encoding=\"utf-8\"))\n assert refreshed[\"pid\"] == 33333\n\n\ndef test_build_server_command_includes_daemon_and_warmup_flags():\n manager = EmbeddingServerManager(\"leann_backend_hnsw.hnsw_embedding_server\")\n command = manager._build_server_command(\n port=6020,\n model_name=\"m\",\n embedding_mode=\"sentence-transformers\",\n distance_metric=\"mips\",\n enable_warmup=True,\n use_daemon=True,\n daemon_ttl_seconds=321,\n )\n assert \"--enable-warmup\" in command\n assert \"--daemon-mode\" in command\n assert \"--daemon-ttl\" in command\n ttl_idx = command.index(\"--daemon-ttl\")\n assert command[ttl_idx + 1] == \"321\"\n\n command_no_daemon = manager._build_server_command(\n port=6020,\n model_name=\"m\",\n embedding_mode=\"sentence-transformers\",\n enable_warmup=False,\n use_daemon=False,\n )\n assert \"--daemon-mode\" not in command_no_daemon\n assert \"--enable-warmup\" not in command_no_daemon\n\n\ndef test_corrupted_registry_file_is_recovered_on_start(tmp_path, monkeypatch):\n \"\"\"Invalid registry json should not block startup; file is replaced.\"\"\"\n registry_dir = tmp_path / \"servers\"\n registry_dir.mkdir()\n monkeypatch.setattr(EmbeddingServerManager, \"_registry_dir\", staticmethod(lambda: registry_dir))\n monkeypatch.setattr(\n \"leann.embedding_server_manager._get_available_port\",\n lambda start_port: start_port,\n )\n\n starts = []\n\n def fake_start_new_server(self, port, model_name, embedding_mode, **kwargs):\n starts.append(port)\n self.server_process = DummyProcess(pid=44444)\n self.server_port = port\n self._server_config = kwargs.get(\"config_signature\")\n return True, port\n\n monkeypatch.setattr(EmbeddingServerManager, \"_start_new_server\", fake_start_new_server)\n\n manager = EmbeddingServerManager(\"leann_backend_hnsw.hnsw_embedding_server\")\n signature = manager._build_config_signature(\n model_name=\"test-model\",\n embedding_mode=\"sentence-transformers\",\n provider_options=None,\n passages_file=None,\n distance_metric=None,\n )\n record_path = registry_dir / f\"{manager._registry_key(signature)}.json\"\n record_path.write_text(\"{invalid-json\", encoding=\"utf-8\")\n\n monkeypatch.setattr(\"leann.embedding_server_manager._pid_is_alive\", lambda pid: False)\n monkeypatch.setattr(\"leann.embedding_server_manager._check_port\", lambda port: False)\n\n ok, port = manager.start_server(port=6030, model_name=\"test-model\", use_daemon=True)\n assert ok and port == 6030\n assert starts == [6030]\n data = json.loads(record_path.read_text(encoding=\"utf-8\"))\n assert data[\"pid\"] == 44444\n\n\ndef test_stop_server_detaches_when_daemon_mode(monkeypatch):\n \"\"\"Daemon mode should detach manager without terminating shared process.\"\"\"\n manager = EmbeddingServerManager(\"leann_backend_hnsw.hnsw_embedding_server\")\n manager.server_process = cast(Any, DummyProcess(pid=55555))\n manager.server_port = 6031\n manager._server_config = {\"model_name\": \"m\"}\n manager._daemon_mode = True\n\n # If terminate is called this test should fail.\n called = {\"terminate\": 0}\n\n def fail_terminate():\n called[\"terminate\"] += 1\n raise AssertionError(\"terminate should not be called in daemon detach path\")\n\n manager.server_process.terminate = fail_terminate # type: ignore[method-assign]\n\n manager.stop_server()\n assert called[\"terminate\"] == 0\n assert manager.server_process is None\n assert manager.server_port is None\n assert manager._server_config is None\n\n\ndef test_concurrent_daemon_start_only_spawns_once(tmp_path, monkeypatch):\n \"\"\"Concurrent calls should be serialized by registry lock for same signature.\"\"\"\n registry_dir = tmp_path / \"servers\"\n registry_dir.mkdir()\n monkeypatch.setattr(EmbeddingServerManager, \"_registry_dir\", staticmethod(lambda: registry_dir))\n monkeypatch.setattr(\n \"leann.embedding_server_manager._get_available_port\",\n lambda start_port: start_port,\n )\n monkeypatch.setattr(\"leann.embedding_server_manager._check_port\", lambda port: True)\n monkeypatch.setattr(\"leann.embedding_server_manager._pid_is_alive\", lambda pid: pid in (77777,))\n\n starts = []\n\n def fake_start_new_server(self, port, model_name, embedding_mode, **kwargs):\n # Force overlap window between two starters.\n time.sleep(0.05)\n starts.append((self, port))\n self.server_process = DummyProcess(pid=77777)\n self.server_port = port\n self._server_config = kwargs.get(\"config_signature\")\n return True, port\n\n monkeypatch.setattr(EmbeddingServerManager, \"_start_new_server\", fake_start_new_server)\n\n results = []\n\n def runner():\n manager = EmbeddingServerManager(\"leann_backend_hnsw.hnsw_embedding_server\")\n ok, port = manager.start_server(port=6040, model_name=\"test-model\", use_daemon=True)\n results.append((ok, port))\n\n t1 = threading.Thread(target=runner)\n t2 = threading.Thread(target=runner)\n t1.start()\n t2.start()\n t1.join(timeout=5)\n t2.join(timeout=5)\n\n assert len(results) == 2\n assert all(ok and port == 6040 for ok, port in results)\n # Exactly one actual process start, one adopts registry record.\n assert len(starts) == 1\n\n\ndef test_registry_record_write_is_atomic(tmp_path, monkeypatch):\n \"\"\"Registry writes should go through temp file + os.replace.\"\"\"\n registry_dir = tmp_path / \"servers\"\n registry_dir.mkdir()\n monkeypatch.setattr(EmbeddingServerManager, \"_registry_dir\", staticmethod(lambda: registry_dir))\n\n manager = EmbeddingServerManager(\"leann_backend_hnsw.hnsw_embedding_server\")\n manager.server_process = cast(Any, DummyProcess(pid=88888))\n\n calls = []\n real_replace = os.replace\n\n def tracked_replace(src, dst):\n calls.append((str(src), str(dst)))\n return real_replace(src, dst)\n\n monkeypatch.setattr(os, \"replace\", tracked_replace)\n\n path = manager._write_registry_record(\n port=6060,\n config_signature={\"model_name\": \"m\"},\n daemon_ttl_seconds=123,\n )\n assert path.exists()\n assert calls, \"os.replace should be used for atomic write\"\n src, dst = calls[0]\n assert src.endswith(\".json.tmp\")\n assert dst.endswith(\".json\")\n\n\ndef test_stale_lock_info_removed_when_pid_dead(tmp_path, monkeypatch):\n registry_dir = tmp_path / \"servers\"\n registry_dir.mkdir()\n monkeypatch.setattr(EmbeddingServerManager, \"_registry_dir\", staticmethod(lambda: registry_dir))\n monkeypatch.setattr(\"leann.embedding_server_manager._pid_is_alive\", lambda pid: False)\n\n manager = EmbeddingServerManager(\"leann_backend_hnsw.hnsw_embedding_server\")\n signature = manager._build_config_signature(\n model_name=\"x\",\n embedding_mode=\"sentence-transformers\",\n provider_options=None,\n passages_file=None,\n distance_metric=None,\n )\n key = manager._registry_key(signature)\n lock_info = registry_dir / f\"{key}.lockinfo.json\"\n lock_info.write_text(json.dumps({\"pid\": 999999, \"ts\": time.time()}), encoding=\"utf-8\")\n\n with manager._registry_lock(signature):\n pass\n\n assert not lock_info.exists()\n" }, { "path": "tests/test_embedding_server_manager_e2e.py", "content": "import os\nimport time\nfrom pathlib import Path\n\nfrom leann.embedding_server_manager import EmbeddingServerManager\n\n\ndef _configure_fake_module_env(monkeypatch):\n root = Path(__file__).resolve().parents[1]\n support_path = str(root / \"tests\" / \"support\")\n existing = os.environ.get(\"PYTHONPATH\", \"\")\n joined = support_path if not existing else f\"{support_path}{os.pathsep}{existing}\"\n monkeypatch.setenv(\"PYTHONPATH\", joined)\n\n\ndef _wait_until(predicate, timeout=5.0, interval=0.05):\n deadline = time.time() + timeout\n while time.time() < deadline:\n if predicate():\n return True\n time.sleep(interval)\n return False\n\n\ndef test_daemon_reuse_with_real_subprocess(tmp_path, monkeypatch):\n _configure_fake_module_env(monkeypatch)\n monkeypatch.setattr(\n EmbeddingServerManager, \"_registry_dir\", staticmethod(lambda: tmp_path / \"servers\")\n )\n\n manager1 = EmbeddingServerManager(\"fake_embedding_server_module\")\n ok1, port1 = manager1.start_server(\n port=6151,\n model_name=\"fake-model\",\n use_daemon=True,\n daemon_ttl_seconds=60,\n enable_warmup=True,\n )\n assert ok1 and port1 == 6151\n assert manager1.server_process is not None\n pid1 = manager1.server_process.pid\n\n manager2 = EmbeddingServerManager(\"fake_embedding_server_module\")\n ok2, port2 = manager2.start_server(\n port=6151,\n model_name=\"fake-model\",\n use_daemon=True,\n daemon_ttl_seconds=60,\n )\n assert ok2 and port2 == 6151\n # Adoption path: no newly spawned process object attached.\n assert manager2.server_process is None\n\n records = EmbeddingServerManager.list_daemons()\n assert len(records) == 1\n assert int(records[0][\"pid\"]) == pid1\n\n stopped = EmbeddingServerManager.stop_daemons(\n backend_module_name=\"fake_embedding_server_module\"\n )\n assert stopped == 1\n\n\ndef test_daemon_ttl_expiry_with_real_subprocess(tmp_path, monkeypatch):\n _configure_fake_module_env(monkeypatch)\n monkeypatch.setattr(\n EmbeddingServerManager, \"_registry_dir\", staticmethod(lambda: tmp_path / \"servers\")\n )\n\n manager = EmbeddingServerManager(\"fake_embedding_server_module\")\n ok, port = manager.start_server(\n port=6152,\n model_name=\"fake-model\",\n use_daemon=True,\n daemon_ttl_seconds=1,\n enable_warmup=False,\n )\n assert ok and port == 6152\n\n # Fake daemon should self-exit after idle TTL.\n expired = _wait_until(lambda: len(EmbeddingServerManager.list_daemons()) == 0, timeout=4.0)\n assert expired\n" }, { "path": "tests/test_hybrid_search.py", "content": "\"\"\"\nComprehensive tests for hybrid search functionality.\n\nThis module tests the hybrid search feature that combines vector search\nwith BM25 keyword search using the gemma parameter.\n\"\"\"\n\nimport os\nimport tempfile\nfrom pathlib import Path\n\nimport pytest\n\n\n@pytest.mark.skipif(\n os.environ.get(\"CI\") == \"true\", reason=\"Skip model tests in CI to avoid MPS memory issues\"\n)\nclass TestHybridSearch:\n \"\"\"Test suite for hybrid search functionality.\"\"\"\n\n @pytest.fixture\n def sample_index(self):\n \"\"\"Create a sample index for testing.\"\"\"\n from leann.api import LeannBuilder, LeannSearcher\n\n with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as temp_dir:\n index_path = str(Path(temp_dir) / \"test_hybrid.hnsw\")\n\n # Create documents with diverse content for testing\n # Some documents are keyword-rich, others are semantically similar\n texts = [\n \"The quick brown fox jumps over the lazy dog\",\n \"A fast auburn canine leaps above a sleepy hound\", # Semantically similar to first\n \"Python programming language is great for data science\",\n \"Machine learning and artificial intelligence are transforming technology\",\n \"The weather today is sunny and warm\",\n \"Climate conditions are pleasant with clear skies\", # Semantically similar to weather\n \"Database management systems store and retrieve data efficiently\",\n \"SQL queries help extract information from databases\", # Related to databases\n \"Cooking recipes require precise measurements and timing\",\n \"Baking bread needs flour water yeast and patience\", # Related to cooking\n ]\n\n builder = LeannBuilder(\n backend_name=\"hnsw\",\n embedding_model=\"facebook/contriever\",\n embedding_mode=\"sentence-transformers\",\n M=16,\n efConstruction=200,\n )\n\n for i, text in enumerate(texts):\n builder.add_text(text, metadata={\"id\": str(i), \"doc_num\": i})\n\n builder.build_index(index_path)\n\n searcher = LeannSearcher(index_path)\n yield searcher, texts\n\n searcher.cleanup()\n\n def test_pure_vector_search(self, sample_index):\n \"\"\"Test pure vector search (gemma=1.0, default).\"\"\"\n searcher, texts = sample_index\n\n # Search with gemma=1.0 (pure vector search)\n results = searcher.search(\"canine animal\", top_k=3, gemma=1.0)\n\n assert len(results) > 0\n assert len(results) <= 3\n # Should find semantically similar documents about animals/dogs\n assert any(\n \"fox\" in r.text.lower() or \"dog\" in r.text.lower() or \"canine\" in r.text.lower()\n for r in results\n )\n\n def test_pure_keyword_search(self, sample_index):\n \"\"\"Test pure keyword search (gemma=0.0).\"\"\"\n searcher, texts = sample_index\n\n # Search with gemma=0.0 (pure BM25 keyword search)\n results = searcher.search(\"database SQL\", top_k=3, gemma=0.0)\n\n assert len(results) > 0\n assert len(results) <= 3\n # Should find documents with exact keyword matches\n # BM25 should prioritize documents containing \"database\" or \"SQL\"\n top_result_text = results[0].text.lower()\n assert \"database\" in top_result_text or \"sql\" in top_result_text\n\n def test_hybrid_search_balanced(self, sample_index):\n \"\"\"Test balanced hybrid search (gemma=0.5).\"\"\"\n searcher, texts = sample_index\n\n # Search with gemma=0.5 (balanced hybrid)\n results = searcher.search(\"programming Python code\", top_k=5, gemma=0.5)\n\n assert len(results) > 0\n assert len(results) <= 5\n # Should combine both semantic and keyword matching\n # At least one result should contain \"Python\" or \"programming\"\n assert any(\"python\" in r.text.lower() or \"programming\" in r.text.lower() for r in results)\n\n def test_hybrid_search_vector_heavy(self, sample_index):\n \"\"\"Test vector-heavy hybrid search (gemma=0.8).\"\"\"\n searcher, texts = sample_index\n\n # Search with gemma=0.8 (mostly vector, some keyword)\n results = searcher.search(\"sunny weather conditions\", top_k=3, gemma=0.8)\n\n assert len(results) > 0\n # Should prioritize semantic similarity but consider keywords\n # Should find weather-related documents\n assert any(\n \"weather\" in r.text.lower() or \"sunny\" in r.text.lower() or \"climate\" in r.text.lower()\n for r in results\n )\n\n def test_hybrid_search_keyword_heavy(self, sample_index):\n \"\"\"Test keyword-heavy hybrid search (gemma=0.2).\"\"\"\n searcher, texts = sample_index\n\n # Search with gemma=0.2 (mostly keyword, some vector)\n results = searcher.search(\"bread flour baking\", top_k=3, gemma=0.2)\n\n assert len(results) > 0\n # Should prioritize keyword matches\n # Should find documents with exact keyword matches\n top_results_text = \" \".join([r.text.lower() for r in results[:2]])\n assert (\n \"bread\" in top_results_text\n or \"flour\" in top_results_text\n or \"baking\" in top_results_text\n )\n\n def test_hybrid_search_score_combination(self, sample_index):\n \"\"\"Test that hybrid search properly combines scores.\"\"\"\n searcher, texts = sample_index\n\n # Get results with different gemma values\n pure_vector = searcher.search(\"machine learning AI\", top_k=5, gemma=1.0)\n pure_keyword = searcher.search(\"machine learning AI\", top_k=5, gemma=0.0)\n hybrid = searcher.search(\"machine learning AI\", top_k=5, gemma=0.5)\n\n # All should return results\n assert len(pure_vector) > 0\n assert len(pure_keyword) > 0\n assert len(hybrid) > 0\n\n # Hybrid results should potentially differ from pure approaches\n # (though with small dataset, there might be overlap)\n assert all(r.score > 0 for r in hybrid)\n\n def test_hybrid_search_with_metadata_filters(self, sample_index):\n \"\"\"Test hybrid search combined with metadata filtering.\"\"\"\n searcher, texts = sample_index\n\n # Search with hybrid and metadata filter\n results = searcher.search(\n \"data information\", top_k=5, gemma=0.6, metadata_filters={\"doc_num\": {\"<\": 8}}\n )\n\n assert len(results) > 0\n # All results should satisfy the metadata filter\n for r in results:\n assert r.metadata.get(\"doc_num\", 999) < 8\n" }, { "path": "tests/test_incremental_build.py", "content": "\"\"\"\nTests for incremental build (Feature #89).\n\nWhen an index already exists and build is run again without --force,\nonly new files are indexed and appended to the existing index (HNSW, non-compact only).\nChange detection uses content-hash (merkle tree) via FileSynchronizer.\n\"\"\"\n\nimport os\nfrom pathlib import Path\n\nimport pytest\nfrom leann.cli import _normalize_path\nfrom leann.sync import FileSynchronizer\n\n\ndef test_normalize_path():\n assert _normalize_path(\"\") == \"\"\n assert _normalize_path(\"/a/b\") == \"/a/b\" or \"a\" in _normalize_path(\"/a/b\")\n rel = \"foo/bar\"\n out = _normalize_path(rel)\n assert Path(out).is_absolute() or out == rel\n\n\ndef test_file_synchronizer_detect_changes(tmp_path):\n \"\"\"FileSynchronizer detect_changes returns all files as added when no snapshot exists.\"\"\"\n docs = tmp_path / \"docs\"\n docs.mkdir()\n (docs / \"a.txt\").write_text(\"hello\", encoding=\"utf-8\")\n snapshot = str(tmp_path / \"test.pickle\")\n fs = FileSynchronizer(root_dir=str(docs), snapshot_path=snapshot)\n added, removed, modified = fs.detect_changes()\n assert len(added) == 1\n assert len(removed) == 0\n assert len(modified) == 0\n\n\ndef test_file_synchronizer_no_changes_after_commit(tmp_path):\n \"\"\"After commit, detect_changes should report no changes if files haven't changed.\"\"\"\n docs = tmp_path / \"docs\"\n docs.mkdir()\n (docs / \"a.txt\").write_text(\"hello\", encoding=\"utf-8\")\n snapshot = str(tmp_path / \"test.pickle\")\n fs = FileSynchronizer(root_dir=str(docs), snapshot_path=snapshot)\n fs.detect_changes()\n fs.commit()\n\n fs2 = FileSynchronizer(root_dir=str(docs), snapshot_path=snapshot)\n added, removed, modified = fs2.detect_changes()\n assert not added and not removed and not modified\n\n\ndef test_file_synchronizer_detects_new_file(tmp_path):\n \"\"\"Adding a file after commit should be detected as added.\"\"\"\n docs = tmp_path / \"docs\"\n docs.mkdir()\n (docs / \"a.txt\").write_text(\"hello\", encoding=\"utf-8\")\n snapshot = str(tmp_path / \"test.pickle\")\n fs = FileSynchronizer(root_dir=str(docs), snapshot_path=snapshot)\n fs.detect_changes()\n fs.commit()\n\n (docs / \"b.txt\").write_text(\"world\", encoding=\"utf-8\")\n fs2 = FileSynchronizer(root_dir=str(docs), snapshot_path=snapshot)\n added, removed, modified = fs2.detect_changes()\n assert len(added) == 1\n assert len(removed) == 0\n assert len(modified) == 0\n\n\ndef test_file_synchronizer_detects_modification(tmp_path):\n \"\"\"Changing file content should be detected as modified.\"\"\"\n docs = tmp_path / \"docs\"\n docs.mkdir()\n (docs / \"a.txt\").write_text(\"hello\", encoding=\"utf-8\")\n snapshot = str(tmp_path / \"test.pickle\")\n fs = FileSynchronizer(root_dir=str(docs), snapshot_path=snapshot)\n fs.detect_changes()\n fs.commit()\n\n (docs / \"a.txt\").write_text(\"changed\", encoding=\"utf-8\")\n fs2 = FileSynchronizer(root_dir=str(docs), snapshot_path=snapshot)\n added, removed, modified = fs2.detect_changes()\n assert len(modified) == 1\n assert len(added) == 0\n\n\ndef test_file_synchronizer_touch_no_false_positive(tmp_path):\n \"\"\"Touching a file (mtime change, same content) should NOT report as modified.\"\"\"\n docs = tmp_path / \"docs\"\n docs.mkdir()\n f = docs / \"a.txt\"\n f.write_text(\"hello\", encoding=\"utf-8\")\n snapshot = str(tmp_path / \"test.pickle\")\n fs = FileSynchronizer(root_dir=str(docs), snapshot_path=snapshot)\n fs.detect_changes()\n fs.commit()\n\n os.utime(f, None)\n fs2 = FileSynchronizer(root_dir=str(docs), snapshot_path=snapshot)\n added, removed, modified = fs2.detect_changes()\n assert not added and not removed and not modified\n\n\n@pytest.mark.skipif(\n os.environ.get(\"CI\") == \"true\",\n reason=\"Skip in CI to avoid embedding/model load\",\n)\ndef test_incremental_build_adds_only_new_files(tmp_path):\n \"\"\"Build once with one file, add a second file, run build again without --force; index grows.\"\"\"\n import asyncio\n\n from leann.api import LeannSearcher\n from leann.cli import LeannCLI\n\n docs_dir = tmp_path / \"docs\"\n docs_dir.mkdir()\n (docs_dir / \"a.txt\").write_text(\"First document content for indexing.\", encoding=\"utf-8\")\n\n cli = LeannCLI()\n cli.indexes_dir = tmp_path / \".leann\" / \"indexes\"\n cli.indexes_dir.mkdir(parents=True, exist_ok=True)\n index_name = \"incr_test\"\n index_dir = cli.indexes_dir / index_name\n index_path = cli.get_index_path(index_name)\n\n parser = cli.create_parser()\n args = parser.parse_args(\n [\n \"build\",\n index_name,\n \"--docs\",\n str(docs_dir),\n \"--backend-name\",\n \"hnsw\",\n \"--no-compact\",\n \"--embedding-model\",\n \"all-MiniLM-L6-v2\",\n \"--embedding-mode\",\n \"sentence-transformers\",\n \"--force\",\n ]\n )\n asyncio.run(cli.build_index(args))\n assert index_dir.exists()\n assert (index_dir / \"documents.leann.meta.json\").exists()\n\n # Add second file\n (docs_dir / \"b.txt\").write_text(\"Second document content.\", encoding=\"utf-8\")\n\n # Build again without --force (incremental)\n args2 = parser.parse_args(\n [\n \"build\",\n index_name,\n \"--docs\",\n str(docs_dir),\n \"--backend-name\",\n \"hnsw\",\n \"--no-compact\",\n \"--embedding-model\",\n \"all-MiniLM-L6-v2\",\n \"--embedding-mode\",\n \"sentence-transformers\",\n ]\n )\n asyncio.run(cli.build_index(args2))\n\n # Index should still be searchable and contain both files\n searcher = LeannSearcher(index_path)\n results = searcher.search(\"Second document\", top_k=3)\n searcher.cleanup()\n assert len(results) >= 1\n assert \"Second\" in results[0].text or \"document\" in results[0].text\n\n\n@pytest.mark.skipif(\n os.environ.get(\"CI\") == \"true\",\n reason=\"Skip in CI to avoid embedding/model load\",\n)\ndef test_ivf_incremental_add_then_remove_searchable(tmp_path):\n \"\"\"IVF: add content, search finds it; delete content, incremental build, search no longer finds it.\"\"\"\n import asyncio\n\n from leann.api import LeannSearcher\n from leann.cli import LeannCLI\n\n docs_dir = tmp_path / \"docs\"\n docs_dir.mkdir()\n unique_phrase = \"XYZZY_PLUGH_INCR_TEST_12345\"\n (docs_dir / \"f.txt\").write_text(\n f\"Initial content.\\n\\n{unique_phrase}\\n\\nMore text here.\",\n encoding=\"utf-8\",\n )\n # Add extra files so IVF has enough training points (nlist=100)\n for i in range(110):\n (docs_dir / f\"filler_{i:03d}.txt\").write_text(\n f\"Filler document {i} with some content to create enough chunks.\",\n encoding=\"utf-8\",\n )\n\n cli = LeannCLI()\n cli.indexes_dir = tmp_path / \".leann\" / \"indexes\"\n cli.indexes_dir.mkdir(parents=True, exist_ok=True)\n index_name = \"ivf_incr_test\"\n index_path = cli.get_index_path(index_name)\n\n parser = cli.create_parser()\n build_args = [\n \"build\",\n index_name,\n \"--docs\",\n str(docs_dir),\n \"--backend-name\",\n \"ivf\",\n \"--embedding-model\",\n \"all-MiniLM-L6-v2\",\n \"--embedding-mode\",\n \"sentence-transformers\",\n ]\n\n asyncio.run(cli.build_index(parser.parse_args([*build_args, \"--force\"])))\n\n searcher = LeannSearcher(index_path)\n results = searcher.search(unique_phrase, top_k=5)\n searcher.cleanup()\n assert len(results) >= 1, \"Added content should be searchable\"\n assert unique_phrase in results[0].text\n\n # Remove the unique phrase from the file\n (docs_dir / \"f.txt\").write_text(\"Initial content.\\n\\nMore text here.\", encoding=\"utf-8\")\n\n # Incremental build (IVF remove+add)\n asyncio.run(cli.build_index(parser.parse_args(build_args)))\n\n # Deleted content should no longer be searchable\n searcher = LeannSearcher(index_path)\n results = searcher.search(unique_phrase, top_k=5)\n searcher.cleanup()\n assert all(unique_phrase not in r.text for r in results), (\n \"Deleted content should not appear in search results\"\n )\n\n\n@pytest.mark.skipif(\n os.environ.get(\"CI\") == \"true\",\n reason=\"Skip in CI to avoid embedding/model load\",\n)\ndef test_ivf_multiple_incremental_no_duplicates(tmp_path):\n \"\"\"IVF: modifying the same file across multiple incremental builds must not create duplicate chunks.\"\"\"\n import asyncio\n import json\n import pickle\n\n from leann.api import LeannSearcher\n from leann.cli import LeannCLI\n\n docs_dir = tmp_path / \"docs\"\n docs_dir.mkdir()\n\n target_phrase_v1 = \"UNIQUE_TARGET_V1_ALPHA_BRAVO\"\n target_phrase_v2 = \"UNIQUE_TARGET_V2_CHARLIE_DELTA\"\n target_phrase_v3 = \"UNIQUE_TARGET_V3_ECHO_FOXTROT\"\n\n (docs_dir / \"target.txt\").write_text(\n f\"Version one content.\\n\\n{target_phrase_v1}\\n\\nEnd of version one.\",\n encoding=\"utf-8\",\n )\n # Filler files for IVF training (nlist=100)\n for i in range(110):\n (docs_dir / f\"filler_{i:03d}.txt\").write_text(\n f\"Filler document {i} with unique content for padding the IVF index.\",\n encoding=\"utf-8\",\n )\n\n cli = LeannCLI()\n cli.indexes_dir = tmp_path / \".leann\" / \"indexes\"\n cli.indexes_dir.mkdir(parents=True, exist_ok=True)\n index_name = \"ivf_dup_test\"\n index_path = cli.get_index_path(index_name)\n index_dir = cli.indexes_dir / index_name\n\n parser = cli.create_parser()\n build_args = [\n \"build\",\n index_name,\n \"--docs\",\n str(docs_dir),\n \"--backend-name\",\n \"ivf\",\n \"--embedding-model\",\n \"all-MiniLM-L6-v2\",\n \"--embedding-mode\",\n \"sentence-transformers\",\n ]\n\n # --- Initial build (--force) ---\n asyncio.run(cli.build_index(parser.parse_args([*build_args, \"--force\"])))\n\n searcher = LeannSearcher(index_path)\n results = searcher.search(target_phrase_v1, top_k=10)\n searcher.cleanup()\n v1_hits = [r for r in results if target_phrase_v1 in r.text]\n assert len(v1_hits) >= 1, \"V1 content should be searchable after initial build\"\n\n # --- Modify target file to V2, incremental build ---\n (docs_dir / \"target.txt\").write_text(\n f\"Version two content.\\n\\n{target_phrase_v2}\\n\\nEnd of version two.\",\n encoding=\"utf-8\",\n )\n asyncio.run(cli.build_index(parser.parse_args(build_args)))\n\n searcher = LeannSearcher(index_path)\n results_v2 = searcher.search(target_phrase_v1, top_k=10)\n searcher.cleanup()\n assert all(target_phrase_v1 not in r.text for r in results_v2), (\n \"V1 content should be gone after first incremental update\"\n )\n\n searcher = LeannSearcher(index_path)\n results_v2b = searcher.search(target_phrase_v2, top_k=10)\n searcher.cleanup()\n v2_hits = [r for r in results_v2b if target_phrase_v2 in r.text]\n assert len(v2_hits) >= 1, \"V2 content should be searchable\"\n\n # --- Modify target file to V3, second incremental build ---\n (docs_dir / \"target.txt\").write_text(\n f\"Version three content.\\n\\n{target_phrase_v3}\\n\\nEnd of version three.\",\n encoding=\"utf-8\",\n )\n asyncio.run(cli.build_index(parser.parse_args(build_args)))\n\n # V1 and V2 should be gone, only V3 present\n searcher = LeannSearcher(index_path)\n results_v3_check_v1 = searcher.search(target_phrase_v1, top_k=10)\n searcher.cleanup()\n assert all(target_phrase_v1 not in r.text for r in results_v3_check_v1), (\n \"V1 content should NOT appear after two incremental updates\"\n )\n\n searcher = LeannSearcher(index_path)\n results_v3_check_v2 = searcher.search(target_phrase_v2, top_k=10)\n searcher.cleanup()\n assert all(target_phrase_v2 not in r.text for r in results_v3_check_v2), (\n \"V2 content should NOT appear after second incremental update\"\n )\n\n searcher = LeannSearcher(index_path)\n results_v3 = searcher.search(target_phrase_v3, top_k=10)\n searcher.cleanup()\n v3_hits = [r for r in results_v3 if target_phrase_v3 in r.text]\n assert len(v3_hits) >= 1, \"V3 content should be searchable\"\n\n # --- Verify passages.jsonl has no stale entries ---\n passages_file = index_dir / \"documents.leann.passages.jsonl\"\n offset_file = index_dir / \"documents.leann.passages.idx\"\n with open(offset_file, \"rb\") as f:\n offset_map = pickle.load(f)\n live_ids = set(offset_map.keys())\n\n jsonl_ids = []\n with open(passages_file, encoding=\"utf-8\") as f:\n for line in f:\n line = line.strip()\n if not line:\n continue\n data = json.loads(line)\n jsonl_ids.append(data[\"id\"])\n\n stale_ids = [pid for pid in jsonl_ids if pid not in live_ids]\n assert len(stale_ids) == 0, (\n f\"passages.jsonl has {len(stale_ids)} stale entries not in offset_map: {stale_ids[:5]}\"\n )\n" }, { "path": "tests/test_lmstudio_bridge.py", "content": "\"\"\"Unit tests for LM Studio TypeScript SDK bridge functionality.\n\nThis test suite defines the contract for the LM Studio SDK bridge that queries\nmodel context length via Node.js subprocess. These tests verify:\n\n1. Successful SDK query returns context length\n2. Graceful fallback when Node.js not installed (FileNotFoundError)\n3. Graceful fallback when SDK not installed (npm error)\n4. Timeout handling (subprocess.TimeoutExpired)\n5. Invalid JSON response handling\n\nAll tests are written in Red Phase - they should FAIL initially because the\n`_query_lmstudio_context_limit` function does not exist yet.\n\nThe function contract:\n- Inputs: model_name (str), base_url (str, WebSocket format \"ws://localhost:1234\")\n- Outputs: context_length (int) or None on error\n- Requirements:\n 1. Call Node.js with inline JavaScript using @lmstudio/sdk\n 2. 10-second timeout (accounts for Node.js startup)\n 3. Graceful fallback on any error (returns None, doesn't raise)\n 4. Parse JSON response with contextLength field\n 5. Log errors at debug level (not warning/error)\n\"\"\"\n\nimport subprocess\nfrom unittest.mock import Mock\n\nimport pytest\n\n# Try to import the function - if it doesn't exist, tests will fail as expected\ntry:\n from leann.embedding_compute import _query_lmstudio_context_limit\nexcept ImportError:\n # Function doesn't exist yet (Red Phase) - create a placeholder that will fail\n def _query_lmstudio_context_limit(*args, **kwargs):\n raise NotImplementedError(\n \"_query_lmstudio_context_limit not implemented yet - this is the Red Phase\"\n )\n\n\nclass TestLMStudioBridge:\n \"\"\"Tests for LM Studio TypeScript SDK bridge integration.\"\"\"\n\n def test_query_lmstudio_success(self, monkeypatch):\n \"\"\"Verify successful SDK query returns context length.\n\n When the Node.js subprocess successfully queries the LM Studio SDK,\n it should return a JSON response with contextLength field. The function\n should parse this and return the integer context length.\n \"\"\"\n\n def mock_run(*args, **kwargs):\n # Verify timeout is set to 10 seconds\n assert kwargs.get(\"timeout\") == 10, \"Should use 10-second timeout for Node.js startup\"\n\n # Verify capture_output and text=True are set\n assert kwargs.get(\"capture_output\") is True, \"Should capture stdout/stderr\"\n assert kwargs.get(\"text\") is True, \"Should decode output as text\"\n\n # Return successful JSON response\n mock_result = Mock()\n mock_result.returncode = 0\n mock_result.stdout = '{\"contextLength\": 8192, \"identifier\": \"custom-model\"}'\n mock_result.stderr = \"\"\n return mock_result\n\n monkeypatch.setattr(\"subprocess.run\", mock_run)\n\n # Test with typical LM Studio model\n limit = _query_lmstudio_context_limit(\n model_name=\"custom-model\", base_url=\"ws://localhost:1234\"\n )\n\n assert limit == 8192, \"Should return context length from SDK response\"\n\n def test_query_lmstudio_nodejs_not_found(self, monkeypatch):\n \"\"\"Verify graceful fallback when Node.js not installed.\n\n When Node.js is not installed, subprocess.run will raise FileNotFoundError.\n The function should catch this and return None (graceful fallback to registry).\n \"\"\"\n\n def mock_run(*args, **kwargs):\n raise FileNotFoundError(\"node: command not found\")\n\n monkeypatch.setattr(\"subprocess.run\", mock_run)\n\n limit = _query_lmstudio_context_limit(\n model_name=\"custom-model\", base_url=\"ws://localhost:1234\"\n )\n\n assert limit is None, \"Should return None when Node.js not installed\"\n\n def test_query_lmstudio_sdk_not_installed(self, monkeypatch):\n \"\"\"Verify graceful fallback when @lmstudio/sdk not installed.\n\n When the SDK npm package is not installed, Node.js will return non-zero\n exit code with error message in stderr. The function should detect this\n and return None.\n \"\"\"\n\n def mock_run(*args, **kwargs):\n mock_result = Mock()\n mock_result.returncode = 1\n mock_result.stdout = \"\"\n mock_result.stderr = (\n \"Error: Cannot find module '@lmstudio/sdk'\\nRequire stack:\\n- /path/to/script.js\"\n )\n return mock_result\n\n monkeypatch.setattr(\"subprocess.run\", mock_run)\n\n limit = _query_lmstudio_context_limit(\n model_name=\"custom-model\", base_url=\"ws://localhost:1234\"\n )\n\n assert limit is None, \"Should return None when SDK not installed\"\n\n def test_query_lmstudio_timeout(self, monkeypatch):\n \"\"\"Verify graceful fallback when subprocess times out.\n\n When the Node.js process takes longer than 10 seconds (e.g., LM Studio\n not responding), subprocess.TimeoutExpired should be raised. The function\n should catch this and return None.\n \"\"\"\n\n def mock_run(*args, **kwargs):\n raise subprocess.TimeoutExpired(cmd=[\"node\", \"lmstudio_bridge.js\"], timeout=10)\n\n monkeypatch.setattr(\"subprocess.run\", mock_run)\n\n limit = _query_lmstudio_context_limit(\n model_name=\"custom-model\", base_url=\"ws://localhost:1234\"\n )\n\n assert limit is None, \"Should return None on timeout\"\n\n def test_query_lmstudio_invalid_json(self, monkeypatch):\n \"\"\"Verify graceful fallback when response is invalid JSON.\n\n When the subprocess returns malformed JSON (e.g., due to SDK error),\n json.loads will raise ValueError/JSONDecodeError. The function should\n catch this and return None.\n \"\"\"\n\n def mock_run(*args, **kwargs):\n mock_result = Mock()\n mock_result.returncode = 0\n mock_result.stdout = \"This is not valid JSON\"\n mock_result.stderr = \"\"\n return mock_result\n\n monkeypatch.setattr(\"subprocess.run\", mock_run)\n\n limit = _query_lmstudio_context_limit(\n model_name=\"custom-model\", base_url=\"ws://localhost:1234\"\n )\n\n assert limit is None, \"Should return None when JSON parsing fails\"\n\n def test_query_lmstudio_missing_context_length_field(self, monkeypatch):\n \"\"\"Verify graceful fallback when JSON lacks contextLength field.\n\n When the SDK returns valid JSON but without the expected contextLength\n field (e.g., error response), the function should return None.\n \"\"\"\n\n def mock_run(*args, **kwargs):\n mock_result = Mock()\n mock_result.returncode = 0\n mock_result.stdout = '{\"identifier\": \"test-model\", \"error\": \"Model not found\"}'\n mock_result.stderr = \"\"\n return mock_result\n\n monkeypatch.setattr(\"subprocess.run\", mock_run)\n\n limit = _query_lmstudio_context_limit(\n model_name=\"nonexistent-model\", base_url=\"ws://localhost:1234\"\n )\n\n assert limit is None, \"Should return None when contextLength field missing\"\n\n def test_query_lmstudio_null_context_length(self, monkeypatch):\n \"\"\"Verify graceful fallback when contextLength is null.\n\n When the SDK returns contextLength: null (model couldn't be loaded),\n the function should return None for registry fallback.\n \"\"\"\n\n def mock_run(*args, **kwargs):\n mock_result = Mock()\n mock_result.returncode = 0\n mock_result.stdout = '{\"contextLength\": null, \"identifier\": \"test-model\"}'\n mock_result.stderr = \"\"\n return mock_result\n\n monkeypatch.setattr(\"subprocess.run\", mock_run)\n\n limit = _query_lmstudio_context_limit(\n model_name=\"test-model\", base_url=\"ws://localhost:1234\"\n )\n\n assert limit is None, \"Should return None when contextLength is null\"\n\n def test_query_lmstudio_zero_context_length(self, monkeypatch):\n \"\"\"Verify graceful fallback when contextLength is zero.\n\n When the SDK returns contextLength: 0 (invalid value), the function\n should return None to trigger registry fallback.\n \"\"\"\n\n def mock_run(*args, **kwargs):\n mock_result = Mock()\n mock_result.returncode = 0\n mock_result.stdout = '{\"contextLength\": 0, \"identifier\": \"test-model\"}'\n mock_result.stderr = \"\"\n return mock_result\n\n monkeypatch.setattr(\"subprocess.run\", mock_run)\n\n limit = _query_lmstudio_context_limit(\n model_name=\"test-model\", base_url=\"ws://localhost:1234\"\n )\n\n assert limit is None, \"Should return None when contextLength is zero\"\n\n def test_query_lmstudio_with_custom_port(self, monkeypatch):\n \"\"\"Verify SDK query works with non-default WebSocket port.\n\n LM Studio can run on custom ports. The function should pass the\n provided base_url to the Node.js subprocess.\n \"\"\"\n\n def mock_run(*args, **kwargs):\n # Verify the base_url argument is passed correctly\n command = args[0] if args else kwargs.get(\"args\", [])\n assert \"ws://localhost:8080\" in \" \".join(command), (\n \"Should pass custom port to subprocess\"\n )\n\n mock_result = Mock()\n mock_result.returncode = 0\n mock_result.stdout = '{\"contextLength\": 4096, \"identifier\": \"custom-model\"}'\n mock_result.stderr = \"\"\n return mock_result\n\n monkeypatch.setattr(\"subprocess.run\", mock_run)\n\n limit = _query_lmstudio_context_limit(\n model_name=\"custom-model\", base_url=\"ws://localhost:8080\"\n )\n\n assert limit == 4096, \"Should work with custom WebSocket port\"\n\n @pytest.mark.parametrize(\n \"context_length,expected\",\n [\n (512, 512), # Small context\n (2048, 2048), # Common context\n (8192, 8192), # Large context\n (32768, 32768), # Very large context\n ],\n )\n def test_query_lmstudio_various_context_lengths(self, monkeypatch, context_length, expected):\n \"\"\"Verify SDK query handles various context length values.\n\n Different models have different context lengths. The function should\n correctly parse and return any positive integer value.\n \"\"\"\n\n def mock_run(*args, **kwargs):\n mock_result = Mock()\n mock_result.returncode = 0\n mock_result.stdout = f'{{\"contextLength\": {context_length}, \"identifier\": \"test\"}}'\n mock_result.stderr = \"\"\n return mock_result\n\n monkeypatch.setattr(\"subprocess.run\", mock_run)\n\n limit = _query_lmstudio_context_limit(\n model_name=\"test-model\", base_url=\"ws://localhost:1234\"\n )\n\n assert limit == expected, f\"Should return {expected} for context length {context_length}\"\n\n def test_query_lmstudio_logs_at_debug_level(self, monkeypatch, caplog):\n \"\"\"Verify errors are logged at DEBUG level, not WARNING/ERROR.\n\n Following the graceful fallback pattern from Ollama implementation,\n errors should be logged at debug level to avoid alarming users when\n fallback to registry works fine.\n \"\"\"\n import logging\n\n caplog.set_level(logging.DEBUG, logger=\"leann.embedding_compute\")\n\n def mock_run(*args, **kwargs):\n raise FileNotFoundError(\"node: command not found\")\n\n monkeypatch.setattr(\"subprocess.run\", mock_run)\n\n _query_lmstudio_context_limit(model_name=\"test-model\", base_url=\"ws://localhost:1234\")\n\n # Check that debug logging occurred (not warning/error)\n debug_logs = [record for record in caplog.records if record.levelname == \"DEBUG\"]\n assert len(debug_logs) > 0, \"Should log error at DEBUG level\"\n\n # Verify no WARNING or ERROR logs\n warning_or_error_logs = [\n record for record in caplog.records if record.levelname in [\"WARNING\", \"ERROR\"]\n ]\n assert len(warning_or_error_logs) == 0, (\n \"Should not log at WARNING/ERROR level for expected failures\"\n )\n" }, { "path": "tests/test_mcp_integration.py", "content": "#!/usr/bin/env python3\n\"\"\"\nTest script for MCP integration implementations.\n\nThis script tests the basic functionality of the MCP readers and RAG applications\nwithout requiring actual MCP servers to be running.\n\"\"\"\n\nimport sys\nfrom pathlib import Path\n\n# Add the parent directory to the path so we can import from apps\nsys.path.append(str(Path(__file__).parent.parent))\n\nfrom apps.slack_data.slack_mcp_reader import SlackMCPReader\nfrom apps.slack_rag import SlackMCPRAG\nfrom apps.twitter_data.twitter_mcp_reader import TwitterMCPReader\nfrom apps.twitter_rag import TwitterMCPRAG\n\n\ndef test_slack_reader_initialization():\n \"\"\"Test that SlackMCPReader can be initialized with various parameters.\"\"\"\n print(\"Testing SlackMCPReader initialization...\")\n\n # Test basic initialization\n reader = SlackMCPReader(\"slack-mcp-server\")\n assert reader.mcp_server_command == \"slack-mcp-server\"\n assert reader.concatenate_conversations\n assert reader.max_messages_per_conversation == 100\n\n # Test with custom parameters\n reader = SlackMCPReader(\n \"custom-slack-server\",\n workspace_name=\"test-workspace\",\n concatenate_conversations=False,\n max_messages_per_conversation=50,\n )\n assert reader.workspace_name == \"test-workspace\"\n assert not reader.concatenate_conversations\n assert reader.max_messages_per_conversation == 50\n\n print(\"✅ SlackMCPReader initialization tests passed\")\n\n\ndef test_twitter_reader_initialization():\n \"\"\"Test that TwitterMCPReader can be initialized with various parameters.\"\"\"\n print(\"Testing TwitterMCPReader initialization...\")\n\n # Test basic initialization\n reader = TwitterMCPReader(\"twitter-mcp-server\")\n assert reader.mcp_server_command == \"twitter-mcp-server\"\n assert reader.include_tweet_content\n assert reader.include_metadata\n assert reader.max_bookmarks == 1000\n\n # Test with custom parameters\n reader = TwitterMCPReader(\n \"custom-twitter-server\",\n username=\"testuser\",\n include_tweet_content=False,\n include_metadata=False,\n max_bookmarks=500,\n )\n assert reader.username == \"testuser\"\n assert not reader.include_tweet_content\n assert not reader.include_metadata\n assert reader.max_bookmarks == 500\n\n print(\"✅ TwitterMCPReader initialization tests passed\")\n\n\ndef test_slack_message_formatting():\n \"\"\"Test Slack message formatting functionality.\"\"\"\n print(\"Testing Slack message formatting...\")\n\n reader = SlackMCPReader(\"slack-mcp-server\")\n\n # Test basic message formatting\n message = {\n \"text\": \"Hello, world!\",\n \"user\": \"john_doe\",\n \"channel\": \"general\",\n \"ts\": \"1234567890.123456\",\n }\n\n formatted = reader._format_message(message)\n assert \"Channel: #general\" in formatted\n assert \"User: john_doe\" in formatted\n assert \"Message: Hello, world!\" in formatted\n assert \"Time:\" in formatted\n\n # Test with missing fields\n message = {\"text\": \"Simple message\"}\n formatted = reader._format_message(message)\n assert \"Message: Simple message\" in formatted\n\n print(\"✅ Slack message formatting tests passed\")\n\n\ndef test_twitter_bookmark_formatting():\n \"\"\"Test Twitter bookmark formatting functionality.\"\"\"\n print(\"Testing Twitter bookmark formatting...\")\n\n reader = TwitterMCPReader(\"twitter-mcp-server\")\n\n # Test basic bookmark formatting\n bookmark = {\n \"text\": \"This is a great article about AI!\",\n \"author\": \"ai_researcher\",\n \"created_at\": \"2024-01-01T12:00:00Z\",\n \"url\": \"https://twitter.com/ai_researcher/status/123456789\",\n \"likes\": 42,\n \"retweets\": 15,\n }\n\n formatted = reader._format_bookmark(bookmark)\n assert \"=== Twitter Bookmark ===\" in formatted\n assert \"Author: @ai_researcher\" in formatted\n assert \"Content:\" in formatted\n assert \"This is a great article about AI!\" in formatted\n assert \"URL: https://twitter.com\" in formatted\n assert \"Likes: 42\" in formatted\n assert \"Retweets: 15\" in formatted\n\n # Test with minimal data\n bookmark = {\"text\": \"Simple tweet\"}\n formatted = reader._format_bookmark(bookmark)\n assert \"=== Twitter Bookmark ===\" in formatted\n assert \"Simple tweet\" in formatted\n\n print(\"✅ Twitter bookmark formatting tests passed\")\n\n\ndef test_slack_rag_initialization():\n \"\"\"Test that SlackMCPRAG can be initialized.\"\"\"\n print(\"Testing SlackMCPRAG initialization...\")\n\n app = SlackMCPRAG()\n assert app.default_index_name == \"slack_messages\"\n assert hasattr(app, \"parser\")\n\n print(\"✅ SlackMCPRAG initialization tests passed\")\n\n\ndef test_twitter_rag_initialization():\n \"\"\"Test that TwitterMCPRAG can be initialized.\"\"\"\n print(\"Testing TwitterMCPRAG initialization...\")\n\n app = TwitterMCPRAG()\n assert app.default_index_name == \"twitter_bookmarks\"\n assert hasattr(app, \"parser\")\n\n print(\"✅ TwitterMCPRAG initialization tests passed\")\n\n\ndef test_concatenated_content_creation():\n \"\"\"Test creation of concatenated content from multiple messages.\"\"\"\n print(\"Testing concatenated content creation...\")\n\n reader = SlackMCPReader(\"slack-mcp-server\", workspace_name=\"test-workspace\")\n\n messages = [\n {\"text\": \"First message\", \"user\": \"alice\", \"ts\": \"1000\"},\n {\"text\": \"Second message\", \"user\": \"bob\", \"ts\": \"2000\"},\n {\"text\": \"Third message\", \"user\": \"charlie\", \"ts\": \"3000\"},\n ]\n\n content = reader._create_concatenated_content(messages, \"general\")\n\n assert \"Slack Channel: #general\" in content\n assert \"Message Count: 3\" in content\n assert \"Workspace: test-workspace\" in content\n assert \"First message\" in content\n assert \"Second message\" in content\n assert \"Third message\" in content\n\n print(\"✅ Concatenated content creation tests passed\")\n\n\ndef main():\n \"\"\"Run all tests.\"\"\"\n print(\"🧪 Running MCP Integration Tests\")\n print(\"=\" * 50)\n\n try:\n test_slack_reader_initialization()\n test_twitter_reader_initialization()\n test_slack_message_formatting()\n test_twitter_bookmark_formatting()\n test_slack_rag_initialization()\n test_twitter_rag_initialization()\n test_concatenated_content_creation()\n\n print(\"\\n\" + \"=\" * 50)\n print(\"🎉 All tests passed! MCP integration is working correctly.\")\n print(\"\\nNext steps:\")\n print(\"1. Install actual MCP servers for Slack and Twitter\")\n print(\"2. Configure API credentials\")\n print(\"3. Test with --test-connection flag\")\n print(\"4. Start indexing your live data!\")\n\n except Exception as e:\n print(f\"\\n❌ Test failed: {e}\")\n sys.exit(1)\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "tests/test_mcp_standalone.py", "content": "#!/usr/bin/env python3\n\"\"\"\nStandalone test script for MCP integration implementations.\n\nThis script tests the basic functionality of the MCP readers\nwithout requiring LEANN core dependencies.\n\"\"\"\n\nimport json\nimport sys\nfrom pathlib import Path\n\n# Add the parent directory to the path so we can import from apps\nsys.path.append(str(Path(__file__).parent.parent))\n\n\ndef test_slack_reader_basic():\n \"\"\"Test basic SlackMCPReader functionality without async operations.\"\"\"\n print(\"Testing SlackMCPReader basic functionality...\")\n\n # Import and test initialization\n from apps.slack_data.slack_mcp_reader import SlackMCPReader\n\n reader = SlackMCPReader(\"slack-mcp-server\")\n assert reader.mcp_server_command == \"slack-mcp-server\"\n assert reader.concatenate_conversations\n\n # Test message formatting\n message = {\n \"text\": \"Hello team! How's the project going?\",\n \"user\": \"john_doe\",\n \"channel\": \"general\",\n \"ts\": \"1234567890.123456\",\n }\n\n formatted = reader._format_message(message)\n assert \"Channel: #general\" in formatted\n assert \"User: john_doe\" in formatted\n assert \"Message: Hello team!\" in formatted\n\n # Test concatenated content creation\n messages = [\n {\"text\": \"First message\", \"user\": \"alice\", \"ts\": \"1000\"},\n {\"text\": \"Second message\", \"user\": \"bob\", \"ts\": \"2000\"},\n ]\n\n content = reader._create_concatenated_content(messages, \"dev-team\")\n assert \"Slack Channel: #dev-team\" in content\n assert \"Message Count: 2\" in content\n assert \"First message\" in content\n assert \"Second message\" in content\n\n print(\"✅ SlackMCPReader basic tests passed\")\n\n\ndef test_twitter_reader_basic():\n \"\"\"Test basic TwitterMCPReader functionality.\"\"\"\n print(\"Testing TwitterMCPReader basic functionality...\")\n\n from apps.twitter_data.twitter_mcp_reader import TwitterMCPReader\n\n reader = TwitterMCPReader(\"twitter-mcp-server\")\n assert reader.mcp_server_command == \"twitter-mcp-server\"\n assert reader.include_tweet_content\n assert reader.max_bookmarks == 1000\n\n # Test bookmark formatting\n bookmark = {\n \"text\": \"Amazing article about the future of AI! Must read for everyone interested in tech.\",\n \"author\": \"tech_guru\",\n \"created_at\": \"2024-01-15T14:30:00Z\",\n \"url\": \"https://twitter.com/tech_guru/status/123456789\",\n \"likes\": 156,\n \"retweets\": 42,\n \"replies\": 23,\n \"hashtags\": [\"AI\", \"tech\", \"future\"],\n \"mentions\": [\"@openai\", \"@anthropic\"],\n }\n\n formatted = reader._format_bookmark(bookmark)\n assert \"=== Twitter Bookmark ===\" in formatted\n assert \"Author: @tech_guru\" in formatted\n assert \"Amazing article about the future of AI!\" in formatted\n assert \"Likes: 156\" in formatted\n assert \"Retweets: 42\" in formatted\n assert \"Hashtags: AI, tech, future\" in formatted\n assert \"Mentions: @openai, @anthropic\" in formatted\n\n # Test with minimal data\n simple_bookmark = {\"text\": \"Short tweet\", \"author\": \"user123\"}\n formatted_simple = reader._format_bookmark(simple_bookmark)\n assert \"=== Twitter Bookmark ===\" in formatted_simple\n assert \"Short tweet\" in formatted_simple\n assert \"Author: @user123\" in formatted_simple\n\n print(\"✅ TwitterMCPReader basic tests passed\")\n\n\ndef test_mcp_request_format():\n \"\"\"Test MCP request formatting.\"\"\"\n print(\"Testing MCP request formatting...\")\n\n # Test initialization request format\n init_request = {\n \"jsonrpc\": \"2.0\",\n \"id\": 1,\n \"method\": \"initialize\",\n \"params\": {\n \"protocolVersion\": \"2024-11-05\",\n \"capabilities\": {},\n \"clientInfo\": {\"name\": \"leann-slack-reader\", \"version\": \"1.0.0\"},\n },\n }\n\n # Verify it's valid JSON\n json_str = json.dumps(init_request)\n parsed = json.loads(json_str)\n assert parsed[\"jsonrpc\"] == \"2.0\"\n assert parsed[\"method\"] == \"initialize\"\n assert parsed[\"params\"][\"protocolVersion\"] == \"2024-11-05\"\n\n # Test tools/list request\n list_request = {\"jsonrpc\": \"2.0\", \"id\": 2, \"method\": \"tools/list\", \"params\": {}}\n\n json_str = json.dumps(list_request)\n parsed = json.loads(json_str)\n assert parsed[\"method\"] == \"tools/list\"\n\n print(\"✅ MCP request formatting tests passed\")\n\n\ndef test_data_processing():\n \"\"\"Test data processing capabilities.\"\"\"\n print(\"Testing data processing capabilities...\")\n\n from apps.slack_data.slack_mcp_reader import SlackMCPReader\n from apps.twitter_data.twitter_mcp_reader import TwitterMCPReader\n\n # Test Slack message processing with various formats\n slack_reader = SlackMCPReader(\"test-server\")\n\n messages_with_timestamps = [\n {\"text\": \"Meeting in 5 minutes\", \"user\": \"alice\", \"ts\": \"1000.123\"},\n {\"text\": \"On my way!\", \"user\": \"bob\", \"ts\": \"1001.456\"},\n {\"text\": \"Starting now\", \"user\": \"charlie\", \"ts\": \"1002.789\"},\n ]\n\n content = slack_reader._create_concatenated_content(messages_with_timestamps, \"meetings\")\n assert \"Meeting in 5 minutes\" in content\n assert \"On my way!\" in content\n assert \"Starting now\" in content\n\n # Test Twitter bookmark processing with engagement data\n twitter_reader = TwitterMCPReader(\"test-server\", include_metadata=True)\n\n high_engagement_bookmark = {\n \"text\": \"Thread about startup lessons learned 🧵\",\n \"author\": \"startup_founder\",\n \"likes\": 1250,\n \"retweets\": 340,\n \"replies\": 89,\n }\n\n formatted = twitter_reader._format_bookmark(high_engagement_bookmark)\n assert \"Thread about startup lessons learned\" in formatted\n assert \"Likes: 1250\" in formatted\n assert \"Retweets: 340\" in formatted\n assert \"Replies: 89\" in formatted\n\n # Test with metadata disabled\n twitter_reader_no_meta = TwitterMCPReader(\"test-server\", include_metadata=False)\n formatted_no_meta = twitter_reader_no_meta._format_bookmark(high_engagement_bookmark)\n assert \"Thread about startup lessons learned\" in formatted_no_meta\n assert \"Likes:\" not in formatted_no_meta\n assert \"Retweets:\" not in formatted_no_meta\n\n print(\"✅ Data processing tests passed\")\n\n\ndef main():\n \"\"\"Run all standalone tests.\"\"\"\n print(\"🧪 Running MCP Integration Standalone Tests\")\n print(\"=\" * 60)\n print(\"Testing core functionality without LEANN dependencies...\")\n print()\n\n try:\n test_slack_reader_basic()\n test_twitter_reader_basic()\n test_mcp_request_format()\n test_data_processing()\n\n print(\"\\n\" + \"=\" * 60)\n print(\"🎉 All standalone tests passed!\")\n print(\"\\n✨ MCP Integration Summary:\")\n print(\"- SlackMCPReader: Ready for Slack message processing\")\n print(\"- TwitterMCPReader: Ready for Twitter bookmark processing\")\n print(\"- MCP Protocol: Properly formatted JSON-RPC requests\")\n print(\"- Data Processing: Handles various message/bookmark formats\")\n\n print(\"\\n🚀 Next Steps:\")\n print(\"1. Install MCP servers: npm install -g slack-mcp-server twitter-mcp-server\")\n print(\"2. Configure API credentials for Slack and Twitter\")\n print(\"3. Test connections: python -m apps.slack_rag --test-connection\")\n print(\"4. Start indexing live data from your platforms!\")\n\n print(\"\\n📖 Documentation:\")\n print(\"- Check README.md for detailed setup instructions\")\n print(\"- Run examples/mcp_integration_demo.py for usage examples\")\n print(\"- Explore apps/slack_rag.py and apps/twitter_rag.py for implementation details\")\n\n except Exception as e:\n print(f\"\\n❌ Test failed: {e}\")\n import traceback\n\n traceback.print_exc()\n sys.exit(1)\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "tests/test_metadata_filtering.py", "content": "\"\"\"\nComprehensive tests for metadata filtering functionality.\n\nThis module tests the MetadataFilterEngine class and its integration\nwith the LEANN search system.\n\"\"\"\n\nimport os\n\n# Import the modules we're testing\nimport sys\nfrom unittest.mock import Mock, patch\n\nsys.path.insert(0, os.path.join(os.path.dirname(__file__), \"../packages/leann-core/src\"))\n\nfrom leann.api import PassageManager, SearchResult\nfrom leann.metadata_filter import MetadataFilterEngine\n\n\nclass TestMetadataFilterEngine:\n \"\"\"Test suite for the MetadataFilterEngine class.\"\"\"\n\n def setup_method(self):\n \"\"\"Setup test fixtures.\"\"\"\n self.engine = MetadataFilterEngine()\n\n # Sample search results for testing\n self.sample_results = [\n {\n \"id\": \"doc1\",\n \"score\": 0.95,\n \"text\": \"This is chapter 1 content\",\n \"metadata\": {\n \"chapter\": 1,\n \"character\": \"Alice\",\n \"tags\": [\"adventure\", \"fantasy\"],\n \"word_count\": 150,\n \"is_published\": True,\n \"genre\": \"fiction\",\n },\n },\n {\n \"id\": \"doc2\",\n \"score\": 0.87,\n \"text\": \"This is chapter 3 content\",\n \"metadata\": {\n \"chapter\": 3,\n \"character\": \"Bob\",\n \"tags\": [\"mystery\", \"thriller\"],\n \"word_count\": 250,\n \"is_published\": True,\n \"genre\": \"fiction\",\n },\n },\n {\n \"id\": \"doc3\",\n \"score\": 0.82,\n \"text\": \"This is chapter 5 content\",\n \"metadata\": {\n \"chapter\": 5,\n \"character\": \"Alice\",\n \"tags\": [\"romance\", \"drama\"],\n \"word_count\": 300,\n \"is_published\": False,\n \"genre\": \"non-fiction\",\n },\n },\n {\n \"id\": \"doc4\",\n \"score\": 0.78,\n \"text\": \"This is chapter 10 content\",\n \"metadata\": {\n \"chapter\": 10,\n \"character\": \"Charlie\",\n \"tags\": [\"action\", \"adventure\"],\n \"word_count\": 400,\n \"is_published\": True,\n \"genre\": \"fiction\",\n },\n },\n ]\n\n def test_engine_initialization(self):\n \"\"\"Test that the filter engine initializes correctly.\"\"\"\n assert self.engine is not None\n assert len(self.engine.operators) > 0\n assert \"==\" in self.engine.operators\n assert \"contains\" in self.engine.operators\n assert \"in\" in self.engine.operators\n\n def test_direct_instantiation(self):\n \"\"\"Test direct instantiation of the engine.\"\"\"\n engine = MetadataFilterEngine()\n assert isinstance(engine, MetadataFilterEngine)\n\n def test_no_filters_returns_all_results(self):\n \"\"\"Test that passing None or empty filters returns all results.\"\"\"\n # Test with None\n result = self.engine.apply_filters(self.sample_results, None)\n assert len(result) == len(self.sample_results)\n\n # Test with empty dict\n result = self.engine.apply_filters(self.sample_results, {})\n assert len(result) == len(self.sample_results)\n\n # Test comparison operators\n def test_equals_filter(self):\n \"\"\"Test equals (==) filter.\"\"\"\n filters = {\"chapter\": {\"==\": 1}}\n result = self.engine.apply_filters(self.sample_results, filters)\n assert len(result) == 1\n assert result[0][\"id\"] == \"doc1\"\n\n def test_not_equals_filter(self):\n \"\"\"Test not equals (!=) filter.\"\"\"\n filters = {\"genre\": {\"!=\": \"fiction\"}}\n result = self.engine.apply_filters(self.sample_results, filters)\n assert len(result) == 1\n assert result[0][\"metadata\"][\"genre\"] == \"non-fiction\"\n\n def test_less_than_filter(self):\n \"\"\"Test less than (<) filter.\"\"\"\n filters = {\"chapter\": {\"<\": 5}}\n result = self.engine.apply_filters(self.sample_results, filters)\n assert len(result) == 2\n chapters = [r[\"metadata\"][\"chapter\"] for r in result]\n assert all(ch < 5 for ch in chapters)\n\n def test_less_than_or_equal_filter(self):\n \"\"\"Test less than or equal (<=) filter.\"\"\"\n filters = {\"chapter\": {\"<=\": 5}}\n result = self.engine.apply_filters(self.sample_results, filters)\n assert len(result) == 3\n chapters = [r[\"metadata\"][\"chapter\"] for r in result]\n assert all(ch <= 5 for ch in chapters)\n\n def test_greater_than_filter(self):\n \"\"\"Test greater than (>) filter.\"\"\"\n filters = {\"word_count\": {\">\": 200}}\n result = self.engine.apply_filters(self.sample_results, filters)\n assert len(result) == 3 # Documents with word_count 250, 300, 400\n word_counts = [r[\"metadata\"][\"word_count\"] for r in result]\n assert all(wc > 200 for wc in word_counts)\n\n def test_greater_than_or_equal_filter(self):\n \"\"\"Test greater than or equal (>=) filter.\"\"\"\n filters = {\"word_count\": {\">=\": 250}}\n result = self.engine.apply_filters(self.sample_results, filters)\n assert len(result) == 3\n word_counts = [r[\"metadata\"][\"word_count\"] for r in result]\n assert all(wc >= 250 for wc in word_counts)\n\n # Test membership operators\n def test_in_filter(self):\n \"\"\"Test in filter.\"\"\"\n filters = {\"character\": {\"in\": [\"Alice\", \"Bob\"]}}\n result = self.engine.apply_filters(self.sample_results, filters)\n assert len(result) == 3\n characters = [r[\"metadata\"][\"character\"] for r in result]\n assert all(ch in [\"Alice\", \"Bob\"] for ch in characters)\n\n def test_not_in_filter(self):\n \"\"\"Test not_in filter.\"\"\"\n filters = {\"character\": {\"not_in\": [\"Alice\", \"Bob\"]}}\n result = self.engine.apply_filters(self.sample_results, filters)\n assert len(result) == 1\n assert result[0][\"metadata\"][\"character\"] == \"Charlie\"\n\n # Test string operators\n def test_contains_filter(self):\n \"\"\"Test contains filter.\"\"\"\n filters = {\"genre\": {\"contains\": \"fiction\"}}\n result = self.engine.apply_filters(self.sample_results, filters)\n assert len(result) == 4 # Both \"fiction\" and \"non-fiction\"\n\n def test_starts_with_filter(self):\n \"\"\"Test starts_with filter.\"\"\"\n filters = {\"genre\": {\"starts_with\": \"non\"}}\n result = self.engine.apply_filters(self.sample_results, filters)\n assert len(result) == 1\n assert result[0][\"metadata\"][\"genre\"] == \"non-fiction\"\n\n def test_ends_with_filter(self):\n \"\"\"Test ends_with filter.\"\"\"\n filters = {\"text\": {\"ends_with\": \"content\"}}\n result = self.engine.apply_filters(self.sample_results, filters)\n assert len(result) == 4 # All sample texts end with \"content\"\n\n # Test boolean operators\n def test_is_true_filter(self):\n \"\"\"Test is_true filter.\"\"\"\n filters = {\"is_published\": {\"is_true\": True}}\n result = self.engine.apply_filters(self.sample_results, filters)\n assert len(result) == 3\n assert all(r[\"metadata\"][\"is_published\"] for r in result)\n\n def test_is_false_filter(self):\n \"\"\"Test is_false filter.\"\"\"\n filters = {\"is_published\": {\"is_false\": False}}\n result = self.engine.apply_filters(self.sample_results, filters)\n assert len(result) == 1\n assert not result[0][\"metadata\"][\"is_published\"]\n\n # Test compound filters (AND logic)\n def test_compound_filters(self):\n \"\"\"Test multiple filters applied together (AND logic).\"\"\"\n filters = {\"genre\": {\"==\": \"fiction\"}, \"chapter\": {\"<=\": 5}}\n result = self.engine.apply_filters(self.sample_results, filters)\n assert len(result) == 2\n for r in result:\n assert r[\"metadata\"][\"genre\"] == \"fiction\"\n assert r[\"metadata\"][\"chapter\"] <= 5\n\n def test_multiple_operators_same_field(self):\n \"\"\"Test multiple operators on the same field.\"\"\"\n filters = {\"word_count\": {\">=\": 200, \"<=\": 350}}\n result = self.engine.apply_filters(self.sample_results, filters)\n assert len(result) == 2\n for r in result:\n wc = r[\"metadata\"][\"word_count\"]\n assert 200 <= wc <= 350\n\n # Test edge cases\n def test_missing_field_fails_filter(self):\n \"\"\"Test that missing metadata fields fail filters.\"\"\"\n filters = {\"nonexistent_field\": {\"==\": \"value\"}}\n result = self.engine.apply_filters(self.sample_results, filters)\n assert len(result) == 0\n\n def test_invalid_operator(self):\n \"\"\"Test that invalid operators are handled gracefully.\"\"\"\n filters = {\"chapter\": {\"invalid_op\": 1}}\n result = self.engine.apply_filters(self.sample_results, filters)\n assert len(result) == 0 # Should filter out all results\n\n def test_type_coercion_numeric(self):\n \"\"\"Test numeric type coercion in comparisons.\"\"\"\n # Add a result with string chapter number\n test_results = [\n *self.sample_results,\n {\n \"id\": \"doc5\",\n \"score\": 0.75,\n \"text\": \"String chapter test\",\n \"metadata\": {\"chapter\": \"2\", \"genre\": \"test\"},\n },\n ]\n\n filters = {\"chapter\": {\"<\": 3}}\n result = self.engine.apply_filters(test_results, filters)\n # Should include doc1 (chapter=1) and doc5 (chapter=\"2\")\n assert len(result) == 2\n ids = [r[\"id\"] for r in result]\n assert \"doc1\" in ids\n assert \"doc5\" in ids\n\n def test_list_membership_with_nested_tags(self):\n \"\"\"Test membership operations with list metadata.\"\"\"\n # Note: This tests the metadata structure, not list field filtering\n # For list field filtering, we'd need to modify the test data\n filters = {\"character\": {\"in\": [\"Alice\"]}}\n result = self.engine.apply_filters(self.sample_results, filters)\n assert len(result) == 2\n assert all(r[\"metadata\"][\"character\"] == \"Alice\" for r in result)\n\n def test_empty_results_list(self):\n \"\"\"Test filtering on empty results list.\"\"\"\n filters = {\"chapter\": {\"==\": 1}}\n result = self.engine.apply_filters([], filters)\n assert len(result) == 0\n\n\nclass TestPassageManagerFiltering:\n \"\"\"Test suite for PassageManager filtering integration.\"\"\"\n\n def setup_method(self):\n \"\"\"Setup test fixtures.\"\"\"\n # Mock the passage manager without actual file I/O\n self.passage_manager = Mock(spec=PassageManager)\n self.passage_manager.filter_engine = MetadataFilterEngine()\n\n # Sample SearchResult objects\n self.search_results = [\n SearchResult(\n id=\"doc1\",\n score=0.95,\n text=\"Chapter 1 content\",\n metadata={\"chapter\": 1, \"character\": \"Alice\"},\n ),\n SearchResult(\n id=\"doc2\",\n score=0.87,\n text=\"Chapter 5 content\",\n metadata={\"chapter\": 5, \"character\": \"Bob\"},\n ),\n SearchResult(\n id=\"doc3\",\n score=0.82,\n text=\"Chapter 10 content\",\n metadata={\"chapter\": 10, \"character\": \"Alice\"},\n ),\n ]\n\n def test_search_result_filtering(self):\n \"\"\"Test filtering SearchResult objects.\"\"\"\n # Create a real PassageManager instance just for the filtering method\n # We'll mock the file operations\n with patch(\"builtins.open\"), patch(\"json.loads\"), patch(\"pickle.load\"):\n pm = PassageManager([{\"type\": \"jsonl\", \"path\": \"test.jsonl\"}])\n\n filters = {\"chapter\": {\"<=\": 5}}\n result = pm.filter_search_results(self.search_results, filters)\n\n assert len(result) == 2\n chapters = [r.metadata[\"chapter\"] for r in result]\n assert all(ch <= 5 for ch in chapters)\n\n def test_filter_search_results_no_filters(self):\n \"\"\"Test that None filters return all results.\"\"\"\n with patch(\"builtins.open\"), patch(\"json.loads\"), patch(\"pickle.load\"):\n pm = PassageManager([{\"type\": \"jsonl\", \"path\": \"test.jsonl\"}])\n\n result = pm.filter_search_results(self.search_results, None)\n assert len(result) == len(self.search_results)\n\n def test_filter_maintains_search_result_type(self):\n \"\"\"Test that filtering returns SearchResult objects.\"\"\"\n with patch(\"builtins.open\"), patch(\"json.loads\"), patch(\"pickle.load\"):\n pm = PassageManager([{\"type\": \"jsonl\", \"path\": \"test.jsonl\"}])\n\n filters = {\"character\": {\"==\": \"Alice\"}}\n result = pm.filter_search_results(self.search_results, filters)\n\n assert len(result) == 2\n for r in result:\n assert isinstance(r, SearchResult)\n assert r.metadata[\"character\"] == \"Alice\"\n\n\n# Integration tests would go here, but they require actual LEANN backend setup\n# These would test the full pipeline from LeannSearcher.search() with metadata_filters\n\nif __name__ == \"__main__\":\n # Run basic smoke tests\n engine = MetadataFilterEngine()\n\n sample_data = [\n {\n \"id\": \"test1\",\n \"score\": 0.9,\n \"text\": \"Test content\",\n \"metadata\": {\"chapter\": 1, \"published\": True},\n }\n ]\n\n # Test basic filtering\n result = engine.apply_filters(sample_data, {\"chapter\": {\"==\": 1}})\n assert len(result) == 1\n print(\"✅ Basic filtering test passed\")\n\n result = engine.apply_filters(sample_data, {\"chapter\": {\"==\": 2}})\n assert len(result) == 0\n print(\"✅ No match filtering test passed\")\n\n print(\"🎉 All smoke tests passed!\")\n" }, { "path": "tests/test_minimax_provider.py", "content": "\"\"\"\nTests for MiniMax provider integration.\n\nThese tests validate MiniMax provider settings, chat class, and factory integration.\nWe import from leann.settings and leann.chat directly to avoid triggering\nthe full leann.__init__ import chain which requires C++ backend builds.\n\"\"\"\n\nimport os\nimport sys\nfrom unittest.mock import MagicMock, patch\n\nimport pytest\n\n# Add the leann-core source to sys.path so we can import submodules\n# without triggering __init__.py's backend imports.\n_LEANN_SRC = os.path.join(os.path.dirname(__file__), \"..\", \"packages\", \"leann-core\", \"src\")\nif _LEANN_SRC not in sys.path:\n sys.path.insert(0, os.path.abspath(_LEANN_SRC))\n\n# Prevent leann.__init__ from running its heavy imports by pre-registering\n# a lightweight stub in sys.modules (if not already present).\nif \"leann\" not in sys.modules:\n import types\n\n _stub = types.ModuleType(\"leann\")\n _stub.__path__ = [os.path.join(os.path.abspath(_LEANN_SRC), \"leann\")]\n sys.modules[\"leann\"] = _stub\n\n# Now we can safely import the modules we actually test.\nfrom leann.settings import ( # noqa: E402\n resolve_minimax_api_key,\n resolve_minimax_base_url,\n)\n\n\nclass TestMiniMaxSettings:\n \"\"\"Test MiniMax settings resolver functions.\"\"\"\n\n def test_resolve_minimax_api_key_explicit(self):\n assert resolve_minimax_api_key(\"test-key\") == \"test-key\"\n\n def test_resolve_minimax_api_key_from_env(self):\n with patch.dict(os.environ, {\"MINIMAX_API_KEY\": \"env-key\"}):\n assert resolve_minimax_api_key() == \"env-key\"\n\n def test_resolve_minimax_api_key_none(self):\n with patch.dict(os.environ, {}, clear=True):\n assert resolve_minimax_api_key() is None\n\n def test_resolve_minimax_base_url_default(self):\n with patch.dict(os.environ, {}, clear=True):\n assert resolve_minimax_base_url() == \"https://api.minimax.io/v1\"\n\n def test_resolve_minimax_base_url_explicit(self):\n assert resolve_minimax_base_url(\"https://custom.url/v1\") == \"https://custom.url/v1\"\n\n def test_resolve_minimax_base_url_from_env(self):\n with patch.dict(os.environ, {\"MINIMAX_BASE_URL\": \"https://env.url/v1\"}):\n assert resolve_minimax_base_url() == \"https://env.url/v1\"\n\n def test_resolve_minimax_base_url_leann_env(self):\n with patch.dict(os.environ, {\"LEANN_MINIMAX_BASE_URL\": \"https://leann.url/v1\"}):\n assert resolve_minimax_base_url() == \"https://leann.url/v1\"\n\n def test_resolve_minimax_base_url_strips_trailing_slash(self):\n assert resolve_minimax_base_url(\"https://api.minimax.io/v1/\") == \"https://api.minimax.io/v1\"\n\n\nclass TestMiniMaxChat:\n \"\"\"Test MiniMaxChat class.\"\"\"\n\n def test_init_requires_api_key(self):\n from leann.chat import MiniMaxChat\n\n with patch.dict(os.environ, {}, clear=True):\n with pytest.raises(ValueError, match=\"MiniMax API key is required\"):\n MiniMaxChat(api_key=None)\n\n @patch(\"openai.OpenAI\")\n def test_init_with_api_key(self, mock_openai_cls):\n from leann.chat import MiniMaxChat\n\n chat = MiniMaxChat(api_key=\"test-key\")\n assert chat.model == \"MiniMax-M2.5\"\n assert chat.api_key == \"test-key\"\n assert chat.base_url == \"https://api.minimax.io/v1\"\n mock_openai_cls.assert_called_once_with(\n api_key=\"test-key\", base_url=\"https://api.minimax.io/v1\"\n )\n\n @patch(\"openai.OpenAI\")\n def test_init_custom_model(self, mock_openai_cls):\n from leann.chat import MiniMaxChat\n\n chat = MiniMaxChat(model=\"MiniMax-M2.5-highspeed\", api_key=\"test-key\")\n assert chat.model == \"MiniMax-M2.5-highspeed\"\n\n @patch(\"openai.OpenAI\")\n def test_init_custom_base_url(self, mock_openai_cls):\n from leann.chat import MiniMaxChat\n\n chat = MiniMaxChat(api_key=\"test-key\", base_url=\"https://api.minimaxi.com/v1\")\n assert chat.base_url == \"https://api.minimaxi.com/v1\"\n\n @patch(\"openai.OpenAI\")\n def test_ask_returns_response(self, mock_openai_cls):\n from leann.chat import MiniMaxChat\n\n # Mock the response chain\n mock_client = MagicMock()\n mock_openai_cls.return_value = mock_client\n\n mock_response = MagicMock()\n mock_response.choices = [MagicMock()]\n mock_response.choices[0].message.content = \"Hello from MiniMax!\"\n mock_response.choices[0].finish_reason = \"stop\"\n mock_response.usage.total_tokens = 100\n mock_response.usage.prompt_tokens = 50\n mock_response.usage.completion_tokens = 50\n mock_client.chat.completions.create.return_value = mock_response\n\n chat = MiniMaxChat(api_key=\"test-key\")\n result = chat.ask(\"Hello\")\n\n assert result == \"Hello from MiniMax!\"\n mock_client.chat.completions.create.assert_called_once()\n\n @patch(\"openai.OpenAI\")\n def test_ask_with_kwargs(self, mock_openai_cls):\n from leann.chat import MiniMaxChat\n\n mock_client = MagicMock()\n mock_openai_cls.return_value = mock_client\n\n mock_response = MagicMock()\n mock_response.choices = [MagicMock()]\n mock_response.choices[0].message.content = \"Response\"\n mock_response.choices[0].finish_reason = \"stop\"\n mock_response.usage.total_tokens = 50\n mock_response.usage.prompt_tokens = 25\n mock_response.usage.completion_tokens = 25\n mock_client.chat.completions.create.return_value = mock_response\n\n chat = MiniMaxChat(api_key=\"test-key\")\n chat.ask(\"Hello\", temperature=0.5, max_tokens=500, top_p=0.9)\n\n call_kwargs = mock_client.chat.completions.create.call_args[1]\n assert call_kwargs[\"temperature\"] == 0.5\n assert call_kwargs[\"max_tokens\"] == 500\n assert call_kwargs[\"top_p\"] == 0.9\n\n @patch(\"openai.OpenAI\")\n def test_ask_handles_error(self, mock_openai_cls):\n from leann.chat import MiniMaxChat\n\n mock_client = MagicMock()\n mock_openai_cls.return_value = mock_client\n mock_client.chat.completions.create.side_effect = Exception(\"API error\")\n\n chat = MiniMaxChat(api_key=\"test-key\")\n result = chat.ask(\"Hello\")\n\n assert \"Error\" in result\n assert \"MiniMax\" in result\n\n\nclass TestGetLLMFactory:\n \"\"\"Test get_llm factory function with minimax type.\"\"\"\n\n @patch(\"openai.OpenAI\")\n def test_get_llm_minimax(self, mock_openai_cls):\n from leann.chat import MiniMaxChat, get_llm\n\n llm = get_llm({\"type\": \"minimax\", \"api_key\": \"test-key\"})\n assert isinstance(llm, MiniMaxChat)\n assert llm.model == \"MiniMax-M2.5\"\n\n @patch(\"openai.OpenAI\")\n def test_get_llm_minimax_custom_model(self, mock_openai_cls):\n from leann.chat import MiniMaxChat, get_llm\n\n llm = get_llm({\"type\": \"minimax\", \"model\": \"MiniMax-M2.5-highspeed\", \"api_key\": \"test-key\"})\n assert isinstance(llm, MiniMaxChat)\n assert llm.model == \"MiniMax-M2.5-highspeed\"\n\n @patch(\"openai.OpenAI\")\n def test_get_llm_minimax_custom_base_url(self, mock_openai_cls):\n from leann.chat import MiniMaxChat, get_llm\n\n llm = get_llm(\n {\n \"type\": \"minimax\",\n \"api_key\": \"test-key\",\n \"base_url\": \"https://api.minimaxi.com/v1\",\n }\n )\n assert isinstance(llm, MiniMaxChat)\n assert llm.base_url == \"https://api.minimaxi.com/v1\"\n\n\n@pytest.mark.skipif(\n not os.getenv(\"MINIMAX_API_KEY\"),\n reason=\"MINIMAX_API_KEY not set; skipping live API test\",\n)\nclass TestMiniMaxLiveAPI:\n \"\"\"Live API tests for MiniMax provider (requires MINIMAX_API_KEY).\"\"\"\n\n def test_minimax_m25_live(self):\n from leann.chat import MiniMaxChat\n\n chat = MiniMaxChat(model=\"MiniMax-M2.5\")\n response = chat.ask(\"Say hello in one word.\", max_tokens=10)\n assert isinstance(response, str)\n assert len(response) > 0\n\n def test_minimax_m25_highspeed_live(self):\n from leann.chat import MiniMaxChat\n\n chat = MiniMaxChat(model=\"MiniMax-M2.5-highspeed\")\n response = chat.ask(\"Say hello in one word.\", max_tokens=10)\n assert isinstance(response, str)\n assert len(response) > 0\n\n def test_minimax_via_get_llm_live(self):\n from leann.chat import get_llm\n\n llm = get_llm({\"type\": \"minimax\"})\n response = llm.ask(\"What is 1+1? Reply with just the number.\", max_tokens=10)\n assert isinstance(response, str)\n assert len(response) > 0\n" }, { "path": "tests/test_prompt_template_e2e.py", "content": "\"\"\"End-to-end integration tests for prompt template and token limit features.\n\nThese tests verify real-world functionality with live services:\n- OpenAI-compatible APIs (OpenAI, LM Studio) with prompt template support\n- Ollama with dynamic token limit detection\n- Hybrid token limit discovery mechanism\n\nRun with: pytest tests/test_prompt_template_e2e.py -v -s\nSkip if services unavailable: pytest tests/test_prompt_template_e2e.py -m \"not integration\"\n\nPrerequisites:\n1. LM Studio running with embedding model: http://localhost:1234\n2. [Optional] Ollama running: ollama serve\n3. [Optional] Ollama model: ollama pull nomic-embed-text\n4. [Optional] Node.js + @lmstudio/sdk for context length detection\n\"\"\"\n\nimport logging\nimport socket\n\nimport numpy as np\nimport pytest\nimport requests\nfrom leann.embedding_compute import (\n compute_embeddings_ollama,\n compute_embeddings_openai,\n get_model_token_limit,\n)\n\n# Test markers for conditional execution\npytestmark = pytest.mark.integration\n\nlogger = logging.getLogger(__name__)\n\n\ndef check_service_available(host: str, port: int, timeout: float = 2.0) -> bool:\n \"\"\"Check if a service is available on the given host:port.\"\"\"\n try:\n sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)\n sock.settimeout(timeout)\n result = sock.connect_ex((host, port))\n sock.close()\n return result == 0\n except Exception:\n return False\n\n\ndef check_ollama_available() -> bool:\n \"\"\"Check if Ollama service is available.\"\"\"\n if not check_service_available(\"localhost\", 11434):\n return False\n try:\n response = requests.get(\"http://localhost:11434/api/tags\", timeout=2.0)\n return response.status_code == 200\n except Exception:\n return False\n\n\ndef check_lmstudio_available() -> bool:\n \"\"\"Check if LM Studio service is available.\"\"\"\n if not check_service_available(\"localhost\", 1234):\n return False\n try:\n response = requests.get(\"http://localhost:1234/v1/models\", timeout=2.0)\n return response.status_code == 200\n except Exception:\n return False\n\n\ndef get_lmstudio_first_model() -> str | None:\n \"\"\"Get the first available model from LM Studio.\"\"\"\n try:\n response = requests.get(\"http://localhost:1234/v1/models\", timeout=5.0)\n data = response.json()\n models = data.get(\"data\", [])\n if models:\n return models[0][\"id\"]\n except Exception:\n pass\n return None\n\n\nclass TestPromptTemplateOpenAI:\n \"\"\"End-to-end tests for prompt template with OpenAI-compatible APIs (LM Studio).\"\"\"\n\n @pytest.mark.skipif(\n not check_lmstudio_available(), reason=\"LM Studio service not available on localhost:1234\"\n )\n def test_lmstudio_embedding_with_prompt_template(self):\n \"\"\"Test prompt templates with LM Studio using OpenAI-compatible API.\"\"\"\n model_name = get_lmstudio_first_model()\n if not model_name:\n pytest.skip(\"No models loaded in LM Studio\")\n assert model_name is not None # Type narrowing for type checker\n\n texts = [\"artificial intelligence\", \"machine learning\"]\n prompt_template = \"search_query: \"\n\n # Get embeddings with prompt template via provider_options\n provider_options = {\"prompt_template\": prompt_template}\n embeddings = compute_embeddings_openai(\n texts=texts,\n model_name=model_name,\n base_url=\"http://localhost:1234/v1\",\n api_key=\"lm-studio\", # LM Studio doesn't require real key\n provider_options=provider_options,\n )\n\n assert embeddings is not None\n assert len(embeddings) == 2\n assert all(isinstance(emb, np.ndarray) for emb in embeddings)\n assert all(len(emb) > 0 for emb in embeddings)\n\n logger.info(\n f\"✓ LM Studio embeddings with prompt template: {len(embeddings)} vectors, {len(embeddings[0])} dimensions\"\n )\n\n @pytest.mark.skipif(not check_lmstudio_available(), reason=\"LM Studio service not available\")\n def test_lmstudio_prompt_template_affects_embeddings(self):\n \"\"\"Verify that prompt templates actually change embedding values.\"\"\"\n model_name = get_lmstudio_first_model()\n if not model_name:\n pytest.skip(\"No models loaded in LM Studio\")\n assert model_name is not None # Type narrowing for type checker\n\n text = \"machine learning\"\n base_url = \"http://localhost:1234/v1\"\n api_key = \"lm-studio\"\n\n # Get embeddings without template\n embeddings_no_template = compute_embeddings_openai(\n texts=[text],\n model_name=model_name,\n base_url=base_url,\n api_key=api_key,\n provider_options={},\n )\n\n # Get embeddings with template\n embeddings_with_template = compute_embeddings_openai(\n texts=[text],\n model_name=model_name,\n base_url=base_url,\n api_key=api_key,\n provider_options={\"prompt_template\": \"search_query: \"},\n )\n\n # Embeddings should be different when template is applied\n assert not np.allclose(embeddings_no_template[0], embeddings_with_template[0])\n\n logger.info(\"✓ Prompt template changes embedding values as expected\")\n\n\nclass TestPromptTemplateOllama:\n \"\"\"End-to-end tests for prompt template with Ollama.\"\"\"\n\n @pytest.mark.skipif(\n not check_ollama_available(), reason=\"Ollama service not available on localhost:11434\"\n )\n def test_ollama_embedding_with_prompt_template(self):\n \"\"\"Test prompt templates with Ollama using any available embedding model.\"\"\"\n # Get any available embedding model\n try:\n response = requests.get(\"http://localhost:11434/api/tags\", timeout=2.0)\n models = response.json().get(\"models\", [])\n\n embedding_models = []\n for model in models:\n name = model[\"name\"]\n base_name = name.split(\":\")[0]\n if any(emb in base_name for emb in [\"embed\", \"bge\", \"minilm\", \"e5\", \"nomic\"]):\n embedding_models.append(name)\n\n if not embedding_models:\n pytest.skip(\"No embedding models available in Ollama\")\n\n model_name = embedding_models[0]\n\n texts = [\"artificial intelligence\", \"machine learning\"]\n prompt_template = \"search_query: \"\n\n # Get embeddings with prompt template via provider_options\n provider_options = {\"prompt_template\": prompt_template}\n embeddings = compute_embeddings_ollama(\n texts=texts,\n model_name=model_name,\n is_build=False,\n host=\"http://localhost:11434\",\n provider_options=provider_options,\n )\n\n assert embeddings is not None\n assert len(embeddings) == 2\n assert all(isinstance(emb, np.ndarray) for emb in embeddings)\n assert all(len(emb) > 0 for emb in embeddings)\n\n logger.info(\n f\"✓ Ollama embeddings with prompt template: {len(embeddings)} vectors, {len(embeddings[0])} dimensions\"\n )\n\n except Exception as e:\n pytest.skip(f\"Could not test Ollama prompt template: {e}\")\n\n @pytest.mark.skipif(not check_ollama_available(), reason=\"Ollama service not available\")\n def test_ollama_prompt_template_affects_embeddings(self):\n \"\"\"Verify that prompt templates actually change embedding values with Ollama.\"\"\"\n # Get any available embedding model\n try:\n response = requests.get(\"http://localhost:11434/api/tags\", timeout=2.0)\n models = response.json().get(\"models\", [])\n\n embedding_models = []\n for model in models:\n name = model[\"name\"]\n base_name = name.split(\":\")[0]\n if any(emb in base_name for emb in [\"embed\", \"bge\", \"minilm\", \"e5\", \"nomic\"]):\n embedding_models.append(name)\n\n if not embedding_models:\n pytest.skip(\"No embedding models available in Ollama\")\n\n model_name = embedding_models[0]\n text = \"machine learning\"\n host = \"http://localhost:11434\"\n\n # Get embeddings without template\n embeddings_no_template = compute_embeddings_ollama(\n texts=[text], model_name=model_name, is_build=False, host=host, provider_options={}\n )\n\n # Get embeddings with template\n embeddings_with_template = compute_embeddings_ollama(\n texts=[text],\n model_name=model_name,\n is_build=False,\n host=host,\n provider_options={\"prompt_template\": \"search_query: \"},\n )\n\n # Embeddings should be different when template is applied\n assert not np.allclose(embeddings_no_template[0], embeddings_with_template[0])\n\n logger.info(\"✓ Ollama prompt template changes embedding values as expected\")\n\n except Exception as e:\n pytest.skip(f\"Could not test Ollama prompt template: {e}\")\n\n\nclass TestLMStudioSDK:\n \"\"\"End-to-end tests for LM Studio SDK integration.\"\"\"\n\n @pytest.mark.skipif(not check_lmstudio_available(), reason=\"LM Studio service not available\")\n def test_lmstudio_model_listing(self):\n \"\"\"Test that we can list models from LM Studio.\"\"\"\n try:\n response = requests.get(\"http://localhost:1234/v1/models\", timeout=5.0)\n assert response.status_code == 200\n\n data = response.json()\n assert \"data\" in data\n\n models = data[\"data\"]\n logger.info(f\"✓ LM Studio models available: {len(models)}\")\n\n if models:\n logger.info(f\" First model: {models[0].get('id', 'unknown')}\")\n except Exception as e:\n pytest.skip(f\"LM Studio API error: {e}\")\n\n @pytest.mark.skipif(not check_lmstudio_available(), reason=\"LM Studio service not available\")\n def test_lmstudio_sdk_context_length_detection(self):\n \"\"\"Test context length detection via LM Studio SDK bridge (requires Node.js + SDK).\"\"\"\n model_name = get_lmstudio_first_model()\n if not model_name:\n pytest.skip(\"No models loaded in LM Studio\")\n assert model_name is not None # Type narrowing for type checker\n\n try:\n from leann.embedding_compute import _query_lmstudio_context_limit\n\n # SDK requires WebSocket URL (ws://)\n context_length = _query_lmstudio_context_limit(\n model_name=model_name, base_url=\"ws://localhost:1234\"\n )\n\n if context_length is None:\n logger.warning(\n \"⚠ LM Studio SDK bridge returned None (Node.js or SDK may not be available)\"\n )\n pytest.skip(\"Node.js or @lmstudio/sdk not available - SDK bridge unavailable\")\n else:\n assert context_length > 0\n logger.info(\n f\"✓ LM Studio context length detected via SDK: {context_length} for {model_name}\"\n )\n\n except ImportError:\n pytest.skip(\"_query_lmstudio_context_limit not implemented yet\")\n except Exception as e:\n logger.error(f\"LM Studio SDK test error: {e}\")\n raise\n\n\nclass TestOllamaTokenLimit:\n \"\"\"End-to-end tests for Ollama token limit discovery.\"\"\"\n\n @pytest.mark.skipif(not check_ollama_available(), reason=\"Ollama service not available\")\n def test_ollama_token_limit_detection(self):\n \"\"\"Test dynamic token limit detection from Ollama /api/show endpoint.\"\"\"\n # Get any available embedding model\n try:\n response = requests.get(\"http://localhost:11434/api/tags\", timeout=2.0)\n models = response.json().get(\"models\", [])\n\n embedding_models = []\n for model in models:\n name = model[\"name\"]\n base_name = name.split(\":\")[0]\n if any(emb in base_name for emb in [\"embed\", \"bge\", \"minilm\", \"e5\", \"nomic\"]):\n embedding_models.append(name)\n\n if not embedding_models:\n pytest.skip(\"No embedding models available in Ollama\")\n\n test_model = embedding_models[0]\n\n # Test token limit detection\n limit = get_model_token_limit(model_name=test_model, base_url=\"http://localhost:11434\")\n\n assert limit > 0\n logger.info(f\"✓ Ollama token limit detected: {limit} for {test_model}\")\n\n except Exception as e:\n pytest.skip(f\"Could not test Ollama token detection: {e}\")\n\n\nclass TestHybridTokenLimit:\n \"\"\"End-to-end tests for hybrid token limit discovery mechanism.\"\"\"\n\n def test_hybrid_discovery_registry_fallback(self):\n \"\"\"Test fallback to static registry for known OpenAI models.\"\"\"\n # Use a known OpenAI model (should be in registry)\n limit = get_model_token_limit(\n model_name=\"text-embedding-3-small\",\n base_url=\"http://fake-server:9999\", # Fake URL to force registry lookup\n )\n\n # text-embedding-3-small should have 8192 in registry\n assert limit == 8192\n logger.info(f\"✓ Hybrid discovery (registry fallback): {limit} tokens\")\n\n def test_hybrid_discovery_default_fallback(self):\n \"\"\"Test fallback to safe default for completely unknown models.\"\"\"\n limit = get_model_token_limit(\n model_name=\"completely-unknown-model-xyz-12345\",\n base_url=\"http://fake-server:9999\",\n default=512,\n )\n\n # Should get the specified default\n assert limit == 512\n logger.info(f\"✓ Hybrid discovery (default fallback): {limit} tokens\")\n\n @pytest.mark.skipif(not check_ollama_available(), reason=\"Ollama service not available\")\n def test_hybrid_discovery_ollama_dynamic_first(self):\n \"\"\"Test that Ollama models use dynamic discovery first.\"\"\"\n # Get any available embedding model\n try:\n response = requests.get(\"http://localhost:11434/api/tags\", timeout=2.0)\n models = response.json().get(\"models\", [])\n\n embedding_models = []\n for model in models:\n name = model[\"name\"]\n base_name = name.split(\":\")[0]\n if any(emb in base_name for emb in [\"embed\", \"bge\", \"minilm\", \"e5\", \"nomic\"]):\n embedding_models.append(name)\n\n if not embedding_models:\n pytest.skip(\"No embedding models available in Ollama\")\n\n test_model = embedding_models[0]\n\n # Should query Ollama /api/show dynamically\n limit = get_model_token_limit(model_name=test_model, base_url=\"http://localhost:11434\")\n\n assert limit > 0\n logger.info(f\"✓ Hybrid discovery (Ollama dynamic): {limit} tokens for {test_model}\")\n\n except Exception as e:\n pytest.skip(f\"Could not test hybrid Ollama discovery: {e}\")\n\n\nif __name__ == \"__main__\":\n print(\"\\n\" + \"=\" * 70)\n print(\"INTEGRATION TEST SUITE - Real Service Testing\")\n print(\"=\" * 70)\n print(\"\\nThese tests require live services:\")\n print(\" • LM Studio: http://localhost:1234 (with embedding model loaded)\")\n print(\" • [Optional] Ollama: http://localhost:11434\")\n print(\" • [Optional] Node.js + @lmstudio/sdk for SDK bridge tests\")\n print(\"\\nRun with: pytest tests/test_prompt_template_e2e.py -v -s\")\n print(\"=\" * 70 + \"\\n\")\n" }, { "path": "tests/test_prompt_template_persistence.py", "content": "\"\"\"\nIntegration tests for prompt template metadata persistence and reuse.\n\nThese tests verify the complete lifecycle of prompt template persistence:\n1. Template is saved to .meta.json during index build\n2. Template is automatically loaded during search operations\n3. Template can be overridden with explicit flag during search\n4. Template is reused during chat/ask operations\n\nThese are integration tests that:\n- Use real file system with temporary directories\n- Run actual build and search operations\n- Inspect .meta.json file contents directly\n- Mock embedding servers to avoid external dependencies\n- Use small test codebases for fast execution\n\nExpected to FAIL in Red Phase because metadata persistence verification is not yet implemented.\n\"\"\"\n\nimport json\nimport tempfile\nfrom pathlib import Path\nfrom unittest.mock import Mock, patch\n\nimport numpy as np\nimport pytest\nfrom leann.api import LeannBuilder, LeannSearcher\n\n\nclass TestPromptTemplateMetadataPersistence:\n \"\"\"Tests for prompt template storage in .meta.json during build.\"\"\"\n\n @pytest.fixture\n def temp_index_dir(self):\n \"\"\"Create temporary directory for test indexes.\"\"\"\n with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir:\n yield Path(tmpdir)\n\n @pytest.fixture\n def mock_embeddings(self):\n \"\"\"Mock compute_embeddings to return dummy embeddings.\"\"\"\n with patch(\"leann.api.compute_embeddings\") as mock_compute:\n # Return dummy embeddings as numpy array\n mock_compute.return_value = np.array([[0.1, 0.2, 0.3]], dtype=np.float32)\n yield mock_compute\n\n def test_prompt_template_saved_to_metadata(self, temp_index_dir, mock_embeddings):\n \"\"\"\n Verify that when build is run with embedding_options containing prompt_template,\n the template value is saved to .meta.json file.\n\n This is the core persistence requirement - templates must be saved to allow\n reuse in subsequent search operations without re-specifying the flag.\n\n Expected failure: .meta.json exists but doesn't contain embedding_options\n with prompt_template, or the value is not persisted correctly.\n \"\"\"\n # Setup test data\n index_path = temp_index_dir / \"test_index.leann\"\n template = \"search_document: \"\n\n # Build index with prompt template in embedding_options\n builder = LeannBuilder(\n backend_name=\"hnsw\",\n embedding_model=\"text-embedding-3-small\",\n embedding_mode=\"openai\",\n embedding_options={\"prompt_template\": template},\n )\n\n # Add a simple document\n builder.add_text(\"This is a test document for indexing\")\n\n # Build the index\n builder.build_index(str(index_path))\n\n # Verify .meta.json was created and contains the template\n meta_path = temp_index_dir / \"test_index.leann.meta.json\"\n assert meta_path.exists(), \".meta.json file should be created during build\"\n\n # Read and parse metadata\n with open(meta_path, encoding=\"utf-8\") as f:\n meta_data = json.load(f)\n\n # Verify embedding_options exists in metadata\n assert \"embedding_options\" in meta_data, (\n \"embedding_options should be saved to .meta.json when provided\"\n )\n\n # Verify prompt_template is in embedding_options\n embedding_options = meta_data[\"embedding_options\"]\n assert \"prompt_template\" in embedding_options, (\n \"prompt_template should be saved within embedding_options\"\n )\n\n # Verify the template value matches what we provided\n assert embedding_options[\"prompt_template\"] == template, (\n f\"Template should be '{template}', got '{embedding_options.get('prompt_template')}'\"\n )\n\n def test_prompt_template_absent_when_not_provided(self, temp_index_dir, mock_embeddings):\n \"\"\"\n Verify that when no prompt template is provided during build,\n .meta.json either doesn't have embedding_options or prompt_template key.\n\n This ensures clean metadata without unnecessary keys when features aren't used.\n\n Expected behavior: Build succeeds, .meta.json doesn't contain prompt_template.\n \"\"\"\n index_path = temp_index_dir / \"test_no_template.leann\"\n\n # Build index WITHOUT prompt template\n builder = LeannBuilder(\n backend_name=\"hnsw\",\n embedding_model=\"text-embedding-3-small\",\n embedding_mode=\"openai\",\n # No embedding_options provided\n )\n\n builder.add_text(\"Document without template\")\n builder.build_index(str(index_path))\n\n # Verify metadata\n meta_path = temp_index_dir / \"test_no_template.leann.meta.json\"\n assert meta_path.exists()\n\n with open(meta_path, encoding=\"utf-8\") as f:\n meta_data = json.load(f)\n\n # If embedding_options exists, it should not contain prompt_template\n if \"embedding_options\" in meta_data:\n embedding_options = meta_data[\"embedding_options\"]\n assert \"prompt_template\" not in embedding_options, (\n \"prompt_template should not be in metadata when not provided\"\n )\n\n\nclass TestPromptTemplateAutoLoadOnSearch:\n \"\"\"Tests for automatic loading of prompt template during search operations.\n\n NOTE: Over-mocked test removed (test_prompt_template_auto_loaded_on_search).\n This functionality is now comprehensively tested by TestQueryPromptTemplateAutoLoad\n which uses simpler mocking and doesn't hang.\n \"\"\"\n\n @pytest.fixture\n def temp_index_dir(self):\n \"\"\"Create temporary directory for test indexes.\"\"\"\n with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir:\n yield Path(tmpdir)\n\n @pytest.fixture\n def mock_embeddings(self):\n \"\"\"Mock compute_embeddings to capture calls and return dummy embeddings.\"\"\"\n with patch(\"leann.api.compute_embeddings\") as mock_compute:\n mock_compute.return_value = np.array([[0.1, 0.2, 0.3]], dtype=np.float32)\n yield mock_compute\n\n def test_search_without_template_in_metadata(self, temp_index_dir, mock_embeddings):\n \"\"\"\n Verify that searching an index built WITHOUT a prompt template\n works correctly (backward compatibility).\n\n The searcher should handle missing prompt_template gracefully.\n\n Expected behavior: Search succeeds, no template is used.\n \"\"\"\n # Build index without template\n index_path = temp_index_dir / \"no_template.leann\"\n\n builder = LeannBuilder(\n backend_name=\"hnsw\",\n embedding_model=\"text-embedding-3-small\",\n embedding_mode=\"openai\",\n )\n builder.add_text(\"Document without template\")\n builder.build_index(str(index_path))\n\n # Reset mocks\n mock_embeddings.reset_mock()\n\n # Create searcher and search\n searcher = LeannSearcher(index_path=str(index_path))\n\n # Verify no template in embedding_options\n assert \"prompt_template\" not in searcher.embedding_options, (\n \"Searcher should not have prompt_template when not in metadata\"\n )\n\n\nclass TestQueryPromptTemplateAutoLoad:\n \"\"\"Tests for automatic loading of separate query_prompt_template during search (R2).\n\n These tests verify the new two-template system where:\n - build_prompt_template: Applied during index building\n - query_prompt_template: Applied during search operations\n\n Expected to FAIL in Red Phase (R2) because query template extraction\n and application is not yet implemented in LeannSearcher.search().\n \"\"\"\n\n @pytest.fixture\n def temp_index_dir(self):\n \"\"\"Create temporary directory for test indexes.\"\"\"\n with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir:\n yield Path(tmpdir)\n\n @pytest.fixture\n def mock_compute_embeddings(self):\n \"\"\"Mock compute_embeddings to capture calls and return dummy embeddings.\"\"\"\n with patch(\"leann.embedding_compute.compute_embeddings\") as mock_compute:\n mock_compute.return_value = np.array([[0.1, 0.2, 0.3]], dtype=np.float32)\n yield mock_compute\n\n def test_search_auto_loads_query_template(self, temp_index_dir, mock_compute_embeddings):\n \"\"\"\n Verify that search() automatically loads and applies query_prompt_template from .meta.json.\n\n Given: Index built with separate build_prompt_template and query_prompt_template\n When: LeannSearcher.search(\"my query\") is called\n Then: Query embedding is computed with \"query: my query\" (query template applied)\n\n This is the core R2 requirement - query templates must be auto-loaded and applied\n during search without user intervention.\n\n Expected failure: compute_embeddings called with raw \"my query\" instead of\n \"query: my query\" because query template extraction is not implemented.\n \"\"\"\n # Setup: Build index with separate templates in new format\n index_path = temp_index_dir / \"query_template.leann\"\n\n builder = LeannBuilder(\n backend_name=\"hnsw\",\n embedding_model=\"text-embedding-3-small\",\n embedding_mode=\"openai\",\n embedding_options={\n \"build_prompt_template\": \"doc: \",\n \"query_prompt_template\": \"query: \",\n },\n )\n builder.add_text(\"Test document\")\n builder.build_index(str(index_path))\n\n # Reset mock to ignore build calls\n mock_compute_embeddings.reset_mock()\n\n # Act: Search with query\n searcher = LeannSearcher(index_path=str(index_path))\n\n # Mock the backend search to avoid actual search\n with patch.object(searcher.backend_impl, \"search\") as mock_backend_search:\n mock_backend_search.return_value = {\n \"labels\": [[\"test_id_0\"]], # IDs (nested list for batch support)\n \"distances\": [[0.9]], # Distances (nested list for batch support)\n }\n\n searcher.search(\"my query\", top_k=1, recompute_embeddings=False)\n\n # Assert: compute_embeddings was called with query template applied\n assert mock_compute_embeddings.called, \"compute_embeddings should be called during search\"\n\n # Get the actual text passed to compute_embeddings\n call_args = mock_compute_embeddings.call_args\n texts_arg = call_args[0][0] # First positional arg (list of texts)\n\n assert len(texts_arg) == 1, \"Should compute embedding for one query\"\n assert texts_arg[0] == \"query: my query\", (\n f\"Query template should be applied: expected 'query: my query', got '{texts_arg[0]}'\"\n )\n\n def test_search_backward_compat_single_template(self, temp_index_dir, mock_compute_embeddings):\n \"\"\"\n Verify backward compatibility with old single prompt_template format.\n\n Given: Index with old format (single prompt_template, no query_prompt_template)\n When: LeannSearcher.search(\"my query\") is called\n Then: Query embedding computed with \"doc: my query\" (old template applied)\n\n This ensures indexes built with the old single-template system continue\n to work correctly with the new search implementation.\n\n Expected failure: Old template not recognized/applied because backward\n compatibility logic is not implemented.\n \"\"\"\n # Setup: Build index with old single-template format\n index_path = temp_index_dir / \"old_template.leann\"\n\n builder = LeannBuilder(\n backend_name=\"hnsw\",\n embedding_model=\"text-embedding-3-small\",\n embedding_mode=\"openai\",\n embedding_options={\"prompt_template\": \"doc: \"}, # Old format\n )\n builder.add_text(\"Test document\")\n builder.build_index(str(index_path))\n\n # Reset mock\n mock_compute_embeddings.reset_mock()\n\n # Act: Search\n searcher = LeannSearcher(index_path=str(index_path))\n\n with patch.object(searcher.backend_impl, \"search\") as mock_backend_search:\n mock_backend_search.return_value = {\"labels\": [[\"test_id_0\"]], \"distances\": [[0.9]]}\n\n searcher.search(\"my query\", top_k=1, recompute_embeddings=False)\n\n # Assert: Old template was applied\n call_args = mock_compute_embeddings.call_args\n texts_arg = call_args[0][0]\n\n assert texts_arg[0] == \"doc: my query\", (\n f\"Old prompt_template should be applied for backward compatibility: \"\n f\"expected 'doc: my query', got '{texts_arg[0]}'\"\n )\n\n def test_search_backward_compat_no_template(self, temp_index_dir, mock_compute_embeddings):\n \"\"\"\n Verify backward compatibility when no template is present in .meta.json.\n\n Given: Index with no template in .meta.json (very old indexes)\n When: LeannSearcher.search(\"my query\") is called\n Then: Query embedding computed with \"my query\" (no template, raw query)\n\n This ensures the most basic backward compatibility - indexes without\n any template support continue to work as before.\n\n Expected failure: May fail if default template is incorrectly applied,\n or if missing template causes error.\n \"\"\"\n # Setup: Build index without any template\n index_path = temp_index_dir / \"no_template.leann\"\n\n builder = LeannBuilder(\n backend_name=\"hnsw\",\n embedding_model=\"text-embedding-3-small\",\n embedding_mode=\"openai\",\n # No embedding_options at all\n )\n builder.add_text(\"Test document\")\n builder.build_index(str(index_path))\n\n # Reset mock\n mock_compute_embeddings.reset_mock()\n\n # Act: Search\n searcher = LeannSearcher(index_path=str(index_path))\n\n with patch.object(searcher.backend_impl, \"search\") as mock_backend_search:\n mock_backend_search.return_value = {\"labels\": [[\"test_id_0\"]], \"distances\": [[0.9]]}\n\n searcher.search(\"my query\", top_k=1, recompute_embeddings=False)\n\n # Assert: No template applied (raw query)\n call_args = mock_compute_embeddings.call_args\n texts_arg = call_args[0][0]\n\n assert texts_arg[0] == \"my query\", (\n f\"No template should be applied when missing from metadata: \"\n f\"expected 'my query', got '{texts_arg[0]}'\"\n )\n\n def test_search_override_via_provider_options(self, temp_index_dir, mock_compute_embeddings):\n \"\"\"\n Verify that explicit provider_options can override stored query template.\n\n Given: Index with query_prompt_template: \"query: \"\n When: search() called with provider_options={\"prompt_template\": \"override: \"}\n Then: Query embedding computed with \"override: test\" (override takes precedence)\n\n This enables users to experiment with different query templates without\n rebuilding the index, or to handle special query types differently.\n\n Expected failure: provider_options parameter is accepted via **kwargs but\n not used. Query embedding computed with raw \"test\" instead of \"override: test\"\n because override logic is not implemented.\n \"\"\"\n # Setup: Build index with query template\n index_path = temp_index_dir / \"override_template.leann\"\n\n builder = LeannBuilder(\n backend_name=\"hnsw\",\n embedding_model=\"text-embedding-3-small\",\n embedding_mode=\"openai\",\n embedding_options={\n \"build_prompt_template\": \"doc: \",\n \"query_prompt_template\": \"query: \",\n },\n )\n builder.add_text(\"Test document\")\n builder.build_index(str(index_path))\n\n # Reset mock\n mock_compute_embeddings.reset_mock()\n\n # Act: Search with override\n searcher = LeannSearcher(index_path=str(index_path))\n\n with patch.object(searcher.backend_impl, \"search\") as mock_backend_search:\n mock_backend_search.return_value = {\"labels\": [[\"test_id_0\"]], \"distances\": [[0.9]]}\n\n # This should accept provider_options parameter\n searcher.search(\n \"test\",\n top_k=1,\n recompute_embeddings=False,\n provider_options={\"prompt_template\": \"override: \"},\n )\n\n # Assert: Override template was applied\n call_args = mock_compute_embeddings.call_args\n texts_arg = call_args[0][0]\n\n assert texts_arg[0] == \"override: test\", (\n f\"Override template should take precedence: \"\n f\"expected 'override: test', got '{texts_arg[0]}'\"\n )\n\n\nclass TestPromptTemplateReuseInChat:\n \"\"\"Tests for prompt template reuse in chat/ask operations.\"\"\"\n\n @pytest.fixture\n def temp_index_dir(self):\n \"\"\"Create temporary directory for test indexes.\"\"\"\n with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir:\n yield Path(tmpdir)\n\n @pytest.fixture\n def mock_embeddings(self):\n \"\"\"Mock compute_embeddings to return dummy embeddings.\"\"\"\n with patch(\"leann.api.compute_embeddings\") as mock_compute:\n mock_compute.return_value = np.array([[0.1, 0.2, 0.3]], dtype=np.float32)\n yield mock_compute\n\n @pytest.fixture\n def mock_embedding_server_manager(self):\n \"\"\"Mock EmbeddingServerManager for chat tests.\"\"\"\n with patch(\"leann.searcher_base.EmbeddingServerManager\") as mock_manager_class:\n mock_manager = Mock()\n mock_manager.start_server.return_value = (True, 5557)\n mock_manager_class.return_value = mock_manager\n yield mock_manager\n\n @pytest.fixture\n def index_with_template(self, temp_index_dir, mock_embeddings):\n \"\"\"Build an index with a prompt template.\"\"\"\n index_path = temp_index_dir / \"chat_template_index.leann\"\n template = \"document_query: \"\n\n builder = LeannBuilder(\n backend_name=\"hnsw\",\n embedding_model=\"text-embedding-3-small\",\n embedding_mode=\"openai\",\n embedding_options={\"prompt_template\": template},\n )\n\n builder.add_text(\"Test document for chat\")\n builder.build_index(str(index_path))\n\n return str(index_path), template\n\n\nclass TestPromptTemplateIntegrationWithEmbeddingModes:\n \"\"\"Tests for prompt template compatibility with different embedding modes.\"\"\"\n\n @pytest.fixture\n def temp_index_dir(self):\n \"\"\"Create temporary directory for test indexes.\"\"\"\n with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir:\n yield Path(tmpdir)\n\n @pytest.mark.parametrize(\n \"mode,model,template,filename_prefix\",\n [\n (\n \"openai\",\n \"text-embedding-3-small\",\n \"Represent this for searching: \",\n \"openai_template\",\n ),\n (\"ollama\", \"nomic-embed-text\", \"search_query: \", \"ollama_template\"),\n (\"sentence-transformers\", \"facebook/contriever\", \"query: \", \"st_template\"),\n ],\n )\n def test_prompt_template_metadata_with_embedding_modes(\n self, temp_index_dir, mode, model, template, filename_prefix\n ):\n \"\"\"Verify prompt template is saved correctly across different embedding modes.\n\n Tests that prompt templates are persisted to .meta.json for:\n - OpenAI mode (primary use case)\n - Ollama mode (also supports templates)\n - Sentence-transformers mode (saved for forward compatibility)\n\n Expected behavior: Template is saved to .meta.json regardless of mode.\n \"\"\"\n with patch(\"leann.api.compute_embeddings\") as mock_compute:\n mock_compute.return_value = np.array([[0.1, 0.2, 0.3]], dtype=np.float32)\n\n index_path = temp_index_dir / f\"{filename_prefix}.leann\"\n\n builder = LeannBuilder(\n backend_name=\"hnsw\",\n embedding_model=model,\n embedding_mode=mode,\n embedding_options={\"prompt_template\": template},\n )\n\n builder.add_text(f\"{mode.capitalize()} test document\")\n builder.build_index(str(index_path))\n\n # Verify metadata\n meta_path = temp_index_dir / f\"{filename_prefix}.leann.meta.json\"\n with open(meta_path, encoding=\"utf-8\") as f:\n meta_data = json.load(f)\n\n assert meta_data[\"embedding_mode\"] == mode\n # Template should be saved for all modes (even if not used by some)\n if \"embedding_options\" in meta_data:\n assert meta_data[\"embedding_options\"][\"prompt_template\"] == template\n\n\nclass TestQueryTemplateApplicationInComputeEmbedding:\n \"\"\"Tests for query template application in compute_query_embedding() (Bug Fix).\n\n These tests verify that query templates are applied consistently in BOTH\n code paths (server and fallback) when computing query embeddings.\n\n This addresses the bug where query templates were only applied in the\n fallback path, not when using the embedding server (the default path).\n\n Bug Context:\n - Issue: Query templates were stored in metadata but only applied during\n fallback (direct) computation, not when using embedding server\n - Fix: Move template application to BEFORE any computation path in\n compute_query_embedding() (searcher_base.py:107-110)\n - Impact: Critical for models like EmbeddingGemma that require task-specific\n templates for optimal performance\n\n These tests ensure the fix works correctly and prevent regression.\n \"\"\"\n\n @pytest.fixture\n def temp_index_with_template(self):\n \"\"\"Create a temporary index with query template in metadata\"\"\"\n with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir:\n index_dir = Path(tmpdir)\n index_file = index_dir / \"test.leann\"\n meta_file = index_dir / \"test.leann.meta.json\"\n\n # Create minimal metadata with query template\n metadata = {\n \"version\": \"1.0\",\n \"backend_name\": \"hnsw\",\n \"embedding_model\": \"text-embedding-embeddinggemma-300m-qat\",\n \"dimensions\": 768,\n \"embedding_mode\": \"openai\",\n \"backend_kwargs\": {\n \"graph_degree\": 32,\n \"complexity\": 64,\n \"distance_metric\": \"cosine\",\n },\n \"embedding_options\": {\n \"base_url\": \"http://localhost:1234/v1\",\n \"api_key\": \"test-key\",\n \"build_prompt_template\": \"title: none | text: \",\n \"query_prompt_template\": \"task: search result | query: \",\n },\n }\n\n meta_file.write_text(json.dumps(metadata, indent=2))\n\n # Create minimal HNSW index file (empty is okay for this test)\n index_file.write_bytes(b\"\")\n\n yield str(index_file)\n\n def test_query_template_applied_in_fallback_path(self, temp_index_with_template):\n \"\"\"Test that query template is applied when using fallback (direct) path\"\"\"\n from leann.searcher_base import BaseSearcher\n\n # Create a concrete implementation for testing\n class TestSearcher(BaseSearcher):\n def search(\n self,\n query,\n top_k,\n complexity=64,\n beam_width=1,\n prune_ratio=0.0,\n recompute_embeddings=False,\n pruning_strategy=\"global\",\n zmq_port=None,\n **kwargs,\n ):\n return {\"labels\": [], \"distances\": []}\n\n searcher = object.__new__(TestSearcher)\n searcher.index_path = Path(temp_index_with_template)\n searcher.index_dir = searcher.index_path.parent\n\n # Load metadata\n meta_file = searcher.index_dir / f\"{searcher.index_path.name}.meta.json\"\n with open(meta_file) as f:\n searcher.meta = json.load(f)\n\n searcher.embedding_model = searcher.meta[\"embedding_model\"]\n searcher.embedding_mode = searcher.meta.get(\"embedding_mode\", \"sentence-transformers\")\n searcher.embedding_options = searcher.meta.get(\"embedding_options\", {})\n searcher.enable_warmup = False\n searcher.use_daemon = False\n searcher.daemon_ttl_seconds = 0\n\n # Mock compute_embeddings to capture the query text\n captured_queries = []\n\n def mock_compute_embeddings(texts, model, mode, provider_options=None):\n captured_queries.extend(texts)\n return np.random.rand(len(texts), 768).astype(np.float32)\n\n with patch(\n \"leann.embedding_compute.compute_embeddings\", side_effect=mock_compute_embeddings\n ):\n # Call compute_query_embedding with template (fallback path)\n result = searcher.compute_query_embedding(\n query=\"vector database\",\n use_server_if_available=False, # Force fallback path\n query_template=\"task: search result | query: \",\n )\n\n # Verify template was applied\n assert len(captured_queries) == 1\n assert captured_queries[0] == \"task: search result | query: vector database\"\n assert result.shape == (1, 768)\n\n def test_query_template_applied_in_server_path(self, temp_index_with_template):\n \"\"\"Test that query template is applied when using server path\"\"\"\n from leann.searcher_base import BaseSearcher\n\n # Create a concrete implementation for testing\n class TestSearcher(BaseSearcher):\n def search(\n self,\n query,\n top_k,\n complexity=64,\n beam_width=1,\n prune_ratio=0.0,\n recompute_embeddings=False,\n pruning_strategy=\"global\",\n zmq_port=None,\n **kwargs,\n ):\n return {\"labels\": [], \"distances\": []}\n\n searcher = object.__new__(TestSearcher)\n searcher.index_path = Path(temp_index_with_template)\n searcher.index_dir = searcher.index_path.parent\n\n # Load metadata\n meta_file = searcher.index_dir / f\"{searcher.index_path.name}.meta.json\"\n with open(meta_file) as f:\n searcher.meta = json.load(f)\n\n searcher.embedding_model = searcher.meta[\"embedding_model\"]\n searcher.embedding_mode = searcher.meta.get(\"embedding_mode\", \"sentence-transformers\")\n searcher.embedding_options = searcher.meta.get(\"embedding_options\", {})\n searcher.enable_warmup = False\n searcher.use_daemon = False\n searcher.daemon_ttl_seconds = 0\n\n # Mock the server methods to capture the query text\n captured_queries = []\n\n def mock_ensure_server_running(passages_file, port, **kwargs):\n return port\n\n def mock_compute_embedding_via_server(chunks, port):\n captured_queries.extend(chunks)\n return np.random.rand(len(chunks), 768).astype(np.float32)\n\n searcher._ensure_server_running = mock_ensure_server_running\n searcher._compute_embedding_via_server = mock_compute_embedding_via_server\n\n # Call compute_query_embedding with template (server path)\n result = searcher.compute_query_embedding(\n query=\"vector database\",\n use_server_if_available=True, # Use server path\n query_template=\"task: search result | query: \",\n )\n\n # Verify template was applied BEFORE calling server\n assert len(captured_queries) == 1\n assert captured_queries[0] == \"task: search result | query: vector database\"\n assert result.shape == (1, 768)\n\n def test_query_template_without_template_parameter(self, temp_index_with_template):\n \"\"\"Test that query is unchanged when no template is provided\"\"\"\n from leann.searcher_base import BaseSearcher\n\n class TestSearcher(BaseSearcher):\n def search(\n self,\n query,\n top_k,\n complexity=64,\n beam_width=1,\n prune_ratio=0.0,\n recompute_embeddings=False,\n pruning_strategy=\"global\",\n zmq_port=None,\n **kwargs,\n ):\n return {\"labels\": [], \"distances\": []}\n\n searcher = object.__new__(TestSearcher)\n searcher.index_path = Path(temp_index_with_template)\n searcher.index_dir = searcher.index_path.parent\n\n meta_file = searcher.index_dir / f\"{searcher.index_path.name}.meta.json\"\n with open(meta_file) as f:\n searcher.meta = json.load(f)\n\n searcher.embedding_model = searcher.meta[\"embedding_model\"]\n searcher.embedding_mode = searcher.meta.get(\"embedding_mode\", \"sentence-transformers\")\n searcher.embedding_options = searcher.meta.get(\"embedding_options\", {})\n searcher.enable_warmup = False\n searcher.use_daemon = False\n searcher.daemon_ttl_seconds = 0\n\n captured_queries = []\n\n def mock_compute_embeddings(texts, model, mode, provider_options=None):\n captured_queries.extend(texts)\n return np.random.rand(len(texts), 768).astype(np.float32)\n\n with patch(\n \"leann.embedding_compute.compute_embeddings\", side_effect=mock_compute_embeddings\n ):\n searcher.compute_query_embedding(\n query=\"vector database\",\n use_server_if_available=False,\n query_template=None, # No template\n )\n\n # Verify query is unchanged\n assert len(captured_queries) == 1\n assert captured_queries[0] == \"vector database\"\n\n def test_query_template_consistency_between_paths(self, temp_index_with_template):\n \"\"\"Test that both paths apply template identically\"\"\"\n from leann.searcher_base import BaseSearcher\n\n class TestSearcher(BaseSearcher):\n def search(\n self,\n query,\n top_k,\n complexity=64,\n beam_width=1,\n prune_ratio=0.0,\n recompute_embeddings=False,\n pruning_strategy=\"global\",\n zmq_port=None,\n **kwargs,\n ):\n return {\"labels\": [], \"distances\": []}\n\n searcher = object.__new__(TestSearcher)\n searcher.index_path = Path(temp_index_with_template)\n searcher.index_dir = searcher.index_path.parent\n\n meta_file = searcher.index_dir / f\"{searcher.index_path.name}.meta.json\"\n with open(meta_file) as f:\n searcher.meta = json.load(f)\n\n searcher.embedding_model = searcher.meta[\"embedding_model\"]\n searcher.embedding_mode = searcher.meta.get(\"embedding_mode\", \"sentence-transformers\")\n searcher.embedding_options = searcher.meta.get(\"embedding_options\", {})\n searcher.enable_warmup = False\n searcher.use_daemon = False\n searcher.daemon_ttl_seconds = 0\n\n query_template = \"task: search result | query: \"\n original_query = \"vector database\"\n\n # Capture queries from fallback path\n fallback_queries = []\n\n def mock_compute_embeddings(texts, model, mode, provider_options=None):\n fallback_queries.extend(texts)\n return np.random.rand(len(texts), 768).astype(np.float32)\n\n with patch(\n \"leann.embedding_compute.compute_embeddings\", side_effect=mock_compute_embeddings\n ):\n searcher.compute_query_embedding(\n query=original_query,\n use_server_if_available=False,\n query_template=query_template,\n )\n\n # Capture queries from server path\n server_queries = []\n\n def mock_ensure_server_running(passages_file, port, **kwargs):\n return port\n\n def mock_compute_embedding_via_server(chunks, port):\n server_queries.extend(chunks)\n return np.random.rand(len(chunks), 768).astype(np.float32)\n\n searcher._ensure_server_running = mock_ensure_server_running\n searcher._compute_embedding_via_server = mock_compute_embedding_via_server\n\n searcher.compute_query_embedding(\n query=original_query,\n use_server_if_available=True,\n query_template=query_template,\n )\n\n # Verify both paths produced identical templated queries\n assert len(fallback_queries) == 1\n assert len(server_queries) == 1\n assert fallback_queries[0] == server_queries[0]\n assert fallback_queries[0] == f\"{query_template}{original_query}\"\n\n def test_query_template_with_empty_string(self, temp_index_with_template):\n \"\"\"Test behavior with empty template string\"\"\"\n from leann.searcher_base import BaseSearcher\n\n class TestSearcher(BaseSearcher):\n def search(\n self,\n query,\n top_k,\n complexity=64,\n beam_width=1,\n prune_ratio=0.0,\n recompute_embeddings=False,\n pruning_strategy=\"global\",\n zmq_port=None,\n **kwargs,\n ):\n return {\"labels\": [], \"distances\": []}\n\n searcher = object.__new__(TestSearcher)\n searcher.index_path = Path(temp_index_with_template)\n searcher.index_dir = searcher.index_path.parent\n\n meta_file = searcher.index_dir / f\"{searcher.index_path.name}.meta.json\"\n with open(meta_file) as f:\n searcher.meta = json.load(f)\n\n searcher.embedding_model = searcher.meta[\"embedding_model\"]\n searcher.embedding_mode = searcher.meta.get(\"embedding_mode\", \"sentence-transformers\")\n searcher.embedding_options = searcher.meta.get(\"embedding_options\", {})\n searcher.enable_warmup = False\n searcher.use_daemon = False\n searcher.daemon_ttl_seconds = 0\n\n captured_queries = []\n\n def mock_compute_embeddings(texts, model, mode, provider_options=None):\n captured_queries.extend(texts)\n return np.random.rand(len(texts), 768).astype(np.float32)\n\n with patch(\n \"leann.embedding_compute.compute_embeddings\", side_effect=mock_compute_embeddings\n ):\n searcher.compute_query_embedding(\n query=\"vector database\",\n use_server_if_available=False,\n query_template=\"\", # Empty string\n )\n\n # Empty string is falsy, so no template should be applied\n assert captured_queries[0] == \"vector database\"\n" }, { "path": "tests/test_readme_examples.py", "content": "\"\"\"\nTest examples from README.md to ensure documentation is accurate.\n\"\"\"\n\nimport os\nimport platform\nimport tempfile\nfrom pathlib import Path\n\nimport pytest\n\n\n@pytest.mark.parametrize(\"backend_name\", [\"hnsw\", \"diskann\"])\ndef test_readme_basic_example(backend_name):\n \"\"\"Test the basic example from README.md with both backends.\"\"\"\n # Skip on macOS CI due to MPS environment issues with all-MiniLM-L6-v2\n if os.environ.get(\"CI\") == \"true\" and platform.system() == \"Darwin\":\n pytest.skip(\"Skipping on macOS CI due to MPS environment issues with all-MiniLM-L6-v2\")\n # Skip DiskANN on CI (Linux runners) due to C++ extension memory/hardware constraints\n if os.environ.get(\"CI\") == \"true\" and backend_name == \"diskann\":\n pytest.skip(\"Skip DiskANN tests in CI due to resource constraints and instability\")\n\n # This is the exact code from README (with smaller model for CI)\n from leann import LeannBuilder, LeannChat, LeannSearcher\n from leann.api import SearchResult\n\n with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as temp_dir:\n INDEX_PATH = str(Path(temp_dir) / f\"demo_{backend_name}.leann\")\n\n if os.environ.get(\"CI\") == \"true\":\n builder = LeannBuilder(\n backend_name=backend_name,\n embedding_model=\"sentence-transformers/all-MiniLM-L6-v2\",\n dimensions=384,\n )\n else:\n builder = LeannBuilder(backend_name=backend_name)\n builder.add_text(\"LEANN saves 97% storage compared to traditional vector databases.\")\n builder.add_text(\"Tung Tung Tung Sahur called—they need their banana-crocodile hybrid back\")\n builder.build_index(INDEX_PATH)\n\n index_dir = Path(INDEX_PATH).parent\n assert index_dir.exists()\n index_files = list(index_dir.glob(f\"{Path(INDEX_PATH).stem}.*\"))\n assert len(index_files) > 0\n\n with LeannSearcher(INDEX_PATH) as searcher:\n results = searcher.search(\"fantastical AI-generated creatures\", top_k=1)\n\n assert len(results) > 0\n assert isinstance(results[0], SearchResult)\n assert results[0].score != float(\"-inf\"), (\n f\"should return valid scores, got {results[0].score}\"\n )\n assert \"banana\" in results[0].text or \"crocodile\" in results[0].text\n\n chat = LeannChat(INDEX_PATH, llm_config={\"type\": \"simulated\"})\n response = chat.ask(\"How much storage does LEANN save?\", top_k=1)\n\n # Verify chat works\n assert isinstance(response, str)\n assert len(response) > 0\n # Cleanup chat resources\n chat.cleanup()\n\n\ndef test_readme_imports():\n \"\"\"Test that the imports shown in README work correctly.\"\"\"\n # These are the imports shown in README\n from leann import LeannBuilder, LeannChat, LeannSearcher\n\n # Verify they are the correct types\n assert callable(LeannBuilder)\n assert callable(LeannSearcher)\n assert callable(LeannChat)\n\n\ndef test_backend_options():\n \"\"\"Test different backend options mentioned in documentation.\"\"\"\n # Skip on macOS CI due to MPS environment issues with all-MiniLM-L6-v2\n if os.environ.get(\"CI\") == \"true\" and platform.system() == \"Darwin\":\n pytest.skip(\"Skipping on macOS CI due to MPS environment issues with all-MiniLM-L6-v2\")\n\n from leann import LeannBuilder\n\n with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as temp_dir:\n is_ci = os.environ.get(\"CI\") == \"true\"\n embedding_model = (\n \"sentence-transformers/all-MiniLM-L6-v2\" if is_ci else \"facebook/contriever\"\n )\n dimensions = 384 if is_ci else None\n\n hnsw_path = str(Path(temp_dir) / \"test_hnsw.leann\")\n builder_hnsw = LeannBuilder(\n backend_name=\"hnsw\", embedding_model=embedding_model, dimensions=dimensions\n )\n builder_hnsw.add_text(\"Test document for HNSW backend\")\n builder_hnsw.build_index(hnsw_path)\n assert Path(hnsw_path).parent.exists()\n assert len(list(Path(hnsw_path).parent.glob(f\"{Path(hnsw_path).stem}.*\"))) > 0\n\n if is_ci:\n pytest.skip(\n \"Skip DiskANN portion in CI - small datasets trigger MKL parameter \"\n \"errors and pytest-timeout thread kills cause segfaults on Windows\"\n )\n\n diskann_path = str(Path(temp_dir) / \"test_diskann.leann\")\n builder_diskann = LeannBuilder(\n backend_name=\"diskann\", embedding_model=embedding_model, dimensions=dimensions\n )\n builder_diskann.add_text(\"Test document for DiskANN backend\")\n builder_diskann.build_index(diskann_path)\n assert Path(diskann_path).parent.exists()\n assert len(list(Path(diskann_path).parent.glob(f\"{Path(diskann_path).stem}.*\"))) > 0\n\n\n@pytest.mark.parametrize(\"backend_name\", [\"hnsw\", \"diskann\"])\ndef test_llm_config_simulated(backend_name):\n \"\"\"Test simulated LLM configuration option with both backends.\"\"\"\n # Skip on macOS CI due to MPS environment issues with all-MiniLM-L6-v2\n if os.environ.get(\"CI\") == \"true\" and platform.system() == \"Darwin\":\n pytest.skip(\"Skipping on macOS CI due to MPS environment issues with all-MiniLM-L6-v2\")\n\n # Skip DiskANN tests in CI due to hardware requirements\n if os.environ.get(\"CI\") == \"true\" and backend_name == \"diskann\":\n pytest.skip(\"Skip DiskANN tests in CI - requires specific hardware and large memory\")\n\n from leann import LeannBuilder, LeannChat\n\n with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as temp_dir:\n index_path = str(Path(temp_dir) / f\"test_{backend_name}.leann\")\n if os.environ.get(\"CI\") == \"true\":\n builder = LeannBuilder(\n backend_name=backend_name,\n embedding_model=\"sentence-transformers/all-MiniLM-L6-v2\",\n dimensions=384,\n )\n else:\n builder = LeannBuilder(backend_name=backend_name)\n builder.add_text(\"Test document for LLM testing\")\n builder.build_index(index_path)\n\n llm_config = {\"type\": \"simulated\"}\n chat = LeannChat(index_path, llm_config=llm_config)\n response = chat.ask(\"What is this document about?\", top_k=1)\n\n assert isinstance(response, str)\n assert len(response) > 0\n\n\n@pytest.mark.skip(reason=\"Requires HF model download and may timeout\")\ndef test_llm_config_hf():\n \"\"\"Test HuggingFace LLM configuration option.\"\"\"\n from leann import LeannBuilder, LeannChat\n\n pytest.importorskip(\"transformers\") # Skip if transformers not installed\n\n with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as temp_dir:\n index_path = str(Path(temp_dir) / \"test.leann\")\n builder = LeannBuilder(backend_name=\"hnsw\")\n builder.add_text(\"Test document for LLM testing\")\n builder.build_index(index_path)\n\n # Test HF LLM config\n llm_config = {\"type\": \"hf\", \"model\": \"Qwen/Qwen3-0.6B\"}\n chat = LeannChat(index_path, llm_config=llm_config)\n response = chat.ask(\"What is this document about?\", top_k=1)\n\n assert isinstance(response, str)\n assert len(response) > 0\n" }, { "path": "tests/test_sync.py", "content": "import os\nimport tempfile\nimport unittest\nfrom unittest.mock import Mock, patch\n\nfrom leann.sync import FileSynchronizer, MerkleTree, hash_data\n\n\nclass TestMerkleTreeCompare(unittest.TestCase):\n def test_no_changes_if_root_hash_same(self):\n tree1 = Mock()\n tree2 = Mock()\n\n tree1.root = Mock(hash=\"root_hash\")\n tree2.root = Mock(hash=\"root_hash\")\n\n added, removed, modified = MerkleTree.compare_with(tree1, tree2)\n\n self.assertEqual(added, [])\n self.assertEqual(removed, [])\n self.assertEqual(modified, [])\n\n def test_added_removed_modified(self):\n tree1 = Mock()\n tree2 = Mock()\n\n # Mock file nodes\n file_a_new = Mock()\n file_b_new = Mock()\n file_a_old = Mock()\n file_c_old = Mock()\n\n # Equality behavior\n file_a_new.__eq__ = Mock(return_value=False)\n\n tree1.root = Mock(\n hash=\"new_root\",\n children={\n \"a.txt\": file_a_new,\n \"b.txt\": file_b_new,\n },\n )\n\n tree2.root = Mock(\n hash=\"old_root\",\n children={\n \"a.txt\": file_a_old,\n \"c.txt\": file_c_old,\n },\n )\n\n added, removed, modified = MerkleTree.compare_with(tree1, tree2)\n\n self.assertEqual(added, [\"c.txt\"])\n self.assertEqual(removed, [\"b.txt\"])\n self.assertEqual(modified, [\"a.txt\"])\n\n\nclass TestFileSynchronizer(unittest.TestCase):\n def test_generate_file_hashes(self):\n temp_dir = tempfile.gettempdir()\n fs = FileSynchronizer(temp_dir, auto_load=False)\n\n mock_file = Mock()\n mock_file.text = \"hello world\"\n mock_file.metadata = {\"file_path\": os.path.join(temp_dir, \"file.txt\")}\n\n mock_reader_instance = Mock()\n mock_reader_instance.iter_data.return_value = [\n [mock_file],\n ]\n\n with patch(\"leann.sync.SimpleDirectoryReader\") as mock_reader:\n mock_reader.return_value = mock_reader_instance\n\n result = fs.generate_file_hashes()\n\n assert result == {os.path.join(temp_dir, \"file.txt\"): hash_data(\"hello world\")}\n\n def test_build_merkle_tree(self):\n fs = FileSynchronizer(\".\", auto_load=False)\n\n file_hashes = {\n \"a.txt\": \"hashA\",\n \"b.txt\": \"hashB\",\n }\n\n tree = fs.build_merkle_tree(file_hashes)\n\n # Root exists\n assert tree.root is not None\n\n # Children added correctly\n assert set(tree.root.children.keys()) == {\"a.txt\", \"b.txt\"}\n\n # Child nodes have correct data\n assert tree.root.children[\"a.txt\"].data == \"hashA\"\n assert tree.root.children[\"b.txt\"].data == \"hashB\"\n\n expected_root_data = \"a.txt\" + \"hashA\" + \"b.txt\" + \"hashB\"\n assert tree.root.hash == hash_data(expected_root_data)\n\n def test_check_for_changes_detected(self):\n fs = FileSynchronizer.__new__(FileSynchronizer)\n\n fs.generate_file_hashes = Mock(return_value={\"a.txt\": \"hash\"})\n fs.build_merkle_tree = Mock(return_value=Mock())\n\n old_tree = Mock()\n new_tree = fs.build_merkle_tree.return_value\n\n old_tree.compare_with.return_value = ([\"a.txt\"], [], [])\n fs.tree = old_tree\n\n fs.save_snapshot = Mock()\n\n changes = fs.check_for_changes()\n\n assert changes == ([\"a.txt\"], [], [])\n\n fs.build_merkle_tree.assert_called_once_with({\"a.txt\": \"hash\"})\n old_tree.compare_with.assert_called_once_with(new_tree)\n\n fs.save_snapshot.assert_called_once()\n assert fs.tree is new_tree\n" }, { "path": "tests/test_token_truncation.py", "content": "\"\"\"Unit tests for token-aware truncation functionality.\n\nThis test suite defines the contract for token truncation functions that prevent\n500 errors from Ollama when text exceeds model token limits. These tests verify:\n\n1. Model token limit retrieval (known and unknown models)\n2. Text truncation behavior for single and multiple texts\n3. Token counting and truncation accuracy using tiktoken\n\nAll tests are written in Red Phase - they should FAIL initially because the\nimplementation does not exist yet.\n\"\"\"\n\nimport pytest\nimport tiktoken\nfrom leann.embedding_compute import (\n EMBEDDING_MODEL_LIMITS,\n get_model_token_limit,\n truncate_to_token_limit,\n)\n\n\nclass TestModelTokenLimits:\n \"\"\"Tests for retrieving model-specific token limits.\"\"\"\n\n def test_get_model_token_limit_known_model(self):\n \"\"\"Verify correct token limit is returned for known models.\n\n Known models should return their specific token limits from\n EMBEDDING_MODEL_LIMITS dictionary.\n \"\"\"\n # Test nomic-embed-text (2048 tokens)\n limit = get_model_token_limit(\"nomic-embed-text\")\n assert limit == 2048, \"nomic-embed-text should have 2048 token limit\"\n\n # Test nomic-embed-text-v1.5 (2048 tokens)\n limit = get_model_token_limit(\"nomic-embed-text-v1.5\")\n assert limit == 2048, \"nomic-embed-text-v1.5 should have 2048 token limit\"\n\n # Test nomic-embed-text-v2 (512 tokens)\n limit = get_model_token_limit(\"nomic-embed-text-v2\")\n assert limit == 512, \"nomic-embed-text-v2 should have 512 token limit\"\n\n # Test OpenAI models (8192 tokens)\n limit = get_model_token_limit(\"text-embedding-3-small\")\n assert limit == 8192, \"text-embedding-3-small should have 8192 token limit\"\n\n def test_get_model_token_limit_unknown_model(self):\n \"\"\"Verify default token limit is returned for unknown models.\n\n Unknown models should return the default limit (2048) to allow\n operation with reasonable safety margin.\n \"\"\"\n # Test with completely unknown model\n limit = get_model_token_limit(\"unknown-model-xyz\")\n assert limit == 2048, \"Unknown models should return default 2048\"\n\n # Test with empty string\n limit = get_model_token_limit(\"\")\n assert limit == 2048, \"Empty model name should return default 2048\"\n\n def test_get_model_token_limit_custom_default(self):\n \"\"\"Verify custom default can be specified for unknown models.\n\n Allow callers to specify their own default token limit when\n model is not in the known models dictionary.\n \"\"\"\n limit = get_model_token_limit(\"unknown-model\", default=4096)\n assert limit == 4096, \"Should return custom default for unknown models\"\n\n # Known model should ignore custom default\n limit = get_model_token_limit(\"nomic-embed-text\", default=4096)\n assert limit == 2048, \"Known model should ignore custom default\"\n\n def test_embedding_model_limits_dictionary_exists(self):\n \"\"\"Verify EMBEDDING_MODEL_LIMITS dictionary contains expected models.\n\n The dictionary should be importable and contain at least the\n known nomic models with correct token limits.\n \"\"\"\n assert isinstance(EMBEDDING_MODEL_LIMITS, dict), \"Should be a dictionary\"\n assert \"nomic-embed-text\" in EMBEDDING_MODEL_LIMITS, \"Should contain nomic-embed-text\"\n assert \"nomic-embed-text-v1.5\" in EMBEDDING_MODEL_LIMITS, (\n \"Should contain nomic-embed-text-v1.5\"\n )\n assert EMBEDDING_MODEL_LIMITS[\"nomic-embed-text\"] == 2048\n assert EMBEDDING_MODEL_LIMITS[\"nomic-embed-text-v1.5\"] == 2048\n assert EMBEDDING_MODEL_LIMITS[\"nomic-embed-text-v2\"] == 512\n # OpenAI models\n assert EMBEDDING_MODEL_LIMITS[\"text-embedding-3-small\"] == 8192\n\n\nclass TestTokenTruncation:\n \"\"\"Tests for truncating texts to token limits.\"\"\"\n\n @pytest.fixture\n def tokenizer(self):\n \"\"\"Provide tiktoken tokenizer for token counting verification.\"\"\"\n return tiktoken.get_encoding(\"cl100k_base\")\n\n def test_truncate_single_text_under_limit(self, tokenizer):\n \"\"\"Verify text under token limit remains unchanged.\n\n When text is already within the token limit, it should be\n returned unchanged with no truncation.\n \"\"\"\n text = \"This is a short text that is well under the token limit.\"\n token_count = len(tokenizer.encode(text))\n assert token_count < 100, f\"Test setup: text should be short (has {token_count} tokens)\"\n\n # Truncate with generous limit\n result = truncate_to_token_limit([text], token_limit=512)\n\n assert len(result) == 1, \"Should return same number of texts\"\n assert result[0] == text, \"Text under limit should be unchanged\"\n\n def test_truncate_single_text_over_limit(self, tokenizer):\n \"\"\"Verify text over token limit is truncated correctly.\n\n When text exceeds the token limit, it should be truncated to\n fit within the limit while maintaining valid token boundaries.\n \"\"\"\n # Create a text that definitely exceeds limit\n text = \"word \" * 200 # ~200 tokens (each \"word \" is typically 1-2 tokens)\n original_token_count = len(tokenizer.encode(text))\n assert original_token_count > 50, (\n f\"Test setup: text should be long (has {original_token_count} tokens)\"\n )\n\n # Truncate to 50 tokens\n result = truncate_to_token_limit([text], token_limit=50)\n\n assert len(result) == 1, \"Should return same number of texts\"\n assert result[0] != text, \"Text over limit should be truncated\"\n assert len(result[0]) < len(text), \"Truncated text should be shorter\"\n\n # Verify truncated text is within token limit\n truncated_token_count = len(tokenizer.encode(result[0]))\n assert truncated_token_count <= 50, (\n f\"Truncated text should be ≤50 tokens, got {truncated_token_count}\"\n )\n\n def test_truncate_multiple_texts_mixed_lengths(self, tokenizer):\n \"\"\"Verify multiple texts with mixed lengths are handled correctly.\n\n When processing multiple texts:\n - Texts under limit should remain unchanged\n - Texts over limit should be truncated independently\n - Output list should maintain same order and length\n \"\"\"\n texts = [\n \"Short text.\", # Under limit\n \"word \" * 200, # Over limit\n \"Another short one.\", # Under limit\n \"token \" * 150, # Over limit\n ]\n\n # Verify test setup\n for i, text in enumerate(texts):\n token_count = len(tokenizer.encode(text))\n if i in [1, 3]:\n assert token_count > 50, f\"Text {i} should be over limit (has {token_count} tokens)\"\n else:\n assert token_count < 50, (\n f\"Text {i} should be under limit (has {token_count} tokens)\"\n )\n\n # Truncate with 50 token limit\n result = truncate_to_token_limit(texts, token_limit=50)\n\n assert len(result) == len(texts), \"Should return same number of texts\"\n\n # Verify each text individually\n for i, (original, truncated) in enumerate(zip(texts, result)):\n token_count = len(tokenizer.encode(truncated))\n assert token_count <= 50, f\"Text {i} should be ≤50 tokens, got {token_count}\"\n\n # Short texts should be unchanged\n if i in [0, 2]:\n assert truncated == original, f\"Short text {i} should be unchanged\"\n # Long texts should be truncated\n else:\n assert len(truncated) < len(original), f\"Long text {i} should be truncated\"\n\n def test_truncate_empty_list(self):\n \"\"\"Verify empty input list returns empty output list.\n\n Edge case: empty list should return empty list without errors.\n \"\"\"\n result = truncate_to_token_limit([], token_limit=512)\n assert result == [], \"Empty input should return empty output\"\n\n def test_truncate_preserves_order(self, tokenizer):\n \"\"\"Verify truncation preserves original text order.\n\n Output list should maintain the same order as input list,\n regardless of which texts were truncated.\n \"\"\"\n texts = [\n \"First text \" * 50, # Will be truncated\n \"Second text.\", # Won't be truncated\n \"Third text \" * 50, # Will be truncated\n ]\n\n result = truncate_to_token_limit(texts, token_limit=20)\n\n assert len(result) == 3, \"Should preserve list length\"\n # Check that order is maintained by looking for distinctive words\n assert \"First\" in result[0], \"First text should remain in first position\"\n assert \"Second\" in result[1], \"Second text should remain in second position\"\n assert \"Third\" in result[2], \"Third text should remain in third position\"\n\n def test_truncate_extremely_long_text(self, tokenizer):\n \"\"\"Verify extremely long texts are truncated efficiently.\n\n Test with text that far exceeds token limit to ensure\n truncation handles extreme cases without performance issues.\n \"\"\"\n # Create very long text (simulate real-world scenario)\n text = \"token \" * 5000 # ~5000+ tokens\n original_token_count = len(tokenizer.encode(text))\n assert original_token_count > 1000, \"Test setup: text should be very long\"\n\n # Truncate to small limit\n result = truncate_to_token_limit([text], token_limit=100)\n\n assert len(result) == 1\n truncated_token_count = len(tokenizer.encode(result[0]))\n assert truncated_token_count <= 100, (\n f\"Should truncate to ≤100 tokens, got {truncated_token_count}\"\n )\n assert len(result[0]) < len(text) // 10, \"Should significantly reduce text length\"\n\n def test_truncate_exact_token_limit(self, tokenizer):\n \"\"\"Verify text at exactly token limit is handled correctly.\n\n Edge case: text with exactly the token limit should either\n remain unchanged or be safely truncated by 1 token.\n \"\"\"\n # Create text with approximately 50 tokens\n # We'll adjust to get exactly 50\n target_tokens = 50\n text = \"word \" * 50\n tokens = tokenizer.encode(text)\n\n # Adjust to get exactly target_tokens\n if len(tokens) > target_tokens:\n tokens = tokens[:target_tokens]\n text = tokenizer.decode(tokens)\n elif len(tokens) < target_tokens:\n # Add more words\n while len(tokenizer.encode(text)) < target_tokens:\n text += \"word \"\n tokens = tokenizer.encode(text)[:target_tokens]\n text = tokenizer.decode(tokens)\n\n # Verify we have exactly target_tokens\n assert len(tokenizer.encode(text)) == target_tokens, (\n \"Test setup: should have exactly 50 tokens\"\n )\n\n result = truncate_to_token_limit([text], token_limit=target_tokens)\n\n assert len(result) == 1\n result_tokens = len(tokenizer.encode(result[0]))\n assert result_tokens <= target_tokens, (\n f\"Should be ≤{target_tokens} tokens, got {result_tokens}\"\n )\n\n\nclass TestLMStudioHybridDiscovery:\n \"\"\"Tests for LM Studio integration in get_model_token_limit() hybrid discovery.\n\n These tests verify that get_model_token_limit() properly integrates with\n the LM Studio SDK bridge for dynamic token limit discovery. The integration\n should:\n\n 1. Detect LM Studio URLs (port 1234 or 'lmstudio'/'lm.studio' in URL)\n 2. Convert HTTP URLs to WebSocket format for SDK queries\n 3. Query LM Studio SDK and use discovered limit\n 4. Fall back to registry when SDK returns None\n 5. Execute AFTER Ollama detection but BEFORE registry fallback\n\n All tests are written in Red Phase - they should FAIL initially because the\n LM Studio detection and integration logic does not exist yet in get_model_token_limit().\n \"\"\"\n\n def test_get_model_token_limit_lmstudio_success(self, monkeypatch):\n \"\"\"Verify LM Studio SDK query succeeds and returns detected limit.\n\n When a LM Studio base_url is detected and the SDK query succeeds,\n get_model_token_limit() should return the dynamically discovered\n context length without falling back to the registry.\n \"\"\"\n\n # Mock _query_lmstudio_context_limit to return successful SDK query\n def mock_query_lmstudio(model_name, base_url):\n # Verify WebSocket URL was passed (not HTTP)\n assert base_url.startswith(\"ws://\"), (\n f\"Should convert HTTP to WebSocket format, got: {base_url}\"\n )\n return 8192 # Successful SDK query\n\n monkeypatch.setattr(\n \"leann.embedding_compute._query_lmstudio_context_limit\",\n mock_query_lmstudio,\n )\n\n # Test with HTTP URL that should be converted to WebSocket\n limit = get_model_token_limit(\n model_name=\"custom-model\", base_url=\"http://localhost:1234/v1\"\n )\n\n assert limit == 8192, \"Should return limit from LM Studio SDK query\"\n\n def test_get_model_token_limit_lmstudio_fallback_to_registry(self, monkeypatch):\n \"\"\"Verify fallback to registry when LM Studio SDK returns None.\n\n When LM Studio SDK query fails (returns None), get_model_token_limit()\n should fall back to the EMBEDDING_MODEL_LIMITS registry.\n \"\"\"\n\n # Mock _query_lmstudio_context_limit to return None (SDK failure)\n def mock_query_lmstudio(model_name, base_url):\n return None # SDK query failed\n\n monkeypatch.setattr(\n \"leann.embedding_compute._query_lmstudio_context_limit\",\n mock_query_lmstudio,\n )\n\n # Test with known model that exists in registry\n limit = get_model_token_limit(\n model_name=\"nomic-embed-text\", base_url=\"http://localhost:1234/v1\"\n )\n\n # Should fall back to registry value\n assert limit == 2048, \"Should fall back to registry when SDK returns None\"\n\n def test_get_model_token_limit_lmstudio_port_detection(self, monkeypatch):\n \"\"\"Verify detection of LM Studio via port 1234.\n\n get_model_token_limit() should recognize port 1234 as a LM Studio\n server and attempt SDK query, regardless of hostname.\n \"\"\"\n query_called = False\n\n def mock_query_lmstudio(model_name, base_url):\n nonlocal query_called\n query_called = True\n return 4096\n\n monkeypatch.setattr(\n \"leann.embedding_compute._query_lmstudio_context_limit\",\n mock_query_lmstudio,\n )\n\n # Test with port 1234 (default LM Studio port)\n limit = get_model_token_limit(model_name=\"test-model\", base_url=\"http://127.0.0.1:1234/v1\")\n\n assert query_called, \"Should detect port 1234 and call LM Studio SDK query\"\n assert limit == 4096, \"Should return SDK query result\"\n\n @pytest.mark.parametrize(\n \"test_url,expected_limit,keyword\",\n [\n (\"http://lmstudio.local:8080/v1\", 16384, \"lmstudio\"),\n (\"http://api.lm.studio:5000/v1\", 32768, \"lm.studio\"),\n ],\n )\n def test_get_model_token_limit_lmstudio_url_keyword_detection(\n self, monkeypatch, test_url, expected_limit, keyword\n ):\n \"\"\"Verify detection of LM Studio via keywords in URL.\n\n get_model_token_limit() should recognize 'lmstudio' or 'lm.studio'\n in the URL as indicating a LM Studio server.\n \"\"\"\n query_called = False\n\n def mock_query_lmstudio(model_name, base_url):\n nonlocal query_called\n query_called = True\n return expected_limit\n\n monkeypatch.setattr(\n \"leann.embedding_compute._query_lmstudio_context_limit\",\n mock_query_lmstudio,\n )\n\n limit = get_model_token_limit(model_name=\"test-model\", base_url=test_url)\n\n assert query_called, f\"Should detect '{keyword}' keyword and call SDK query\"\n assert limit == expected_limit, f\"Should return SDK query result for {keyword}\"\n\n @pytest.mark.parametrize(\n \"input_url,expected_protocol,expected_host\",\n [\n (\"http://localhost:1234/v1\", \"ws://\", \"localhost:1234\"),\n (\"https://lmstudio.example.com:1234/v1\", \"wss://\", \"lmstudio.example.com:1234\"),\n ],\n )\n def test_get_model_token_limit_protocol_conversion(\n self, monkeypatch, input_url, expected_protocol, expected_host\n ):\n \"\"\"Verify HTTP/HTTPS URL is converted to WebSocket format for SDK query.\n\n LM Studio SDK requires WebSocket URLs. get_model_token_limit() should:\n 1. Convert 'http://' to 'ws://'\n 2. Convert 'https://' to 'wss://'\n 3. Remove '/v1' or other path suffixes (SDK expects base URL)\n 4. Preserve host and port\n \"\"\"\n conversions_tested = []\n\n def mock_query_lmstudio(model_name, base_url):\n conversions_tested.append(base_url)\n return 8192\n\n monkeypatch.setattr(\n \"leann.embedding_compute._query_lmstudio_context_limit\",\n mock_query_lmstudio,\n )\n\n get_model_token_limit(model_name=\"test-model\", base_url=input_url)\n\n # Verify conversion happened\n assert len(conversions_tested) == 1, \"Should have called SDK query once\"\n assert conversions_tested[0].startswith(expected_protocol), (\n f\"Should convert to {expected_protocol}\"\n )\n assert expected_host in conversions_tested[0], (\n f\"Should preserve host and port: {expected_host}\"\n )\n\n def test_get_model_token_limit_lmstudio_executes_after_ollama(self, monkeypatch):\n \"\"\"Verify LM Studio detection happens AFTER Ollama detection.\n\n The hybrid discovery order should be:\n 1. Ollama dynamic discovery (port 11434 or 'ollama' in URL)\n 2. LM Studio dynamic discovery (port 1234 or 'lmstudio' in URL)\n 3. Registry fallback\n\n If both Ollama and LM Studio patterns match, Ollama should take precedence.\n This test verifies that LM Studio is checked but doesn't interfere with Ollama.\n \"\"\"\n ollama_called = False\n lmstudio_called = False\n\n def mock_query_ollama(model_name, base_url):\n nonlocal ollama_called\n ollama_called = True\n return 2048 # Ollama query succeeds\n\n def mock_query_lmstudio(model_name, base_url):\n nonlocal lmstudio_called\n lmstudio_called = True\n return None # Should not be reached if Ollama succeeds\n\n monkeypatch.setattr(\n \"leann.embedding_compute._query_ollama_context_limit\",\n mock_query_ollama,\n )\n monkeypatch.setattr(\n \"leann.embedding_compute._query_lmstudio_context_limit\",\n mock_query_lmstudio,\n )\n\n # Test with Ollama URL\n limit = get_model_token_limit(\n model_name=\"test-model\", base_url=\"http://localhost:11434/api\"\n )\n\n assert ollama_called, \"Should attempt Ollama query first\"\n assert not lmstudio_called, \"Should not attempt LM Studio query when Ollama succeeds\"\n assert limit == 2048, \"Should return Ollama result\"\n\n def test_get_model_token_limit_lmstudio_not_detected_for_non_lmstudio_urls(self, monkeypatch):\n \"\"\"Verify LM Studio SDK query is NOT called for non-LM Studio URLs.\n\n Only URLs with port 1234 or 'lmstudio'/'lm.studio' keywords should\n trigger LM Studio SDK queries. Other URLs should skip to registry fallback.\n \"\"\"\n lmstudio_called = False\n\n def mock_query_lmstudio(model_name, base_url):\n nonlocal lmstudio_called\n lmstudio_called = True\n return 8192\n\n monkeypatch.setattr(\n \"leann.embedding_compute._query_lmstudio_context_limit\",\n mock_query_lmstudio,\n )\n\n # Test with non-LM Studio URLs\n test_cases = [\n \"http://localhost:8080/v1\", # Different port\n \"http://openai.example.com/v1\", # Different service\n \"http://localhost:3000/v1\", # Another port\n ]\n\n for base_url in test_cases:\n lmstudio_called = False # Reset for each test\n get_model_token_limit(model_name=\"nomic-embed-text\", base_url=base_url)\n assert not lmstudio_called, f\"Should NOT call LM Studio SDK for URL: {base_url}\"\n\n def test_get_model_token_limit_lmstudio_case_insensitive_detection(self, monkeypatch):\n \"\"\"Verify LM Studio detection is case-insensitive for keywords.\n\n Keywords 'lmstudio' and 'lm.studio' should be detected regardless\n of case (LMStudio, LMSTUDIO, LmStudio, etc.).\n \"\"\"\n query_called = False\n\n def mock_query_lmstudio(model_name, base_url):\n nonlocal query_called\n query_called = True\n return 8192\n\n monkeypatch.setattr(\n \"leann.embedding_compute._query_lmstudio_context_limit\",\n mock_query_lmstudio,\n )\n\n # Test various case variations\n test_cases = [\n \"http://LMStudio.local:8080/v1\",\n \"http://LMSTUDIO.example.com/v1\",\n \"http://LmStudio.local/v1\",\n \"http://api.LM.STUDIO:5000/v1\",\n ]\n\n for base_url in test_cases:\n query_called = False # Reset for each test\n limit = get_model_token_limit(model_name=\"test-model\", base_url=base_url)\n assert query_called, f\"Should detect LM Studio in URL: {base_url}\"\n assert limit == 8192, f\"Should return SDK result for URL: {base_url}\"\n\n\nclass TestTokenLimitCaching:\n \"\"\"Tests for token limit caching to prevent repeated SDK/API calls.\n\n Caching prevents duplicate SDK/API calls within the same Python process,\n which is important because:\n 1. LM Studio SDK load() can load duplicate model instances\n 2. Ollama /api/show queries add latency\n 3. Registry lookups are pure overhead\n\n Cache is process-scoped and resets between leann build invocations.\n \"\"\"\n\n def setup_method(self):\n \"\"\"Clear cache before each test.\"\"\"\n from leann.embedding_compute import _token_limit_cache\n\n _token_limit_cache.clear()\n\n def test_registry_lookup_is_cached(self):\n \"\"\"Verify that registry lookups are cached.\"\"\"\n from leann.embedding_compute import _token_limit_cache\n\n # First call\n limit1 = get_model_token_limit(\"text-embedding-3-small\")\n assert limit1 == 8192\n\n # Verify it's in cache\n cache_key = (\"text-embedding-3-small\", \"\")\n assert cache_key in _token_limit_cache\n assert _token_limit_cache[cache_key] == 8192\n\n # Second call should use cache\n limit2 = get_model_token_limit(\"text-embedding-3-small\")\n assert limit2 == 8192\n\n def test_default_fallback_is_cached(self):\n \"\"\"Verify that default fallbacks are cached.\"\"\"\n from leann.embedding_compute import _token_limit_cache\n\n # First call with unknown model\n limit1 = get_model_token_limit(\"unknown-model-xyz\", default=512)\n assert limit1 == 512\n\n # Verify it's in cache\n cache_key = (\"unknown-model-xyz\", \"\")\n assert cache_key in _token_limit_cache\n assert _token_limit_cache[cache_key] == 512\n\n # Second call should use cache\n limit2 = get_model_token_limit(\"unknown-model-xyz\", default=512)\n assert limit2 == 512\n\n def test_different_urls_create_separate_cache_entries(self):\n \"\"\"Verify that different base_urls create separate cache entries.\"\"\"\n from leann.embedding_compute import _token_limit_cache\n\n # Same model, different URLs\n limit1 = get_model_token_limit(\"nomic-embed-text\", base_url=\"http://localhost:11434\")\n limit2 = get_model_token_limit(\"nomic-embed-text\", base_url=\"http://localhost:1234/v1\")\n\n # Both should find the model in registry (2048)\n assert limit1 == 2048\n assert limit2 == 2048\n\n # But they should be separate cache entries\n cache_key1 = (\"nomic-embed-text\", \"http://localhost:11434\")\n cache_key2 = (\"nomic-embed-text\", \"http://localhost:1234/v1\")\n\n assert cache_key1 in _token_limit_cache\n assert cache_key2 in _token_limit_cache\n assert len(_token_limit_cache) == 2\n\n def test_cache_prevents_repeated_lookups(self):\n \"\"\"Verify that cache prevents repeated registry/API lookups.\"\"\"\n from leann.embedding_compute import _token_limit_cache\n\n model_name = \"text-embedding-ada-002\"\n\n # First call - should add to cache\n assert len(_token_limit_cache) == 0\n limit1 = get_model_token_limit(model_name)\n\n cache_size_after_first = len(_token_limit_cache)\n assert cache_size_after_first == 1\n\n # Multiple subsequent calls - cache size should not change\n for _ in range(5):\n limit = get_model_token_limit(model_name)\n assert limit == limit1\n assert len(_token_limit_cache) == cache_size_after_first\n\n def test_versioned_model_names_cached_correctly(self):\n \"\"\"Verify that versioned model names (e.g., model:tag) are cached.\"\"\"\n from leann.embedding_compute import _token_limit_cache\n\n # Model with version tag\n limit = get_model_token_limit(\"nomic-embed-text:latest\", base_url=\"http://localhost:11434\")\n assert limit == 2048\n\n # Should be cached with full name including version\n cache_key = (\"nomic-embed-text:latest\", \"http://localhost:11434\")\n assert cache_key in _token_limit_cache\n assert _token_limit_cache[cache_key] == 2048\n" } ]