[ { "path": ".github/workflows/fuzz.yaml", "content": "name: \"Fuzz\"\non:\n push:\n branches: [main]\n schedule:\n # Nightly at 2am UTC for longer fuzzing sessions\n - cron: \"0 2 * * *\"\n workflow_dispatch:\n inputs:\n duration:\n description: \"Fuzz duration per target (seconds)\"\n default: \"60\"\n\npermissions:\n contents: read\n\njobs:\n fuzz-linux:\n runs-on: ubuntu-22.04\n steps:\n - uses: actions/checkout@v4\n - name: Install LLVM 18\n run: |\n wget -qO- https://apt.llvm.org/llvm.sh | sudo bash -s -- 18\n echo \"FUZZ_CC=clang-18\" >> $GITHUB_ENV\n - run: ./scripts/vendor.sh\n - name: Generate sqlite-vec.h\n run: make sqlite-vec.h\n - name: Build fuzz targets\n run: make -C tests/fuzz all FUZZ_CC=$FUZZ_CC FUZZ_LDFLAGS=\n - name: Run fuzz targets\n run: |\n DURATION=${{ github.event.inputs.duration || '60' }}\n EXIT_CODE=0\n for target in tests/fuzz/targets/*; do\n [ -f \"$target\" ] && [ -x \"$target\" ] || continue\n name=$(basename \"$target\")\n echo \"::group::Fuzzing $name ($DURATION seconds)\"\n corpus=\"tests/fuzz/corpus/$name\"\n mkdir -p \"$corpus\"\n dict=\"tests/fuzz/${name//_/-}.dict\"\n dict_flag=\"\"\n [ -f \"$dict\" ] && dict_flag=\"-dict=$dict\"\n if ! ASAN_OPTIONS=detect_leaks=1 \"$target\" $dict_flag \\\n -max_total_time=\"$DURATION\" \"$corpus\" 2>&1; then\n echo \"::error::Fuzz target $name found a crash!\"\n EXIT_CODE=1\n fi\n echo \"::endgroup::\"\n done\n exit $EXIT_CODE\n - name: Upload crash artifacts\n if: failure()\n uses: actions/upload-artifact@v4\n with:\n name: fuzz-crashes-linux\n path: |\n crash-*\n leak-*\n timeout-*\n\n fuzz-macos:\n runs-on: macos-14\n # Best-effort: Homebrew LLVM 18 runtime dylibs pull in\n # __ZnwmSt19__type_descriptor_t (typed allocation ABI) which\n # macOS 14's system libc++ doesn't provide, causing dyld to abort.\n # Xcode clang doesn't ship libclang_rt.fuzzer_osx.a (no libFuzzer).\n # TODO: fix macOS fuzzing (pin older compiler-rt, or static runtime).\n continue-on-error: true\n steps:\n - uses: actions/checkout@v4\n - name: Install LLVM 18\n run: brew install llvm@18\n - run: ./scripts/vendor.sh\n - name: Generate sqlite-vec.h\n run: make sqlite-vec.h\n - name: Build fuzz targets\n run: |\n LLVM=/opt/homebrew/opt/llvm@18\n make -C tests/fuzz all \\\n FUZZ_CC=$LLVM/bin/clang \\\n FUZZ_LDFLAGS=\"-Wl,-ld_classic\"\n - name: Run fuzz targets\n run: |\n DURATION=${{ github.event.inputs.duration || '60' }}\n EXIT_CODE=0\n for target in tests/fuzz/targets/*; do\n [ -f \"$target\" ] && [ -x \"$target\" ] || continue\n name=$(basename \"$target\")\n echo \"::group::Fuzzing $name ($DURATION seconds)\"\n corpus=\"tests/fuzz/corpus/$name\"\n mkdir -p \"$corpus\"\n dict=\"tests/fuzz/${name//_/-}.dict\"\n dict_flag=\"\"\n [ -f \"$dict\" ] && dict_flag=\"-dict=$dict\"\n if ! \"$target\" $dict_flag \\\n -max_total_time=\"$DURATION\" \"$corpus\" 2>&1; then\n echo \"::error::Fuzz target $name found a crash!\"\n EXIT_CODE=1\n fi\n echo \"::endgroup::\"\n done\n exit $EXIT_CODE\n - name: Upload crash artifacts\n if: failure()\n uses: actions/upload-artifact@v4\n with:\n name: fuzz-crashes-macos\n path: |\n crash-*\n leak-*\n timeout-*\n\n fuzz-windows:\n # Best-effort: libFuzzer works on Windows via LLVM but ASAN/UBSAN\n # support is less reliable. Leak detection is not available.\n runs-on: windows-2022\n continue-on-error: true\n steps:\n - uses: actions/checkout@v4\n - name: Install LLVM\n run: choco install llvm -y\n - run: bash ./scripts/vendor.sh\n shell: bash\n - name: Generate sqlite-vec.h\n shell: bash\n run: make sqlite-vec.h\n - name: Build fuzz targets\n shell: bash\n run: |\n export PATH=\"/c/Program Files/LLVM/bin:$PATH\"\n cd tests/fuzz\n mkdir -p targets\n for src in *.c; do\n name=\"${src%.c}\"\n target_name=\"${name//-/_}\"\n echo \"Building $target_name from $src\"\n clang -fsanitize=address,fuzzer \\\n -I ../../ -I ../../vendor -DSQLITE_CORE -g \\\n ../../vendor/sqlite3.c ../../sqlite-vec.c \\\n \"$src\" -o \"targets/${target_name}.exe\" || {\n echo \"Warning: failed to build $target_name (best-effort)\"\n }\n done\n - name: Run fuzz targets\n shell: bash\n run: |\n export PATH=\"/c/Program Files/LLVM/bin:$PATH\"\n DURATION=${{ github.event.inputs.duration || '60' }}\n for target in tests/fuzz/targets/*.exe; do\n [ -f \"$target\" ] || continue\n name=$(basename \"$target\" .exe)\n echo \"=== Fuzzing $name ($DURATION seconds) ===\"\n corpus=\"tests/fuzz/corpus/$name\"\n mkdir -p \"$corpus\"\n \"$target\" -max_total_time=\"$DURATION\" \"$corpus\" 2>&1 || {\n echo \"Warning: $name found an issue or failed\"\n }\n done\n - name: Upload crash artifacts\n if: failure()\n uses: actions/upload-artifact@v4\n with:\n name: fuzz-crashes-windows\n path: |\n tests/fuzz/crash-*\n tests/fuzz/leak-*\n" }, { "path": ".github/workflows/release.yaml", "content": "name: \"Release\"\non:\n release:\n types: [published]\npermissions:\n contents: read\njobs:\n build-linux-x86_64-extension:\n runs-on: ubuntu-22.04\n steps:\n - uses: actions/checkout@v4\n - run: ./scripts/vendor.sh\n - run: make loadable static\n - uses: actions/upload-artifact@v4\n with:\n name: sqlite-vec-linux-x86_64-extension\n path: dist/*\n build-macos-x86_64-extension:\n runs-on: macos-15-intel\n steps:\n - uses: actions/checkout@v4\n - run: ./scripts/vendor.sh\n - run: make loadable static\n - uses: actions/upload-artifact@v4\n with:\n name: sqlite-vec-macos-x86_64-extension\n path: dist/*\n build-macos-aarch64-extension:\n runs-on: macos-14\n steps:\n - uses: actions/checkout@v4\n - run: ./scripts/vendor.sh\n - run: make loadable static\n - uses: actions/upload-artifact@v4\n with:\n name: sqlite-vec-macos-aarch64-extension\n path: dist/*\n build-windows-x86_64-extension:\n runs-on: windows-2022\n permissions:\n contents: write\n steps:\n - uses: actions/checkout@v4\n - uses: ilammy/msvc-dev-cmd@v1\n - uses: actions/setup-python@v5\n with:\n python-version: \"3.12\"\n - run: ./scripts/vendor.sh\n shell: bash\n - run: make sqlite-vec.h\n - run: mkdir dist\n - run: cl.exe /fPIC -shared /W4 /Ivendor/ /O2 /LD sqlite-vec.c -o dist/vec0.dll\n - uses: actions/upload-artifact@v4\n with:\n name: sqlite-vec-windows-x86_64-extension\n path: dist/*\n build-linux-aarch64-extension:\n runs-on: ubuntu-22.04-arm\n steps:\n - uses: actions/checkout@v4\n - run: ./scripts/vendor.sh\n - run: make loadable static\n - uses: actions/upload-artifact@v4\n with:\n name: sqlite-vec-linux-aarch64-extension\n path: dist/*\n build-cosmopolitan:\n runs-on: macos-latest\n permissions:\n contents: write\n steps:\n - uses: actions/checkout@v4\n - run: |\n mkdir $HOME/cosmo\n curl -L -o cosmocc-$COSMO_VERSION.zip https://github.com/jart/cosmopolitan/releases/download/$COSMO_VERSION/cosmocc-$COSMO_VERSION.zip\n unzip cosmocc-$COSMO_VERSION.zip -d $HOME/cosmo\n env:\n COSMO_VERSION: \"3.5.4\"\n - run: ./scripts/vendor.sh\n - run: make cli CC=$HOME/cosmo/bin/cosmocc AR=$HOME/cosmo/bin/cosmoar OMIT_SIMD=1\n - run: tar -czvf sqlite-vec-$(cat VERSION)-cli-cosmopolitan.tar.gz dist/sqlite3\n - run: gh release upload ${{ github.ref_name }} sqlite-vec-$(cat VERSION)-cli-cosmopolitan.tar.gz\n env:\n GH_TOKEN: ${{ github.token }}\n - uses: actions/upload-artifact@v4\n with:\n name: sqlite-vec-cosmopolitan\n path: dist/*\n build-wasm32-emscripten:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v4\n - uses: mymindstorm/setup-emsdk@v12\n with:\n version: \"latest\"\n - run: ./scripts/vendor.sh\n - run: make sqlite-vec.h\n - run: make wasm\n - uses: actions/upload-artifact@v4\n with:\n name: sqlite-vec-wasm32-emscripten\n path: dist/.wasm/*\n include-hidden-files: true\n build-android-extensions:\n runs-on: ubuntu-latest\n strategy:\n matrix:\n platforms:\n [\n { name: android-aarch64, cc: aarch64-linux-android21-clang },\n { name: android-i686, cc: i686-linux-android21-clang },\n { name: android-x86_64, cc: x86_64-linux-android21-clang },\n { name: android-armv7a, cc: armv7a-linux-androideabi21-clang },\n ]\n steps:\n - uses: actions/checkout@v4\n - run: ./scripts/vendor.sh\n - uses: actions/setup-java@v4\n with:\n java-version: \"17\"\n distribution: \"temurin\"\n - uses: android-actions/setup-android@v3\n - run: |\n sdkmanager --install \"ndk;27.0.12077973\"\n echo \"ANDROID_NDK_HOME=$ANDROID_SDK_ROOT/ndk/27.0.12077973\" >> $GITHUB_ENV\n - run: |\n ls $ANDROID_NDK_HOME/toolchains/llvm/prebuilt/\n ls $ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/bin\n - run: |\n export PATH=$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/bin:$PATH\n make CC=${{ matrix.platforms.cc }} \\\n AR=llvm-ar \\\n RANLIB=llvm-ranlib \\\n STRIP=llvm-strip loadable\n - uses: actions/upload-artifact@v4\n with:\n name: sqlite-vec-${{ matrix.platforms.name }}-extension\n path: dist/*.so\n build-ios-extensions:\n runs-on: ${{ matrix.platforms.runner }}\n strategy:\n fail-fast: false\n matrix:\n platforms: [\n {\n name: ios-aarch64,\n target: arm64-apple-ios,\n sdk: iphoneos,\n runner: macos-14,\n },\n #{\n # name: ios-x86_64,\n # target: x86_64-apple-ios,\n # sdk: iphoneos,\n # runner: macos-12,\n #},\n {\n name: iossimulator-aarch64,\n target: arm64-apple-ios-simulator,\n sdk: iphonesimulator,\n runner: macos-14,\n },\n {\n name: iossimulator-x86_64,\n target: x86_64-apple-ios-simulator,\n sdk: iphonesimulator,\n runner: macos-14,\n },\n ]\n steps:\n - uses: actions/checkout@v4\n - uses: maxim-lobanov/setup-xcode@v1\n with:\n xcode-version: latest-stable\n - run: ./scripts/vendor.sh\n\n - run: make CFLAGS=\"-target ${{ matrix.platforms.target }} -isysroot $(xcrun -sdk ${{ matrix.platforms.sdk }} --show-sdk-path) -fembed-bitcode -DNDEBUG=1\" loadable static\n - uses: actions/upload-artifact@v4\n with:\n name: sqlite-vec-${{ matrix.platforms.name }}-extension\n path: dist/*\n dist:\n runs-on: ubuntu-latest\n needs:\n [\n build-linux-x86_64-extension,\n build-linux-aarch64-extension,\n build-macos-x86_64-extension,\n build-macos-aarch64-extension,\n build-windows-x86_64-extension,\n build-wasm32-emscripten,\n build-android-extensions,\n build-ios-extensions,\n ]\n environment:\n name: release\n permissions:\n contents: write\n id-token: write\n steps:\n - uses: actions/checkout@v4\n - uses: actions/download-artifact@v4\n with:\n name: sqlite-vec-linux-x86_64-extension\n path: dist/linux-x86_64\n - uses: actions/download-artifact@v4\n with:\n name: sqlite-vec-linux-aarch64-extension\n path: dist/linux-aarch64\n - uses: actions/download-artifact@v4\n with:\n name: sqlite-vec-macos-x86_64-extension\n path: dist/macos-x86_64\n - uses: actions/download-artifact@v4\n with:\n name: sqlite-vec-macos-aarch64-extension\n path: dist/macos-aarch64\n - uses: actions/download-artifact@v4\n with:\n name: sqlite-vec-windows-x86_64-extension\n path: dist/windows-x86_64\n - uses: actions/download-artifact@v4\n with:\n name: sqlite-vec-wasm32-emscripten\n path: dist/wasm32-emscripten\n - uses: actions/download-artifact@v4\n with:\n name: sqlite-vec-android-aarch64-extension\n path: dist/android-aarch64\n - uses: actions/download-artifact@v4\n with:\n name: sqlite-vec-android-i686-extension\n path: dist/android-i686\n - uses: actions/download-artifact@v4\n with:\n name: sqlite-vec-android-x86_64-extension\n path: dist/android-x86_64\n - uses: actions/download-artifact@v4\n with:\n name: sqlite-vec-android-armv7a-extension\n path: dist/android-armv7a\n - uses: actions/download-artifact@v4\n with:\n name: sqlite-vec-ios-aarch64-extension\n path: dist/ios-aarch64\n - uses: actions/download-artifact@v4\n with:\n name: sqlite-vec-iossimulator-aarch64-extension\n path: dist/iossimulator-aarch64\n - uses: actions/download-artifact@v4\n with:\n name: sqlite-vec-iossimulator-x86_64-extension\n path: dist/iossimulator-x86_64\n - run: make sqlite-vec.h\n - uses: asg017/setup-sqlite-dist@fadb0183a6ec70c3f1942de7d232b087ff2bacd1\n - run: sqlite-dist build --set-version $(cat VERSION)\n - run: |\n gh release upload ${{ github.ref_name }} \\\n .sqlite-dist/amalgamation/* \\\n .sqlite-dist/github_releases/* \\\n .sqlite-dist/spm/* \\\n .sqlite-dist/sqlpkg/* \\\n .sqlite-dist/checksums.txt \\\n .sqlite-dist/sqlite-dist-manifest.json \\\n .sqlite-dist/install.sh\n env:\n GH_TOKEN: ${{ github.token }}\n - name: Install node\n uses: actions/setup-node@v4\n with:\n node-version: \"24\"\n registry-url: \"https://registry.npmjs.org\"\n - run: |\n VERSION=$(cat VERSION)\n if echo \"$VERSION\" | grep -q \"alpha\"; then\n TAG=alpha\n elif echo \"$VERSION\" | grep -q \"beta\"; then\n TAG=beta\n else\n TAG=latest\n fi\n npm publish --provenance --access public --tag $TAG .sqlite-dist/npm/sqlite-vec-darwin-arm64.tar.gz\n npm publish --provenance --access public --tag $TAG .sqlite-dist/npm/sqlite-vec-darwin-x64.tar.gz\n npm publish --provenance --access public --tag $TAG .sqlite-dist/npm/sqlite-vec-linux-x64.tar.gz\n npm publish --provenance --access public --tag $TAG .sqlite-dist/npm/sqlite-vec-linux-arm64.tar.gz\n npm publish --provenance --access public --tag $TAG .sqlite-dist/npm/sqlite-vec-windows-x64.tar.gz\n npm publish --provenance --access public --tag $TAG .sqlite-dist/npm/sqlite-vec-wasm-demo.tar.gz\n npm publish --provenance --access public --tag $TAG .sqlite-dist/npm/sqlite-vec.tar.gz\n env:\n NODE_AUTH_TOKEN: \"\"\n - uses: ruby/setup-ruby@v1\n with:\n ruby-version: 3.2\n - run: |\n for file in .sqlite-dist/gem/*; do\n gem push \"$file\"\n done\n env:\n GEM_HOST_API_KEY: ${{ secrets.GEM_HOST_API_KEY }}\n - uses: actions/setup-python@v5\n with:\n python-version: \"3.12\"\n - run: pip install twine\n - run: |\n twine upload .sqlite-dist/pip/*\n twine upload .sqlite-dist/datasette/*\n twine upload .sqlite-dist/sqlite_utils/*\n env:\n TWINE_USERNAME: __token__\n TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}\n upload-crate:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v4\n - uses: actions-rs/toolchain@v1\n with:\n toolchain: stable\n - run: ./scripts/vendor.sh\n - run: make sqlite-vec.h\n - run: make deps\n working-directory: ./bindings/rust\n - run: cargo publish --no-verify\n working-directory: ./bindings/rust\n env:\n CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}\n\n build-ncruces-go:\n if: false\n runs-on: ubuntu-latest\n permissions:\n contents: write\n steps:\n - uses: actions/checkout@v4\n - run: make sqlite-vec.h\n - uses: actions/checkout@v4\n with:\n repository: ncruces/go-sqlite3\n path: go-sqlite3\n - run: git apply ../bindings/go/ncruces/go-sqlite3.patch\n working-directory: go-sqlite3/\n - run: |\n mkdir -p tools/\n [ -d \"tools/wasi-sdk\" ] || curl -#L \"$WASI_SDK\" | tar xzC tools &\n [ -d \"tools/binaryen\" ] || curl -#L \"$BINARYEN\" | tar xzC tools &\n wait\n\n mv \"tools/wasi-sdk\"* \"tools/wasi-sdk\"\n mv \"tools/binaryen\"* \"tools/binaryen\"\n\n sqlite3/download.sh\n embed/build.sh\n env:\n WASI_SDK: \"https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-23/wasi-sdk-23.0-x86_64-linux.tar.gz\"\n BINARYEN: \"https://github.com/WebAssembly/binaryen/releases/download/version_118/binaryen-version_118-x86_64-linux.tar.gz\"\n working-directory: go-sqlite3/\n - uses: actions/checkout@v4\n with:\n repository: asg017/sqlite-vec-go-bindings\n path: sqlite-vec-go-bindings\n token: ${{secrets.NCRUCES_BINDINGS_REPO_PAT}}\n - run: |\n cp go-sqlite3/embed/sqlite3.wasm sqlite-vec-go-bindings/ncruces/sqlite3.wasm\n cp sqlite-vec.c sqlite-vec-go-bindings/cgo/sqlite-vec.c\n cp sqlite-vec.h sqlite-vec-go-bindings/cgo/sqlite-vec.h\n - run: |\n git config user.name \"Alex Garcia\"\n git config user.email \"alexsebastian.garcia@gmail.com\"\n git add .\n git commit --allow-empty -m \"AUTOMATED ${{ github.ref_name }}\" || exit 0\n git tag \"${{ github.ref_name }}\"\n git push origin main \"${{ github.ref_name }}\"\n working-directory: sqlite-vec-go-bindings\n env:\n GITHUB_TOKEN: ${{secrets.NCRUCES_BINDINGS_REPO_PAT}}\n" }, { "path": ".github/workflows/site.yaml", "content": "name: Deploy Site\non:\n workflow_dispatch: {}\n push:\n branches:\n - main\n paths:\n - \"site/**\"\n - \".github/**\"\n - \"VERSION\"\n - \"reference.yaml\"\njobs:\n deploy:\n runs-on: ubuntu-latest\n permissions:\n pages: write\n id-token: write\n environment:\n name: github-pages\n url: ${{ steps.deployment.outputs.page_url }}\n steps:\n - uses: actions/checkout@v4\n - uses: actions/setup-node@v4\n with:\n cache: npm\n cache-dependency-path: site/package-lock.json\n - run: npm ci\n working-directory: site/\n - run: make site-build\n - uses: actions/configure-pages@v2\n - uses: actions/upload-pages-artifact@v4\n with:\n path: site/.vitepress/dist\n - id: deployment\n uses: actions/deploy-pages@v4\n" }, { "path": ".github/workflows/test.yaml", "content": "name: \"Test\"\non:\n push:\n branches:\n - main\npermissions:\n contents: read\njobs:\n build-linux-x86_64-extension:\n runs-on: ubuntu-22.04\n steps:\n - uses: actions/checkout@v4\n - uses: astral-sh/setup-uv@v7\n - run: ./scripts/vendor.sh\n - run: make loadable static\n - run: uv sync --directory tests\n - run: make test-loadable\n - uses: actions/upload-artifact@v4\n with:\n name: sqlite-vec-linux-x86_64-extension\n path: dist/*\n build-macos-x86_64-extension:\n runs-on: macos-15-intel\n steps:\n - uses: actions/checkout@v4\n - uses: astral-sh/setup-uv@v7\n with:\n enable-cache: true\n - run: uv python install 3.12\n - run: ./scripts/vendor.sh\n - run: make loadable static\n - run: uv sync --directory tests\n - run: make test-loadable\n - uses: actions/upload-artifact@v4\n with:\n name: sqlite-vec-macos-x86_64-extension\n path: dist/*\n build-macos-aarch64-extension:\n runs-on: macos-14\n steps:\n - uses: actions/checkout@v4\n - uses: astral-sh/setup-uv@v7\n with:\n enable-cache: true\n - run: ./scripts/vendor.sh\n - run: make loadable static\n - run: uv sync --directory tests\n - run: make test-loadable\n - uses: actions/upload-artifact@v4\n with:\n name: sqlite-vec-macos-aarch64-extension\n path: dist/*\n build-windows-x86_64-extension:\n runs-on: windows-2022\n steps:\n - uses: actions/checkout@v4\n - uses: ilammy/msvc-dev-cmd@v1\n - uses: astral-sh/setup-uv@v7\n with:\n enable-cache: true\n - run: ./scripts/vendor.sh\n shell: bash\n - run: make sqlite-vec.h\n - run: mkdir dist\n - run: cl.exe /fPIC -shared /W4 /Ivendor/ /O2 /LD sqlite-vec.c -o dist/vec0.dll\n - run: uv sync --directory tests\n - run: make test-loadable\n shell: bash\n - uses: actions/upload-artifact@v4\n with:\n name: sqlite-vec-windows-x86_64-extension\n path: dist/*\n build-linux-aarch64-extension:\n runs-on: ubuntu-22.04-arm\n steps:\n - uses: actions/checkout@v4\n - uses: astral-sh/setup-uv@v7\n - run: ./scripts/vendor.sh\n - run: make loadable static\n - run: uv sync --directory tests\n - run: make test-loadable\n - uses: actions/upload-artifact@v4\n with:\n name: sqlite-vec-linux-aarch64-extension\n path: dist/*\n build-android-extensions:\n runs-on: ubuntu-latest\n strategy:\n matrix:\n platforms:\n [\n { name: android-aarch64, cc: aarch64-linux-android21-clang },\n { name: android-i686, cc: i686-linux-android21-clang },\n { name: android-x86_64, cc: x86_64-linux-android21-clang },\n { name: android-armv7a, cc: armv7a-linux-androideabi21-clang },\n ]\n steps:\n - uses: actions/checkout@v4\n - run: ./scripts/vendor.sh\n - uses: actions/setup-java@v4\n with:\n java-version: \"17\"\n distribution: \"temurin\"\n - uses: android-actions/setup-android@v3\n - run: |\n sdkmanager --install \"ndk;27.0.12077973\"\n echo \"ANDROID_NDK_HOME=$ANDROID_SDK_ROOT/ndk/27.0.12077973\" >> $GITHUB_ENV\n - run: |\n ls $ANDROID_NDK_HOME/toolchains/llvm/prebuilt/\n ls $ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/bin\n - run: |\n export PATH=$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/bin:$PATH\n make CC=${{ matrix.platforms.cc }} \\\n AR=llvm-ar \\\n RANLIB=llvm-ranlib \\\n STRIP=llvm-strip loadable\n - uses: actions/upload-artifact@v4\n with:\n name: sqlite-vec-android-${{ matrix.platforms.name }}-extension\n path: dist/*.so\n build-ios-extensions:\n runs-on: ${{ matrix.platforms.runner }}\n strategy:\n fail-fast: false\n matrix:\n platforms: [\n {\n name: ios-arm64,\n target: arm64-apple-ios,\n sdk: iphoneos,\n runner: macos-14,\n },\n #{\n # name: ios-x86_64,\n # target: x86_64-apple-ios,\n # sdk: iphoneos,\n # runner: macos-12,\n #},\n {\n name: ios-simulator-arm64,\n target: arm64-apple-ios-simulator,\n sdk: iphonesimulator,\n runner: macos-14,\n },\n {\n name: ios-simulator-x86_64,\n target: x86_64-apple-ios-simulator,\n sdk: iphonesimulator,\n runner: macos-14,\n },\n ]\n steps:\n - uses: actions/checkout@v4\n - uses: maxim-lobanov/setup-xcode@v1\n with:\n xcode-version: latest-stable\n - run: ./scripts/vendor.sh\n\n - run: make CFLAGS=\"-target ${{ matrix.platforms.target }} -isysroot $(xcrun -sdk ${{ matrix.platforms.sdk }} --show-sdk-path) -fembed-bitcode -DNDEBUG=1\" loadable static\n - uses: actions/upload-artifact@v4\n with:\n name: sqlite-vec-${{ matrix.platforms.name }}-extension\n path: dist/*\n build-wasm32-emscripten:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v4\n - uses: mymindstorm/setup-emsdk@v12\n with:\n version: \"latest\"\n - run: ./scripts/vendor.sh\n - run: make sqlite-vec.h\n - run: make wasm\n - run: ls; ls dist; ls dist/.wasm\n - uses: actions/upload-artifact@v4\n with:\n name: sqlite-vec-wasm32-emscripten\n path: dist/.wasm/*\n include-hidden-files: true\n build-pyodide:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v4\n - uses: mymindstorm/setup-emsdk@v12\n with:\n version: \"latest\"\n - run: ./scripts/vendor.sh\n - run: make sqlite-vec.h\n - run: |\n emcc -c ./sqlite-vec.c -o ./sqlite-vec.o -I ./vendor -fPIC -g3 \\\n -DSTDC_HEADERS=1 -DHAVE_SYS_TYPES_H=1 -DHAVE_SYS_STAT_H=1 -DHAVE_STDLIB_H=1 -DHAVE_STRING_H=1 -DHAVE_MEMORY_H=1 -DHAVE_STRINGS_H=1 -DHAVE_INTTYPES_H=1 -DHAVE_STDINT_H=1 -DHAVE_UNISTD_H=1 -DHAVE_FDATASYNC=1 -DHAVE_USLEEP=1 -DHAVE_LOCALTIME_R=1 -DHAVE_GMTIME_R=1 -DHAVE_DECL_STRERROR_R=1 -DHAVE_STRERROR_R=1 -DHAVE_POSIX_FALLOCATE=1 -DSQLITE_ENABLE_MATH_FUNCTIONS=1 -DSQLITE_ENABLE_FTS4=1 -DSQLITE_ENABLE_FTS5=1 -DSQLITE_ENABLE_RTREE=1 -DSQLITE_ENABLE_GEOPOLY=1 -DSQLITE_OMIT_POPEN=1 -DSQLITE_THREADSAFE=0\n emcc ./sqlite-vec.o -o vec0.so -s SIDE_MODULE=1 -g3 -s WASM_BIGINT=1\n - run: ls\n - uses: actions/upload-artifact@v4\n with:\n name: sqlite-vec-pyodide\n path: vec0.so\n build-ncruces-go:\n if: false\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v4\n - run: make sqlite-vec.h\n - uses: actions/checkout@v4\n with:\n repository: ncruces/go-sqlite3\n path: go-sqlite3\n - run: git apply ../bindings/go/ncruces/go-sqlite3.patch\n working-directory: go-sqlite3/\n - run: |\n mkdir -p tools/\n [ -d \"tools/wasi-sdk\" ] || curl -#L \"$WASI_SDK\" | tar xzC tools &\n [ -d \"tools/binaryen\" ] || curl -#L \"$BINARYEN\" | tar xzC tools &\n wait\n\n mv \"tools/wasi-sdk\"* \"tools/wasi-sdk\"\n mv \"tools/binaryen\"* \"tools/binaryen\"\n\n sqlite3/download.sh\n embed/build.sh\n env:\n WASI_SDK: \"https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-23/wasi-sdk-23.0-x86_64-linux.tar.gz\"\n BINARYEN: \"https://github.com/WebAssembly/binaryen/releases/download/version_118/binaryen-version_118-x86_64-linux.tar.gz\"\n working-directory: go-sqlite3/\n - uses: actions/upload-artifact@v4\n with:\n name: sqlite-vec-ncruces-go\n path: go-sqlite3/embed/sqlite3.wasm\n build-cosmopolitan:\n runs-on: macos-latest\n steps:\n - uses: actions/checkout@v4\n - run: |\n mkdir $HOME/cosmo\n curl -L -o cosmocc-$COSMO_VERSION.zip https://github.com/jart/cosmopolitan/releases/download/$COSMO_VERSION/cosmocc-$COSMO_VERSION.zip\n unzip cosmocc-$COSMO_VERSION.zip -d $HOME/cosmo\n env:\n COSMO_VERSION: \"3.5.4\"\n - run: ./scripts/vendor.sh\n - run: make cli CC=$HOME/cosmo/bin/cosmocc AR=$HOME/cosmo/bin/cosmoar OMIT_SIMD=1\n - uses: actions/upload-artifact@v4\n with:\n name: sqlite-vec-cosmopolitan\n path: dist/*\n test-minimum:\n runs-on: macos-latest\n steps:\n - uses: actions/checkout@v4\n - run: make sqlite-vec.h\n - run: make test\n working-directory: tests/minimum\n" }, { "path": ".gitignore", "content": "/target\n.vscode\nsift/\n*.tar.gz\n*.db\n*.npy\n*.bin\n*.out\nvenv/\n\nvendor/\ndist/\n\n*.pyc\n*.db-journal\n\nalexandria/\nopenai/\nexamples/supabase-dbpedia\nexamples/ann-filtering\nexamples/dbpedia-openai\nexamples/imdb\nexamples/sotu\n\nsqlite-vec.h\ntmp/\n\npoetry.lock\n\n*.jsonl\n\nmemstat.c\nmemstat.*\n" }, { "path": "ARCHITECTURE.md", "content": "# `sqlite-vec` Architecture\n\nInternal documentation for how `sqlite-vec` works under-the-hood. Not meant for\nusers of the `sqlite-vec` project, consult\n[the official `sqlite-vec` documentation](https://alexgarcia.xyz/sqlite-vec) for\nhow-to-guides. Rather, this is for people interested in how `sqlite-vec` works\nand some guidelines to any future contributors.\n\nVery much a WIP.\n\n## `vec0`\n\n### Shadow Tables\n\n#### `xyz_chunks`\n\n- `chunk_id INTEGER`\n- `size INTEGER`\n- `validity BLOB`\n- `rowids BLOB`\n\n#### `xyz_rowids`\n\n- `rowid INTEGER`\n- `id`\n- `chunk_id INTEGER`\n- `chunk_offset INTEGER`\n\n#### `xyz_vector_chunksNN`\n\n- `rowid INTEGER`\n- `vector BLOB`\n\n#### `xyz_auxiliary`\n\n- `rowid INTEGER`\n- `valueNN [type]`\n\n#### `xyz_metadatachunksNN`\n\n- `rowid INTEGER`\n- `data BLOB`\n\n#### `xyz_metadatatextNN`\n\n- `rowid INTEGER`\n- `data TEXT`\n\n### idxStr\n\nThe `vec0` idxStr is a string composed of single \"header\" character and 0 or\nmore \"blocks\" of 4 characters each.\n\nThe \"header\" charcter denotes the type of query plan, as determined by the\n`enum vec0_query_plan` values. The current possible values are:\n\n| Name | Value | Description |\n| -------------------------- | ----- | ---------------------------------------------------------------------- |\n| `VEC0_QUERY_PLAN_FULLSCAN` | `'1'` | Perform a full-scan on all rows |\n| `VEC0_QUERY_PLAN_POINT` | `'2'` | Perform a single-lookup point query for the provided rowid |\n| `VEC0_QUERY_PLAN_KNN` | `'3'` | Perform a KNN-style query on the provided query vector and parameters. |\n\nEach 4-character \"block\" is associated with a corresponding value in `argv[]`.\nFor example, the 1st block at byte offset `1-4` (inclusive) is the 1st block and\nis associated with `argv[1]`. The 2nd block at byte offset `5-8` (inclusive) is\nassociated with `argv[2]` and so on. Each block describes what kind of value or\nfilter the given `argv[i]` value is.\n\n#### `VEC0_IDXSTR_KIND_KNN_MATCH` (`'{'`)\n\n`argv[i]` is the query vector of the KNN query.\n\nThe remaining 3 characters of the block are `_` fillers.\n\n#### `VEC0_IDXSTR_KIND_KNN_K` (`'}'`)\n\n`argv[i]` is the limit/k value of the KNN query.\n\nThe remaining 3 characters of the block are `_` fillers.\n\n#### `VEC0_IDXSTR_KIND_KNN_ROWID_IN` (`'['`)\n\n`argv[i]` is the optional `rowid in (...)` value, and must be handled with\n[`sqlite3_vtab_in_first()` / `sqlite3_vtab_in_next()`](https://www.sqlite.org/c3ref/vtab_in_first.html).\n\nThe remaining 3 characters of the block are `_` fillers.\n\n#### `VEC0_IDXSTR_KIND_KNN_PARTITON_CONSTRAINT` (`']'`)\n\n`argv[i]` is a \"constraint\" on a specific partition key.\n\nThe second character of the block denotes which partition key to filter on,\nusing `A` to denote the first partition key column, `B` for the second, etc. It\nis encoded with `'A' + partition_idx` and can be decoded with `c - 'A'`.\n\nThe third character of the block denotes which operator is used in the\nconstraint. It will be one of the values of `enum vec0_partition_operator`, as\nonly a subset of operations are supported on partition keys.\n\nThe fourth character of the block is a `_` filler.\n\n#### `VEC0_IDXSTR_KIND_POINT_ID` (`'!'`)\n\n`argv[i]` is the value of the rowid or id to match against for the point query.\n\nThe remaining 3 characters of the block are `_` fillers.\n\n#### `VEC0_IDXSTR_KIND_METADATA_CONSTRAINT` (`'&'`)\n\n`argv[i]` is the value of the `WHERE` constraint for a metdata column in a KNN\nquery.\n\nThe second character of the block denotes which metadata column the constraint\nbelongs to, using `A` to denote the first metadata column column, `B` for the\nsecond, etc. It is encoded with `'A' + metadata_idx` and can be decoded with\n`c - 'A'`.\n\nThe third character of the block is the constraint operator. It will be one of\n`enum vec0_metadata_operator`, as only a subset of operators are supported on\nmetadata column KNN filters.\n\nThe foruth character of the block is a `_` filler.\n" }, { "path": "LICENSE-APACHE", "content": " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\nTERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\nEND OF TERMS AND CONDITIONS\n\nAPPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\nCopyright 2024 Alex Garcia\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n" }, { "path": "LICENSE-MIT", "content": "MIT License\n\nCopyright (c) 2024 Alex Garcia\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" }, { "path": "Makefile", "content": "\nCOMMIT=$(shell git rev-parse HEAD)\nVERSION=$(shell cat VERSION)\nDATE=$(shell date +'%FT%TZ%z')\n\nINSTALL_LIB_DIR = /usr/local/lib\nINSTALL_INCLUDE_DIR = /usr/local/include\nINSTALL_BIN_DIR = /usr/local/bin\n\nifndef CC\nCC=gcc\nendif\nifndef AR\nAR=ar\nendif\n\nifeq ($(shell uname -s),Darwin)\nCONFIG_DARWIN=y\nelse ifeq ($(OS),Windows_NT)\nCONFIG_WINDOWS=y\nelse\nCONFIG_LINUX=y\nendif\n\nifdef CONFIG_DARWIN\nLOADABLE_EXTENSION=dylib\nendif\n\nifdef CONFIG_LINUX\nLOADABLE_EXTENSION=so\nCFLAGS += -lm\nendif\n\nifdef CONFIG_WINDOWS\nLOADABLE_EXTENSION=dll\nendif\n\nifndef OMIT_SIMD\n\tifeq ($(shell uname -sm),Darwin x86_64)\n\tCFLAGS += -mavx -DSQLITE_VEC_ENABLE_AVX\n\tendif\n\tifeq ($(shell uname -sm),Darwin arm64)\n\tCFLAGS += -mcpu=apple-m1 -DSQLITE_VEC_ENABLE_NEON\n\tendif\nendif\n\nifdef USE_BREW_SQLITE\n\tSQLITE_INCLUDE_PATH=-I/opt/homebrew/opt/sqlite/include\n\tSQLITE_LIB_PATH=-L/opt/homebrew/opt/sqlite/lib\n\tCFLAGS += $(SQLITE_INCLUDE_PATH) $(SQLITE_LIB_PATH)\nendif\n\nifdef IS_MACOS_ARM\nRENAME_WHEELS_ARGS=--is-macos-arm\nelse\nRENAME_WHEELS_ARGS=\nendif\n\nprefix=dist\n$(prefix):\n\tmkdir -p $(prefix)\n\nTARGET_LOADABLE=$(prefix)/vec0.$(LOADABLE_EXTENSION)\nTARGET_STATIC=$(prefix)/libsqlite_vec0.a\nTARGET_STATIC_H=$(prefix)/sqlite-vec.h\nTARGET_CLI=$(prefix)/sqlite3\n\nloadable: $(TARGET_LOADABLE)\nstatic: $(TARGET_STATIC)\ncli: $(TARGET_CLI)\n\nall: loadable static cli\n\nOBJS_DIR=$(prefix)/.objs\nLIBS_DIR=$(prefix)/.libs\nBUILD_DIR=$(prefix)/.build\n\n$(OBJS_DIR): $(prefix)\n\tmkdir -p $@\n\n$(LIBS_DIR): $(prefix)\n\tmkdir -p $@\n\n$(BUILD_DIR): $(prefix)\n\tmkdir -p $@\n\n\n$(TARGET_LOADABLE): sqlite-vec.c sqlite-vec.h $(prefix)\n\t$(CC) \\\n\t\t-fPIC -shared \\\n\t\t-Wall -Wextra \\\n\t\t-Ivendor/ \\\n\t\t-O3 \\\n\t\t$(CFLAGS) \\\n\t\t$< -o $@\n\n$(TARGET_STATIC): sqlite-vec.c sqlite-vec.h $(prefix) $(OBJS_DIR)\n\t$(CC) -Ivendor/ $(CFLAGS) -DSQLITE_CORE -DSQLITE_VEC_STATIC \\\n\t-O3 -c $< -o $(OBJS_DIR)/vec.o\n\t$(AR) rcs $@ $(OBJS_DIR)/vec.o\n\n$(TARGET_STATIC_H): sqlite-vec.h $(prefix)\n\tcp $< $@\n\n\n$(OBJS_DIR)/sqlite3.o: vendor/sqlite3.c $(OBJS_DIR)\n\t$(CC) -c -g3 -O3 -DSQLITE_EXTRA_INIT=core_init -DSQLITE_CORE -DSQLITE_ENABLE_STMT_SCANSTATUS -DSQLITE_ENABLE_BYTECODE_VTAB -DSQLITE_ENABLE_EXPLAIN_COMMENTS -I./vendor $< -o $@\n\n$(LIBS_DIR)/sqlite3.a: $(OBJS_DIR)/sqlite3.o $(LIBS_DIR)\n\t$(AR) rcs $@ $<\n\n$(BUILD_DIR)/shell-new.c: vendor/shell.c $(BUILD_DIR)\n\tsed 's/\\/\\*extra-version-info\\*\\//EXTRA_TODO/g' $< > $@\n\n$(OBJS_DIR)/shell.o: $(BUILD_DIR)/shell-new.c $(OBJS_DIR)\n\t$(CC) -c -g3 -O3 \\\n\t\t-I./vendor \\\n\t\t-DSQLITE_ENABLE_STMT_SCANSTATUS -DSQLITE_ENABLE_BYTECODE_VTAB -DSQLITE_ENABLE_EXPLAIN_COMMENTS \\\n\t\t-DEXTRA_TODO=\"\\\"CUSTOMBUILD:sqlite-vec\\n\\\"\" \\\n\t\t$< -o $@\n\n$(LIBS_DIR)/shell.a: $(OBJS_DIR)/shell.o $(LIBS_DIR)\n\t$(AR) rcs $@ $<\n\n$(OBJS_DIR)/sqlite-vec.o: sqlite-vec.c $(OBJS_DIR)\n\t$(CC) -c -g3 -Ivendor/ -I./ $(CFLAGS) $< -o $@\n\n$(LIBS_DIR)/sqlite-vec.a: $(OBJS_DIR)/sqlite-vec.o $(LIBS_DIR)\n\t$(AR) rcs $@ $<\n\n\n$(TARGET_CLI): sqlite-vec.h $(LIBS_DIR)/sqlite-vec.a $(LIBS_DIR)/shell.a $(LIBS_DIR)/sqlite3.a examples/sqlite3-cli/core_init.c $(prefix)\n\t$(CC) -g3 \\\n\t-Ivendor/ -I./ \\\n\t-DSQLITE_CORE \\\n\t-DSQLITE_VEC_STATIC \\\n\t-DSQLITE_THREADSAFE=0 -DSQLITE_ENABLE_FTS4 \\\n\t-DSQLITE_ENABLE_STMT_SCANSTATUS -DSQLITE_ENABLE_BYTECODE_VTAB -DSQLITE_ENABLE_EXPLAIN_COMMENTS \\\n\t-DSQLITE_EXTRA_INIT=core_init \\\n\t$(CFLAGS) \\\n\t-ldl -lm \\\n\texamples/sqlite3-cli/core_init.c $(LIBS_DIR)/shell.a $(LIBS_DIR)/sqlite3.a $(LIBS_DIR)/sqlite-vec.a -o $@\n\n\nsqlite-vec.h: sqlite-vec.h.tmpl VERSION\n\tVERSION=$(shell cat VERSION) \\\n\tDATE=$(shell date -r VERSION +'%FT%TZ%z') \\\n\tSOURCE=$(shell git log -n 1 --pretty=format:%H -- VERSION) \\\n\tVERSION_MAJOR=$$(echo $$VERSION | cut -d. -f1) \\\n\tVERSION_MINOR=$$(echo $$VERSION | cut -d. -f2) \\\n\tVERSION_PATCH=$$(echo $$VERSION | cut -d. -f3 | cut -d- -f1) \\\n\tenvsubst < $< > $@\n\nclean:\n\trm -rf dist\n\n\nFORMAT_FILES=sqlite-vec.h sqlite-vec.c\nformat: $(FORMAT_FILES)\n\tclang-format -i $(FORMAT_FILES)\n\tblack tests/test-loadable.py\n\nlint: SHELL:=/bin/bash\nlint:\n\tdiff -u <(cat $(FORMAT_FILES)) <(clang-format $(FORMAT_FILES))\n\nprogress:\n\tdeno run --allow-read=sqlite-vec.c scripts/progress.ts\n\n\nevidence-of:\n\t@echo \"EVIDENCE-OF: V$(shell printf \"%05d\" $$((RANDOM % 100000)))_$(shell printf \"%05d\" $$((RANDOM % 100000)))\"\n\ntest:\n\tsqlite3 :memory: '.read test.sql'\n\n.PHONY: version loadable static test clean gh-release evidence-of install uninstall\n\npublish-release:\n\t./scripts/publish-release.sh\n\n# -k test_vec0_update\ntest-loadable: loadable\n\tuv run --managed-python --project tests pytest -vv -s -x . tests/test-*.py\n\ntest-loadable-snapshot-update: loadable\n\tuv run --managed-python --project tests pytest -vv tests/test-loadable.py --snapshot-update\n\ntest-loadable-watch:\n\twatchexec --exts c,py,Makefile --clear -- make test-loadable\n\ntest-unit:\n\t$(CC) -DSQLITE_CORE -DSQLITE_VEC_TEST tests/test-unit.c sqlite-vec.c vendor/sqlite3.c -I./ -Ivendor -o $(prefix)/test-unit && $(prefix)/test-unit\n\nfuzz-build:\n\t$(MAKE) -C tests/fuzz all\n\nfuzz-quick: fuzz-build\n\t@echo \"Running all fuzz targets for 30 seconds each...\"\n\t@for target in tests/fuzz/targets/*; do \\\n\t [ -f \"$$target\" ] && [ -x \"$$target\" ] || continue; \\\n\t name=$$(basename $$target); \\\n\t echo \"=== Fuzzing $$name ===\"; \\\n\t corpus=\"tests/fuzz/corpus/$$name\"; \\\n\t mkdir -p \"$$corpus\"; \\\n\t dict=\"tests/fuzz/$${name//_/-}.dict\"; \\\n\t dict_flag=\"\"; \\\n\t [ -f \"$$dict\" ] && dict_flag=\"-dict=$$dict\"; \\\n\t \"$$target\" $$dict_flag \\\n\t -max_total_time=30 \"$$corpus\" 2>&1 || true; \\\n\tdone\n\nfuzz-long: fuzz-build\n\t@echo \"Running all fuzz targets for 5 minutes each...\"\n\t@for target in tests/fuzz/targets/*; do \\\n\t [ -f \"$$target\" ] && [ -x \"$$target\" ] || continue; \\\n\t name=$$(basename $$target); \\\n\t echo \"=== Fuzzing $$name ===\"; \\\n\t corpus=\"tests/fuzz/corpus/$$name\"; \\\n\t mkdir -p \"$$corpus\"; \\\n\t dict=\"tests/fuzz/$${name//_/-}.dict\"; \\\n\t dict_flag=\"\"; \\\n\t [ -f \"$$dict\" ] && dict_flag=\"-dict=$$dict\"; \\\n\t \"$$target\" $$dict_flag \\\n\t -max_total_time=300 \"$$corpus\" 2>&1 || true; \\\n\tdone\n\nsite-dev:\n\tnpm --prefix site run dev\n\nsite-build:\n\tnpm --prefix site run build\n\ninstall:\n\tinstall -d $(INSTALL_LIB_DIR)\n\tinstall -d $(INSTALL_INCLUDE_DIR)\n\tinstall -m 644 sqlite-vec.h $(INSTALL_INCLUDE_DIR)\n\t@if [ -f $(TARGET_LOADABLE) ]; then \\\n\t\tinstall -m 644 $(TARGET_LOADABLE) $(INSTALL_LIB_DIR); \\\n\tfi\n\t@if [ -f $(TARGET_STATIC) ]; then \\\n\t\tinstall -m 644 $(TARGET_STATIC) $(INSTALL_LIB_DIR); \\\n\tfi\n\t@if [ -f $(TARGET_CLI) ]; then \\\n\t\tsudo install -m 755 $(TARGET_CLI) $(INSTALL_BIN_DIR); \\\n\tfi\n\tldconfig\n\nuninstall:\n\trm -f $(INSTALL_LIB_DIR)/$(notdir $(TARGET_LOADABLE))\n\trm -f $(INSTALL_LIB_DIR)/$(notdir $(TARGET_STATIC))\n\trm -f $(INSTALL_LIB_DIR)/$(notdir $(TARGET_CLI))\n\trm -f $(INSTALL_INCLUDE_DIR)/sqlite-vec.h\n\tldconfig\n\n# ███████████████████████████████ WASM SECTION ███████████████████████████████\n\nWASM_DIR=$(prefix)/.wasm\n\n$(WASM_DIR): $(prefix)\n\tmkdir -p $@\n\nSQLITE_WASM_VERSION=3450300\nSQLITE_WASM_YEAR=2024\nSQLITE_WASM_SRCZIP=$(BUILD_DIR)/sqlite-src.zip\nSQLITE_WASM_COMPILED_SQLITE3C=$(BUILD_DIR)/sqlite-src-$(SQLITE_WASM_VERSION)/sqlite3.c\nSQLITE_WASM_COMPILED_MJS=$(BUILD_DIR)/sqlite-src-$(SQLITE_WASM_VERSION)/ext/wasm/jswasm/sqlite3.mjs\nSQLITE_WASM_COMPILED_WASM=$(BUILD_DIR)/sqlite-src-$(SQLITE_WASM_VERSION)/ext/wasm/jswasm/sqlite3.wasm\n\nTARGET_WASM_LIB=$(WASM_DIR)/libsqlite_vec.wasm.a\nTARGET_WASM_MJS=$(WASM_DIR)/sqlite3.mjs\nTARGET_WASM_WASM=$(WASM_DIR)/sqlite3.wasm\nTARGET_WASM=$(TARGET_WASM_MJS) $(TARGET_WASM_WASM)\n\n$(SQLITE_WASM_SRCZIP): $(BUILD_DIR)\n\tcurl -o $@ https://www.sqlite.org/$(SQLITE_WASM_YEAR)/sqlite-src-$(SQLITE_WASM_VERSION).zip\n\ttouch $@\n\n$(SQLITE_WASM_COMPILED_SQLITE3C): $(SQLITE_WASM_SRCZIP) $(BUILD_DIR)\n\trm -rf $(BUILD_DIR)/sqlite-src-$(SQLITE_WASM_VERSION)/ || true\n\tunzip -q -o $< -d $(BUILD_DIR)\n\t(cd $(BUILD_DIR)/sqlite-src-$(SQLITE_WASM_VERSION)/ && ./configure --enable-all && make sqlite3.c)\n\ttouch $@\n\n$(TARGET_WASM_LIB): examples/wasm/wasm.c sqlite-vec.c $(BUILD_DIR) $(WASM_DIR)\n\temcc -O3 -I./ -Ivendor -DSQLITE_CORE -c examples/wasm/wasm.c -o $(BUILD_DIR)/wasm.wasm.o\n\temcc -O3 -I./ -Ivendor -DSQLITE_CORE -c sqlite-vec.c -o $(BUILD_DIR)/sqlite-vec.wasm.o\n\temar rcs $@ $(BUILD_DIR)/wasm.wasm.o $(BUILD_DIR)/sqlite-vec.wasm.o\n\n$(SQLITE_WASM_COMPILED_MJS) $(SQLITE_WASM_COMPILED_WASM): $(SQLITE_WASM_COMPILED_SQLITE3C) $(TARGET_WASM_LIB)\n\t(cd $(BUILD_DIR)/sqlite-src-$(SQLITE_WASM_VERSION)/ext/wasm && \\\n\t\tmake sqlite3_wasm_extra_init.c=../../../../.wasm/libsqlite_vec.wasm.a jswasm/sqlite3.mjs jswasm/sqlite3.wasm \\\n\t)\n\n$(TARGET_WASM_MJS): $(SQLITE_WASM_COMPILED_MJS)\n\tcp $< $@\n\n$(TARGET_WASM_WASM): $(SQLITE_WASM_COMPILED_WASM)\n\tcp $< $@\n\nwasm: $(TARGET_WASM)\n\n# ███████████████████████████████ END WASM ███████████████████████████████\n" }, { "path": "README.md", "content": "# `sqlite-vec`\n\n[![](https://dcbadge.vercel.app/api/server/VCtQ8cGhUs)](https://discord.gg/Ve7WeCJFXk)\n\nAn extremely small, \"fast enough\" vector search SQLite extension that runs\nanywhere! A successor to [`sqlite-vss`](https://github.com/asg017/sqlite-vss)\n\n\n\n> [!IMPORTANT]\n> _`sqlite-vec` is a pre-v1, so expect breaking changes!_\n\n\n\n- Store and query float, int8, and binary vectors in `vec0` virtual tables\n- Written in pure C, no dependencies, runs anywhere SQLite runs\n (Linux/MacOS/Windows, in the browser with WASM, Raspberry Pis, etc.)\n- Store non-vector data in metadata, auxiliary, or partition key columns\n\n

\n \n \n \n \n \"Mozilla\n \n \n

\n\n

\n\nsqlite-vec is a\nMozilla Builders project,\nwith additional sponsorship from\n Fly.io ,\n Turso,\n SQLite Cloud, and\n Shinkai.\nSee the Sponsors section for more details.\n\n

\n\n## Installing\n\nSee [Installing `sqlite-vec`](https://alexgarcia.xyz/sqlite-vec/installation.html)\nfor more details.\n\n| Language | Install | More Info | |\n| -------------- | ---------------------------------------------------- | ------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |\n| Python | `pip install sqlite-vec` | [`sqlite-vec` with Python](https://alexgarcia.xyz/sqlite-vec/python.html) | [![PyPI](https://img.shields.io/pypi/v/sqlite-vec.svg?color=blue&logo=python&logoColor=white)](https://pypi.org/project/sqlite-vec/) |\n| Node.js | `npm install sqlite-vec` | [`sqlite-vec` with Node.js](https://alexgarcia.xyz/sqlite-vec/js.html) | [![npm](https://img.shields.io/npm/v/sqlite-vec.svg?color=green&logo=nodedotjs&logoColor=white)](https://www.npmjs.com/package/sqlite-vec) |\n| Ruby | `gem install sqlite-vec` | [`sqlite-vec` with Ruby](https://alexgarcia.xyz/sqlite-vec/ruby.html) | ![Gem](https://img.shields.io/gem/v/sqlite-vec?color=red&logo=rubygems&logoColor=white) |\n| Go | `go get -u github.com/asg017/sqlite-vec/bindings/go` | [`sqlite-vec` with Go](https://alexgarcia.xyz/sqlite-vec/go.html) | [![Go Reference](https://pkg.go.dev/badge/github.com/asg017/sqlite-vec-go-bindings/cgo.svg)](https://pkg.go.dev/github.com/asg017/asg017/sqlite-vec-go-bindings/cgo) |\n| Rust | `cargo add sqlite-vec` | [`sqlite-vec` with Rust](https://alexgarcia.xyz/sqlite-vec/rust.html) | [![Crates.io](https://img.shields.io/crates/v/sqlite-vec?logo=rust)](https://crates.io/crates/sqlite-vec) |\n| Datasette | `datasette install datasette-sqlite-vec` | [`sqlite-vec` with Datasette](https://alexgarcia.xyz/sqlite-vec/datasette.html) | [![Datasette](https://img.shields.io/pypi/v/datasette-sqlite-vec.svg?color=B6B6D9&label=Datasette+plugin&logoColor=white&logo=python)](https://datasette.io/plugins/datasette-sqlite-vec) |\n| rqlite | `rqlited -extensions-path=sqlite-vec.tar.gz` | [`sqlite-vec` with rqlite](https://alexgarcia.xyz/sqlite-vec/rqlite.html) | [![rqlite](https://img.shields.io/badge/rqlite-sqlite_extensions-blue)](https://rqlite.io/docs/guides/extensions/) |\n| `sqlite-utils` | `sqlite-utils install sqlite-utils-sqlite-vec` | [`sqlite-vec` with sqlite-utils](https://alexgarcia.xyz/sqlite-vec/sqlite-utils.html) | [![sqlite-utils](https://img.shields.io/pypi/v/sqlite-utils-sqlite-vec.svg?color=B6B6D9&label=sqlite-utils+plugin&logoColor=white&logo=python)](https://datasette.io/plugins/datasette-sqlite-vec) |\n| Github Release | | | ![GitHub tag (latest SemVer pre-release)](https://img.shields.io/github/v/tag/asg017/sqlite-vec?color=lightgrey&include_prereleases&label=Github+release&logo=github) |\n\n\n## Sample usage\n\n```sql\n.load ./vec0\n\ncreate virtual table vec_examples using vec0(\n sample_embedding float[8]\n);\n\n-- vectors can be provided as JSON or in a compact binary format\ninsert into vec_examples(rowid, sample_embedding)\n values\n (1, '[-0.200, 0.250, 0.341, -0.211, 0.645, 0.935, -0.316, -0.924]'),\n (2, '[0.443, -0.501, 0.355, -0.771, 0.707, -0.708, -0.185, 0.362]'),\n (3, '[0.716, -0.927, 0.134, 0.052, -0.669, 0.793, -0.634, -0.162]'),\n (4, '[-0.710, 0.330, 0.656, 0.041, -0.990, 0.726, 0.385, -0.958]');\n\n\n-- KNN style query\nselect\n rowid,\n distance\nfrom vec_examples\nwhere sample_embedding match '[0.890, 0.544, 0.825, 0.961, 0.358, 0.0196, 0.521, 0.175]'\norder by distance\nlimit 2;\n/*\n┌───────┬──────────────────┐\n│ rowid │ distance │\n├───────┼──────────────────┤\n│ 2 │ 2.38687372207642 │\n│ 1 │ 2.38978505134583 │\n└───────┴──────────────────┘\n*/\n```\n\n## Sponsors\n\nDevelopment of `sqlite-vec` is supported by multiple generous sponsors! Mozilla\nis the main sponsor through the new Builders project.\n

\n \n \n \n \n \"Mozilla\n \n \n

\n\n`sqlite-vec` is also sponsored by the following companies:\n\n\n\n \n \n \"Fly.io\n\n\n\n\n\n \n \n \"Turso\n\n\n\n\n\n \n \n \"SQLite\n\n\n\n\n\n \n \n\n \"Shinkai\n\n\n\nAs well as multiple individual supporters on\n[Github sponsors](https://github.com/sponsors/asg017/)!\n\nIf your company interested in sponsoring `sqlite-vec` development, send me an\nemail to get more info: https://alexgarcia.xyz\n\n## See Also\n\n- [**`sqlite-ecosystem`**](https://github.com/asg017/sqlite-ecosystem), Maybe\n more 3rd party SQLite extensions I've developed\n- [**`sqlite-rembed`**](https://github.com/asg017/sqlite-rembed), Generate text\n embeddings from remote APIs like OpenAI/Nomic/Ollama, meant for testing and\n SQL scripts\n- [**`sqlite-lembed`**](https://github.com/asg017/sqlite-lembed), Generate text\n embeddings locally from embedding models in the `.gguf` format\n" }, { "path": "SECURITY.md", "content": "Please report any security vulnerabilities to alexsebastian.garcia@gmail.com . Avould using public Github issues whenever possible. I will get back to you as quickly as possible.\n" }, { "path": "TODO", "content": "- [ ] add `xyz_info` shadow table with version etc.\n\n- later\n - [ ] partition: UPDATE support\n - [ ] skip invalid validity entries in knn filter?\n - [ ] nulls in metadata\n - [ ] partition `x in (...)` handling\n - [ ] blobs/date/datetime\n - [ ] uuid/ulid perf\n - [ ] Aux columns: `NOT NULL` constraint\n - [ ] Metadata columns: `NOT NULL` constraint\n - [ ] Partiion key: `NOT NULL` constraint\n - [ ] dictionary encoding?\n - [ ] properly sqlite3_vtab_nochange / sqlite3_value_nochange handling\n - [ ] perf\n - [ ] aux: cache INSERT\n - [ ] aux: LEFT JOIN on `_rowids` queries to avoid N lookup queries\n" }, { "path": "VERSION", "content": "0.1.8-alpha.1" }, { "path": "benchmarks/README.md", "content": "" }, { "path": "benchmarks/exhaustive-memory/.gitignore", "content": "data/\n" }, { "path": "benchmarks/exhaustive-memory/Makefile", "content": "\n\n\ndata/:\n\tmkdir -p $@\n\ndata/sift: data/\n\tcurl -o data/sift.tar.gz ftp://ftp.irisa.fr/local/texmex/corpus/sift.tar.gz\n\ttar -xvzf data/sift.tar.gz -C data/\n\trm data/sift.tar.gz\n\ndata/gist: data/\n\tcurl -o data/gist.tar.gz ftp://ftp.irisa.fr/local/texmex/corpus/gist.tar.gz\n\ttar -xvzf data/gist.tar.gz -C data/\n\trm data/gist.tar.gz\n" }, { "path": "benchmarks/exhaustive-memory/README.md", "content": "# `sqlite-vec` In-memory benchmark comparisions\n\nThis repo contains a benchmarks that compares KNN queries of `sqlite-vec` to other in-process vector search tools using **brute force linear scans only**. These include:\n\n\n- [Faiss IndexFlatL2](https://faiss.ai/)\n- [usearch with `exact=True`](https://github.com/unum-cloud/usearch)\n- [libsql vector search with `vector_distance_cos`](https://turso.tech/vector)\n- [numpy](https://numpy.org/), using [this approach](https://github.com/EthanRosenthal/nn-vs-ann)\n- [duckdb with `list_cosine_similarity`](https://duckdb.org/docs/sql/functions/nested.html#list_cosine_similaritylist1-list2)\n- [`sentence_transformers.util.semantic_search`](https://sbert.net/docs/package_reference/util.html#sentence_transformers.util.semantic_search)\n- [hnswlib BFIndex](https://github.com/nmslib/hnswlib/blob/c1b9b79af3d10c6ee7b5d0afa1ce851ae975254c/TESTING_RECALL.md?plain=1#L8)\n\n\nAgain **ONLY BRUTE FORCE LINEAR SCANS ARE TESTED**. This benchmark does **not** test approximate nearest neighbors (ANN) implementations. This benchmarks is extremely narrow to just testing KNN searches using brute force.\n\nA few other caveats:\n\n- Only brute-force linear scans, no ANN\n- Only CPU is used. The only tool that does offer GPU is Faiss anyway.\n- Only in-memory datasets are used. Many of these tools do support serializing and reading from disk (including `sqlite-vec`) and possibly `mmap`'ing, but this only tests in-memory datasets. Mostly because of numpy\n- Queries are made one after the other, **not batched.** Some tools offer APIs to query multiple inputs at the same time, but this benchmark runs queries sequentially. This was done to emulate \"server request\"-style queries, but multiple users would send queries at different times, making batching more difficult. To note, `sqlite-vec` does **not** support batched queries yet.\n\n\nThese tests are run in Python. Vectors are provided as an in-memory numpy array, and each test converts that numpy array into whatever makes sense for the given tool. For example, `sqlite-vec` tests will read those vectors into a SQLite table. DuckDB will read them into an Array array then create a DuckDB table from that.\n" }, { "path": "benchmarks/exhaustive-memory/bench.py", "content": "import numpy as np\nimport numpy.typing as npt\nimport time\nimport sqlite3\nimport pandas as pd\nfrom dataclasses import dataclass\nfrom rich.console import Console\nfrom rich.table import Table\nfrom typing import List, Optional\n\n\n@dataclass\nclass BenchResult:\n tool: str\n build_time_ms: float\n query_times_ms: List[float]\n\n\ndef duration(seconds: float):\n ms = seconds * 1000\n return f\"{int(ms)}ms\"\n\n\ndef cosine_similarity(\n vec: npt.NDArray[np.float32], mat: npt.NDArray[np.float32], do_norm: bool = True\n) -> npt.NDArray[np.float32]:\n sim = vec @ mat.T\n if do_norm:\n sim /= np.linalg.norm(vec) * np.linalg.norm(mat, axis=1)\n return sim\n\n\ndef topk(\n vec: npt.NDArray[np.float32],\n mat: npt.NDArray[np.float32],\n k: int = 5,\n do_norm: bool = True,\n) -> tuple[npt.NDArray[np.int32], npt.NDArray[np.float32]]:\n sim = cosine_similarity(vec, mat, do_norm=do_norm)\n # Rather than sorting all similarities and taking the top K, it's faster to\n # argpartition and then just sort the top K.\n # The difference is O(N logN) vs O(N + k logk)\n indices = np.argpartition(-sim, kth=k)[:k]\n top_indices = np.argsort(-sim[indices])\n return indices[top_indices], sim[top_indices]\n\n\ndef ivecs_read(fname):\n a = np.fromfile(fname, dtype=\"int32\",)\n d = a[0]\n return a.reshape(-1, d + 1)[:, 1:].copy()\n\n\ndef fvecs_read(fname, sample):\n return ivecs_read(fname).view(\"float32\")[:sample]\n\n\ndef bench_hnsw(base, query):\n import hnswlib\n t0 = time.time()\n p = hnswlib.Index(space=\"ip\", dim=128) # possible options are l2, cosine or ip\n\n # NOTE: Use default settings from the README.\n print(\"buildings hnsw\")\n p.init_index(max_elements=base.shape[0], ef_construction=200, M=16)\n ids = np.arange(base.shape[0])\n p.add_items(base, ids)\n p.set_ef(50)\n\n print(\"build time\", time.time() - t0)\n\n results = []\n times = []\n t = time.time()\n for idx, q in enumerate(query):\n t0 = time.time()\n result = p.knn_query(q, k=5)\n results.append(result)\n times.append(time.time() - t0)\n print(time.time() - t)\n print(\"hnsw avg\", np.mean(times))\n return results\n\n\ndef bench_hnsw_bf(base, query, k) -> BenchResult:\n import hnswlib\n print(\"hnswlib-bf\")\n dimensions = base.shape[1]\n t0 = time.time()\n p = hnswlib.BFIndex(space=\"l2\", dim=dimensions)\n\n p.init_index(max_elements=base.shape[0])\n ids = np.arange(base.shape[0])\n p.add_items(base, ids)\n\n build_time = time.time() - t0\n\n results = []\n times = []\n t = time.time()\n for idx, q in enumerate(query):\n t0 = time.time()\n result = p.knn_query(q, k=k)\n results.append(result)\n times.append(time.time() - t0)\n return BenchResult(\"hnswlib-bf\", build_time, times)\n\n\ndef bench_numpy(base, query, k) -> BenchResult:\n print(\"numpy...\")\n times = []\n results = []\n for idx, q in enumerate(query):\n t0 = time.time()\n result = topk(q, base, k=k)\n results.append(result)\n times.append(time.time() - t0)\n return BenchResult(\"numpy\", 0, times)\n\n\ndef bench_sqlite_vec(base, query, page_size, chunk_size, k) -> BenchResult:\n dimensions = base.shape[1]\n print(f\"sqlite-vec {page_size} {chunk_size}...\")\n\n db = sqlite3.connect(\":memory:\")\n db.execute(f\"PRAGMA page_size = {page_size}\")\n db.enable_load_extension(True)\n db.load_extension(\"../../dist/vec0\")\n db.execute(\n f\"\"\"\n create virtual table vec_sift1m using vec0(\n chunk_size={chunk_size},\n vector float[{dimensions}]\n )\n \"\"\"\n )\n\n t = time.time()\n with db:\n db.executemany(\n \"insert into vec_sift1m(vector) values (?)\",\n list(map(lambda x: [x.tobytes()], base)),\n )\n build_time = time.time() - t\n times = []\n results = []\n for (\n idx,\n q,\n ) in enumerate(query):\n t0 = time.time()\n result = db.execute(\n \"\"\"\n select\n rowid,\n distance\n from vec_sift1m\n where vector match ?\n and k = ?\n order by distance\n \"\"\",\n [q.tobytes(), k],\n ).fetchall()\n assert len(result) == k\n times.append(time.time() - t0)\n return BenchResult(f\"sqlite-vec vec0 ({page_size}|{chunk_size})\", build_time, times)\n\n\ndef bench_sqlite_vec_scalar(base, query, page_size, k) -> BenchResult:\n print(f\"sqlite-vec-scalar...\")\n\n db = sqlite3.connect(\":memory:\")\n db.enable_load_extension(True)\n db.load_extension(\"../../dist/vec0\")\n db.execute(f\"PRAGMA page_size={page_size}\")\n db.execute(f\"create table sift1m(vector);\")\n\n t = time.time()\n with db:\n db.executemany(\n \"insert into sift1m(vector) values (?)\",\n list(map(lambda x: [x.tobytes()], base)),\n )\n build_time = time.time() - t\n times = []\n results = []\n t = time.time()\n for (\n idx,\n q,\n ) in enumerate(query):\n t0 = time.time()\n result = db.execute(\n \"\"\"\n select\n rowid,\n vec_distance_l2(?, vector) as distance\n from sift1m\n order by distance\n limit ?\n \"\"\",\n [q.tobytes(), k],\n ).fetchall()\n assert len(result) == k\n times.append(time.time() - t0)\n return BenchResult(f\"sqlite-vec-scalar ({page_size})\", build_time, times)\n\ndef bench_libsql(base, query, page_size, k) -> BenchResult:\n print(f\"libsql ...\")\n dimensions = base.shape[1]\n\n db = sqlite3.connect(\":memory:\")\n db.enable_load_extension(True)\n assert db.execute(\"select 'vector' in (select name from pragma_function_list)\").fetchone()[0] == 1\n db.execute(f\"PRAGMA page_size={page_size}\")\n db.execute(f\"create table vectors(vector f32_blob({dimensions}));\")\n\n # TODO: only does DiskANN?\n #db.execute(\"CREATE INDEX vectors_idx ON vectors (libsql_vector_idx(vector, 'metric=cosine'))\")\n\n t = time.time()\n with db:\n db.executemany(\n \"insert into vectors(vector) values (?)\",\n list(map(lambda x: [x.tobytes()], base)),\n )\n build_time = time.time() - t\n times = []\n results = []\n t = time.time()\n for (\n idx,\n q,\n ) in enumerate(query):\n t0 = time.time()\n result = db.execute(\n \"\"\"\n select\n rowid,\n vector_distance_cos(?, vector) as distance\n FROM vectors\n order by 2\n limit ?\n \"\"\",\n [q.tobytes(), k],\n ).fetchall()\n times.append(time.time() - t0)\n return BenchResult(f\"libsql ({page_size})\", build_time, times)\n\n\ndef register_np(db, array, name):\n ptr = array.__array_interface__[\"data\"][0]\n nvectors, dimensions = array.__array_interface__[\"shape\"]\n element_type = array.__array_interface__[\"typestr\"]\n\n assert element_type == \" BenchResult:\n print(f\"sqlite-vec static...\")\n\n db = sqlite3.connect(\":memory:\")\n db.enable_load_extension(True)\n db.load_extension(\"../../dist/vec0\")\n\n\n\n t = time.time()\n register_np(db, base, \"base\")\n build_time = time.time() - t\n\n times = []\n results = []\n for (\n idx,\n q,\n ) in enumerate(query):\n t0 = time.time()\n result = db.execute(\n \"\"\"\n select\n rowid\n from base\n where vector match ?\n and k = ?\n order by distance\n \"\"\",\n [q.tobytes(), k],\n ).fetchall()\n assert len(result) == k\n times.append(time.time() - t0)\n return BenchResult(f\"sqlite-vec static\", build_time, times)\n\ndef bench_faiss(base, query, k) -> BenchResult:\n import faiss\n dimensions = base.shape[1]\n print(\"faiss...\")\n t = time.time()\n index = faiss.IndexFlatL2(dimensions)\n index.add(base)\n build_time = time.time() - t\n times = []\n results = []\n t = time.time()\n for idx, q in enumerate(query):\n t0 = time.time()\n distances, rowids = index.search(x=np.array([q]), k=k)\n results.append(rowids)\n times.append(time.time() - t0)\n return BenchResult(\"faiss\", build_time, times)\n\n\ndef bench_lancedb(base, query, k) -> BenchResult:\n import lancedb\n print('lancedb...')\n dimensions = base.shape[1]\n db = lancedb.connect(\"a\")\n data = [{\"vector\": row.reshape(1, -1)[0]} for row in base]\n # Create a DataFrame where each row is a 1D array\n df = pd.DataFrame(data=data, columns=[\"vector\"])\n t = time.time()\n db.create_table(\"t\", data=df)\n build_time = time.time() - t\n tbl = db.open_table(\"t\")\n times = []\n for q in query:\n t0 = time.time()\n result = tbl.search(q).limit(k).to_arrow()\n times.append(time.time() - t0)\n return BenchResult(\"lancedb\", build_time, times)\n\ndef bench_duckdb(base, query, k) -> BenchResult:\n import duckdb\n import pyarrow as pa\n print(\"duckdb...\")\n dimensions = base.shape[1]\n db = duckdb.connect(\":memory:\")\n db.execute(f\"CREATE TABLE t(vector float[{dimensions}])\")\n\n t0 = time.time()\n pa_base = pa.Table.from_arrays([pa.array(list(base))], names=['vector'])\n pa_base\n db.execute(f\"INSERT INTO t(vector) SELECT vector::float[{dimensions}] FROM pa_base\")\n build_time = time.time() - t0\n times = []\n for q in query:\n t0 = time.time()\n result = db.execute(\n f\"\"\"\n SELECT\n rowid,\n array_cosine_similarity(vector, ?::float[{dimensions}])\n FROM t\n ORDER BY 2 DESC\n LIMIT ?\n \"\"\", [q, k]).fetchall()\n times.append(time.time() - t0)\n return BenchResult(\"duckdb\", build_time, times)\n\ndef bench_sentence_transformers(base, query, k) -> BenchResult:\n from sentence_transformers.util import semantic_search\n print(\"sentence-transformers\")\n dimensions = base.shape[1]\n t0 = time.time()\n build_time = time.time() - t0\n\n times = []\n for q in query:\n t0 = time.time()\n result = semantic_search(q, base, top_k=k)\n times.append(time.time() - t0)\n\n return BenchResult(\"sentence-transformers\", build_time, times)\n\n\ndef bench_chroma(base, query, k):\n import chromadb\n from chromadb.utils.batch_utils import create_batches\n chroma_client = chromadb.EphemeralClient()\n collection = chroma_client.create_collection(name=\"my_collection\")\n\n t = time.time()\n for batch in create_batches(api=chroma_client, ids=[str(x) for x in range(len(base))], embeddings=base.tolist()):\n collection.add(*batch)\n build_time = time.time() - t\n times = []\n for q in query:\n t0 = time.time()\n result = collection.query(\n query_embeddings=[q.tolist()],\n n_results=k,\n )\n times.append(time.time() - t0)\n #print(\"chroma avg\", duration(np.mean(times)))\n return BenchResult(\"chroma\", build_time, times)\n\ndef bench_usearch_npy(base, query, k) -> BenchResult:\n from usearch.index import Index, search, MetricKind\n times = []\n for q in query:\n t0 = time.time()\n # result = index.search(q, exact=True)\n result = search(base, q, k, MetricKind.L2sq, exact=True)\n times.append(time.time() - t0)\n return BenchResult(\"usearch numpy exact=True\", 0, times)\n\n\ndef bench_usearch_special(base, query, k) -> BenchResult:\n from usearch.index import Index, search, MetricKind\n dimensions = base.shape[1]\n index = Index(ndim=dimensions)\n t = time.time()\n index.add(np.arange(len(base)), base)\n build_time = time.time() - t\n\n times = []\n for q in query:\n t0 = time.time()\n result = index.search(q, exact=True)\n times.append(time.time() - t0)\n return BenchResult(\"usuearch index\", build_time, times)\n\n\ndef suite(name, base, query, k, benchmarks):\n print(f\"Starting benchmark suite: {name} {base.shape}, k={k}\")\n results = []\n\n for b in benchmarks:\n if b == \"faiss\":\n results.append(bench_faiss(base, query, k=k))\n elif b == \"vec-static\":\n results.append(bench_sqlite_vec_static(base, query, k=k))\n elif b.startswith(\"vec-scalar\"):\n _, page_size = b.split('.')\n results.append(bench_sqlite_vec_scalar(base, query, page_size, k=k))\n elif b.startswith(\"libsql\"):\n _, page_size = b.split('.')\n results.append(bench_libsql(base, query, page_size, k=k))\n elif b.startswith(\"vec-vec0\"):\n _, page_size, chunk_size = b.split('.')\n results.append(bench_sqlite_vec(base, query, int(page_size), int(chunk_size), k=k))\n elif b == \"usearch\":\n results.append(bench_usearch_npy(base, query, k=k))\n elif b == \"hnswlib\":\n results.append(bench_hnsw_bf(base, query, k=k))\n elif b == \"numpy\":\n results.append(bench_numpy(base, query, k=k))\n elif b == \"duckdb\":\n results.append(bench_duckdb(base, query, k=k))\n elif b == \"sentence-transformers\":\n results.append(bench_sentence_transformers(base, query, k=k))\n elif b == \"chroma\":\n results.append(bench_chroma(base, query, k=k))\n else:\n raise Exception(f\"unknown benchmark {b}\")\n\n #results.append(bench_sqlite_vec(base, query, 32768, 512, k=k))\n #results.append(bench_sqlite_vec(base, query, 32768, 256, k=k))\n\n\n #results.append(bench_sqlite_vec_expo(base, query, k=k))\n\n # n = bench_chroma(base[:40000], query, k=k)\n\n # n = bench_usearch_special(base, query, k=k)\n\n\n\n # n = bench_sqlite_vec(base, query, 4096, 1024, k=k)\n # n = bench_sqlite_vec(base, query, 32768, 1024, k=k)\n\n\n\n # blessed\n\n ### #for pgsz in [4096, 8192, 16384, 32768, 65536]:\n ### # for chunksz in [8, 32, 128, 512, 1024, 2048]:\n ### # results.append(bench_sqlite_vec(base, query, pgsz, chunksz, k=k))\n ### # n = bench_sqlite_vec(base, query, 16384, 64, k=k)\n ### # n = bench_sqlite_vec(base, query, 16384, 32, k=k)\n ### results.append(bench_sqlite_normal(base, query, 8192, k=k))\n ### results.append(bench_lancedb(base, query, k=k))\n\n ### #h = bench_hnsw(base, query)\n\n table = Table(\n title=f\"{name}: {base.shape[0]:,} {base.shape[1]}-dimension vectors, k={k}\"\n )\n\n table.add_column(\"Tool\")\n table.add_column(\"Build Time (ms)\", justify=\"right\")\n table.add_column(\"Query time (ms)\", justify=\"right\")\n for res in sorted(results, key=lambda x: np.mean(x.query_times_ms)):\n table.add_row(\n res.tool, duration(res.build_time_ms), duration(np.mean(res.query_times_ms))\n )\n\n console = Console()\n console.print(table)\n\n\nimport argparse\n\n\ndef parse_args():\n parser = argparse.ArgumentParser(description=\"Benchmark processing script.\")\n # Required arguments\n parser.add_argument(\"-n\", \"--name\", required=True, help=\"Name of the benchmark.\")\n parser.add_argument(\n \"-i\", \"--input\", required=True, help=\"Path to input file (.npy).\"\n )\n parser.add_argument(\n \"-k\", type=int, required=True, help=\"Parameter k to use in benchmark.\"\n )\n\n # Optional arguments\n parser.add_argument(\n \"-q\", \"--query\", required=False, help=\"Path to query file (.npy).\"\n )\n parser.add_argument(\n \"--sample\",\n type=int,\n required=False,\n help=\"Number of entries in base to use. Defaults all\",\n default=-1\n )\n parser.add_argument(\n \"--qsample\",\n type=int,\n required=False,\n help=\"Number of queries to use. Defaults all\",\n )\n parser.add_argument(\n \"-x\", help=\"type of runs to make\", default=\"faiss,vec-scalar.4096,vec-static,vec-vec0.4096.16,usearch,duckdb,hnswlib,numpy\"\n )\n\n args = parser.parse_args()\n return args\n\n\nfrom pathlib import Path\n\n\ndef cli_read_input(input, sample):\n input_path = Path(input)\n if input_path.suffix == \".fvecs\":\n return fvecs_read(input_path, sample)\n if input_path.suffx == \".npy\":\n return np.fromfile(input_path, dtype=\"float32\", count=sample)\n raise Exception(\"unknown filetype\", input)\n\n\ndef cli_read_query(query, base):\n if query is None:\n return base[np.random.choice(base.shape[0], 100, replace=False), :]\n return cli_read_input(query, -1)\n\n\n\n@dataclass\nclass Config:\n name: str\n input: str\n k: int\n queries: str\n qsample: int\n tests: List[str]\n sample: Optional[int]\n\ndef parse_config_file(path:str) -> Config:\n name = None\n input = None\n k = None\n queries = None\n qsample = None\n sample = None\n tests = []\n\n for line in open(path, 'r'):\n line = line.strip()\n if not line or line.startswith('#'):\n continue\n elif line.startswith('@name='):\n name = line.removeprefix('@name=')\n elif line.startswith('@k='):\n k = line.removeprefix('@k=')\n elif line.startswith('@input='):\n input = line.removeprefix('@input=')\n elif line.startswith('@queries='):\n queries = line.removeprefix('@queries=')\n elif line.startswith('@qsample='):\n qsample = line.removeprefix('@qsample=')\n elif line.startswith('@sample='):\n sample = line.removeprefix('@sample=')\n elif line.startswith('@'):\n raise Exception(f\"unknown config line '{line}'\")\n else:\n tests.append(line)\n return Config(name, input, int(k), queries, int(qsample), tests, int(sample) if sample is not None else None)\n\n\n\nfrom sys import argv\nif __name__ == \"__main__\":\n config = parse_config_file(argv[1])\n print(config)\n #args = parse_args()\n #print(args)\n base = cli_read_input(config.input, config.sample)\n queries = cli_read_query(config.queries, base)[: config.qsample]\n suite(config.name, base, queries, config.k, config.tests)\n\n #main()\n" }, { "path": "benchmarks/exhaustive-memory/gist.suite", "content": "@name=gist\n@input=data/gist/gist_base.fvecs\n@queries=data/gist/gist_query.fvecs\n@sample=500000\n@qsample=20\n@k=20\n\nfaiss\nusearch\nvec-static\n#duckdb\n#vec-vec0.8192.1024\n#vec-vec0.8192.2048\n#vec-scalar.8192\n#numpy\n" }, { "path": "benchmarks/exhaustive-memory/requirements.txt", "content": "annotated-types==0.7.0\nanyio==4.4.0\nasgiref==3.8.1\nattrs==23.2.0\nbackoff==2.2.1\nbcrypt==4.2.0\nbuild==1.2.1\ncachetools==5.4.0\ncertifi==2024.7.4\ncharset-normalizer==3.3.2\nchroma-hnswlib==0.7.6\nchromadb==0.5.5\nclick==8.1.7\ncoloredlogs==15.0.1\ndecorator==5.1.1\ndeprecated==1.2.14\ndeprecation==2.1.0\ndnspython==2.6.1\nduckdb==1.0.0\nemail-validator==2.2.0\nfaiss-cpu==1.8.0.post1\nfastapi==0.111.1\nfastapi-cli==0.0.4\nfilelock==3.15.4\nflatbuffers==24.3.25\nfsspec==2024.6.1\ngoogle-auth==2.32.0\ngoogleapis-common-protos==1.63.2\ngrpcio==1.65.1\nh11==0.14.0\nhnswlib==0.8.0\nhttpcore==1.0.5\nhttptools==0.6.1\nhttpx==0.27.0\nhuggingface-hub==0.24.1\nhumanfriendly==10.0\nidna==3.7\nimportlib-metadata==8.0.0\nimportlib-resources==6.4.0\njinja2==3.1.4\njoblib==1.4.2\nkubernetes==30.1.0\nlancedb==0.10.2\nmarkdown-it-py==3.0.0\nmarkupsafe==2.1.5\nmdurl==0.1.2\nmmh3==4.1.0\nmonotonic==1.6\nmpmath==1.3.0\nnetworkx==3.3\nnumpy==1.26.4\noauthlib==3.2.2\nonnxruntime==1.18.1\nopentelemetry-api==1.26.0\nopentelemetry-exporter-otlp-proto-common==1.26.0\nopentelemetry-exporter-otlp-proto-grpc==1.26.0\nopentelemetry-instrumentation==0.47b0\nopentelemetry-instrumentation-asgi==0.47b0\nopentelemetry-instrumentation-fastapi==0.47b0\nopentelemetry-proto==1.26.0\nopentelemetry-sdk==1.26.0\nopentelemetry-semantic-conventions==0.47b0\nopentelemetry-util-http==0.47b0\norjson==3.10.6\noverrides==7.7.0\npackaging==24.1\npandas==2.2.2\npillow==10.4.0\nposthog==3.5.0\nprotobuf==4.25.4\npy==1.11.0\npyarrow==15.0.0\npyasn1==0.6.0\npyasn1-modules==0.4.0\npydantic==2.8.2\npydantic-core==2.20.1\npygments==2.18.0\npylance==0.14.1\npypika==0.48.9\npyproject-hooks==1.1.0\npython-dateutil==2.9.0.post0\npython-dotenv==1.0.1\npython-multipart==0.0.9\npytz==2024.1\npyyaml==6.0.1\nratelimiter==1.2.0.post0\nregex==2024.5.15\nrequests==2.32.3\nrequests-oauthlib==2.0.0\nretry==0.9.2\nrich==13.7.1\nrsa==4.9\nsafetensors==0.4.3\nscikit-learn==1.5.1\nscipy==1.14.0\nsentence-transformers==3.0.1\nsetuptools==71.1.0\nshellingham==1.5.4\nsix==1.16.0\nsniffio==1.3.1\nstarlette==0.37.2\nsympy==1.13.1\ntenacity==8.5.0\nthreadpoolctl==3.5.0\ntokenizers==0.19.1\ntorch==2.3.1\ntqdm==4.66.4\ntransformers==4.43.1\ntyper==0.12.3\ntyping-extensions==4.12.2\ntzdata==2024.1\nurllib3==2.2.2\nusearch==2.12.0\nuvicorn==0.30.3\nuvloop==0.19.0\nwatchfiles==0.22.0\nwebsocket-client==1.8.0\nwebsockets==12.0\nwrapt==1.16.0\nzipp==3.19.2\n" }, { "path": "benchmarks/exhaustive-memory/sift.suite", "content": "@name=sift1m\n@input=data/sift/sift_base.fvecs\n@queries=data/sift/sift_query.fvecs\n@qsample=100\n@k=20\n\nfaiss\nusearch\nduckdb\nvec-static\nvec-vec0.8192.1024\nvec-vec0.8192.2048\nvec-scalar.8192\nnumpy\n\n# #libsql.4096\n# #libsql.8192\n# faiss\n# vec-scalar.4096\n# vec-static\n# vec-vec0.4096.16\n# vec-vec0.8192.1024\n# vec-vec0.4096.2048\n# usearch\n# duckdb\n# hnswlib\n# numpy\n# chroma\n" }, { "path": "benchmarks/micro/.gitignore", "content": "target/\n" }, { "path": "benchmarks/micro/Cargo.toml", "content": "[package]\nname = \"micro\"\nversion = \"0.1.0\"\nedition = \"2021\"\n\n[dependencies]\nrusqlite = {version=\"0.31.0\", features=[\"bundled\"]}\n\n[dev-dependencies]\ncriterion = \"0.3\"\nrand = \"0.8.5\"\nzerocopy = \"0.7.34\"\n\n[build-dependencies]\ncc = \"1.0.99\"\n\n[[bench]]\nname = \"my_benchmark\"\nharness = false\n" }, { "path": "benchmarks/micro/benches/my_benchmark.rs", "content": "use criterion::{black_box, criterion_group, criterion_main, Criterion};\nuse micro::init_vec;\nuse rand::Rng;\nuse rusqlite::Connection;\nuse zerocopy::AsBytes;\n\nfn random_vector(n: usize) -> Vec {\n let mut rng = rand::thread_rng();\n (0..n).map(|_| rng.gen()).collect()\n}\n\nfn setup_base(page_size: usize, d: usize, n: i32) -> Connection {\n let base: Vec> = (0..n).map(|_| random_vector(d)).collect();\n\n let mut db = Connection::open_in_memory().unwrap();\n db.pragma_update(\n Some(rusqlite::DatabaseName::Main),\n \"page_size\",\n page_size, //,\n //|row| Ok(assert!(row.get::(0).unwrap() == page_size)),\n )\n .unwrap();\n assert_eq!(\n db.pragma_query_value(Some(rusqlite::DatabaseName::Main), \"page_size\", |v| {\n Ok(v.get::(0).unwrap())\n })\n .unwrap(),\n page_size,\n );\n db.execute(\n format!(\"create virtual table vec_base using vec0(a float[{d}])\").as_str(),\n [],\n )\n .unwrap();\n\n let tx = db.transaction().unwrap();\n for item in &base {\n tx.execute(\"insert into vec_base(a) values (?)\", [item.as_bytes()])\n .unwrap();\n }\n tx.commit().unwrap();\n db\n}\npub fn criterion_benchmark(c: &mut Criterion) {\n init_vec();\n\n let n = 1_000_000;\n let d = 1536;\n let k = 10;\n let page_size = 8192;\n\n let page_sizes = [4096, 8192, 16384, 32768];\n for page_size in page_sizes {\n let db = setup_base(page_size, d, n);\n\n let mut stmt = db\n .prepare(\"select rowid, a from vec_base where rowid = ?\")\n .unwrap();\n\n c.bench_function(\n format!(\"point page_size={page_size} n={n} dimension={d} k={k}\").as_str(),\n |b| {\n let mut rng = rand::thread_rng();\n let query: i64 = rng.gen_range(0..n.into());\n\n b.iter(|| {\n let result: (i64, Vec) = stmt\n .query_row(rusqlite::params![query], |r| {\n Ok((r.get(0).unwrap(), r.get(1).unwrap()))\n })\n .unwrap();\n assert_eq!(result.0, query);\n });\n },\n );\n /*\n c.bench_function(\n format!(\"KNN page_size={page_size} n={n} dimension={d} k={k}\").as_str(),\n |b| {\n let query: Vec = random_vector(d);\n let db = setup_base(page_size, d, n);\n\n let mut stmt = db.prepare(\n \"select rowid, distance from vec_base where a match ? order by distance limit ?\",\n )\n .unwrap();\n\n b.iter(|| {\n let result: Vec<(i64, f64)> = stmt\n .query_map(rusqlite::params![query.as_bytes(), k], |r| {\n Ok((r.get(0).unwrap(), r.get(1).unwrap()))\n })\n .unwrap()\n .collect::, _>>()\n .unwrap();\n assert_eq!(result.len(), 10);\n });\n stmt.finalize().unwrap()\n },\n ); */\n }\n}\n\ncriterion_group!(benches, criterion_benchmark);\ncriterion_main!(benches);\n" }, { "path": "benchmarks/micro/build.rs", "content": "fn main() {\n cc::Build::new()\n .file(\"../../sqlite-vec.c\")\n .compile(\"sqlite_vec0\");\n}\n" }, { "path": "benchmarks/micro/src/lib.rs", "content": "use rusqlite::ffi::sqlite3_auto_extension;\n\n#[link(name = \"sqlite_vec0\")]\nextern \"C\" {\n pub fn sqlite3_vec_init();\n}\n\npub fn init_vec() {\n unsafe {\n sqlite3_auto_extension(Some(std::mem::transmute(sqlite3_vec_init as *const ())));\n }\n}\n" }, { "path": "benchmarks/profiling/build-from-npy.sql", "content": ".timer on\npragma page_size = 32768;\n--pragma page_size = 16384;\n--pragma page_size = 16384;\n--pragma page_size = 4096;\n\ncreate virtual table vec_items using vec0(\n embedding float[1536]\n);\n\n-- 65s (limit 1e5), ~615MB on disk\ninsert into vec_items\n select\n rowid,\n vector\n from vec_npy_each(vec_npy_file('examples/dbpedia-openai/data/vectors.npy'))\n limit 1e5;\n" }, { "path": "benchmarks/profiling/query-k.sql", "content": ".timer on\n\nselect rowid, distance\nfrom vec_items\nwhere embedding match (select embedding from vec_items where rowid = 100)\n and k = :k\norder by distance;\n\nselect rowid, distance\nfrom vec_items\nwhere embedding match (select embedding from vec_items where rowid = 100)\n and k = :k\norder by distance;\n\nselect rowid, distance\nfrom vec_items\nwhere embedding match (select embedding from vec_items where rowid = 100)\n and k = :k\norder by distance;\n\nselect rowid, distance\nfrom vec_items\nwhere embedding match (select embedding from vec_items where rowid = 100)\n and k = :k\norder by distance;\n\nselect rowid, distance\nfrom vec_items\nwhere embedding match (select embedding from vec_items where rowid = 100)\n and k = :k\norder by distance;\n" }, { "path": "benchmarks/self-params/build.py", "content": "import sqlite3\nimport time\n\n\ndef connect(path):\n db = sqlite3.connect(path)\n db.enable_load_extension(True)\n db.load_extension(\"../dist/vec0\")\n db.execute(\"select load_extension('../dist/vec0', 'sqlite3_vec_fs_read_init')\")\n db.enable_load_extension(False)\n return db\n\n\npage_sizes = [ # 4096, 8192,\n 16384,\n 32768,\n]\nchunk_sizes = [128, 256, 1024, 2048]\ntypes = [\"f32\", \"int8\", \"bit\"]\n\nSRC = \"../examples/dbpedia-openai/data/vectors.npy\"\n\nfor page_size in page_sizes:\n for chunk_size in chunk_sizes:\n for t in types:\n print(f\"{t} page_size={page_size}, chunk_size={chunk_size}\")\n\n t0 = time.time()\n db = connect(f\"dbs/test.{page_size}.{chunk_size}.{t}.db\")\n db.execute(f\"pragma page_size = {page_size}\")\n with db:\n db.execute(\n f\"\"\"\n create virtual table vec_items using vec0(\n embedding {t}[1536],\n chunk_size={chunk_size}\n )\n \"\"\"\n )\n func = \"vector\"\n if t == \"int8\":\n func = \"vec_quantize_i8(vector, 'unit')\"\n if t == \"bit\":\n func = \"vec_quantize_binary(vector)\"\n db.execute(\n f\"\"\"\n insert into vec_items\n select rowid, {func}\n from vec_npy_each(vec_npy_file(?))\n limit 100000\n \"\"\",\n [SRC],\n )\n elapsed = time.time() - t0\n print(elapsed)\n\n\"\"\"\n\n# for 100_000\n\npage_size=4096, chunk_size=256\n3.5894200801849365\npage_size=4096, chunk_size=1024\n60.70046401023865\npage_size=4096, chunk_size=2048\n201.04426288604736\npage_size=8192, chunk_size=256\n7.034514904022217\npage_size=8192, chunk_size=1024\n9.983598947525024\npage_size=8192, chunk_size=2048\n12.318921089172363\npage_size=16384, chunk_size=256\n4.97080397605896\npage_size=16384, chunk_size=1024\n6.051195859909058\npage_size=16384, chunk_size=2048\n8.492683172225952\npage_size=32768, chunk_size=256\n5.906642198562622\npage_size=32768, chunk_size=1024\n5.876632213592529\npage_size=32768, chunk_size=2048\n5.420510292053223\n\"\"\"\n" }, { "path": "benchmarks/self-params/knn.py", "content": "import sqlite3\nimport time\nfrom random import randrange\nfrom statistics import mean\n\n\ndef connect(path):\n print(path)\n db = sqlite3.connect(path)\n db.enable_load_extension(True)\n db.load_extension(\"../dist/vec0\")\n db.execute(\"select load_extension('../dist/vec0', 'sqlite3_vec_fs_read_init')\")\n db.enable_load_extension(False)\n return db\n\n\npage_sizes = [ # 4096, 8192,\n 16384,\n 32768,\n]\nchunk_sizes = [128, 256, 1024, 2048]\ntypes = [\"f32\", \"int8\", \"bit\"]\n\ntypes.reverse()\n\nfor t in types:\n for page_size in page_sizes:\n for chunk_size in chunk_sizes:\n print(f\"page_size={page_size}, chunk_size={chunk_size}\")\n\n func = \"embedding\"\n if t == \"int8\":\n func = \"vec_quantize_i8(embedding, 'unit')\"\n if t == \"bit\":\n func = \"vec_quantize_binary(embedding)\"\n\n times = []\n trials = 20\n db = connect(f\"dbs/test.{page_size}.{chunk_size}.{t}.db\")\n\n for trial in range(trials):\n t0 = time.time()\n results = db.execute(\n f\"\"\"\n select rowid\n from vec_items\n where embedding match (select {func} from vec_items where rowid = ?)\n and k = 10\n order by distance\n \"\"\",\n [randrange(100000)],\n ).fetchall()\n\n times.append(time.time() - t0)\n print(mean(times))\n\n\"\"\"\n\npage_size=4096, chunk_size=256\n0.2635102152824402\npage_size=4096, chunk_size=1024\n0.2609449863433838\npage_size=4096, chunk_size=2048\n0.275589919090271\npage_size=8192, chunk_size=256\n0.18621582984924318\npage_size=8192, chunk_size=1024\n0.20939643383026124\npage_size=8192, chunk_size=2048\n0.22376316785812378\npage_size=16384, chunk_size=256\n0.16012665033340454\npage_size=16384, chunk_size=1024\n0.18346318006515502\npage_size=16384, chunk_size=2048\n0.18224761486053467\npage_size=32768, chunk_size=256\n0.14202518463134767\npage_size=32768, chunk_size=1024\n0.15340715646743774\npage_size=32768, chunk_size=2048\n0.18018823862075806\n\"\"\"\n" }, { "path": "benchmarks/self-params/test.py", "content": "import sqlite3\nimport time\n\n\ndef connect(path):\n db = sqlite3.connect(path)\n db.enable_load_extension(True)\n db.load_extension(\"../dist/vec0\")\n db.execute(\"select load_extension('../dist/vec0', 'sqlite3_vec_fs_read_init')\")\n db.enable_load_extension(False)\n return db\n\n\npage_sizes = [4096, 8192, 16384, 32768]\nchunk_sizes = [256, 1024, 2048]\n\nfor page_size in page_sizes:\n for chunk_size in chunk_sizes:\n print(f\"page_size={page_size}, chunk_size={chunk_size}\")\n\n t0 = time.time()\n db = connect(f\"dbs/test.{page_size}.{chunk_size}.db\")\n print(db.execute(\"pragma page_size\").fetchone()[0])\n print(db.execute(\"select count(*) from vec_items_rowids\").fetchone()[0])\n" }, { "path": "bindings/go/ncruces/go-sqlite3.patch", "content": "diff --git a/embed/build.sh b/embed/build.sh\nindex ed2aaec..4cc0b0e 100755\n--- a/embed/build.sh\n+++ b/embed/build.sh\n@@ -23,6 +23,7 @@ trap 'rm -f sqlite3.tmp' EXIT\n \t-Wl,--initial-memory=327680 \\\n \t-D_HAVE_SQLITE_CONFIG_H \\\n \t-DSQLITE_CUSTOM_INCLUDE=sqlite_opt.h \\\n+ -DSQLITE_VEC_OMIT_FS=1 \\\n \t$(awk '{print \"-Wl,--export=\"$0}' exports.txt)\n\n \"$BINARYEN/wasm-ctor-eval\" -g -c _initialize sqlite3.wasm -o sqlite3.tmp\ndiff --git a/sqlite3/main.c b/sqlite3/main.c\nindex c732937..7c9002a 100644\n--- a/sqlite3/main.c\n+++ b/sqlite3/main.c\n@@ -19,6 +19,7 @@\n #include \"time.c\"\n #include \"vfs.c\"\n #include \"vtab.c\"\n+#include \"../../sqlite-vec.c\"\n\n __attribute__((constructor)) void init() {\n sqlite3_initialize();\n@@ -30,4 +31,5 @@ __attribute__((constructor)) void init() {\n sqlite3_auto_extension((void (*)(void))sqlite3_spellfix_init);\n sqlite3_auto_extension((void (*)(void))sqlite3_uint_init);\n sqlite3_auto_extension((void (*)(void))sqlite3_time_init);\n+ sqlite3_auto_extension((void (*)(void))sqlite3_vec_init);\n }\n\\ No newline at end of file\n" }, { "path": "bindings/python/extra_init.py", "content": "from typing import List\nfrom struct import pack\nfrom sqlite3 import Connection\n\n\ndef serialize_float32(vector: List[float]) -> bytes:\n \"\"\"Serializes a list of floats into the \"raw bytes\" format sqlite-vec expects\"\"\"\n return pack(\"%sf\" % len(vector), *vector)\n\n\ndef serialize_int8(vector: List[int]) -> bytes:\n \"\"\"Serializes a list of integers into the \"raw bytes\" format sqlite-vec expects\"\"\"\n return pack(\"%sb\" % len(vector), *vector)\n\n\ntry:\n import numpy.typing as npt\n\n def register_numpy(db: Connection, name: str, array: npt.NDArray):\n \"\"\"ayoo\"\"\"\n\n ptr = array.__array_interface__[\"data\"][0]\n nvectors, dimensions = array.__array_interface__[\"shape\"]\n element_type = array.__array_interface__[\"typestr\"]\n\n assert element_type == \"\"]\ndescription = \"FFI bindings to the sqlite-vec SQLite extension\"\nhomepage = \"https://github.com/asg017/sqlite-vec\"\nrepository = \"https://github.com/asg017/sqlite-vec\"\nkeywords = [\"sqlite\", \"sqlite-extension\"]\nlicense = \"MIT/Apache-2.0\"\n\n\n[dependencies]\n\n[build-dependencies]\ncc = \"1.0\"\n\n[dev-dependencies]\nrusqlite = \"0.31.0\"\n" }, { "path": "bindings/rust/Makefile", "content": "VERSION=$(shell cat ../../VERSION)\n\ndeps: Cargo.toml sqlite-vec.c sqlite-vec.h sqlite3ext.h sqlite3.h\n\nCargo.toml: ../../VERSION Cargo.toml.tmpl\n\t\tVERSION=$(VERSION) envsubst < Cargo.toml.tmpl > $@\n\nsqlite-vec.c: ../../sqlite-vec.c\n\t\tcp $< $@\n\nsqlite-vec.h: ../../sqlite-vec.h\n\t\tcp $< $@\n\nsqlite3ext.h: ../../vendor/sqlite3ext.h\n\t\tcp $< $@\n\nsqlite3.h: ../../vendor/sqlite3.h\n\t\tcp $< $@\n\n.PHONY: deps\n" }, { "path": "bindings/rust/build.rs", "content": "fn main() {\n cc::Build::new().file(\"sqlite-vec.c\").define(\"SQLITE_CORE\", None).compile(\"sqlite_vec0\");\n}\n" }, { "path": "bindings/rust/src/lib.rs", "content": "#[link(name = \"sqlite_vec0\")]\nextern \"C\" {\n pub fn sqlite3_vec_init();\n}\n\n#[cfg(test)]\nmod tests {\n use super::*;\n\n use rusqlite::{ffi::sqlite3_auto_extension, Connection};\n\n #[test]\n fn test_rusqlite_auto_extension() {\n unsafe {\n sqlite3_auto_extension(Some(std::mem::transmute(sqlite3_vec_init as *const ())));\n }\n\n let conn = Connection::open_in_memory().unwrap();\n\n let result: String = conn\n .query_row(\"select vec_version()\", [], |x| x.get(0))\n .unwrap();\n\n assert!(result.starts_with(\"v\"));\n }\n}\n" }, { "path": "examples/nbc-headlines/.gitignore", "content": "*.dylib\n*.so\n*.dll\n*.gguf\n" }, { "path": "examples/nbc-headlines/1_scrape.ipynb", "content": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"# NBC News Headlines: Scraper\\n\",\n \"\\n\",\n \"This notebooks implements a scraper for [NBC News](https://www.nbcnews.com) headlines. It uses [this sitemap](https://www.nbcnews.com/archive/articles/2024/march), which provides a list of article headlines + URLs\\n\",\n \"for every month for the past few years. \\n\",\n \"\\n\",\n \"This dataset is mostly to get a simple, real-world small text dataset for testing embeddings. \\n\",\n \"They're small pieces of text (~dozen words), have a wide range of semantic meaning, and are more \\\"real-world\\\"\\n\",\n \"them some other embeddings datasets out there.\\n\",\n \"\\n\",\n \"This notebook uses [Deno](https://deno.com/), [linkedom](https://github.com/WebReflection/linkedom), and a few \\n\",\n \"SQLite extensions to scrape the headlines for a given date range. It creates a single SQL table, `articles`, \\n\",\n \"with a few columns like `headline` and `url`. By default it will get all article headlines from January 2024 -> present\\n\",\n \"and save them to a database called `headlines-2024.db`. Feel free to copy+paste this code into your own custom scraper. \\n\",\n \"\\n\",\n \"This notebook also just scrapes the data into a SQLite database, it does NOT do any embeddings + vector search. \\n\",\n \"For those examples of those, see [`./2_build.ipynb`](./2_build.ipynb) and [`./3_search.ipynb`](./3_search.ipynb).\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 43,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"import { Database, Statement } from \\\"jsr:@db/sqlite@0.11\\\";\\n\",\n \"import { parseHTML } from \\\"npm:linkedom\\\";\\n\",\n \"import * as d3 from \\\"npm:d3-time\\\";\\n\",\n \"import * as sqlitePath from \\\"npm:sqlite-path\\\";\\n\",\n \"import * as sqliteUrl from \\\"npm:sqlite-url\\\";\\n\",\n \"import * as sqliteRegex from \\\"npm:sqlite-regex\\\";\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 47,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"const months = [\\\"january\\\", \\\"february\\\", \\\"march\\\", \\\"april\\\", \\\"may\\\", \\\"june\\\", \\\"july\\\", \\\"august\\\", \\\"september\\\", \\\"october\\\", \\\"november\\\", \\\"december\\\"]\\n\",\n \"\\n\",\n \"class Db {\\n\",\n \" db: Database;\\n\",\n \" #stmtInsertArticle: Statement;\\n\",\n \"\\n\",\n \" constructor(path:string) {\\n\",\n \" this.db = new Database(path);\\n\",\n \" this.db.enableLoadExtension = true;\\n\",\n \" this.db.loadExtension(sqlitePath.getLoadablePath());\\n\",\n \" this.db.loadExtension(sqliteUrl.getLoadablePath());\\n\",\n \" this.db.loadExtension(sqliteRegex.getLoadablePath());\\n\",\n \" this.db.enableLoadExtension = false;\\n\",\n \"\\n\",\n \" this.db.exec(`\\n\",\n \" CREATE TABLE IF NOT EXISTS articles(\\n\",\n \" id integer primary key autoincrement,\\n\",\n \" year integer,\\n\",\n \" month integer,\\n\",\n \" slug TEXT,\\n\",\n \" slug_id TEXT,\\n\",\n \" headline TEXT,\\n\",\n \" url TEXT,\\n\",\n \" category1 TEXT,\\n\",\n \" category2 TEXT\\n\",\n \" )\\n\",\n \" `);\\n\",\n \"\\n\",\n \" this.#stmtInsertArticle = this.db.prepare(`\\n\",\n \" insert into articles(year, month, slug, slug_id, headline, url, category1, category2)\\n\",\n \" select\\n\",\n \" :year as year,\\n\",\n \" :month as month,\\n\",\n \" regex_capture(\\n\",\n \" '(?P.+)-(?P[^-]+)$',\\n\",\n \" path_at(url_path(:url), -1),\\n\",\n \" 'slug'\\n\",\n \" ) as slug,\\n\",\n \" regex_capture(\\n\",\n \" '(?P.+)-(?P[^-]+)$',\\n\",\n \" path_at(url_path(:url), -1),\\n\",\n \" 'id'\\n\",\n \" ) as slug_id,\\n\",\n \" :headline as headline,\\n\",\n \" :url as url,\\n\",\n \" path_at(url_path(:url), 0) as category1,\\n\",\n \" iif(\\n\",\n \" path_length(url_path(:url)) > 2,\\n\",\n \" path_at(url_path(:url), 1),\\n\",\n \" null\\n\",\n \" ) as category2\\n\",\n \" `);\\n\",\n \" }\\n\",\n \"\\n\",\n \" insertArticles(year:number, month:text, articles:{url: string, year: number, month: number}[]) {\\n\",\n \" const tx = this.db.transaction((year, month, articles) => {\\n\",\n \" for(const article of articles) {\\n\",\n \" this.#stmtInsertArticle.run({...article, year, month})\\n\",\n \" }\\n\",\n \" });\\n\",\n \" tx(year, month, articles);\\n\",\n \" }\\n\",\n \"}\\n\",\n \"\\n\",\n \"async function insertMonth(db: Db, year:number, month: text) {\\n\",\n \" let url = `https://www.nbcnews.com/archive/articles/${year}/${month}`;\\n\",\n \" while(true) {\\n\",\n \" const monthPage = await fetch(url).then(r=>r.text())\\n\",\n \" const {document:monthPageDoc} = parseHTML(monthPage);\\n\",\n \" const monthEntries = monthPageDoc\\n\",\n \" .querySelectorAll('.MonthPage a')\\n\",\n \" .map(a => ({headline: a.innerText, url: a.getAttribute('href')}));\\n\",\n \" db.insertArticles(year, months.findIndex(m => m === month)+1, monthEntries);\\n\",\n \" const next = monthPageDoc.querySelector('a.Pagination__next.Pagination__enable');\\n\",\n \" if(!next) {\\n\",\n \" break;\\n\",\n \" }\\n\",\n \" url = `https://www.nbcnews.com${next.getAttribute('href')}`;\\n\",\n \" }\\n\",\n \"\\n\",\n \"}\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 48,\n \"metadata\": {},\n \"outputs\": [],\n \"source\": [\n \"\\n\",\n \"async function backfill(db, start: Date, end: Date) {\\n\",\n \" const targets = d3.timeMonths(start, end)\\n\",\n \" .map(date => ({year: date.getFullYear(), monthIndex: date.getMonth()}));\\n\",\n \" for(const target of targets) {\\n\",\n \" console.log(`${target.year} ${target.monthIndex}`)\\n\",\n \" await insertMonth(db, target.year, months[target.monthIndex]);\\n\",\n \" }\\n\",\n \"}\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 49,\n \"metadata\": {},\n \"outputs\": [\n {\n \"name\": \"stdout\",\n \"output_type\": \"stream\",\n \"text\": [\n \"2024 0\\n\",\n \"2024 1\\n\",\n \"2024 2\\n\",\n \"2024 3\\n\",\n \"2024 4\\n\",\n \"2024 5\\n\",\n \"2024 6\\n\",\n \"2024 7\\n\"\n ]\n },\n {\n \"data\": {\n \"text/plain\": [\n \"\\u001b[33m1\\u001b[39m\"\n ]\n },\n \"execution_count\": 49,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"source\": [\n \"const db = new Db(\\\":memory:\\\");\\n\",\n \"await backfill(db, new Date('2024-01-01'), new Date())\\n\",\n \"db.db.exec(\\\"vacuum into 'headlines-2024.db'\\\")\"\n ]\n }\n ],\n \"metadata\": {\n \"kernelspec\": {\n \"display_name\": \"Deno\",\n \"language\": \"typescript\",\n \"name\": \"deno\"\n },\n \"language_info\": {\n \"codemirror_mode\": \"typescript\",\n \"file_extension\": \".ts\",\n \"mimetype\": \"text/x.typescript\",\n \"name\": \"typescript\",\n \"nbconvert_exporter\": \"script\",\n \"pygments_lexer\": \"typescript\",\n \"version\": \"5.5.2\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 2\n}\n" }, { "path": "examples/nbc-headlines/2_build.ipynb", "content": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"# NBC News Headlines: Building FTS5 + `vec0` indexes\\n\",\n \"\\n\",\n \"Using the dataset built in [the previous `./1_scrape.ipynb` notebook](./1_scrape.ipynb), \\n\",\n \"this notebook will enrich that dataset with a full-text search index and a semantic search index,\\n\",\n \"using [FTS5](https://www.sqlite.org/fts5.html), \\n\",\n \"[`sqlite-vec`](https://github.com/asg017/sqlite-vec), and \\n\",\n \"[`sqlite-lembed`](https://github.com/asg017/sqlite-lembed).\\n\",\n \"\\n\",\n \"This example will use pure SQL for everything. You can do the same exact thing in Python/JavaScript/Go/Rust/etc., or use\\n\",\n \"your own embeddings providers like Ollama/llamafile/OpenAI/etc. The core mechanics of FTS5 and `sqlite-vec` will remain the same. \\n\",\n \"\\n\",\n \"We will use the [Snowflake Artic Embed v1.5](https://huggingface.co/Snowflake/snowflake-arctic-embed-m-v1.5) embeddings model to generate embeddings. \"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 1,\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"text/plain\": [\n \"[no code]\"\n ]\n },\n \"execution_count\": 1,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"source\": [\n \".open tmp-artic2.db\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"# Step 1: Create a FTS5 index\\n\",\n \"\\n\",\n \"Creating a full-text search index is as simple as 3 SQL commands! We already have the headlines stored in the `articles` \\n\",\n \"table under the `headline` column, so it's just a matter of initializing the FTS5 virtual table and inserting the data.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 3,\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"text/html\": [\n \"
\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"
\\n\",\n \"
\\n\",\n \"0 row × 0 column\\n\",\n \"
\\n\",\n \"
\\n\"\n ],\n \"text/plain\": [\n \"\\u001b[0m┌\\u001b[0m\\u001b[0m\\u001b[0m├\\u001b[0m\\u001b[0m\"\n ]\n },\n \"execution_count\": 3,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"source\": [\n \"create virtual table fts_articles using fts5(\\n\",\n \" headline,\\n\",\n \" content='articles', content_rowid='id'\\n\",\n \");\\n\",\n \"\\n\",\n \"insert into fts_articles(rowid, headline)\\n\",\n \" select rowid, headline\\n\",\n \" from articles;\\n\",\n \"\\n\",\n \"insert into fts_articles(fts_articles) values('optimize');\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"By convention we name the FTS5 table `fts_articles`, where the `fts_` prefix says \\\"this virtual table is full-text search of the `articles` table\\\". We are only searching the `headline` column, the rest can be ignored. \\n\",\n \"\\n\",\n \"Here we are using the [\\\"external content tables\\\"](https://www.sqlite.org/fts5.html#external_content_tables)\\n\",\n \"feature in FTS5 tables, which will avoid storing the headlines a 2nd time, since they already exist in the `articles` table. \\n\",\n \"This part isn't required, but saves us a bit of storage. \\n\",\n \"\\n\",\n \"We also use the [`'optimize'`](https://www.sqlite.org/fts5.html#the_optimize_command) command\\n\",\n \" to keep things tidy. This doesn't do much on such a small dataset, but is important to remember for larger tables!\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 25,\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"text/html\": [\n \"
\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"
\\n\",\n \"headline\\n\",\n \"
\\n\",\n \"Kamala Harris visits Planned Parenthood clinic\\n\",\n \"
\\n\",\n \"Former Marine sentenced to 9 years in prison for firebombing Planned Parenthood clinic\\n\",\n \"
\\n\",\n \"
\\n\",\n \"2 rows × 1 column\\n\",\n \"
\\n\",\n \"
\\n\"\n ],\n \"text/plain\": [\n \"\\u001b[0m┌\\u001b[0m\\u001b[0m────────────────────────────────────────────────────────────────────────────────────────\\u001b[0m\\u001b[0m┐\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m\\u001b[0m\\u001b[1mheadline\\u001b[0m \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m├\\u001b[0m\\u001b[0m────────────────────────────────────────────────────────────────────────────────────────\\u001b[0m\\u001b[0m┤\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mKamala Harris visits Planned Parenthood clinic \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mFormer Marine sentenced to 9 years in prison for firebombing Planned Parenthood clinic\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m└\\u001b[0m\\u001b[0m────────────────────────────────────────────────────────────────────────────────────────\\u001b[0m\\u001b[0m┘\\n\",\n \"\\u001b[0m\\u001b[0m\"\n ]\n },\n \"execution_count\": 25,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"source\": [\n \"select *\\n\",\n \"from fts_articles\\n\",\n \"where headline match 'planned parenthood'\\n\",\n \"limit 10;\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## Step 2: Create a \\\"semantic index\\\"\\n\",\n \"\\n\",\n \"\\\"Semantic index\\\" in this case is just a fancy way of saying \\\"vector store\\\", which we will do with a `sqlite-vec` `vec0` virtual table. \\n\",\n \"\\n\",\n \"Now, `sqlite-vec` just stores vectors, it doesn't generate embeddings for us. There are hundreds of different remote APIs or local inference runtimes you can use to generate embeddings,\\n\",\n \"but here we will use [`sqlite-lembed`](https://github.com/asg017/sqlite-lembed) to keep everything local and everything in pure SQL. \\n\",\n \"\\n\",\n \"We will need to choose an embeddings model in the [GGUF format](https://huggingface.co/docs/hub/en/gguf),\\n\",\n \"since `sqlite-lembed` uses [llama.cpp](https://github.com/ggerganov/llama.cpp) under the hood. \\n\",\n \"Here we will use [`Snowflake/snowflake-arctic-embed-m-v1.5`](https://huggingface.co/Snowflake/snowflake-arctic-embed-m-v1.5),\\n\",\n \"where we can find a GGUF version [here](https://huggingface.co/asg017/sqlite-lembed-model-examples/tree/main/snowflake-arctic-embed-m-v1.5). \\n\",\n \"This model is small-sh (`436MB` full-sized, `118MB` at `Q8_0` quantized), and is trained on fairly recent data so it understands\\n\",\n \"recent events like \\\"COVID-19\\\" or \\\"Kamala Harris\\\". \\n\",\n \"\\n\",\n \"You can download a `.gguf` quantized version of this model with:\\n\",\n \"\\n\",\n \"```bash\\n\",\n \"wget https://huggingface.co/asg017/sqlite-lembed-model-examples/resolve/main/snowflake-arctic-embed-m-v1.5/snowflake-arctic-embed-m-v1.5.d70deb40.f16.gguf\\n\",\n \"```\\n\",\n \"\\n\",\n \"And we can configure `sqlite-lembed` to use this model like so:\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 6,\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"text/html\": [\n \"
\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"
\\n\",\n \"
\\n\",\n \"0 row × 0 column\\n\",\n \"
\\n\",\n \"
\\n\"\n ],\n \"text/plain\": [\n \"\\u001b[0m┌\\u001b[0m\\u001b[0m\\u001b[0m├\\u001b[0m\\u001b[0m\"\n ]\n },\n \"execution_count\": 6,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"source\": [\n \".load ./lembed0\\n\",\n \".load ../../dist/vec0\\n\",\n \"\\n\",\n \"insert into lembed_models(name, model) values\\n\",\n \" ('default', lembed_model_from_file('./snowflake-arctic-embed-m-v1.5.d70deb40.f16.gguf'));\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"It's embeddings time! We can use the `lembed()` function, which takes in text and returns a vector representation of that text,\\n\",\n \"as an embeddings BLOB that we can insert directly into a `vec0` virtul table. \\n\",\n \"\\n\",\n \"We'll declare this new `vec_articles` table, using the `vec_` prefix as convention. This matches the `fts_articles` table above. \\n\",\n \"The Snowflake embedding model generate vectors with `768` dimensions, which we we store as-as. \\n\",\n \"\\n\",\n \"Embedding and inserting into this vector store is as easy as a single `INSERT INTO` and `lembed()` call.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 9,\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"text/html\": [\n \"
\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"
\\n\",\n \"
\\n\",\n \"0 row × 0 column\\n\",\n \"
\\n\",\n \"
\\n\"\n ],\n \"text/plain\": [\n \"\\u001b[0m┌\\u001b[0m\\u001b[0m\\u001b[0m├\\u001b[0m\\u001b[0m\"\n ]\n },\n \"execution_count\": 9,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"source\": [\n \"\\n\",\n \"create virtual table vec_articles using vec0(\\n\",\n \" article_id integer primary key,\\n\",\n \" headline_embedding float[768]\\n\",\n \");\\n\",\n \"\\n\",\n \"insert into vec_articles(article_id, headline_embedding)\\n\",\n \"select\\n\",\n \" rowid,\\n\",\n \" lembed(headline)\\n\",\n \"from articles;\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"This took ~13 minutes for ~14,500 embeddings on my older 2019 Macbook, but newer computers with better CPUs will finish quicker (it took `2m20s` on my newer Mac M1 Mini). \\n\",\n \"\\n\",\n \"Once the `vec_articles` is ready, we can perform a KNN query like so:\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 27,\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"text/html\": [\n \"
\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"
\\n\",\n \"headline\\n\",\n \"\\n\",\n \"distance\\n\",\n \"
\\n\",\n \"Kamala Harris visits Planned Parenthood clinic\\n\",\n \"\\n\",\n \"0.492593914270401\\n\",\n \"
\\n\",\n \"After Dobbs decision, more women are managing their own abortions\\n\",\n \"\\n\",\n \"0.5789032578468323\\n\",\n \"
\\n\",\n \"Transforming Healthcare\\n\",\n \"\\n\",\n \"0.5822411179542542\\n\",\n \"
\\n\",\n \"A timeline of Trump's many, many positions on abortion\\n\",\n \"\\n\",\n \"0.6101462841033936\\n\",\n \"
\\n\",\n \"How a network of abortion pill providers works together in the wake of new threats\\n\",\n \"\\n\",\n \"0.6196886897087097\\n\",\n \"
\\n\",\n \"'Major hurdles': The reality check behind Biden's big abortion promise\\n\",\n \"\\n\",\n \"0.6198344826698303\\n\",\n \"
\\n\",\n \"Trump's conflicting abortion stances are coming back to haunt him — and his party\\n\",\n \"\\n\",\n \"0.6198986768722534\\n\",\n \"
\\n\",\n \"Where abortion rights could be on the ballot this fall: From the Politics Desk\\n\",\n \"\\n\",\n \"0.6201764345169067\\n\",\n \"
\\n\",\n \"How the Biden campaign quickly mobilized on Trump's abortion stance\\n\",\n \"\\n\",\n \"0.633980393409729\\n\",\n \"
\\n\",\n \"Battle over abortion heats up in Arizona — and could be on the 2024 ballot\\n\",\n \"\\n\",\n \"0.6341449022293091\\n\",\n \"
\\n\",\n \"
\\n\",\n \"10 rows × 2 columns\\n\",\n \"
\\n\",\n \"
\\n\"\n ],\n \"text/plain\": [\n \"\\u001b[0m┌\\u001b[0m\\u001b[0m────────────────────────────────────────────────────────────────────────────────────\\u001b[0m\\u001b[0m┬\\u001b[0m\\u001b[0m────────────────────\\u001b[0m\\u001b[0m┐\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m\\u001b[0m\\u001b[1mheadline\\u001b[0m \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m\\u001b[0m\\u001b[1mdistance\\u001b[0m \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m├\\u001b[0m\\u001b[0m────────────────────────────────────────────────────────────────────────────────────\\u001b[0m\\u001b[0m┼\\u001b[0m\\u001b[0m────────────────────\\u001b[0m\\u001b[0m┤\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mKamala Harris visits Planned Parenthood clinic \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 0.492593914270401\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mAfter Dobbs decision, more women are managing their own abortions \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m0.5789032578468323\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mTransforming Healthcare \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m0.5822411179542542\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mA timeline of Trump's many, many positions on abortion \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m0.6101462841033936\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mHow a network of abortion pill providers works together in the wake of new threats\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m0.6196886897087097\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m'Major hurdles': The reality check behind Biden's big abortion promise \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m0.6198344826698303\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mTrump's conflicting abortion stances are coming back to haunt him — and his party \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m0.6198986768722534\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mWhere abortion rights could be on the ballot this fall: From the Politics Desk \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m0.6201764345169067\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mHow the Biden campaign quickly mobilized on Trump's abortion stance \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 0.633980393409729\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mBattle over abortion heats up in Arizona — and could be on the 2024 ballot \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m0.6341449022293091\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m└\\u001b[0m\\u001b[0m────────────────────────────────────────────────────────────────────────────────────\\u001b[0m\\u001b[0m┴\\u001b[0m\\u001b[0m────────────────────\\u001b[0m\\u001b[0m┘\\n\",\n \"\\u001b[0m\\u001b[0m\"\n ]\n },\n \"execution_count\": 27,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"source\": [\n \"select\\n\",\n \" articles.headline,\\n\",\n \" vec_articles.distance\\n\",\n \"from vec_articles\\n\",\n \"left join articles on articles.rowid = vec_articles.article_id\\n\",\n \"where headline_embedding match lembed(\\\"planned parenthood\\\")\\n\",\n \" and k = 10;\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## Slim it down with Binary Quantization\\n\",\n \"\\n\",\n \"The vectors in the `vec_articles` table take up a lot of space. A vector with `768` dimensions take up `786 * 4 = 3072` bytes of space each, or around `45MB` of space for these ~14,500 entries. \\n\",\n \"\\n\",\n \"That's a lot — the original text dataset was only `~4MB`!\\n\",\n \"\\n\",\n \"If you want to make the database smaller, there's a number of quantization or other methods to do so, by trading accuracy. \\n\",\n \"Here's an example of performing [binary quantization](https://alexgarcia.xyz/sqlite-vec/guides/binary-quant.html)\\n\",\n \"on this dataset, storing 768-dimensional bit-vectors instead of floating-point vectors, a `32x` size reduction, at the expense of accuracy. \\n\",\n \"\\n\",\n \"We'll keep the current SQLite database as-is, and instead make a copy into a new SQLite database file, and change the `vec_articles` table\\n\",\n \"to store bit-vectors instead. \\n\",\n \"\\n\",\n \"First, we'll make a copy of the current database into a new file:\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 15,\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"text/html\": [\n \"
\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"
\\n\",\n \"
\\n\",\n \"0 row × 0 column\\n\",\n \"
\\n\",\n \"
\\n\"\n ],\n \"text/plain\": [\n \"\\u001b[0m┌\\u001b[0m\\u001b[0m\\u001b[0m├\\u001b[0m\\u001b[0m\"\n ]\n },\n \"execution_count\": 15,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"source\": [\n \"vacuum into 'tmp-artic2.slim.db';\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"Now we'll make a connection to this new file, and drop the old `vec_articles` table that contains the large `float[768]` vectors.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 16,\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"text/html\": [\n \"
\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"
\\n\",\n \"
\\n\",\n \"0 row × 0 column\\n\",\n \"
\\n\",\n \"
\\n\"\n ],\n \"text/plain\": [\n \"\\u001b[0m┌\\u001b[0m\\u001b[0m\\u001b[0m├\\u001b[0m\\u001b[0m\"\n ]\n },\n \"execution_count\": 16,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"source\": [\n \"attach database 'tmp-artic2.slim.db' as slim;\\n\",\n \"drop table slim.vec_articles;\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"Now we can create a new `vec0` table, storing `bit[768]` vectors instead! \\n\",\n \"We can insert the original `float[768]` from the `main.vec_articles` table (original table),\\n\",\n \"calling [`vec_quantize_binary()`](https://alexgarcia.xyz/sqlite-vec/api-reference.html#vec_quantize_binary) to convert the floats to bits. \"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 21,\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"text/html\": [\n \"
\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"
\\n\",\n \"
\\n\",\n \"0 row × 0 column\\n\",\n \"
\\n\",\n \"
\\n\"\n ],\n \"text/plain\": [\n \"\\u001b[0m┌\\u001b[0m\\u001b[0m\\u001b[0m├\\u001b[0m\\u001b[0m\"\n ]\n },\n \"execution_count\": 21,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"source\": [\n \"\\n\",\n \"create virtual table slim.vec_articles using vec0(\\n\",\n \" article_id integer primary key,\\n\",\n \" headline_embedding bit[768]\\n\",\n \");\\n\",\n \"\\n\",\n \"insert into slim.vec_articles(article_id, headline_embedding)\\n\",\n \"select\\n\",\n \" article_id,\\n\",\n \" vec_quantize_binary(headline_embedding)\\n\",\n \"from main.vec_articles;\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"Then we can `VACUUM` the new `slim` database to shrink the file, delete the `DROP`'ed pages from the older `vec0` table. \"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 22,\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"text/html\": [\n \"
\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"
\\n\",\n \"
\\n\",\n \"0 row × 0 column\\n\",\n \"
\\n\",\n \"
\\n\"\n ],\n \"text/plain\": [\n \"\\u001b[0m┌\\u001b[0m\\u001b[0m\\u001b[0m├\\u001b[0m\\u001b[0m\"\n ]\n },\n \"execution_count\": 22,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"source\": [\n \"vacuum slim;\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"And there we have it! This file is `7.1MB`, a large reduction from the original `53MB` table. \\n\",\n \"\\n\",\n \"KNN queries are similar, only adding the `vec_quantize_binary()` function to the query vector.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 28,\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"text/html\": [\n \"
\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"
\\n\",\n \"headline\\n\",\n \"\\n\",\n \"distance\\n\",\n \"
\\n\",\n \"Kamala Harris visits Planned Parenthood clinic\\n\",\n \"\\n\",\n \"139\\n\",\n \"
\\n\",\n \"How a network of abortion pill providers works together in the wake of new threats\\n\",\n \"\\n\",\n \"151\\n\",\n \"
\\n\",\n \"After Dobbs decision, more women are managing their own abortions\\n\",\n \"\\n\",\n \"153\\n\",\n \"
\\n\",\n \"A timeline of Trump's many, many positions on abortion\\n\",\n \"\\n\",\n \"156\\n\",\n \"
\\n\",\n \"Two of the country’s largest transgender rights organizations will merge\\n\",\n \"\\n\",\n \"158\\n\",\n \"
\\n\",\n \"Transforming Healthcare\\n\",\n \"\\n\",\n \"158\\n\",\n \"
\\n\",\n \"With Harris and Walz, Democrats put abortion rights at the top of the agenda\\n\",\n \"\\n\",\n \"159\\n\",\n \"
\\n\",\n \"In states with strict abortion policies, simply seeing an OB/GYN for regular care can be difficult\\n\",\n \"\\n\",\n \"160\\n\",\n \"
\\n\",\n \"Where abortion rights could be on the ballot this fall: From the Politics Desk\\n\",\n \"\\n\",\n \"161\\n\",\n \"
\\n\",\n \"Map: Where medication abortion is and isn’t legal\\n\",\n \"\\n\",\n \"162\\n\",\n \"
\\n\",\n \"
\\n\",\n \"10 rows × 2 columns\\n\",\n \"
\\n\",\n \"
\\n\"\n ],\n \"text/plain\": [\n \"\\u001b[0m┌\\u001b[0m\\u001b[0m────────────────────────────────────────────────────────────────────────────────────────────────────\\u001b[0m\\u001b[0m┬\\u001b[0m\\u001b[0m──────────\\u001b[0m\\u001b[0m┐\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m\\u001b[0m\\u001b[1mheadline\\u001b[0m \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m\\u001b[0m\\u001b[1mdistance\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m├\\u001b[0m\\u001b[0m────────────────────────────────────────────────────────────────────────────────────────────────────\\u001b[0m\\u001b[0m┼\\u001b[0m\\u001b[0m──────────\\u001b[0m\\u001b[0m┤\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mKamala Harris visits Planned Parenthood clinic \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 139\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mHow a network of abortion pill providers works together in the wake of new threats \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 151\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mAfter Dobbs decision, more women are managing their own abortions \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 153\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mA timeline of Trump's many, many positions on abortion \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 156\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mTwo of the country’s largest transgender rights organizations will merge \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 158\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mTransforming Healthcare \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 158\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mWith Harris and Walz, Democrats put abortion rights at the top of the agenda \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 159\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mIn states with strict abortion policies, simply seeing an OB/GYN for regular care can be difficult\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 160\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mWhere abortion rights could be on the ballot this fall: From the Politics Desk \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 161\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mMap: Where medication abortion is and isn’t legal \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 162\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m└\\u001b[0m\\u001b[0m────────────────────────────────────────────────────────────────────────────────────────────────────\\u001b[0m\\u001b[0m┴\\u001b[0m\\u001b[0m──────────\\u001b[0m\\u001b[0m┘\\n\",\n \"\\u001b[0m\\u001b[0m\"\n ]\n },\n \"execution_count\": 28,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"source\": [\n \"select\\n\",\n \" slim.articles.headline,\\n\",\n \" slim.vec_articles.distance\\n\",\n \"from slim.vec_articles\\n\",\n \"left join slim.articles on slim.articles.rowid = slim.vec_articles.article_id\\n\",\n \"where headline_embedding match vec_quantize_binary(lembed(\\\"planned parenthood\\\"))\\n\",\n \" and k = 10;\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"You'll notice the results differ slightly to the full-sized query from above. Some results are ordered differently, some are missing. \\n\",\n \"The `distance` in this binary KNN search is hamming distance, not the default L2 distance. \"\n ]\n }\n ],\n \"metadata\": {\n \"kernelspec\": {\n \"display_name\": \"Solite\",\n \"language\": \"sql\",\n \"name\": \"solite\"\n },\n \"language_info\": {\n \"file_extension\": \".sql\",\n \"mimetype\": \"text/x.sqlite\",\n \"name\": \"sql\",\n \"nb_converter\": \"script\",\n \"pygments_lexer\": \"sql\",\n \"version\": \"TODO\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 2\n}\n" }, { "path": "examples/nbc-headlines/3_search.ipynb", "content": "{\n \"cells\": [\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"# NBC News Headlines: Exploring Hybrod FTS5 + Vector Search\\n\",\n \"\\n\",\n \"This notebooks explore a few different ways one could combine FTS5 and vector search results, when querying \\n\",\n \"[FTS5](https://www.sqlite.org/fts5.html) and\\n\",\n \"[`sqlite-vec`](https://github.com/asg017/sqlite-vec) virtual table.\\n\",\n \"\\n\",\n \"This dataset is a small list of headines scraped from NBC News, found in the [`./1_scrape.ipynb`](./1_scrape.ipynb) notebook.\\n\",\n \"To see how the `fts_articles` and `vec_articles` tables were created, see the [`./3_search.ipynb`](./3_search.ipynb) notebook.\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 1,\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"text/html\": [\n \"
\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"
\\n\",\n \"vec_version()\\n\",\n \"\\n\",\n \"lembed_version()\\n\",\n \"
\\n\",\n \"v0.1.3-alpha.2\\n\",\n \"\\n\",\n \"v0.0.1-alpha.8\\n\",\n \"
\\n\",\n \"
\\n\",\n \"1 row × 2 columns\\n\",\n \"
\\n\",\n \"
\\n\"\n ],\n \"text/plain\": [\n \"\\u001b[0m┌\\u001b[0m\\u001b[0m────────────────\\u001b[0m\\u001b[0m┬\\u001b[0m\\u001b[0m──────────────────\\u001b[0m\\u001b[0m┐\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m\\u001b[0m\\u001b[1mvec_version()\\u001b[0m \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m\\u001b[0m\\u001b[1mlembed_version()\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m├\\u001b[0m\\u001b[0m────────────────\\u001b[0m\\u001b[0m┼\\u001b[0m\\u001b[0m──────────────────\\u001b[0m\\u001b[0m┤\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mv0.1.3-alpha.2\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mv0.0.1-alpha.8 \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m└\\u001b[0m\\u001b[0m────────────────\\u001b[0m\\u001b[0m┴\\u001b[0m\\u001b[0m──────────────────\\u001b[0m\\u001b[0m┘\\n\",\n \"\\u001b[0m\\u001b[0m\"\n ]\n },\n \"execution_count\": 1,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"source\": [\n \".open tmp-artic2.db\\n\",\n \"\\n\",\n \".load ../../dist/vec0\\n\",\n \".load ./lembed0\\n\",\n \"\\n\",\n \"insert into lembed_models(name, model)\\n\",\n \" values (\\n\",\n \" 'default',\\n\",\n \" lembed_model_from_file('snowflake-arctic-embed-m-v1.5.d70deb40.f16.gguf')\\n\",\n \" );\\n\",\n \"\\n\",\n \"select vec_version(), lembed_version();\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## Full-text Search Only\\n\",\n \"\\n\",\n \"A simple FTS query on the `fts_articles` virutal table can be made like so:\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 19,\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"text/html\": [\n \"
\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"
\\n\",\n \"rowid\\n\",\n \"\\n\",\n \"headline\\n\",\n \"\\n\",\n \"rank\\n\",\n \"
\\n\",\n \"4666\\n\",\n \"\\n\",\n \"Kamala Harris visits Planned Parenthood clinic\\n\",\n \"\\n\",\n \"-18.9139950477264\\n\",\n \"
\\n\",\n \"6521\\n\",\n \"\\n\",\n \"Former Marine sentenced to 9 years in prison for firebombing Planned Parenthood clinic\\n\",\n \"\\n\",\n \"-14.807022703838651\\n\",\n \"
\\n\",\n \"
\\n\",\n \"2 rows × 3 columns\\n\",\n \"
\\n\",\n \"
\\n\"\n ],\n \"text/plain\": [\n \"\\u001b[0m┌\\u001b[0m\\u001b[0m───────\\u001b[0m\\u001b[0m┬\\u001b[0m\\u001b[0m────────────────────────────────────────────────────────────────────────────────────────\\u001b[0m\\u001b[0m┬\\u001b[0m\\u001b[0m─────────────────────\\u001b[0m\\u001b[0m┐\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m\\u001b[0m\\u001b[1mrowid\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m\\u001b[0m\\u001b[1mheadline\\u001b[0m \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m\\u001b[0m\\u001b[1mrank\\u001b[0m \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m├\\u001b[0m\\u001b[0m───────\\u001b[0m\\u001b[0m┼\\u001b[0m\\u001b[0m────────────────────────────────────────────────────────────────────────────────────────\\u001b[0m\\u001b[0m┼\\u001b[0m\\u001b[0m─────────────────────\\u001b[0m\\u001b[0m┤\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 4666\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mKamala Harris visits Planned Parenthood clinic \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m -18.9139950477264\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 6521\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mFormer Marine sentenced to 9 years in prison for firebombing Planned Parenthood clinic\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m-14.807022703838651\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m└\\u001b[0m\\u001b[0m───────\\u001b[0m\\u001b[0m┴\\u001b[0m\\u001b[0m────────────────────────────────────────────────────────────────────────────────────────\\u001b[0m\\u001b[0m┴\\u001b[0m\\u001b[0m─────────────────────\\u001b[0m\\u001b[0m┘\\n\",\n \"\\u001b[0m\\u001b[0m\"\n ]\n },\n \"execution_count\": 19,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"source\": [\n \".param set query planned parenthood\\n\",\n \"\\n\",\n \"select\\n\",\n \" rowid,\\n\",\n \" headline,\\n\",\n \" rank\\n\",\n \"from fts_articles\\n\",\n \"where headline match :query\\n\",\n \"order by rank\\n\",\n \"limit 10;\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"The `rank` column is the negative BM25 score of the query + document. \"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## Vector Search Only\\n\",\n \"\\n\",\n \"A KNN vector search can be made on the `vec_articles` virtual table like so:\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 6,\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"text/html\": [\n \"
\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"
\\n\",\n \"article_id\\n\",\n \"\\n\",\n \"headline\\n\",\n \"\\n\",\n \"distance\\n\",\n \"
\\n\",\n \"4666\\n\",\n \"\\n\",\n \"Kamala Harris visits Planned Parenthood clinic\\n\",\n \"\\n\",\n \"0.492593914270401\\n\",\n \"
\\n\",\n \"13928\\n\",\n \"\\n\",\n \"After Dobbs decision, more women are managing their own abortions\\n\",\n \"\\n\",\n \"0.5789032578468323\\n\",\n \"
\\n\",\n \"12636\\n\",\n \"\\n\",\n \"Transforming Healthcare\\n\",\n \"\\n\",\n \"0.5822411179542542\\n\",\n \"
\\n\",\n \"6979\\n\",\n \"\\n\",\n \"A timeline of Trump's many, many positions on abortion\\n\",\n \"\\n\",\n \"0.6101462841033936\\n\",\n \"
\\n\",\n \"7038\\n\",\n \"\\n\",\n \"How a network of abortion pill providers works together in the wake of new threats\\n\",\n \"\\n\",\n \"0.6196886897087097\\n\",\n \"
\\n\",\n \"6914\\n\",\n \"\\n\",\n \"'Major hurdles': The reality check behind Biden's big abortion promise\\n\",\n \"\\n\",\n \"0.6198344826698303\\n\",\n \"
\\n\",\n \"6794\\n\",\n \"\\n\",\n \"Trump's conflicting abortion stances are coming back to haunt him — and his party\\n\",\n \"\\n\",\n \"0.6198986768722534\\n\",\n \"
\\n\",\n \"7381\\n\",\n \"\\n\",\n \"Where abortion rights could be on the ballot this fall: From the Politics Desk\\n\",\n \"\\n\",\n \"0.6201764345169067\\n\",\n \"
\\n\",\n \"6871\\n\",\n \"\\n\",\n \"How the Biden campaign quickly mobilized on Trump's abortion stance\\n\",\n \"\\n\",\n \"0.633980393409729\\n\",\n \"
\\n\",\n \"5496\\n\",\n \"\\n\",\n \"Battle over abortion heats up in Arizona — and could be on the 2024 ballot\\n\",\n \"\\n\",\n \"0.6341449022293091\\n\",\n \"
\\n\",\n \"
\\n\",\n \"10 rows × 3 columns\\n\",\n \"
\\n\",\n \"
\\n\"\n ],\n \"text/plain\": [\n \"\\u001b[0m┌\\u001b[0m\\u001b[0m────────────\\u001b[0m\\u001b[0m┬\\u001b[0m\\u001b[0m────────────────────────────────────────────────────────────────────────────────────\\u001b[0m\\u001b[0m┬\\u001b[0m\\u001b[0m────────────────────\\u001b[0m\\u001b[0m┐\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m\\u001b[0m\\u001b[1marticle_id\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m\\u001b[0m\\u001b[1mheadline\\u001b[0m \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m\\u001b[0m\\u001b[1mdistance\\u001b[0m \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m├\\u001b[0m\\u001b[0m────────────\\u001b[0m\\u001b[0m┼\\u001b[0m\\u001b[0m────────────────────────────────────────────────────────────────────────────────────\\u001b[0m\\u001b[0m┼\\u001b[0m\\u001b[0m────────────────────\\u001b[0m\\u001b[0m┤\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 4666\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mKamala Harris visits Planned Parenthood clinic \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 0.492593914270401\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 13928\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mAfter Dobbs decision, more women are managing their own abortions \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m0.5789032578468323\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 12636\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mTransforming Healthcare \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m0.5822411179542542\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 6979\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mA timeline of Trump's many, many positions on abortion \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m0.6101462841033936\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 7038\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mHow a network of abortion pill providers works together in the wake of new threats\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m0.6196886897087097\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 6914\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m'Major hurdles': The reality check behind Biden's big abortion promise \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m0.6198344826698303\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 6794\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mTrump's conflicting abortion stances are coming back to haunt him — and his party \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m0.6198986768722534\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 7381\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mWhere abortion rights could be on the ballot this fall: From the Politics Desk \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m0.6201764345169067\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 6871\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mHow the Biden campaign quickly mobilized on Trump's abortion stance \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 0.633980393409729\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 5496\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mBattle over abortion heats up in Arizona — and could be on the 2024 ballot \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m0.6341449022293091\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m└\\u001b[0m\\u001b[0m────────────\\u001b[0m\\u001b[0m┴\\u001b[0m\\u001b[0m────────────────────────────────────────────────────────────────────────────────────\\u001b[0m\\u001b[0m┴\\u001b[0m\\u001b[0m────────────────────\\u001b[0m\\u001b[0m┘\\n\",\n \"\\u001b[0m\\u001b[0m\"\n ]\n },\n \"execution_count\": 6,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"source\": [\n \".param set query planned parenthood\\n\",\n \"\\n\",\n \"select\\n\",\n \" article_id,\\n\",\n \" articles.headline,\\n\",\n \" distance\\n\",\n \"from vec_articles\\n\",\n \"left join articles on articles.rowid = vec_articles.article_id\\n\",\n \"where headline_embedding match lembed(:query)\\n\",\n \" and k = 10;\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"The `distance` column is the L2 distance between the query vector and the headline embedding. \\n\",\n \"\\n\",\n \"The rest of this notebook explore different ways of combining these FTS5 and vector search results. \\n\",\n \"The core queries are similar, and only really different on different `JOIN` or `ORDER BY` techniques.\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## Combination Technique #1: Keyword-first\\n\",\n \"\\n\",\n \"In many search-engine cases, you may way to display keyword matches first, and supplement the rest wih with vector search results. \\n\",\n \"This makes some intuitive sense — keyword matches are what uses expect, but you'll want to display more result if there are only a few matching documents. \\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 11,\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"text/html\": [\n \"
\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"
\\n\",\n \"id\\n\",\n \"\\n\",\n \"headline\\n\",\n \"\\n\",\n \"match_type\\n\",\n \"\\n\",\n \"article_id\\n\",\n \"\\n\",\n \"rank_number\\n\",\n \"\\n\",\n \"score\\n\",\n \"
\\n\",\n \"10098\\n\",\n \"\\n\",\n \"Kamala Harris says abortion bans are creating 'a health care crisis'\\n\",\n \"\\n\",\n \"fts\\n\",\n \"\\n\",\n \"10098\\n\",\n \"\\n\",\n \"1\\n\",\n \"\\n\",\n \"-10.678829270936067\\n\",\n \"
\\n\",\n \"9776\\n\",\n \"\\n\",\n \"States with abortion bans saw birth control prescriptions fall post-Dobbs, study finds\\n\",\n \"\\n\",\n \"fts\\n\",\n \"\\n\",\n \"9776\\n\",\n \"\\n\",\n \"2\\n\",\n \"\\n\",\n \"-10.016316725971112\\n\",\n \"
\\n\",\n \"2292\\n\",\n \"\\n\",\n \"Ohio GOP Senate candidates pitch federal abortion bans even after voters protected reproductive rights\\n\",\n \"\\n\",\n \"fts\\n\",\n \"\\n\",\n \"2292\\n\",\n \"\\n\",\n \"3\\n\",\n \"\\n\",\n \"-9.7149595994016\\n\",\n \"
\\n\",\n \"452\\n\",\n \"\\n\",\n \"64K women and girls became pregnant due to rape in states with abortion bans, study estimates\\n\",\n \"\\n\",\n \"fts\\n\",\n \"\\n\",\n \"452\\n\",\n \"\\n\",\n \"4\\n\",\n \"\\n\",\n \"-9.163558569425538\\n\",\n \"
\\n\",\n \"9187\\n\",\n \"\\n\",\n \"Abortion bans drive away up to half of young talent, CNBC/Generation Lab youth survey finds\\n\",\n \"\\n\",\n \"fts\\n\",\n \"\\n\",\n \"9187\\n\",\n \"\\n\",\n \"5\\n\",\n \"\\n\",\n \"-9.163558569425538\\n\",\n \"
\\n\",\n \"6989\\n\",\n \"\\n\",\n \"Trump says abortion restrictions should be left to states, dodging a national ban\\n\",\n \"\\n\",\n \"vec\\n\",\n \"\\n\",\n \"6989\\n\",\n \"\\n\",\n \"1\\n\",\n \"\\n\",\n \"0.4930749833583832\\n\",\n \"
\\n\",\n \"13928\\n\",\n \"\\n\",\n \"After Dobbs decision, more women are managing their own abortions\\n\",\n \"\\n\",\n \"vec\\n\",\n \"\\n\",\n \"13928\\n\",\n \"\\n\",\n \"2\\n\",\n \"\\n\",\n \"0.5120846629142761\\n\",\n \"
\\n\",\n \"11822\\n\",\n \"\\n\",\n \"Iowa now bans most abortions after about 6 weeks\\n\",\n \"\\n\",\n \"vec\\n\",\n \"\\n\",\n \"11822\\n\",\n \"\\n\",\n \"3\\n\",\n \"\\n\",\n \"0.512569785118103\\n\",\n \"
\\n\",\n \"7381\\n\",\n \"\\n\",\n \"Where abortion rights could be on the ballot this fall: From the Politics Desk\\n\",\n \"\\n\",\n \"vec\\n\",\n \"\\n\",\n \"7381\\n\",\n \"\\n\",\n \"4\\n\",\n \"\\n\",\n \"0.5168291926383972\\n\",\n \"
\\n\",\n \"14009\\n\",\n \"\\n\",\n \"Trump signals openness to banning abortion pill\\n\",\n \"\\n\",\n \"vec\\n\",\n \"\\n\",\n \"14009\\n\",\n \"\\n\",\n \"5\\n\",\n \"\\n\",\n \"0.5288293957710266\\n\",\n \"
\\n\",\n \"4426\\n\",\n \"\\n\",\n \"Medication abortions rose in year after Dobbs decision, report finds\\n\",\n \"\\n\",\n \"vec\\n\",\n \"\\n\",\n \"4426\\n\",\n \"\\n\",\n \"6\\n\",\n \"\\n\",\n \"0.5305097699165344\\n\",\n \"
\\n\",\n \"4328\\n\",\n \"\\n\",\n \"Trump signals support for a national 15-week abortion ban\\n\",\n \"\\n\",\n \"vec\\n\",\n \"\\n\",\n \"4328\\n\",\n \"\\n\",\n \"7\\n\",\n \"\\n\",\n \"0.532848060131073\\n\",\n \"
\\n\",\n \"6979\\n\",\n \"\\n\",\n \"A timeline of Trump's many, many positions on abortion\\n\",\n \"\\n\",\n \"vec\\n\",\n \"\\n\",\n \"6979\\n\",\n \"\\n\",\n \"8\\n\",\n \"\\n\",\n \"0.533357560634613\\n\",\n \"
\\n\",\n \"2092\\n\",\n \"\\n\",\n \"For the first time in years, Sen. Graham hasn't introduced a national abortion ban\\n\",\n \"\\n\",\n \"vec\\n\",\n \"\\n\",\n \"2092\\n\",\n \"\\n\",\n \"9\\n\",\n \"\\n\",\n \"0.5336830615997314\\n\",\n \"
\\n\",\n \"6794\\n\",\n \"\\n\",\n \"Trump's conflicting abortion stances are coming back to haunt him — and his party\\n\",\n \"\\n\",\n \"vec\\n\",\n \"\\n\",\n \"6794\\n\",\n \"\\n\",\n \"10\\n\",\n \"\\n\",\n \"0.5347095131874084\\n\",\n \"
\\n\",\n \"
\\n\",\n \"15 rows × 6 columns\\n\",\n \"
\\n\",\n \"
\\n\"\n ],\n \"text/plain\": [\n \"\\u001b[0m┌\\u001b[0m\\u001b[0m───────\\u001b[0m\\u001b[0m┬\\u001b[0m\\u001b[0m────────────────────────────────────────────────────────────────────────────────────────────────────────\\u001b[0m\\u001b[0m┬\\u001b[0m\\u001b[0m────────────\\u001b[0m\\u001b[0m┬\\u001b[0m\\u001b[0m────────────\\u001b[0m\\u001b[0m┬\\u001b[0m\\u001b[0m─────────────\\u001b[0m\\u001b[0m┬\\u001b[0m\\u001b[0m─────────────────────\\u001b[0m\\u001b[0m┐\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m\\u001b[0m\\u001b[1mid\\u001b[0m \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m\\u001b[0m\\u001b[1mheadline\\u001b[0m \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m\\u001b[0m\\u001b[1mmatch_type\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m\\u001b[0m\\u001b[1marticle_id\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m\\u001b[0m\\u001b[1mrank_number\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m\\u001b[0m\\u001b[1mscore\\u001b[0m \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m├\\u001b[0m\\u001b[0m───────\\u001b[0m\\u001b[0m┼\\u001b[0m\\u001b[0m────────────────────────────────────────────────────────────────────────────────────────────────────────\\u001b[0m\\u001b[0m┼\\u001b[0m\\u001b[0m────────────\\u001b[0m\\u001b[0m┼\\u001b[0m\\u001b[0m────────────\\u001b[0m\\u001b[0m┼\\u001b[0m\\u001b[0m─────────────\\u001b[0m\\u001b[0m┼\\u001b[0m\\u001b[0m─────────────────────\\u001b[0m\\u001b[0m┤\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m10098\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mKamala Harris says abortion bans are creating 'a health care crisis' \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mfts \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 10098\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 1\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m-10.678829270936067\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 9776\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mStates with abortion bans saw birth control prescriptions fall post-Dobbs, study finds \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mfts \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 9776\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 2\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m-10.016316725971112\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 2292\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mOhio GOP Senate candidates pitch federal abortion bans even after voters protected reproductive rights\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mfts \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 2292\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 3\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m -9.7149595994016\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 452\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m64K women and girls became pregnant due to rape in states with abortion bans, study estimates \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mfts \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 452\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 4\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m -9.163558569425538\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 9187\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mAbortion bans drive away up to half of young talent, CNBC/Generation Lab youth survey finds \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mfts \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 9187\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 5\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m -9.163558569425538\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 6989\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mTrump says abortion restrictions should be left to states, dodging a national ban \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mvec \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 6989\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 1\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 0.4930749833583832\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m13928\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mAfter Dobbs decision, more women are managing their own abortions \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mvec \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 13928\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 2\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 0.5120846629142761\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m11822\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mIowa now bans most abortions after about 6 weeks \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mvec \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 11822\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 3\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 0.512569785118103\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 7381\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mWhere abortion rights could be on the ballot this fall: From the Politics Desk \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mvec \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 7381\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 4\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 0.5168291926383972\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m14009\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mTrump signals openness to banning abortion pill \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mvec \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 14009\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 5\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 0.5288293957710266\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 4426\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mMedication abortions rose in year after Dobbs decision, report finds \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mvec \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 4426\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 6\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 0.5305097699165344\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 4328\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mTrump signals support for a national 15-week abortion ban \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mvec \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 4328\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 7\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 0.532848060131073\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 6979\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mA timeline of Trump's many, many positions on abortion \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mvec \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 6979\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 8\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 0.533357560634613\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 2092\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mFor the first time in years, Sen. Graham hasn't introduced a national abortion ban \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mvec \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 2092\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 9\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 0.5336830615997314\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 6794\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mTrump's conflicting abortion stances are coming back to haunt him — and his party \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mvec \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 6794\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 10\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 0.5347095131874084\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m└\\u001b[0m\\u001b[0m───────\\u001b[0m\\u001b[0m┴\\u001b[0m\\u001b[0m────────────────────────────────────────────────────────────────────────────────────────────────────────\\u001b[0m\\u001b[0m┴\\u001b[0m\\u001b[0m────────────\\u001b[0m\\u001b[0m┴\\u001b[0m\\u001b[0m────────────\\u001b[0m\\u001b[0m┴\\u001b[0m\\u001b[0m─────────────\\u001b[0m\\u001b[0m┴\\u001b[0m\\u001b[0m─────────────────────\\u001b[0m\\u001b[0m┘\\n\",\n \"\\u001b[0m\\u001b[0m\"\n ]\n },\n \"execution_count\": 11,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"source\": [\n \".param set query abortion bans\\n\",\n \".param set k 10\\n\",\n \"\\n\",\n \"\\n\",\n \"with fts_matches as (\\n\",\n \" select\\n\",\n \" rowid as article_id,\\n\",\n \" row_number() over (order by rank) as rank_number,\\n\",\n \" rank as score\\n\",\n \" from fts_articles\\n\",\n \" where headline match :query\\n\",\n \" limit :k\\n\",\n \"),\\n\",\n \"vec_matches as (\\n\",\n \" select\\n\",\n \" article_id,\\n\",\n \" row_number() over (order by distance) as rank_number,\\n\",\n \" distance as score\\n\",\n \" from vec_articles\\n\",\n \" where\\n\",\n \" headline_embedding match lembed(:query)\\n\",\n \" and k = :k\\n\",\n \" order by distance\\n\",\n \"),\\n\",\n \"combined as (\\n\",\n \" select 'fts' as match_type, * from fts_matches\\n\",\n \" union all\\n\",\n \" select 'vec' as match_type, * from vec_matches\\n\",\n \"),\\n\",\n \"final as (\\n\",\n \" select\\n\",\n \" articles.id,\\n\",\n \" articles.headline,\\n\",\n \" combined.*\\n\",\n \" from combined\\n\",\n \" left join articles on articles.rowid = combined.article_id\\n\",\n \")\\n\",\n \"select * from final;\\n\",\n \"\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"We do this with a verbose CTE: one step for the FTS5 query, another for the vector search, one to \\\"combine\\\" the results with a `UNION ALL`, and one last one to `LEFT JOIN` back to the base `articles` table to get the headline.\\n\",\n \"\\n\",\n \"Here we have 5 FTS results and 10 additional vector results. This seems pretty natural, a fallback to vector search when keywords matches lack a bit.\\n\",\n \"\\n\",\n \"One note: this example doesn't do any de-duplication, so you may get the same results twice. So you may want to add a `DISTINCT` or `GROUP BY` somehwere to handle that. \"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"## Combination Technique #2: Reciprocal Rank Fusion (RRF)\\n\",\n \"\\n\",\n \"[Reciprocal Rank Fusion](https://learn.microsoft.com/en-us/azure/search/hybrid-search-ranking) \\n\",\n \"is another combination technique, where matches that are both FTS matches and vector matches\\n\",\n \"are ranked higher than other. The CTE logic is a bit more involved, but can still be represented in a few steps:\\n\"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 14,\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"text/html\": [\n \"
\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"
\\n\",\n \"id\\n\",\n \"\\n\",\n \"headline\\n\",\n \"\\n\",\n \"vec_rank\\n\",\n \"\\n\",\n \"fts_rank\\n\",\n \"\\n\",\n \"combined_rank\\n\",\n \"\\n\",\n \"vec_distance\\n\",\n \"\\n\",\n \"fts_score\\n\",\n \"
\\n\",\n \"4328\\n\",\n \"\\n\",\n \"Trump signals support for a national 15-week abortion ban\\n\",\n \"\\n\",\n \"2\\n\",\n \"\\n\",\n \"3\\n\",\n \"\\n\",\n \"0.03200204813108039\\n\",\n \"\\n\",\n \"0.5334203839302063\\n\",\n \"\\n\",\n \"-9.841645168493953\\n\",\n \"
\\n\",\n \"5769\\n\",\n \"\\n\",\n \"Mitch McConnell shies away from supporting national abortion ban\\n\",\n \"\\n\",\n \"8\\n\",\n \"\\n\",\n \"2\\n\",\n \"\\n\",\n \"0.030834914611005692\\n\",\n \"\\n\",\n \"0.5501425266265869\\n\",\n \"\\n\",\n \"-10.19017787567105\\n\",\n \"
\\n\",\n \"9507\\n\",\n \"\\n\",\n \"Arizona Senate passes repeal of 1864 abortion ban\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"1\\n\",\n \"\\n\",\n \"0.01639344262295082\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"-10.564302831642667\\n\",\n \"
\\n\",\n \"6989\\n\",\n \"\\n\",\n \"Trump says abortion restrictions should be left to states, dodging a national ban\\n\",\n \"\\n\",\n \"1\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"0.01639344262295082\\n\",\n \"\\n\",\n \"0.5142395496368408\\n\",\n \"\\n\",\n \"\\n\",\n \"
\\n\",\n \"10717\\n\",\n \"\\n\",\n \"Supreme Court rejects bid to restrict access to abortion pill\\n\",\n \"\\n\",\n \"3\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"0.015873015873015872\\n\",\n \"\\n\",\n \"0.5351248383522034\\n\",\n \"\\n\",\n \"\\n\",\n \"
\\n\",\n \"5981\\n\",\n \"\\n\",\n \"Arizona state House passes bill to repeal 1864 abortion ban\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"4\\n\",\n \"\\n\",\n \"0.015625\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"-9.841645168493953\\n\",\n \"
\\n\",\n \"14009\\n\",\n \"\\n\",\n \"Trump signals openness to banning abortion pill\\n\",\n \"\\n\",\n \"4\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"0.015625\\n\",\n \"\\n\",\n \"0.5364335179328918\\n\",\n \"\\n\",\n \"\\n\",\n \"
\\n\",\n \"6375\\n\",\n \"\\n\",\n \"Arizona Republicans again quash effort to repeal 1864 abortion ban\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"5\\n\",\n \"\\n\",\n \"0.015384615384615385\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"-9.841645168493953\\n\",\n \"
\\n\",\n \"7381\\n\",\n \"\\n\",\n \"Where abortion rights could be on the ballot this fall: From the Politics Desk\\n\",\n \"\\n\",\n \"5\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"0.015384615384615385\\n\",\n \"\\n\",\n \"0.5462378859519958\\n\",\n \"\\n\",\n \"\\n\",\n \"
\\n\",\n \"9443\\n\",\n \"\\n\",\n \"Arizona Gov. Katie Hobbs signs repeal of 1864 abortion ban\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"6\\n\",\n \"\\n\",\n \"0.015151515151515152\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"-9.841645168493953\\n\",\n \"
\\n\",\n \"13928\\n\",\n \"\\n\",\n \"After Dobbs decision, more women are managing their own abortions\\n\",\n \"\\n\",\n \"6\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"0.015151515151515152\\n\",\n \"\\n\",\n \"0.5467031002044678\\n\",\n \"\\n\",\n \"\\n\",\n \"
\\n\",\n \"1821\\n\",\n \"\\n\",\n \"Dominican women fight child marriage, teen pregancy amid total abortion ban\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"7\\n\",\n \"\\n\",\n \"0.014925373134328358\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"-9.51616557526609\\n\",\n \"
\\n\",\n \"2092\\n\",\n \"\\n\",\n \"For the first time in years, Sen. Graham hasn't introduced a national abortion ban\\n\",\n \"\\n\",\n \"7\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"0.014925373134328358\\n\",\n \"\\n\",\n \"0.5477523803710938\\n\",\n \"\\n\",\n \"\\n\",\n \"
\\n\",\n \"7150\\n\",\n \"\\n\",\n \"Tennessee court weighs challenge to abortion ban’s narrow medical exception\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"8\\n\",\n \"\\n\",\n \"0.014705882352941176\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"-9.51616557526609\\n\",\n \"
\\n\",\n \"8690\\n\",\n \"\\n\",\n \"Arizona Supreme Court pushes back enforcement date for 1864 abortion ban\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"9\\n\",\n \"\\n\",\n \"0.014492753623188406\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"-9.51616557526609\\n\",\n \"
\\n\",\n \"11822\\n\",\n \"\\n\",\n \"Iowa now bans most abortions after about 6 weeks\\n\",\n \"\\n\",\n \"9\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"0.014492753623188406\\n\",\n \"\\n\",\n \"0.5557170510292053\\n\",\n \"\\n\",\n \"\\n\",\n \"
\\n\",\n \"2646\\n\",\n \"\\n\",\n \"Trump campaign scrambles over abortion ban report as Democrats seize the moment\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"10\\n\",\n \"\\n\",\n \"0.014285714285714285\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"-9.211525101866211\\n\",\n \"
\\n\",\n \"5538\\n\",\n \"\\n\",\n \"Map: Where medication abortion is and isn’t legal\\n\",\n \"\\n\",\n \"10\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"0.014285714285714285\\n\",\n \"\\n\",\n \"0.5588464140892029\\n\",\n \"\\n\",\n \"\\n\",\n \"
\\n\",\n \"
\\n\",\n \"18 rows × 7 columns\\n\",\n \"
\\n\",\n \"
\\n\"\n ],\n \"text/plain\": [\n \"\\u001b[0m┌\\u001b[0m\\u001b[0m───────\\u001b[0m\\u001b[0m┬\\u001b[0m\\u001b[0m────────────────────────────────────────────────────────────────────────────────────\\u001b[0m\\u001b[0m┬\\u001b[0m\\u001b[0m──────────\\u001b[0m\\u001b[0m┬\\u001b[0m\\u001b[0m──────────\\u001b[0m\\u001b[0m┬\\u001b[0m\\u001b[0m──────────────────────\\u001b[0m\\u001b[0m┬\\u001b[0m\\u001b[0m────────────────────\\u001b[0m\\u001b[0m┬\\u001b[0m\\u001b[0m─────────────────────\\u001b[0m\\u001b[0m┐\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m\\u001b[0m\\u001b[1mid\\u001b[0m \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m\\u001b[0m\\u001b[1mheadline\\u001b[0m \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m\\u001b[0m\\u001b[1mvec_rank\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m\\u001b[0m\\u001b[1mfts_rank\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m\\u001b[0m\\u001b[1mcombined_rank\\u001b[0m \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m\\u001b[0m\\u001b[1mvec_distance\\u001b[0m \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m\\u001b[0m\\u001b[1mfts_score\\u001b[0m \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m├\\u001b[0m\\u001b[0m───────\\u001b[0m\\u001b[0m┼\\u001b[0m\\u001b[0m────────────────────────────────────────────────────────────────────────────────────\\u001b[0m\\u001b[0m┼\\u001b[0m\\u001b[0m──────────\\u001b[0m\\u001b[0m┼\\u001b[0m\\u001b[0m──────────\\u001b[0m\\u001b[0m┼\\u001b[0m\\u001b[0m──────────────────────\\u001b[0m\\u001b[0m┼\\u001b[0m\\u001b[0m────────────────────\\u001b[0m\\u001b[0m┼\\u001b[0m\\u001b[0m─────────────────────\\u001b[0m\\u001b[0m┤\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 4328\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mTrump signals support for a national 15-week abortion ban \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 2\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 3\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 0.03200204813108039\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m0.5334203839302063\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m -9.841645168493953\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 5769\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mMitch McConnell shies away from supporting national abortion ban \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 8\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 2\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m0.030834914611005692\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m0.5501425266265869\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m -10.19017787567105\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 9507\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mArizona Senate passes repeal of 1864 abortion ban \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 1\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 0.01639344262295082\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m-10.564302831642667\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 6989\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mTrump says abortion restrictions should be left to states, dodging a national ban \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 1\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 0.01639344262295082\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m0.5142395496368408\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m10717\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mSupreme Court rejects bid to restrict access to abortion pill \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 3\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m0.015873015873015872\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m0.5351248383522034\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 5981\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mArizona state House passes bill to repeal 1864 abortion ban \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 4\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 0.015625\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m -9.841645168493953\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m14009\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mTrump signals openness to banning abortion pill \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 4\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 0.015625\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m0.5364335179328918\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 6375\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mArizona Republicans again quash effort to repeal 1864 abortion ban \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 5\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m0.015384615384615385\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m -9.841645168493953\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 7381\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mWhere abortion rights could be on the ballot this fall: From the Politics Desk \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 5\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m0.015384615384615385\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m0.5462378859519958\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 9443\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mArizona Gov. Katie Hobbs signs repeal of 1864 abortion ban \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 6\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m0.015151515151515152\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m -9.841645168493953\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m13928\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mAfter Dobbs decision, more women are managing their own abortions \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 6\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m0.015151515151515152\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m0.5467031002044678\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 1821\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mDominican women fight child marriage, teen pregancy amid total abortion ban \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 7\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m0.014925373134328358\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m -9.51616557526609\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 2092\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mFor the first time in years, Sen. Graham hasn't introduced a national abortion ban\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 7\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m0.014925373134328358\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m0.5477523803710938\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 7150\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mTennessee court weighs challenge to abortion ban’s narrow medical exception \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 8\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m0.014705882352941176\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m -9.51616557526609\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 8690\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mArizona Supreme Court pushes back enforcement date for 1864 abortion ban \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 9\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m0.014492753623188406\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m -9.51616557526609\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m11822\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mIowa now bans most abortions after about 6 weeks \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 9\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m0.014492753623188406\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m0.5557170510292053\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 2646\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mTrump campaign scrambles over abortion ban report as Democrats seize the moment \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 10\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m0.014285714285714285\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m -9.211525101866211\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 5538\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mMap: Where medication abortion is and isn’t legal \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 10\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m0.014285714285714285\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m0.5588464140892029\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m└\\u001b[0m\\u001b[0m───────\\u001b[0m\\u001b[0m┴\\u001b[0m\\u001b[0m────────────────────────────────────────────────────────────────────────────────────\\u001b[0m\\u001b[0m┴\\u001b[0m\\u001b[0m──────────\\u001b[0m\\u001b[0m┴\\u001b[0m\\u001b[0m──────────\\u001b[0m\\u001b[0m┴\\u001b[0m\\u001b[0m──────────────────────\\u001b[0m\\u001b[0m┴\\u001b[0m\\u001b[0m────────────────────\\u001b[0m\\u001b[0m┴\\u001b[0m\\u001b[0m─────────────────────\\u001b[0m\\u001b[0m┘\\n\",\n \"\\u001b[0m\\u001b[0m\"\n ]\n },\n \"execution_count\": 14,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"source\": [\n \".param set query abortion ban\\n\",\n \"\\n\",\n \"\\n\",\n \".param set k 10\\n\",\n \".param set rrf_k 60\\n\",\n \".param set weight_fts 1.0\\n\",\n \".param set weight_vec 1.0\\n\",\n \"\\n\",\n \"with vec_matches as (\\n\",\n \" select\\n\",\n \" article_id,\\n\",\n \" row_number() over (order by distance) as rank_number,\\n\",\n \" distance\\n\",\n \" from vec_articles\\n\",\n \" where\\n\",\n \" headline_embedding match lembed(:query)\\n\",\n \" and k = :k\\n\",\n \"),\\n\",\n \"fts_matches as (\\n\",\n \" select\\n\",\n \" rowid,\\n\",\n \" row_number() over (order by rank) as rank_number,\\n\",\n \" rank as score\\n\",\n \" from fts_articles\\n\",\n \" where headline match :query\\n\",\n \" limit :k\\n\",\n \"),\\n\",\n \"final as (\\n\",\n \" select\\n\",\n \" articles.id,\\n\",\n \" articles.headline,\\n\",\n \" vec_matches.rank_number as vec_rank,\\n\",\n \" fts_matches.rank_number as fts_rank,\\n\",\n \" coalesce(1.0 / (:rrf_k + fts_matches.rank_number), 0.0) * :weight_fts\\n\",\n \" + coalesce(1.0 / (:rrf_k + vec_matches.rank_number), 0.0) * :weight_vec\\n\",\n \" as combined_rank,\\n\",\n \" vec_matches.distance as vec_distance,\\n\",\n \" fts_matches.score as fts_score\\n\",\n \" from fts_matches\\n\",\n \" full outer join vec_matches on vec_matches.article_id = fts_matches.rowid\\n\",\n \" join articles on articles.rowid = coalesce(fts_matches.rowid, vec_matches.article_id)\\n\",\n \" order by combined_rank desc\\n\",\n \")\\n\",\n \"select * from final;\\n\",\n \"\\n\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"The first two CTE steps are identical to the \\\"keyword-first\\\" approach, just a normal FTS5 + vector KNN queries. \\n\",\n \"\\n\",\n \"The combination CTE step is more involved, and is described in detail in [this \\\"Hybrid Search\\\" Supabase docs page](https://supabase.com/docs/guides/ai/hybrid-search). \\n\",\n \"What's nice about this approach is that you can configure the \\\"weights\\\" of FTS or vector results with a normal SQL parameter. \\n\",\n \"\\n\",\n \"In this query, we can see the top result `\\\"Trump signals support for a national 15-week abortion ban\\\"` was neither a top FTS result or vector result — only ranked `2` and `3` respectively. \\n\",\n \"But since it appeared in both the FTS and vector results, it's ranked higher than others, same with `\\\"Mitch McConnell shies away from supporting national abortion ban\\\"`. The rest of the results are\\n\",\n \"FTS + vector results interwoven together, pretty nice!\"\n ]\n },\n {\n \"cell_type\": \"markdown\",\n \"metadata\": {},\n \"source\": [\n \"### Combination Technique #3: Re-rank by semantics\\n\",\n \"\\n\",\n \"Here we use FTS5 results are the \\\"source truth\\\", but we re-order them based on semantic similarity between \"\n ]\n },\n {\n \"cell_type\": \"code\",\n \"execution_count\": 18,\n \"metadata\": {},\n \"outputs\": [\n {\n \"data\": {\n \"text/html\": [\n \"
\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"\\n\",\n \"
\\n\",\n \"id\\n\",\n \"\\n\",\n \"headline\\n\",\n \"\\n\",\n \"rowid\\n\",\n \"\\n\",\n \"fts_rank_number\\n\",\n \"\\n\",\n \"score\\n\",\n \"
\\n\",\n \"4328\\n\",\n \"\\n\",\n \"Trump signals support for a national 15-week abortion ban\\n\",\n \"\\n\",\n \"4328\\n\",\n \"\\n\",\n \"3\\n\",\n \"\\n\",\n \"-9.841645168493953\\n\",\n \"
\\n\",\n \"5769\\n\",\n \"\\n\",\n \"Mitch McConnell shies away from supporting national abortion ban\\n\",\n \"\\n\",\n \"5769\\n\",\n \"\\n\",\n \"2\\n\",\n \"\\n\",\n \"-10.19017787567105\\n\",\n \"
\\n\",\n \"2646\\n\",\n \"\\n\",\n \"Trump campaign scrambles over abortion ban report as Democrats seize the moment\\n\",\n \"\\n\",\n \"2646\\n\",\n \"\\n\",\n \"10\\n\",\n \"\\n\",\n \"-9.211525101866211\\n\",\n \"
\\n\",\n \"7150\\n\",\n \"\\n\",\n \"Tennessee court weighs challenge to abortion ban’s narrow medical exception\\n\",\n \"\\n\",\n \"7150\\n\",\n \"\\n\",\n \"8\\n\",\n \"\\n\",\n \"-9.51616557526609\\n\",\n \"
\\n\",\n \"1821\\n\",\n \"\\n\",\n \"Dominican women fight child marriage, teen pregancy amid total abortion ban\\n\",\n \"\\n\",\n \"1821\\n\",\n \"\\n\",\n \"7\\n\",\n \"\\n\",\n \"-9.51616557526609\\n\",\n \"
\\n\",\n \"6375\\n\",\n \"\\n\",\n \"Arizona Republicans again quash effort to repeal 1864 abortion ban\\n\",\n \"\\n\",\n \"6375\\n\",\n \"\\n\",\n \"5\\n\",\n \"\\n\",\n \"-9.841645168493953\\n\",\n \"
\\n\",\n \"9507\\n\",\n \"\\n\",\n \"Arizona Senate passes repeal of 1864 abortion ban\\n\",\n \"\\n\",\n \"9507\\n\",\n \"\\n\",\n \"1\\n\",\n \"\\n\",\n \"-10.564302831642667\\n\",\n \"
\\n\",\n \"8690\\n\",\n \"\\n\",\n \"Arizona Supreme Court pushes back enforcement date for 1864 abortion ban\\n\",\n \"\\n\",\n \"8690\\n\",\n \"\\n\",\n \"9\\n\",\n \"\\n\",\n \"-9.51616557526609\\n\",\n \"
\\n\",\n \"5981\\n\",\n \"\\n\",\n \"Arizona state House passes bill to repeal 1864 abortion ban\\n\",\n \"\\n\",\n \"5981\\n\",\n \"\\n\",\n \"4\\n\",\n \"\\n\",\n \"-9.841645168493953\\n\",\n \"
\\n\",\n \"9443\\n\",\n \"\\n\",\n \"Arizona Gov. Katie Hobbs signs repeal of 1864 abortion ban\\n\",\n \"\\n\",\n \"9443\\n\",\n \"\\n\",\n \"6\\n\",\n \"\\n\",\n \"-9.841645168493953\\n\",\n \"
\\n\",\n \"
\\n\",\n \"10 rows × 5 columns\\n\",\n \"
\\n\",\n \"
\\n\"\n ],\n \"text/plain\": [\n \"\\u001b[0m┌\\u001b[0m\\u001b[0m──────\\u001b[0m\\u001b[0m┬\\u001b[0m\\u001b[0m─────────────────────────────────────────────────────────────────────────────────\\u001b[0m\\u001b[0m┬\\u001b[0m\\u001b[0m───────\\u001b[0m\\u001b[0m┬\\u001b[0m\\u001b[0m─────────────────\\u001b[0m\\u001b[0m┬\\u001b[0m\\u001b[0m─────────────────────\\u001b[0m\\u001b[0m┐\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m\\u001b[0m\\u001b[1mid\\u001b[0m \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m\\u001b[0m\\u001b[1mheadline\\u001b[0m \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m\\u001b[0m\\u001b[1mrowid\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m\\u001b[0m\\u001b[1mfts_rank_number\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m\\u001b[0m\\u001b[1mscore\\u001b[0m \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m├\\u001b[0m\\u001b[0m──────\\u001b[0m\\u001b[0m┼\\u001b[0m\\u001b[0m─────────────────────────────────────────────────────────────────────────────────\\u001b[0m\\u001b[0m┼\\u001b[0m\\u001b[0m───────\\u001b[0m\\u001b[0m┼\\u001b[0m\\u001b[0m─────────────────\\u001b[0m\\u001b[0m┼\\u001b[0m\\u001b[0m─────────────────────\\u001b[0m\\u001b[0m┤\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m4328\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mTrump signals support for a national 15-week abortion ban \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 4328\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 3\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m -9.841645168493953\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m5769\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mMitch McConnell shies away from supporting national abortion ban \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 5769\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 2\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m -10.19017787567105\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m2646\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mTrump campaign scrambles over abortion ban report as Democrats seize the moment\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 2646\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 10\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m -9.211525101866211\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m7150\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mTennessee court weighs challenge to abortion ban’s narrow medical exception \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 7150\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 8\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m -9.51616557526609\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m1821\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mDominican women fight child marriage, teen pregancy amid total abortion ban \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 1821\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 7\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m -9.51616557526609\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m6375\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mArizona Republicans again quash effort to repeal 1864 abortion ban \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 6375\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 5\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m -9.841645168493953\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m9507\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mArizona Senate passes repeal of 1864 abortion ban \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 9507\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 1\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m-10.564302831642667\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m8690\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mArizona Supreme Court pushes back enforcement date for 1864 abortion ban \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 8690\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 9\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m -9.51616557526609\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m5981\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mArizona state House passes bill to repeal 1864 abortion ban \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 5981\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 4\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m -9.841645168493953\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m9443\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0mArizona Gov. Katie Hobbs signs repeal of 1864 abortion ban \\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 9443\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m 6\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m \\u001b[0m\\u001b[0m\\u001b[0m -9.841645168493953\\u001b[0m \\u001b[0m\\u001b[0m│\\u001b[0m\\u001b[0m\\n\",\n \"\\u001b[0m\\u001b[0m\\u001b[0m└\\u001b[0m\\u001b[0m──────\\u001b[0m\\u001b[0m┴\\u001b[0m\\u001b[0m─────────────────────────────────────────────────────────────────────────────────\\u001b[0m\\u001b[0m┴\\u001b[0m\\u001b[0m───────\\u001b[0m\\u001b[0m┴\\u001b[0m\\u001b[0m─────────────────\\u001b[0m\\u001b[0m┴\\u001b[0m\\u001b[0m─────────────────────\\u001b[0m\\u001b[0m┘\\n\",\n \"\\u001b[0m\\u001b[0m\"\n ]\n },\n \"execution_count\": 18,\n \"metadata\": {},\n \"output_type\": \"execute_result\"\n }\n ],\n \"source\": [\n \".param set query abortion ban\\n\",\n \".param set k 10\\n\",\n \"\\n\",\n \"\\n\",\n \"with fts_matches as (\\n\",\n \" select\\n\",\n \" rowid,\\n\",\n \" row_number() over (order by rank) as fts_rank_number,\\n\",\n \" rank as score\\n\",\n \" from fts_articles\\n\",\n \" where headline match :query\\n\",\n \" limit :k\\n\",\n \"),\\n\",\n \"final as (\\n\",\n \" select\\n\",\n \" articles.id,\\n\",\n \" articles.headline,\\n\",\n \" fts_matches.*\\n\",\n \" from fts_matches\\n\",\n \" left join articles on articles.rowid = fts_matches.rowid\\n\",\n \" order by vec_distance_cosine(lembed(:query), lembed(articles.headline))\\n\",\n \")\\n\",\n \"select * from final;\\n\",\n \"\\n\"\n ]\n }\n ],\n \"metadata\": {\n \"kernelspec\": {\n \"display_name\": \"Solite\",\n \"language\": \"sql\",\n \"name\": \"solite\"\n },\n \"language_info\": {\n \"file_extension\": \".sql\",\n \"mimetype\": \"text/x.sqlite\",\n \"name\": \"sql\",\n \"nb_converter\": \"script\",\n \"pygments_lexer\": \"sql\",\n \"version\": \"TODO\"\n }\n },\n \"nbformat\": 4,\n \"nbformat_minor\": 2\n}\n" }, { "path": "examples/nbc-headlines/Makefile", "content": "all-MiniLM-L6-v2.e4ce9877.q8_0.gguf:\n\tcurl -L -o $@ https://huggingface.co/asg017/sqlite-lembed-model-examples/resolve/main/all-MiniLM-L6-v2/all-MiniLM-L6-v2.e4ce9877.q8_0.gguf\n" }, { "path": "examples/nbc-headlines/README.md", "content": "- `headlines-2024.db`\n - 14.5k rows\n - 4.4MB\n\n" }, { "path": "examples/python-recipes/openai-sample.py", "content": "# pip install openai sqlite-vec\n\nfrom openai import OpenAI\nimport sqlite3\nimport sqlite_vec\nimport struct\nfrom typing import List\n\n\ndef serialize(vector: List[float]) -> bytes:\n \"\"\"serializes a list of floats into a compact \"raw bytes\" format\"\"\"\n return struct.pack(\"%sf\" % len(vector), *vector)\n\n\nsentences = [\n \"Capri-Sun is a brand of juice concentrate–based drinks manufactured by the German company Wild and regional licensees.\",\n \"George V was King of the United Kingdom and the British Dominions, and Emperor of India, from 6 May 1910 until his death in 1936.\",\n \"Alaqua Cox is a Native American (Menominee) actress.\",\n \"Shohei Ohtani is a Japanese professional baseball pitcher and designated hitter for the Los Angeles Dodgers of Major League Baseball.\",\n \"Tamarindo, also commonly known as agua de tamarindo, is a non-alcoholic beverage made of tamarind, sugar, and water.\",\n]\n\n\nclient = OpenAI()\n\n# change ':memory:' to a filepath to persist data\ndb = sqlite3.connect(\":memory:\")\ndb.enable_load_extension(True)\nsqlite_vec.load(db)\ndb.enable_load_extension(False)\n\ndb.execute(\n \"\"\"\n CREATE TABLE sentences(\n id INTEGER PRIMARY KEY,\n sentence TEXT\n );\n \"\"\"\n)\n\nwith db:\n for i, sentence in enumerate(sentences):\n db.execute(\"INSERT INTO sentences(id, sentence) VALUES(?, ?)\", [i, sentence])\n\ndb.execute(\n \"\"\"\n CREATE VIRTUAL TABLE vec_sentences USING vec0(\n id INTEGER PRIMARY KEY,\n sentence_embedding FLOAT[1536]\n );\n \"\"\"\n)\n\n\nwith db:\n sentence_rows = db.execute(\"SELECT id, sentence FROM sentences\").fetchall()\n response = client.embeddings.create(\n input=[row[1] for row in sentence_rows], model=\"text-embedding-3-small\"\n )\n for (id, _), embedding in zip(sentence_rows, response.data):\n db.execute(\n \"INSERT INTO vec_sentences(id, sentence_embedding) VALUES(?, ?)\",\n [id, serialize(embedding.embedding)],\n )\n\n\nquery = \"fruity liquids\"\nquery_embedding = (\n client.embeddings.create(input=query, model=\"text-embedding-3-small\")\n .data[0]\n .embedding\n)\n\nresults = db.execute(\n \"\"\"\n SELECT\n vec_sentences.id,\n distance,\n sentence\n FROM vec_sentences\n LEFT JOIN sentences ON sentences.id = vec_sentences.id\n WHERE sentence_embedding MATCH ?\n AND k = 3\n ORDER BY distance\n \"\"\",\n [serialize(query_embedding)],\n).fetchall()\n\nfor row in results:\n print(row)\n" }, { "path": "examples/simple-bun/.gitignore", "content": "node_modules/\nbun.lockb\n" }, { "path": "examples/simple-bun/demo.ts", "content": "import { Database } from \"bun:sqlite\";\nDatabase.setCustomSQLite(\"/usr/local/opt/sqlite3/lib/libsqlite3.dylib\");\n\nconst db = new Database(\":memory:\");\n//sqliteVec.load(db);\ndb.loadExtension(\"../../dist/vec0\");\n\nconst { sqlite_version, vec_version } = db\n .prepare(\n \"select sqlite_version() as sqlite_version, vec_version() as vec_version;\",\n )\n .get();\n\nconsole.log(`sqlite_version=${sqlite_version}, vec_version=${vec_version}`);\n\nconst items = [\n [1, [0.1, 0.1, 0.1, 0.1]],\n [2, [0.2, 0.2, 0.2, 0.2]],\n [3, [0.3, 0.3, 0.3, 0.3]],\n [4, [0.4, 0.4, 0.4, 0.4]],\n [5, [0.5, 0.5, 0.5, 0.5]],\n];\nconst query = [0.3, 0.3, 0.3, 0.3];\n\ndb.exec(\"CREATE VIRTUAL TABLE vec_items USING vec0(embedding float[4])\");\n\nconst insertStmt = db.prepare(\n \"INSERT INTO vec_items(rowid, embedding) VALUES (?, vec_f32(?))\",\n);\n\nconst insertVectors = db.transaction((items) => {\n for (const [id, vector] of items) {\n insertStmt.run(BigInt(id), new Float32Array(vector));\n }\n});\n\ninsertVectors(items);\n\nconst rows = db\n .prepare(\n `\n SELECT\n rowid,\n distance\n FROM vec_items\n WHERE embedding MATCH ?\n ORDER BY distance\n LIMIT 3\n`,\n )\n .all(new Float32Array(query));\n\nconsole.log(rows);\n" }, { "path": "examples/simple-bun/package.json", "content": "{\n \"name\": \"simple-bun\",\n \"module\": \"index.ts\",\n \"type\": \"module\",\n \"dependencies\": {\n \"sqlite-vec\": \"latest\"\n }\n}\n" }, { "path": "examples/simple-c/.gitignore", "content": "demo\n" }, { "path": "examples/simple-c/Makefile", "content": "demo: demo.c\n\tgcc \\\n\t\t-O3 -DSQLITE_CORE \\\n\t\t-I../../ -I../../vendor \\\n\t\tdemo.c ../../sqlite-vec.c ../../vendor/sqlite3.c \\\n\t\t-o $@\n" }, { "path": "examples/simple-c/demo.c", "content": "#include \"sqlite3.h\"\n#include \"sqlite-vec.h\"\n#include \n#include \n#include \n\nint main(int argc, char *argv[]) {\n int rc = SQLITE_OK;\n sqlite3 *db;\n sqlite3_stmt *stmt;\n\n rc = sqlite3_auto_extension((void (*)())sqlite3_vec_init);\n assert(rc == SQLITE_OK);\n\n rc = sqlite3_open(\":memory:\", &db);\n assert(rc == SQLITE_OK);\n\n rc = sqlite3_prepare_v2(db, \"SELECT sqlite_version(), vec_version()\", -1, &stmt, NULL);\n assert(rc == SQLITE_OK);\n\n rc = sqlite3_step(stmt);\n printf(\"sqlite_version=%s, vec_version=%s\\n\", sqlite3_column_text(stmt, 0), sqlite3_column_text(stmt, 1));\n sqlite3_finalize(stmt);\n\n static const struct {\n sqlite3_int64 id;\n float vector[4];\n } items[] = {\n {1, {0.1, 0.1, 0.1, 0.1}},\n {2, {0.2, 0.2, 0.2, 0.2}},\n {3, {0.3, 0.3, 0.3, 0.3}},\n {4, {0.4, 0.4, 0.4, 0.4}},\n {5, {0.5, 0.5, 0.5, 0.5}},\n };\n float query[4] = {0.3, 0.3, 0.3, 0.3};\n\n\n rc = sqlite3_prepare_v2(db, \"CREATE VIRTUAL TABLE vec_items USING vec0(embedding float[4])\", -1, &stmt, NULL);\n assert(rc == SQLITE_OK);\n rc = sqlite3_step(stmt);\n assert(rc == SQLITE_DONE);\n sqlite3_finalize(stmt);\n\n rc = sqlite3_exec(db, \"BEGIN\", NULL, NULL, NULL);\n assert(rc == SQLITE_OK);\n rc = sqlite3_prepare_v2(db, \"INSERT INTO vec_items(rowid, embedding) VALUES (?, ?)\", -1, &stmt, NULL);\n assert(rc == SQLITE_OK);\n for (unsigned long i = 0; i < sizeof(items) / sizeof(items[0]); i++) {\n sqlite3_bind_int64(stmt, 1, items[i].id);\n sqlite3_bind_blob(stmt, 2, items[i].vector, sizeof(items[i].vector), SQLITE_STATIC);\n rc = sqlite3_step(stmt);\n assert(rc == SQLITE_DONE);\n sqlite3_reset(stmt);\n }\n sqlite3_finalize(stmt);\n rc = sqlite3_exec(db, \"COMMIT\", NULL, NULL, NULL);\n assert(rc == SQLITE_OK);\n\n rc = sqlite3_prepare_v2(db,\n \"SELECT \"\n \" rowid, \"\n \" distance \"\n \"FROM vec_items \"\n \"WHERE embedding MATCH ?1 \"\n \"ORDER BY distance \"\n \"LIMIT 3 \"\n , -1, &stmt, NULL);\n assert(rc == SQLITE_OK);\n\n sqlite3_bind_blob(stmt, 1, query, sizeof(query), SQLITE_STATIC);\n\n while(1) {\n rc = sqlite3_step(stmt);\n if(rc == SQLITE_DONE) break;\n assert(rc==SQLITE_ROW);\n sqlite3_int64 rowid = sqlite3_column_int64(stmt, 0);\n double distance = sqlite3_column_double(stmt, 1);\n printf(\"rowid=%lld distance=%f\\n\", rowid, distance);\n }\n sqlite3_finalize(stmt);\n sqlite3_close(db);\n return 0;\n}\n" }, { "path": "examples/simple-deno/demo.ts", "content": "import { Database } from \"jsr:@db/sqlite@0.11\";\nimport * as sqliteVec from \"npm:sqlite-vec@0.0.1-alpha.9\";\n\nconst db = new Database(\":memory:\");\ndb.enableLoadExtension = true;\nsqliteVec.load(db);\ndb.enableLoadExtension = false;\n\nconst [sqlite_version, vec_version] = db\n .prepare(\"select sqlite_version(), vec_version()\")\n .value<[string, string]>()!;\nconsole.log(`sqlite_version=${sqlite_version}, vec_version=${vec_version}`);\n\nconst items = [\n [1, [0.1, 0.1, 0.1, 0.1]],\n [2, [0.2, 0.2, 0.2, 0.2]],\n [3, [0.3, 0.3, 0.3, 0.3]],\n [4, [0.4, 0.4, 0.4, 0.4]],\n [5, [0.5, 0.5, 0.5, 0.5]],\n];\nconst query = [0.3, 0.3, 0.3, 0.3];\n\ndb.exec(\"CREATE VIRTUAL TABLE vec_items USING vec0(embedding float[4])\");\n\nconst insertStmt = db.prepare(\n \"INSERT INTO vec_items(rowid, embedding) VALUES (?, ?)\"\n);\n\nconst insertVectors = db.transaction((items) => {\n for (const [id, vector] of items) {\n insertStmt.run(BigInt(id), new Uint8Array(new Float32Array(vector).buffer));\n }\n});\n\ninsertVectors(items);\n\nconst rows = db\n .prepare(\n `\n SELECT\n rowid,\n distance\n FROM vec_items\n WHERE embedding MATCH ?\n ORDER BY distance\n LIMIT 5\n`\n )\n .all([new Uint8Array(new Float32Array(query).buffer)]);\n\nconsole.log(rows);\n\ndb.close();\n" }, { "path": "examples/simple-go-cgo/.gitignore", "content": "demo\n" }, { "path": "examples/simple-go-cgo/Makefile", "content": "demo: demo.go go.mod go.sum\n\tgo build -o $@\n" }, { "path": "examples/simple-go-cgo/demo.go", "content": "package main\n\nimport (\n\t\"database/sql\"\n\t\"fmt\"\n\t\"log\"\n\n\tsqlite_vec \"github.com/asg017/sqlite-vec-go-bindings/cgo\"\n\t_ \"github.com/mattn/go-sqlite3\"\n)\n\nfunc main() {\n\tsqlite_vec.Auto()\n\tdb, err := sql.Open(\"sqlite3\", \":memory:\")\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\tdefer db.Close()\n\n\tvar sqliteVersion string\n\tvar vecVersion string\n\terr = db.QueryRow(\"select sqlite_version(), vec_version()\").Scan(&sqliteVersion, &vecVersion)\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\tfmt.Printf(\"sqlite_version=%s, vec_version=%s\\n\", sqliteVersion, vecVersion)\n\n\t_, err = db.Exec(\"CREATE VIRTUAL TABLE vec_items USING vec0(embedding float[4])\")\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\n\titems := map[int][]float32{\n\t\t1: {0.1, 0.1, 0.1, 0.1},\n\t\t2: {0.2, 0.2, 0.2, 0.2},\n\t\t3: {0.3, 0.3, 0.3, 0.3},\n\t\t4: {0.4, 0.4, 0.4, 0.4},\n\t\t5: {0.5, 0.5, 0.5, 0.5},\n\t}\n\tq := []float32{0.3, 0.3, 0.3, 0.3}\n\n\tfor id, values := range items {\n\t\tv, err := sqlite_vec.SerializeFloat32(values)\n\t\tif err != nil {\n\t\t\tlog.Fatal(err)\n\t\t}\n\t\t_, err = db.Exec(\"INSERT INTO vec_items(rowid, embedding) VALUES (?, ?)\", id, v)\n\t\tif err != nil {\n\t\t\tlog.Fatal(err)\n\t\t}\n\t}\n\n\tquery, err := sqlite_vec.SerializeFloat32(q)\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\n\trows, err := db.Query(`\n\t\tSELECT\n\t\t\trowid,\n\t\t\tdistance\n\t\tFROM vec_items\n\t\tWHERE embedding MATCH ?\n\t\tORDER BY distance\n\t\tLIMIT 3\n\t`, query)\n\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\n\tfor rows.Next() {\n\t\tvar rowid int64\n\t\tvar distance float64\n\t\terr = rows.Scan(&rowid, &distance)\n\t\tif err != nil {\n\t\t\tlog.Fatal(err)\n\t\t}\n\t\tfmt.Printf(\"rowid=%d, distance=%f\\n\", rowid, distance)\n\t}\n\terr = rows.Err()\n\tif err != nil {\n\t\tlog.Fatal((err))\n\t}\n\n}\n" }, { "path": "examples/simple-go-cgo/go.mod", "content": "module github.com/asg017/sqlite-vec/examples/go\n\ngo 1.22.5\n\nrequire github.com/mattn/go-sqlite3 v1.14.22\n\nrequire github.com/asg017/sqlite-vec-go-bindings v0.0.1-alpha.36 // indirect\n" }, { "path": "examples/simple-go-cgo/go.sum", "content": "github.com/asg017/sqlite-vec-go-bindings v0.0.1-alpha.36 h1:FMGkKAA7nZL8gr/dvIx1uc54J3v2gbLVa+mLqZDCvjk=\ngithub.com/asg017/sqlite-vec-go-bindings v0.0.1-alpha.36/go.mod h1:A8+cTt/nKFsYCQF6OgzSNpKZrzNo5gQsXBTfsXHXY0Q=\ngithub.com/asg017/sqlite-vec/bindings/go/cgo v0.0.0-20240511043328-3d763f499859 h1:6jeFy/tSnyNJUrTHoIaFTYkjrHtwVAojvCGkr9G8d4o=\ngithub.com/asg017/sqlite-vec/bindings/go/cgo v0.0.0-20240511043328-3d763f499859/go.mod h1:Go89G54PaautWRwxvAa1fmKeYoSuUyIvSYpvlfXQaNU=\ngithub.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=\ngithub.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=\n" }, { "path": "examples/simple-go-ncruces/.gitignore", "content": "demo\n*.wasm\n" }, { "path": "examples/simple-go-ncruces/Makefile", "content": "\ndemo: demo.go\n\tgo build -o $@ $<\n\nclean:\n\trm demo\n\n.PHONY: clean\n" }, { "path": "examples/simple-go-ncruces/demo.go", "content": "package main\n\nimport (\n\t_ \"embed\"\n\t\"log\"\n\n\tsqlite_vec \"github.com/asg017/sqlite-vec-go-bindings/ncruces\"\n\t\"github.com/ncruces/go-sqlite3\"\n)\n\nfunc main() {\n\tdb, err := sqlite3.Open(\":memory:\")\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\n\tstmt, _, err := db.Prepare(`SELECT sqlite_version(), vec_version()`)\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\n\tstmt.Step()\n\n\tlog.Printf(\"sqlite_version=%s, vec_version=%s\\n\", stmt.ColumnText(0), stmt.ColumnText(1))\n\tstmt.Close()\n\n\n\terr = db.Exec(\"CREATE VIRTUAL TABLE vec_items USING vec0(embedding float[4])\")\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\titems := map[int][]float32{\n\t\t1: {0.1, 0.1, 0.1, 0.1},\n\t\t2: {0.2, 0.2, 0.2, 0.2},\n\t\t3: {0.3, 0.3, 0.3, 0.3},\n\t\t4: {0.4, 0.4, 0.4, 0.4},\n\t\t5: {0.5, 0.5, 0.5, 0.5},\n\t}\n\tq := []float32{0.3, 0.3, 0.3, 0.3}\n\n\tstmt, _, err = db.Prepare(\"INSERT INTO vec_items(rowid, embedding) VALUES (?, ?)\")\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\n\tfor id, values := range items {\n\t\tv, err := sqlite_vec.SerializeFloat32(values)\n\t\tif err != nil {\n\t\t\tlog.Fatal(err)\n\t\t}\n\t\tstmt.BindInt(1, id)\n\t\tstmt.BindBlob(2, v)\n\t\terr = stmt.Exec()\n\t\tif err != nil {\n\t\t\tlog.Fatal(err)\n\t\t}\n\t\tstmt.Reset()\n\t}\n\tstmt.Close()\n\n\n\n\tstmt, _, err = db.Prepare(`\n\t\tSELECT\n\t\t\trowid,\n\t\t\tdistance\n\t\tFROM vec_items\n\t\tWHERE embedding MATCH ?\n\t\tORDER BY distance\n\t\tLIMIT 3\n\t`);\n\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\n\tquery, err := sqlite_vec.SerializeFloat32(q)\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\tstmt.BindBlob(1, query)\n\n\tfor stmt.Step() {\n\t\trowid := stmt.ColumnInt64(0)\n\t\tdistance := stmt.ColumnFloat(1)\n\t\tlog.Printf(\"rowid=%d, distance=%f\\n\", rowid, distance)\n\t}\n\tif err := stmt.Err(); err != nil {\n\t\tlog.Fatal(err)\n\t}\n\n\terr = stmt.Close()\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\n\terr = db.Close()\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n}\n" }, { "path": "examples/simple-go-ncruces/go.mod", "content": "module asg017.com/ex1\n\ngo 1.22.5\n\nrequire (\n\tgithub.com/asg017/sqlite-vec-go-bindings v0.0.1-alpha.37\n\tgithub.com/ncruces/go-sqlite3 v0.17.2-0.20240711235451-21de85e849b7\n)\n\nrequire (\n\tgithub.com/ncruces/julianday v1.0.0 // indirect\n\tgithub.com/tetratelabs/wazero v1.7.3 // indirect\n\tgolang.org/x/sys v0.22.0 // indirect\n)\n" }, { "path": "examples/simple-go-ncruces/go.sum", "content": "github.com/asg017/sqlite-vec-go-bindings v0.0.1-alpha.37 h1:Gz6YkDCs60k5VwbBPKDfAPPeIBcuaN3qriAozAaIIZI=\ngithub.com/asg017/sqlite-vec-go-bindings v0.0.1-alpha.37/go.mod h1:A8+cTt/nKFsYCQF6OgzSNpKZrzNo5gQsXBTfsXHXY0Q=\ngithub.com/ncruces/go-sqlite3 v0.17.2-0.20240711235451-21de85e849b7 h1:ssM02uUFDfz0V2TMg2du2BjbW9cpOhFJK0kpDN+X768=\ngithub.com/ncruces/go-sqlite3 v0.17.2-0.20240711235451-21de85e849b7/go.mod h1:FnCyui8SlDoL0mQZ5dTouNo7s7jXS0kJv9lBt1GlM9w=\ngithub.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt7M=\ngithub.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g=\ngithub.com/tetratelabs/wazero v1.7.3 h1:PBH5KVahrt3S2AHgEjKu4u+LlDbbk+nsGE3KLucy6Rw=\ngithub.com/tetratelabs/wazero v1.7.3/go.mod h1:ytl6Zuh20R/eROuyDaGPkp82O9C/DJfXAwJfQ3X6/7Y=\ngolang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=\ngolang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=\ngolang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=\ngolang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=\n" }, { "path": "examples/simple-node/.gitignore", "content": "node_modules/\npackage-lock.json\n" }, { "path": "examples/simple-node/demo.mjs", "content": "import * as sqliteVec from \"sqlite-vec\";\nimport Database from \"better-sqlite3\";\n\nconst db = new Database(\":memory:\");\nsqliteVec.load(db);\n\nconst { sqlite_version, vec_version } = db\n .prepare(\n \"select sqlite_version() as sqlite_version, vec_version() as vec_version;\",\n )\n .get();\n\nconsole.log(`sqlite_version=${sqlite_version}, vec_version=${vec_version}`);\n\nconst items = [\n [1, [0.1, 0.1, 0.1, 0.1]],\n [2, [0.2, 0.2, 0.2, 0.2]],\n [3, [0.3, 0.3, 0.3, 0.3]],\n [4, [0.4, 0.4, 0.4, 0.4]],\n [5, [0.5, 0.5, 0.5, 0.5]],\n];\nconst query = [0.3, 0.3, 0.3, 0.3];\n\ndb.exec(\"CREATE VIRTUAL TABLE vec_items USING vec0(embedding float[4])\");\n\nconst insertStmt = db.prepare(\n \"INSERT INTO vec_items(rowid, embedding) VALUES (?, ?)\",\n);\n\nconst insertVectors = db.transaction((items) => {\n for (const [id, vector] of items) {\n insertStmt.run(BigInt(id), new Float32Array(vector));\n }\n});\n\ninsertVectors(items);\n\nconst rows = db\n .prepare(\n `\n SELECT\n rowid,\n distance\n FROM vec_items\n WHERE embedding MATCH ?\n ORDER BY distance\n LIMIT 3\n`,\n )\n .all(new Float32Array(query));\n\nconsole.log(rows);\n" }, { "path": "examples/simple-node/package.json", "content": "{\n \"name\": \"node\",\n \"version\": \"1.0.0\",\n \"description\": \"\",\n \"main\": \"index.js\",\n \"scripts\": {\n \"test\": \"echo \\\"Error: no test specified\\\" && exit 1\"\n },\n \"keywords\": [],\n \"author\": \"\",\n \"license\": \"ISC\",\n \"dependencies\": {\n \"better-sqlite3\": \"^9.6.0\",\n \"sqlite-vec\": \"latest\"\n }\n}\n" }, { "path": "examples/simple-node2/.gitignore", "content": "node_modules/\npackage-lock.json" }, { "path": "examples/simple-node2/demo.mjs", "content": "/**\n * This demo Node.js script shows how you can use sqlite-vec with\n * the new builtin node:sqlite module.\n * Note that this requires Node v23.5.0 or above.\n */\nimport { DatabaseSync } from \"node:sqlite\";\nimport * as sqliteVec from \"sqlite-vec\";\n\n// allExtension is required to enable extension support\nconst db = new DatabaseSync(\":memory:\", { allowExtension: true });\nsqliteVec.load(db);\n\nconst { sqlite_version, vec_version } = db\n .prepare(\n \"select sqlite_version() as sqlite_version, vec_version() as vec_version;\",\n )\n .get();\n\nconsole.log(`sqlite_version=${sqlite_version}, vec_version=${vec_version}`);\n\nconst items = [\n [1, [0.1, 0.1, 0.1, 0.1]],\n [2, [0.2, 0.2, 0.2, 0.2]],\n [3, [0.3, 0.3, 0.3, 0.3]],\n [4, [0.4, 0.4, 0.4, 0.4]],\n [5, [0.5, 0.5, 0.5, 0.5]],\n];\nconst query = [0.3, 0.3, 0.3, 0.3];\n\ndb.exec(\"CREATE VIRTUAL TABLE vec_items USING vec0(embedding float[4])\");\n\nconst insertStmt = db.prepare(\n \"INSERT INTO vec_items(rowid, embedding) VALUES (?, ?)\",\n);\n\n// TODO node:sqlite doesn't have `.transaction()` support yet\nfor (const [id, vector] of items) {\n // node:sqlite requires Uint8Array for BLOB values, so a bit awkward\n insertStmt.run(BigInt(id), new Uint8Array(new Float32Array(vector).buffer));\n}\n\nconst rows = db\n .prepare(\n `\n SELECT\n rowid,\n distance\n FROM vec_items\n WHERE embedding MATCH ?\n ORDER BY distance\n LIMIT 3\n`,\n )\n .all(new Uint8Array(new Float32Array(query).buffer));\n\nconsole.log(rows);\n" }, { "path": "examples/simple-node2/package.json", "content": "{\n \"name\": \"simple-node2\",\n \"version\": \"1.0.0\",\n \"main\": \"demo.mjs\",\n \"engines\": {\n \"node\": \">=23.5.0\"\n },\n \"dependencies\": {\n \"sqlite-vec\": \"latest\"\n }\n}\n" }, { "path": "examples/simple-node2/tmp.mjs", "content": "import { DatabaseSync } from \"node:sqlite\";\nimport * as sqliteVec from \"sqlite-vec\";\n\nconst db = new DatabaseSync(\":memory:\", { allowExtension: true });\nsqliteVec.load(db);\n\nconst embedding = new Float32Array([0.1, 0.2, 0.3, 0.4]);\nconst { result } = db\n .prepare(\"select vec_length(?) as result\")\n .get(new Uint8Array(embedding.buffer));\n\nconsole.log(result); // 4" }, { "path": "examples/simple-python/.gitignore", "content": ".venv\n" }, { "path": "examples/simple-python/demo.py", "content": "import sqlite3\nimport sqlite_vec\n\nfrom typing import List\nimport struct\n\n\ndef serialize_f32(vector: List[float]) -> bytes:\n \"\"\"serializes a list of floats into a compact \"raw bytes\" format\"\"\"\n return struct.pack(\"%sf\" % len(vector), *vector)\n\n\ndb = sqlite3.connect(\":memory:\")\ndb.enable_load_extension(True)\nsqlite_vec.load(db)\ndb.enable_load_extension(False)\n\nsqlite_version, vec_version = db.execute(\n \"select sqlite_version(), vec_version()\"\n).fetchone()\nprint(f\"sqlite_version={sqlite_version}, vec_version={vec_version}\")\n\nitems = [\n (1, [0.1, 0.1, 0.1, 0.1]),\n (2, [0.2, 0.2, 0.2, 0.2]),\n (3, [0.3, 0.3, 0.3, 0.3]),\n (4, [0.4, 0.4, 0.4, 0.4]),\n (5, [0.5, 0.5, 0.5, 0.5]),\n]\nquery = [0.3, 0.3, 0.3, 0.3]\n\ndb.execute(\"CREATE VIRTUAL TABLE vec_items USING vec0(embedding float[4])\")\n\nwith db:\n for item in items:\n db.execute(\n \"INSERT INTO vec_items(rowid, embedding) VALUES (?, ?)\",\n [item[0], serialize_f32(item[1])],\n )\n\nrows = db.execute(\n \"\"\"\n SELECT\n rowid,\n distance\n FROM vec_items\n WHERE embedding MATCH ?\n ORDER BY distance\n LIMIT 3\n \"\"\",\n [serialize_f32(query)],\n).fetchall()\n\nprint(rows)\n" }, { "path": "examples/simple-python/requirements.txt", "content": "sqlite-vec\n" }, { "path": "examples/simple-ruby/.gitignore", "content": "Gemfile.lock\n" }, { "path": "examples/simple-ruby/Gemfile", "content": "source 'https://rubygems.org'\n\nruby '>= 3.0'\n\ngem 'sqlite3', '~> 2.0', '>= 2.0.1'\ngem 'sqlite-vec'\n" }, { "path": "examples/simple-ruby/demo.rb", "content": "require 'sqlite3'\nrequire 'sqlite_vec'\n\n\ndb = SQLite3::Database.new(':memory:')\ndb.enable_load_extension(true)\nSqliteVec.load(db)\ndb.enable_load_extension(false)\n\nsqlite_version, vec_version = db.execute(\"select sqlite_version(), vec_version()\").first\nputs \"sqlite_version=#{sqlite_version}, vec_version=#{vec_version}\"\n\ndb.execute(\"CREATE VIRTUAL TABLE vec_items USING vec0(embedding float[4])\")\n\nitems = [\n [1, [0.1, 0.1, 0.1, 0.1]],\n [2, [0.2, 0.2, 0.2, 0.2]],\n [3, [0.3, 0.3, 0.3, 0.3]],\n [4, [0.4, 0.4, 0.4, 0.4]],\n [5, [0.5, 0.5, 0.5, 0.5]],\n]\n\ndb.transaction do\n items.each do |item|\n db.execute(\"INSERT INTO vec_items(rowid, embedding) VALUES (?, ?)\", [item[0], item[1].pack(\"f*\")])\n end\nend\n\nquery = [0.3, 0.3, 0.3, 0.3]\nrows = db.execute(<<-SQL, [query.pack(\"f*\")])\n SELECT\n rowid,\n distance\n FROM vec_items\n WHERE embedding MATCH ?\n ORDER BY distance\n LIMIT 3\nSQL\n\nputs rows\n" }, { "path": "examples/simple-rust/.gitignore", "content": "target/\nCargo.lock\n" }, { "path": "examples/simple-rust/Cargo.toml", "content": "[package]\nname = \"sqlite-vec-demo\"\nedition = \"2021\"\n\n[dependencies]\nsqlite-vec={version=\"0.0.1-alpha.7\"}\nrusqlite = {version=\"0.31.0\", features=[\"bundled\"]}\nzerocopy = \"0.7.33\"\n\n[[bin]]\nname=\"demo\"\npath=\"demo.rs\"\n" }, { "path": "examples/simple-rust/demo.rs", "content": "use rusqlite::{ffi::sqlite3_auto_extension, Connection, Result};\nuse sqlite_vec::sqlite3_vec_init;\nuse zerocopy::AsBytes;\n\nfn main() -> Result<()> {\n unsafe {\n sqlite3_auto_extension(Some(std::mem::transmute(sqlite3_vec_init as *const ())));\n }\n\n let db = Connection::open_in_memory()?;\n let v: Vec = vec![0.1, 0.2, 0.3];\n\n let (sqlite_version, vec_version, x): (String, String, String) = db.query_row(\n \"select sqlite_version(), vec_version(), vec_to_json(?)\",\n &[v.as_bytes()],\n |x| Ok((x.get(0)?, x.get(1)?, x.get(2)?)),\n )?;\n\n println!(\"sqlite_version={sqlite_version}, vec_version={vec_version}\");\n\n let items: Vec<(usize, Vec)> = vec![\n (1, vec![0.1, 0.1, 0.1, 0.1]),\n (2, vec![0.2, 0.2, 0.2, 0.2]),\n (3, vec![0.3, 0.3, 0.3, 0.3]),\n (4, vec![0.4, 0.4, 0.4, 0.4]),\n (5, vec![0.5, 0.5, 0.5, 0.5]),\n ];\n println!(\"{x}\");\n\n db.execute(\n \"CREATE VIRTUAL TABLE vec_items USING vec0(embedding float[4])\",\n [],\n )?;\n let mut stmt = db.prepare(\"INSERT INTO vec_items(rowid, embedding) VALUES (?, ?)\")?;\n for item in items {\n stmt.execute(rusqlite::params![item.0, item.1.as_bytes()])?;\n }\n\n let query: Vec = vec![0.3, 0.3, 0.3, 0.3];\n let result: Vec<(i64, f64)> = db\n .prepare(\n r\"\n SELECT\n rowid,\n distance\n FROM vec_items\n WHERE embedding MATCH ?1\n ORDER BY distance\n LIMIT 3\n \",\n )?\n .query_map([query.as_bytes()], |r| Ok((r.get(0)?, r.get(1)?)))?\n .collect::, _>>()?;\n println!(\"{:?}\", result);\n Ok(())\n}\n" }, { "path": "examples/simple-sqlite/demo.sql", "content": ".load ../../dist/vec0\n.mode table\n.header on\n\nselect sqlite_version(), vec_version();\n\nCREATE VIRTUAL TABLE vec_items USING vec0(embedding float[4]);\n\nINSERT INTO vec_items(rowid, embedding)\n select\n value ->> 0,\n value ->> 1\n from json_each('[\n [1, [0.1, 0.1, 0.1, 0.1]],\n [2, [0.2, 0.2, 0.2, 0.2]],\n [3, [0.3, 0.3, 0.3, 0.3]],\n [4, [0.4, 0.4, 0.4, 0.4]],\n [5, [0.5, 0.5, 0.5, 0.5]]\n ]');\n\nSELECT\n rowid,\n distance\nFROM vec_items\nWHERE embedding MATCH '[0.3, 0.3, 0.3, 0.3]'\nORDER BY distance\nLIMIT 3;\n" }, { "path": "examples/simple-wasm/index.html", "content": "\n \n

sqlite-vec demo/simple-wasm

\n\n
\n \n \n\n`\n" }, { "path": "examples/sqlite3-cli/README.md", "content": "# `sqlite-vec` statically compiled in the SQLite CLI\n\nYou can compile your own version of the `sqlite3` CLI with `sqlite-vec` builtin.\nThe process is not well documented, but the special `SQLITE_EXTRA_INIT` compile\noption can be used to \"inject\" code at initialization time. See the `Makefile`\nat the root of this project for some more info.\n\nThe `core_init.c` file here demonstrates auto-loading the `sqlite-vec`\nentrypoints at startup.\n" }, { "path": "examples/sqlite3-cli/core_init.c", "content": "#include \"sqlite3.h\"\n#include \"sqlite-vec.h\"\n#include \nint core_init(const char *dummy) {\n return sqlite3_auto_extension((void *)sqlite3_vec_init);\n}\n" }, { "path": "examples/wasm/README.md", "content": "# `sqlite-vec` statically compiled into WASM builds\n\nYou can compile your own version of SQLite's WASM build with `sqlite-vec`\nbuiltin. Dynamically loading SQLite extensions is not supported in the official\nWASM build yet, but you can statically compile extensions in. It's not well\ndocumented, but the `sqlite3_wasm_extra_init` option in the SQLite `ext/wasm`\nMakefile allows you to inject your own code at initialization time. See the\n`Makefile` at the room of the project for more info.\n\nThe `wasm.c` file here demonstrates auto-loading the `sqlite-vec` entrypoints at\nstartup.\n" }, { "path": "examples/wasm/wasm.c", "content": "#include \"sqlite3.h\"\n#include \"sqlite-vec.h\"\n\nint sqlite3_wasm_extra_init(const char * unused) {\n return sqlite3_auto_extension((void (*)(void)) sqlite3_vec_init);\n}\n" }, { "path": "reference.yaml", "content": "sections:\n constructors:\n title: Constructors\n desc: |\n SQL functions that \"construct\" vectors with different element types.\n\n Currently, only `float32`, `int8`, and `bit` vectors are supported.\n\n op:\n title: Operations\n desc: |\n Different operations and utilities for working with vectors.\n distance:\n title: Distance functions\n desc: Various algorithms to calculate distance between two vectors.\n quantization:\n title: Quantization\n desc: Various techniques to \"compress\" a vector by reducing precision and accuracy.\n numpy:\n title: \"NumPy Utilities\"\n desc: Functions to read data from or work with [NumPy arrays](https://numpy.org/doc/stable/reference/generated/numpy.array.html).\n meta:\n title: Meta\n desc: Helper functions to debug `sqlite-vec` installations.\n entrypoints:\n title: Entrypoints\n desc: All the named entrypoints that load in different `sqlite-vec` functions and options.\n# vec0:\n# title: \"vec0 Virtual Table\"\n# desc: TODO\nmeta:\n vec_version:\n params: []\n desc: Returns a version string of the current `sqlite-vec` installation.\n example: select vec_version();\n vec_debug:\n params: []\n desc: Returns debugging information of the current `sqlite-vec` installation.\n example: select vec_debug();\nconstructors:\n vec_f32:\n params: [vector]\n desc: |\n Creates a float vector from a BLOB or JSON text. If a BLOB is provided,\n the length must be divisible by 4, as a float takes up 4 bytes of space each.\n\n The returned value is a BLOB with 4 bytes per element, with a special [subtype](https://www.sqlite.org/c3ref/result_subtype.html)\n of `223`.\n example:\n - select vec_f32('[.1, .2, .3, 4]');\n - select subtype(vec_f32('[.1, .2, .3, 4]'));\n - select vec_f32(X'AABBCCDD');\n - select vec_to_json(vec_f32(X'AABBCCDD'));\n - select vec_f32(X'AA');\n vec_int8:\n params: [vector]\n desc: |\n Creates a 8-bit integer vector from a BLOB or JSON text. If a BLOB is provided,\n the length must be divisible by 4, as a float takes up 4 bytes of space each.\n If JSON text is provided, each element must be an integer between -128 and 127 inclusive.\n\n The returned value is a BLOB with 1 byte per element, with a special [subtype](https://www.sqlite.org/c3ref/result_subtype.html)\n of `225`.\n\n example:\n - select vec_int8('[1, 2, 3, 4]');\n - select subtype(vec_int8('[1, 2, 3, 4]'));\n - select vec_int8(X'AABBCCDD');\n - select vec_to_json(vec_int8(X'AABBCCDD'));\n - select vec_int8('[999]');\n\n vec_bit:\n params: [vector]\n desc: |\n Creates a binary vector from a BLOB.\n\n The returned value is a BLOB with 1 byte per 8 elements, with a special [subtype](https://www.sqlite.org/c3ref/result_subtype.html)\n of `224`.\n example:\n - select vec_bit(X'F0');\n - select subtype(vec_bit(X'F0'));\n - select vec_to_json(vec_bit(X'F0'));\nop:\n vec_length:\n params: [vector]\n desc: |\n Returns the number of elements in the given vector.\n The vector can be `JSON`, `BLOB`, or the result of a [constructor function](#constructors).\n\n This function will return an error if `vector` is invalid.\n example:\n - select vec_length('[.1, .2]');\n - select vec_length(X'AABBCCDD');\n - select vec_length(vec_int8(X'AABBCCDD'));\n - select vec_length(vec_bit(X'AABBCCDD'));\n - select vec_length(X'CCDD');\n vec_type:\n params: [vector]\n desc: |\n Returns the name of the type of `vector` as text. One of `'float32'`, `'int8'`, or `'bit'`.\n\n This function will return an error if `vector` is invalid.\n example:\n - select vec_type('[.1, .2]');\n - select vec_type(X'AABBCCDD');\n - select vec_type(vec_int8(X'AABBCCDD'));\n - select vec_type(vec_bit(X'AABBCCDD'));\n - select vec_type(X'CCDD');\n vec_add:\n params: [a, b]\n desc: |\n Adds every element in vector `a` with vector `b`, returning a new vector `c`. Both vectors\n must be of the same type and same length. Only `float32` and `int8` vectors are supported.\n\n An error is raised if either `a` or `b` are invalid, or if they are not the same type or same length.\n\n See also [`vec_sub()`](#vec_sub).\n\n example:\n - |\n select vec_add(\n '[.1, .2, .3]',\n '[.4, .5, .6]'\n );\n - |\n select vec_to_json(\n vec_add(\n '[.1, .2, .3]',\n '[.4, .5, .6]'\n )\n );\n - |\n select vec_to_json(\n vec_add(\n vec_int8('[1, 2, 3]'),\n vec_int8('[4, 5, 6]')\n )\n );\n - select vec_add('[.1]', vec_int8('[1]'));\n - select vec_add(vec_bit(X'AA'), vec_bit(X'BB'));\n vec_sub:\n params: [a, b]\n desc: |\n Subtracts every element in vector `a` with vector `b`, returning a new vector `c`. Both vectors\n must be of the same type and same length. Only `float32` and `int8` vectors are supported.\n\n An error is raised if either `a` or `b` are invalid, or if they are not the same type or same length.\n\n See also [`vec_add()`](#vec_add).\n\n example:\n - |\n select vec_sub(\n '[.1, .2, .3]',\n '[.4, .5, .6]'\n );\n - |\n select vec_to_json(\n vec_sub(\n '[.1, .2, .3]',\n '[.4, .5, .6]'\n )\n );\n - |\n select vec_to_json(\n vec_sub(\n vec_int8('[1, 2, 3]'),\n vec_int8('[4, 5, 6]')\n )\n );\n - select vec_sub('[.1]', vec_int8('[1]'));\n - select vec_sub(vec_bit(X'AA'), vec_bit(X'BB'));\n vec_normalize:\n params: [vector]\n desc: |\n Performs L2 normalization on the given vector. Only float32 vectors are currently supported.\n\n Returns an error if the input is an invalid vector or not a float32 vector.\n example:\n - select vec_normalize('[2, 3, 1, -4]');\n - |\n select vec_to_json(\n vec_normalize('[2, 3, 1, -4]')\n );\n - |\n -- for matryoshka embeddings - slice then normalize\n select vec_to_json(\n vec_normalize(\n vec_slice('[2, 3, 1, -4]', 0, 2)\n )\n );\n vec_slice:\n params: [vector, start, end]\n desc: |\n Extract a subset of `vector` from the `start` element (inclusive) to the `end` element (exclusive). TODO check\n\n This is especially useful for [Matryoshka embeddings](#TODO), also known as \"adaptive length\" embeddings.\n Use with [`vec_normalize()`](#vec_normalize) to get proper results.\n\n Returns an error in the following conditions:\n - If `vector` is not a valid vector\n - If `start` is less than zero or greater than or equal to `end`\n - If `end` is greater than the length of `vector`, or less than or equal to `start`.\n - If `vector` is a bitvector, `start` and `end` must be divisible by 8.\n example:\n - select vec_slice('[1, 2,3, 4]', 0, 2);\n - |\n select vec_to_json(\n vec_slice('[1, 2,3, 4]', 0, 2)\n );\n - |\n select vec_to_json(\n vec_slice('[1, 2,3, 4]', 2, 4)\n );\n - |\n select vec_to_json(\n vec_slice('[1, 2,3, 4]', -1, 4)\n );\n - |\n select vec_to_json(\n vec_slice('[1, 2,3, 4]', 0, 5)\n );\n - |\n select vec_to_json(\n vec_slice('[1, 2,3, 4]', 0, 0)\n );\n vec_to_json:\n params: [vector]\n desc: |\n Represents a vector as JSON text. The input vector can be a vector BLOB or JSON text.\n\n Returns an error if `vector` is an invalid vector, or when memory cannot be allocated.\n example:\n - select vec_to_json(X'AABBCCDD');\n - select vec_to_json(vec_int8(X'AABBCCDD'));\n - select vec_to_json(vec_bit(X'AABBCCDD'));\n - select vec_to_json('[1,2,3,4]');\n - select vec_to_json('invalid');\n vec_each:\n params: [vector]\n desc: |\n A table function to iterate through every element in a vector. One row id returned per element in a vector.\n\n ```sql\n CREATE TABLE vec_each(\n rowid int, -- The\n vector HIDDEN -- input parameter: A well-formed vector value\n )\n ```\n\n Returns an error if `vector` is not a valid vector.\n example:\n - select rowid, value from vec_each('[1,2,3,4]');\n - select rowid, value from vec_each(X'AABBCCDD00112233');\n - select rowid, value from vec_each(vec_int8(X'AABBCCDD'));\n - select rowid, value from vec_each(vec_bit(X'F0'));\n\ndistance:\n vec_distance_L2:\n params: [a, b]\n desc: |\n Calculates the L2 euclidian distance between vectors `a` and `b`. Only valid for float32 or int8 vectors.\n\n Returns an error under the following conditions:\n - `a` or `b` are invalid vectors\n - `a` or `b` do not share the same vector element types (ex float32 or int8)\n - `a` or `b` are bit vectors. Use [`vec_distance_hamming()`](#vec_distance_hamming) for distance calculations between two bitvectors.\n - `a` or `b` do not have the same length.\n example:\n - select vec_distance_L2('[1, 1]', '[2, 2]');\n - select vec_distance_L2('[1, 1]', '[-2, -2]');\n - select vec_distance_L2('[1.1, 2.2, 3.3]', '[4.4, 5.5, 6.6]');\n - select vec_distance_L2(X'AABBCCDD', X'00112233');\n - select vec_distance_L2('[1, 1]', vec_int8('[2, 2]'));\n - select vec_distance_L2(vec_bit(X'AA'), vec_bit(X'BB'));\n vec_distance_cosine:\n params: [a, b]\n desc: |\n Calculates the cosine distance between vectors `a` and `b`. Only valid for float32 or int8 vectors.\n\n Returns an error under the following conditions:\n - `a` or `b` are invalid vectors\n - `a` or `b` do not share the same vector element types (ex float32 or int8)\n - `a` or `b` are bit vectors. Use [`vec_distance_hamming()`](#vec_distance_hamming) for distance calculations between two bitvectors.\n - `a` or `b` do not have the same length.\n example:\n - select vec_distance_cosine('[1, 1]', '[2, 2]');\n - select vec_distance_cosine('[1, 1]', '[-2, -2]');\n - select vec_distance_cosine('[1.1, 2.2, 3.3]', '[4.4, 5.5, 6.6]');\n - select vec_distance_cosine(X'AABBCCDD', X'00112233');\n - select vec_distance_cosine('[1, 1]', vec_int8('[2, 2]'));\n - select vec_distance_cosine(vec_bit(X'AA'), vec_bit(X'BB'));\n vec_distance_hamming:\n params: [a, b]\n desc: |\n Calculates the hamming distance between two bitvectors `a` and `b`. Only valid for bitvectors.\n\n Returns an error under the following conditions:\n - `a` or `b` are not bitvectors\n - `a` and `b` do not share the same length\n - Memory cannot be allocated\n example:\n - select vec_distance_hamming(vec_bit(X'00'), vec_bit(X'FF'));\n - select vec_distance_hamming(vec_bit(X'FF'), vec_bit(X'FF'));\n - select vec_distance_hamming(vec_bit(X'F0'), vec_bit(X'44'));\n - select vec_distance_hamming('[1, 1]', '[0, 0]');\n\nquantization:\n vec_quantize_binary:\n params: [vector]\n desc: |\n Quantize a float32 or int8 vector into a bitvector.\n For every element in the vector, a `1` is assigned to positive numbers and a `0` is assigned to negative numbers.\n These values are then packed into a bit vector.\n\n Returns an error if `vector` is invalid, or if `vector` is not a float32 or int8 vector.\n example:\n - select vec_quantize_binary('[1, 2, 3, 4, 5, 6, 7, 8]');\n - select vec_quantize_binary('[1, 2, 3, 4, -5, -6, -7, -8]');\n - select vec_quantize_binary('[-1, -2, -3, -4, -5, -6, -7, -8]');\n - select vec_quantize_binary('[-1, -2, -3, -4, -5, -6, -7, -8]');\n - select vec_quantize_binary(vec_int8(X'11223344'));\n - select vec_quantize_binary(vec_bit(X'FF'));\n vec_quantize_i8:\n params: [vector, \"[start]\", \"[end]\"]\n desc: x\n example: select 'todo';\nvec0:\n vec0:\n params: []\n desc: TODO\n example:\n - |\n create virtual table vec_items using vec0(\n contents_embedding float[4]\n );\n - |\n insert into vec_items(rowid, contents_embedding)\n values (1, '[1, 1, 1, 1]'),\n (2, '[2, 2, 2, 2]'),\n (3, '[3, 3, 3, 3]');\n" }, { "path": "scripts/progress.ts", "content": "const src = Deno.readTextFileSync(\"sqlite-vec.c\");\n\nfunction numOccuranges(rg) {\n return [...src.matchAll(rg)].length;\n}\nconst numAsserts = numOccuranges(/todo_assert/g);\nconst numComments = numOccuranges(/TODO/g);\nconst numHandles = numOccuranges(/todo\\(/g);\n\nconst realTodos = numOccuranges(/TODO\\(/g);\n\nconst numTotal = numAsserts + numComments + numHandles - realTodos;\n\nconsole.log(\"Number of todo_assert()'s: \", numAsserts);\nconsole.log('Number of \"// TODO\" comments: ', numComments);\nconsole.log(\"Number of todo panics: \", numHandles);\nconsole.log(\"Total TODOs: \", numTotal);\n\nconsole.log();\n\nconst TOTAL = 246; // as of e5b0f4c0c5 (2024-04-20)\nconst progress = (TOTAL - numTotal) / TOTAL;\nconst width = 60;\n\nconsole.log(\n \"▓\".repeat((progress < 0 ? 0 : progress) * width) +\n \"░\".repeat((1 - progress) * width) +\n ` (${TOTAL - numTotal}/${TOTAL})`,\n);\nconsole.log();\nconsole.log(\n `${(progress * 100.0).toPrecision(2)}% complete to sqlite-vec v0.1.0`,\n);\n" }, { "path": "scripts/publish-release.sh", "content": "#!/bin/bash\n\nset -euo pipefail xtrace\n\nif [[ -n $(git status --porcelain | grep -v VERSION | grep -v sqlite-dist.toml) ]]; then\n echo \"❌ There are other un-staged changes to the repository besides VERSION and sqlite-dist.toml\"\n exit 1\nfi\n\nVERSION=\"$(cat VERSION)\"\n\necho \"Publishing version v$VERSION...\"\n\nmake version\ngit add --all\ngit commit -m \"v$VERSION\"\ngit tag v$VERSION\ngit push origin main v$VERSION\n\nif grep -qE \"alpha|beta\" VERSION; then\n gh release create v$VERSION --title=v$VERSION --prerelease\nelse\n gh release create v$VERSION --title=v$VERSION\nfi\n\n\necho \"✅ Published! version v$VERSION\"\n" }, { "path": "scripts/vendor.sh", "content": "#!/bin/bash\nmkdir -p vendor\ncurl -o sqlite-amalgamation.zip https://www.sqlite.org/2024/sqlite-amalgamation-3450300.zip\nunzip -d\nunzip sqlite-amalgamation.zip\nmv sqlite-amalgamation-3450300/* vendor/\nrmdir sqlite-amalgamation-3450300\nrm sqlite-amalgamation.zip\n" }, { "path": "site/.gitignore", "content": "node_modules\n.vitepress/cache\n" }, { "path": "site/.vitepress/config.mts", "content": "import { DefaultTheme, defineConfig, HeadConfig } from \"vitepress\";\nimport { readFileSync } from \"node:fs\";\nimport { dirname, join } from \"node:path\";\nimport { fileURLToPath } from \"node:url\";\n\nconst PROJECT = \"sqlite-vec\";\nconst description = \"A vector search SQLite extension that runs anywhere!\";\n\nconst VERSION = readFileSync(\n join(dirname(fileURLToPath(import.meta.url)), \"..\", \"..\", \"VERSION\"),\n \"utf8\"\n);\n\nconst sqliteLanuage = JSON.parse(\n readFileSync(\n join(\n dirname(fileURLToPath(import.meta.url)),\n \"..\",\n \"sqlite.tmlanguage.json\"\n ),\n \"utf8\"\n )\n);\n\nfunction head(): HeadConfig[] {\n return [\n [\n \"link\",\n {\n rel: \"shortcut icon\",\n type: \"image/svg+xml\",\n href: \"./logo.light.svg\",\n },\n ],\n [\n \"script\",\n {\n defer: \"\",\n \"data-domain\": \"alexgarcia.xyz/sqlite-vec\",\n src: \"https://plausible.io/js/script.js\",\n },\n ],\n ];\n}\n\nconst guides = {\n text: \"Guides\",\n collapsed: true,\n items: [\n { text: \"Performance\", link: \"/guides/performance\" },\n\n {\n text: \"Vector operations\",\n items: [\n { text: \"Vector Arithmetic\", link: \"/guides/arithmetic\" },\n { text: \"Binary Quantization\", link: \"/guides/binary-quant\" },\n { text: \"Scalar Quantization\", link: \"/guides/scalar-quant\" },\n {\n text: \"Matryoshka Embeddings\",\n link: \"/guides/matryoshka\",\n },\n ],\n },\n\n /* {\n text: \"Build with sqlite-vec\",\n items: [\n { text: \"Semantic Search\", link: \"/guides/semantic-search\" },\n { text: \"Hybrid Search\", link: \"/guides/hybrid-search\" },\n { text: \"Retrival Augmented Generation (RAG)\", link: \"/guides/rag\" },\n { text: \"Classifiers\", link: \"/guides/classifiers\" },\n ],\n },*/\n ],\n};\n\nfunction nav(): DefaultTheme.NavItem[] {\n return [\n { text: \"API Reference\", link: \"/api-reference\" },\n { text: \"♥ Sponsor\", link: \"https://github.com/sponsors/asg017\" },\n {\n text: `v${VERSION}`,\n items: [\n {\n text: \"Github Release\",\n link: `https://github.com/asg017/${PROJECT}/releases/${VERSION}`,\n },\n {\n text: \"Bindings\",\n items: [\n {\n text: \"Python: PyPi package\",\n link: `https://pypi.org/project/${PROJECT}`,\n },\n {\n text: \"Node.js: NPM package\",\n link: `https://www.npmjs.com/package/${PROJECT}`,\n },\n {\n text: \"Ruby: Ruby gem\",\n link: `https://rubygems.org/gems/${PROJECT}`,\n },\n {\n text: \"Rust: Cargo crate\",\n link: `https://crates.io/crates/${PROJECT}`,\n },\n {\n text: \"Golang: Go module (CGO)\",\n link: `https://pkg.go.dev/github.com/asg017/${PROJECT}-go-bindings/cgo`,\n },\n {\n text: \"Golang: Go module (WASM ncruces)\",\n link: `https://pkg.go.dev/github.com/asg017/${PROJECT}-go-bindings/ncruces`,\n },\n {\n text: \"Datasette: Plugin\",\n link: `https://datasette.io/plugins/datasette-${PROJECT}`,\n },\n {\n text: \"sqlite-utils: Plugin\",\n link: `https://datasette.io/plugins/datasette-${PROJECT}`,\n },\n ],\n },\n ],\n },\n ];\n}\n\nfunction sidebar(): DefaultTheme.SidebarItem[] {\n return [\n {\n text: \"Getting Started\",\n collapsed: true,\n items: [\n {\n text: \"Introduction\",\n link: \"/introduction\",\n },\n {\n text: \"Installation\",\n link: \"/installation\",\n },\n ],\n },\n\n {\n text: \"Using with...\",\n collapsed: true,\n items: [\n { text: \"Python\", link: \"/python\" },\n { text: \"JavaScript\", link: \"/js\" },\n { text: \"Ruby\", link: \"/ruby\" },\n { text: \"Rust\", link: \"/rust\" },\n { text: \"Go\", link: \"/go\" },\n { text: \"C/C++\", link: \"/c\" },\n { text: \"Browser (WASM)\", link: \"/wasm\" },\n { text: \"Datasette\", link: \"/datasette\" },\n { text: \"sqlite-utils\", link: \"/sqlite-utils\" },\n { text: \"rqlite\", link: \"/rqlite\" },\n { text: \"Android+iOS\", link: \"/android-ios\" },\n ],\n },\n {\n text: \"Features\",\n collapsed: true,\n items: [\n { text: \"Vector formats\", link: \"/features/vector-formats\" },\n { text: \"KNN queries\", link: \"/features/knn\" },\n { text: \"vec0 Virtual Tables\", link: \"/features/vec0\" , items: [\n { text: \"Constructor\", link: \"/features/vec0#TODO\" },\n { text: \"KNN Queries\", link: \"/features/vec0#TODO\", items: [{text: \"a\", link:\"\"}] },\n { text: \"Metadata Columns\", link: \"/features/vec0#metadata\" },\n { text: \"Partition Keys\", link: \"/features/vec0#partition-keys\" },\n { text: \"Auxiliary Columns\", link: \"/features/vec0#aux\" },\n \n ]},\n //{ text: \"Static blobs\", link: \"/features/static-blobs\" },\n ],\n },\n guides,\n {\n text: \"Documentation\",\n items: [\n { text: \"Compiling\", link: \"/compiling\" },\n { text: \"API Reference\", link: \"/api-reference\" },\n ],\n },\n {\n text: \"See also\",\n items: [\n {\n text: \"sqlite-ecosystem\",\n link: \"https://github.com/asg017/sqlite-ecosystem\",\n },\n {\n text: \"sqlite-lembed\",\n link: \"https://github.com/asg017/sqlite-lembed\",\n },\n {\n text: \"sqlite-rembed\",\n link: \"https://github.com/asg017/sqlite-rembed\",\n },\n ],\n },\n ];\n}\n\n// https://vitepress.dev/reference/site-config\nexport default defineConfig({\n title: `${PROJECT}`,\n description,\n lastUpdated: true,\n head: head(),\n base: \"/sqlite-vec/\",\n themeConfig: {\n logo: {\n light: \"/logo.dark.svg\",\n dark: \"/logo.light.svg\",\n alt: \"sqlite-vec logo\",\n },\n\n nav: nav(),\n sidebar: sidebar(),\n footer: {\n message: \"MIT/Apache-2 License\",\n copyright:\n 'Copyright © 2024 Alex Garcia',\n },\n outline: \"deep\",\n search: {\n provider: \"local\",\n },\n socialLinks: [\n { icon: \"github\", link: `https://github.com/asg017/${PROJECT}` },\n { icon: \"discord\", link: `https://discord.gg/Ve7WeCJFXk` },\n ],\n editLink: {\n pattern: `https://github.com/asg017/${PROJECT}/edit/main/site/:path`,\n },\n },\n rewrites: {\n \"using/:pkg.md\": \":pkg.md\",\n \"getting-started/:pkg.md\": \":pkg.md\",\n //\"guides/:pkg.md\": \":pkg.md\",\n },\n markdown: {\n languages: [sqliteLanuage],\n },\n});\n" }, { "path": "site/.vitepress/theme/HeroImg.vue", "content": "\n\n\n" }, { "path": "site/.vitepress/theme/Sponsors.vue", "content": "\n\n\n\n\n" }, { "path": "site/.vitepress/theme/index.ts", "content": "// https://vitepress.dev/guide/custom-theme\nimport { h } from \"vue\";\nimport type { Theme } from \"vitepress\";\nimport DefaultTheme from \"vitepress/theme\";\nimport \"./style.css\";\nimport Sponsors from \"./Sponsors.vue\";\nimport HeroImg from \"./HeroImg.vue\";\n\nexport default {\n extends: DefaultTheme,\n Layout: () => {\n return h(DefaultTheme.Layout, null, {\n // https://vitepress.dev/guide/extending-default-theme#layout-slots\n \"layout-top\": () =>\n h(\"marquee\", { class: \"banner\", scrollamount: \"10\" }, [\n \"🚧🚧🚧 This documentation is a work-in-progress! 🚧🚧🚧\",\n ]),\n //\"home-hero-image\": () => h(HeroImg),\n \"aside-ads-before\": () => h(Sponsors),\n });\n },\n enhanceApp({ app, router, siteData }) {\n // ...\n },\n} satisfies Theme;\n" }, { "path": "site/.vitepress/theme/style.css", "content": "/*@import \"https://code.cdn.mozilla.net/fonts/zilla-slab.css\";*/\n\n@font-face {\n font-family: \"ZillaSlab-SemiBold\";\n src: url(\"/fonts/ZillaSlab-SemiBold.woff\");\n src: url(\"/fonts/ZillaSlab-SemiBold.woff2\") format(\"woff2\"),\n url(\"/fonts/ZillaSlab-SemiBold.woff\") format(\"woff\"),\n url(\"/fonts/ZillaSlab(-SemiBold).otf\") format(\"opentype\"),\n url(\"/fonts/ZillaSlab-SemiBold.ttf\") format(\"truetype\");\n font-weight: 600;\n font-style: normal;\n}\n\n.VPHero h1,\n.VPNavBarTitle .title {\n font-family: \"ZillaSlab-SemiBold\";\n font-size: 1.5rem;\n}\n/**\n * Customize default theme styling by overriding CSS variables:\n * https://github.com/vuejs/vitepress/blob/main/src/client/theme-default/styles/vars.css\n */\n\n/**\n * Colors\n *\n * Each colors have exact same color scale system with 3 levels of solid\n * colors with different brightness, and 1 soft color.\n *\n * - `XXX-1`: The most solid color used mainly for colored text. It must\n * satisfy the contrast ratio against when used on top of `XXX-soft`.\n *\n * - `XXX-2`: The color used mainly for hover state of the button.\n *\n * - `XXX-3`: The color for solid background, such as bg color of the button.\n * It must satisfy the contrast ratio with pure white (#ffffff) text on\n * top of it.\n *\n * - `XXX-soft`: The color used for subtle background such as custom container\n * or badges. It must satisfy the contrast ratio when putting `XXX-1` colors\n * on top of it.\n *\n * The soft color must be semi transparent alpha channel. This is crucial\n * because it allows adding multiple \"soft\" colors on top of each other\n * to create a accent, such as when having inline code block inside\n * custom containers.\n *\n * - `default`: The color used purely for subtle indication without any\n * special meanings attched to it such as bg color for menu hover state.\n *\n * - `brand`: Used for primary brand colors, such as link text, button with\n * brand theme, etc.\n *\n * - `tip`: Used to indicate useful information. The default theme uses the\n * brand color for this by default.\n *\n * - `warning`: Used to indicate warning to the users. Used in custom\n * container, badges, etc.\n *\n * - `danger`: Used to show error, or dangerous message to the users. Used\n * in custom container, badges, etc.\n * -------------------------------------------------------------------------- */\n\n:root {\n --vp-c-default-1: var(--vp-c-gray-1);\n --vp-c-default-2: var(--vp-c-gray-2);\n --vp-c-default-3: var(--vp-c-gray-3);\n --vp-c-default-soft: var(--vp-c-gray-soft);\n\n --vp-c-brand-1: var(--vp-c-indigo-1);\n --vp-c-brand-2: var(--vp-c-indigo-2);\n --vp-c-brand-3: var(--vp-c-indigo-3);\n --vp-c-brand-soft: var(--vp-c-indigo-soft);\n\n --vp-c-tip-1: var(--vp-c-brand-1);\n --vp-c-tip-2: var(--vp-c-brand-2);\n --vp-c-tip-3: var(--vp-c-brand-3);\n --vp-c-tip-soft: var(--vp-c-brand-soft);\n\n --vp-c-warning-1: var(--vp-c-yellow-1);\n --vp-c-warning-2: var(--vp-c-yellow-2);\n --vp-c-warning-3: var(--vp-c-yellow-3);\n --vp-c-warning-soft: var(--vp-c-yellow-soft);\n\n --vp-c-danger-1: var(--vp-c-red-1);\n --vp-c-danger-2: var(--vp-c-red-2);\n --vp-c-danger-3: var(--vp-c-red-3);\n --vp-c-danger-soft: var(--vp-c-red-soft);\n\n --vp-c-brand-1x: #a6d189;\n --vp-c-brand-1x: #a6da95;\n --vp-c-brand-1x: #a6e3a1;\n}\n\n:root {\n --vp-c-brand-1: #1e66f5;\n}\n.dark {\n --vp-c-brand-1: #89b4fa;\n}\n\n/**\n * Component: Button\n * -------------------------------------------------------------------------- */\n\n:root {\n --vp-button-brand-border: transparent;\n --vp-button-brand-text: var(--vp-c-white);\n --vp-button-brand-bg: var(--vp-c-brand-3);\n --vp-button-brand-hover-border: transparent;\n --vp-button-brand-hover-text: var(--vp-c-white);\n --vp-button-brand-hover-bg: var(--vp-c-brand-2);\n --vp-button-brand-active-border: transparent;\n --vp-button-brand-active-text: var(--vp-c-white);\n --vp-button-brand-active-bg: var(--vp-c-brand-1);\n}\n\n/**\n * Component: Home\n * -------------------------------------------------------------------------- */\n\n:root {\n --vp-home-hero-name-color: transparent;\n --vp-home-hero-name-background: black;\n\n /*\n --vp-home-hero-image-background-image: linear-gradient(\n -45deg,\n #bd34fe 50%,\n #47caff 50%\n );\n --vp-home-hero-image-filter: blur(44px);\n */\n}\n\n.dark {\n --vp-home-hero-name-background: white;\n}\n\n@media (min-width: 640px) {\n :root {\n --vp-home-hero-image-filter: blur(56px);\n }\n}\n\n@media (min-width: 960px) {\n :root {\n --vp-home-hero-image-filter: blur(68px);\n }\n}\n\n.VPContent.is-home .language-sqlite {\n max-width: 640px;\n margin: 0 auto;\n margin-top: 2rem;\n}\n\n/**\n * Component: Custom Block\n * -------------------------------------------------------------------------- */\n\n:root {\n --vp-custom-block-tip-border: transparent;\n --vp-custom-block-tip-text: var(--vp-c-text-1);\n --vp-custom-block-tip-bg: var(--vp-c-brand-soft);\n --vp-custom-block-tip-code-bg: var(--vp-c-brand-soft);\n}\n\n/**\n * Component: Algolia\n * -------------------------------------------------------------------------- */\n\n.DocSearch {\n --docsearch-primary-color: var(--vp-c-brand-1) !important;\n}\n\n:root {\n --vp-layout-top-height: 30px;\n}\n.banner {\n position: fixed;\n z-index: var(--vp-z-index-layout-top);\n top: 0;\n left: 0;\n right: 0;\n text-align: center;\n height: var(--vp-layout-top-height);\n line-height: var(--vp-layout-top-height);\n background: #f9e2af;\n color: black;\n}\n" }, { "path": "site/api-reference.md", "content": "---\noutline: 2\n---\n\n# API Reference\n\nA complete reference to all the SQL scalar functions, table functions, and virtual tables inside `sqlite-vec`.\n\n::: warning\nsqlite-vec is pre-v1, so expect breaking changes.\n:::\n\n[[toc]]\n\n## Constructors {#constructors} \n\nSQL functions that \"construct\" vectors with different element types.\n\nCurrently, only `float32`, `int8`, and `bit` vectors are supported.\n\n\n### `vec_f32(vector)` {#vec_f32}\n\nCreates a float vector from a BLOB or JSON text. If a BLOB is provided,\nthe length must be divisible by 4, as a float takes up 4 bytes of space each.\n\nThe returned value is a BLOB with 4 bytes per element, with a special [subtype](https://www.sqlite.org/c3ref/result_subtype.html)\nof `223`.\n\n\n```sql\nselect vec_f32('[.1, .2, .3, 4]');\n-- X'CDCCCC3DCDCC4C3E9A99993E00008040'\n\nselect subtype(vec_f32('[.1, .2, .3, 4]'));\n-- 223\n\nselect vec_f32(X'AABBCCDD');\n-- X'AABBCCDD'\n\nselect vec_to_json(vec_f32(X'AABBCCDD'));\n-- '[-1844071490169864000.000000]'\n\nselect vec_f32(X'AA');\n-- ❌ invalid float32 vector BLOB length. Must be divisible by 4, found 1\n\n\n```\n\n### `vec_int8(vector)` {#vec_int8}\n\nCreates a 8-bit integer vector from a BLOB or JSON text. If a BLOB is provided,\nthe length must be divisible by 4, as a float takes up 4 bytes of space each.\nIf JSON text is provided, each element must be an integer between -128 and 127 inclusive.\n\nThe returned value is a BLOB with 1 byte per element, with a special [subtype](https://www.sqlite.org/c3ref/result_subtype.html)\nof `225`.\n\n\n```sql\nselect vec_int8('[1, 2, 3, 4]');\n-- X'01020304'\n\nselect subtype(vec_int8('[1, 2, 3, 4]'));\n-- 225\n\nselect vec_int8(X'AABBCCDD');\n-- X'AABBCCDD'\n\nselect vec_to_json(vec_int8(X'AABBCCDD'));\n-- '[-86,-69,-52,-35]'\n\nselect vec_int8('[999]');\n-- ❌ JSON parsing error: value out of range for int8\n\n\n```\n\n### `vec_bit(vector)` {#vec_bit}\n\nCreates a binary vector from a BLOB.\n\nThe returned value is a BLOB with 1 byte per 8 elements, with a special [subtype](https://www.sqlite.org/c3ref/result_subtype.html)\nof `224`.\n\n\n```sql\nselect vec_bit(X'F0');\n-- X'F0'\n\nselect subtype(vec_bit(X'F0'));\n-- 224\n\nselect vec_to_json(vec_bit(X'F0'));\n-- '[0,0,0,0,1,1,1,1]'\n\n\n```\n\n## Operations {#op} \n\nDifferent operations and utilities for working with vectors.\n\n\n### `vec_length(vector)` {#vec_length}\n\nReturns the number of elements in the given vector.\nThe vector can be `JSON`, `BLOB`, or the result of a [constructor function](#constructors).\n\nThis function will return an error if `vector` is invalid.\n\n\n```sql\nselect vec_length('[.1, .2]');\n-- 2\n\nselect vec_length(X'AABBCCDD');\n-- 1\n\nselect vec_length(vec_int8(X'AABBCCDD'));\n-- 4\n\nselect vec_length(vec_bit(X'AABBCCDD'));\n-- 32\n\nselect vec_length(X'CCDD');\n-- ❌ invalid float32 vector BLOB length. Must be divisible by 4, found 2\n\n\n```\n\n### `vec_type(vector)` {#vec_type}\n\nReturns the name of the type of `vector` as text. One of `'float32'`, `'int8'`, or `'bit'`.\n\nThis function will return an error if `vector` is invalid.\n\n\n```sql\nselect vec_type('[.1, .2]');\n-- 'float32'\n\nselect vec_type(X'AABBCCDD');\n-- 'float32'\n\nselect vec_type(vec_int8(X'AABBCCDD'));\n-- 'int8'\n\nselect vec_type(vec_bit(X'AABBCCDD'));\n-- 'bit'\n\nselect vec_type(X'CCDD');\n-- ❌ invalid float32 vector BLOB length. Must be divisible by 4, found 2\n\n\n```\n\n### `vec_add(a, b)` {#vec_add}\n\nAdds every element in vector `a` with vector `b`, returning a new vector `c`. Both vectors\nmust be of the same type and same length. Only `float32` and `int8` vectors are supported.\n\nAn error is raised if either `a` or `b` are invalid, or if they are not the same type or same length.\n\nSee also [`vec_sub()`](#vec_sub).\n\n\n```sql\nselect vec_add(\n '[.1, .2, .3]',\n '[.4, .5, .6]'\n);\n-- X'0000003F3333333F6766663F'\n\nselect vec_to_json(\n vec_add(\n '[.1, .2, .3]',\n '[.4, .5, .6]'\n )\n);\n-- '[0.500000,0.700000,0.900000]'\n\nselect vec_to_json(\n vec_add(\n vec_int8('[1, 2, 3]'),\n vec_int8('[4, 5, 6]')\n )\n);\n-- '[5,7,9]'\n\nselect vec_add('[.1]', vec_int8('[1]'));\n-- ❌ Vector type mistmatch. First vector has type float32, while the second has type int8.\n\nselect vec_add(vec_bit(X'AA'), vec_bit(X'BB'));\n-- ❌ Cannot add two bitvectors together.\n\n\n```\n\n### `vec_sub(a, b)` {#vec_sub}\n\nSubtracts every element in vector `a` with vector `b`, returning a new vector `c`. Both vectors\nmust be of the same type and same length. Only `float32` and `int8` vectors are supported.\n\nAn error is raised if either `a` or `b` are invalid, or if they are not the same type or same length.\n\nSee also [`vec_add()`](#vec_add).\n\n\n```sql\nselect vec_sub(\n '[.1, .2, .3]',\n '[.4, .5, .6]'\n);\n-- X'9A9999BE9A9999BE9A9999BE'\n\nselect vec_to_json(\n vec_sub(\n '[.1, .2, .3]',\n '[.4, .5, .6]'\n )\n);\n-- '[-0.300000,-0.300000,-0.300000]'\n\nselect vec_to_json(\n vec_sub(\n vec_int8('[1, 2, 3]'),\n vec_int8('[4, 5, 6]')\n )\n);\n-- '[-3,-3,-3]'\n\nselect vec_sub('[.1]', vec_int8('[1]'));\n-- ❌ Vector type mistmatch. First vector has type float32, while the second has type int8.\n\nselect vec_sub(vec_bit(X'AA'), vec_bit(X'BB'));\n-- ❌ Cannot subtract two bitvectors together.\n\n\n```\n\n### `vec_normalize(vector)` {#vec_normalize}\n\nPerforms L2 normalization on the given vector. Only float32 vectors are currently supported.\n\nReturns an error if the input is an invalid vector or not a float32 vector.\n\n\n```sql\nselect vec_normalize('[2, 3, 1, -4]');\n-- X'BAF4BA3E8B370C3FBAF43A3EBAF43ABF'\n\nselect vec_to_json(\n vec_normalize('[2, 3, 1, -4]')\n);\n-- '[0.365148,0.547723,0.182574,-0.730297]'\n\n-- for matryoshka embeddings - slice then normalize\nselect vec_to_json(\n vec_normalize(\n vec_slice('[2, 3, 1, -4]', 0, 2)\n )\n);\n-- '[0.554700,0.832050]'\n\n\n```\n\n### `vec_slice(vector, start, end)` {#vec_slice}\n\nExtract a subset of `vector` from the `start` element (inclusive) to the `end` element (exclusive). TODO check\n\nThis is especially useful for [Matryoshka embeddings](#TODO), also known as \"adaptive length\" embeddings.\nUse with [`vec_normalize()`](#vec_normalize) to get proper results.\n\nReturns an error in the following conditions:\n - If `vector` is not a valid vector\n - If `start` is less than zero or greater than or equal to `end`\n - If `end` is greater than the length of `vector`, or less than or equal to `start`.\n - If `vector` is a bitvector, `start` and `end` must be divisible by 8.\n\n\n```sql\nselect vec_slice('[1, 2,3, 4]', 0, 2);\n-- X'0000803F00000040'\n\nselect vec_to_json(\n vec_slice('[1, 2,3, 4]', 0, 2)\n);\n-- '[1.000000,2.000000]'\n\nselect vec_to_json(\n vec_slice('[1, 2,3, 4]', 2, 4)\n);\n-- '[3.000000,4.000000]'\n\nselect vec_to_json(\n vec_slice('[1, 2,3, 4]', -1, 4)\n);\n-- ❌ slice 'start' index must be a postive number.\n\nselect vec_to_json(\n vec_slice('[1, 2,3, 4]', 0, 5)\n);\n-- ❌ slice 'end' index is greater than the number of dimensions\n\nselect vec_to_json(\n vec_slice('[1, 2,3, 4]', 0, 0)\n);\n-- ❌ slice 'start' index is equal to the 'end' index, vectors must have non-zero length\n\n\n```\n\n### `vec_to_json(vector)` {#vec_to_json}\n\nRepresents a vector as JSON text. The input vector can be a vector BLOB or JSON text.\n\nReturns an error if `vector` is an invalid vector, or when memory cannot be allocated.\n\n\n```sql\nselect vec_to_json(X'AABBCCDD');\n-- '[-1844071490169864000.000000]'\n\nselect vec_to_json(vec_int8(X'AABBCCDD'));\n-- '[-86,-69,-52,-35]'\n\nselect vec_to_json(vec_bit(X'AABBCCDD'));\n-- '[0,1,0,1,0,1,0,1,1,1,0,1,1,1,0,1,0,0,1,1,0,0,1,1,1,0,1,1,1,0,1,1]'\n\nselect vec_to_json('[1,2,3,4]');\n-- '[1.000000,2.000000,3.000000,4.000000]'\n\nselect vec_to_json('invalid');\n-- ❌ JSON array parsing error: Input does not start with '['\n\n\n```\n\n### `vec_each(vector)` {#vec_each}\n\nA table function to iterate through every element in a vector. One row id returned per element in a vector.\n\n```sql\nCREATE TABLE vec_each(\n rowid int, -- The\n vector HIDDEN -- input parameter: A well-formed vector value\n)\n```\n\nReturns an error if `vector` is not a valid vector.\n\n\n```sql\nselect rowid, value from vec_each('[1,2,3,4]');\n/*\n┌───────┬───────┐\n│ rowid │ value │\n├───────┼───────┤\n│ 0 │ 1 │\n├───────┼───────┤\n│ 1 │ 2 │\n├───────┼───────┤\n│ 2 │ 3 │\n├───────┼───────┤\n│ 3 │ 4 │\n└───────┴───────┘\n\n*/\n\n\nselect rowid, value from vec_each(X'AABBCCDD00112233');\n/*\n┌───────┬──────────────────────┐\n│ rowid │ value │\n├───────┼──────────────────────┤\n│ 0 │ -1844071490169864200 │\n├───────┼──────────────────────┤\n│ 1 │ 3.773402568185702e-8 │\n└───────┴──────────────────────┘\n\n*/\n\n\nselect rowid, value from vec_each(vec_int8(X'AABBCCDD'));\n/*\n┌───────┬───────┐\n│ rowid │ value │\n├───────┼───────┤\n│ 0 │ -86 │\n├───────┼───────┤\n│ 1 │ -69 │\n├───────┼───────┤\n│ 2 │ -52 │\n├───────┼───────┤\n│ 3 │ -35 │\n└───────┴───────┘\n\n*/\n\n\nselect rowid, value from vec_each(vec_bit(X'F0'));\n/*\n┌───────┬───────┐\n│ rowid │ value │\n├───────┼───────┤\n│ 0 │ 1 │\n├───────┼───────┤\n│ 1 │ 1 │\n├───────┼───────┤\n│ 2 │ 1 │\n├───────┼───────┤\n│ 3 │ 1 │\n├───────┼───────┤\n│ 4 │ 0 │\n├───────┼───────┤\n│ 5 │ 0 │\n├───────┼───────┤\n│ 6 │ 0 │\n├───────┼───────┤\n│ 7 │ 0 │\n└───────┴───────┘\n\n*/\n\n\n\n```\n\n## Distance functions {#distance} \n\nVarious algorithms to calculate distance between two vectors.\n\n### `vec_distance_L2(a, b)` {#vec_distance_L2}\n\nCalculates the L2 euclidian distance between vectors `a` and `b`. Only valid for float32 or int8 vectors.\n\nReturns an error under the following conditions:\n- `a` or `b` are invalid vectors\n- `a` or `b` do not share the same vector element types (ex float32 or int8)\n- `a` or `b` are bit vectors. Use [`vec_distance_hamming()`](#vec_distance_hamming) for distance calculations between two bitvectors.\n- `a` or `b` do not have the same length.\n\n\n```sql\nselect vec_distance_L2('[1, 1]', '[2, 2]');\n-- 1.4142135381698608\n\nselect vec_distance_L2('[1, 1]', '[-2, -2]');\n-- 4.242640495300293\n\nselect vec_distance_L2('[1.1, 2.2, 3.3]', '[4.4, 5.5, 6.6]');\n-- 5.7157673835754395\n\nselect vec_distance_L2(X'AABBCCDD', X'00112233');\n-- 1844071490169864200\n\nselect vec_distance_L2('[1, 1]', vec_int8('[2, 2]'));\n-- ❌ Vector type mistmatch. First vector has type float32, while the second has type int8.\n\nselect vec_distance_L2(vec_bit(X'AA'), vec_bit(X'BB'));\n-- ❌ Cannot calculate L2 distance between two bitvectors.\n\n\n```\n\n### `vec_distance_cosine(a, b)` {#vec_distance_cosine}\n\nCalculates the cosine distance between vectors `a` and `b`. Only valid for float32 or int8 vectors.\n\nReturns an error under the following conditions:\n - `a` or `b` are invalid vectors\n - `a` or `b` do not share the same vector element types (ex float32 or int8)\n - `a` or `b` are bit vectors. Use [`vec_distance_hamming()`](#vec_distance_hamming) for distance calculations between two bitvectors.\n - `a` or `b` do not have the same length.\n\n\n```sql\nselect vec_distance_cosine('[1, 1]', '[2, 2]');\n-- 2.220446049250313e-16\n\nselect vec_distance_cosine('[1, 1]', '[-2, -2]');\n-- 2\n\nselect vec_distance_cosine('[1.1, 2.2, 3.3]', '[4.4, 5.5, 6.6]');\n-- 0.02536807395517826\n\nselect vec_distance_cosine(X'AABBCCDD', X'00112233');\n-- 2\n\nselect vec_distance_cosine('[1, 1]', vec_int8('[2, 2]'));\n-- ❌ Vector type mistmatch. First vector has type float32, while the second has type int8.\n\nselect vec_distance_cosine(vec_bit(X'AA'), vec_bit(X'BB'));\n-- ❌ Cannot calculate cosine distance between two bitvectors.\n\n\n```\n\n### `vec_distance_hamming(a, b)` {#vec_distance_hamming}\n\nCalculates the hamming distance between two bitvectors `a` and `b`. Only valid for bitvectors.\n\nReturns an error under the following conditions:\n- `a` or `b` are not bitvectors\n- `a` and `b` do not share the same length\n- Memory cannot be allocated\n\n\n```sql\nselect vec_distance_hamming(vec_bit(X'00'), vec_bit(X'FF'));\n-- 8\n\nselect vec_distance_hamming(vec_bit(X'FF'), vec_bit(X'FF'));\n-- 0\n\nselect vec_distance_hamming(vec_bit(X'F0'), vec_bit(X'44'));\n-- 4\n\nselect vec_distance_hamming('[1, 1]', '[0, 0]');\n-- ❌ Cannot calculate hamming distance between two float32 vectors.\n\n\n```\n\n## Quantization {#quantization} \n\nVarious techniques to \"compress\" a vector by reducing precision and accuracy.\n\n### `vec_quantize_binary(vector)` {#vec_quantize_binary}\n\nQuantize a float32 or int8 vector into a bitvector.\nFor every element in the vector, a `1` is assigned to positive numbers and a `0` is assigned to negative numbers.\nThese values are then packed into a bit vector.\n\nReturns an error if `vector` is invalid, or if `vector` is not a float32 or int8 vector.\n\n\n```sql\nselect vec_quantize_binary('[1, 2, 3, 4, 5, 6, 7, 8]');\n-- X'FF'\n\nselect vec_quantize_binary('[1, 2, 3, 4, -5, -6, -7, -8]');\n-- X'0F'\n\nselect vec_quantize_binary('[-1, -2, -3, -4, -5, -6, -7, -8]');\n-- X'00'\n\nselect vec_quantize_binary('[-1, -2, -3, -4, -5, -6, -7, -8]');\n-- X'00'\n\nselect vec_quantize_binary(vec_int8(X'11223344'));\n-- ❌ Binary quantization requires vectors with a length divisible by 8\n\nselect vec_quantize_binary(vec_bit(X'FF'));\n-- ❌ Can only binary quantize float or int8 vectors\n\n\n```\n\n### `vec_quantize_i8(vector, [start], [end])` {#vec_quantize_i8}\n\nx\n\n```sql\nselect 'todo';\n-- 'todo'\n\n\n```\n\n## NumPy Utilities {#numpy} \n\nFunctions to read data from or work with [NumPy arrays](https://numpy.org/doc/stable/reference/generated/numpy.array.html).\n\n### `vec_npy_each(vector)` {#vec_npy_each}\n\nxxx\n\n\n```sql\n-- db.execute('select quote(?)', [to_npy(np.array([[1.0], [2.0], [3.0]], dtype=np.float32))]).fetchone()\nselect\n rowid,\n vector,\n vec_type(vector),\n vec_to_json(vector)\nfrom vec_npy_each(\n X'934E554D5059010076007B276465736372273A20273C6634272C2027666F727472616E5F6F72646572273A2046616C73652C20277368617065273A2028332C2031292C207D202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020200A0000803F0000004000004040'\n)\n/*\n┌───────┬─────────────┬──────────────────┬─────────────────────┐\n│ rowid │ vector │ vec_type(vector) │ vec_to_json(vector) │\n├───────┼─────────────┼──────────────────┼─────────────────────┤\n│ 0 │ X'0000803F' │ 'float32' │ '[1.000000]' │\n├───────┼─────────────┼──────────────────┼─────────────────────┤\n│ 1 │ X'00000040' │ 'float32' │ '[2.000000]' │\n├───────┼─────────────┼──────────────────┼─────────────────────┤\n│ 2 │ X'00004040' │ 'float32' │ '[3.000000]' │\n└───────┴─────────────┴──────────────────┴─────────────────────┘\n\n*/\n\n\n-- db.execute('select quote(?)', [to_npy(np.array([[1.0], [2.0], [3.0]], dtype=np.float32))]).fetchone()\nselect\n rowid,\n vector,\n vec_type(vector),\n vec_to_json(vector)\nfrom vec_npy_each(\n X'934E554D5059010076007B276465736372273A20273C6634272C2027666F727472616E5F6F72646572273A2046616C73652C20277368617065273A2028332C2031292C207D202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020202020200A0000803F0000004000004040'\n)\n/*\n┌───────┬─────────────┬──────────────────┬─────────────────────┐\n│ rowid │ vector │ vec_type(vector) │ vec_to_json(vector) │\n├───────┼─────────────┼──────────────────┼─────────────────────┤\n│ 0 │ X'0000803F' │ 'float32' │ '[1.000000]' │\n├───────┼─────────────┼──────────────────┼─────────────────────┤\n│ 1 │ X'00000040' │ 'float32' │ '[2.000000]' │\n├───────┼─────────────┼──────────────────┼─────────────────────┤\n│ 2 │ X'00004040' │ 'float32' │ '[3.000000]' │\n└───────┴─────────────┴──────────────────┴─────────────────────┘\n\n*/\n\n\n\n```\n\n## Meta {#meta} \n\nHelper functions to debug `sqlite-vec` installations.\n\n### `vec_version()` {#vec_version}\n\nReturns a version string of the current `sqlite-vec` installation.\n\n```sql\nselect vec_version();\n-- 'v0.0.1-alpha.37'\n\n\n```\n\n### `vec_debug()` {#vec_debug}\n\nReturns debugging information of the current `sqlite-vec` installation.\n\n```sql\nselect vec_debug();\n/*\n'Version: v0.0.1-alpha.37\nDate: 2024-07-23T14:09:43Z-0700\nCommit: 77f9b0374c8129056b344854de2dff6b103e5729\nBuild flags: avx '\n*/\n\n\n```\n\n## Entrypoints {#entrypoints} \n\nAll the named entrypoints that load in different `sqlite-vec` functions and options.\n\n" }, { "path": "site/build-ref.mjs", "content": "import Database from \"better-sqlite3\";\nimport { load } from \"js-yaml\";\nimport { fileURLToPath } from \"node:url\";\nimport { dirname, resolve } from \"node:path\";\nimport { readFileSync, writeFileSync } from \"node:fs\";\nimport * as v from \"valibot\";\nimport { table } from \"table\";\n\nconst HEADER = `---\noutline: 2\n---\n\n# API Reference\n\nA complete reference to all the SQL scalar functions, table functions, and virtual tables inside \\`sqlite-vec\\`.\n\n::: warning\nsqlite-vec is pre-v1, so expect breaking changes.\n:::\n\n[[toc]]\n\n`;\n\nconst REF_PATH = resolve(\n dirname(fileURLToPath(import.meta.url)),\n \"../reference.yaml\",\n);\nconst EXT_PATH = resolve(\n dirname(fileURLToPath(import.meta.url)),\n \"../dist/vec0\",\n);\n\nconst DocSchema = v.objectWithRest(\n {\n sections: v.record(\n v.string(),\n v.object({\n title: v.string(),\n desc: v.string(),\n }),\n ),\n },\n v.record(\n v.string(),\n v.object({\n params: v.array(v.string()),\n desc: v.string(),\n example: v.union([v.string(), v.array(v.string())]),\n }),\n ),\n);\n\nconst tableConfig = {\n border: {\n topBody: `─`,\n topJoin: `┬`,\n topLeft: `┌`,\n topRight: `┐`,\n\n bottomBody: `─`,\n bottomJoin: `┴`,\n bottomLeft: `└`,\n bottomRight: `┘`,\n\n bodyLeft: `│`,\n bodyRight: `│`,\n bodyJoin: `│`,\n\n joinBody: `─`,\n joinLeft: `├`,\n joinRight: `┤`,\n joinJoin: `┼`,\n },\n};\n\nfunction formatSingleValue(value) {\n if (typeof value === \"string\") {\n const s = `'${value.replace(/'/g, \"''\")}'`;\n if (s.split(\"\\n\").length > 1) {\n return `/*\\n${s}\\n*/`;\n }\n return `-- ${s}`;\n }\n if (typeof value === \"number\") return `-- ${value.toString()}`;\n if (value === null) return \"-- NULL\";\n if (value instanceof Uint8Array) {\n let s = \"X'\";\n for (const v of value) {\n s += v.toString(16).toUpperCase().padStart(2, \"0\");\n }\n s += \"'\";\n return `-- ${s}`;\n }\n if (typeof value === \"object\" || Array.isArray(value)) {\n return \"-- \" + JSON.stringify(value, null, 2);\n }\n}\nfunction formatValue(value) {\n if (typeof value === \"string\") return `'${value}'`;\n if (typeof value === \"number\") return value;\n if (value === null) return \"NULL\";\n if (value instanceof Uint8Array) {\n let s = \"X'\";\n for (const v of value) {\n s += v.toString(16).toUpperCase().padStart(2, \"0\");\n }\n s += \"'\";\n return s;\n }\n if (typeof value === \"object\" || Array.isArray(value)) {\n return JSON.stringify(value, null, 2);\n }\n}\nfunction tableize(stmt, results) {\n const columnNames = stmt.columns().map((c) => c.name);\n const rows = results.map((row) =>\n row.map((value) => {\n return formatValue(value);\n })\n );\n return table([columnNames, ...rows], tableConfig);\n}\n\nfunction renderExamples(db, name, example) {\n let md = \"```sql\\n\";\n\n const examples = Array.isArray(example) ? example : [example];\n for (const example of examples) {\n const sql = example\n /* Strip any '```sql' markdown at the beginning */\n .replace(/^\\w*```sql/, \"\")\n /* Strip any '```' markdown at the end */\n .replace(/```\\w*$/m, \"\")\n .trim();\n let stmt, results, error;\n results = null;\n try {\n stmt = db.prepare(sql);\n try {\n stmt.raw(true);\n } catch (err) {\n 1;\n }\n } catch (error) {\n console.error(`Error preparing statement for ${name}:`);\n console.error(error);\n throw Error();\n }\n\n try {\n results = stmt.all();\n } catch (e) {\n error = e.message;\n }\n\n md += sql + \"\\n\";\n\n if (!results) {\n md += `-- ❌ ${error}\\n\\n`;\n continue;\n }\n\n const result = results.length > 1 || stmt.columns().length > 1\n ? `/*\\n${tableize(stmt, results)}\\n*/\\n`\n : formatSingleValue(results[0][0]);\n md += result + \"\\n\\n\";\n }\n\n md += \"\\n```\\n\\n\";\n\n return md;\n}\n\nlet md = HEADER;\nconst doc = v.parse(DocSchema, load(readFileSync(REF_PATH, \"utf8\")));\n\nconst db = new Database();\ndb.loadExtension(EXT_PATH);\n\nfor (const section in doc.sections) {\n md += `## ${doc.sections[section].title} {#${section}} \\n\\n`;\n md += doc.sections[section].desc;\n md += \"\\n\\n\";\n\n for (\n const [name, { params, desc, example }] of Object.entries(\n doc[section],\n )\n ) {\n const headerText = `\\`${name}(${(params ?? []).join(\", \")})\\` {#${name}}`;\n\n md += \"### \" + headerText + \"\\n\\n\";\n\n md += desc + \"\\n\\n\";\n md += renderExamples(db, name, example);\n }\n}\n\nwriteFileSync(\"api-reference.md\", md, \"utf8\");\nconsole.log(\"done\");\n" }, { "path": "site/compiling.md", "content": "\n\n# Compiling `sqlite-vec`\n\n`sqlite-vec` is is easy to compile yourself! It's a single C file with no dependencies, so the process should be straightforward.\n\n## From Source\n\nTo compile `sqlite-vec` as a loadable SQLite extension, you can `git clone` the source repository and run the following commands:\n\n```bash\ngit clone https://github.com/asg017/sqlite-vec\ncd sqlite-vec\n./scripts/vendor.sh\nmake loadable\n```\n\nThe `./scripts/vendor.sh` command will download a recent version of [SQLite's amalgammation builds](https://www.sqlite.org/amalgamation.html), to ensure you have an up-to-date `sqlite3ext.h` available on your system. \n\nThen `make loadable` will generate the `sqlite-vec.h` file and a dynamically loadable library at `dist/vec.$SUFFIX`. The suffix will be `.dylib` for MacOS, `.so` for Linux, and `.dll` for Windows.\n\n\n## From the amalgamation build\n\nThe \"amalgamation\" build of `sqlite-vec` is a `.zip` or `.tar.gz` archive with the pre-configured `sqlite-vec.c` and `sqlite-vec.h` source files. \n\nThe amalgamation builds can be found in [`sqlite-vec` Releases](https://github.com/asg017/sqlite-vec/releases). You can also download the latest amalgamation build with this command:\n\n```-vue\nwget https://github.com/asg017/sqlite-vec/releases/download/v{{data.VERSION}}/sqlite-vec-{{data.VERSION}}-amalgamation.zip\nunzip sqlite-vec-{{data.VERSION}}-amalgamation.zip\n```\n\nThere will now be `sqlite-vec.c` and `sqlite-vec.h` available in your current directory. To compile it manually, follow the [official SQLite extension compilation instructions](https://www.sqlite.org/loadext.html#compiling_a_loadable_extension), which will be something like:\n\n```bash\n# Linux \ngcc -g -fPIC -shared sqlite-vec.c -o vec0.so\n\n# MacOS\ngcc -g -fPIC -dynamiclib sqlite-vec.c -o vec0.dylib\n\n# Windows, MSVC compiler\ncl sqlite-vec.c -link -dll -out:sqlite-vec.dll\n\n# Windows, MinGW\ngcc -g -shared sqlite-vec.c -o vec0.dll\n```\n\nDifferent platforms, compiler, or architectures may require different compilation flags.\n\n## Compile-time options\n\nThere are a few compilation options available for `sqlite-vec`, but they're currently unstable and may change in the future. They aren't tracked with [`sqlite-vec`'s semantic versioning policy ](./versioning.md), so options may break in patch version updates.\n\nThe current compile-time flags are:\n\n- `SQLITE_VEC_ENABLE_AVX`, enables AVX CPU instructions for some vector search operations\n- `SQLITE_VEC_ENABLE_NEON`, enables NEON CPU instructions for some vector search operations\n- `SQLITE_VEC_OMIT_FS`, removes some obsure SQL functions and features that use the filesystem, meant for some WASM builds where there's no available filesystem\n- `SQLITE_VEC_STATIC`, meant for statically linking `sqlite-vec` \n" }, { "path": "site/features/knn.md", "content": "# KNN queries\n\nThe most common use-case for vectors in databases is for K-nearest-neighbors (KNN) queries.\nYou'll have a table of vectors, and you'll want to find the K closest\n\nCurrently there are two ways to to perform KNN queries with `sqlite-vec`:\nWith `vec0` virtual tables and \"manually\" with regular tables.\n\nThe `vec0` virtual table is faster and more compact, but is less flexible and requires `JOIN`s back to your source tables.\nThe \"manual\" method is more flexible and\n\n\n\n## `vec0` virtual tables\n\n```sql\ncreate virtual table vec_documents using vec0(\n document_id integer primary key,\n contents_embedding float[768]\n);\n\ninsert into vec_documents(document_id, contents_embedding)\n select id, embed(contents)\n from documents;\n```\n\n\n```sql\nselect\n document_id,\n distance\nfrom vec_documents\nwhere contents_embedding match :query\n and k = 10;\n```\n\n```sql\n-- This example ONLY works in SQLite versions 3.41+\n-- Otherwise, use the `k = 10` method described above!\nselect\n document_id,\n distance\nfrom vec_documents\nwhere contents_embedding match :query\nlimit 10; -- LIMIT only works on SQLite versions 3.41+\n```\n\n```sql\nwith knn_matches as (\n select\n document_id,\n distance\n from vec_documents\n where contents_embedding match :query\n and k = 10\n)\nselect\n documents.id,\n documents.contents,\n knn_matches.distance\nfrom knn_matches\nleft join documents on documents.id = knn_matches.document_id\n```\n\n\n```sql\ncreate virtual table vec_documents using vec0(\n document_id integer primary key,\n contents_embedding float[768] distance_metric=cosine\n);\n\n-- insert vectors into vec_documents...\n\n\n-- this MATCH will now use cosine distance instead of the default L2 distance\nselect\n document_id,\n distance\nfrom vec_documents\nwhere contents_embedding match :query\n and k = 10;\n```\n\n\n\n\n## Manually with SQL scalar functions\n\nYou don't need a `vec0` virtual table to perform KNN searches with `sqlite-vec`.\nYou could store vectors in regular columns in a regular tables, like so:\n\n```sql\ncreate table documents(\n id integer primary key,\n contents text,\n -- a 4-dimensional floating-point vector\n contents_embedding blob\n);\n\ninsert into documents values\n (1, 'alex', vec_f32('[1.1, 1.1, 1.1, 1.1]')),\n (2, 'brian', vec_f32('[2.2, 2.2, 2.2, 2.2]')),\n (3, 'craig', vec_f32('[3.3, 3.3, 3.3, 3.3]'));\n```\n\nWhen you want to find similar vectors, you can manually use\n[`vec_distance_L2()`](../api-reference.md#vec_distance_l2),\n[`vec_distance_L1()`](../api-reference.md#vec_distance_l1),\nor [`vec_distance_cosine()`](../api-reference.md#vec_distance_cosine),\nand an `ORDER BY` clause to perform a brute-force KNN query.\n\n```sql\nselect\n id,\n contents,\n vec_distance_L2(contents_embedding, '[2.2, 2.2, 2.2, 2.2]') as distance\nfrom documents\norder by distance;\n\n/*\n┌────┬──────────┬──────────────────┐\n│ id │ contents │ distance │\n├────┼──────────┼──────────────────┤\n│ 2 │ 'brian' │ 0.0 │\n│ 3 │ 'craig' │ 2.19999980926514 │\n│ 1 │ 'alex' │ 2.20000004768372 │\n└────┴──────────┴──────────────────┘\n*/\n```\n\n\n\n\nIf you choose this approach, it is recommended to define the \"vector column\" with its element type (`float`, `bit`, etc.) and dimension, for better documentation.\nIt's also recommended to include a\n[`CHECK` constraint](https://www.sqlite.org/lang_createtable.html#check_constraints),\nto ensure only vectors of the correct element type and dimension exist in the table.\n\n```sql\ncreate table documents(\n id integer primary key,\n contents text,\n contents_embedding float[4]\n check(\n typeof(contents_embedding) == 'blob'\n and vec_length(contents_embedding) == 4\n )\n);\n\n-- ❌ Fails, needs to be a BLOB input\ninsert into documents values (1, 'alex', '[1.1, 1.1, 1.1, 1.1]');\n\n-- ❌ Fails, 3 dimensions, needs 4\ninsert into documents values (1, 'alex', vec_f32('[1.1, 1.1, 1.1]'));\n\n-- ❌ Fails, needs to be a float32 vector\ninsert into documents values (1, 'alex', vec_bit('[1.1, 1.1, 1.1, 1.1]'));\n\n-- ✅ Success! \ninsert into documents values (1, 'alex', vec_f32('[1.1, 1.1, 1.1, 1.1]'));\n```\n\nKeep in mind: **SQLite does not support custom types.**\nThe example above may look like that the `contents_embedding` column has a \"custom type\"\nof `float[4]`, but SQLite allows for *anything* to appear as a \"column type\".\n\n```sql\n-- these \"column types\" are totally legal in SQLite\ncreate table students(\n name ham_sandwich,\n age minions[42]\n);\n```\n\nSee [Datatypes in SQLite](https://www.sqlite.org/datatype3.html) for more info.\n\nSo by itself, `float[4]` as a \"column type\" is not enforced by SQLite at all.\nThis is why we recommend including `CHECK` constraints, to enforce that values in your vector column\nare of the correct type and length.\n\nFor [strict tables](https://www.sqlite.org/stricttables.html), use the `BLOB` type and include the same `CHECK` constraints.\n\n```sql\n\ncreate table documents(\n id integer primary key,\n contents text,\n contents_embedding blob check(vec_length(contents_embedding) == 4)\n) strict;\n```\n\n\n\n\n" }, { "path": "site/features/vec0.md", "content": "# `vec0` Virtual Table\n\n## Metadata in `vec0` Virtual Tables {#vec0_metadata}\n\nThere are three ways to store non-vector columns in `vec0` virtual tables:\nmetadata columns, partition keys, and auxiliary columns. Each option has its\nown benefits and limitations.\n\n```sql\ncreate virtual table vec_chunks using vec0(\n document_id integer partition key,\n contents_embedding float[768],\n\n -- partition key column, denoted by 'partition key'\n user_id integer partition key,\n\n -- metadata column, appears as normal column definition\n label text,\n\n -- auxiliary column, denoted by '+'\n +contents text\n);\n```\n\nA quick summary of each option:\n\n| Column Type | Description | Benefits | Limitations |\n| ----------------- | ----------------------------------------------------------------------- | ---------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------- |\n| Metadata columns | Stores boolean, integer, floating point, or text data alongside vectors | Can be included in the `WHERE` clause of a KNN query | Slower full scan, slightly inefficient with long strings (`> 12` characters) |\n| Auxiliary columns | Stores any kind of data in a separate internal table | Eliminates need for an external `JOIN` | Cannot appear in the `WHERE` clause of a KNN query |\n| Partition Key | Internally shards vector index on a given key | Make selective queries much faster | Can cause oversharding and slow KNN if not used carefully. Should be +100's of vectors per unique partition key value |\n\n### Metadata Columns {#metadata}\n\nMetadata columns are extra \"regular\" columns that you can include in a `vec0`\ntable definition. These columns will be indexed along with declared vector\ncolumns, and allow you to include extra `WHERE` constraints during KNN queries.\n\n```sql\ncreate virtual table vec_movies using vec0(\n movie_id integer primary key,\n synopsis_embedding float[1024],\n genre text,\n num_reviews int,\n mean_rating float,\n contains_violence boolean\n);\n```\n\nIn the `vec0` constructor, the `genre`, `num_reviews`, `mean_rating`, and\n`contains_violence` columns are metadata columns, with their specified types.\n\nA sample KNN query on this table could look like:\n\n```sql\nselect *\nfrom vec_movies\nwhere synopsis_embedding match '[...]'\n and k = 5\n and genre = 'scifi'\n and num_reviews between 100 and 500\n and mean_rating > 3.5\n and contains_violence = false;\n```\n\nThe first two conditions in the `WHERE` clause (`synopsis_embedding match` and\n`k = 5`) denote that the query is a KNN query. The other conditions are metadata\nconstraints that `sqlite-vec` will recognize and apply during the KNN\ncalculation. In other words, for the above query, a maximum of 5 rows would be\nreturned, all of which would match all the `WHERE` constraints for their\nmetadata column values.\n\n#### Metadata Column Declaration\n\nMetatadata columns are declared in the `vec0` constructor just like regular\ncolumn definitions, with the column name first then the column type.\n\nOnly the following column types are supported in metadata columns. All these\ncolumns are strictly typed.\n\n- `TEXT` for text and strings\n- `INTEGER` for 8-byte integers\n- `FLOAT` for 8-byte floating-point numbers\n- `BOOLEAN` for 1-bit `0` or `1`\n\nOther column types may be supported in the future. Column type names are case\ninsensitive.\n\nAdditional column constraints like `UNIQUE` or `NOT NULL` are not supported.\n\nA maximum of 16 metadata columns can be declared in a `vec0` virtual table.\n\n#### Supported operations\n\nMetadata column `WHERE` conditions in a KNN query will only work on the\nfollowing operators:\n\n- `=` Equals to\n- `!=` Not equals to\n- `>` Greater than\n- `>=` Greater than or equal to\n- `<` Less than\n- `<=` Less than or equal to\n\nUsing any other operator like `IS NULL`, `LIKE`, `GLOB`, `REGEXP`, or any scalar\nfunction will result in an error or incorrect results.\n\nBoolean columns only support `=` and `!=` operators.\n\n### Partition Key Columns {#partition-keys}\n\nPartition key columns allow one to internally shard a vector indexed based on a\ngiven key. Any `=` constraint in a `WHERE` clause on a partition key column will \nrestrict the search to that clause.\n\nFor example, say you're performing vector search on a large dataset of\ndocuments. However, each document belongs to a user, and users can only search\ntheir own documents. It would be wasteful to perform a brute-force search over all\ndocuments if you only care about 1 user at a time. So, you can partition the\nvector index based on user ID like so:\n\n```sql\ncreate virtual table vec_documents using vec0(\n document_id integer primary key,\n user_id integer partition key,\n contents_embedding float[1024]\n)\n```\n\nThen, during a KNN query, you can constrain results to a specific user in the\n`WHERE` clause like so:\n\n```sql\nselect\n document_id,\n user_id,\n distance\nfrom vec_documents\nwhere contents_embedding match :query\n and k = 20\n and user_id = 123;\n```\n\n`sqlite-vec` will recognize the `user_id = 123` constraint and pre-filter\nvectors during a KNN search. Vectors with the same partition key values are\ncollocated together, so this is a fast operation.\n\nAnother example: say you're performing vector search on a large dataset of news\nheadlines of the past 100 years. However, in your application, most users only\nwant to search a subset of articles based on when they were written, like \"in\nthe past ten years\" or \"during the obama administration.\" You can paritition\nbased on published date like so:\n\n```sql\ncreate virtual table vec_articles using vec0(\n article_id integer primary key,\n published_date text partition key,\n headline_embedding float[1024]\n);\n```\n\nAnd a KNN query:\n\n```sql\nselect\n article_id,\n published_date,\n distance\nfrom vec_articles\nwhere headline_embedding match :query\n and published_date between '2009-01-20' and '2017-01-20'; -- Obama administration\n```\n\nBut be careful! over-using partition key columns can lead to over-sharding and\nslower KNN queries. As a rule of thumb, make sure that every unique partition\nkey value has ~100s of vectors associated with it. In the above examples, make\nsure that every user has on the magnitude of dozens or hundreds of documents\neach, or that there are dozens or, preferably, hundreds of articles per day. If they\ndon't and you're noticing slow queries, try a more broad partition key value,\nlike `organization_id` or `published_month`.\n\nA maximum of 4 partition key columns can be declared in a `vec0` virtual table,\nbut use caution if you find yourself using more than 1 partition key column. Vectors are sharded\nalong each unique combination, so over-sharding is more common with more\npartition key columns.\n\n### Auxiliary Columns {#aux}\n\nAuxiliary columns store additional unindexed data separate from the internal\nvector index. They are meant for larger metadata that will never appear in a\n`WHERE` clause of a KNN query, but can be retrieved in the result set without needing a separate `JOIN`.\n\nAuxiliary columns are denoted by a `+` prefix in their column definition, like\nso:\n\n```sql\ncreate virtual table vec_chunks using vec0(\n contents_embedding float[1024],\n +contents text\n);\n\nselect\n rowid,\n contents,\n distance\nfrom vec_chunks\nwhere contents_embedding match :query\n and k = 10;\n```\n\nHere we store the text contents of each chunk in the `contents` auxiliary\ncolumn. When we perform a KNN query, we can reference the `contents` column in\nthe `SELECT` clause, to get the raw text contents of the most relevant chunks.\n\nA similar approach can be used for image embeddings:\n\n```sql\ncreate virtual table vec_image_chunks using vec0(\n image_embedding float[1024],\n +image blob\n);\n\nselect\n rowid,\n contents,\n distance\nfrom vec_chunks\nwhere contents_embedding match :query\n and k = 10;\n```\n\nHere the `image` auxiliary column can store the raw image file in a large `BLOB`\ncolumn. It can appear in the `SELECT` clause of the KNN query, to get the most\nrelevant raw images.\n\nIn general, auxiliary columns are good for large text, blobs, URLs, or other\ndatatypes that won't be a part of a `WHERE` clause of a KNN query. Auxiliary columns are a good fit for columns\nthat will appear often in a `SELECT` clause but not in the `WHERE` clause.\n\nA maximum of 16 auxiliary columns can be declared in a `vec0` virtual table.\n" }, { "path": "site/getting-started/installation.md", "content": "# Installing\n\nYou have several options to include `sqlite-vec` into your projects, including\nPyPi packages for Python, NPM packages for Node.js, Gems for Ruby, and more.\n\n## With popular package managers\n\n::: code-group\n\n```bash [Python]\npip install sqlite-vec\n```\n\n```bash [Node.js]\nnpm install sqlite-vec\n```\n\n```bash [Bun]\nbun install sqlite-vec\n```\n\n```bash [Deno]\ndeno add npm:sqlite-vec\n```\n\n```bash [Ruby]\ngem install sqlite-vec\n```\n\n```bash [Rust]\ncargo add sqlite-vec\n```\n\n```bash [Go (CGO)]\ngo get -u github.com/asg017/sqlite-vec-go-bindings/cgo\n```\n```bash [Go (ncruces WASM)]\ngo get -u github.com/asg017/sqlite-vec-go-bindings/ncruces\n```\n\n```bash [Datasette]\ndatasette install datasette-sqlite-vec\n```\n\n```bash [sqlite-utils]\nsqlite-utils install sqlite-utils-sqlite-vec\n```\n\n:::\n\n## Pre-compiled extensions\n\nAlternatively, you can download pre-compiled loadable extensions from the\n[`sqlite-vec` Github Releases](https://github.com/asg017/sqlite-vec/releases/latest).\n\nThere's also an `install.sh` script that will automatically download the appropriate pre-compiled extension from Github Releases to your machine.\n\n\n```sh\n# yolo\ncurl -L 'https://github.com/asg017/sqlite-vec/releases/latest/download/install.sh' | sh\n```\n\n```sh\n# ok lets play it safe\ncurl -o install.sh -L https://github.com/asg017/sqlite-vec/releases/latest/download/install.sh\n# inspect your scripts\ncat install.sh\n# TODO Test if execute permissions?\n./install.sh\n```\n\n\n## Compiling\n\n`sqlite-vec` is a single `sqlite-vec.c` and `sqlite-vec.h`, and can be easily compiled for different platforms, or statically linked into larger applications.\n\nSee [*Compiling `sqlite-vec`*](#compiling) for more information.\n" }, { "path": "site/getting-started/introduction.md", "content": "# Introduction to `sqlite-vec`\n\n## Intro to Vector Databases\n\n## Vector Search in SQLite with `sqlite-vec`\n\n## Getting help\n" }, { "path": "site/guides/arithmetic.md", "content": "# Vector Arithmetic\n\n- `vec_add()`\n- `vec_sub()`\n- `vec_mean()`\n" }, { "path": "site/guides/binary-quant.md", "content": "# Binary Quantization\n\n\"Quantization\" refers to a variety of methods and techniques for reducing the\nsize of vectors in a vector index. **Binary quantization** (BQ) refers to a\nspecific technique where each individual floating point element in a vector is\nreduced to a single bit, typically by assigning `0` to negative numbers and `1`\nto positive numbers.\n\nFor example, in this 8-dimensional `float32` vector:\n\n```json\n[-0.73, -0.80, 0.12, -0.73, 0.79, -0.11, 0.23, 0.97]\n```\n\nApplying binary quantization would result in the following `bit` vector:\n\n```json\n[0, 0, 1, 0, 1, 0, 1, 1]\n```\n\nThe original 8-dimensional `float32` vector requires `8 * 4 = 32` bytes of space\nto store. For 1 million vectors, that would be `32MB`. On the other hand, the\nbinary quantized 8-dimensional vector can be stored in a single byte — one bit\nper element. For 1 million vectors, that would be just `1MB`, a 32x reduction!\n\nThough keep in mind, you're bound to lose a lot quality when reducing 32 bits of\ninformation to 1 bit. [Oversampling and re-scoring](#re-scoring) will help a\nlot.\n\nThe main goal of BQ is to dramatically reduce the size of your vector index,\nresulting in faster searches with less resources. This is especially useful in\n`sqlite-vec`, which is (currently) brute-force only and meant to run on small\ndevices. BQ is an easy low-cost method to make larger vector datasets easier to\nmanage.\n\n## Binary Quantization `sqlite-vec`\n\nThe `sqlite-vec` extension offers a `vec_quantize_binary()` SQL scalar function,\nwhich applies binary quanitization to a `float32` or `int8` vector. For every\nelement in a given vector, it will apply `0` to negative values and `1` to\npositive values, and pack them into a `BLOB`.\n\n```sqlite\nselect vec_quantize_binary(\n '[-0.73, -0.80, 0.12, -0.73, 0.79, -0.11, 0.23, 0.97]'\n);\n-- X'd4`\n```\n\nThe single byte `0xd4` in hexadecimal is `11010100` in binary.\n\n\n\n## Demo\n\nHere's an end-to-end example of using binary quantization with `vec0` virtual\ntables in `sqlite-vec`.\n\n```sqlite\ncreate virtual table vec_movies using vec0(\n synopsis_embedding bit[768]\n);\n```\n\n```sqlite\ninsert into vec_movies(rowid, synopsis_embedding)\n VALUES (:id, vec_quantize_binary(:vector));\n```\n\n```sqlite\nselect\n rowid,\n distance\nfrom vec_movies\nwhere synopsis_embedding match vec_quantize_binary(:query)\norder by distance\nlimit 20;\n```\n\n### Re-scoring\n\n```sqlite\ncreate virtual table vec_movies using vec0(\n synopsis_embedding float[768],\n synopsis_embedding_coarse bit[768]\n);\n```\n\n```sqlite\ninsert into vec_movies(rowid, synopsis_embedding, synopsis_embedding_coarse)\n VALUES (:id, :vector, vec_quantize_binary(:vector));\n```\n\n```sqlite\nwith coarse_matches as (\n select\n rowid,\n synopsis_embedding\n from vec_movies\n where synopsis_embedding_coarse match vec_quantize_binary(:query)\n order by distance\n limit 20 * 8\n),\nselect\n rowid,\n vec_distance_L2(synopsis_embedding, :query)\nfrom coarse_matches\norder by 2\nlimit 20;\n```\n\n# Benchmarks\n\n## Model support\n\nCertain embedding models, like [Nomic](https://nomic.ai/)'s\n[`nomic-embed-text-v1.5`](https://huggingface.co/nomic-ai/nomic-embed-text-v1.5)\ntext embedding model and\n[mixedbread.ai](https://www.mixedbread.ai/blog/mxbai-embed-2d-large-v1)'s\n[`mxbai-embed-large-v1`](https://huggingface.co/mixedbread-ai/mxbai-embed-large-v1)\nare specifically trained to perform well after binary quantization.\n\nOther embeddings models may not, but you can still try BQ and see if it works\nfor your datasets. Chances are, if your vectors are normalized (ie between\n`-1.0` and `1.0`) there's a good chance you will see acceptable results with BQ.\n" }, { "path": "site/guides/classifiers.md", "content": "" }, { "path": "site/guides/hybrid-search.md", "content": "" }, { "path": "site/guides/matryoshka.md", "content": "# Matryoshka (Adaptive-Length) Embeddings\n\nMatryoshka embeddings are a new class of embedding models introduced in the\nTODO-YYY paper [_TODO title_](https://arxiv.org/abs/2205.13147). They allow one\nto truncate excess dimensions in large vector, without sacrificing much quality.\n\nLet's say your embedding model generate 1024-dimensional vectors. If you have 1\nmillion of these 1024-dimensional vectors, they would take up `4.096 GB` of\nspace! You're not able to reduce the dimensions without losing a lot of\nquality - if you were to remove half of the dimensions 512-dimensional vectors,\nyou could expect to also lose 50% or more of the quality of results. There are\nother dimensional-reduction techniques, like [PCA](#TODO) or [Product Quantization](#TODO), but they typically require\ncomplicated and expensive training processes.\n\nMatryoshka embeddings, on the other hand, _can_ be truncated, without losing much\nquality. Using [`mixedbread.ai`](#TODO) `mxbai-embed-large-v1` model, they claim\nthat\n\nThey are called \"Matryoshka\" embeddings because ... TODO\n\n## Matryoshka Embeddings with `sqlite-vec`\n\nYou can use a combination of [`vec_slice()`](../api-reference.md#vec_slice) and\n[`vec_normalize()`](../api-reference.md#vec_slice) on Matryoshka embeddings to\ntruncate.\n\n```sql\nselect\n vec_normalize(\n vec_slice(title_embeddings, 0, 256)\n ) as title_embeddings_256d\nfrom vec_articles;\n```\n\n[`vec_slice()`](../api-reference.md#vec_slice) will cut down the vector to the first 256 dimensions. Then [`vec_normalize()`](../api-reference.md#vec_normalize) will normalize that truncated vector, which is typically a required step for Matryoshka embeddings.\n\n## Benchmarks\n\n## Suppported Models\n\nhttps://supabase.com/blog/matryoshka-embeddings#which-granularities-were-openais-text-embedding-3-models-trained-on\n\n`text-embedding-3-small`: 1536, 512 `text-embedding-3-large`: 3072, 1024, 256\n\nhttps://x.com/ZainHasan6/status/1757519325202686255\n\n`text-embeddings-3-large:` 3072, 1536, 1024, 512\n\nhttps://www.mixedbread.ai/blog/binary-mrl\n\n`mxbai-embed-large-v1`: 1024, 512, 256, 128, 64\n\n`nomic-embed-text-v1.5`: 768, 512, 256, 128, 64\n\n```\n# TODO new snowflake model\n```\n" }, { "path": "site/guides/performance.md", "content": "- page_size\n- memory mapping\n- in-memory index\n- chunk_size (?)\n" }, { "path": "site/guides/rag.md", "content": "# Retrival Augmented Generation (RAG)\n\n- \"memories\"?\n- chunking\n" }, { "path": "site/guides/scalar-quant.md", "content": "# Scalar Quantization (SQ)\n\n\"Quantization\" refers to a variety of methods and techniques for reducing the\nsize of vectors in a vector index. **Scalar quantization** (SQ) refers to a\nspecific technique where each individual floating point element in a vector is\nscaled to a small element type, like `float16`, `int8`.\n\nMost embedding models generate `float32` vectors. Each `float32` takes up 4\nbytes of space. This can add up, especially when working with a large amount of\nvectors or vectors with many dimensions. However, if you scale them to `float16`\nor `int8` vectors, they only take up 2 bytes of space and 1 bytes of space\nrespectively, saving you precious space at the expense of some quality.\n\n```sql\nselect vec_quantize_float16(vec_f32('[]'), 'unit');\nselect vec_quantize_int8(vec_f32('[]'), 'unit');\n\nselect vec_quantize('float16', vec_f32('...'));\nselect vec_quantize('int8', vec_f32('...'));\nselect vec_quantize('bit', vec_f32('...'));\n\nselect vec_quantize('sqf16', vec_f32('...'));\nselect vec_quantize('sqi8', vec_f32('...'));\nselect vec_quantize('bq2', vec_f32('...'));\n```\n\n## Benchmarks\n" }, { "path": "site/guides/semantic-search.md", "content": "" }, { "path": "site/index.md", "content": "---\n# https://vitepress.dev/reference/default-theme-home-page\nlayout: home\n\nhero:\n name: \"sqlite-vec\"\n text: \"\"\n tagline: A vector search SQLite extension that runs anywhere!\n actions:\n - theme: brand\n text: Getting Started\n link: /introduction\n - theme: alt\n text: API Reference\n link: /api-reference\n\nfeatures:\n - title: Runs everywhere\n details: On laptops, servers, mobile devices, browsers with WASM, Raspberry Pis, and more!\n - title: Bindings for many languages\n details: Python, Ruby, Node.js/Deno/Bun, Go, Rust, and more!\n - title: Pure SQL\n details: No extra configuration or server required — only CREATE, INSERT, and SELECT statements\n---\n\n```sqlite\n-- store 768-dimensional vectors in a vec0 virtual table\ncreate virtual table vec_movies using vec0(\n synopsis_embedding float[768]\n);\n\n-- insert vectors into the table, as JSON or compact BLOBs\ninsert into vec_movies(rowid, synopsis_embedding)\n select\n rowid,\n embed(synopsis) as synopsis_embedding\n from movies;\n\n-- KNN search!\nselect\n rowid,\n distance\nfrom vec_movies\nwhere synopsis_embedding match embed('scary futuristic movies')\norder by distance\nlimit 20;\n```\n" }, { "path": "site/package.json", "content": "{\n \"scripts\": {\n \"ref\": \"node build-ref.mjs\",\n \"dev\": \"vitepress dev\",\n \"build\": \"vitepress build\",\n \"preview\": \"vitepress preview\"\n },\n \"devDependencies\": {\n \"vue\": \"^3.4.26\"\n },\n \"dependencies\": {\n \"@types/node\": \"^20.12.8\",\n \"better-sqlite3\": \"^11.1.2\",\n \"js-yaml\": \"^4.1.0\",\n \"table\": \"^6.8.2\",\n \"valibot\": \"^0.36.0\",\n \"vitepress\": \"^1.1.4\"\n }\n}\n" }, { "path": "site/project.data.ts", "content": "import { readFileSync } from \"node:fs\";\nimport { dirname, join } from \"node:path\";\nimport { fileURLToPath } from \"node:url\";\n\nconst PROJECT = \"sqlite-vec\";\n\nconst VERSION = readFileSync(\n join(dirname(fileURLToPath(import.meta.url)), \"..\", \"VERSION\"),\n \"utf8\",\n);\n\nexport default {\n load() {\n return {\n PROJECT,\n VERSION,\n };\n },\n};\n" }, { "path": "site/sqlite.tmlanguage.json", "content": "{\n \"information_for_contributors\": [\n \"This file has been converted from https://github.com/microsoft/vscode-mssql/blob/master/syntaxes/SQL.plist\",\n \"If you want to provide a fix or improvement, please create a pull request against the original repository.\",\n \"Once accepted there, we are happy to receive an update request.\"\n ],\n \"version\": \"https://github.com/microsoft/vscode-mssql/commit/3929516cce0a570e91ee1be74b09ed886cb360f4\",\n \"name\": \"sqlite\",\n \"scopeName\": \"source.sqlite\",\n \"patterns\": [\n {\n \"match\": \"((?]?=|<>|<|>\",\n \"name\": \"keyword.operator.comparison.sql\"\n },\n {\n \"match\": \"-|\\\\+|/\",\n \"name\": \"keyword.operator.math.sql\"\n },\n {\n \"match\": \"\\\\|\\\\|\",\n \"name\": \"keyword.operator.concatenator.sql\"\n },\n {\n \"match\": \"(?i)\\\\b(approx_count_distinct|approx_percentile_cont|approx_percentile_disc|avg|checksum_agg|count|count_big|group|grouping|grouping_id|max|min|sum|stdev|stdevp|var|varp)\\\\b\\\\s*\\\\(\",\n \"captures\": {\n \"1\": {\n \"name\": \"support.function.aggregate.sql\"\n }\n }\n },\n {\n \"match\": \"(?i)\\\\b(cume_dist|first_value|lag|last_value|lead|percent_rank|percentile_cont|percentile_disc)\\\\b\\\\s*\\\\(\",\n \"captures\": {\n \"1\": {\n \"name\": \"support.function.analytic.sql\"\n }\n }\n },\n {\n \"match\": \"(?i)\\\\b(bit_count|get_bit|left_shift|right_shift|set_bit)\\\\b\\\\s*\\\\(\",\n \"captures\": {\n \"1\": {\n \"name\": \"support.function.bitmanipulation.sql\"\n }\n }\n },\n {\n \"match\": \"(?i)\\\\b(cast|convert|parse|try_cast|try_convert|try_parse)\\\\b\\\\s*\\\\(\",\n \"captures\": {\n \"1\": {\n \"name\": \"support.function.conversion.sql\"\n }\n }\n },\n {\n \"match\": \"(?i)\\\\b(collationproperty|tertiary_weights)\\\\b\\\\s*\\\\(\",\n \"captures\": {\n \"1\": {\n \"name\": \"support.function.collation.sql\"\n }\n }\n },\n {\n \"match\": \"(?i)\\\\b(asymkey_id|asymkeyproperty|certproperty|cert_id|crypt_gen_random|decryptbyasymkey|decryptbycert|decryptbykey|decryptbykeyautoasymkey|decryptbykeyautocert|decryptbypassphrase|encryptbyasymkey|encryptbycert|encryptbykey|encryptbypassphrase|hashbytes|is_objectsigned|key_guid|key_id|key_name|signbyasymkey|signbycert|symkeyproperty|verifysignedbycert|verifysignedbyasymkey)\\\\b\\\\s*\\\\(\",\n \"captures\": {\n \"1\": {\n \"name\": \"support.function.cryptographic.sql\"\n }\n }\n },\n {\n \"match\": \"(?i)\\\\b(cursor_status)\\\\b\\\\s*\\\\(\",\n \"captures\": {\n \"1\": {\n \"name\": \"support.function.cursor.sql\"\n }\n }\n },\n {\n \"match\": \"(?i)\\\\b(sysdatetime|sysdatetimeoffset|sysutcdatetime|current_time(stamp)?|getdate|getutcdate|datename|datepart|day|month|year|datefromparts|datetime2fromparts|datetimefromparts|datetimeoffsetfromparts|smalldatetimefromparts|timefromparts|datediff|dateadd|datetrunc|eomonth|switchoffset|todatetimeoffset|isdate|date_bucket)\\\\b\\\\s*\\\\(\",\n \"captures\": {\n \"1\": {\n \"name\": \"support.function.datetime.sql\"\n }\n }\n },\n {\n \"match\": \"(?i)\\\\b(datalength|ident_current|ident_incr|ident_seed|identity|sql_variant_property)\\\\b\\\\s*\\\\(\",\n \"captures\": {\n \"1\": {\n \"name\": \"support.function.datatype.sql\"\n }\n }\n },\n {\n \"match\": \"(?i)\\\\b(coalesce|nullif)\\\\b\\\\s*\\\\(\",\n \"captures\": {\n \"1\": {\n \"name\": \"support.function.expression.sql\"\n }\n }\n },\n {\n \"match\": \"(?(new Uint8Array(embedding.buffer)!);\nconsole.log(result); // 4\n```\n\nSee\n[`simple-deno/demo.ts`](https://github.com/asg017/sqlite-vec/blob/main/examples/simple-deno/demo.ts)\nfor a more complete Deno demo.\n\nThe `better-sqlite3` example above also works in Deno, when the `better-sqlite3` import is prefixed with `npm:`:\n\n```js\nimport * from \"better-sqlite3\"; // [!code --]\nimport * from \"npm:better-sqlite3\"; // [!code ++]\n```\n\n## Bun\n\nHere's a quick recipe of using `sqlite-vec` with\n[`bun:sqlite`](https://bun.sh/docs/api/sqlite) in Bun. \n\n```ts\nimport { Database } from \"bun:sqlite\";\nimport * as sqliteVec from \"sqlite-vec\";\n\n// MacOS *might* have to do this, as the builtin SQLite library on MacOS doesn't allow extensions\nDatabase.setCustomSQLite(\"/usr/local/opt/sqlite3/lib/libsqlite3.dylib\");\n\nconst db = new Database(\":memory:\");\nsqliteVec.load(db);\n\nconst embedding = new Float32Array([0.1, 0.2, 0.3, 0.4]);\nconst { result } = db\n .prepare(\"select vec_length(?) as result\",)\n .get(embedding);\n\nconsole.log(result); // 4\n\n```\n\nSee\n[`simple-bun/demo.ts`](https://github.com/asg017/sqlite-vec/blob/main/examples/simple-bun/demo.ts)\nfor a more complete Bun demo.\n\nThe `better-sqlite3`\nexample above also works with Bun." }, { "path": "site/using/python.md", "content": "---\ntitle: sqlite-vec in Python\n---\n\n# Using `sqlite-vec` in Python\n\n[![PyPI](https://img.shields.io/pypi/v/sqlite-vec.svg?color=blue&logo=python&logoColor=white)](https://pypi.org/project/sqlite-vec/)\n\nTo use `sqlite-vec` from Python, install the\n[`sqlite-vec` PyPi package](https://pypi.org/project/sqlite-vec/) using your\nfavorite Python package manager:\n\n```bash\npip install sqlite-vec\n```\n\nOnce installed, use the `sqlite_vec.load()` function to load `sqlite-vec` SQL\nfunctions into a SQLite connection.\n\n```python\nimport sqlite3\nimport sqlite_vec\n\ndb = sqlite3.connect(\":memory:\")\ndb.enable_load_extension(True)\nsqlite_vec.load(db)\ndb.enable_load_extension(False)\n\nvec_version, = db.execute(\"select vec_version()\").fetchone()\nprint(f\"vec_version={vec_version}\")\n```\n\nSee\n[`simple-python/demo.py`](https://github.com/asg017/sqlite-vec/blob/main/examples/simple-python/demo.py)\nfor a more complete Python demo.\n\n## Working with Vectors\n\n### Lists\n\nIf your vectors in Python are provided as a list of floats, you can\nconvert them into the compact BLOB format that `sqlite-vec` uses with\n`serialize_float32()`. This internally calls [`struct.pack()`](https://docs.python.org/3/library/struct.html#struct.pack).\n\n```python\nfrom sqlite_vec import serialize_float32\n\nembedding = [0.1, 0.2, 0.3, 0.4]\nresult = db.execute('select vec_length(?)', [serialize_float32(embedding)])\n\nprint(result.fetchone()[0]) # 4\n```\n\n### NumPy Arrays\n\nIf your vectors are NumPy arrays, the Python SQLite package allows you to\npass it along as-is, since NumPy arrays implement [the Buffer protocol](https://docs.python.org/3/c-api/buffer.html). Make sure you cast your array elements to 32-bit floats\nwith\n[`.astype(np.float32)`](https://numpy.org/doc/stable/reference/generated/numpy.ndarray.astype.html),\nas some embeddings will use `np.float64`.\n\n```python\nimport numpy as np\nembedding = np.array([0.1, 0.2, 0.3, 0.4])\ndb.execute(\n \"SELECT vec_length(?)\", [embedding.astype(np.float32)]\n) # 4\n```\n\n\n## Using an up-to-date version of SQLite {#updated-sqlite}\n\nSome features of `sqlite-vec` will require an up-to-date SQLite library. You can\nsee what version of SQLite your Python environment uses with\n[`sqlite3.sqlite_version`](https://docs.python.org/3/library/sqlite3.html#sqlite3.sqlite_version),\nor with this one-line command:\n\n```bash\npython -c 'import sqlite3; print(sqlite3.sqlite_version)'\n```\n\nCurrently, **SQLite version 3.41 or higher** is recommended but not required.\n`sqlite-vec` will work with older versions, but certain features and queries will\nonly work correctly in >=3.41.\n\nTo \"upgrade\" the SQLite version your Python installation uses, you have a few\noptions.\n\n### Compile your own SQLite version\n\nYou can compile an up-to-date version of SQLite and use some system environment\nvariables (like `LD_PRELOAD` and `DYLD_LIBRARY_PATH`) to force Python to use a\ndifferent SQLite library.\n[This guide](https://til.simonwillison.net/sqlite/sqlite-version-macos-python)\ngoes into this approach in more details.\n\nAlthough compiling SQLite can be straightforward, there are a lot of different\ncompilation options to consider, which makes it confusing. This also doesn't\nwork with Windows, which statically compiles its own SQLite library.\n\n### Use `pysqlite3`\n\n[`pysqlite3`](https://github.com/coleifer/pysqlite3) is a 3rd party PyPi package\nthat bundles an up-to-date SQLite library as a separate pip package.\n\nWhile it's mostly compatible with the Python `sqlite3` module, there are a few\nrare edge cases where the APIs don't match.\n\n### Upgrading your Python version\n\nSometimes installing a latest version of Python will \"magically\" upgrade your\nSQLite version as well. This is a nuclear option, as upgrading Python\ninstallations can be quite the hassle, but most Python 3.12 builds will have a\nvery recent SQLite version.\n\n\n## MacOS blocks SQLite extensions by default\n\nThe default SQLite library that is bundled with Mac operating systems do not include support for SQLite extensions. That means the default Python library that is bundled with MacOS also does not support SQLite extensions.\n\nThis is the case if you come across the following error message:\n\n```\nAttributeError: 'sqlite3.Connection' object has no attribute 'enable_load_extension'\n```\n\nAs a workaround, use the Homebrew version of Python (`brew install python`, new version at `/opt/homebrew/bin/python3`), which will use the Homebrew version of SQLite that allows SQLite extensions.\n\nOther workarounds can be found at [Using an up-to-date version of SQLite](#updated-sqlite);\n" }, { "path": "site/using/rqlite.md", "content": "# Using `sqlite-vec` in rqlite\n\n[rqlite](https://rqlite.io/) users can use `sqlite-vec` with rqlite by loading the extension when they launch their rqlite node:\n\n```bash\n# Download a sqlite-vec release.\ncurl -L https://github.com/asg017/sqlite-vec/releases/download/v0.1.1/sqlite-vec-0.1.1-loadable-linux-x86_64.tar.gz -o sqlite-vec.tar.gz\n\n# Tell rqlite to load sqlite-vec at launch time.\nrqlited -extensions-path=sqlite-vec.tar.gz data\n```\n\nOnce loaded you can use `sqlite-vec` functionality within rqlite. For example, you can perform searches via the [rqlite shell](https://rqlite.io/docs/cli/):\n\n```\n$ rqlite\nWelcome to the rqlite CLI.\nEnter \".help\" for usage hints.\nConnected to http://127.0.0.1:4001 running version 8\n127.0.0.1:4001> create virtual table vec_examples using vec0(sample_embedding float[8]);\n1 row affected\n127.0.0.1:4001> insert into vec_examples(rowid, sample_embedding) values (1, '[-0.200, 0.250, 0.341, -0.211, 0.645, 0.935, -0.316, -0.924]'), (2, '[0.443, -0.501, 0.355, -0.771, 0.707, -0.708, -0.185, 0.362]'), (3, '[0.716, -0.927, 0.134, 0.052, -0.669, 0.793, -0.634, -0.162]'), (4, '[-0.710, 0.330, 0.656, 0.041, -0.990, 0.726, 0.385, -0.958]')\n4 rows affected\n127.0.0.1:4001> select rowid, distance from vec_examples where sample_embedding match '[0.890, 0.544, 0.825, 0.961, 0.358, 0.0196, 0.521, 0.175]' order by distance limit 2\n+-------+-------------------+\n| rowid | distance |\n+-------+-------------------+\n| 2 | 2.386873722076416 |\n+-------+-------------------+\n| 1 | 2.389785051345825 |\n+-------+-------------------+\n```\n\nYou can learn more from the [rqlite website](https://rqlite.io/docs/guides/extensions/).\n\n" }, { "path": "site/using/ruby.md", "content": "# Using `sqlite-vec` in Ruby\n\n![Gem](https://img.shields.io/gem/v/sqlite-vec?color=red&logo=rubygems&logoColor=white)\n\nRuby developers can use `sqlite-vec` with the [`sqlite-vec` Gem](https://rubygems.org/gems/sqlite-vec).\n\n\n```bash\ngem install sqlite-vec\n```\n\nYou can then use `SqliteVec.load()` to load `sqlite-vec` SQL functions in a given SQLite connection.\n\n```ruby\nrequire 'sqlite3'\nrequire 'sqlite_vec'\n\ndb = SQLite3::Database.new(':memory:')\ndb.enable_load_extension(true)\nSqliteVec.load(db)\ndb.enable_load_extension(false)\n\nresult = db.execute('SELECT vec_version()')\nputs result.first.first\n\n```\n\nSee\n[`simple-ruby/demo.rb`](https://github.com/asg017/sqlite-vec/blob/main/examples/simple-ruby/demo.rb)\nfor a more complete Ruby demo.\n\n## Working with vectors in Ruby\n\nIf your embeddings are provided as a list of numbers, use `.pack(\"f*\")` to convert them into the compact BLOB format that `sqlite-vec` uses.\n\n```ruby\nembedding = [0.1, 0.2, 0.3, 0.4]\nresult = db.execute(\"SELECT vec_length(?)\", [query.pack(\"f*\")]])\nputs result.first.first # 4\n```\n" }, { "path": "site/using/rust.md", "content": "# Using `sqlite-vec` in Rust\n[![Crates.io](https://img.shields.io/crates/v/sqlite-vec?logo=rust)](https://crates.io/crates/sqlite-vec)\n\nYou can embed `sqlite-vec` into your Rust projects using the official\n[`sqlite-vec` crate](https://crates.io/crates/sqlite-vec).\n\n```bash\ncargo add sqlite-vec\n```\n\nThe crate embeds the `sqlite-vec` C source code, and uses the\n[`cc` crate](https://crates.io/crates/sqlite-vec) to compile and statically link\n`sqlite-vec` at build-time.\n\nThe `sqlite-vec` crate exposes a single function `sqlite3_vec_init`, which is\nthe C entrypoint for the SQLite extension. You can \"register\" with your Rust\nSQLite library's `sqlite3_auto_extension()` function.\n\nThis example registers sqlite-vec using [rusqlite](https://docs.rs/rusqlite/0.32.1/rusqlite/). First, enable the `\"bundled\"` feature in your Cargo file entry for rusqlite:\n\n```diff\n# Cargo.toml\n[dependencies]\n+ rusqlite = { version = \"VERSION\", features = [\"bundled\"] }\n```\n\nThen, you can verify your installation was successful by embedding your first vector. This example uses [zerocopy](https://docs.rs/zerocopy/latest/zerocopy/) to efficiently pass the vector as bytes, and prints the resulting vector and library version as Strings:\n\n```rs\nuse sqlite_vec::sqlite3_vec_init;\nuse rusqlite::{ffi::sqlite3_auto_extension, Result};\nuse zerocopy::AsBytes;\n\nfn main()-> Result<()> {\n unsafe {\n sqlite3_auto_extension(Some(std::mem::transmute(sqlite3_vec_init as *const ())));\n }\n\n let db = Connection::open_in_memory()?;\n let v: Vec = vec![0.1, 0.2, 0.3];\n\n let (vec_version, embedding): (String, String) = db.query_row(\n \"select vec_version(), vec_to_json(?)\",\n &[v.as_bytes()],\n |x| Ok((x.get(0)?, x.get(1)?)),\n )?;\n\n println!(\"vec_version={vec_version}, embedding={embedding}\");\n Ok(())\n}\n```\n\nSee\n[`simple-rust/demo.rs`](https://github.com/asg017/sqlite-vec/blob/main/examples/simple-rust/demo.rs)\nfor a more complete Rust demo.\n\n## Working with vectors in Rust\n\nIf your vectors are provided as a `Vec` type, the [`zerocopy` crate](https://crates.io/crates/zerocopy) is recommended, specifically `zerocopy::AsBytes`. This will allow you to pass in vectors into `sqlite-vec` without any copying.\n\n```rs\nlet query: Vec = vec![0.1, 0.2, 0.3, 0.4];\nlet mut stmt = db.prepare(\"SELECT vec_length(?)\")?;\nstmt.execute(&[item.1.as_bytes()])?;\n```\n" }, { "path": "site/using/sqlite-utils.md", "content": "# Using `sqlite-vec` in `sqlite-utils`\n\n![sqlite-utils](https://img.shields.io/pypi/v/sqlite-utils-sqlite-vec.svg?color=B6B6D9&label=sqlite-utils+plugin&logoColor=white&logo=python)\n\n[`sqlite-utils`](https://sqlite-utils.datasette.io/en/stable/) users can install `sqlite-vec` into their `sqlite-utils` projects with the `sqlite-utils-sqlite-vec` plugin:\n\n\n```bash\nsqlite-utils install sqlite-utils-sqlite-vec\n```\n" }, { "path": "site/using/wasm.md", "content": "# `sqlite-vec` in the Browser with WebAssembly\n\n`sqlite-vec` can be statically compiled into [official SQLite WASM](https://sqlite.org/wasm/doc/trunk/index.md) builds. The process is a bit complicated, but the result is a vector search in the browser, which is pretty cool!\n\n```html\n\n \n \n \n\n```\n[*Open in CodePen*](https://codepen.io/asg017_ucsd/pen/MWMpJNY)\n\n\nIt's not possibly to dynamically load a SQLite extension into a WASM build of SQLite. So `sqlite-vec` must be statically compiled into custom WASM builds.\n\n## The `sqlite-vec-wasm-demo` NPM package\n\nA **demonstration** of `sqlite-vec` in WASM is provided with the `sqlite-vec-wasm-demo` NPM package. This package is a demonstration and may change at any time. It doesn't follow the [Semantic version of `sqlite-vec`](./versioning.md).\n\n\nSee\n[`simple-wasm/index.html`](https://github.com/asg017/sqlite-vec/blob/main/examples/simple-wasm/index.html)\nfor a more complete WASM demo using this package.\n" }, { "path": "site/versioning.md", "content": "# Semantic Versioning for `sqlite-vec`\n\n`sqlite-vec` is pre-v1, so according to the rules of\n[Semantic Versioning](https://semver.org/), so \"minor\" release like \"0.2.0\" or\n\"0.3.0\" may contain breaking changes.\n\nOnly SQL functions, table functions, and virtual tables that are defined in the default `sqlite3_vec_init` entrypoint are considered as the `sqlite-vec` API for semantic versioning. This means that other entrypoints and other SQL functions should be considered unstable, untested, and possibly dangerous.\n\nFor the SQL API, a \"breaking change\" would include:\n\n- Removing a function or module\n- Changing the number or types of arguments for an SQL function\n- Changing the require arguments of position of a table functions\n- Changing the `CREATE VIRTUAL TABLE` constructor of a virtual table in a backwards-incompatible way\n- Removing columns from a virtual table or table function\n\n\nThe official \"bindings\" to `sqlite-vec`, including the Python/Node.js/Ruby/Go/Rust are subject to change and are not covered by semantic versioning.\nThough I have no plans to change or break them, and would include notes in changelogs if that ever needs to happen.\n" }, { "path": "sqlite-dist.toml", "content": "[package]\nname = \"sqlite-vec\"\nlicense = \"MIT OR Apache\"\nhomepage = \"https://alexgarcia.xyz/sqlite-vec\"\nrepo = \"https://github.com/asg017/sqlite-vec\"\ndescription = \"A vector search SQLite extension.\"\nauthors = [\"Alex Garcia\"]\ngit_tag_format = \"v$VERSION\"\n\n[targets]\ngithub_releases = {}\nsqlpkg = {}\nspm = {}\namalgamation = {include=[\"sqlite-vec.c\", \"sqlite-vec.h\"]}\n\npip = { extra_init_py = \"bindings/python/extra_init.py\" }\ndatasette = {}\nsqlite_utils = {}\n\nnpm = {}\n\ngem = { module_name = \"SqliteVec\" }\n" }, { "path": "sqlite-vec.c", "content": "#include \"sqlite-vec.h\"\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n\n#ifndef SQLITE_VEC_OMIT_FS\n#include \n#endif\n\n#ifndef SQLITE_CORE\n#include \"sqlite3ext.h\"\nSQLITE_EXTENSION_INIT1\n#else\n#include \"sqlite3.h\"\n#endif\n\n#ifndef UINT32_TYPE\n#ifdef HAVE_UINT32_T\n#define UINT32_TYPE uint32_t\n#else\n#define UINT32_TYPE unsigned int\n#endif\n#endif\n#ifndef UINT16_TYPE\n#ifdef HAVE_UINT16_T\n#define UINT16_TYPE uint16_t\n#else\n#define UINT16_TYPE unsigned short int\n#endif\n#endif\n#ifndef INT16_TYPE\n#ifdef HAVE_INT16_T\n#define INT16_TYPE int16_t\n#else\n#define INT16_TYPE short int\n#endif\n#endif\n#ifndef UINT8_TYPE\n#ifdef HAVE_UINT8_T\n#define UINT8_TYPE uint8_t\n#else\n#define UINT8_TYPE unsigned char\n#endif\n#endif\n#ifndef INT8_TYPE\n#ifdef HAVE_INT8_T\n#define INT8_TYPE int8_t\n#else\n#define INT8_TYPE signed char\n#endif\n#endif\n#ifndef LONGDOUBLE_TYPE\n#define LONGDOUBLE_TYPE long double\n#endif\n\n#ifndef _WIN32\n#ifndef __EMSCRIPTEN__\n#ifndef __COSMOPOLITAN__\n#ifndef __wasi__\ntypedef u_int8_t uint8_t;\ntypedef u_int16_t uint16_t;\ntypedef u_int64_t uint64_t;\n#endif\n#endif\n#endif\n#endif\n\ntypedef int8_t i8;\ntypedef uint8_t u8;\ntypedef int16_t i16;\ntypedef int32_t i32;\ntypedef sqlite3_int64 i64;\ntypedef uint32_t u32;\ntypedef uint64_t u64;\ntypedef float f32;\ntypedef size_t usize;\n\n#ifndef UNUSED_PARAMETER\n#define UNUSED_PARAMETER(X) (void)(X)\n#endif\n\n// sqlite3_vtab_in() was added in SQLite version 3.38 (2022-02-22)\n// https://www.sqlite.org/changes.html#version_3_38_0\n#if SQLITE_VERSION_NUMBER >= 3038000\n#define COMPILER_SUPPORTS_VTAB_IN 1\n#endif\n\n#ifndef SQLITE_SUBTYPE\n#define SQLITE_SUBTYPE 0x000100000\n#endif\n\n#ifndef SQLITE_RESULT_SUBTYPE\n#define SQLITE_RESULT_SUBTYPE 0x001000000\n#endif\n\n#ifndef SQLITE_INDEX_CONSTRAINT_LIMIT\n#define SQLITE_INDEX_CONSTRAINT_LIMIT 73\n#endif\n\n#ifndef SQLITE_INDEX_CONSTRAINT_OFFSET\n#define SQLITE_INDEX_CONSTRAINT_OFFSET 74\n#endif\n\n#define countof(x) (sizeof(x) / sizeof((x)[0]))\n#define min(a, b) (((a) <= (b)) ? (a) : (b))\n\nenum VectorElementType {\n // clang-format off\n SQLITE_VEC_ELEMENT_TYPE_FLOAT32 = 223 + 0,\n SQLITE_VEC_ELEMENT_TYPE_BIT = 223 + 1,\n SQLITE_VEC_ELEMENT_TYPE_INT8 = 223 + 2,\n // clang-format on\n};\n\n#ifdef SQLITE_VEC_ENABLE_AVX\n#include \n#define PORTABLE_ALIGN32 __attribute__((aligned(32)))\n#define PORTABLE_ALIGN64 __attribute__((aligned(64)))\n\nstatic f32 l2_sqr_float_avx(const void *pVect1v, const void *pVect2v,\n const void *qty_ptr) {\n f32 *pVect1 = (f32 *)pVect1v;\n f32 *pVect2 = (f32 *)pVect2v;\n size_t qty = *((size_t *)qty_ptr);\n f32 PORTABLE_ALIGN32 TmpRes[8];\n size_t qty16 = qty >> 4;\n\n const f32 *pEnd1 = pVect1 + (qty16 << 4);\n\n __m256 diff, v1, v2;\n __m256 sum = _mm256_set1_ps(0);\n\n while (pVect1 < pEnd1) {\n v1 = _mm256_loadu_ps(pVect1);\n pVect1 += 8;\n v2 = _mm256_loadu_ps(pVect2);\n pVect2 += 8;\n diff = _mm256_sub_ps(v1, v2);\n sum = _mm256_add_ps(sum, _mm256_mul_ps(diff, diff));\n\n v1 = _mm256_loadu_ps(pVect1);\n pVect1 += 8;\n v2 = _mm256_loadu_ps(pVect2);\n pVect2 += 8;\n diff = _mm256_sub_ps(v1, v2);\n sum = _mm256_add_ps(sum, _mm256_mul_ps(diff, diff));\n }\n\n _mm256_store_ps(TmpRes, sum);\n return sqrt(TmpRes[0] + TmpRes[1] + TmpRes[2] + TmpRes[3] + TmpRes[4] +\n TmpRes[5] + TmpRes[6] + TmpRes[7]);\n}\n#endif\n\n#ifdef SQLITE_VEC_ENABLE_NEON\n#include \n\n#define PORTABLE_ALIGN32 __attribute__((aligned(32)))\n\n// thx https://github.com/nmslib/hnswlib/pull/299/files\nstatic f32 l2_sqr_float_neon(const void *pVect1v, const void *pVect2v,\n const void *qty_ptr) {\n f32 *pVect1 = (f32 *)pVect1v;\n f32 *pVect2 = (f32 *)pVect2v;\n size_t qty = *((size_t *)qty_ptr);\n size_t qty16 = qty >> 4;\n\n const f32 *pEnd1 = pVect1 + (qty16 << 4);\n\n float32x4_t diff, v1, v2;\n float32x4_t sum0 = vdupq_n_f32(0);\n float32x4_t sum1 = vdupq_n_f32(0);\n float32x4_t sum2 = vdupq_n_f32(0);\n float32x4_t sum3 = vdupq_n_f32(0);\n\n while (pVect1 < pEnd1) {\n v1 = vld1q_f32(pVect1);\n pVect1 += 4;\n v2 = vld1q_f32(pVect2);\n pVect2 += 4;\n diff = vsubq_f32(v1, v2);\n sum0 = vfmaq_f32(sum0, diff, diff);\n\n v1 = vld1q_f32(pVect1);\n pVect1 += 4;\n v2 = vld1q_f32(pVect2);\n pVect2 += 4;\n diff = vsubq_f32(v1, v2);\n sum1 = vfmaq_f32(sum1, diff, diff);\n\n v1 = vld1q_f32(pVect1);\n pVect1 += 4;\n v2 = vld1q_f32(pVect2);\n pVect2 += 4;\n diff = vsubq_f32(v1, v2);\n sum2 = vfmaq_f32(sum2, diff, diff);\n\n v1 = vld1q_f32(pVect1);\n pVect1 += 4;\n v2 = vld1q_f32(pVect2);\n pVect2 += 4;\n diff = vsubq_f32(v1, v2);\n sum3 = vfmaq_f32(sum3, diff, diff);\n }\n\n f32 sum_scalar =\n vaddvq_f32(vaddq_f32(vaddq_f32(sum0, sum1), vaddq_f32(sum2, sum3)));\n const f32 *pEnd2 = pVect1 + (qty - (qty16 << 4));\n while (pVect1 < pEnd2) {\n f32 diff = *pVect1 - *pVect2;\n sum_scalar += diff * diff;\n pVect1++;\n pVect2++;\n }\n\n return sqrt(sum_scalar);\n}\n\nstatic f32 l2_sqr_int8_neon(const void *pVect1v, const void *pVect2v,\n const void *qty_ptr) {\n i8 *pVect1 = (i8 *)pVect1v;\n i8 *pVect2 = (i8 *)pVect2v;\n size_t qty = *((size_t *)qty_ptr);\n\n const i8 *pEnd1 = pVect1 + qty;\n i32 sum_scalar = 0;\n\n while (pVect1 < pEnd1 - 7) {\n // loading 8 at a time\n int8x8_t v1 = vld1_s8(pVect1);\n int8x8_t v2 = vld1_s8(pVect2);\n pVect1 += 8;\n pVect2 += 8;\n\n // widen to protect against overflow\n int16x8_t v1_wide = vmovl_s8(v1);\n int16x8_t v2_wide = vmovl_s8(v2);\n\n int16x8_t diff = vsubq_s16(v1_wide, v2_wide);\n int16x8_t squared_diff = vmulq_s16(diff, diff);\n int32x4_t sum = vpaddlq_s16(squared_diff);\n\n sum_scalar += vgetq_lane_s32(sum, 0) + vgetq_lane_s32(sum, 1) +\n vgetq_lane_s32(sum, 2) + vgetq_lane_s32(sum, 3);\n }\n\n // handle leftovers\n while (pVect1 < pEnd1) {\n i16 diff = (i16)*pVect1 - (i16)*pVect2;\n sum_scalar += diff * diff;\n pVect1++;\n pVect2++;\n }\n\n return sqrtf(sum_scalar);\n}\n\nstatic i32 l1_int8_neon(const void *pVect1v, const void *pVect2v,\n const void *qty_ptr) {\n i8 *pVect1 = (i8 *)pVect1v;\n i8 *pVect2 = (i8 *)pVect2v;\n size_t qty = *((size_t *)qty_ptr);\n\n const int8_t *pEnd1 = pVect1 + qty;\n\n int32x4_t acc1 = vdupq_n_s32(0);\n int32x4_t acc2 = vdupq_n_s32(0);\n int32x4_t acc3 = vdupq_n_s32(0);\n int32x4_t acc4 = vdupq_n_s32(0);\n\n while (pVect1 < pEnd1 - 63) {\n int8x16_t v1 = vld1q_s8(pVect1);\n int8x16_t v2 = vld1q_s8(pVect2);\n int8x16_t diff1 = vabdq_s8(v1, v2);\n acc1 = vaddq_s32(acc1, vpaddlq_u16(vpaddlq_u8(diff1)));\n\n v1 = vld1q_s8(pVect1 + 16);\n v2 = vld1q_s8(pVect2 + 16);\n int8x16_t diff2 = vabdq_s8(v1, v2);\n acc2 = vaddq_s32(acc2, vpaddlq_u16(vpaddlq_u8(diff2)));\n\n v1 = vld1q_s8(pVect1 + 32);\n v2 = vld1q_s8(pVect2 + 32);\n int8x16_t diff3 = vabdq_s8(v1, v2);\n acc3 = vaddq_s32(acc3, vpaddlq_u16(vpaddlq_u8(diff3)));\n\n v1 = vld1q_s8(pVect1 + 48);\n v2 = vld1q_s8(pVect2 + 48);\n int8x16_t diff4 = vabdq_s8(v1, v2);\n acc4 = vaddq_s32(acc4, vpaddlq_u16(vpaddlq_u8(diff4)));\n\n pVect1 += 64;\n pVect2 += 64;\n }\n\n while (pVect1 < pEnd1 - 15) {\n int8x16_t v1 = vld1q_s8(pVect1);\n int8x16_t v2 = vld1q_s8(pVect2);\n int8x16_t diff = vabdq_s8(v1, v2);\n acc1 = vaddq_s32(acc1, vpaddlq_u16(vpaddlq_u8(diff)));\n pVect1 += 16;\n pVect2 += 16;\n }\n\n int32x4_t acc = vaddq_s32(vaddq_s32(acc1, acc2), vaddq_s32(acc3, acc4));\n\n int32_t sum = 0;\n while (pVect1 < pEnd1) {\n int32_t diff = abs((int32_t)*pVect1 - (int32_t)*pVect2);\n sum += diff;\n pVect1++;\n pVect2++;\n }\n\n return vaddvq_s32(acc) + sum;\n}\n\nstatic double l1_f32_neon(const void *pVect1v, const void *pVect2v,\n const void *qty_ptr) {\n f32 *pVect1 = (f32 *)pVect1v;\n f32 *pVect2 = (f32 *)pVect2v;\n size_t qty = *((size_t *)qty_ptr);\n\n const f32 *pEnd1 = pVect1 + qty;\n float64x2_t acc = vdupq_n_f64(0);\n\n while (pVect1 < pEnd1 - 3) {\n float32x4_t v1 = vld1q_f32(pVect1);\n float32x4_t v2 = vld1q_f32(pVect2);\n pVect1 += 4;\n pVect2 += 4;\n\n // f32x4 -> f64x2 pad for overflow\n float64x2_t low_diff = vabdq_f64(vcvt_f64_f32(vget_low_f32(v1)),\n vcvt_f64_f32(vget_low_f32(v2)));\n float64x2_t high_diff =\n vabdq_f64(vcvt_high_f64_f32(v1), vcvt_high_f64_f32(v2));\n\n acc = vaddq_f64(acc, vaddq_f64(low_diff, high_diff));\n }\n\n double sum = 0;\n while (pVect1 < pEnd1) {\n sum += fabs((double)*pVect1 - (double)*pVect2);\n pVect1++;\n pVect2++;\n }\n\n return vaddvq_f64(acc) + sum;\n}\n#endif\n\nstatic f32 l2_sqr_float(const void *pVect1v, const void *pVect2v,\n const void *qty_ptr) {\n f32 *pVect1 = (f32 *)pVect1v;\n f32 *pVect2 = (f32 *)pVect2v;\n size_t qty = *((size_t *)qty_ptr);\n\n f32 res = 0;\n for (size_t i = 0; i < qty; i++) {\n f32 t = *pVect1 - *pVect2;\n pVect1++;\n pVect2++;\n res += t * t;\n }\n return sqrt(res);\n}\n\nstatic f32 l2_sqr_int8(const void *pA, const void *pB, const void *pD) {\n i8 *a = (i8 *)pA;\n i8 *b = (i8 *)pB;\n size_t d = *((size_t *)pD);\n\n f32 res = 0;\n for (size_t i = 0; i < d; i++) {\n f32 t = *a - *b;\n a++;\n b++;\n res += t * t;\n }\n return sqrt(res);\n}\n\nstatic f32 distance_l2_sqr_float(const void *a, const void *b, const void *d) {\n#ifdef SQLITE_VEC_ENABLE_NEON\n if ((*(const size_t *)d) > 16) {\n return l2_sqr_float_neon(a, b, d);\n }\n#endif\n#ifdef SQLITE_VEC_ENABLE_AVX\n if (((*(const size_t *)d) % 16 == 0)) {\n return l2_sqr_float_avx(a, b, d);\n }\n#endif\n return l2_sqr_float(a, b, d);\n}\n\nstatic f32 distance_l2_sqr_int8(const void *a, const void *b, const void *d) {\n#ifdef SQLITE_VEC_ENABLE_NEON\n if ((*(const size_t *)d) > 7) {\n return l2_sqr_int8_neon(a, b, d);\n }\n#endif\n return l2_sqr_int8(a, b, d);\n}\n\nstatic i32 l1_int8(const void *pA, const void *pB, const void *pD) {\n i8 *a = (i8 *)pA;\n i8 *b = (i8 *)pB;\n size_t d = *((size_t *)pD);\n\n i32 res = 0;\n for (size_t i = 0; i < d; i++) {\n res += abs(*a - *b);\n a++;\n b++;\n }\n\n return res;\n}\n\nstatic i32 distance_l1_int8(const void *a, const void *b, const void *d) {\n#ifdef SQLITE_VEC_ENABLE_NEON\n if ((*(const size_t *)d) > 15) {\n return l1_int8_neon(a, b, d);\n }\n#endif\n return l1_int8(a, b, d);\n}\n\nstatic double l1_f32(const void *pA, const void *pB, const void *pD) {\n f32 *a = (f32 *)pA;\n f32 *b = (f32 *)pB;\n size_t d = *((size_t *)pD);\n\n double res = 0;\n for (size_t i = 0; i < d; i++) {\n res += fabs((double)*a - (double)*b);\n a++;\n b++;\n }\n\n return res;\n}\n\nstatic double distance_l1_f32(const void *a, const void *b, const void *d) {\n#ifdef SQLITE_VEC_ENABLE_NEON\n if ((*(const size_t *)d) > 3) {\n return l1_f32_neon(a, b, d);\n }\n#endif\n return l1_f32(a, b, d);\n}\n\nstatic f32 distance_cosine_float(const void *pVect1v, const void *pVect2v,\n const void *qty_ptr) {\n f32 *pVect1 = (f32 *)pVect1v;\n f32 *pVect2 = (f32 *)pVect2v;\n size_t qty = *((size_t *)qty_ptr);\n\n f32 dot = 0;\n f32 aMag = 0;\n f32 bMag = 0;\n for (size_t i = 0; i < qty; i++) {\n dot += *pVect1 * *pVect2;\n aMag += *pVect1 * *pVect1;\n bMag += *pVect2 * *pVect2;\n pVect1++;\n pVect2++;\n }\n return 1 - (dot / (sqrt(aMag) * sqrt(bMag)));\n}\nstatic f32 distance_cosine_int8(const void *pA, const void *pB,\n const void *pD) {\n i8 *a = (i8 *)pA;\n i8 *b = (i8 *)pB;\n size_t d = *((size_t *)pD);\n\n f32 dot = 0;\n f32 aMag = 0;\n f32 bMag = 0;\n for (size_t i = 0; i < d; i++) {\n dot += *a * *b;\n aMag += *a * *a;\n bMag += *b * *b;\n a++;\n b++;\n }\n return 1 - (dot / (sqrt(aMag) * sqrt(bMag)));\n}\n\n// https://github.com/facebookresearch/faiss/blob/77e2e79cd0a680adc343b9840dd865da724c579e/faiss/utils/hamming_distance/common.h#L34\nstatic u8 hamdist_table[256] = {\n 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, 1, 2, 2, 3, 2, 3, 3, 4,\n 2, 3, 3, 4, 3, 4, 4, 5, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,\n 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 1, 2, 2, 3, 2, 3, 3, 4,\n 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,\n 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6,\n 4, 5, 5, 6, 5, 6, 6, 7, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,\n 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 2, 3, 3, 4, 3, 4, 4, 5,\n 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,\n 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6,\n 4, 5, 5, 6, 5, 6, 6, 7, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,\n 4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8};\n\nstatic f32 distance_hamming_u8(u8 *a, u8 *b, size_t n) {\n int same = 0;\n for (unsigned long i = 0; i < n; i++) {\n same += hamdist_table[a[i] ^ b[i]];\n }\n return (f32)same;\n}\n\n#ifdef _MSC_VER\n#if !defined(__clang__) && (defined(_M_ARM) || defined(_M_ARM64))\n// From\n// https://github.com/ngtcp2/ngtcp2/blob/b64f1e77b5e0d880b93d31f474147fae4a1d17cc/lib/ngtcp2_ringbuf.c,\n// line 34-43\nstatic unsigned int __builtin_popcountl(unsigned int x) {\n unsigned int c = 0;\n for (; x; ++c) {\n x &= x - 1;\n }\n return c;\n}\n#else\n#include \n#define __builtin_popcountl __popcnt64\n#endif\n#endif\n\nstatic f32 distance_hamming_u64(u64 *a, u64 *b, size_t n) {\n int same = 0;\n for (unsigned long i = 0; i < n; i++) {\n same += __builtin_popcountl(a[i] ^ b[i]);\n }\n return (f32)same;\n}\n\n/**\n * @brief Calculate the hamming distance between two bitvectors.\n *\n * @param a - first bitvector, MUST have d dimensions\n * @param b - second bitvector, MUST have d dimensions\n * @param d - pointer to size_t, MUST be divisible by CHAR_BIT\n * @return f32\n */\nstatic f32 distance_hamming(const void *a, const void *b, const void *d) {\n size_t dimensions = *((size_t *)d);\n\n if ((dimensions % 64) == 0) {\n return distance_hamming_u64((u64 *)a, (u64 *)b, dimensions / 8 / CHAR_BIT);\n }\n return distance_hamming_u8((u8 *)a, (u8 *)b, dimensions / CHAR_BIT);\n}\n\n#ifdef SQLITE_VEC_TEST\nf32 _test_distance_l2_sqr_float(const f32 *a, const f32 *b, size_t dims) {\n return distance_l2_sqr_float(a, b, &dims);\n}\nf32 _test_distance_cosine_float(const f32 *a, const f32 *b, size_t dims) {\n return distance_cosine_float(a, b, &dims);\n}\nf32 _test_distance_hamming(const u8 *a, const u8 *b, size_t dims) {\n return distance_hamming(a, b, &dims);\n}\n#endif\n\n// from SQLite source:\n// https://github.com/sqlite/sqlite/blob/a509a90958ddb234d1785ed7801880ccb18b497e/src/json.c#L153\nstatic const char vecJsonIsSpaceX[] = {\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n};\n\n#define vecJsonIsspace(x) (vecJsonIsSpaceX[(unsigned char)x])\n\ntypedef void (*vector_cleanup)(void *p);\n\nvoid vector_cleanup_noop(void *_) { UNUSED_PARAMETER(_); }\n\n#define JSON_SUBTYPE 74\n\nvoid vtab_set_error(sqlite3_vtab *pVTab, const char *zFormat, ...) {\n va_list args;\n sqlite3_free(pVTab->zErrMsg);\n va_start(args, zFormat);\n pVTab->zErrMsg = sqlite3_vmprintf(zFormat, args);\n va_end(args);\n}\nstruct Array {\n size_t element_size;\n size_t length;\n size_t capacity;\n void *z;\n};\n\n/**\n * @brief Initial an array with the given element size and capacity.\n *\n * @param array\n * @param element_size\n * @param init_capacity\n * @return SQLITE_OK on success, error code on failure. Only error is\n * SQLITE_NOMEM\n */\nint array_init(struct Array *array, size_t element_size, size_t init_capacity) {\n int sz = element_size * init_capacity;\n void *z = sqlite3_malloc(sz);\n if (!z) {\n return SQLITE_NOMEM;\n }\n memset(z, 0, sz);\n\n array->element_size = element_size;\n array->length = 0;\n array->capacity = init_capacity;\n array->z = z;\n return SQLITE_OK;\n}\n\nint array_append(struct Array *array, const void *element) {\n if (array->length == array->capacity) {\n size_t new_capacity = array->capacity * 2 + 100;\n void *z = sqlite3_realloc64(array->z, array->element_size * new_capacity);\n if (z) {\n array->capacity = new_capacity;\n array->z = z;\n } else {\n return SQLITE_NOMEM;\n }\n }\n memcpy(&((unsigned char *)array->z)[array->length * array->element_size],\n element, array->element_size);\n array->length++;\n return SQLITE_OK;\n}\n\nvoid array_cleanup(struct Array *array) {\n if (!array)\n return;\n array->element_size = 0;\n array->length = 0;\n array->capacity = 0;\n sqlite3_free(array->z);\n array->z = NULL;\n}\n\nchar *vector_subtype_name(int subtype) {\n switch (subtype) {\n case SQLITE_VEC_ELEMENT_TYPE_FLOAT32:\n return \"float32\";\n case SQLITE_VEC_ELEMENT_TYPE_INT8:\n return \"int8\";\n case SQLITE_VEC_ELEMENT_TYPE_BIT:\n return \"bit\";\n }\n return \"\";\n}\nchar *type_name(int type) {\n switch (type) {\n case SQLITE_INTEGER:\n return \"INTEGER\";\n case SQLITE_BLOB:\n return \"BLOB\";\n case SQLITE_TEXT:\n return \"TEXT\";\n case SQLITE_FLOAT:\n return \"FLOAT\";\n case SQLITE_NULL:\n return \"NULL\";\n }\n return \"\";\n}\n\ntypedef void (*fvec_cleanup)(void *vector);\n\nvoid fvec_cleanup_noop(void *_) { UNUSED_PARAMETER(_); }\n\nstatic int fvec_from_value(sqlite3_value *value, f32 **vector,\n size_t *dimensions, fvec_cleanup *cleanup,\n char **pzErr) {\n int value_type = sqlite3_value_type(value);\n\n if (value_type == SQLITE_BLOB) {\n const void *blob = sqlite3_value_blob(value);\n int bytes = sqlite3_value_bytes(value);\n if (bytes == 0) {\n *pzErr = sqlite3_mprintf(\"zero-length vectors are not supported.\");\n return SQLITE_ERROR;\n }\n if ((bytes % sizeof(f32)) != 0) {\n *pzErr = sqlite3_mprintf(\"invalid float32 vector BLOB length. Must be \"\n \"divisible by %d, found %d\",\n sizeof(f32), bytes);\n return SQLITE_ERROR;\n }\n f32 *buf = sqlite3_malloc(bytes);\n if (!buf) {\n *pzErr = sqlite3_mprintf(\"out of memory\");\n return SQLITE_NOMEM;\n }\n memcpy(buf, blob, bytes);\n *vector = buf;\n *dimensions = bytes / sizeof(f32);\n *cleanup = sqlite3_free;\n return SQLITE_OK;\n }\n\n if (value_type == SQLITE_TEXT) {\n const char *source = (const char *)sqlite3_value_text(value);\n int source_len = sqlite3_value_bytes(value);\n if (source_len == 0) {\n *pzErr = sqlite3_mprintf(\"zero-length vectors are not supported.\");\n return SQLITE_ERROR;\n }\n int i = 0;\n\n struct Array x;\n int rc = array_init(&x, sizeof(f32), ceil(source_len / 2.0));\n if (rc != SQLITE_OK) {\n return rc;\n }\n\n // advance leading whitespace to first '['\n while (i < source_len) {\n if (vecJsonIsspace(source[i])) {\n i++;\n continue;\n }\n if (source[i] == '[') {\n break;\n }\n\n *pzErr = sqlite3_mprintf(\n \"JSON array parsing error: Input does not start with '['\");\n array_cleanup(&x);\n return SQLITE_ERROR;\n }\n if (source[i] != '[') {\n *pzErr = sqlite3_mprintf(\n \"JSON array parsing error: Input does not start with '['\");\n array_cleanup(&x);\n return SQLITE_ERROR;\n }\n int offset = i + 1;\n\n while (offset < source_len) {\n char *ptr = (char *)&source[offset];\n char *endptr;\n\n errno = 0;\n double result = strtod(ptr, &endptr);\n if ((errno != 0 && result == 0) // some interval error?\n || (errno == ERANGE &&\n (result == HUGE_VAL || result == -HUGE_VAL)) // too big / smalls\n ) {\n sqlite3_free(x.z);\n *pzErr = sqlite3_mprintf(\"JSON parsing error\");\n return SQLITE_ERROR;\n }\n\n if (endptr == ptr) {\n if (*ptr != ']') {\n sqlite3_free(x.z);\n *pzErr = sqlite3_mprintf(\"JSON parsing error\");\n return SQLITE_ERROR;\n }\n goto done;\n }\n\n f32 res = (f32)result;\n array_append(&x, (const void *)&res);\n\n offset += (endptr - ptr);\n while (offset < source_len) {\n if (vecJsonIsspace(source[offset])) {\n offset++;\n continue;\n }\n if (source[offset] == ',') {\n offset++;\n continue;\n }\n if (source[offset] == ']')\n goto done;\n break;\n }\n }\n\n done:\n\n if (x.length > 0) {\n *vector = (f32 *)x.z;\n *dimensions = x.length;\n *cleanup = sqlite3_free;\n return SQLITE_OK;\n }\n sqlite3_free(x.z);\n *pzErr = sqlite3_mprintf(\"zero-length vectors are not supported.\");\n return SQLITE_ERROR;\n }\n\n *pzErr = sqlite3_mprintf(\n \"Input must have type BLOB (compact format) or TEXT (JSON), found %s\",\n type_name(value_type));\n return SQLITE_ERROR;\n}\n\nstatic int bitvec_from_value(sqlite3_value *value, u8 **vector,\n size_t *dimensions, vector_cleanup *cleanup,\n char **pzErr) {\n int value_type = sqlite3_value_type(value);\n if (value_type == SQLITE_BLOB) {\n const void *blob = sqlite3_value_blob(value);\n int bytes = sqlite3_value_bytes(value);\n if (bytes == 0) {\n *pzErr = sqlite3_mprintf(\"zero-length vectors are not supported.\");\n return SQLITE_ERROR;\n }\n *vector = (u8 *)blob;\n *dimensions = bytes * CHAR_BIT;\n *cleanup = vector_cleanup_noop;\n return SQLITE_OK;\n }\n *pzErr = sqlite3_mprintf(\"Unknown type for bitvector.\");\n return SQLITE_ERROR;\n}\n\nstatic int int8_vec_from_value(sqlite3_value *value, i8 **vector,\n size_t *dimensions, vector_cleanup *cleanup,\n char **pzErr) {\n int value_type = sqlite3_value_type(value);\n if (value_type == SQLITE_BLOB) {\n const void *blob = sqlite3_value_blob(value);\n int bytes = sqlite3_value_bytes(value);\n if (bytes == 0) {\n *pzErr = sqlite3_mprintf(\"zero-length vectors are not supported.\");\n return SQLITE_ERROR;\n }\n *vector = (i8 *)blob;\n *dimensions = bytes;\n *cleanup = vector_cleanup_noop;\n return SQLITE_OK;\n }\n\n if (value_type == SQLITE_TEXT) {\n const char *source = (const char *)sqlite3_value_text(value);\n int source_len = sqlite3_value_bytes(value);\n int i = 0;\n\n if (source_len == 0) {\n *pzErr = sqlite3_mprintf(\"zero-length vectors are not supported.\");\n return SQLITE_ERROR;\n }\n\n struct Array x;\n int rc = array_init(&x, sizeof(i8), ceil(source_len / 2.0));\n if (rc != SQLITE_OK) {\n return rc;\n }\n\n // advance leading whitespace to first '['\n while (i < source_len) {\n if (vecJsonIsspace(source[i])) {\n i++;\n continue;\n }\n if (source[i] == '[') {\n break;\n }\n\n *pzErr = sqlite3_mprintf(\n \"JSON array parsing error: Input does not start with '['\");\n array_cleanup(&x);\n return SQLITE_ERROR;\n }\n if (source[i] != '[') {\n *pzErr = sqlite3_mprintf(\n \"JSON array parsing error: Input does not start with '['\");\n array_cleanup(&x);\n return SQLITE_ERROR;\n }\n int offset = i + 1;\n\n while (offset < source_len) {\n char *ptr = (char *)&source[offset];\n char *endptr;\n\n errno = 0;\n long result = strtol(ptr, &endptr, 10);\n if ((errno != 0 && result == 0) ||\n (errno == ERANGE && (result == LONG_MAX || result == LONG_MIN))) {\n sqlite3_free(x.z);\n *pzErr = sqlite3_mprintf(\"JSON parsing error\");\n return SQLITE_ERROR;\n }\n\n if (endptr == ptr) {\n if (*ptr != ']') {\n sqlite3_free(x.z);\n *pzErr = sqlite3_mprintf(\"JSON parsing error\");\n return SQLITE_ERROR;\n }\n goto done;\n }\n\n if (result < INT8_MIN || result > INT8_MAX) {\n sqlite3_free(x.z);\n *pzErr =\n sqlite3_mprintf(\"JSON parsing error: value out of range for int8\");\n return SQLITE_ERROR;\n }\n\n i8 res = (i8)result;\n array_append(&x, (const void *)&res);\n\n offset += (endptr - ptr);\n while (offset < source_len) {\n if (vecJsonIsspace(source[offset])) {\n offset++;\n continue;\n }\n if (source[offset] == ',') {\n offset++;\n continue;\n }\n if (source[offset] == ']')\n goto done;\n break;\n }\n }\n\n done:\n\n if (x.length > 0) {\n *vector = (i8 *)x.z;\n *dimensions = x.length;\n *cleanup = (vector_cleanup)sqlite3_free;\n return SQLITE_OK;\n }\n sqlite3_free(x.z);\n *pzErr = sqlite3_mprintf(\"zero-length vectors are not supported.\");\n return SQLITE_ERROR;\n }\n\n *pzErr = sqlite3_mprintf(\"Unknown type for int8 vector.\");\n return SQLITE_ERROR;\n}\n\n/**\n * @brief Extract a vector from a sqlite3_value. Can be a float32, int8, or bit\n * vector.\n *\n * @param value: the sqlite3_value to read from.\n * @param vector: Output pointer to vector data.\n * @param dimensions: Output number of dimensions\n * @param dimensions: Output vector element type\n * @param cleanup\n * @param pzErrorMessage\n * @return int SQLITE_OK on success, error code otherwise\n */\nint vector_from_value(sqlite3_value *value, void **vector, size_t *dimensions,\n enum VectorElementType *element_type,\n vector_cleanup *cleanup, char **pzErrorMessage) {\n int subtype = sqlite3_value_subtype(value);\n if (!subtype || (subtype == SQLITE_VEC_ELEMENT_TYPE_FLOAT32) ||\n (subtype == JSON_SUBTYPE)) {\n int rc = fvec_from_value(value, (f32 **)vector, dimensions,\n (fvec_cleanup *)cleanup, pzErrorMessage);\n if (rc == SQLITE_OK) {\n *element_type = SQLITE_VEC_ELEMENT_TYPE_FLOAT32;\n }\n return rc;\n }\n\n if (subtype == SQLITE_VEC_ELEMENT_TYPE_BIT) {\n int rc = bitvec_from_value(value, (u8 **)vector, dimensions, cleanup,\n pzErrorMessage);\n if (rc == SQLITE_OK) {\n *element_type = SQLITE_VEC_ELEMENT_TYPE_BIT;\n }\n return rc;\n }\n if (subtype == SQLITE_VEC_ELEMENT_TYPE_INT8) {\n int rc = int8_vec_from_value(value, (i8 **)vector, dimensions, cleanup,\n pzErrorMessage);\n if (rc == SQLITE_OK) {\n *element_type = SQLITE_VEC_ELEMENT_TYPE_INT8;\n }\n return rc;\n }\n *pzErrorMessage = sqlite3_mprintf(\"Unknown subtype: %d\", subtype);\n return SQLITE_ERROR;\n}\n\nint ensure_vector_match(sqlite3_value *aValue, sqlite3_value *bValue, void **a,\n void **b, enum VectorElementType *element_type,\n size_t *dimensions, vector_cleanup *outACleanup,\n vector_cleanup *outBCleanup, char **outError) {\n int rc;\n enum VectorElementType aType, bType;\n size_t aDims, bDims;\n char *error = NULL;\n vector_cleanup aCleanup, bCleanup;\n\n rc = vector_from_value(aValue, a, &aDims, &aType, &aCleanup, &error);\n if (rc != SQLITE_OK) {\n *outError = sqlite3_mprintf(\"Error reading 1st vector: %s\", error);\n sqlite3_free(error);\n return SQLITE_ERROR;\n }\n\n rc = vector_from_value(bValue, b, &bDims, &bType, &bCleanup, &error);\n if (rc != SQLITE_OK) {\n *outError = sqlite3_mprintf(\"Error reading 2nd vector: %s\", error);\n sqlite3_free(error);\n aCleanup(*a);\n return SQLITE_ERROR;\n }\n\n if (aType != bType) {\n *outError =\n sqlite3_mprintf(\"Vector type mistmatch. First vector has type %s, \"\n \"while the second has type %s.\",\n vector_subtype_name(aType), vector_subtype_name(bType));\n aCleanup(*a);\n bCleanup(*b);\n return SQLITE_ERROR;\n }\n if (aDims != bDims) {\n *outError = sqlite3_mprintf(\n \"Vector dimension mistmatch. First vector has %ld dimensions, \"\n \"while the second has %ld dimensions.\",\n aDims, bDims);\n aCleanup(*a);\n bCleanup(*b);\n return SQLITE_ERROR;\n }\n *element_type = aType;\n *dimensions = aDims;\n *outACleanup = aCleanup;\n *outBCleanup = bCleanup;\n return SQLITE_OK;\n}\n\nint _cmp(const void *a, const void *b) { return (*(i64 *)a - *(i64 *)b); }\n\nstruct VecNpyFile {\n char *path;\n size_t pathLength;\n};\n#define SQLITE_VEC_NPY_FILE_NAME \"vec0-npy-file\"\n\n#ifndef SQLITE_VEC_OMIT_FS\nstatic void vec_npy_file(sqlite3_context *context, int argc,\n sqlite3_value **argv) {\n assert(argc == 1);\n char *path = (char *)sqlite3_value_text(argv[0]);\n size_t pathLength = sqlite3_value_bytes(argv[0]);\n struct VecNpyFile *f;\n\n f = sqlite3_malloc(sizeof(*f));\n if (!f) {\n sqlite3_result_error_nomem(context);\n return;\n }\n memset(f, 0, sizeof(*f));\n\n f->path = path;\n f->pathLength = pathLength;\n sqlite3_result_pointer(context, f, SQLITE_VEC_NPY_FILE_NAME, sqlite3_free);\n}\n#endif\n\n#pragma region scalar functions\nstatic void vec_f32(sqlite3_context *context, int argc, sqlite3_value **argv) {\n assert(argc == 1);\n int rc;\n f32 *vector = NULL;\n size_t dimensions;\n fvec_cleanup cleanup;\n char *errmsg;\n rc = fvec_from_value(argv[0], &vector, &dimensions, &cleanup, &errmsg);\n if (rc != SQLITE_OK) {\n sqlite3_result_error(context, errmsg, -1);\n sqlite3_free(errmsg);\n return;\n }\n sqlite3_result_blob(context, vector, dimensions * sizeof(f32),\n (void (*)(void *))cleanup);\n sqlite3_result_subtype(context, SQLITE_VEC_ELEMENT_TYPE_FLOAT32);\n}\n\nstatic void vec_bit(sqlite3_context *context, int argc, sqlite3_value **argv) {\n assert(argc == 1);\n int rc;\n u8 *vector;\n size_t dimensions;\n vector_cleanup cleanup;\n char *errmsg;\n rc = bitvec_from_value(argv[0], &vector, &dimensions, &cleanup, &errmsg);\n if (rc != SQLITE_OK) {\n sqlite3_result_error(context, errmsg, -1);\n sqlite3_free(errmsg);\n return;\n }\n sqlite3_result_blob(context, vector, dimensions / CHAR_BIT, SQLITE_TRANSIENT);\n sqlite3_result_subtype(context, SQLITE_VEC_ELEMENT_TYPE_BIT);\n cleanup(vector);\n}\nstatic void vec_int8(sqlite3_context *context, int argc, sqlite3_value **argv) {\n assert(argc == 1);\n int rc;\n i8 *vector;\n size_t dimensions;\n vector_cleanup cleanup;\n char *errmsg;\n rc = int8_vec_from_value(argv[0], &vector, &dimensions, &cleanup, &errmsg);\n if (rc != SQLITE_OK) {\n sqlite3_result_error(context, errmsg, -1);\n sqlite3_free(errmsg);\n return;\n }\n sqlite3_result_blob(context, vector, dimensions, SQLITE_TRANSIENT);\n sqlite3_result_subtype(context, SQLITE_VEC_ELEMENT_TYPE_INT8);\n cleanup(vector);\n}\n\nstatic void vec_length(sqlite3_context *context, int argc,\n sqlite3_value **argv) {\n assert(argc == 1);\n int rc;\n void *vector;\n size_t dimensions;\n vector_cleanup cleanup;\n char *errmsg;\n enum VectorElementType elementType;\n rc = vector_from_value(argv[0], &vector, &dimensions, &elementType, &cleanup,\n &errmsg);\n if (rc != SQLITE_OK) {\n sqlite3_result_error(context, errmsg, -1);\n sqlite3_free(errmsg);\n return;\n }\n sqlite3_result_int64(context, dimensions);\n cleanup(vector);\n}\n\nstatic void vec_distance_cosine(sqlite3_context *context, int argc,\n sqlite3_value **argv) {\n assert(argc == 2);\n int rc;\n void *a = NULL, *b = NULL;\n size_t dimensions;\n vector_cleanup aCleanup, bCleanup;\n char *error;\n enum VectorElementType elementType;\n rc = ensure_vector_match(argv[0], argv[1], &a, &b, &elementType, &dimensions,\n &aCleanup, &bCleanup, &error);\n if (rc != SQLITE_OK) {\n sqlite3_result_error(context, error, -1);\n sqlite3_free(error);\n return;\n }\n\n switch (elementType) {\n case SQLITE_VEC_ELEMENT_TYPE_BIT: {\n sqlite3_result_error(\n context, \"Cannot calculate cosine distance between two bitvectors.\",\n -1);\n goto finish;\n }\n case SQLITE_VEC_ELEMENT_TYPE_FLOAT32: {\n f32 result = distance_cosine_float(a, b, &dimensions);\n sqlite3_result_double(context, result);\n goto finish;\n }\n case SQLITE_VEC_ELEMENT_TYPE_INT8: {\n f32 result = distance_cosine_int8(a, b, &dimensions);\n sqlite3_result_double(context, result);\n goto finish;\n }\n }\n\nfinish:\n aCleanup(a);\n bCleanup(b);\n return;\n}\n\nstatic void vec_distance_l2(sqlite3_context *context, int argc,\n sqlite3_value **argv) {\n assert(argc == 2);\n int rc;\n void *a = NULL, *b = NULL;\n size_t dimensions;\n vector_cleanup aCleanup, bCleanup;\n char *error;\n enum VectorElementType elementType;\n rc = ensure_vector_match(argv[0], argv[1], &a, &b, &elementType, &dimensions,\n &aCleanup, &bCleanup, &error);\n if (rc != SQLITE_OK) {\n sqlite3_result_error(context, error, -1);\n sqlite3_free(error);\n return;\n }\n\n switch (elementType) {\n case SQLITE_VEC_ELEMENT_TYPE_BIT: {\n sqlite3_result_error(\n context, \"Cannot calculate L2 distance between two bitvectors.\", -1);\n goto finish;\n }\n case SQLITE_VEC_ELEMENT_TYPE_FLOAT32: {\n f32 result = distance_l2_sqr_float(a, b, &dimensions);\n sqlite3_result_double(context, result);\n goto finish;\n }\n case SQLITE_VEC_ELEMENT_TYPE_INT8: {\n f32 result = distance_l2_sqr_int8(a, b, &dimensions);\n sqlite3_result_double(context, result);\n goto finish;\n }\n }\n\nfinish:\n aCleanup(a);\n bCleanup(b);\n return;\n}\n\nstatic void vec_distance_l1(sqlite3_context *context, int argc,\n sqlite3_value **argv) {\n assert(argc == 2);\n int rc;\n void *a, *b;\n size_t dimensions;\n vector_cleanup aCleanup, bCleanup;\n char *error;\n enum VectorElementType elementType;\n rc = ensure_vector_match(argv[0], argv[1], &a, &b, &elementType, &dimensions,\n &aCleanup, &bCleanup, &error);\n if (rc != SQLITE_OK) {\n sqlite3_result_error(context, error, -1);\n sqlite3_free(error);\n return;\n }\n\n switch (elementType) {\n case SQLITE_VEC_ELEMENT_TYPE_BIT: {\n sqlite3_result_error(\n context, \"Cannot calculate L1 distance between two bitvectors.\", -1);\n goto finish;\n }\n case SQLITE_VEC_ELEMENT_TYPE_FLOAT32: {\n double result = distance_l1_f32(a, b, &dimensions);\n sqlite3_result_double(context, result);\n goto finish;\n }\n case SQLITE_VEC_ELEMENT_TYPE_INT8: {\n i64 result = distance_l1_int8(a, b, &dimensions);\n sqlite3_result_int(context, result);\n goto finish;\n }\n }\n\nfinish:\n aCleanup(a);\n bCleanup(b);\n return;\n}\n\nstatic void vec_distance_hamming(sqlite3_context *context, int argc,\n sqlite3_value **argv) {\n assert(argc == 2);\n int rc;\n void *a = NULL, *b = NULL;\n size_t dimensions;\n vector_cleanup aCleanup, bCleanup;\n char *error;\n enum VectorElementType elementType;\n rc = ensure_vector_match(argv[0], argv[1], &a, &b, &elementType, &dimensions,\n &aCleanup, &bCleanup, &error);\n if (rc != SQLITE_OK) {\n sqlite3_result_error(context, error, -1);\n sqlite3_free(error);\n return;\n }\n\n switch (elementType) {\n case SQLITE_VEC_ELEMENT_TYPE_BIT: {\n sqlite3_result_double(context, distance_hamming(a, b, &dimensions));\n goto finish;\n }\n case SQLITE_VEC_ELEMENT_TYPE_FLOAT32: {\n sqlite3_result_error(\n context,\n \"Cannot calculate hamming distance between two float32 vectors.\", -1);\n goto finish;\n }\n case SQLITE_VEC_ELEMENT_TYPE_INT8: {\n sqlite3_result_error(\n context, \"Cannot calculate hamming distance between two int8 vectors.\",\n -1);\n goto finish;\n }\n }\n\nfinish:\n aCleanup(a);\n bCleanup(b);\n return;\n}\n\nchar *vec_type_name(enum VectorElementType elementType) {\n switch (elementType) {\n case SQLITE_VEC_ELEMENT_TYPE_FLOAT32:\n return \"float32\";\n case SQLITE_VEC_ELEMENT_TYPE_INT8:\n return \"int8\";\n case SQLITE_VEC_ELEMENT_TYPE_BIT:\n return \"bit\";\n }\n return \"\";\n}\n\nstatic void vec_type(sqlite3_context *context, int argc, sqlite3_value **argv) {\n assert(argc == 1);\n void *vector;\n size_t dimensions;\n vector_cleanup cleanup;\n char *pzError;\n enum VectorElementType elementType;\n int rc = vector_from_value(argv[0], &vector, &dimensions, &elementType,\n &cleanup, &pzError);\n if (rc != SQLITE_OK) {\n sqlite3_result_error(context, pzError, -1);\n sqlite3_free(pzError);\n return;\n }\n sqlite3_result_text(context, vec_type_name(elementType), -1, SQLITE_STATIC);\n cleanup(vector);\n}\nstatic void vec_quantize_binary(sqlite3_context *context, int argc,\n sqlite3_value **argv) {\n assert(argc == 1);\n void *vector;\n size_t dimensions;\n vector_cleanup vectorCleanup;\n char *pzError;\n enum VectorElementType elementType;\n int rc = vector_from_value(argv[0], &vector, &dimensions, &elementType,\n &vectorCleanup, &pzError);\n if (rc != SQLITE_OK) {\n sqlite3_result_error(context, pzError, -1);\n sqlite3_free(pzError);\n return;\n }\n\n if (dimensions <= 0) {\n sqlite3_result_error(context, \"Zero length vectors are not supported.\", -1);\n goto cleanup;\n return;\n }\n if ((dimensions % CHAR_BIT) != 0) {\n sqlite3_result_error(\n context,\n \"Binary quantization requires vectors with a length divisible by 8\",\n -1);\n goto cleanup;\n return;\n }\n\n int sz = dimensions / CHAR_BIT;\n u8 *out = sqlite3_malloc(sz);\n if (!out) {\n sqlite3_result_error_code(context, SQLITE_NOMEM);\n goto cleanup;\n return;\n }\n memset(out, 0, sz);\n\n switch (elementType) {\n case SQLITE_VEC_ELEMENT_TYPE_FLOAT32: {\n\n for (size_t i = 0; i < dimensions; i++) {\n int res = ((f32 *)vector)[i] > 0.0;\n out[i / 8] |= (res << (i % 8));\n }\n break;\n }\n case SQLITE_VEC_ELEMENT_TYPE_INT8: {\n for (size_t i = 0; i < dimensions; i++) {\n int res = ((i8 *)vector)[i] > 0;\n out[i / 8] |= (res << (i % 8));\n }\n break;\n }\n case SQLITE_VEC_ELEMENT_TYPE_BIT: {\n sqlite3_result_error(context,\n \"Can only binary quantize float or int8 vectors\", -1);\n sqlite3_free(out);\n return;\n }\n }\n sqlite3_result_blob(context, out, sz, sqlite3_free);\n sqlite3_result_subtype(context, SQLITE_VEC_ELEMENT_TYPE_BIT);\n\ncleanup:\n vectorCleanup(vector);\n}\n\nstatic void vec_quantize_int8(sqlite3_context *context, int argc,\n sqlite3_value **argv) {\n assert(argc == 2);\n f32 *srcVector;\n size_t dimensions;\n fvec_cleanup srcCleanup;\n char *err;\n i8 *out = NULL;\n int rc = fvec_from_value(argv[0], &srcVector, &dimensions, &srcCleanup, &err);\n if (rc != SQLITE_OK) {\n sqlite3_result_error(context, err, -1);\n sqlite3_free(err);\n return;\n }\n\n int sz = dimensions * sizeof(i8);\n out = sqlite3_malloc(sz);\n if (!out) {\n sqlite3_result_error_nomem(context);\n goto cleanup;\n }\n memset(out, 0, sz);\n\n if ((sqlite3_value_type(argv[1]) != SQLITE_TEXT) ||\n (sqlite3_value_bytes(argv[1]) != strlen(\"unit\")) ||\n (sqlite3_stricmp((const char *)sqlite3_value_text(argv[1]), \"unit\") !=\n 0)) {\n sqlite3_result_error(\n context, \"2nd argument to vec_quantize_int8() must be 'unit'.\", -1);\n sqlite3_free(out);\n goto cleanup;\n }\n f32 step = (1.0 - (-1.0)) / 255;\n for (size_t i = 0; i < dimensions; i++) {\n double val = ((srcVector[i] - (-1.0)) / step) - 128;\n if (!(val <= 127.0)) val = 127.0; /* also clamps NaN */\n if (!(val >= -128.0)) val = -128.0;\n out[i] = (i8)val;\n }\n\n sqlite3_result_blob(context, out, dimensions * sizeof(i8), sqlite3_free);\n sqlite3_result_subtype(context, SQLITE_VEC_ELEMENT_TYPE_INT8);\n\ncleanup:\n srcCleanup(srcVector);\n}\n\nstatic void vec_add(sqlite3_context *context, int argc, sqlite3_value **argv) {\n assert(argc == 2);\n int rc;\n void *a = NULL, *b = NULL;\n size_t dimensions;\n vector_cleanup aCleanup, bCleanup;\n char *error;\n enum VectorElementType elementType;\n rc = ensure_vector_match(argv[0], argv[1], &a, &b, &elementType, &dimensions,\n &aCleanup, &bCleanup, &error);\n if (rc != SQLITE_OK) {\n sqlite3_result_error(context, error, -1);\n sqlite3_free(error);\n return;\n }\n\n switch (elementType) {\n case SQLITE_VEC_ELEMENT_TYPE_BIT: {\n sqlite3_result_error(context, \"Cannot add two bitvectors together.\", -1);\n goto finish;\n }\n case SQLITE_VEC_ELEMENT_TYPE_FLOAT32: {\n size_t outSize = dimensions * sizeof(f32);\n f32 *out = sqlite3_malloc(outSize);\n if (!out) {\n sqlite3_result_error_nomem(context);\n goto finish;\n }\n memset(out, 0, outSize);\n for (size_t i = 0; i < dimensions; i++) {\n out[i] = ((f32 *)a)[i] + ((f32 *)b)[i];\n }\n sqlite3_result_blob(context, out, outSize, sqlite3_free);\n sqlite3_result_subtype(context, SQLITE_VEC_ELEMENT_TYPE_FLOAT32);\n goto finish;\n }\n case SQLITE_VEC_ELEMENT_TYPE_INT8: {\n size_t outSize = dimensions * sizeof(i8);\n i8 *out = sqlite3_malloc(outSize);\n if (!out) {\n sqlite3_result_error_nomem(context);\n goto finish;\n }\n memset(out, 0, outSize);\n for (size_t i = 0; i < dimensions; i++) {\n out[i] = ((i8 *)a)[i] + ((i8 *)b)[i];\n }\n sqlite3_result_blob(context, out, outSize, sqlite3_free);\n sqlite3_result_subtype(context, SQLITE_VEC_ELEMENT_TYPE_INT8);\n goto finish;\n }\n }\nfinish:\n aCleanup(a);\n bCleanup(b);\n return;\n}\nstatic void vec_sub(sqlite3_context *context, int argc, sqlite3_value **argv) {\n assert(argc == 2);\n int rc;\n void *a = NULL, *b = NULL;\n size_t dimensions;\n vector_cleanup aCleanup, bCleanup;\n char *error;\n enum VectorElementType elementType;\n rc = ensure_vector_match(argv[0], argv[1], &a, &b, &elementType, &dimensions,\n &aCleanup, &bCleanup, &error);\n if (rc != SQLITE_OK) {\n sqlite3_result_error(context, error, -1);\n sqlite3_free(error);\n return;\n }\n\n switch (elementType) {\n case SQLITE_VEC_ELEMENT_TYPE_BIT: {\n sqlite3_result_error(context, \"Cannot subtract two bitvectors together.\",\n -1);\n goto finish;\n }\n case SQLITE_VEC_ELEMENT_TYPE_FLOAT32: {\n size_t outSize = dimensions * sizeof(f32);\n f32 *out = sqlite3_malloc(outSize);\n if (!out) {\n sqlite3_result_error_nomem(context);\n goto finish;\n }\n memset(out, 0, outSize);\n for (size_t i = 0; i < dimensions; i++) {\n out[i] = ((f32 *)a)[i] - ((f32 *)b)[i];\n }\n sqlite3_result_blob(context, out, outSize, sqlite3_free);\n sqlite3_result_subtype(context, SQLITE_VEC_ELEMENT_TYPE_FLOAT32);\n goto finish;\n }\n case SQLITE_VEC_ELEMENT_TYPE_INT8: {\n size_t outSize = dimensions * sizeof(i8);\n i8 *out = sqlite3_malloc(outSize);\n if (!out) {\n sqlite3_result_error_nomem(context);\n goto finish;\n }\n memset(out, 0, outSize);\n for (size_t i = 0; i < dimensions; i++) {\n out[i] = ((i8 *)a)[i] - ((i8 *)b)[i];\n }\n sqlite3_result_blob(context, out, outSize, sqlite3_free);\n sqlite3_result_subtype(context, SQLITE_VEC_ELEMENT_TYPE_INT8);\n goto finish;\n }\n }\nfinish:\n aCleanup(a);\n bCleanup(b);\n return;\n}\nstatic void vec_slice(sqlite3_context *context, int argc,\n sqlite3_value **argv) {\n assert(argc == 3);\n\n void *vector;\n size_t dimensions;\n vector_cleanup cleanup;\n char *err;\n enum VectorElementType elementType;\n\n int rc = vector_from_value(argv[0], &vector, &dimensions, &elementType,\n &cleanup, &err);\n if (rc != SQLITE_OK) {\n sqlite3_result_error(context, err, -1);\n sqlite3_free(err);\n return;\n }\n\n int start = sqlite3_value_int(argv[1]);\n int end = sqlite3_value_int(argv[2]);\n\n if (start < 0) {\n sqlite3_result_error(context,\n \"slice 'start' index must be a postive number.\", -1);\n goto done;\n }\n if (end < 0) {\n sqlite3_result_error(context, \"slice 'end' index must be a postive number.\",\n -1);\n goto done;\n }\n if (((size_t)start) > dimensions) {\n sqlite3_result_error(\n context, \"slice 'start' index is greater than the number of dimensions\",\n -1);\n goto done;\n }\n if (((size_t)end) > dimensions) {\n sqlite3_result_error(\n context, \"slice 'end' index is greater than the number of dimensions\",\n -1);\n goto done;\n }\n if (start > end) {\n sqlite3_result_error(context,\n \"slice 'start' index is greater than 'end' index\", -1);\n goto done;\n }\n if (start == end) {\n sqlite3_result_error(context,\n \"slice 'start' index is equal to the 'end' index, \"\n \"vectors must have non-zero length\",\n -1);\n goto done;\n }\n size_t n = end - start;\n\n switch (elementType) {\n case SQLITE_VEC_ELEMENT_TYPE_FLOAT32: {\n int outSize = n * sizeof(f32);\n f32 *out = sqlite3_malloc(outSize);\n if (!out) {\n sqlite3_result_error_nomem(context);\n goto done;\n }\n memset(out, 0, outSize);\n for (size_t i = 0; i < n; i++) {\n out[i] = ((f32 *)vector)[start + i];\n }\n sqlite3_result_blob(context, out, outSize, sqlite3_free);\n sqlite3_result_subtype(context, SQLITE_VEC_ELEMENT_TYPE_FLOAT32);\n goto done;\n }\n case SQLITE_VEC_ELEMENT_TYPE_INT8: {\n int outSize = n * sizeof(i8);\n i8 *out = sqlite3_malloc(outSize);\n if (!out) {\n sqlite3_result_error_nomem(context);\n return;\n }\n memset(out, 0, outSize);\n for (size_t i = 0; i < n; i++) {\n out[i] = ((i8 *)vector)[start + i];\n }\n sqlite3_result_blob(context, out, outSize, sqlite3_free);\n sqlite3_result_subtype(context, SQLITE_VEC_ELEMENT_TYPE_INT8);\n goto done;\n }\n case SQLITE_VEC_ELEMENT_TYPE_BIT: {\n if ((start % CHAR_BIT) != 0) {\n sqlite3_result_error(context, \"start index must be divisible by 8.\", -1);\n goto done;\n }\n if ((end % CHAR_BIT) != 0) {\n sqlite3_result_error(context, \"end index must be divisible by 8.\", -1);\n goto done;\n }\n int outSize = n / CHAR_BIT;\n u8 *out = sqlite3_malloc(outSize);\n if (!out) {\n sqlite3_result_error_nomem(context);\n return;\n }\n memset(out, 0, outSize);\n for (size_t i = 0; i < n / CHAR_BIT; i++) {\n out[i] = ((u8 *)vector)[(start / CHAR_BIT) + i];\n }\n sqlite3_result_blob(context, out, outSize, sqlite3_free);\n sqlite3_result_subtype(context, SQLITE_VEC_ELEMENT_TYPE_BIT);\n goto done;\n }\n }\ndone:\n cleanup(vector);\n}\n\nstatic void vec_to_json(sqlite3_context *context, int argc,\n sqlite3_value **argv) {\n assert(argc == 1);\n void *vector;\n size_t dimensions;\n vector_cleanup cleanup;\n char *err;\n enum VectorElementType elementType;\n\n int rc = vector_from_value(argv[0], &vector, &dimensions, &elementType,\n &cleanup, &err);\n if (rc != SQLITE_OK) {\n sqlite3_result_error(context, err, -1);\n sqlite3_free(err);\n return;\n }\n\n sqlite3_str *str = sqlite3_str_new(sqlite3_context_db_handle(context));\n sqlite3_str_appendall(str, \"[\");\n for (size_t i = 0; i < dimensions; i++) {\n if (i != 0) {\n sqlite3_str_appendall(str, \",\");\n }\n if (elementType == SQLITE_VEC_ELEMENT_TYPE_FLOAT32) {\n f32 value = ((f32 *)vector)[i];\n if (isnan(value)) {\n sqlite3_str_appendall(str, \"null\");\n } else {\n sqlite3_str_appendf(str, \"%f\", value);\n }\n\n } else if (elementType == SQLITE_VEC_ELEMENT_TYPE_INT8) {\n sqlite3_str_appendf(str, \"%d\", ((i8 *)vector)[i]);\n } else if (elementType == SQLITE_VEC_ELEMENT_TYPE_BIT) {\n u8 b = (((u8 *)vector)[i / 8] >> (i % CHAR_BIT)) & 1;\n sqlite3_str_appendf(str, \"%d\", b);\n }\n }\n sqlite3_str_appendall(str, \"]\");\n int len = sqlite3_str_length(str);\n char *s = sqlite3_str_finish(str);\n if (s) {\n sqlite3_result_text(context, s, len, sqlite3_free);\n sqlite3_result_subtype(context, JSON_SUBTYPE);\n } else {\n sqlite3_result_error_nomem(context);\n }\n cleanup(vector);\n}\n\nstatic void vec_normalize(sqlite3_context *context, int argc,\n sqlite3_value **argv) {\n assert(argc == 1);\n void *vector;\n size_t dimensions;\n vector_cleanup cleanup;\n char *err;\n enum VectorElementType elementType;\n\n int rc = vector_from_value(argv[0], &vector, &dimensions, &elementType,\n &cleanup, &err);\n if (rc != SQLITE_OK) {\n sqlite3_result_error(context, err, -1);\n sqlite3_free(err);\n return;\n }\n\n if (elementType != SQLITE_VEC_ELEMENT_TYPE_FLOAT32) {\n sqlite3_result_error(\n context, \"only float32 vectors are supported when normalizing\", -1);\n cleanup(vector);\n return;\n }\n\n int outSize = dimensions * sizeof(f32);\n f32 *out = sqlite3_malloc(outSize);\n if (!out) {\n cleanup(vector);\n sqlite3_result_error_code(context, SQLITE_NOMEM);\n return;\n }\n memset(out, 0, outSize);\n\n f32 *v = (f32 *)vector;\n\n f32 norm = 0;\n for (size_t i = 0; i < dimensions; i++) {\n norm += v[i] * v[i];\n }\n norm = sqrt(norm);\n for (size_t i = 0; i < dimensions; i++) {\n out[i] = v[i] / norm;\n }\n\n sqlite3_result_blob(context, out, dimensions * sizeof(f32), sqlite3_free);\n sqlite3_result_subtype(context, SQLITE_VEC_ELEMENT_TYPE_FLOAT32);\n cleanup(vector);\n}\n\nstatic void _static_text_func(sqlite3_context *context, int argc,\n sqlite3_value **argv) {\n UNUSED_PARAMETER(argc);\n UNUSED_PARAMETER(argv);\n sqlite3_result_text(context, sqlite3_user_data(context), -1, SQLITE_STATIC);\n}\n\n#pragma endregion\n\nenum Vec0TokenType {\n TOKEN_TYPE_IDENTIFIER,\n TOKEN_TYPE_DIGIT,\n TOKEN_TYPE_LBRACKET,\n TOKEN_TYPE_RBRACKET,\n TOKEN_TYPE_PLUS,\n TOKEN_TYPE_EQ,\n TOKEN_TYPE_LPAREN,\n TOKEN_TYPE_RPAREN,\n TOKEN_TYPE_COMMA,\n};\nstruct Vec0Token {\n enum Vec0TokenType token_type;\n char *start;\n char *end;\n};\n\nint is_alpha(char x) {\n return (x >= 'a' && x <= 'z') || (x >= 'A' && x <= 'Z');\n}\nint is_digit(char x) { return (x >= '0' && x <= '9'); }\nint is_whitespace(char x) {\n return x == ' ' || x == '\\t' || x == '\\n' || x == '\\r';\n}\n\n#define VEC0_TOKEN_RESULT_EOF 1\n#define VEC0_TOKEN_RESULT_SOME 2\n#define VEC0_TOKEN_RESULT_ERROR 3\n\nint vec0_token_next(char *start, char *end, struct Vec0Token *out) {\n char *ptr = start;\n while (ptr < end) {\n char curr = *ptr;\n if (is_whitespace(curr)) {\n ptr++;\n continue;\n } else if (curr == '+') {\n ptr++;\n out->start = ptr;\n out->end = ptr;\n out->token_type = TOKEN_TYPE_PLUS;\n return VEC0_TOKEN_RESULT_SOME;\n } else if (curr == '[') {\n ptr++;\n out->start = ptr;\n out->end = ptr;\n out->token_type = TOKEN_TYPE_LBRACKET;\n return VEC0_TOKEN_RESULT_SOME;\n } else if (curr == ']') {\n ptr++;\n out->start = ptr;\n out->end = ptr;\n out->token_type = TOKEN_TYPE_RBRACKET;\n return VEC0_TOKEN_RESULT_SOME;\n } else if (curr == '=') {\n ptr++;\n out->start = ptr;\n out->end = ptr;\n out->token_type = TOKEN_TYPE_EQ;\n return VEC0_TOKEN_RESULT_SOME;\n } else if (curr == '(') {\n ptr++;\n out->start = ptr;\n out->end = ptr;\n out->token_type = TOKEN_TYPE_LPAREN;\n return VEC0_TOKEN_RESULT_SOME;\n } else if (curr == ')') {\n ptr++;\n out->start = ptr;\n out->end = ptr;\n out->token_type = TOKEN_TYPE_RPAREN;\n return VEC0_TOKEN_RESULT_SOME;\n } else if (curr == ',') {\n ptr++;\n out->start = ptr;\n out->end = ptr;\n out->token_type = TOKEN_TYPE_COMMA;\n return VEC0_TOKEN_RESULT_SOME;\n } else if (is_alpha(curr)) {\n char *start = ptr;\n while (ptr < end && (is_alpha(*ptr) || is_digit(*ptr) || *ptr == '_')) {\n ptr++;\n }\n out->start = start;\n out->end = ptr;\n out->token_type = TOKEN_TYPE_IDENTIFIER;\n return VEC0_TOKEN_RESULT_SOME;\n } else if (is_digit(curr)) {\n char *start = ptr;\n while (ptr < end && (is_digit(*ptr))) {\n ptr++;\n }\n out->start = start;\n out->end = ptr;\n out->token_type = TOKEN_TYPE_DIGIT;\n return VEC0_TOKEN_RESULT_SOME;\n } else {\n return VEC0_TOKEN_RESULT_ERROR;\n }\n }\n return VEC0_TOKEN_RESULT_EOF;\n}\n\nstruct Vec0Scanner {\n char *start;\n char *end;\n char *ptr;\n};\n\nvoid vec0_scanner_init(struct Vec0Scanner *scanner, const char *source,\n int source_length) {\n scanner->start = (char *)source;\n scanner->end = (char *)source + source_length;\n scanner->ptr = (char *)source;\n}\nint vec0_scanner_next(struct Vec0Scanner *scanner, struct Vec0Token *out) {\n int rc = vec0_token_next(scanner->start, scanner->end, out);\n if (rc == VEC0_TOKEN_RESULT_SOME) {\n scanner->start = out->end;\n }\n return rc;\n}\n\nint vec0_parse_table_option(const char *source, int source_length,\n char **out_key, int *out_key_length,\n char **out_value, int *out_value_length) {\n int rc;\n struct Vec0Scanner scanner;\n struct Vec0Token token;\n char *key;\n char *value;\n int keyLength, valueLength;\n\n vec0_scanner_init(&scanner, source, source_length);\n\n rc = vec0_scanner_next(&scanner, &token);\n if (rc != VEC0_TOKEN_RESULT_SOME &&\n token.token_type != TOKEN_TYPE_IDENTIFIER) {\n return SQLITE_EMPTY;\n }\n key = token.start;\n keyLength = token.end - token.start;\n\n rc = vec0_scanner_next(&scanner, &token);\n if (rc != VEC0_TOKEN_RESULT_SOME && token.token_type != TOKEN_TYPE_EQ) {\n return SQLITE_EMPTY;\n }\n\n rc = vec0_scanner_next(&scanner, &token);\n if (rc != VEC0_TOKEN_RESULT_SOME &&\n !((token.token_type == TOKEN_TYPE_IDENTIFIER) ||\n (token.token_type == TOKEN_TYPE_DIGIT))) {\n return SQLITE_ERROR;\n }\n value = token.start;\n valueLength = token.end - token.start;\n\n rc = vec0_scanner_next(&scanner, &token);\n if (rc == VEC0_TOKEN_RESULT_EOF) {\n *out_key = key;\n *out_key_length = keyLength;\n *out_value = value;\n *out_value_length = valueLength;\n return SQLITE_OK;\n }\n return SQLITE_ERROR;\n}\n/**\n * @brief Parse an argv[i] entry of a vec0 virtual table definition, and see if\n * it's a PARTITION KEY definition.\n *\n * @param source: argv[i] source string\n * @param source_length: length of the source string\n * @param out_column_name: If it is a partition key, the output column name. Same lifetime\n * as source, points to specific char *\n * @param out_column_name_length: Length of out_column_name in bytes\n * @param out_column_type: SQLITE_TEXT or SQLITE_INTEGER.\n * @return int: SQLITE_EMPTY if not a PK, SQLITE_OK if it is.\n */\nint vec0_parse_partition_key_definition(const char *source, int source_length,\n char **out_column_name,\n int *out_column_name_length,\n int *out_column_type) {\n struct Vec0Scanner scanner;\n struct Vec0Token token;\n char *column_name;\n int column_name_length;\n int column_type;\n vec0_scanner_init(&scanner, source, source_length);\n\n // Check first token is identifier, will be the column name\n int rc = vec0_scanner_next(&scanner, &token);\n if (rc != VEC0_TOKEN_RESULT_SOME &&\n token.token_type != TOKEN_TYPE_IDENTIFIER) {\n return SQLITE_EMPTY;\n }\n\n column_name = token.start;\n column_name_length = token.end - token.start;\n\n // Check the next token matches \"text\" or \"integer\", as column type\n rc = vec0_scanner_next(&scanner, &token);\n if (rc != VEC0_TOKEN_RESULT_SOME &&\n token.token_type != TOKEN_TYPE_IDENTIFIER) {\n return SQLITE_EMPTY;\n }\n if (sqlite3_strnicmp(token.start, \"text\", token.end - token.start) == 0) {\n column_type = SQLITE_TEXT;\n } else if (sqlite3_strnicmp(token.start, \"int\", token.end - token.start) ==\n 0 ||\n sqlite3_strnicmp(token.start, \"integer\",\n token.end - token.start) == 0) {\n column_type = SQLITE_INTEGER;\n } else {\n return SQLITE_EMPTY;\n }\n\n // Check the next token is identifier and matches \"partition\"\n rc = vec0_scanner_next(&scanner, &token);\n if (rc != VEC0_TOKEN_RESULT_SOME &&\n token.token_type != TOKEN_TYPE_IDENTIFIER) {\n return SQLITE_EMPTY;\n }\n if (sqlite3_strnicmp(token.start, \"partition\", token.end - token.start) != 0) {\n return SQLITE_EMPTY;\n }\n\n // Check the next token is identifier and matches \"key\"\n rc = vec0_scanner_next(&scanner, &token);\n if (rc != VEC0_TOKEN_RESULT_SOME &&\n token.token_type != TOKEN_TYPE_IDENTIFIER) {\n return SQLITE_EMPTY;\n }\n if (sqlite3_strnicmp(token.start, \"key\", token.end - token.start) != 0) {\n return SQLITE_EMPTY;\n }\n\n *out_column_name = column_name;\n *out_column_name_length = column_name_length;\n *out_column_type = column_type;\n\n return SQLITE_OK;\n}\n\n/**\n * @brief Parse an argv[i] entry of a vec0 virtual table definition, and see if\n * it's an auxiliar column definition, ie `+[name] [type]` like `+contents text`\n *\n * @param source: argv[i] source string\n * @param source_length: length of the source string\n * @param out_column_name: If it is a partition key, the output column name. Same lifetime\n * as source, points to specific char *\n * @param out_column_name_length: Length of out_column_name in bytes\n * @param out_column_type: SQLITE_TEXT, SQLITE_INTEGER, SQLITE_FLOAT, or SQLITE_BLOB.\n * @return int: SQLITE_EMPTY if not an aux column, SQLITE_OK if it is.\n */\nint vec0_parse_auxiliary_column_definition(const char *source, int source_length,\n char **out_column_name,\n int *out_column_name_length,\n int *out_column_type) {\n struct Vec0Scanner scanner;\n struct Vec0Token token;\n char *column_name;\n int column_name_length;\n int column_type;\n vec0_scanner_init(&scanner, source, source_length);\n\n // Check first token is '+', which denotes aux columns\n int rc = vec0_scanner_next(&scanner, &token);\n if (rc != VEC0_TOKEN_RESULT_SOME ||\n token.token_type != TOKEN_TYPE_PLUS) {\n return SQLITE_EMPTY;\n }\n\n rc = vec0_scanner_next(&scanner, &token);\n if (rc != VEC0_TOKEN_RESULT_SOME &&\n token.token_type != TOKEN_TYPE_IDENTIFIER) {\n return SQLITE_EMPTY;\n }\n\n column_name = token.start;\n column_name_length = token.end - token.start;\n\n // Check the next token matches \"text\" or \"integer\", as column type\n rc = vec0_scanner_next(&scanner, &token);\n if (rc != VEC0_TOKEN_RESULT_SOME &&\n token.token_type != TOKEN_TYPE_IDENTIFIER) {\n return SQLITE_EMPTY;\n }\n if (sqlite3_strnicmp(token.start, \"text\", token.end - token.start) == 0) {\n column_type = SQLITE_TEXT;\n } else if (sqlite3_strnicmp(token.start, \"int\", token.end - token.start) ==\n 0 ||\n sqlite3_strnicmp(token.start, \"integer\",\n token.end - token.start) == 0) {\n column_type = SQLITE_INTEGER;\n } else if (sqlite3_strnicmp(token.start, \"float\", token.end - token.start) ==\n 0 ||\n sqlite3_strnicmp(token.start, \"double\",\n token.end - token.start) == 0) {\n column_type = SQLITE_FLOAT;\n } else if (sqlite3_strnicmp(token.start, \"blob\", token.end - token.start) ==0) {\n column_type = SQLITE_BLOB;\n } else {\n return SQLITE_EMPTY;\n }\n\n *out_column_name = column_name;\n *out_column_name_length = column_name_length;\n *out_column_type = column_type;\n\n return SQLITE_OK;\n}\n\ntypedef enum {\n VEC0_METADATA_COLUMN_KIND_BOOLEAN,\n VEC0_METADATA_COLUMN_KIND_INTEGER,\n VEC0_METADATA_COLUMN_KIND_FLOAT,\n VEC0_METADATA_COLUMN_KIND_TEXT,\n // future: blob, date, datetime\n} vec0_metadata_column_kind;\n\n/**\n * @brief Parse an argv[i] entry of a vec0 virtual table definition, and see if\n * it's an metadata column definition, ie `[name] [type]` like `is_released boolean`\n *\n * @param source: argv[i] source string\n * @param source_length: length of the source string\n * @param out_column_name: If it is a metadata column, the output column name. Same lifetime\n * as source, points to specific char *\n * @param out_column_name_length: Length of out_column_name in bytes\n * @param out_column_type: one of vec0_metadata_column_kind\n * @return int: SQLITE_EMPTY if not an metadata column, SQLITE_OK if it is.\n */\nint vec0_parse_metadata_column_definition(const char *source, int source_length,\n char **out_column_name,\n int *out_column_name_length,\n vec0_metadata_column_kind *out_column_type) {\n struct Vec0Scanner scanner;\n struct Vec0Token token;\n char *column_name;\n int column_name_length;\n vec0_metadata_column_kind column_type;\n int rc;\n vec0_scanner_init(&scanner, source, source_length);\n\n rc = vec0_scanner_next(&scanner, &token);\n if (rc != VEC0_TOKEN_RESULT_SOME ||\n token.token_type != TOKEN_TYPE_IDENTIFIER) {\n return SQLITE_EMPTY;\n }\n\n column_name = token.start;\n column_name_length = token.end - token.start;\n\n // Check the next token matches a valid metadata type\n rc = vec0_scanner_next(&scanner, &token);\n if (rc != VEC0_TOKEN_RESULT_SOME ||\n token.token_type != TOKEN_TYPE_IDENTIFIER) {\n return SQLITE_EMPTY;\n }\n char * t = token.start;\n int n = token.end - token.start;\n if (sqlite3_strnicmp(t, \"boolean\", n) == 0 || sqlite3_strnicmp(t, \"bool\", n) == 0) {\n column_type = VEC0_METADATA_COLUMN_KIND_BOOLEAN;\n }else if (sqlite3_strnicmp(t, \"int64\", n) == 0 || sqlite3_strnicmp(t, \"integer64\", n) == 0 || sqlite3_strnicmp(t, \"integer\", n) == 0 || sqlite3_strnicmp(t, \"int\", n) == 0) {\n column_type = VEC0_METADATA_COLUMN_KIND_INTEGER;\n }else if (sqlite3_strnicmp(t, \"float\", n) == 0 || sqlite3_strnicmp(t, \"double\", n) == 0 || sqlite3_strnicmp(t, \"float64\", n) == 0 || sqlite3_strnicmp(t, \"f64\", n) == 0) {\n column_type = VEC0_METADATA_COLUMN_KIND_FLOAT;\n } else if (sqlite3_strnicmp(t, \"text\", n) == 0) {\n column_type = VEC0_METADATA_COLUMN_KIND_TEXT;\n } else {\n return SQLITE_EMPTY;\n }\n\n *out_column_name = column_name;\n *out_column_name_length = column_name_length;\n *out_column_type = column_type;\n\n return SQLITE_OK;\n}\n\n/**\n * @brief Parse an argv[i] entry of a vec0 virtual table definition, and see if\n * it's a PRIMARY KEY definition.\n *\n * @param source: argv[i] source string\n * @param source_length: length of the source string\n * @param out_column_name: If it is a PK, the output column name. Same lifetime\n * as source, points to specific char *\n * @param out_column_name_length: Length of out_column_name in bytes\n * @param out_column_type: SQLITE_TEXT or SQLITE_INTEGER.\n * @return int: SQLITE_EMPTY if not a PK, SQLITE_OK if it is.\n */\nint vec0_parse_primary_key_definition(const char *source, int source_length,\n char **out_column_name,\n int *out_column_name_length,\n int *out_column_type) {\n struct Vec0Scanner scanner;\n struct Vec0Token token;\n char *column_name;\n int column_name_length;\n int column_type;\n vec0_scanner_init(&scanner, source, source_length);\n\n // Check first token is identifier, will be the column name\n int rc = vec0_scanner_next(&scanner, &token);\n if (rc != VEC0_TOKEN_RESULT_SOME &&\n token.token_type != TOKEN_TYPE_IDENTIFIER) {\n return SQLITE_EMPTY;\n }\n\n column_name = token.start;\n column_name_length = token.end - token.start;\n\n // Check the next token matches \"text\" or \"integer\", as column type\n rc = vec0_scanner_next(&scanner, &token);\n if (rc != VEC0_TOKEN_RESULT_SOME &&\n token.token_type != TOKEN_TYPE_IDENTIFIER) {\n return SQLITE_EMPTY;\n }\n if (sqlite3_strnicmp(token.start, \"text\", token.end - token.start) == 0) {\n column_type = SQLITE_TEXT;\n } else if (sqlite3_strnicmp(token.start, \"int\", token.end - token.start) ==\n 0 ||\n sqlite3_strnicmp(token.start, \"integer\",\n token.end - token.start) == 0) {\n column_type = SQLITE_INTEGER;\n } else {\n return SQLITE_EMPTY;\n }\n\n // Check the next token is identifier and matches \"primary\"\n rc = vec0_scanner_next(&scanner, &token);\n if (rc != VEC0_TOKEN_RESULT_SOME &&\n token.token_type != TOKEN_TYPE_IDENTIFIER) {\n return SQLITE_EMPTY;\n }\n if (sqlite3_strnicmp(token.start, \"primary\", token.end - token.start) != 0) {\n return SQLITE_EMPTY;\n }\n\n // Check the next token is identifier and matches \"key\"\n rc = vec0_scanner_next(&scanner, &token);\n if (rc != VEC0_TOKEN_RESULT_SOME &&\n token.token_type != TOKEN_TYPE_IDENTIFIER) {\n return SQLITE_EMPTY;\n }\n if (sqlite3_strnicmp(token.start, \"key\", token.end - token.start) != 0) {\n return SQLITE_EMPTY;\n }\n\n *out_column_name = column_name;\n *out_column_name_length = column_name_length;\n *out_column_type = column_type;\n\n return SQLITE_OK;\n}\n\nenum Vec0DistanceMetrics {\n VEC0_DISTANCE_METRIC_L2 = 1,\n VEC0_DISTANCE_METRIC_COSINE = 2,\n VEC0_DISTANCE_METRIC_L1 = 3,\n};\n\nstruct VectorColumnDefinition {\n char *name;\n int name_length;\n size_t dimensions;\n enum VectorElementType element_type;\n enum Vec0DistanceMetrics distance_metric;\n};\n\nstruct Vec0PartitionColumnDefinition {\n int type;\n char * name;\n int name_length;\n};\n\nstruct Vec0AuxiliaryColumnDefinition {\n int type;\n char * name;\n int name_length;\n};\nstruct Vec0MetadataColumnDefinition {\n vec0_metadata_column_kind kind;\n char * name;\n int name_length;\n};\n\nsize_t vector_byte_size(enum VectorElementType element_type,\n size_t dimensions) {\n switch (element_type) {\n case SQLITE_VEC_ELEMENT_TYPE_FLOAT32:\n return dimensions * sizeof(f32);\n case SQLITE_VEC_ELEMENT_TYPE_INT8:\n return dimensions * sizeof(i8);\n case SQLITE_VEC_ELEMENT_TYPE_BIT:\n return dimensions / CHAR_BIT;\n }\n return 0;\n}\n\nsize_t vector_column_byte_size(struct VectorColumnDefinition column) {\n return vector_byte_size(column.element_type, column.dimensions);\n}\n\n/**\n * @brief Parse an vec0 vtab argv[i] column definition and see if\n * it's a vector column defintion, ex `contents_embedding float[768]`.\n *\n * @param source vec0 argv[i] item\n * @param source_length length of source in bytes\n * @param outColumn Output the parse vector column to this struct, if success\n * @return int SQLITE_OK on success, SQLITE_EMPTY is it's not a vector column\n * definition, SQLITE_ERROR on error.\n */\nint vec0_parse_vector_column(const char *source, int source_length,\n struct VectorColumnDefinition *outColumn) {\n // parses a vector column definition like so:\n // \"abc float[123]\", \"abc_123 bit[1234]\", eetc.\n // https://github.com/asg017/sqlite-vec/issues/46\n int rc;\n struct Vec0Scanner scanner;\n struct Vec0Token token;\n\n char *name;\n int nameLength;\n enum VectorElementType elementType;\n enum Vec0DistanceMetrics distanceMetric = VEC0_DISTANCE_METRIC_L2;\n int dimensions;\n\n vec0_scanner_init(&scanner, source, source_length);\n\n // starts with an identifier\n rc = vec0_scanner_next(&scanner, &token);\n\n if (rc != VEC0_TOKEN_RESULT_SOME &&\n token.token_type != TOKEN_TYPE_IDENTIFIER) {\n return SQLITE_EMPTY;\n }\n\n name = token.start;\n nameLength = token.end - token.start;\n\n // vector column type comes next: float, int, or bit\n rc = vec0_scanner_next(&scanner, &token);\n\n if (rc != VEC0_TOKEN_RESULT_SOME ||\n token.token_type != TOKEN_TYPE_IDENTIFIER) {\n return SQLITE_EMPTY;\n }\n if (sqlite3_strnicmp(token.start, \"float\", 5) == 0 ||\n sqlite3_strnicmp(token.start, \"f32\", 3) == 0) {\n elementType = SQLITE_VEC_ELEMENT_TYPE_FLOAT32;\n } else if (sqlite3_strnicmp(token.start, \"int8\", 4) == 0 ||\n sqlite3_strnicmp(token.start, \"i8\", 2) == 0) {\n elementType = SQLITE_VEC_ELEMENT_TYPE_INT8;\n } else if (sqlite3_strnicmp(token.start, \"bit\", 3) == 0) {\n elementType = SQLITE_VEC_ELEMENT_TYPE_BIT;\n } else {\n return SQLITE_EMPTY;\n }\n\n // left '[' bracket\n rc = vec0_scanner_next(&scanner, &token);\n if (rc != VEC0_TOKEN_RESULT_SOME && token.token_type != TOKEN_TYPE_LBRACKET) {\n return SQLITE_EMPTY;\n }\n\n // digit, for vector dimension length\n rc = vec0_scanner_next(&scanner, &token);\n if (rc != VEC0_TOKEN_RESULT_SOME && token.token_type != TOKEN_TYPE_DIGIT) {\n return SQLITE_ERROR;\n }\n dimensions = atoi(token.start);\n if (dimensions <= 0) {\n return SQLITE_ERROR;\n }\n\n // // right ']' bracket\n rc = vec0_scanner_next(&scanner, &token);\n if (rc != VEC0_TOKEN_RESULT_SOME && token.token_type != TOKEN_TYPE_RBRACKET) {\n return SQLITE_ERROR;\n }\n\n // any other tokens left should be column-level options , ex `key=value`\n // ex `distance_metric=L2 distance_metric=cosine` should error\n while (1) {\n // should be EOF or identifier (option key)\n rc = vec0_scanner_next(&scanner, &token);\n if (rc == VEC0_TOKEN_RESULT_EOF) {\n break;\n }\n\n if (rc != VEC0_TOKEN_RESULT_SOME &&\n token.token_type != TOKEN_TYPE_IDENTIFIER) {\n return SQLITE_ERROR;\n }\n\n char *key = token.start;\n int keyLength = token.end - token.start;\n\n if (sqlite3_strnicmp(key, \"distance_metric\", keyLength) == 0) {\n\n if (elementType == SQLITE_VEC_ELEMENT_TYPE_BIT) {\n return SQLITE_ERROR;\n }\n // ensure equal sign after distance_metric\n rc = vec0_scanner_next(&scanner, &token);\n if (rc != VEC0_TOKEN_RESULT_SOME && token.token_type != TOKEN_TYPE_EQ) {\n return SQLITE_ERROR;\n }\n\n // distance_metric value, an identifier (L2, cosine, etc)\n rc = vec0_scanner_next(&scanner, &token);\n if (rc != VEC0_TOKEN_RESULT_SOME &&\n token.token_type != TOKEN_TYPE_IDENTIFIER) {\n return SQLITE_ERROR;\n }\n\n char *value = token.start;\n int valueLength = token.end - token.start;\n if (sqlite3_strnicmp(value, \"l2\", valueLength) == 0) {\n distanceMetric = VEC0_DISTANCE_METRIC_L2;\n } else if (sqlite3_strnicmp(value, \"l1\", valueLength) == 0) {\n distanceMetric = VEC0_DISTANCE_METRIC_L1;\n } else if (sqlite3_strnicmp(value, \"cosine\", valueLength) == 0) {\n distanceMetric = VEC0_DISTANCE_METRIC_COSINE;\n } else {\n return SQLITE_ERROR;\n }\n }\n // unknown key\n else {\n return SQLITE_ERROR;\n }\n }\n\n outColumn->name = sqlite3_mprintf(\"%.*s\", nameLength, name);\n if (!outColumn->name) {\n return SQLITE_ERROR;\n }\n outColumn->name_length = nameLength;\n outColumn->distance_metric = distanceMetric;\n outColumn->element_type = elementType;\n outColumn->dimensions = dimensions;\n return SQLITE_OK;\n}\n\n#pragma region vec_each table function\n\ntypedef struct vec_each_vtab vec_each_vtab;\nstruct vec_each_vtab {\n sqlite3_vtab base;\n};\n\ntypedef struct vec_each_cursor vec_each_cursor;\nstruct vec_each_cursor {\n sqlite3_vtab_cursor base;\n i64 iRowid;\n enum VectorElementType vector_type;\n void *vector;\n size_t dimensions;\n vector_cleanup cleanup;\n};\n\nstatic int vec_eachConnect(sqlite3 *db, void *pAux, int argc,\n const char *const *argv, sqlite3_vtab **ppVtab,\n char **pzErr) {\n UNUSED_PARAMETER(pAux);\n UNUSED_PARAMETER(argc);\n UNUSED_PARAMETER(argv);\n UNUSED_PARAMETER(pzErr);\n vec_each_vtab *pNew;\n int rc;\n\n rc = sqlite3_declare_vtab(db, \"CREATE TABLE x(value, vector hidden)\");\n#define VEC_EACH_COLUMN_VALUE 0\n#define VEC_EACH_COLUMN_VECTOR 1\n if (rc == SQLITE_OK) {\n pNew = sqlite3_malloc(sizeof(*pNew));\n *ppVtab = (sqlite3_vtab *)pNew;\n if (pNew == 0)\n return SQLITE_NOMEM;\n memset(pNew, 0, sizeof(*pNew));\n }\n return rc;\n}\n\nstatic int vec_eachDisconnect(sqlite3_vtab *pVtab) {\n vec_each_vtab *p = (vec_each_vtab *)pVtab;\n sqlite3_free(p);\n return SQLITE_OK;\n}\n\nstatic int vec_eachOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor) {\n UNUSED_PARAMETER(p);\n vec_each_cursor *pCur;\n pCur = sqlite3_malloc(sizeof(*pCur));\n if (pCur == 0)\n return SQLITE_NOMEM;\n memset(pCur, 0, sizeof(*pCur));\n *ppCursor = &pCur->base;\n return SQLITE_OK;\n}\n\nstatic int vec_eachClose(sqlite3_vtab_cursor *cur) {\n vec_each_cursor *pCur = (vec_each_cursor *)cur;\n if(pCur->vector) {\n pCur->cleanup(pCur->vector);\n }\n sqlite3_free(pCur);\n return SQLITE_OK;\n}\n\nstatic int vec_eachBestIndex(sqlite3_vtab *pVTab,\n sqlite3_index_info *pIdxInfo) {\n UNUSED_PARAMETER(pVTab);\n int hasVector = 0;\n for (int i = 0; i < pIdxInfo->nConstraint; i++) {\n const struct sqlite3_index_constraint *pCons = &pIdxInfo->aConstraint[i];\n // printf(\"i=%d iColumn=%d, op=%d, usable=%d\\n\", i, pCons->iColumn,\n // pCons->op, pCons->usable);\n switch (pCons->iColumn) {\n case VEC_EACH_COLUMN_VECTOR: {\n if (pCons->op == SQLITE_INDEX_CONSTRAINT_EQ && pCons->usable) {\n hasVector = 1;\n pIdxInfo->aConstraintUsage[i].argvIndex = 1;\n pIdxInfo->aConstraintUsage[i].omit = 1;\n }\n break;\n }\n }\n }\n if (!hasVector) {\n return SQLITE_CONSTRAINT;\n }\n\n pIdxInfo->estimatedCost = (double)100000;\n pIdxInfo->estimatedRows = 100000;\n\n return SQLITE_OK;\n}\n\nstatic int vec_eachFilter(sqlite3_vtab_cursor *pVtabCursor, int idxNum,\n const char *idxStr, int argc, sqlite3_value **argv) {\n UNUSED_PARAMETER(idxNum);\n UNUSED_PARAMETER(idxStr);\n assert(argc == 1);\n vec_each_cursor *pCur = (vec_each_cursor *)pVtabCursor;\n\n if (pCur->vector) {\n pCur->cleanup(pCur->vector);\n pCur->vector = NULL;\n }\n\n char *pzErrMsg;\n int rc = vector_from_value(argv[0], &pCur->vector, &pCur->dimensions,\n &pCur->vector_type, &pCur->cleanup, &pzErrMsg);\n if (rc != SQLITE_OK) {\n sqlite3_free(pzErrMsg);\n return SQLITE_ERROR;\n }\n pCur->iRowid = 0;\n return SQLITE_OK;\n}\n\nstatic int vec_eachRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid) {\n vec_each_cursor *pCur = (vec_each_cursor *)cur;\n *pRowid = pCur->iRowid;\n return SQLITE_OK;\n}\n\nstatic int vec_eachEof(sqlite3_vtab_cursor *cur) {\n vec_each_cursor *pCur = (vec_each_cursor *)cur;\n return pCur->iRowid >= (i64)pCur->dimensions;\n}\n\nstatic int vec_eachNext(sqlite3_vtab_cursor *cur) {\n vec_each_cursor *pCur = (vec_each_cursor *)cur;\n pCur->iRowid++;\n return SQLITE_OK;\n}\n\nstatic int vec_eachColumn(sqlite3_vtab_cursor *cur, sqlite3_context *context,\n int i) {\n vec_each_cursor *pCur = (vec_each_cursor *)cur;\n switch (i) {\n case VEC_EACH_COLUMN_VALUE:\n switch (pCur->vector_type) {\n case SQLITE_VEC_ELEMENT_TYPE_FLOAT32: {\n sqlite3_result_double(context, ((f32 *)pCur->vector)[pCur->iRowid]);\n break;\n }\n case SQLITE_VEC_ELEMENT_TYPE_BIT: {\n u8 x = ((u8 *)pCur->vector)[pCur->iRowid / CHAR_BIT];\n sqlite3_result_int(context,\n (x & (0b10000000 >> ((pCur->iRowid % CHAR_BIT)))) > 0);\n break;\n }\n case SQLITE_VEC_ELEMENT_TYPE_INT8: {\n sqlite3_result_int(context, ((i8 *)pCur->vector)[pCur->iRowid]);\n break;\n }\n }\n\n break;\n }\n return SQLITE_OK;\n}\n\nstatic sqlite3_module vec_eachModule = {\n /* iVersion */ 0,\n /* xCreate */ 0,\n /* xConnect */ vec_eachConnect,\n /* xBestIndex */ vec_eachBestIndex,\n /* xDisconnect */ vec_eachDisconnect,\n /* xDestroy */ 0,\n /* xOpen */ vec_eachOpen,\n /* xClose */ vec_eachClose,\n /* xFilter */ vec_eachFilter,\n /* xNext */ vec_eachNext,\n /* xEof */ vec_eachEof,\n /* xColumn */ vec_eachColumn,\n /* xRowid */ vec_eachRowid,\n /* xUpdate */ 0,\n /* xBegin */ 0,\n /* xSync */ 0,\n /* xCommit */ 0,\n /* xRollback */ 0,\n /* xFindMethod */ 0,\n /* xRename */ 0,\n /* xSavepoint */ 0,\n /* xRelease */ 0,\n /* xRollbackTo */ 0,\n /* xShadowName */ 0,\n#if SQLITE_VERSION_NUMBER >= 3044000\n /* xIntegrity */ 0\n#endif\n};\n\n#pragma endregion\n\n#pragma region vec_npy_each table function\n\nenum NpyTokenType {\n NPY_TOKEN_TYPE_IDENTIFIER,\n NPY_TOKEN_TYPE_NUMBER,\n NPY_TOKEN_TYPE_LPAREN,\n NPY_TOKEN_TYPE_RPAREN,\n NPY_TOKEN_TYPE_LBRACE,\n NPY_TOKEN_TYPE_RBRACE,\n NPY_TOKEN_TYPE_COLON,\n NPY_TOKEN_TYPE_COMMA,\n NPY_TOKEN_TYPE_STRING,\n NPY_TOKEN_TYPE_FALSE,\n};\n\nstruct NpyToken {\n enum NpyTokenType token_type;\n unsigned char *start;\n unsigned char *end;\n};\n\nint npy_token_next(unsigned char *start, unsigned char *end,\n struct NpyToken *out) {\n unsigned char *ptr = start;\n while (ptr < end) {\n unsigned char curr = *ptr;\n if (is_whitespace(curr)) {\n ptr++;\n continue;\n } else if (curr == '(') {\n out->start = ptr++;\n out->end = ptr;\n out->token_type = NPY_TOKEN_TYPE_LPAREN;\n return VEC0_TOKEN_RESULT_SOME;\n } else if (curr == ')') {\n out->start = ptr++;\n out->end = ptr;\n out->token_type = NPY_TOKEN_TYPE_RPAREN;\n return VEC0_TOKEN_RESULT_SOME;\n } else if (curr == '{') {\n out->start = ptr++;\n out->end = ptr;\n out->token_type = NPY_TOKEN_TYPE_LBRACE;\n return VEC0_TOKEN_RESULT_SOME;\n } else if (curr == '}') {\n out->start = ptr++;\n out->end = ptr;\n out->token_type = NPY_TOKEN_TYPE_RBRACE;\n return VEC0_TOKEN_RESULT_SOME;\n } else if (curr == ':') {\n out->start = ptr++;\n out->end = ptr;\n out->token_type = NPY_TOKEN_TYPE_COLON;\n return VEC0_TOKEN_RESULT_SOME;\n } else if (curr == ',') {\n out->start = ptr++;\n out->end = ptr;\n out->token_type = NPY_TOKEN_TYPE_COMMA;\n return VEC0_TOKEN_RESULT_SOME;\n } else if (curr == '\\'') {\n unsigned char *start = ptr;\n ptr++;\n while (ptr < end) {\n if ((*ptr) == '\\'') {\n break;\n }\n ptr++;\n }\n if (ptr >= end || (*ptr) != '\\'') {\n return VEC0_TOKEN_RESULT_ERROR;\n }\n out->start = start;\n out->end = ++ptr;\n out->token_type = NPY_TOKEN_TYPE_STRING;\n return VEC0_TOKEN_RESULT_SOME;\n } else if (curr == 'F' &&\n strncmp((char *)ptr, \"False\", strlen(\"False\")) == 0) {\n out->start = ptr;\n out->end = (ptr + (int)strlen(\"False\"));\n ptr = out->end;\n out->token_type = NPY_TOKEN_TYPE_FALSE;\n return VEC0_TOKEN_RESULT_SOME;\n } else if (is_digit(curr)) {\n unsigned char *start = ptr;\n while (ptr < end && (is_digit(*ptr))) {\n ptr++;\n }\n out->start = start;\n out->end = ptr;\n out->token_type = NPY_TOKEN_TYPE_NUMBER;\n return VEC0_TOKEN_RESULT_SOME;\n } else {\n return VEC0_TOKEN_RESULT_ERROR;\n }\n }\n return VEC0_TOKEN_RESULT_ERROR;\n}\n\nstruct NpyScanner {\n unsigned char *start;\n unsigned char *end;\n unsigned char *ptr;\n};\n\nvoid npy_scanner_init(struct NpyScanner *scanner, const unsigned char *source,\n int source_length) {\n scanner->start = (unsigned char *)source;\n scanner->end = (unsigned char *)source + source_length;\n scanner->ptr = (unsigned char *)source;\n}\n\nint npy_scanner_next(struct NpyScanner *scanner, struct NpyToken *out) {\n int rc = npy_token_next(scanner->start, scanner->end, out);\n if (rc == VEC0_TOKEN_RESULT_SOME) {\n scanner->start = out->end;\n }\n return rc;\n}\n\n#define NPY_PARSE_ERROR \"Error parsing numpy array: \"\nint parse_npy_header(sqlite3_vtab *pVTab, const unsigned char *header,\n size_t headerLength,\n enum VectorElementType *out_element_type,\n int *fortran_order, size_t *numElements,\n size_t *numDimensions) {\n\n struct NpyScanner scanner;\n struct NpyToken token;\n int rc;\n npy_scanner_init(&scanner, header, headerLength);\n\n if (npy_scanner_next(&scanner, &token) != VEC0_TOKEN_RESULT_SOME &&\n token.token_type != NPY_TOKEN_TYPE_LBRACE) {\n vtab_set_error(pVTab,\n NPY_PARSE_ERROR \"numpy header did not start with '{'\");\n return SQLITE_ERROR;\n }\n while (1) {\n rc = npy_scanner_next(&scanner, &token);\n if (rc != VEC0_TOKEN_RESULT_SOME) {\n vtab_set_error(pVTab, NPY_PARSE_ERROR \"expected key in numpy header\");\n return SQLITE_ERROR;\n }\n\n if (token.token_type == NPY_TOKEN_TYPE_RBRACE) {\n break;\n }\n if (token.token_type != NPY_TOKEN_TYPE_STRING) {\n vtab_set_error(pVTab, NPY_PARSE_ERROR\n \"expected a string as key in numpy header\");\n return SQLITE_ERROR;\n }\n unsigned char *key = token.start;\n\n rc = npy_scanner_next(&scanner, &token);\n if ((rc != VEC0_TOKEN_RESULT_SOME) ||\n (token.token_type != NPY_TOKEN_TYPE_COLON)) {\n vtab_set_error(pVTab, NPY_PARSE_ERROR\n \"expected a ':' after key in numpy header\");\n return SQLITE_ERROR;\n }\n\n if (strncmp((char *)key, \"'descr'\", strlen(\"'descr'\")) == 0) {\n rc = npy_scanner_next(&scanner, &token);\n if ((rc != VEC0_TOKEN_RESULT_SOME) ||\n (token.token_type != NPY_TOKEN_TYPE_STRING)) {\n vtab_set_error(pVTab, NPY_PARSE_ERROR\n \"expected a string value after 'descr' key\");\n return SQLITE_ERROR;\n }\n if (strncmp((char *)token.start, \"'maxChunks = 1024;\n pCur->chunksBufferSize =\n (vector_byte_size(element_type, numDimensions)) * pCur->maxChunks;\n pCur->chunksBuffer = sqlite3_malloc(pCur->chunksBufferSize);\n if (pCur->chunksBufferSize && !pCur->chunksBuffer) {\n return SQLITE_NOMEM;\n }\n\n pCur->currentChunkSize =\n fread(pCur->chunksBuffer, vector_byte_size(element_type, numDimensions),\n pCur->maxChunks, file);\n\n pCur->currentChunkIndex = 0;\n pCur->elementType = element_type;\n pCur->nElements = numElements;\n pCur->nDimensions = numDimensions;\n pCur->input_type = VEC_NPY_EACH_INPUT_FILE;\n\n pCur->eof = pCur->currentChunkSize == 0;\n pCur->file = file;\n return SQLITE_OK;\n}\n#endif\n\nint parse_npy_buffer(sqlite3_vtab *pVTab, const unsigned char *buffer,\n int bufferLength, void **data, size_t *numElements,\n size_t *numDimensions,\n enum VectorElementType *element_type) {\n\n if (bufferLength < 10) {\n // IMP: V03312_20150\n vtab_set_error(pVTab, \"numpy array too short\");\n return SQLITE_ERROR;\n }\n if (memcmp(NPY_MAGIC, buffer, sizeof(NPY_MAGIC)) != 0) {\n // V11954_28792\n vtab_set_error(pVTab, \"numpy array does not contain the 'magic' header\");\n return SQLITE_ERROR;\n }\n\n u8 major = buffer[6];\n u8 minor = buffer[7];\n uint16_t headerLength = 0;\n memcpy(&headerLength, &buffer[8], sizeof(uint16_t));\n\n i32 totalHeaderLength = sizeof(NPY_MAGIC) + sizeof(major) + sizeof(minor) +\n sizeof(headerLength) + headerLength;\n i32 dataSize = bufferLength - totalHeaderLength;\n\n if (dataSize < 0) {\n vtab_set_error(pVTab, \"numpy array header length is invalid\");\n return SQLITE_ERROR;\n }\n\n const unsigned char *header = &buffer[10];\n int fortran_order;\n\n int rc = parse_npy_header(pVTab, header, headerLength, element_type,\n &fortran_order, numElements, numDimensions);\n if (rc != SQLITE_OK) {\n return rc;\n }\n\n i32 expectedDataSize =\n (*numElements * vector_byte_size(*element_type, *numDimensions));\n if (expectedDataSize != dataSize) {\n vtab_set_error(pVTab,\n \"numpy array error: Expected a data size of %d, found %d\",\n expectedDataSize, dataSize);\n return SQLITE_ERROR;\n }\n\n *data = (void *)&buffer[totalHeaderLength];\n return SQLITE_OK;\n}\n\nstatic int vec_npy_eachConnect(sqlite3 *db, void *pAux, int argc,\n const char *const *argv, sqlite3_vtab **ppVtab,\n char **pzErr) {\n UNUSED_PARAMETER(pAux);\n UNUSED_PARAMETER(argc);\n UNUSED_PARAMETER(argv);\n UNUSED_PARAMETER(pzErr);\n vec_npy_each_vtab *pNew;\n int rc;\n\n rc = sqlite3_declare_vtab(db, \"CREATE TABLE x(vector, input hidden)\");\n#define VEC_NPY_EACH_COLUMN_VECTOR 0\n#define VEC_NPY_EACH_COLUMN_INPUT 1\n if (rc == SQLITE_OK) {\n pNew = sqlite3_malloc(sizeof(*pNew));\n *ppVtab = (sqlite3_vtab *)pNew;\n if (pNew == 0)\n return SQLITE_NOMEM;\n memset(pNew, 0, sizeof(*pNew));\n }\n return rc;\n}\n\nstatic int vec_npy_eachDisconnect(sqlite3_vtab *pVtab) {\n vec_npy_each_vtab *p = (vec_npy_each_vtab *)pVtab;\n sqlite3_free(p);\n return SQLITE_OK;\n}\n\nstatic int vec_npy_eachOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor) {\n UNUSED_PARAMETER(p);\n vec_npy_each_cursor *pCur;\n pCur = sqlite3_malloc(sizeof(*pCur));\n if (pCur == 0)\n return SQLITE_NOMEM;\n memset(pCur, 0, sizeof(*pCur));\n *ppCursor = &pCur->base;\n return SQLITE_OK;\n}\n\nstatic int vec_npy_eachClose(sqlite3_vtab_cursor *cur) {\n vec_npy_each_cursor *pCur = (vec_npy_each_cursor *)cur;\n#ifndef SQLITE_VEC_OMIT_FS\n if (pCur->file) {\n fclose(pCur->file);\n pCur->file = NULL;\n }\n#endif\n if (pCur->chunksBuffer) {\n sqlite3_free(pCur->chunksBuffer);\n pCur->chunksBuffer = NULL;\n }\n if (pCur->vector) {\n pCur->vector = NULL;\n }\n sqlite3_free(pCur);\n return SQLITE_OK;\n}\n\nstatic int vec_npy_eachBestIndex(sqlite3_vtab *pVTab,\n sqlite3_index_info *pIdxInfo) {\n int hasInput;\n for (int i = 0; i < pIdxInfo->nConstraint; i++) {\n const struct sqlite3_index_constraint *pCons = &pIdxInfo->aConstraint[i];\n // printf(\"i=%d iColumn=%d, op=%d, usable=%d\\n\", i, pCons->iColumn,\n // pCons->op, pCons->usable);\n switch (pCons->iColumn) {\n case VEC_NPY_EACH_COLUMN_INPUT: {\n if (pCons->op == SQLITE_INDEX_CONSTRAINT_EQ && pCons->usable) {\n hasInput = 1;\n pIdxInfo->aConstraintUsage[i].argvIndex = 1;\n pIdxInfo->aConstraintUsage[i].omit = 1;\n }\n break;\n }\n }\n }\n if (!hasInput) {\n pVTab->zErrMsg = sqlite3_mprintf(\"input argument is required\");\n return SQLITE_ERROR;\n }\n\n pIdxInfo->estimatedCost = (double)100000;\n pIdxInfo->estimatedRows = 100000;\n\n return SQLITE_OK;\n}\n\nstatic int vec_npy_eachFilter(sqlite3_vtab_cursor *pVtabCursor, int idxNum,\n const char *idxStr, int argc,\n sqlite3_value **argv) {\n UNUSED_PARAMETER(idxNum);\n UNUSED_PARAMETER(idxStr);\n assert(argc == 1);\n int rc;\n\n vec_npy_each_cursor *pCur = (vec_npy_each_cursor *)pVtabCursor;\n\n#ifndef SQLITE_VEC_OMIT_FS\n if (pCur->file) {\n fclose(pCur->file);\n pCur->file = NULL;\n }\n#endif\n if (pCur->chunksBuffer) {\n sqlite3_free(pCur->chunksBuffer);\n pCur->chunksBuffer = NULL;\n }\n if (pCur->vector) {\n pCur->vector = NULL;\n }\n\n#ifndef SQLITE_VEC_OMIT_FS\n struct VecNpyFile *f = NULL;\n if ((f = sqlite3_value_pointer(argv[0], SQLITE_VEC_NPY_FILE_NAME))) {\n FILE *file = fopen(f->path, \"r\");\n if (!file) {\n vtab_set_error(pVtabCursor->pVtab, \"Could not open numpy file\");\n return SQLITE_ERROR;\n }\n\n rc = parse_npy_file(pVtabCursor->pVtab, file, pCur);\n if (rc != SQLITE_OK) {\n#ifndef SQLITE_VEC_OMIT_FS\n fclose(file);\n#endif\n return rc;\n }\n\n } else\n#endif\n {\n\n const unsigned char *input = sqlite3_value_blob(argv[0]);\n int inputLength = sqlite3_value_bytes(argv[0]);\n void *data;\n size_t numElements;\n size_t numDimensions;\n enum VectorElementType element_type;\n\n rc = parse_npy_buffer(pVtabCursor->pVtab, input, inputLength, &data,\n &numElements, &numDimensions, &element_type);\n if (rc != SQLITE_OK) {\n return rc;\n }\n\n pCur->vector = data;\n pCur->elementType = element_type;\n pCur->nElements = numElements;\n pCur->nDimensions = numDimensions;\n pCur->input_type = VEC_NPY_EACH_INPUT_BUFFER;\n }\n\n pCur->iRowid = 0;\n return SQLITE_OK;\n}\n\nstatic int vec_npy_eachRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid) {\n vec_npy_each_cursor *pCur = (vec_npy_each_cursor *)cur;\n *pRowid = pCur->iRowid;\n return SQLITE_OK;\n}\n\nstatic int vec_npy_eachEof(sqlite3_vtab_cursor *cur) {\n vec_npy_each_cursor *pCur = (vec_npy_each_cursor *)cur;\n if (pCur->input_type == VEC_NPY_EACH_INPUT_BUFFER) {\n return (!pCur->nElements) || (size_t)pCur->iRowid >= pCur->nElements;\n }\n return pCur->eof;\n}\n\nstatic int vec_npy_eachNext(sqlite3_vtab_cursor *cur) {\n vec_npy_each_cursor *pCur = (vec_npy_each_cursor *)cur;\n pCur->iRowid++;\n if (pCur->input_type == VEC_NPY_EACH_INPUT_BUFFER) {\n return SQLITE_OK;\n }\n\n#ifndef SQLITE_VEC_OMIT_FS\n // else: input is a file\n pCur->currentChunkIndex++;\n if (pCur->currentChunkIndex >= pCur->currentChunkSize) {\n pCur->currentChunkSize =\n fread(pCur->chunksBuffer,\n vector_byte_size(pCur->elementType, pCur->nDimensions),\n pCur->maxChunks, pCur->file);\n if (!pCur->currentChunkSize) {\n pCur->eof = 1;\n }\n pCur->currentChunkIndex = 0;\n }\n#endif\n return SQLITE_OK;\n}\n\nstatic int vec_npy_eachColumnBuffer(vec_npy_each_cursor *pCur,\n sqlite3_context *context, int i) {\n switch (i) {\n case VEC_NPY_EACH_COLUMN_VECTOR: {\n sqlite3_result_subtype(context, pCur->elementType);\n switch (pCur->elementType) {\n case SQLITE_VEC_ELEMENT_TYPE_FLOAT32: {\n sqlite3_result_blob(\n context,\n &((unsigned char *)\n pCur->vector)[pCur->iRowid * pCur->nDimensions * sizeof(f32)],\n pCur->nDimensions * sizeof(f32), SQLITE_TRANSIENT);\n\n break;\n }\n case SQLITE_VEC_ELEMENT_TYPE_INT8:\n case SQLITE_VEC_ELEMENT_TYPE_BIT: {\n // https://github.com/asg017/sqlite-vec/issues/42\n sqlite3_result_error(context,\n \"vec_npy_each only supports float32 vectors\", -1);\n break;\n }\n }\n\n break;\n }\n }\n return SQLITE_OK;\n}\nstatic int vec_npy_eachColumnFile(vec_npy_each_cursor *pCur,\n sqlite3_context *context, int i) {\n switch (i) {\n case VEC_NPY_EACH_COLUMN_VECTOR: {\n switch (pCur->elementType) {\n case SQLITE_VEC_ELEMENT_TYPE_FLOAT32: {\n sqlite3_result_blob(\n context,\n &((unsigned char *)\n pCur->chunksBuffer)[pCur->currentChunkIndex *\n pCur->nDimensions * sizeof(f32)],\n pCur->nDimensions * sizeof(f32), SQLITE_TRANSIENT);\n break;\n }\n case SQLITE_VEC_ELEMENT_TYPE_INT8:\n case SQLITE_VEC_ELEMENT_TYPE_BIT: {\n // https://github.com/asg017/sqlite-vec/issues/42\n sqlite3_result_error(context,\n \"vec_npy_each only supports float32 vectors\", -1);\n break;\n }\n }\n break;\n }\n }\n return SQLITE_OK;\n}\nstatic int vec_npy_eachColumn(sqlite3_vtab_cursor *cur,\n sqlite3_context *context, int i) {\n vec_npy_each_cursor *pCur = (vec_npy_each_cursor *)cur;\n switch (pCur->input_type) {\n case VEC_NPY_EACH_INPUT_BUFFER:\n return vec_npy_eachColumnBuffer(pCur, context, i);\n case VEC_NPY_EACH_INPUT_FILE:\n return vec_npy_eachColumnFile(pCur, context, i);\n }\n return SQLITE_ERROR;\n}\n\nstatic sqlite3_module vec_npy_eachModule = {\n /* iVersion */ 0,\n /* xCreate */ 0,\n /* xConnect */ vec_npy_eachConnect,\n /* xBestIndex */ vec_npy_eachBestIndex,\n /* xDisconnect */ vec_npy_eachDisconnect,\n /* xDestroy */ 0,\n /* xOpen */ vec_npy_eachOpen,\n /* xClose */ vec_npy_eachClose,\n /* xFilter */ vec_npy_eachFilter,\n /* xNext */ vec_npy_eachNext,\n /* xEof */ vec_npy_eachEof,\n /* xColumn */ vec_npy_eachColumn,\n /* xRowid */ vec_npy_eachRowid,\n /* xUpdate */ 0,\n /* xBegin */ 0,\n /* xSync */ 0,\n /* xCommit */ 0,\n /* xRollback */ 0,\n /* xFindMethod */ 0,\n /* xRename */ 0,\n /* xSavepoint */ 0,\n /* xRelease */ 0,\n /* xRollbackTo */ 0,\n /* xShadowName */ 0,\n#if SQLITE_VERSION_NUMBER >= 3044000\n /* xIntegrity */ 0,\n#endif\n};\n\n#pragma endregion\n\n#pragma region vec0 virtual table\n\n#define VEC0_COLUMN_ID 0\n#define VEC0_COLUMN_USERN_START 1\n#define VEC0_COLUMN_OFFSET_DISTANCE 1\n#define VEC0_COLUMN_OFFSET_K 2\n\n#define VEC0_SHADOW_INFO_NAME \"\\\"%w\\\".\\\"%w_info\\\"\"\n\n#define VEC0_SHADOW_CHUNKS_NAME \"\\\"%w\\\".\\\"%w_chunks\\\"\"\n/// 1) schema, 2) original vtab table name\n#define VEC0_SHADOW_CHUNKS_CREATE \\\n \"CREATE TABLE \" VEC0_SHADOW_CHUNKS_NAME \"(\" \\\n \"chunk_id INTEGER PRIMARY KEY AUTOINCREMENT,\" \\\n \"size INTEGER NOT NULL,\" \\\n \"validity BLOB NOT NULL,\" \\\n \"rowids BLOB NOT NULL\" \\\n \");\"\n\n#define VEC0_SHADOW_ROWIDS_NAME \"\\\"%w\\\".\\\"%w_rowids\\\"\"\n/// 1) schema, 2) original vtab table name\n#define VEC0_SHADOW_ROWIDS_CREATE_BASIC \\\n \"CREATE TABLE \" VEC0_SHADOW_ROWIDS_NAME \"(\" \\\n \"rowid INTEGER PRIMARY KEY AUTOINCREMENT,\" \\\n \"id,\" \\\n \"chunk_id INTEGER,\" \\\n \"chunk_offset INTEGER\" \\\n \");\"\n\n// vec0 tables with a text primary keys are still backed by int64 primary keys,\n// since a fixed-length rowid is required for vec0 chunks. But we add a new 'id\n// text unique' column to emulate a text primary key interface.\n#define VEC0_SHADOW_ROWIDS_CREATE_PK_TEXT \\\n \"CREATE TABLE \" VEC0_SHADOW_ROWIDS_NAME \"(\" \\\n \"rowid INTEGER PRIMARY KEY AUTOINCREMENT,\" \\\n \"id TEXT UNIQUE NOT NULL,\" \\\n \"chunk_id INTEGER,\" \\\n \"chunk_offset INTEGER\" \\\n \");\"\n\n/// 1) schema, 2) original vtab table name\n#define VEC0_SHADOW_VECTOR_N_NAME \"\\\"%w\\\".\\\"%w_vector_chunks%02d\\\"\"\n\n/// 1) schema, 2) original vtab table name\n//\n// IMPORTANT: \"rowid\" is declared as PRIMARY KEY but WITHOUT the INTEGER type.\n// This means it is NOT a true SQLite rowid alias — the user-defined \"rowid\"\n// column and the internal SQLite rowid (_rowid_) are two separate values.\n// When inserting, both must be set explicitly to keep them in sync. See the\n// _rowid_ bindings in vec0_new_chunk() and the explanation in\n// SHADOW_TABLE_ROWID_QUIRK below.\n#define VEC0_SHADOW_VECTOR_N_CREATE \\\n \"CREATE TABLE \" VEC0_SHADOW_VECTOR_N_NAME \"(\" \\\n \"rowid PRIMARY KEY,\" \\\n \"vectors BLOB NOT NULL\" \\\n \");\"\n\n#define VEC0_SHADOW_AUXILIARY_NAME \"\\\"%w\\\".\\\"%w_auxiliary\\\"\"\n\n#define VEC0_SHADOW_METADATA_N_NAME \"\\\"%w\\\".\\\"%w_metadatachunks%02d\\\"\"\n#define VEC0_SHADOW_METADATA_TEXT_DATA_NAME \"\\\"%w\\\".\\\"%w_metadatatext%02d\\\"\"\n\n#define VEC_INTERAL_ERROR \"Internal sqlite-vec error: \"\n#define REPORT_URL \"https://github.com/asg017/sqlite-vec/issues/new\"\n\ntypedef struct vec0_vtab vec0_vtab;\n\n#define VEC0_MAX_VECTOR_COLUMNS 16\n#define VEC0_MAX_PARTITION_COLUMNS 4\n#define VEC0_MAX_AUXILIARY_COLUMNS 16\n#define VEC0_MAX_METADATA_COLUMNS 16\n\n#define SQLITE_VEC_VEC0_MAX_DIMENSIONS 8192\n#define VEC0_METADATA_TEXT_VIEW_BUFFER_LENGTH 16\n#define VEC0_METADATA_TEXT_VIEW_DATA_LENGTH 12\n\ntypedef enum {\n // vector column, ie \"contents_embedding float[1024]\"\n SQLITE_VEC0_USER_COLUMN_KIND_VECTOR = 1,\n\n // partition key column, ie \"user_id integer partition key\"\n SQLITE_VEC0_USER_COLUMN_KIND_PARTITION = 2,\n\n //\n SQLITE_VEC0_USER_COLUMN_KIND_AUXILIARY = 3,\n\n // metadata column that can be filtered, ie \"genre text\"\n SQLITE_VEC0_USER_COLUMN_KIND_METADATA = 4,\n} vec0_user_column_kind;\n\nstruct vec0_vtab {\n sqlite3_vtab base;\n\n // the SQLite connection of the host database\n sqlite3 *db;\n\n // True if the primary key of the vec0 table has a column type TEXT.\n // Will change the schema of the _rowids table, and insert/query logic.\n int pkIsText;\n\n // number of defined vector columns.\n int numVectorColumns;\n\n // number of defined PARTITION KEY columns.\n int numPartitionColumns;\n\n // number of defined auxiliary columns\n int numAuxiliaryColumns;\n\n // number of defined metadata columns\n int numMetadataColumns;\n\n\n // Name of the schema the table exists on.\n // Must be freed with sqlite3_free()\n char *schemaName;\n\n // Name of the table the table exists on.\n // Must be freed with sqlite3_free()\n char *tableName;\n\n // Name of the _rowids shadow table.\n // Must be freed with sqlite3_free()\n char *shadowRowidsName;\n\n // Name of the _chunks shadow table.\n // Must be freed with sqlite3_free()\n char *shadowChunksName;\n\n // contains enum vec0_user_column_kind values for up to\n // numVectorColumns + numPartitionColumns entries\n vec0_user_column_kind user_column_kinds[VEC0_MAX_VECTOR_COLUMNS + VEC0_MAX_PARTITION_COLUMNS + VEC0_MAX_AUXILIARY_COLUMNS + VEC0_MAX_METADATA_COLUMNS];\n\n uint8_t user_column_idxs[VEC0_MAX_VECTOR_COLUMNS + VEC0_MAX_PARTITION_COLUMNS + VEC0_MAX_AUXILIARY_COLUMNS + VEC0_MAX_METADATA_COLUMNS];\n\n\n // Name of all the vector chunk shadow tables.\n // Ex '_vector_chunks00'\n // Only the first numVectorColumns entries will be available.\n // The first numVectorColumns entries must be freed with sqlite3_free()\n char *shadowVectorChunksNames[VEC0_MAX_VECTOR_COLUMNS];\n\n // Name of all metadata chunk shadow tables, ie `_metadatachunks00`\n // Only the first numMetadataColumns entries will be available.\n // The first numMetadataColumns entries must be freed with sqlite3_free()\n char *shadowMetadataChunksNames[VEC0_MAX_METADATA_COLUMNS];\n\n struct VectorColumnDefinition vector_columns[VEC0_MAX_VECTOR_COLUMNS];\n struct Vec0PartitionColumnDefinition paritition_columns[VEC0_MAX_PARTITION_COLUMNS];\n struct Vec0AuxiliaryColumnDefinition auxiliary_columns[VEC0_MAX_AUXILIARY_COLUMNS];\n struct Vec0MetadataColumnDefinition metadata_columns[VEC0_MAX_METADATA_COLUMNS];\n\n int chunk_size;\n\n // select latest chunk from _chunks, getting chunk_id\n sqlite3_stmt *stmtLatestChunk;\n\n /**\n * Statement to insert a row into the _rowids table, with a rowid.\n * Parameters:\n * 1: int64, rowid to insert\n * Result columns: none\n * SQL: \"INSERT INTO _rowids(rowid) VALUES (?)\"\n *\n * Must be cleaned up with sqlite3_finalize().\n */\n sqlite3_stmt *stmtRowidsInsertRowid;\n\n /**\n * Statement to insert a row into the _rowids table, with an id.\n * The id column isn't a tradition primary key, but instead a unique\n * column to handle \"text primary key\" vec0 tables. The true int64 rowid\n * can be retrieved after inserting with sqlite3_last_rowid().\n *\n * Parameters:\n * 1: text or null, id to insert\n * Result columns: none\n *\n * Must be cleaned up with sqlite3_finalize().\n */\n sqlite3_stmt *stmtRowidsInsertId;\n\n /**\n * Statement to update the \"position\" columns chunk_id and chunk_offset for\n * a given _rowids row. Used when the \"next available\" chunk position is found\n * for a vector.\n *\n * Parameters:\n * 1: int64, chunk_id value\n * 2: int64, chunk_offset value\n * 3: int64, rowid value\n * Result columns: none\n *\n * Must be cleaned up with sqlite3_finalize().\n */\n sqlite3_stmt *stmtRowidsUpdatePosition;\n\n /**\n * Statement to quickly find the chunk_id + chunk_offset of a given row.\n * Parameters:\n * 1: rowid of the row/vector to lookup\n * Result columns:\n * 0: chunk_id (i64)\n * 1: chunk_offset (i64)\n * SQL: \"SELECT id, chunk_id, chunk_offset FROM _rowids WHERE rowid = ?\"\"\n *\n * Must be cleaned up with sqlite3_finalize().\n */\n sqlite3_stmt *stmtRowidsGetChunkPosition;\n};\n\n/**\n * @brief Finalize all the sqlite3_stmt members in a vec0_vtab.\n *\n * @param p vec0_vtab pointer\n */\nvoid vec0_free_resources(vec0_vtab *p) {\n sqlite3_finalize(p->stmtLatestChunk);\n p->stmtLatestChunk = NULL;\n sqlite3_finalize(p->stmtRowidsInsertRowid);\n p->stmtRowidsInsertRowid = NULL;\n sqlite3_finalize(p->stmtRowidsInsertId);\n p->stmtRowidsInsertId = NULL;\n sqlite3_finalize(p->stmtRowidsUpdatePosition);\n p->stmtRowidsUpdatePosition = NULL;\n sqlite3_finalize(p->stmtRowidsGetChunkPosition);\n p->stmtRowidsGetChunkPosition = NULL;\n}\n\n/**\n * @brief Free all memory and sqlite3_stmt members of a vec0_vtab\n *\n * @param p vec0_vtab pointer\n */\nvoid vec0_free(vec0_vtab *p) {\n vec0_free_resources(p);\n\n sqlite3_free(p->schemaName);\n p->schemaName = NULL;\n sqlite3_free(p->tableName);\n p->tableName = NULL;\n sqlite3_free(p->shadowChunksName);\n p->shadowChunksName = NULL;\n sqlite3_free(p->shadowRowidsName);\n p->shadowRowidsName = NULL;\n\n for (int i = 0; i < p->numVectorColumns; i++) {\n sqlite3_free(p->shadowVectorChunksNames[i]);\n p->shadowVectorChunksNames[i] = NULL;\n\n sqlite3_free(p->vector_columns[i].name);\n p->vector_columns[i].name = NULL;\n }\n\n for (int i = 0; i < p->numPartitionColumns; i++) {\n sqlite3_free(p->paritition_columns[i].name);\n p->paritition_columns[i].name = NULL;\n }\n\n for (int i = 0; i < p->numAuxiliaryColumns; i++) {\n sqlite3_free(p->auxiliary_columns[i].name);\n p->auxiliary_columns[i].name = NULL;\n }\n\n for (int i = 0; i < p->numMetadataColumns; i++) {\n sqlite3_free(p->metadata_columns[i].name);\n p->metadata_columns[i].name = NULL;\n }\n}\n\nint vec0_num_defined_user_columns(vec0_vtab *p) {\n return p->numVectorColumns + p->numPartitionColumns + p->numAuxiliaryColumns + p->numMetadataColumns;\n}\n\n/**\n * @brief Returns the index of the distance hidden column for the given vec0\n * table.\n *\n * @param p vec0 table\n * @return int\n */\nint vec0_column_distance_idx(vec0_vtab *p) {\n return VEC0_COLUMN_USERN_START + (vec0_num_defined_user_columns(p) - 1) +\n VEC0_COLUMN_OFFSET_DISTANCE;\n}\n\n/**\n * @brief Returns the index of the k hidden column for the given vec0 table.\n *\n * @param p vec0 table\n * @return int k column index\n */\nint vec0_column_k_idx(vec0_vtab *p) {\n return VEC0_COLUMN_USERN_START + (vec0_num_defined_user_columns(p) - 1) +\n VEC0_COLUMN_OFFSET_K;\n}\n\n/**\n * Returns 1 if the given column-based index is a valid vector column,\n * 0 otherwise.\n */\nint vec0_column_idx_is_vector(vec0_vtab *pVtab, int column_idx) {\n return column_idx >= VEC0_COLUMN_USERN_START &&\n column_idx <= (VEC0_COLUMN_USERN_START + vec0_num_defined_user_columns(pVtab) - 1) &&\n pVtab->user_column_kinds[column_idx - VEC0_COLUMN_USERN_START] == SQLITE_VEC0_USER_COLUMN_KIND_VECTOR;\n}\n\n/**\n * Returns the vector index of the given user column index.\n * ONLY call if validated with vec0_column_idx_is_vector before\n */\nint vec0_column_idx_to_vector_idx(vec0_vtab *pVtab, int column_idx) {\n UNUSED_PARAMETER(pVtab);\n return pVtab->user_column_idxs[column_idx - VEC0_COLUMN_USERN_START];\n}\n/**\n * Returns 1 if the given column-based index is a \"partition key\" column,\n * 0 otherwise.\n */\nint vec0_column_idx_is_partition(vec0_vtab *pVtab, int column_idx) {\n return column_idx >= VEC0_COLUMN_USERN_START &&\n column_idx <= (VEC0_COLUMN_USERN_START + vec0_num_defined_user_columns(pVtab) - 1) &&\n pVtab->user_column_kinds[column_idx - VEC0_COLUMN_USERN_START] == SQLITE_VEC0_USER_COLUMN_KIND_PARTITION;\n}\n\n/**\n * Returns the partition column index of the given user column index.\n * ONLY call if validated with vec0_column_idx_is_vector before\n */\nint vec0_column_idx_to_partition_idx(vec0_vtab *pVtab, int column_idx) {\n UNUSED_PARAMETER(pVtab);\n return pVtab->user_column_idxs[column_idx - VEC0_COLUMN_USERN_START];\n}\n\n/**\n * Returns 1 if the given column-based index is a auxiliary column,\n * 0 otherwise.\n */\nint vec0_column_idx_is_auxiliary(vec0_vtab *pVtab, int column_idx) {\n return column_idx >= VEC0_COLUMN_USERN_START &&\n column_idx <= (VEC0_COLUMN_USERN_START + vec0_num_defined_user_columns(pVtab) - 1) &&\n pVtab->user_column_kinds[column_idx - VEC0_COLUMN_USERN_START] == SQLITE_VEC0_USER_COLUMN_KIND_AUXILIARY;\n}\n\n/**\n * Returns the auxiliary column index of the given user column index.\n * ONLY call if validated with vec0_column_idx_to_partition_idx before\n */\nint vec0_column_idx_to_auxiliary_idx(vec0_vtab *pVtab, int column_idx) {\n UNUSED_PARAMETER(pVtab);\n return pVtab->user_column_idxs[column_idx - VEC0_COLUMN_USERN_START];\n}\n\n/**\n * Returns 1 if the given column-based index is a metadata column,\n * 0 otherwise.\n */\nint vec0_column_idx_is_metadata(vec0_vtab *pVtab, int column_idx) {\n return column_idx >= VEC0_COLUMN_USERN_START &&\n column_idx <= (VEC0_COLUMN_USERN_START + vec0_num_defined_user_columns(pVtab) - 1) &&\n pVtab->user_column_kinds[column_idx - VEC0_COLUMN_USERN_START] == SQLITE_VEC0_USER_COLUMN_KIND_METADATA;\n}\n\n/**\n * Returns the metadata column index of the given user column index.\n * ONLY call if validated with vec0_column_idx_is_metadata before\n */\nint vec0_column_idx_to_metadata_idx(vec0_vtab *pVtab, int column_idx) {\n UNUSED_PARAMETER(pVtab);\n return pVtab->user_column_idxs[column_idx - VEC0_COLUMN_USERN_START];\n}\n\n/**\n * @brief Retrieve the chunk_id, chunk_offset, and possible \"id\" value\n * of a vec0_vtab row with the provided rowid\n *\n * @param p vec0_vtab\n * @param rowid the rowid of the row to query\n * @param id output, optional sqlite3_value to provide the id.\n * Useful for text PK rows. Must be freed with sqlite3_value_free()\n * @param chunk_id output, the chunk_id the row belongs to\n * @param chunk_offset output, the offset within the chunk the row belongs to\n * @return SQLITE_ROW on success, error code otherwise. SQLITE_EMPTY if row DNE\n */\nint vec0_get_chunk_position(vec0_vtab *p, i64 rowid, sqlite3_value **id,\n i64 *chunk_id, i64 *chunk_offset) {\n int rc;\n\n if (!p->stmtRowidsGetChunkPosition) {\n const char *zSql =\n sqlite3_mprintf(\"SELECT id, chunk_id, chunk_offset \"\n \"FROM \" VEC0_SHADOW_ROWIDS_NAME \" WHERE rowid = ?\",\n p->schemaName, p->tableName);\n if (!zSql) {\n rc = SQLITE_NOMEM;\n goto cleanup;\n }\n rc = sqlite3_prepare_v2(p->db, zSql, -1, &p->stmtRowidsGetChunkPosition, 0);\n sqlite3_free((void *)zSql);\n if (rc != SQLITE_OK) {\n vtab_set_error(\n &p->base, VEC_INTERAL_ERROR\n \"could not initialize 'rowids get chunk position' statement\");\n goto cleanup;\n }\n }\n\n sqlite3_bind_int64(p->stmtRowidsGetChunkPosition, 1, rowid);\n rc = sqlite3_step(p->stmtRowidsGetChunkPosition);\n // special case: when no results, return SQLITE_EMPTY to convey \"that chunk\n // position doesnt exist\"\n if (rc == SQLITE_DONE) {\n rc = SQLITE_EMPTY;\n goto cleanup;\n }\n if (rc != SQLITE_ROW) {\n goto cleanup;\n }\n\n if (id) {\n sqlite3_value *value =\n sqlite3_column_value(p->stmtRowidsGetChunkPosition, 0);\n *id = sqlite3_value_dup(value);\n if (!*id) {\n rc = SQLITE_NOMEM;\n goto cleanup;\n }\n }\n\n if (chunk_id) {\n *chunk_id = sqlite3_column_int64(p->stmtRowidsGetChunkPosition, 1);\n }\n if (chunk_offset) {\n *chunk_offset = sqlite3_column_int64(p->stmtRowidsGetChunkPosition, 2);\n }\n\n rc = SQLITE_OK;\n\ncleanup:\n sqlite3_reset(p->stmtRowidsGetChunkPosition);\n sqlite3_clear_bindings(p->stmtRowidsGetChunkPosition);\n return rc;\n}\n\n/**\n * @brief Return the id value from the _rowids table where _rowids.rowid =\n * rowid.\n *\n * @param pVtab: vec0 table to query\n * @param rowid: rowid of the row to query.\n * @param out: A dup'ed sqlite3_value of the id column. Might be null.\n * Must be cleaned up with sqlite3_value_free().\n * @returns SQLITE_OK on success, error code on failure\n */\nint vec0_get_id_value_from_rowid(vec0_vtab *pVtab, i64 rowid,\n sqlite3_value **out) {\n // PERF: different strategy than get_chunk_position?\n return vec0_get_chunk_position((vec0_vtab *)pVtab, rowid, out, NULL, NULL);\n}\n\nint vec0_rowid_from_id(vec0_vtab *p, sqlite3_value *valueId, i64 *rowid) {\n sqlite3_stmt *stmt = NULL;\n int rc;\n char *zSql;\n zSql = sqlite3_mprintf(\"SELECT rowid\"\n \" FROM \" VEC0_SHADOW_ROWIDS_NAME \" WHERE id = ?\",\n p->schemaName, p->tableName);\n if (!zSql) {\n rc = SQLITE_NOMEM;\n goto cleanup;\n }\n rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, NULL);\n sqlite3_free(zSql);\n if (rc != SQLITE_OK) {\n goto cleanup;\n }\n sqlite3_bind_value(stmt, 1, valueId);\n rc = sqlite3_step(stmt);\n if (rc == SQLITE_DONE) {\n rc = SQLITE_EMPTY;\n goto cleanup;\n }\n if (rc != SQLITE_ROW) {\n goto cleanup;\n }\n *rowid = sqlite3_column_int64(stmt, 0);\n rc = sqlite3_step(stmt);\n if (rc != SQLITE_DONE) {\n goto cleanup;\n }\n\n rc = SQLITE_OK;\n\ncleanup:\n sqlite3_finalize(stmt);\n return rc;\n}\n\nint vec0_result_id(vec0_vtab *p, sqlite3_context *context, i64 rowid) {\n if (!p->pkIsText) {\n sqlite3_result_int64(context, rowid);\n return SQLITE_OK;\n }\n sqlite3_value *valueId;\n int rc = vec0_get_id_value_from_rowid(p, rowid, &valueId);\n if (rc != SQLITE_OK) {\n return rc;\n }\n if (!valueId) {\n sqlite3_result_error_nomem(context);\n } else {\n sqlite3_result_value(context, valueId);\n sqlite3_value_free(valueId);\n }\n return SQLITE_OK;\n}\n\n/**\n * @brief\n *\n * @param pVtab: virtual table to query\n * @param rowid: row to lookup\n * @param vector_column_idx: which vector column to query\n * @param outVector: Output pointer to the vector buffer.\n * Must be sqlite3_free()'ed.\n * @param outVectorSize: Pointer to a int where the size of outVector\n * will be stored.\n * @return int SQLITE_OK on success.\n */\nint vec0_get_vector_data(vec0_vtab *pVtab, i64 rowid, int vector_column_idx,\n void **outVector, int *outVectorSize) {\n vec0_vtab *p = pVtab;\n int rc, brc;\n i64 chunk_id;\n i64 chunk_offset;\n size_t size;\n void *buf = NULL;\n int blobOffset;\n sqlite3_blob *vectorBlob = NULL;\n assert((vector_column_idx >= 0) &&\n (vector_column_idx < pVtab->numVectorColumns));\n\n rc = vec0_get_chunk_position(pVtab, rowid, NULL, &chunk_id, &chunk_offset);\n if (rc == SQLITE_EMPTY) {\n vtab_set_error(&pVtab->base, \"Could not find a row with rowid %lld\", rowid);\n goto cleanup;\n }\n if (rc != SQLITE_OK) {\n goto cleanup;\n }\n\n rc = sqlite3_blob_open(p->db, p->schemaName,\n p->shadowVectorChunksNames[vector_column_idx],\n \"vectors\", chunk_id, 0, &vectorBlob);\n\n if (rc != SQLITE_OK) {\n vtab_set_error(&pVtab->base,\n \"Could not fetch vector data for %lld, opening blob failed\",\n rowid);\n rc = SQLITE_ERROR;\n goto cleanup;\n }\n\n size = vector_column_byte_size(pVtab->vector_columns[vector_column_idx]);\n blobOffset = chunk_offset * size;\n\n buf = sqlite3_malloc(size);\n if (!buf) {\n rc = SQLITE_NOMEM;\n goto cleanup;\n }\n\n rc = sqlite3_blob_read(vectorBlob, buf, size, blobOffset);\n if (rc != SQLITE_OK) {\n sqlite3_free(buf);\n buf = NULL;\n vtab_set_error(\n &pVtab->base,\n \"Could not fetch vector data for %lld, reading from blob failed\",\n rowid);\n rc = SQLITE_ERROR;\n goto cleanup;\n }\n\n *outVector = buf;\n if (outVectorSize) {\n *outVectorSize = size;\n }\n rc = SQLITE_OK;\n\ncleanup:\n brc = sqlite3_blob_close(vectorBlob);\n if ((rc == SQLITE_OK) && (brc != SQLITE_OK)) {\n vtab_set_error(\n &p->base, VEC_INTERAL_ERROR\n \"unknown error, could not close vector blob, please file an issue\");\n return brc;\n }\n\n return rc;\n}\n\n/**\n * @brief Retrieve the sqlite3_value of the i'th partition value for the given row.\n *\n * @param pVtab - the vec0_vtab in questions\n * @param rowid - rowid of target row\n * @param partition_idx - which partition column to retrieve\n * @param outValue - output sqlite3_value\n * @return int - SQLITE_OK on success, otherwise error code\n */\nint vec0_get_partition_value_for_rowid(vec0_vtab *pVtab, i64 rowid, int partition_idx, sqlite3_value ** outValue) {\n int rc;\n i64 chunk_id;\n i64 chunk_offset;\n rc = vec0_get_chunk_position(pVtab, rowid, NULL, &chunk_id, &chunk_offset);\n if(rc != SQLITE_OK) {\n return rc;\n }\n sqlite3_stmt * stmt = NULL;\n char * zSql = sqlite3_mprintf(\"SELECT partition%02d FROM \" VEC0_SHADOW_CHUNKS_NAME \" WHERE chunk_id = ?\", partition_idx, pVtab->schemaName, pVtab->tableName);\n if(!zSql) {\n return SQLITE_NOMEM;\n }\n rc = sqlite3_prepare_v2(pVtab->db, zSql, -1, &stmt, NULL);\n sqlite3_free(zSql);\n if(rc != SQLITE_OK) {\n return rc;\n }\n sqlite3_bind_int64(stmt, 1, chunk_id);\n rc = sqlite3_step(stmt);\n if(rc != SQLITE_ROW) {\n rc = SQLITE_ERROR;\n goto done;\n }\n *outValue = sqlite3_value_dup(sqlite3_column_value(stmt, 0));\n if(!*outValue) {\n rc = SQLITE_NOMEM;\n goto done;\n }\n rc = SQLITE_OK;\n\n done:\n sqlite3_finalize(stmt);\n return rc;\n\n}\n\n/**\n * @brief Get the value of an auxiliary column for the given rowid\n *\n * @param pVtab vec0_vtab\n * @param rowid the rowid of the row to lookup\n * @param auxiliary_idx aux index of the column we care about\n * @param outValue Output sqlite3_value to store\n * @return int SQLITE_OK on success, error code otherwise\n */\nint vec0_get_auxiliary_value_for_rowid(vec0_vtab *pVtab, i64 rowid, int auxiliary_idx, sqlite3_value ** outValue) {\n int rc;\n sqlite3_stmt * stmt = NULL;\n char * zSql = sqlite3_mprintf(\"SELECT value%02d FROM \" VEC0_SHADOW_AUXILIARY_NAME \" WHERE rowid = ?\", auxiliary_idx, pVtab->schemaName, pVtab->tableName);\n if(!zSql) {\n return SQLITE_NOMEM;\n }\n rc = sqlite3_prepare_v2(pVtab->db, zSql, -1, &stmt, NULL);\n sqlite3_free(zSql);\n if(rc != SQLITE_OK) {\n return rc;\n }\n sqlite3_bind_int64(stmt, 1, rowid);\n rc = sqlite3_step(stmt);\n if(rc != SQLITE_ROW) {\n rc = SQLITE_ERROR;\n goto done;\n }\n *outValue = sqlite3_value_dup(sqlite3_column_value(stmt, 0));\n if(!*outValue) {\n rc = SQLITE_NOMEM;\n goto done;\n }\n rc = SQLITE_OK;\n\n done:\n sqlite3_finalize(stmt);\n return rc;\n}\n\n/**\n * @brief Result the given metadata value for the given row and metadata column index.\n * Will traverse the metadatachunksNN table with BLOB I/0 for the given rowid.\n *\n * @param p\n * @param rowid\n * @param metadata_idx\n * @param context\n * @return int\n */\nint vec0_result_metadata_value_for_rowid(vec0_vtab *p, i64 rowid, int metadata_idx, sqlite3_context * context) {\n int rc;\n i64 chunk_id;\n i64 chunk_offset;\n rc = vec0_get_chunk_position(p, rowid, NULL, &chunk_id, &chunk_offset);\n if(rc != SQLITE_OK) {\n return rc;\n }\n sqlite3_blob * blobValue;\n rc = sqlite3_blob_open(p->db, p->schemaName, p->shadowMetadataChunksNames[metadata_idx], \"data\", chunk_id, 0, &blobValue);\n if(rc != SQLITE_OK) {\n return rc;\n }\n\n switch(p->metadata_columns[metadata_idx].kind) {\n case VEC0_METADATA_COLUMN_KIND_BOOLEAN: {\n u8 block;\n rc = sqlite3_blob_read(blobValue, &block, sizeof(block), chunk_offset / CHAR_BIT);\n if(rc != SQLITE_OK) {\n goto done;\n }\n int value = block >> ((chunk_offset % CHAR_BIT)) & 1;\n sqlite3_result_int(context, value);\n break;\n }\n case VEC0_METADATA_COLUMN_KIND_INTEGER: {\n i64 value;\n rc = sqlite3_blob_read(blobValue, &value, sizeof(value), chunk_offset * sizeof(i64));\n if(rc != SQLITE_OK) {\n goto done;\n }\n sqlite3_result_int64(context, value);\n break;\n }\n case VEC0_METADATA_COLUMN_KIND_FLOAT: {\n double value;\n rc = sqlite3_blob_read(blobValue, &value, sizeof(value), chunk_offset * sizeof(double));\n if(rc != SQLITE_OK) {\n goto done;\n }\n sqlite3_result_double(context, value);\n break;\n }\n case VEC0_METADATA_COLUMN_KIND_TEXT: {\n u8 view[VEC0_METADATA_TEXT_VIEW_BUFFER_LENGTH];\n rc = sqlite3_blob_read(blobValue, &view, VEC0_METADATA_TEXT_VIEW_BUFFER_LENGTH, chunk_offset * VEC0_METADATA_TEXT_VIEW_BUFFER_LENGTH);\n if(rc != SQLITE_OK) {\n goto done;\n }\n int length = ((int *)view)[0];\n if(length <= VEC0_METADATA_TEXT_VIEW_DATA_LENGTH) {\n sqlite3_result_text(context, (const char*) (view + 4), length, SQLITE_TRANSIENT);\n }\n else {\n sqlite3_stmt * stmt;\n const char * zSql = sqlite3_mprintf(\"SELECT data FROM \" VEC0_SHADOW_METADATA_TEXT_DATA_NAME \" WHERE rowid = ?\", p->schemaName, p->tableName, metadata_idx);\n if(!zSql) {\n rc = SQLITE_ERROR;\n goto done;\n }\n rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, NULL);\n sqlite3_free((void *) zSql);\n if(rc != SQLITE_OK) {\n goto done;\n }\n sqlite3_bind_int64(stmt, 1, rowid);\n rc = sqlite3_step(stmt);\n if(rc != SQLITE_ROW) {\n sqlite3_finalize(stmt);\n rc = SQLITE_ERROR;\n goto done;\n }\n sqlite3_result_value(context, sqlite3_column_value(stmt, 0));\n sqlite3_finalize(stmt);\n rc = SQLITE_OK;\n }\n break;\n }\n }\n done:\n // blobValue is read-only, will not fail on close\n sqlite3_blob_close(blobValue);\n return rc;\n\n}\n\nint vec0_get_latest_chunk_rowid(vec0_vtab *p, i64 *chunk_rowid, sqlite3_value ** partitionKeyValues) {\n int rc;\n const char *zSql;\n // lazy initialize stmtLatestChunk when needed. May be cleared during xSync()\n if (!p->stmtLatestChunk) {\n if(p->numPartitionColumns > 0) {\n sqlite3_str * s = sqlite3_str_new(NULL);\n sqlite3_str_appendf(s, \"SELECT max(rowid) FROM \" VEC0_SHADOW_CHUNKS_NAME \" WHERE \",\n p->schemaName, p->tableName);\n\n for(int i = 0; i < p->numPartitionColumns; i++) {\n if(i != 0) {\n sqlite3_str_appendall(s, \" AND \");\n }\n sqlite3_str_appendf(s, \" partition%02d = ? \", i);\n }\n zSql = sqlite3_str_finish(s);\n }else {\n zSql = sqlite3_mprintf(\"SELECT max(rowid) FROM \" VEC0_SHADOW_CHUNKS_NAME,\n p->schemaName, p->tableName);\n }\n\n if (!zSql) {\n rc = SQLITE_NOMEM;\n goto cleanup;\n }\n rc = sqlite3_prepare_v2(p->db, zSql, -1, &p->stmtLatestChunk, 0);\n sqlite3_free((void *)zSql);\n if (rc != SQLITE_OK) {\n // IMP: V21406_05476\n vtab_set_error(&p->base, VEC_INTERAL_ERROR\n \"could not initialize 'latest chunk' statement\");\n goto cleanup;\n }\n }\n\n for(int i = 0; i < p->numPartitionColumns; i++) {\n sqlite3_bind_value(p->stmtLatestChunk, i+1, (partitionKeyValues[i]));\n }\n\n rc = sqlite3_step(p->stmtLatestChunk);\n if (rc != SQLITE_ROW) {\n // IMP: V31559_15629\n vtab_set_error(&p->base, VEC_INTERAL_ERROR \"Could not find latest chunk\");\n rc = SQLITE_ERROR;\n goto cleanup;\n }\n if(sqlite3_column_type(p->stmtLatestChunk, 0) == SQLITE_NULL){\n rc = SQLITE_EMPTY;\n goto cleanup;\n }\n *chunk_rowid = sqlite3_column_int64(p->stmtLatestChunk, 0);\n rc = sqlite3_step(p->stmtLatestChunk);\n if (rc != SQLITE_DONE) {\n vtab_set_error(&p->base,\n VEC_INTERAL_ERROR\n \"unknown result code when closing out stmtLatestChunk. \"\n \"Please file an issue: \" REPORT_URL,\n p->schemaName, p->shadowChunksName);\n goto cleanup;\n }\n rc = SQLITE_OK;\n\ncleanup:\n if (p->stmtLatestChunk) {\n sqlite3_reset(p->stmtLatestChunk);\n sqlite3_clear_bindings(p->stmtLatestChunk);\n }\n return rc;\n}\n\nint vec0_rowids_insert_rowid(vec0_vtab *p, i64 rowid) {\n int rc = SQLITE_OK;\n int entered = 0;\n UNUSED_PARAMETER(entered); // temporary\n if (!p->stmtRowidsInsertRowid) {\n const char *zSql =\n sqlite3_mprintf(\"INSERT INTO \" VEC0_SHADOW_ROWIDS_NAME \"(rowid)\"\n \"VALUES (?);\",\n p->schemaName, p->tableName);\n if (!zSql) {\n rc = SQLITE_NOMEM;\n goto cleanup;\n }\n rc = sqlite3_prepare_v2(p->db, zSql, -1, &p->stmtRowidsInsertRowid, 0);\n sqlite3_free((void *)zSql);\n if (rc != SQLITE_OK) {\n vtab_set_error(&p->base, VEC_INTERAL_ERROR\n \"could not initialize 'insert rowids' statement\");\n goto cleanup;\n }\n }\n\n#if SQLITE_THREADSAFE\n if (sqlite3_mutex_enter) {\n sqlite3_mutex_enter(sqlite3_db_mutex(p->db));\n entered = 1;\n }\n#endif\n sqlite3_bind_int64(p->stmtRowidsInsertRowid, 1, rowid);\n rc = sqlite3_step(p->stmtRowidsInsertRowid);\n\n if (rc != SQLITE_DONE) {\n if (sqlite3_extended_errcode(p->db) == SQLITE_CONSTRAINT_PRIMARYKEY) {\n // IMP: V17090_01160\n vtab_set_error(&p->base, \"UNIQUE constraint failed on %s primary key\",\n p->tableName);\n } else {\n // IMP: V04679_21517\n vtab_set_error(&p->base,\n \"Error inserting rowid into rowids shadow table: %s\",\n sqlite3_errmsg(sqlite3_db_handle(p->stmtRowidsInsertId)));\n }\n rc = SQLITE_ERROR;\n goto cleanup;\n }\n\n rc = SQLITE_OK;\n\ncleanup:\n if (p->stmtRowidsInsertRowid) {\n sqlite3_reset(p->stmtRowidsInsertRowid);\n sqlite3_clear_bindings(p->stmtRowidsInsertRowid);\n }\n\n#if SQLITE_THREADSAFE\n if (sqlite3_mutex_leave && entered) {\n sqlite3_mutex_leave(sqlite3_db_mutex(p->db));\n }\n#endif\n return rc;\n}\n\nint vec0_rowids_insert_id(vec0_vtab *p, sqlite3_value *idValue, i64 *rowid) {\n int rc = SQLITE_OK;\n int entered = 0;\n UNUSED_PARAMETER(entered); // temporary\n if (!p->stmtRowidsInsertId) {\n const char *zSql =\n sqlite3_mprintf(\"INSERT INTO \" VEC0_SHADOW_ROWIDS_NAME \"(id)\"\n \"VALUES (?);\",\n p->schemaName, p->tableName);\n if (!zSql) {\n rc = SQLITE_NOMEM;\n goto complete;\n }\n rc = sqlite3_prepare_v2(p->db, zSql, -1, &p->stmtRowidsInsertId, 0);\n sqlite3_free((void *)zSql);\n if (rc != SQLITE_OK) {\n vtab_set_error(&p->base, VEC_INTERAL_ERROR\n \"could not initialize 'insert rowids id' statement\");\n goto complete;\n }\n }\n\n#if SQLITE_THREADSAFE\n if (sqlite3_mutex_enter) {\n sqlite3_mutex_enter(sqlite3_db_mutex(p->db));\n entered = 1;\n }\n#endif\n\n if (idValue) {\n sqlite3_bind_value(p->stmtRowidsInsertId, 1, idValue);\n }\n rc = sqlite3_step(p->stmtRowidsInsertId);\n\n if (rc != SQLITE_DONE) {\n if (sqlite3_extended_errcode(p->db) == SQLITE_CONSTRAINT_UNIQUE) {\n // IMP: V20497_04568\n vtab_set_error(&p->base, \"UNIQUE constraint failed on %s primary key\",\n p->tableName);\n } else {\n // IMP: V24016_08086\n // IMP: V15177_32015\n vtab_set_error(&p->base,\n \"Error inserting id into rowids shadow table: %s\",\n sqlite3_errmsg(sqlite3_db_handle(p->stmtRowidsInsertId)));\n }\n rc = SQLITE_ERROR;\n goto complete;\n }\n\n *rowid = sqlite3_last_insert_rowid(p->db);\n rc = SQLITE_OK;\n\ncomplete:\n if (p->stmtRowidsInsertId) {\n sqlite3_reset(p->stmtRowidsInsertId);\n sqlite3_clear_bindings(p->stmtRowidsInsertId);\n }\n\n#if SQLITE_THREADSAFE\n if (sqlite3_mutex_leave && entered) {\n sqlite3_mutex_leave(sqlite3_db_mutex(p->db));\n }\n#endif\n return rc;\n}\n\nint vec0_metadata_chunk_size(vec0_metadata_column_kind kind, int chunk_size) {\n switch(kind) {\n case VEC0_METADATA_COLUMN_KIND_BOOLEAN:\n return chunk_size / 8;\n case VEC0_METADATA_COLUMN_KIND_INTEGER:\n return chunk_size * sizeof(i64);\n case VEC0_METADATA_COLUMN_KIND_FLOAT:\n return chunk_size * sizeof(double);\n case VEC0_METADATA_COLUMN_KIND_TEXT:\n return chunk_size * VEC0_METADATA_TEXT_VIEW_BUFFER_LENGTH;\n }\n return 0;\n}\n\nint vec0_rowids_update_position(vec0_vtab *p, i64 rowid, i64 chunk_rowid,\n i64 chunk_offset) {\n int rc = SQLITE_OK;\n\n if (!p->stmtRowidsUpdatePosition) {\n const char *zSql = sqlite3_mprintf(\" UPDATE \" VEC0_SHADOW_ROWIDS_NAME\n \" SET chunk_id = ?, chunk_offset = ?\"\n \" WHERE rowid = ?\",\n p->schemaName, p->tableName);\n if (!zSql) {\n rc = SQLITE_NOMEM;\n goto cleanup;\n }\n rc = sqlite3_prepare_v2(p->db, zSql, -1, &p->stmtRowidsUpdatePosition, 0);\n sqlite3_free((void *)zSql);\n if (rc != SQLITE_OK) {\n vtab_set_error(&p->base, VEC_INTERAL_ERROR\n \"could not initialize 'update rowids position' statement\");\n goto cleanup;\n }\n }\n\n sqlite3_bind_int64(p->stmtRowidsUpdatePosition, 1, chunk_rowid);\n sqlite3_bind_int64(p->stmtRowidsUpdatePosition, 2, chunk_offset);\n sqlite3_bind_int64(p->stmtRowidsUpdatePosition, 3, rowid);\n\n rc = sqlite3_step(p->stmtRowidsUpdatePosition);\n if (rc != SQLITE_DONE) {\n // IMP: V21925_05995\n vtab_set_error(&p->base,\n VEC_INTERAL_ERROR\n \"could not update rowids position for rowid=%lld, \"\n \"chunk_rowid=%lld, chunk_offset=%lld\",\n rowid, chunk_rowid, chunk_offset);\n rc = SQLITE_ERROR;\n goto cleanup;\n }\n rc = SQLITE_OK;\n\ncleanup:\n if (p->stmtRowidsUpdatePosition) {\n sqlite3_reset(p->stmtRowidsUpdatePosition);\n sqlite3_clear_bindings(p->stmtRowidsUpdatePosition);\n }\n\n return rc;\n}\n\n/**\n * @brief Adds a new chunk for the vec0 table, and the corresponding vector\n * chunks.\n *\n * Inserts a new row into the _chunks table, with blank data, and uses that new\n * rowid to insert new blank rows into _vector_chunksXX tables.\n *\n * @param p: vec0 table to add new chunk\n * @param paritionKeyValues: Array of partition key valeus for the new chunk, if available\n * @param chunk_rowid: Output pointer, if not NULL, then will be filled with the\n * new chunk rowid.\n * @return int SQLITE_OK on success, error code otherwise.\n */\nint vec0_new_chunk(vec0_vtab *p, sqlite3_value ** partitionKeyValues, i64 *chunk_rowid) {\n int rc;\n char *zSql;\n sqlite3_stmt *stmt;\n i64 rowid;\n\n // Step 1: Insert a new row in _chunks, capture that new rowid\n if(p->numPartitionColumns > 0) {\n sqlite3_str * s = sqlite3_str_new(NULL);\n sqlite3_str_appendf(s, \"INSERT INTO \" VEC0_SHADOW_CHUNKS_NAME, p->schemaName, p->tableName);\n sqlite3_str_appendall(s, \"(size, validity, rowids\");\n for(int i = 0; i < p->numPartitionColumns; i++) {\n sqlite3_str_appendf(s, \", partition%02d\", i);\n }\n sqlite3_str_appendall(s, \") VALUES (?, ?, ?\");\n for(int i = 0; i < p->numPartitionColumns; i++) {\n sqlite3_str_appendall(s, \", ?\");\n }\n sqlite3_str_appendall(s, \")\");\n\n zSql = sqlite3_str_finish(s);\n }else {\n zSql = sqlite3_mprintf(\"INSERT INTO \" VEC0_SHADOW_CHUNKS_NAME\n \"(size, validity, rowids) \"\n \"VALUES (?, ?, ?);\",\n p->schemaName, p->tableName);\n }\n\n if (!zSql) {\n return SQLITE_NOMEM;\n }\n rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, NULL);\n sqlite3_free(zSql);\n if (rc != SQLITE_OK) {\n sqlite3_finalize(stmt);\n return rc;\n }\n\n#if SQLITE_THREADSAFE\n if (sqlite3_mutex_enter) {\n sqlite3_mutex_enter(sqlite3_db_mutex(p->db));\n }\n#endif\n\n sqlite3_bind_int64(stmt, 1, p->chunk_size); // size\n sqlite3_bind_zeroblob(stmt, 2, p->chunk_size / CHAR_BIT); // validity bitmap\n sqlite3_bind_zeroblob(stmt, 3, p->chunk_size * sizeof(i64)); // rowids\n\n for(int i = 0; i < p->numPartitionColumns; i++) {\n sqlite3_bind_value(stmt, 4 + i, partitionKeyValues[i]);\n }\n\n rc = sqlite3_step(stmt);\n int failed = rc != SQLITE_DONE;\n rowid = sqlite3_last_insert_rowid(p->db);\n#if SQLITE_THREADSAFE\n if (sqlite3_mutex_leave) {\n sqlite3_mutex_leave(sqlite3_db_mutex(p->db));\n }\n#endif\n sqlite3_finalize(stmt);\n if (failed) {\n return SQLITE_ERROR;\n }\n\n // Step 2: Create new vector chunks for each vector column, with\n // that new chunk_rowid.\n //\n // SHADOW_TABLE_ROWID_QUIRK: The _vector_chunksNN and _metadatachunksNN\n // shadow tables declare \"rowid PRIMARY KEY\" without the INTEGER type, so\n // the user-defined \"rowid\" column is NOT an alias for the internal SQLite\n // rowid (_rowid_). When only appending rows these two happen to stay in\n // sync, but after a chunk is deleted (vec0Update_Delete_DeleteChunkIfEmpty)\n // and a new one is created, the auto-assigned _rowid_ can diverge from the\n // user \"rowid\" value. Since sqlite3_blob_open() addresses rows by internal\n // _rowid_, we must explicitly set BOTH _rowid_ and \"rowid\" to the same\n // value so that later blob operations can find the row.\n //\n // The correct long-term fix is changing the schema to\n // \"rowid INTEGER PRIMARY KEY\"\n // which makes it a true alias, but that would break existing databases.\n\n for (int i = 0; i < vec0_num_defined_user_columns(p); i++) {\n if(p->user_column_kinds[i] != SQLITE_VEC0_USER_COLUMN_KIND_VECTOR) {\n continue;\n }\n int vector_column_idx = p->user_column_idxs[i];\n i64 vectorsSize =\n p->chunk_size * vector_column_byte_size(p->vector_columns[vector_column_idx]);\n\n // See SHADOW_TABLE_ROWID_QUIRK above for why _rowid_ and rowid are both set.\n zSql = sqlite3_mprintf(\"INSERT INTO \" VEC0_SHADOW_VECTOR_N_NAME\n \"(_rowid_, rowid, vectors)\"\n \"VALUES (?, ?, ?)\",\n p->schemaName, p->tableName, vector_column_idx);\n if (!zSql) {\n return SQLITE_NOMEM;\n }\n rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, NULL);\n sqlite3_free(zSql);\n\n if (rc != SQLITE_OK) {\n sqlite3_finalize(stmt);\n return rc;\n }\n\n sqlite3_bind_int64(stmt, 1, rowid); // _rowid_ (internal SQLite rowid)\n sqlite3_bind_int64(stmt, 2, rowid); // rowid (user-defined column)\n sqlite3_bind_zeroblob64(stmt, 3, vectorsSize);\n\n rc = sqlite3_step(stmt);\n sqlite3_finalize(stmt);\n if (rc != SQLITE_DONE) {\n return rc;\n }\n }\n\n // Step 3: Create new metadata chunks for each metadata column\n for (int i = 0; i < vec0_num_defined_user_columns(p); i++) {\n if(p->user_column_kinds[i] != SQLITE_VEC0_USER_COLUMN_KIND_METADATA) {\n continue;\n }\n int metadata_column_idx = p->user_column_idxs[i];\n // See SHADOW_TABLE_ROWID_QUIRK above for why _rowid_ and rowid are both set.\n zSql = sqlite3_mprintf(\"INSERT INTO \" VEC0_SHADOW_METADATA_N_NAME\n \"(_rowid_, rowid, data)\"\n \"VALUES (?, ?, ?)\",\n p->schemaName, p->tableName, metadata_column_idx);\n if (!zSql) {\n return SQLITE_NOMEM;\n }\n rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, NULL);\n sqlite3_free(zSql);\n\n if (rc != SQLITE_OK) {\n sqlite3_finalize(stmt);\n return rc;\n }\n\n sqlite3_bind_int64(stmt, 1, rowid); // _rowid_ (internal SQLite rowid)\n sqlite3_bind_int64(stmt, 2, rowid); // rowid (user-defined column)\n sqlite3_bind_zeroblob64(stmt, 3, vec0_metadata_chunk_size(p->metadata_columns[metadata_column_idx].kind, p->chunk_size));\n\n rc = sqlite3_step(stmt);\n sqlite3_finalize(stmt);\n if (rc != SQLITE_DONE) {\n return rc;\n }\n }\n\n\n if (chunk_rowid) {\n *chunk_rowid = rowid;\n }\n\n return SQLITE_OK;\n}\n\nstruct vec0_query_fullscan_data {\n sqlite3_stmt *rowids_stmt;\n i8 done;\n};\nvoid vec0_query_fullscan_data_clear(\n struct vec0_query_fullscan_data *fullscan_data) {\n if (!fullscan_data)\n return;\n\n if (fullscan_data->rowids_stmt) {\n sqlite3_finalize(fullscan_data->rowids_stmt);\n fullscan_data->rowids_stmt = NULL;\n }\n}\n\nstruct vec0_query_knn_data {\n i64 k;\n i64 k_used;\n // Array of rowids of size k. Must be freed with sqlite3_free().\n i64 *rowids;\n // Array of distances of size k. Must be freed with sqlite3_free().\n f32 *distances;\n i64 current_idx;\n};\nvoid vec0_query_knn_data_clear(struct vec0_query_knn_data *knn_data) {\n if (!knn_data)\n return;\n\n if (knn_data->rowids) {\n sqlite3_free(knn_data->rowids);\n knn_data->rowids = NULL;\n }\n if (knn_data->distances) {\n sqlite3_free(knn_data->distances);\n knn_data->distances = NULL;\n }\n}\n\nstruct vec0_query_point_data {\n i64 rowid;\n void *vectors[VEC0_MAX_VECTOR_COLUMNS];\n int done;\n};\nvoid vec0_query_point_data_clear(struct vec0_query_point_data *point_data) {\n if (!point_data)\n return;\n for (int i = 0; i < VEC0_MAX_VECTOR_COLUMNS; i++) {\n sqlite3_free(point_data->vectors[i]);\n point_data->vectors[i] = NULL;\n }\n}\n\ntypedef enum {\n // If any values are updated, please update the ARCHITECTURE.md docs accordingly!\n\n VEC0_QUERY_PLAN_FULLSCAN = '1',\n VEC0_QUERY_PLAN_POINT = '2',\n VEC0_QUERY_PLAN_KNN = '3',\n} vec0_query_plan;\n\ntypedef struct vec0_cursor vec0_cursor;\nstruct vec0_cursor {\n sqlite3_vtab_cursor base;\n\n vec0_query_plan query_plan;\n struct vec0_query_fullscan_data *fullscan_data;\n struct vec0_query_knn_data *knn_data;\n struct vec0_query_point_data *point_data;\n};\n\nvoid vec0_cursor_clear(vec0_cursor *pCur) {\n if (pCur->fullscan_data) {\n vec0_query_fullscan_data_clear(pCur->fullscan_data);\n sqlite3_free(pCur->fullscan_data);\n pCur->fullscan_data = NULL;\n }\n if (pCur->knn_data) {\n vec0_query_knn_data_clear(pCur->knn_data);\n sqlite3_free(pCur->knn_data);\n pCur->knn_data = NULL;\n }\n if (pCur->point_data) {\n vec0_query_point_data_clear(pCur->point_data);\n sqlite3_free(pCur->point_data);\n pCur->point_data = NULL;\n }\n}\n\n#define VEC_CONSTRUCTOR_ERROR \"vec0 constructor error: \"\nstatic int vec0_init(sqlite3 *db, void *pAux, int argc, const char *const *argv,\n sqlite3_vtab **ppVtab, char **pzErr, bool isCreate) {\n UNUSED_PARAMETER(pAux);\n vec0_vtab *pNew;\n int rc;\n const char *zSql;\n\n pNew = sqlite3_malloc(sizeof(*pNew));\n if (pNew == 0)\n return SQLITE_NOMEM;\n memset(pNew, 0, sizeof(*pNew));\n\n // Declared chunk_size=N for entire table.\n // -1 to use the defualt, otherwise will get re-assigned on `chunk_size=N`\n // option\n int chunk_size = -1;\n int numVectorColumns = 0;\n int numPartitionColumns = 0;\n int numAuxiliaryColumns = 0;\n int numMetadataColumns = 0;\n int user_column_idx = 0;\n\n // track if a \"primary key\" column is defined\n char *pkColumnName = NULL;\n int pkColumnNameLength;\n int pkColumnType = SQLITE_INTEGER;\n\n for (int i = 3; i < argc; i++) {\n struct VectorColumnDefinition vecColumn;\n struct Vec0PartitionColumnDefinition partitionColumn;\n struct Vec0AuxiliaryColumnDefinition auxColumn;\n struct Vec0MetadataColumnDefinition metadataColumn;\n char *cName = NULL;\n int cNameLength;\n int cType;\n\n // Scenario #1: Constructor argument is a vector column definition, ie `foo float[1024]`\n rc = vec0_parse_vector_column(argv[i], strlen(argv[i]), &vecColumn);\n if (rc == SQLITE_ERROR) {\n *pzErr = sqlite3_mprintf(\n VEC_CONSTRUCTOR_ERROR \"could not parse vector column '%s'\", argv[i]);\n goto error;\n }\n if (rc == SQLITE_OK) {\n if (numVectorColumns >= VEC0_MAX_VECTOR_COLUMNS) {\n sqlite3_free(vecColumn.name);\n *pzErr = sqlite3_mprintf(VEC_CONSTRUCTOR_ERROR\n \"Too many provided vector columns, maximum %d\",\n VEC0_MAX_VECTOR_COLUMNS);\n goto error;\n }\n\n if (vecColumn.dimensions > SQLITE_VEC_VEC0_MAX_DIMENSIONS) {\n sqlite3_free(vecColumn.name);\n *pzErr = sqlite3_mprintf(\n VEC_CONSTRUCTOR_ERROR\n \"Dimension on vector column too large, provided %lld, maximum %lld\",\n (i64)vecColumn.dimensions, SQLITE_VEC_VEC0_MAX_DIMENSIONS);\n goto error;\n }\n pNew->user_column_kinds[user_column_idx] = SQLITE_VEC0_USER_COLUMN_KIND_VECTOR;\n pNew->user_column_idxs[user_column_idx] = numVectorColumns;\n memcpy(&pNew->vector_columns[numVectorColumns], &vecColumn, sizeof(vecColumn));\n numVectorColumns++;\n pNew->numVectorColumns = numVectorColumns;\n user_column_idx++;\n\n continue;\n }\n\n // Scenario #2: Constructor argument is a partition key column definition, ie `user_id text partition key`\n rc = vec0_parse_partition_key_definition(argv[i], strlen(argv[i]), &cName,\n &cNameLength, &cType);\n if (rc == SQLITE_OK) {\n if (numPartitionColumns >= VEC0_MAX_PARTITION_COLUMNS) {\n *pzErr = sqlite3_mprintf(\n VEC_CONSTRUCTOR_ERROR\n \"More than %d partition key columns were provided\",\n VEC0_MAX_PARTITION_COLUMNS);\n goto error;\n }\n partitionColumn.type = cType;\n partitionColumn.name_length = cNameLength;\n partitionColumn.name = sqlite3_mprintf(\"%.*s\", cNameLength, cName);\n if(!partitionColumn.name) {\n rc = SQLITE_NOMEM;\n goto error;\n }\n\n pNew->user_column_kinds[user_column_idx] = SQLITE_VEC0_USER_COLUMN_KIND_PARTITION;\n pNew->user_column_idxs[user_column_idx] = numPartitionColumns;\n memcpy(&pNew->paritition_columns[numPartitionColumns], &partitionColumn, sizeof(partitionColumn));\n numPartitionColumns++;\n pNew->numPartitionColumns = numPartitionColumns;\n user_column_idx++;\n continue;\n }\n\n // Scenario #3: Constructor argument is a primary key column definition, ie `article_id text primary key`\n rc = vec0_parse_primary_key_definition(argv[i], strlen(argv[i]), &cName,\n &cNameLength, &cType);\n if (rc == SQLITE_OK) {\n if (pkColumnName) {\n *pzErr = sqlite3_mprintf(\n VEC_CONSTRUCTOR_ERROR\n \"More than one primary key definition was provided, vec0 only \"\n \"suports a single primary key column\",\n argv[i]);\n goto error;\n }\n pkColumnName = cName;\n pkColumnNameLength = cNameLength;\n pkColumnType = cType;\n continue;\n }\n\n // Scenario #4: Constructor argument is a auxiliary column definition, ie `+contents text`\n rc = vec0_parse_auxiliary_column_definition(argv[i], strlen(argv[i]), &cName,\n &cNameLength, &cType);\n if(rc == SQLITE_OK) {\n if (numAuxiliaryColumns >= VEC0_MAX_AUXILIARY_COLUMNS) {\n *pzErr = sqlite3_mprintf(\n VEC_CONSTRUCTOR_ERROR\n \"More than %d auxiliary columns were provided\",\n VEC0_MAX_AUXILIARY_COLUMNS);\n goto error;\n }\n auxColumn.type = cType;\n auxColumn.name_length = cNameLength;\n auxColumn.name = sqlite3_mprintf(\"%.*s\", cNameLength, cName);\n if(!auxColumn.name) {\n rc = SQLITE_NOMEM;\n goto error;\n }\n\n pNew->user_column_kinds[user_column_idx] = SQLITE_VEC0_USER_COLUMN_KIND_AUXILIARY;\n pNew->user_column_idxs[user_column_idx] = numAuxiliaryColumns;\n memcpy(&pNew->auxiliary_columns[numAuxiliaryColumns], &auxColumn, sizeof(auxColumn));\n numAuxiliaryColumns++;\n pNew->numAuxiliaryColumns = numAuxiliaryColumns;\n user_column_idx++;\n continue;\n }\n\n vec0_metadata_column_kind kind;\n rc = vec0_parse_metadata_column_definition(argv[i], strlen(argv[i]), &cName,\n &cNameLength, &kind);\n if(rc == SQLITE_OK) {\n if (numMetadataColumns >= VEC0_MAX_METADATA_COLUMNS) {\n *pzErr = sqlite3_mprintf(\n VEC_CONSTRUCTOR_ERROR\n \"More than %d metadata columns were provided\",\n VEC0_MAX_METADATA_COLUMNS);\n goto error;\n }\n metadataColumn.kind = kind;\n metadataColumn.name_length = cNameLength;\n metadataColumn.name = sqlite3_mprintf(\"%.*s\", cNameLength, cName);\n if(!metadataColumn.name) {\n rc = SQLITE_NOMEM;\n goto error;\n }\n\n pNew->user_column_kinds[user_column_idx] = SQLITE_VEC0_USER_COLUMN_KIND_METADATA;\n pNew->user_column_idxs[user_column_idx] = numMetadataColumns;\n memcpy(&pNew->metadata_columns[numMetadataColumns], &metadataColumn, sizeof(metadataColumn));\n numMetadataColumns++;\n pNew->numMetadataColumns = numMetadataColumns;\n user_column_idx++;\n continue;\n }\n\n // Scenario #4: Constructor argument is a table-level option, ie `chunk_size`\n\n char *key;\n char *value;\n int keyLength, valueLength;\n rc = vec0_parse_table_option(argv[i], strlen(argv[i]), &key, &keyLength,\n &value, &valueLength);\n if (rc == SQLITE_ERROR) {\n *pzErr = sqlite3_mprintf(\n VEC_CONSTRUCTOR_ERROR \"could not parse table option '%s'\", argv[i]);\n goto error;\n }\n if (rc == SQLITE_OK) {\n if (sqlite3_strnicmp(key, \"chunk_size\", keyLength) == 0) {\n chunk_size = atoi(value);\n if (chunk_size <= 0) {\n // IMP: V01931_18769\n *pzErr =\n sqlite3_mprintf(VEC_CONSTRUCTOR_ERROR\n \"chunk_size must be a non-zero positive integer\");\n goto error;\n }\n if ((chunk_size % 8) != 0) {\n // IMP: V14110_30948\n *pzErr = sqlite3_mprintf(VEC_CONSTRUCTOR_ERROR\n \"chunk_size must be divisible by 8\");\n goto error;\n }\n#define SQLITE_VEC_CHUNK_SIZE_MAX 4096\n if (chunk_size > SQLITE_VEC_CHUNK_SIZE_MAX) {\n *pzErr =\n sqlite3_mprintf(VEC_CONSTRUCTOR_ERROR \"chunk_size too large\");\n goto error;\n }\n } else {\n // IMP: V27642_11712\n *pzErr = sqlite3_mprintf(\n VEC_CONSTRUCTOR_ERROR \"Unknown table option: %.*s\", keyLength, key);\n goto error;\n }\n continue;\n }\n\n // Scenario #5: Unknown constructor argument\n *pzErr =\n sqlite3_mprintf(VEC_CONSTRUCTOR_ERROR \"Could not parse '%s'\", argv[i]);\n goto error;\n }\n\n if (chunk_size < 0) {\n chunk_size = 1024;\n }\n\n if (numVectorColumns <= 0) {\n *pzErr = sqlite3_mprintf(VEC_CONSTRUCTOR_ERROR\n \"At least one vector column is required\");\n goto error;\n }\n\n sqlite3_str *createStr = sqlite3_str_new(NULL);\n sqlite3_str_appendall(createStr, \"CREATE TABLE x(\");\n if (pkColumnName) {\n sqlite3_str_appendf(createStr, \"\\\"%.*w\\\" primary key, \", pkColumnNameLength,\n pkColumnName);\n } else {\n sqlite3_str_appendall(createStr, \"rowid, \");\n }\n for (int i = 0; i < numVectorColumns + numPartitionColumns + numAuxiliaryColumns + numMetadataColumns; i++) {\n switch(pNew->user_column_kinds[i]) {\n case SQLITE_VEC0_USER_COLUMN_KIND_VECTOR: {\n int vector_idx = pNew->user_column_idxs[i];\n sqlite3_str_appendf(createStr, \"\\\"%.*w\\\", \",\n pNew->vector_columns[vector_idx].name_length,\n pNew->vector_columns[vector_idx].name);\n break;\n }\n case SQLITE_VEC0_USER_COLUMN_KIND_PARTITION: {\n int partition_idx = pNew->user_column_idxs[i];\n sqlite3_str_appendf(createStr, \"\\\"%.*w\\\", \",\n pNew->paritition_columns[partition_idx].name_length,\n pNew->paritition_columns[partition_idx].name);\n break;\n }\n case SQLITE_VEC0_USER_COLUMN_KIND_AUXILIARY: {\n int auxiliary_idx = pNew->user_column_idxs[i];\n sqlite3_str_appendf(createStr, \"\\\"%.*w\\\", \",\n pNew->auxiliary_columns[auxiliary_idx].name_length,\n pNew->auxiliary_columns[auxiliary_idx].name);\n break;\n }\n case SQLITE_VEC0_USER_COLUMN_KIND_METADATA: {\n int metadata_idx = pNew->user_column_idxs[i];\n sqlite3_str_appendf(createStr, \"\\\"%.*w\\\", \",\n pNew->metadata_columns[metadata_idx].name_length,\n pNew->metadata_columns[metadata_idx].name);\n break;\n }\n }\n\n }\n sqlite3_str_appendall(createStr, \" distance hidden, k hidden) \");\n if (pkColumnName) {\n sqlite3_str_appendall(createStr, \"without rowid \");\n }\n zSql = sqlite3_str_finish(createStr);\n if (!zSql) {\n goto error;\n }\n rc = sqlite3_declare_vtab(db, zSql);\n sqlite3_free((void *)zSql);\n if (rc != SQLITE_OK) {\n *pzErr = sqlite3_mprintf(VEC_CONSTRUCTOR_ERROR\n \"could not declare virtual table, '%s'\",\n sqlite3_errmsg(db));\n goto error;\n }\n\n const char *schemaName = argv[1];\n const char *tableName = argv[2];\n\n pNew->db = db;\n pNew->pkIsText = pkColumnType == SQLITE_TEXT;\n pNew->schemaName = sqlite3_mprintf(\"%s\", schemaName);\n if (!pNew->schemaName) {\n goto error;\n }\n pNew->tableName = sqlite3_mprintf(\"%s\", tableName);\n if (!pNew->tableName) {\n goto error;\n }\n pNew->shadowRowidsName = sqlite3_mprintf(\"%s_rowids\", tableName);\n if (!pNew->shadowRowidsName) {\n goto error;\n }\n pNew->shadowChunksName = sqlite3_mprintf(\"%s_chunks\", tableName);\n if (!pNew->shadowChunksName) {\n goto error;\n }\n pNew->numVectorColumns = numVectorColumns;\n pNew->numPartitionColumns = numPartitionColumns;\n pNew->numAuxiliaryColumns = numAuxiliaryColumns;\n pNew->numMetadataColumns = numMetadataColumns;\n\n for (int i = 0; i < pNew->numVectorColumns; i++) {\n pNew->shadowVectorChunksNames[i] =\n sqlite3_mprintf(\"%s_vector_chunks%02d\", tableName, i);\n if (!pNew->shadowVectorChunksNames[i]) {\n goto error;\n }\n }\n for (int i = 0; i < pNew->numMetadataColumns; i++) {\n pNew->shadowMetadataChunksNames[i] =\n sqlite3_mprintf(\"%s_metadatachunks%02d\", tableName, i);\n if (!pNew->shadowMetadataChunksNames[i]) {\n goto error;\n }\n }\n pNew->chunk_size = chunk_size;\n\n // if xCreate, then create the necessary shadow tables\n if (isCreate) {\n sqlite3_stmt *stmt;\n int rc;\n\n char * zCreateInfo = sqlite3_mprintf(\"CREATE TABLE \"VEC0_SHADOW_INFO_NAME \" (key text primary key, value any)\", pNew->schemaName, pNew->tableName);\n if(!zCreateInfo) {\n goto error;\n }\n rc = sqlite3_prepare_v2(db, zCreateInfo, -1, &stmt, NULL);\n\n sqlite3_free((void *) zCreateInfo);\n if ((rc != SQLITE_OK) || (sqlite3_step(stmt) != SQLITE_DONE)) {\n // TODO(IMP)\n sqlite3_finalize(stmt);\n *pzErr = sqlite3_mprintf(\"Could not create '_info' shadow table: %s\",\n sqlite3_errmsg(db));\n goto error;\n }\n sqlite3_finalize(stmt);\n\n char * zSeedInfo = sqlite3_mprintf(\n \"INSERT INTO \"VEC0_SHADOW_INFO_NAME \"(key, value) VALUES \"\n \"(?1, ?2), (?3, ?4), (?5, ?6), (?7, ?8) \",\n pNew->schemaName, pNew->tableName\n );\n if(!zSeedInfo) {\n goto error;\n }\n rc = sqlite3_prepare_v2(db, zSeedInfo, -1, &stmt, NULL);\n sqlite3_free((void *) zSeedInfo);\n if (rc != SQLITE_OK) {\n // TODO(IMP)\n sqlite3_finalize(stmt);\n *pzErr = sqlite3_mprintf(\"Could not seed '_info' shadow table: %s\",\n sqlite3_errmsg(db));\n goto error;\n }\n sqlite3_bind_text(stmt, 1, \"CREATE_VERSION\", -1, SQLITE_STATIC);\n sqlite3_bind_text(stmt, 2, SQLITE_VEC_VERSION, -1, SQLITE_STATIC);\n sqlite3_bind_text(stmt, 3, \"CREATE_VERSION_MAJOR\", -1, SQLITE_STATIC);\n sqlite3_bind_int(stmt, 4, SQLITE_VEC_VERSION_MAJOR);\n sqlite3_bind_text(stmt, 5, \"CREATE_VERSION_MINOR\", -1, SQLITE_STATIC);\n sqlite3_bind_int(stmt, 6, SQLITE_VEC_VERSION_MINOR);\n sqlite3_bind_text(stmt, 7, \"CREATE_VERSION_PATCH\", -1, SQLITE_STATIC);\n sqlite3_bind_int(stmt, 8, SQLITE_VEC_VERSION_PATCH);\n\n if(sqlite3_step(stmt) != SQLITE_DONE) {\n // TODO(IMP)\n sqlite3_finalize(stmt);\n *pzErr = sqlite3_mprintf(\"Could not seed '_info' shadow table: %s\",\n sqlite3_errmsg(db));\n goto error;\n }\n sqlite3_finalize(stmt);\n\n\n\n // create the _chunks shadow table\n char *zCreateShadowChunks = NULL;\n if(pNew->numPartitionColumns) {\n sqlite3_str * s = sqlite3_str_new(NULL);\n sqlite3_str_appendf(s, \"CREATE TABLE \" VEC0_SHADOW_CHUNKS_NAME \"(\", pNew->schemaName, pNew->tableName);\n sqlite3_str_appendall(s, \"chunk_id INTEGER PRIMARY KEY AUTOINCREMENT,\" \"size INTEGER NOT NULL,\");\n sqlite3_str_appendall(s, \"sequence_id integer,\");\n for(int i = 0; i < pNew->numPartitionColumns;i++) {\n sqlite3_str_appendf(s, \"partition%02d,\", i);\n }\n sqlite3_str_appendall(s, \"validity BLOB NOT NULL, rowids BLOB NOT NULL);\");\n zCreateShadowChunks = sqlite3_str_finish(s);\n }else {\n zCreateShadowChunks = sqlite3_mprintf(VEC0_SHADOW_CHUNKS_CREATE,\n pNew->schemaName, pNew->tableName);\n }\n if (!zCreateShadowChunks) {\n goto error;\n }\n rc = sqlite3_prepare_v2(db, zCreateShadowChunks, -1, &stmt, 0);\n sqlite3_free((void *)zCreateShadowChunks);\n if ((rc != SQLITE_OK) || (sqlite3_step(stmt) != SQLITE_DONE)) {\n // IMP: V17740_01811\n sqlite3_finalize(stmt);\n *pzErr = sqlite3_mprintf(\"Could not create '_chunks' shadow table: %s\",\n sqlite3_errmsg(db));\n goto error;\n }\n sqlite3_finalize(stmt);\n\n // create the _rowids shadow table\n char *zCreateShadowRowids;\n if (pNew->pkIsText) {\n // adds a \"text unique not null\" constraint to the id column\n zCreateShadowRowids = sqlite3_mprintf(VEC0_SHADOW_ROWIDS_CREATE_PK_TEXT,\n pNew->schemaName, pNew->tableName);\n } else {\n zCreateShadowRowids = sqlite3_mprintf(VEC0_SHADOW_ROWIDS_CREATE_BASIC,\n pNew->schemaName, pNew->tableName);\n }\n if (!zCreateShadowRowids) {\n goto error;\n }\n rc = sqlite3_prepare_v2(db, zCreateShadowRowids, -1, &stmt, 0);\n sqlite3_free((void *)zCreateShadowRowids);\n if ((rc != SQLITE_OK) || (sqlite3_step(stmt) != SQLITE_DONE)) {\n // IMP: V11631_28470\n sqlite3_finalize(stmt);\n *pzErr = sqlite3_mprintf(\"Could not create '_rowids' shadow table: %s\",\n sqlite3_errmsg(db));\n goto error;\n }\n sqlite3_finalize(stmt);\n\n for (int i = 0; i < pNew->numVectorColumns; i++) {\n char *zSql = sqlite3_mprintf(VEC0_SHADOW_VECTOR_N_CREATE,\n pNew->schemaName, pNew->tableName, i);\n if (!zSql) {\n goto error;\n }\n rc = sqlite3_prepare_v2(db, zSql, -1, &stmt, 0);\n sqlite3_free((void *)zSql);\n if ((rc != SQLITE_OK) || (sqlite3_step(stmt) != SQLITE_DONE)) {\n // IMP: V25919_09989\n sqlite3_finalize(stmt);\n *pzErr = sqlite3_mprintf(\n \"Could not create '_vector_chunks%02d' shadow table: %s\", i,\n sqlite3_errmsg(db));\n goto error;\n }\n sqlite3_finalize(stmt);\n }\n\n // See SHADOW_TABLE_ROWID_QUIRK in vec0_new_chunk() — same \"rowid PRIMARY KEY\"\n // without INTEGER type issue applies here.\n for (int i = 0; i < pNew->numMetadataColumns; i++) {\n char *zSql = sqlite3_mprintf(\"CREATE TABLE \" VEC0_SHADOW_METADATA_N_NAME \"(rowid PRIMARY KEY, data BLOB NOT NULL);\",\n pNew->schemaName, pNew->tableName, i);\n if (!zSql) {\n goto error;\n }\n rc = sqlite3_prepare_v2(db, zSql, -1, &stmt, 0);\n sqlite3_free((void *)zSql);\n if ((rc != SQLITE_OK) || (sqlite3_step(stmt) != SQLITE_DONE)) {\n sqlite3_finalize(stmt);\n *pzErr = sqlite3_mprintf(\n \"Could not create '_metata_chunks%02d' shadow table: %s\", i,\n sqlite3_errmsg(db));\n goto error;\n }\n sqlite3_finalize(stmt);\n\n if(pNew->metadata_columns[i].kind == VEC0_METADATA_COLUMN_KIND_TEXT) {\n char *zSql = sqlite3_mprintf(\"CREATE TABLE \" VEC0_SHADOW_METADATA_TEXT_DATA_NAME \"(rowid PRIMARY KEY, data TEXT);\",\n pNew->schemaName, pNew->tableName, i);\n if (!zSql) {\n goto error;\n }\n rc = sqlite3_prepare_v2(db, zSql, -1, &stmt, 0);\n sqlite3_free((void *)zSql);\n if ((rc != SQLITE_OK) || (sqlite3_step(stmt) != SQLITE_DONE)) {\n sqlite3_finalize(stmt);\n *pzErr = sqlite3_mprintf(\n \"Could not create '_metadatatext%02d' shadow table: %s\", i,\n sqlite3_errmsg(db));\n goto error;\n }\n sqlite3_finalize(stmt);\n\n }\n }\n\n if(pNew->numAuxiliaryColumns > 0) {\n sqlite3_stmt * stmt;\n sqlite3_str * s = sqlite3_str_new(NULL);\n sqlite3_str_appendf(s, \"CREATE TABLE \" VEC0_SHADOW_AUXILIARY_NAME \"( rowid integer PRIMARY KEY \", pNew->schemaName, pNew->tableName);\n for(int i = 0; i < pNew->numAuxiliaryColumns; i++) {\n sqlite3_str_appendf(s, \", value%02d\", i);\n }\n sqlite3_str_appendall(s, \")\");\n char *zSql = sqlite3_str_finish(s);\n if(!zSql) {\n goto error;\n }\n rc = sqlite3_prepare_v2(db, zSql, -1, &stmt, NULL);\n if ((rc != SQLITE_OK) || (sqlite3_step(stmt) != SQLITE_DONE)) {\n sqlite3_finalize(stmt);\n *pzErr = sqlite3_mprintf(\n \"Could not create auxiliary shadow table: %s\",\n sqlite3_errmsg(db));\n\n goto error;\n }\n sqlite3_finalize(stmt);\n }\n }\n\n *ppVtab = (sqlite3_vtab *)pNew;\n return SQLITE_OK;\n\nerror:\n vec0_free(pNew);\n sqlite3_free(pNew);\n return SQLITE_ERROR;\n}\n\nstatic int vec0Create(sqlite3 *db, void *pAux, int argc,\n const char *const *argv, sqlite3_vtab **ppVtab,\n char **pzErr) {\n return vec0_init(db, pAux, argc, argv, ppVtab, pzErr, true);\n}\nstatic int vec0Connect(sqlite3 *db, void *pAux, int argc,\n const char *const *argv, sqlite3_vtab **ppVtab,\n char **pzErr) {\n return vec0_init(db, pAux, argc, argv, ppVtab, pzErr, false);\n}\n\nstatic int vec0Disconnect(sqlite3_vtab *pVtab) {\n vec0_vtab *p = (vec0_vtab *)pVtab;\n vec0_free(p);\n sqlite3_free(p);\n return SQLITE_OK;\n}\nstatic int vec0Destroy(sqlite3_vtab *pVtab) {\n vec0_vtab *p = (vec0_vtab *)pVtab;\n sqlite3_stmt *stmt;\n int rc;\n const char *zSql;\n\n // Free up any sqlite3_stmt, otherwise DROPs on those tables will fail\n vec0_free_resources(p);\n\n // TODO(test) later: can't evidence-of here, bc always gives \"SQL logic error\" instead of\n // provided error\n zSql = sqlite3_mprintf(\"DROP TABLE \" VEC0_SHADOW_CHUNKS_NAME, p->schemaName,\n p->tableName);\n rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, 0);\n sqlite3_free((void *)zSql);\n if ((rc != SQLITE_OK) || (sqlite3_step(stmt) != SQLITE_DONE)) {\n rc = SQLITE_ERROR;\n vtab_set_error(pVtab, \"could not drop chunks shadow table\");\n goto done;\n }\n sqlite3_finalize(stmt);\n\n zSql = sqlite3_mprintf(\"DROP TABLE \" VEC0_SHADOW_INFO_NAME, p->schemaName,\n p->tableName);\n rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, 0);\n sqlite3_free((void *)zSql);\n if ((rc != SQLITE_OK) || (sqlite3_step(stmt) != SQLITE_DONE)) {\n rc = SQLITE_ERROR;\n vtab_set_error(pVtab, \"could not drop info shadow table\");\n goto done;\n }\n sqlite3_finalize(stmt);\n\n zSql = sqlite3_mprintf(\"DROP TABLE \" VEC0_SHADOW_ROWIDS_NAME, p->schemaName,\n p->tableName);\n rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, 0);\n sqlite3_free((void *)zSql);\n if ((rc != SQLITE_OK) || (sqlite3_step(stmt) != SQLITE_DONE)) {\n rc = SQLITE_ERROR;\n goto done;\n }\n sqlite3_finalize(stmt);\n\n for (int i = 0; i < p->numVectorColumns; i++) {\n zSql = sqlite3_mprintf(\"DROP TABLE \\\"%w\\\".\\\"%w\\\"\", p->schemaName,\n p->shadowVectorChunksNames[i]);\n rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, 0);\n sqlite3_free((void *)zSql);\n if ((rc != SQLITE_OK) || (sqlite3_step(stmt) != SQLITE_DONE)) {\n rc = SQLITE_ERROR;\n goto done;\n }\n sqlite3_finalize(stmt);\n }\n\n if(p->numAuxiliaryColumns > 0) {\n zSql = sqlite3_mprintf(\"DROP TABLE \" VEC0_SHADOW_AUXILIARY_NAME, p->schemaName, p->tableName);\n rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, 0);\n sqlite3_free((void *)zSql);\n if ((rc != SQLITE_OK) || (sqlite3_step(stmt) != SQLITE_DONE)) {\n rc = SQLITE_ERROR;\n goto done;\n }\n sqlite3_finalize(stmt);\n }\n\n\n for (int i = 0; i < p->numMetadataColumns; i++) {\n zSql = sqlite3_mprintf(\"DROP TABLE \" VEC0_SHADOW_METADATA_N_NAME, p->schemaName,p->tableName, i);\n rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, 0);\n sqlite3_free((void *)zSql);\n if ((rc != SQLITE_OK) || (sqlite3_step(stmt) != SQLITE_DONE)) {\n rc = SQLITE_ERROR;\n goto done;\n }\n sqlite3_finalize(stmt);\n\n if(p->metadata_columns[i].kind == VEC0_METADATA_COLUMN_KIND_TEXT) {\n zSql = sqlite3_mprintf(\"DROP TABLE \" VEC0_SHADOW_METADATA_TEXT_DATA_NAME, p->schemaName,p->tableName, i);\n rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, 0);\n sqlite3_free((void *)zSql);\n if ((rc != SQLITE_OK) || (sqlite3_step(stmt) != SQLITE_DONE)) {\n rc = SQLITE_ERROR;\n goto done;\n }\n sqlite3_finalize(stmt);\n }\n }\n\n stmt = NULL;\n rc = SQLITE_OK;\n\ndone:\n sqlite3_finalize(stmt);\n vec0_free(p);\n // If there was an error\n if (rc == SQLITE_OK) {\n sqlite3_free(p);\n }\n return rc;\n}\n\nstatic int vec0Open(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor) {\n UNUSED_PARAMETER(p);\n vec0_cursor *pCur;\n pCur = sqlite3_malloc(sizeof(*pCur));\n if (pCur == 0)\n return SQLITE_NOMEM;\n memset(pCur, 0, sizeof(*pCur));\n *ppCursor = &pCur->base;\n return SQLITE_OK;\n}\n\nstatic int vec0Close(sqlite3_vtab_cursor *cur) {\n vec0_cursor *pCur = (vec0_cursor *)cur;\n vec0_cursor_clear(pCur);\n sqlite3_free(pCur);\n return SQLITE_OK;\n}\n\n// All the different type of \"values\" provided to argv/argc in vec0Filter.\n// These enums denote the use and purpose of all of them.\ntypedef enum {\n // If any values are updated, please update the ARCHITECTURE.md docs accordingly!\n\n // ~~~ KNN QUERIES ~~~ //\n VEC0_IDXSTR_KIND_KNN_MATCH = '{',\n VEC0_IDXSTR_KIND_KNN_K = '}',\n VEC0_IDXSTR_KIND_KNN_ROWID_IN = '[',\n // argv[i] is a constraint on a PARTITON KEY column in a KNN query\n // \n VEC0_IDXSTR_KIND_KNN_PARTITON_CONSTRAINT = ']',\n\n // argv[i] is a constraint on the distance column in a KNN query\n VEC0_IDXSTR_KIND_KNN_DISTANCE_CONSTRAINT = '*',\n\n // ~~~ POINT QUERIES ~~~ //\n VEC0_IDXSTR_KIND_POINT_ID = '!',\n\n // ~~~ ??? ~~~ //\n VEC0_IDXSTR_KIND_METADATA_CONSTRAINT = '&',\n} vec0_idxstr_kind;\n\n// The different SQLITE_INDEX_CONSTRAINT values that vec0 partition key columns\n// support, but as characters that fit nicely in idxstr.\ntypedef enum {\n // If any values are updated, please update the ARCHITECTURE.md docs accordingly!\n\n // Equality constraint on a PARTITON KEY column, ex `user_id = 123`\n VEC0_PARTITION_OPERATOR_EQ = 'a',\n \n // \"Greater than\" constraint on a PARTITON KEY column, ex `year > 2024`\n VEC0_PARTITION_OPERATOR_GT = 'b',\n \n // \"Less than or equal to\" constraint on a PARTITON KEY column, ex `year <= 2024`\n VEC0_PARTITION_OPERATOR_LE = 'c',\n\n // \"Less than\" constraint on a PARTITON KEY column, ex `year < 2024`\n VEC0_PARTITION_OPERATOR_LT = 'd',\n \n // \"Greater than or equal to\" constraint on a PARTITON KEY column, ex `year >= 2024`\n VEC0_PARTITION_OPERATOR_GE = 'e',\n \n // \"Not equal to\" constraint on a PARTITON KEY column, ex `year != 2024`\n VEC0_PARTITION_OPERATOR_NE = 'f',\n} vec0_partition_operator;\ntypedef enum {\n VEC0_METADATA_OPERATOR_EQ = 'a',\n VEC0_METADATA_OPERATOR_GT = 'b',\n VEC0_METADATA_OPERATOR_LE = 'c',\n VEC0_METADATA_OPERATOR_LT = 'd',\n VEC0_METADATA_OPERATOR_GE = 'e',\n VEC0_METADATA_OPERATOR_NE = 'f',\n VEC0_METADATA_OPERATOR_IN = 'g',\n} vec0_metadata_operator;\n\n\ntypedef enum {\n\n VEC0_DISTANCE_CONSTRAINT_GT = 'a',\n VEC0_DISTANCE_CONSTRAINT_GE = 'b',\n VEC0_DISTANCE_CONSTRAINT_LT = 'c',\n VEC0_DISTANCE_CONSTRAINT_LE = 'd',\n} vec0_distance_constraint_operator;\n\nstatic int vec0BestIndex(sqlite3_vtab *pVTab, sqlite3_index_info *pIdxInfo) {\n vec0_vtab *p = (vec0_vtab *)pVTab;\n /**\n * Possible query plans are:\n * 1. KNN when:\n * a) An `MATCH` op on vector column\n * b) ORDER BY on distance column\n * c) LIMIT\n * d) rowid in (...) OPTIONAL\n * 2. Point when:\n * a) An `EQ` op on rowid column\n * 3. else: fullscan\n *\n */\n int iMatchTerm = -1;\n int iMatchVectorTerm = -1;\n int iLimitTerm = -1;\n int iRowidTerm = -1;\n int iKTerm = -1;\n int iRowidInTerm = -1;\n int hasAuxConstraint = 0;\n\n#ifdef SQLITE_VEC_DEBUG\n printf(\"pIdxInfo->nOrderBy=%d, pIdxInfo->nConstraint=%d\\n\", pIdxInfo->nOrderBy, pIdxInfo->nConstraint);\n#endif\n\n for (int i = 0; i < pIdxInfo->nConstraint; i++) {\n u8 vtabIn = 0;\n\n#if COMPILER_SUPPORTS_VTAB_IN\n if (sqlite3_libversion_number() >= 3038000) {\n vtabIn = sqlite3_vtab_in(pIdxInfo, i, -1);\n }\n#endif\n\n#ifdef SQLITE_VEC_DEBUG\n printf(\"xBestIndex [%d] usable=%d iColumn=%d op=%d vtabin=%d\\n\", i,\n pIdxInfo->aConstraint[i].usable, pIdxInfo->aConstraint[i].iColumn,\n pIdxInfo->aConstraint[i].op, vtabIn);\n#endif\n if (!pIdxInfo->aConstraint[i].usable)\n continue;\n\n int iColumn = pIdxInfo->aConstraint[i].iColumn;\n int op = pIdxInfo->aConstraint[i].op;\n\n if (op == SQLITE_INDEX_CONSTRAINT_LIMIT) {\n iLimitTerm = i;\n }\n if (op == SQLITE_INDEX_CONSTRAINT_MATCH &&\n vec0_column_idx_is_vector(p, iColumn)) {\n if (iMatchTerm > -1) {\n vtab_set_error(\n pVTab, \"only 1 MATCH operator is allowed in a single vec0 query\");\n return SQLITE_ERROR;\n }\n iMatchTerm = i;\n iMatchVectorTerm = vec0_column_idx_to_vector_idx(p, iColumn);\n }\n if (op == SQLITE_INDEX_CONSTRAINT_EQ && iColumn == VEC0_COLUMN_ID) {\n if (vtabIn) {\n if (iRowidInTerm != -1) {\n vtab_set_error(pVTab, \"only 1 'rowid in (..)' operator is allowed in \"\n \"a single vec0 query\");\n return SQLITE_ERROR;\n }\n iRowidInTerm = i;\n\n } else {\n iRowidTerm = i;\n }\n }\n if (op == SQLITE_INDEX_CONSTRAINT_EQ && iColumn == vec0_column_k_idx(p)) {\n iKTerm = i;\n }\n if(\n (op != SQLITE_INDEX_CONSTRAINT_LIMIT && op != SQLITE_INDEX_CONSTRAINT_OFFSET)\n && vec0_column_idx_is_auxiliary(p, iColumn)) {\n hasAuxConstraint = 1;\n }\n }\n\n sqlite3_str *idxStr = sqlite3_str_new(NULL);\n int rc;\n\n if (iMatchTerm >= 0) {\n if (iLimitTerm < 0 && iKTerm < 0) {\n vtab_set_error(\n pVTab,\n \"A LIMIT or 'k = ?' constraint is required on vec0 knn queries.\");\n rc = SQLITE_ERROR;\n goto done;\n }\n if (iLimitTerm >= 0 && iKTerm >= 0) {\n vtab_set_error(pVTab, \"Only LIMIT or 'k =?' can be provided, not both\");\n rc = SQLITE_ERROR;\n goto done;\n }\n\n if (pIdxInfo->nOrderBy) {\n if (pIdxInfo->nOrderBy > 1) {\n vtab_set_error(pVTab, \"Only a single 'ORDER BY distance' clause is \"\n \"allowed on vec0 KNN queries\");\n rc = SQLITE_ERROR;\n goto done;\n }\n if (pIdxInfo->aOrderBy[0].iColumn != vec0_column_distance_idx(p)) {\n vtab_set_error(pVTab,\n \"Only a single 'ORDER BY distance' clause is allowed on \"\n \"vec0 KNN queries, not on other columns\");\n rc = SQLITE_ERROR;\n goto done;\n }\n if (pIdxInfo->aOrderBy[0].desc) {\n vtab_set_error(\n pVTab, \"Only ascending in ORDER BY distance clause is supported, \"\n \"DESC is not supported yet.\");\n rc = SQLITE_ERROR;\n goto done;\n }\n }\n\n if(hasAuxConstraint) {\n // IMP: V25623_09693\n vtab_set_error(pVTab, \"An illegal WHERE constraint was provided on a vec0 auxiliary column in a KNN query.\");\n rc = SQLITE_ERROR;\n goto done;\n }\n\n sqlite3_str_appendchar(idxStr, 1, VEC0_QUERY_PLAN_KNN);\n\n int argvIndex = 1;\n pIdxInfo->aConstraintUsage[iMatchTerm].argvIndex = argvIndex++;\n pIdxInfo->aConstraintUsage[iMatchTerm].omit = 1;\n sqlite3_str_appendchar(idxStr, 1, VEC0_IDXSTR_KIND_KNN_MATCH);\n sqlite3_str_appendchar(idxStr, 3, '_');\n\n if (iLimitTerm >= 0) {\n pIdxInfo->aConstraintUsage[iLimitTerm].argvIndex = argvIndex++;\n pIdxInfo->aConstraintUsage[iLimitTerm].omit = 1;\n } else {\n pIdxInfo->aConstraintUsage[iKTerm].argvIndex = argvIndex++;\n pIdxInfo->aConstraintUsage[iKTerm].omit = 1;\n }\n sqlite3_str_appendchar(idxStr, 1, VEC0_IDXSTR_KIND_KNN_K);\n sqlite3_str_appendchar(idxStr, 3, '_');\n\n#if COMPILER_SUPPORTS_VTAB_IN\n if (iRowidInTerm >= 0) {\n // already validated as >= SQLite 3.38 bc iRowidInTerm is only >= 0 when\n // vtabIn == 1\n sqlite3_vtab_in(pIdxInfo, iRowidInTerm, 1);\n pIdxInfo->aConstraintUsage[iRowidInTerm].argvIndex = argvIndex++;\n pIdxInfo->aConstraintUsage[iRowidInTerm].omit = 1;\n sqlite3_str_appendchar(idxStr, 1, VEC0_IDXSTR_KIND_KNN_ROWID_IN);\n sqlite3_str_appendchar(idxStr, 3, '_');\n }\n#endif\n\n // find any PARTITION KEY column constraints\n for (int i = 0; i < pIdxInfo->nConstraint; i++) {\n if (!pIdxInfo->aConstraint[i].usable)\n continue;\n\n int iColumn = pIdxInfo->aConstraint[i].iColumn;\n int op = pIdxInfo->aConstraint[i].op;\n if(op == SQLITE_INDEX_CONSTRAINT_LIMIT || op == SQLITE_INDEX_CONSTRAINT_OFFSET) {\n continue;\n }\n if(!vec0_column_idx_is_partition(p, iColumn)) {\n continue;\n }\n\n int partition_idx = vec0_column_idx_to_partition_idx(p, iColumn);\n char value = 0;\n\n switch(op) {\n case SQLITE_INDEX_CONSTRAINT_EQ: {\n value = VEC0_PARTITION_OPERATOR_EQ;\n break;\n }\n case SQLITE_INDEX_CONSTRAINT_GT: {\n value = VEC0_PARTITION_OPERATOR_GT;\n break;\n }\n case SQLITE_INDEX_CONSTRAINT_LE: {\n value = VEC0_PARTITION_OPERATOR_LE;\n break;\n }\n case SQLITE_INDEX_CONSTRAINT_LT: {\n value = VEC0_PARTITION_OPERATOR_LT;\n break;\n }\n case SQLITE_INDEX_CONSTRAINT_GE: {\n value = VEC0_PARTITION_OPERATOR_GE;\n break;\n }\n case SQLITE_INDEX_CONSTRAINT_NE: {\n value = VEC0_PARTITION_OPERATOR_NE;\n break;\n }\n }\n\n if(value) {\n pIdxInfo->aConstraintUsage[i].argvIndex = argvIndex++;\n pIdxInfo->aConstraintUsage[i].omit = 1;\n sqlite3_str_appendchar(idxStr, 1, VEC0_IDXSTR_KIND_KNN_PARTITON_CONSTRAINT);\n sqlite3_str_appendchar(idxStr, 1, 'A' + partition_idx);\n sqlite3_str_appendchar(idxStr, 1, value);\n sqlite3_str_appendchar(idxStr, 1, '_');\n }\n\n }\n\n // find any metadata column constraints\n for (int i = 0; i < pIdxInfo->nConstraint; i++) {\n if (!pIdxInfo->aConstraint[i].usable)\n continue;\n\n int iColumn = pIdxInfo->aConstraint[i].iColumn;\n int op = pIdxInfo->aConstraint[i].op;\n if(op == SQLITE_INDEX_CONSTRAINT_LIMIT || op == SQLITE_INDEX_CONSTRAINT_OFFSET) {\n continue;\n }\n if(!vec0_column_idx_is_metadata(p, iColumn)) {\n continue;\n }\n\n int metadata_idx = vec0_column_idx_to_metadata_idx(p, iColumn);\n char value = 0;\n\n switch(op) {\n case SQLITE_INDEX_CONSTRAINT_EQ: {\n int vtabIn = 0;\n #if COMPILER_SUPPORTS_VTAB_IN\n if (sqlite3_libversion_number() >= 3038000) {\n vtabIn = sqlite3_vtab_in(pIdxInfo, i, -1);\n }\n if(vtabIn) {\n switch(p->metadata_columns[metadata_idx].kind) {\n case VEC0_METADATA_COLUMN_KIND_FLOAT:\n case VEC0_METADATA_COLUMN_KIND_BOOLEAN: {\n // IMP: V15248_32086\n rc = SQLITE_ERROR;\n vtab_set_error(pVTab, \"'xxx in (...)' is only available on INTEGER or TEXT metadata columns.\");\n goto done;\n break;\n }\n case VEC0_METADATA_COLUMN_KIND_INTEGER:\n case VEC0_METADATA_COLUMN_KIND_TEXT: {\n break;\n }\n }\n value = VEC0_METADATA_OPERATOR_IN;\n sqlite3_vtab_in(pIdxInfo, i, 1);\n }else\n #endif\n {\n value = VEC0_PARTITION_OPERATOR_EQ;\n }\n break;\n }\n case SQLITE_INDEX_CONSTRAINT_GT: {\n value = VEC0_METADATA_OPERATOR_GT;\n break;\n }\n case SQLITE_INDEX_CONSTRAINT_LE: {\n value = VEC0_METADATA_OPERATOR_LE;\n break;\n }\n case SQLITE_INDEX_CONSTRAINT_LT: {\n value = VEC0_METADATA_OPERATOR_LT;\n break;\n }\n case SQLITE_INDEX_CONSTRAINT_GE: {\n value = VEC0_METADATA_OPERATOR_GE;\n break;\n }\n case SQLITE_INDEX_CONSTRAINT_NE: {\n value = VEC0_METADATA_OPERATOR_NE;\n break;\n }\n default: {\n // IMP: V16511_00582\n rc = SQLITE_ERROR;\n vtab_set_error(pVTab,\n \"An illegal WHERE constraint was provided on a vec0 metadata column in a KNN query. \"\n \"Only one of EQUALS, GREATER_THAN, LESS_THAN_OR_EQUAL, LESS_THAN, GREATER_THAN_OR_EQUAL, NOT_EQUALS is allowed.\"\n );\n goto done;\n }\n }\n\n if(p->metadata_columns[metadata_idx].kind == VEC0_METADATA_COLUMN_KIND_BOOLEAN) {\n if(!(value == VEC0_METADATA_OPERATOR_EQ || value == VEC0_METADATA_OPERATOR_NE)) {\n // IMP: V10145_26984\n rc = SQLITE_ERROR;\n vtab_set_error(pVTab, \"ONLY EQUALS (=) or NOT_EQUALS (!=) operators are allowed on boolean metadata columns.\");\n goto done;\n }\n }\n\n pIdxInfo->aConstraintUsage[i].argvIndex = argvIndex++;\n pIdxInfo->aConstraintUsage[i].omit = 1;\n sqlite3_str_appendchar(idxStr, 1, VEC0_IDXSTR_KIND_METADATA_CONSTRAINT);\n sqlite3_str_appendchar(idxStr, 1, 'A' + metadata_idx);\n sqlite3_str_appendchar(idxStr, 1, value);\n sqlite3_str_appendchar(idxStr, 1, '_');\n\n }\n\n // find any distance column constraints\n for (int i = 0; i < pIdxInfo->nConstraint; i++) {\n if (!pIdxInfo->aConstraint[i].usable)\n continue;\n\n int iColumn = pIdxInfo->aConstraint[i].iColumn;\n int op = pIdxInfo->aConstraint[i].op;\n if(op == SQLITE_INDEX_CONSTRAINT_LIMIT || op == SQLITE_INDEX_CONSTRAINT_OFFSET) {\n continue;\n }\n if(vec0_column_distance_idx(p) != iColumn) {\n continue;\n }\n\n char value = 0;\n switch(op) {\n case SQLITE_INDEX_CONSTRAINT_GT: {\n value = VEC0_DISTANCE_CONSTRAINT_GT;\n break;\n }\n case SQLITE_INDEX_CONSTRAINT_GE: {\n value = VEC0_DISTANCE_CONSTRAINT_GE;\n break;\n }\n case SQLITE_INDEX_CONSTRAINT_LT: {\n value = VEC0_DISTANCE_CONSTRAINT_LT;\n break;\n }\n case SQLITE_INDEX_CONSTRAINT_LE: {\n value = VEC0_DISTANCE_CONSTRAINT_LE;\n break;\n }\n default: {\n // IMP TODO\n rc = SQLITE_ERROR;\n vtab_set_error(\n pVTab, \n \"Illegal WHERE constraint on distance column in a KNN query. \"\n \"Only one of GT, GE, LT, LE constraints are allowed.\"\n );\n goto done;\n }\n }\n\n pIdxInfo->aConstraintUsage[i].argvIndex = argvIndex++;\n pIdxInfo->aConstraintUsage[i].omit = 1;\n sqlite3_str_appendchar(idxStr, 1, VEC0_IDXSTR_KIND_KNN_DISTANCE_CONSTRAINT);\n sqlite3_str_appendchar(idxStr, 1, value);\n sqlite3_str_appendchar(idxStr, 1, '_');\n sqlite3_str_appendchar(idxStr, 1, '_');\n }\n\n\n\n pIdxInfo->idxNum = iMatchVectorTerm;\n pIdxInfo->estimatedCost = 30.0;\n pIdxInfo->estimatedRows = 10;\n\n } else if (iRowidTerm >= 0) {\n sqlite3_str_appendchar(idxStr, 1, VEC0_QUERY_PLAN_POINT);\n pIdxInfo->aConstraintUsage[iRowidTerm].argvIndex = 1;\n pIdxInfo->aConstraintUsage[iRowidTerm].omit = 1;\n sqlite3_str_appendchar(idxStr, 1, VEC0_IDXSTR_KIND_POINT_ID);\n sqlite3_str_appendchar(idxStr, 3, '_');\n pIdxInfo->idxNum = pIdxInfo->colUsed;\n pIdxInfo->estimatedCost = 10.0;\n pIdxInfo->estimatedRows = 1;\n } else {\n sqlite3_str_appendchar(idxStr, 1, VEC0_QUERY_PLAN_FULLSCAN);\n pIdxInfo->estimatedCost = 3000000.0;\n pIdxInfo->estimatedRows = 100000;\n }\n pIdxInfo->idxStr = sqlite3_str_finish(idxStr);\n idxStr = NULL;\n if (!pIdxInfo->idxStr) {\n rc = SQLITE_OK;\n goto done;\n }\n pIdxInfo->needToFreeIdxStr = 1;\n\n rc = SQLITE_OK;\n\n done:\n if(idxStr) {\n sqlite3_str_finish(idxStr);\n }\n return rc;\n}\n\n// forward delcaration bc vec0Filter uses it\nstatic int vec0Next(sqlite3_vtab_cursor *cur);\n\nvoid merge_sorted_lists(f32 *a, i64 *a_rowids, i64 a_length, f32 *b,\n i64 *b_rowids, i32 *b_top_idxs, i64 b_length, f32 *out,\n i64 *out_rowids, i64 out_length, i64 *out_used) {\n // assert((a_length >= out_length) || (b_length >= out_length));\n i64 ptrA = 0;\n i64 ptrB = 0;\n for (int i = 0; i < out_length; i++) {\n if ((ptrA >= a_length) && (ptrB >= b_length)) {\n *out_used = i;\n return;\n }\n if (ptrA >= a_length) {\n out[i] = b[b_top_idxs[ptrB]];\n out_rowids[i] = b_rowids[b_top_idxs[ptrB]];\n ptrB++;\n } else if (ptrB >= b_length) {\n out[i] = a[ptrA];\n out_rowids[i] = a_rowids[ptrA];\n ptrA++;\n } else {\n if (a[ptrA] <= b[b_top_idxs[ptrB]]) {\n out[i] = a[ptrA];\n out_rowids[i] = a_rowids[ptrA];\n ptrA++;\n } else {\n out[i] = b[b_top_idxs[ptrB]];\n out_rowids[i] = b_rowids[b_top_idxs[ptrB]];\n ptrB++;\n }\n }\n }\n\n *out_used = out_length;\n}\n\nu8 *bitmap_new(i32 n) {\n assert(n % 8 == 0);\n u8 *p = sqlite3_malloc(n * sizeof(u8) / CHAR_BIT);\n if (p) {\n memset(p, 0, n * sizeof(u8) / CHAR_BIT);\n }\n return p;\n}\nu8 *bitmap_new_from(i32 n, u8 *from) {\n assert(n % 8 == 0);\n u8 *p = sqlite3_malloc(n * sizeof(u8) / CHAR_BIT);\n if (p) {\n memcpy(p, from, n / CHAR_BIT);\n }\n return p;\n}\n\nvoid bitmap_copy(u8 *base, u8 *from, i32 n) {\n assert(n % 8 == 0);\n memcpy(base, from, n / CHAR_BIT);\n}\n\nvoid bitmap_and_inplace(u8 *base, u8 *other, i32 n) {\n assert((n % 8) == 0);\n for (int i = 0; i < n / CHAR_BIT; i++) {\n base[i] = base[i] & other[i];\n }\n}\n\nvoid bitmap_set(u8 *bitmap, i32 position, int value) {\n if (value) {\n bitmap[position / CHAR_BIT] |= 1 << (position % CHAR_BIT);\n } else {\n bitmap[position / CHAR_BIT] &= ~(1 << (position % CHAR_BIT));\n }\n}\n\nint bitmap_get(u8 *bitmap, i32 position) {\n return (((bitmap[position / CHAR_BIT]) >> (position % CHAR_BIT)) & 1);\n}\n\nvoid bitmap_clear(u8 *bitmap, i32 n) {\n assert((n % 8) == 0);\n memset(bitmap, 0, n / CHAR_BIT);\n}\n\nvoid bitmap_fill(u8 *bitmap, i32 n) {\n assert((n % 8) == 0);\n memset(bitmap, 0xFF, n / CHAR_BIT);\n}\n\n/**\n * @brief Finds the minimum k items in distances, and writes the indicies to\n * out.\n *\n * @param distances input f32 array of size n, the items to consider.\n * @param n: size of distances array.\n * @param out: Output array of size k, will contain at most k element indicies\n * @param k: Size of output array\n * @return int\n */\nint min_idx(const f32 *distances, i32 n, u8 *candidates, i32 *out, i32 k,\n u8 *bTaken, i32 *k_used) {\n assert(k > 0);\n assert(k <= n);\n\n bitmap_clear(bTaken, n);\n\n for (int ik = 0; ik < k; ik++) {\n int min_idx = 0;\n while (min_idx < n &&\n (bitmap_get(bTaken, min_idx) || !bitmap_get(candidates, min_idx))) {\n min_idx++;\n }\n if (min_idx >= n) {\n *k_used = ik;\n return SQLITE_OK;\n }\n\n for (int i = 0; i < n; i++) {\n if (distances[i] <= distances[min_idx] && !bitmap_get(bTaken, i) &&\n (bitmap_get(candidates, i))) {\n min_idx = i;\n }\n }\n\n out[ik] = min_idx;\n bitmap_set(bTaken, min_idx, 1);\n }\n *k_used = k;\n return SQLITE_OK;\n}\n\nint vec0_get_metadata_text_long_value(\n vec0_vtab * p,\n sqlite3_stmt ** stmt,\n int metadata_idx,\n i64 rowid,\n int *n,\n char ** s) {\n int rc;\n if(!(*stmt)) {\n const char * zSql = sqlite3_mprintf(\"select data from \" VEC0_SHADOW_METADATA_TEXT_DATA_NAME \" where rowid = ?\", p->schemaName, p->tableName, metadata_idx);\n if(!zSql) {\n rc = SQLITE_NOMEM;\n goto done;\n }\n rc = sqlite3_prepare_v2(p->db, zSql, -1, stmt, NULL);\n sqlite3_free( (void *) zSql);\n if(rc != SQLITE_OK) {\n goto done;\n }\n }\n\n sqlite3_reset(*stmt);\n sqlite3_bind_int64(*stmt, 1, rowid);\n rc = sqlite3_step(*stmt);\n if(rc != SQLITE_ROW) {\n rc = SQLITE_ERROR;\n goto done;\n }\n *s = (char *) sqlite3_column_text(*stmt, 0);\n *n = sqlite3_column_bytes(*stmt, 0);\n rc = SQLITE_OK;\n done:\n return rc;\n}\n\n/**\n * @brief Crete at \"iterator\" (sqlite3_stmt) of chunks with the given constraints\n *\n * Any VEC0_IDXSTR_KIND_KNN_PARTITON_CONSTRAINT values in idxStr/argv will be applied\n * as WHERE constraints in the underlying stmt SQL, and any consumer of the stmt\n * can freely step through the stmt with all constraints satisfied.\n *\n * @param p - vec0_vtab\n * @param idxStr - the xBestIndex/xFilter idxstr containing VEC0_IDXSTR values\n * @param argc - number of argv values from xFilter\n * @param argv - array of sqlite3_value from xFilter\n * @param outStmt - output sqlite3_stmt of chunks with all filters applied\n * @return int SQLITE_OK on success, error code otherwise\n */\nint vec0_chunks_iter(vec0_vtab * p, const char * idxStr, int argc, sqlite3_value ** argv, sqlite3_stmt** outStmt) {\n // always null terminated, enforced by SQLite\n int idxStrLength = strlen(idxStr);\n // \"1\" refers to the initial vec0_query_plan char, 4 is the number of chars per \"element\"\n int numValueEntries = (idxStrLength-1) / 4;\n assert(argc == numValueEntries);\n\n int rc;\n sqlite3_str * s = sqlite3_str_new(NULL);\n sqlite3_str_appendf(s, \"select chunk_id, validity, rowids \"\n \" from \" VEC0_SHADOW_CHUNKS_NAME,\n p->schemaName, p->tableName);\n\n int appendedWhere = 0;\n for(int i = 0; i < numValueEntries; i++) {\n int idx = 1 + (i * 4);\n char kind = idxStr[idx + 0];\n if(kind != VEC0_IDXSTR_KIND_KNN_PARTITON_CONSTRAINT) {\n continue;\n }\n\n int partition_idx = idxStr[idx + 1] - 'A';\n int operator = idxStr[idx + 2];\n // idxStr[idx + 3] is just null, a '_' placeholder\n\n if(!appendedWhere) {\n sqlite3_str_appendall(s, \" WHERE \");\n appendedWhere = 1;\n }else {\n sqlite3_str_appendall(s, \" AND \");\n }\n switch(operator) {\n case VEC0_PARTITION_OPERATOR_EQ:\n sqlite3_str_appendf(s, \" partition%02d = ? \", partition_idx);\n break;\n case VEC0_PARTITION_OPERATOR_GT:\n sqlite3_str_appendf(s, \" partition%02d > ? \", partition_idx);\n break;\n case VEC0_PARTITION_OPERATOR_LE:\n sqlite3_str_appendf(s, \" partition%02d <= ? \", partition_idx);\n break;\n case VEC0_PARTITION_OPERATOR_LT:\n sqlite3_str_appendf(s, \" partition%02d < ? \", partition_idx);\n break;\n case VEC0_PARTITION_OPERATOR_GE:\n sqlite3_str_appendf(s, \" partition%02d >= ? \", partition_idx);\n break;\n case VEC0_PARTITION_OPERATOR_NE:\n sqlite3_str_appendf(s, \" partition%02d != ? \", partition_idx);\n break;\n default: {\n char * zSql = sqlite3_str_finish(s);\n sqlite3_free(zSql);\n return SQLITE_ERROR;\n }\n\n }\n\n }\n\n char *zSql = sqlite3_str_finish(s);\n if (!zSql) {\n return SQLITE_NOMEM;\n }\n\n rc = sqlite3_prepare_v2(p->db, zSql, -1, outStmt, NULL);\n sqlite3_free(zSql);\n if(rc != SQLITE_OK) {\n return rc;\n }\n\n int n = 1;\n for(int i = 0; i < numValueEntries; i++) {\n int idx = 1 + (i * 4);\n char kind = idxStr[idx + 0];\n if(kind != VEC0_IDXSTR_KIND_KNN_PARTITON_CONSTRAINT) {\n continue;\n }\n sqlite3_bind_value(*outStmt, n++, argv[i]);\n }\n\n return rc;\n}\n\n// a single `xxx in (...)` constraint on a metadata column. TEXT or INTEGER only for now.\nstruct Vec0MetadataIn{\n // index of argv[i]` the constraint is on\n int argv_idx;\n // metadata column index of the constraint, derived from idxStr + argv_idx\n int metadata_idx;\n // array of the copied `(...)` values from sqlite3_vtab_in_first()/sqlite3_vtab_in_next()\n struct Array array;\n};\n\n// Array elements for `xxx in (...)` values for a text column. basically just a string\nstruct Vec0MetadataInTextEntry {\n int n;\n char * zString;\n};\n\n\nint vec0_metadata_filter_text(vec0_vtab * p, sqlite3_value * value, const void * buffer, int size, vec0_metadata_operator op, u8* b, int metadata_idx, int chunk_rowid, struct Array * aMetadataIn, int argv_idx) {\n int rc;\n sqlite3_stmt * stmt = NULL;\n i64 * rowids = NULL;\n sqlite3_blob * rowidsBlob;\n const char * sTarget = (const char *) sqlite3_value_text(value);\n int nTarget = sqlite3_value_bytes(value);\n\n\n // TODO(perf): only text metadata news the rowids BLOB. Make it so that\n // rowids BLOB is re-used when multiple fitlers on text columns,\n // ex \"name BETWEEN 'a' and 'b'\"\"\n rc = sqlite3_blob_open(p->db, p->schemaName, p->shadowChunksName, \"rowids\", chunk_rowid, 0, &rowidsBlob);\n if(rc != SQLITE_OK) {\n return rc;\n }\n assert(sqlite3_blob_bytes(rowidsBlob) % sizeof(i64) == 0);\n assert((sqlite3_blob_bytes(rowidsBlob) / sizeof(i64)) == size);\n\n rowids = sqlite3_malloc(sqlite3_blob_bytes(rowidsBlob));\n if(!rowids) {\n sqlite3_blob_close(rowidsBlob);\n return SQLITE_NOMEM;\n }\n\n rc = sqlite3_blob_read(rowidsBlob, rowids, sqlite3_blob_bytes(rowidsBlob), 0);\n if(rc != SQLITE_OK) {\n sqlite3_blob_close(rowidsBlob);\n return rc;\n }\n sqlite3_blob_close(rowidsBlob);\n\n switch(op) {\n int nPrefix;\n char * sPrefix;\n char *sFull;\n int nFull;\n u8 * view;\n case VEC0_METADATA_OPERATOR_EQ: {\n for(int i = 0; i < size; i++) {\n view = &((u8*) buffer)[i * VEC0_METADATA_TEXT_VIEW_BUFFER_LENGTH];\n nPrefix = ((int*) view)[0];\n sPrefix = (char *) &view[4];\n\n // for EQ the text lengths must match\n if(nPrefix != nTarget) {\n bitmap_set(b, i, 0);\n continue;\n }\n int cmpPrefix = strncmp(sPrefix, sTarget, min(nPrefix, VEC0_METADATA_TEXT_VIEW_DATA_LENGTH));\n\n // for short strings, use the prefix comparison direclty\n if(nPrefix <= VEC0_METADATA_TEXT_VIEW_DATA_LENGTH) {\n bitmap_set(b, i, cmpPrefix == 0);\n continue;\n }\n // for EQ on longs strings, the prefix must match\n if(cmpPrefix) {\n bitmap_set(b, i, 0);\n continue;\n }\n // consult the full string\n rc = vec0_get_metadata_text_long_value(p, &stmt, metadata_idx, rowids[i], &nFull, &sFull);\n if(rc != SQLITE_OK) {\n goto done;\n }\n if(nPrefix != nFull) {\n rc = SQLITE_ERROR;\n goto done;\n }\n bitmap_set(b, i, strncmp(sFull, sTarget, nFull) == 0);\n }\n break;\n }\n case VEC0_METADATA_OPERATOR_NE: {\n for(int i = 0; i < size; i++) {\n view = &((u8*) buffer)[i * VEC0_METADATA_TEXT_VIEW_BUFFER_LENGTH];\n nPrefix = ((int*) view)[0];\n sPrefix = (char *) &view[4];\n\n // for NE if text lengths dont match, it never will\n if(nPrefix != nTarget) {\n bitmap_set(b, i, 1);\n continue;\n }\n\n int cmpPrefix = strncmp(sPrefix, sTarget, min(nPrefix, VEC0_METADATA_TEXT_VIEW_DATA_LENGTH));\n\n // for short strings, use the prefix comparison direclty\n if(nPrefix <= VEC0_METADATA_TEXT_VIEW_DATA_LENGTH) {\n bitmap_set(b, i, cmpPrefix != 0);\n continue;\n }\n // for NE on longs strings, if prefixes dont match, then long string wont\n if(cmpPrefix) {\n bitmap_set(b, i, 1);\n continue;\n }\n // consult the full string\n rc = vec0_get_metadata_text_long_value(p, &stmt, metadata_idx, rowids[i], &nFull, &sFull);\n if(rc != SQLITE_OK) {\n goto done;\n }\n if(nPrefix != nFull) {\n rc = SQLITE_ERROR;\n goto done;\n }\n bitmap_set(b, i, strncmp(sFull, sTarget, nFull) != 0);\n }\n break;\n }\n case VEC0_METADATA_OPERATOR_GT: {\n for(int i = 0; i < size; i++) {\n view = &((u8*) buffer)[i * VEC0_METADATA_TEXT_VIEW_BUFFER_LENGTH];\n nPrefix = ((int*) view)[0];\n sPrefix = (char *) &view[4];\n int cmpPrefix = strncmp(sPrefix, sTarget, min(min(nPrefix, VEC0_METADATA_TEXT_VIEW_DATA_LENGTH), nTarget));\n\n if(nPrefix < VEC0_METADATA_TEXT_VIEW_DATA_LENGTH) {\n // if prefix match, check which is longer\n if(cmpPrefix == 0) {\n bitmap_set(b, i, nPrefix > nTarget);\n }\n else {\n bitmap_set(b, i, cmpPrefix > 0);\n }\n continue;\n }\n // TODO(perf): may not need to compare full text in some cases\n\n rc = vec0_get_metadata_text_long_value(p, &stmt, metadata_idx, rowids[i], &nFull, &sFull);\n if(rc != SQLITE_OK) {\n goto done;\n }\n if(nPrefix != nFull) {\n rc = SQLITE_ERROR;\n goto done;\n }\n bitmap_set(b, i, strncmp(sFull, sTarget, nFull) > 0);\n }\n break;\n }\n case VEC0_METADATA_OPERATOR_GE: {\n for(int i = 0; i < size; i++) {\n view = &((u8*) buffer)[i * VEC0_METADATA_TEXT_VIEW_BUFFER_LENGTH];\n nPrefix = ((int*) view)[0];\n sPrefix = (char *) &view[4];\n int cmpPrefix = strncmp(sPrefix, sTarget, min(min(nPrefix, VEC0_METADATA_TEXT_VIEW_DATA_LENGTH), nTarget));\n\n if(nPrefix < VEC0_METADATA_TEXT_VIEW_DATA_LENGTH) {\n // if prefix match, check which is longer\n if(cmpPrefix == 0) {\n bitmap_set(b, i, nPrefix >= nTarget);\n }\n else {\n bitmap_set(b, i, cmpPrefix >= 0);\n }\n continue;\n }\n // TODO(perf): may not need to compare full text in some cases\n\n rc = vec0_get_metadata_text_long_value(p, &stmt, metadata_idx, rowids[i], &nFull, &sFull);\n if(rc != SQLITE_OK) {\n goto done;\n }\n if(nPrefix != nFull) {\n rc = SQLITE_ERROR;\n goto done;\n }\n bitmap_set(b, i, strncmp(sFull, sTarget, nFull) >= 0);\n }\n break;\n }\n case VEC0_METADATA_OPERATOR_LE: {\n for(int i = 0; i < size; i++) {\n view = &((u8*) buffer)[i * VEC0_METADATA_TEXT_VIEW_BUFFER_LENGTH];\n nPrefix = ((int*) view)[0];\n sPrefix = (char *) &view[4];\n int cmpPrefix = strncmp(sPrefix, sTarget, min(min(nPrefix, VEC0_METADATA_TEXT_VIEW_DATA_LENGTH), nTarget));\n\n if(nPrefix < VEC0_METADATA_TEXT_VIEW_DATA_LENGTH) {\n // if prefix match, check which is longer\n if(cmpPrefix == 0) {\n bitmap_set(b, i, nPrefix <= nTarget);\n }\n else {\n bitmap_set(b, i, cmpPrefix <= 0);\n }\n continue;\n }\n // TODO(perf): may not need to compare full text in some cases\n\n rc = vec0_get_metadata_text_long_value(p, &stmt, metadata_idx, rowids[i], &nFull, &sFull);\n if(rc != SQLITE_OK) {\n goto done;\n }\n if(nPrefix != nFull) {\n rc = SQLITE_ERROR;\n goto done;\n }\n bitmap_set(b, i, strncmp(sFull, sTarget, nFull) <= 0);\n }\n break;\n }\n case VEC0_METADATA_OPERATOR_LT: {\n for(int i = 0; i < size; i++) {\n view = &((u8*) buffer)[i * VEC0_METADATA_TEXT_VIEW_BUFFER_LENGTH];\n nPrefix = ((int*) view)[0];\n sPrefix = (char *) &view[4];\n int cmpPrefix = strncmp(sPrefix, sTarget, min(min(nPrefix, VEC0_METADATA_TEXT_VIEW_DATA_LENGTH), nTarget));\n\n if(nPrefix < VEC0_METADATA_TEXT_VIEW_DATA_LENGTH) {\n // if prefix match, check which is longer\n if(cmpPrefix == 0) {\n bitmap_set(b, i, nPrefix < nTarget);\n }\n else {\n bitmap_set(b, i, cmpPrefix < 0);\n }\n continue;\n }\n // TODO(perf): may not need to compare full text in some cases\n\n rc = vec0_get_metadata_text_long_value(p, &stmt, metadata_idx, rowids[i], &nFull, &sFull);\n if(rc != SQLITE_OK) {\n goto done;\n }\n if(nPrefix != nFull) {\n rc = SQLITE_ERROR;\n goto done;\n }\n bitmap_set(b, i, strncmp(sFull, sTarget, nFull) < 0);\n }\n break;\n }\n\n case VEC0_METADATA_OPERATOR_IN: {\n size_t metadataInIdx = -1;\n for(size_t i = 0; i < aMetadataIn->length; i++) {\n struct Vec0MetadataIn * metadataIn = &(((struct Vec0MetadataIn *) aMetadataIn->z)[i]);\n if(metadataIn->argv_idx == argv_idx) {\n metadataInIdx = i;\n break;\n }\n }\n if(metadataInIdx < 0) {\n rc = SQLITE_ERROR;\n goto done;\n }\n\n struct Vec0MetadataIn * metadataIn = &((struct Vec0MetadataIn *) aMetadataIn->z)[metadataInIdx];\n struct Array * aTarget = &(metadataIn->array);\n\n\n int nPrefix;\n char * sPrefix;\n char *sFull;\n int nFull;\n u8 * view;\n for(int i = 0; i < size; i++) {\n view = &((u8*) buffer)[i * VEC0_METADATA_TEXT_VIEW_BUFFER_LENGTH];\n nPrefix = ((int*) view)[0];\n sPrefix = (char *) &view[4];\n for(size_t target_idx = 0; target_idx < aTarget->length; target_idx++) {\n struct Vec0MetadataInTextEntry * entry = &(((struct Vec0MetadataInTextEntry*)aTarget->z)[target_idx]);\n if(entry->n != nPrefix) {\n continue;\n }\n int cmpPrefix = strncmp(sPrefix, entry->zString, min(nPrefix, VEC0_METADATA_TEXT_VIEW_DATA_LENGTH));\n if(nPrefix <= VEC0_METADATA_TEXT_VIEW_DATA_LENGTH) {\n if(cmpPrefix == 0) {\n bitmap_set(b, i, 1);\n break;\n }\n continue;\n }\n if(cmpPrefix) {\n continue;\n }\n\n rc = vec0_get_metadata_text_long_value(p, &stmt, metadata_idx, rowids[i], &nFull, &sFull);\n if(rc != SQLITE_OK) {\n goto done;\n }\n if(nPrefix != nFull) {\n rc = SQLITE_ERROR;\n goto done;\n }\n if(strncmp(sFull, entry->zString, nFull) == 0) {\n bitmap_set(b, i, 1);\n break;\n }\n }\n }\n break;\n }\n\n }\n rc = SQLITE_OK;\n\n done:\n sqlite3_finalize(stmt);\n sqlite3_free(rowids);\n return rc;\n\n}\n\n/**\n * @brief Fill in bitmap of chunk values, whether or not the values match a metadata constraint\n *\n * @param p vec0_vtab\n * @param metadata_idx index of the metatadata column to perfrom constraints on\n * @param value sqlite3_value of the constraints value\n * @param blob sqlite3_blob that is already opened on the metdata column's shadow chunk table\n * @param chunk_rowid rowid of the chunk to calculate on\n * @param b pre-allocated and zero'd out bitmap to write results to\n * @param size size of the chunk\n * @return int SQLITE_OK on success, error code otherwise\n */\nint vec0_set_metadata_filter_bitmap(\n vec0_vtab *p,\n int metadata_idx,\n vec0_metadata_operator op,\n sqlite3_value * value,\n sqlite3_blob * blob,\n i64 chunk_rowid,\n u8* b,\n int size,\n struct Array * aMetadataIn, int argv_idx) {\n // TODO: shouldn't this skip in-valid entries from the chunk's validity bitmap?\n\n int rc;\n rc = sqlite3_blob_reopen(blob, chunk_rowid);\n if(rc != SQLITE_OK) {\n return rc;\n }\n\n vec0_metadata_column_kind kind = p->metadata_columns[metadata_idx].kind;\n int szMatch = 0;\n int blobSize = sqlite3_blob_bytes(blob);\n switch(kind) {\n case VEC0_METADATA_COLUMN_KIND_BOOLEAN: {\n szMatch = blobSize == size / CHAR_BIT;\n break;\n }\n case VEC0_METADATA_COLUMN_KIND_INTEGER: {\n szMatch = blobSize == size * sizeof(i64);\n break;\n }\n case VEC0_METADATA_COLUMN_KIND_FLOAT: {\n szMatch = blobSize == size * sizeof(double);\n break;\n }\n case VEC0_METADATA_COLUMN_KIND_TEXT: {\n szMatch = blobSize == size * VEC0_METADATA_TEXT_VIEW_BUFFER_LENGTH;\n break;\n }\n }\n if(!szMatch) {\n return SQLITE_ERROR;\n }\n void * buffer = sqlite3_malloc(blobSize);\n if(!buffer) {\n return SQLITE_NOMEM;\n }\n rc = sqlite3_blob_read(blob, buffer, blobSize, 0);\n if(rc != SQLITE_OK) {\n goto done;\n }\n switch(kind) {\n case VEC0_METADATA_COLUMN_KIND_BOOLEAN: {\n int target = sqlite3_value_int(value);\n if( (target && op == VEC0_METADATA_OPERATOR_EQ) || (!target && op == VEC0_METADATA_OPERATOR_NE)) {\n for(int i = 0; i < size; i++) { bitmap_set(b, i, bitmap_get((u8*) buffer, i)); }\n }\n else {\n for(int i = 0; i < size; i++) { bitmap_set(b, i, !bitmap_get((u8*) buffer, i)); }\n }\n break;\n }\n case VEC0_METADATA_COLUMN_KIND_INTEGER: {\n i64 * array = (i64*) buffer;\n i64 target = sqlite3_value_int64(value);\n switch(op) {\n case VEC0_METADATA_OPERATOR_EQ: {\n for(int i = 0; i < size; i++) { bitmap_set(b, i, array[i] == target); }\n break;\n }\n case VEC0_METADATA_OPERATOR_GT: {\n for(int i = 0; i < size; i++) { bitmap_set(b, i, array[i] > target); }\n break;\n }\n case VEC0_METADATA_OPERATOR_LE: {\n for(int i = 0; i < size; i++) { bitmap_set(b, i, array[i] <= target); }\n break;\n }\n case VEC0_METADATA_OPERATOR_LT: {\n for(int i = 0; i < size; i++) { bitmap_set(b, i, array[i] < target); }\n break;\n }\n case VEC0_METADATA_OPERATOR_GE: {\n for(int i = 0; i < size; i++) { bitmap_set(b, i, array[i] >= target); }\n break;\n }\n case VEC0_METADATA_OPERATOR_NE: {\n for(int i = 0; i < size; i++) { bitmap_set(b, i, array[i] != target); }\n break;\n }\n case VEC0_METADATA_OPERATOR_IN: {\n int metadataInIdx = -1;\n for(size_t i = 0; i < aMetadataIn->length; i++) {\n struct Vec0MetadataIn * metadataIn = &((struct Vec0MetadataIn *) aMetadataIn->z)[i];\n if(metadataIn->argv_idx == argv_idx) {\n metadataInIdx = i;\n break;\n }\n }\n if(metadataInIdx < 0) {\n rc = SQLITE_ERROR;\n goto done;\n }\n struct Vec0MetadataIn * metadataIn = &((struct Vec0MetadataIn *) aMetadataIn->z)[metadataInIdx];\n struct Array * aTarget = &(metadataIn->array);\n\n for(int i = 0; i < size; i++) {\n for(size_t target_idx = 0; target_idx < aTarget->length; target_idx++) {\n if( ((i64*)aTarget->z)[target_idx] == array[i]) {\n bitmap_set(b, i, 1);\n break;\n }\n }\n }\n break;\n }\n }\n break;\n }\n case VEC0_METADATA_COLUMN_KIND_FLOAT: {\n double * array = (double*) buffer;\n double target = sqlite3_value_double(value);\n switch(op) {\n case VEC0_METADATA_OPERATOR_EQ: {\n for(int i = 0; i < size; i++) { bitmap_set(b, i, array[i] == target); }\n break;\n }\n case VEC0_METADATA_OPERATOR_GT: {\n for(int i = 0; i < size; i++) { bitmap_set(b, i, array[i] > target); }\n break;\n }\n case VEC0_METADATA_OPERATOR_LE: {\n for(int i = 0; i < size; i++) { bitmap_set(b, i, array[i] <= target); }\n break;\n }\n case VEC0_METADATA_OPERATOR_LT: {\n for(int i = 0; i < size; i++) { bitmap_set(b, i, array[i] < target); }\n break;\n }\n case VEC0_METADATA_OPERATOR_GE: {\n for(int i = 0; i < size; i++) { bitmap_set(b, i, array[i] >= target); }\n break;\n }\n case VEC0_METADATA_OPERATOR_NE: {\n for(int i = 0; i < size; i++) { bitmap_set(b, i, array[i] != target); }\n break;\n }\n case VEC0_METADATA_OPERATOR_IN: {\n // should never be reached\n break;\n }\n }\n break;\n }\n case VEC0_METADATA_COLUMN_KIND_TEXT: {\n rc = vec0_metadata_filter_text(p, value, buffer, size, op, b, metadata_idx, chunk_rowid, aMetadataIn, argv_idx);\n if(rc != SQLITE_OK) {\n goto done;\n }\n break;\n }\n }\n done:\n sqlite3_free(buffer);\n return rc;\n}\n\nint vec0Filter_knn_chunks_iter(vec0_vtab *p, sqlite3_stmt *stmtChunks,\n struct VectorColumnDefinition *vector_column,\n int vectorColumnIdx, struct Array *arrayRowidsIn,\n struct Array * aMetadataIn,\n const char * idxStr, int argc, sqlite3_value ** argv,\n void *queryVector, i64 k, i64 **out_topk_rowids,\n f32 **out_topk_distances, i64 *out_used) {\n // for each chunk, get top min(k, chunk_size) rowid + distances to query vec.\n // then reconcile all topk_chunks for a true top k.\n // output only rowids + distances for now\n\n int rc = SQLITE_OK;\n sqlite3_blob *blobVectors = NULL;\n\n void *baseVectors = NULL; // memory: chunk_size * dimensions * element_size\n\n // OWNED BY CALLER ON SUCCESS\n i64 *topk_rowids = NULL; // memory: k * 4\n // OWNED BY CALLER ON SUCCESS\n f32 *topk_distances = NULL; // memory: k * 4\n\n i64 *tmp_topk_rowids = NULL; // memory: k * 4\n f32 *tmp_topk_distances = NULL; // memory: k * 4\n f32 *chunk_distances = NULL; // memory: chunk_size * 4\n u8 *b = NULL; // memory: chunk_size / 8\n u8 *bTaken = NULL; // memory: chunk_size / 8\n i32 *chunk_topk_idxs = NULL; // memory: k * 4\n u8 *bmRowids = NULL; // memory: chunk_size / 8\n u8 *bmMetadata = NULL; // memory: chunk_size / 8\n // // total: a lot???\n\n // 6 * (k * 4) + (k * 2) + (chunk_size / 8) + (chunk_size * dimensions * 4)\n\n topk_rowids = sqlite3_malloc(k * sizeof(i64));\n if (!topk_rowids) {\n rc = SQLITE_NOMEM;\n goto cleanup;\n }\n memset(topk_rowids, 0, k * sizeof(i64));\n\n topk_distances = sqlite3_malloc(k * sizeof(f32));\n if (!topk_distances) {\n rc = SQLITE_NOMEM;\n goto cleanup;\n }\n memset(topk_distances, 0, k * sizeof(f32));\n\n tmp_topk_rowids = sqlite3_malloc(k * sizeof(i64));\n if (!tmp_topk_rowids) {\n rc = SQLITE_NOMEM;\n goto cleanup;\n }\n memset(tmp_topk_rowids, 0, k * sizeof(i64));\n\n tmp_topk_distances = sqlite3_malloc(k * sizeof(f32));\n if (!tmp_topk_distances) {\n rc = SQLITE_NOMEM;\n goto cleanup;\n }\n memset(tmp_topk_distances, 0, k * sizeof(f32));\n\n i64 k_used = 0;\n i64 baseVectorsSize = p->chunk_size * vector_column_byte_size(*vector_column);\n baseVectors = sqlite3_malloc(baseVectorsSize);\n if (!baseVectors) {\n rc = SQLITE_NOMEM;\n goto cleanup;\n }\n\n chunk_distances = sqlite3_malloc(p->chunk_size * sizeof(f32));\n if (!chunk_distances) {\n rc = SQLITE_NOMEM;\n goto cleanup;\n }\n\n b = bitmap_new(p->chunk_size);\n if (!b) {\n rc = SQLITE_NOMEM;\n goto cleanup;\n }\n\n bTaken = bitmap_new(p->chunk_size);\n if (!bTaken) {\n rc = SQLITE_NOMEM;\n goto cleanup;\n }\n\n chunk_topk_idxs = sqlite3_malloc(k * sizeof(i32));\n if (!chunk_topk_idxs) {\n rc = SQLITE_NOMEM;\n goto cleanup;\n }\n\n bmRowids = arrayRowidsIn ? bitmap_new(p->chunk_size) : NULL;\n if (arrayRowidsIn && !bmRowids) {\n rc = SQLITE_NOMEM;\n goto cleanup;\n }\n\n sqlite3_blob * metadataBlobs[VEC0_MAX_METADATA_COLUMNS];\n memset(metadataBlobs, 0, sizeof(sqlite3_blob*) * VEC0_MAX_METADATA_COLUMNS);\n\n bmMetadata = bitmap_new(p->chunk_size);\n if(!bmMetadata) {\n rc = SQLITE_NOMEM;\n goto cleanup;\n }\n\n int idxStrLength = strlen(idxStr);\n int numValueEntries = (idxStrLength-1) / 4;\n assert(numValueEntries == argc);\n int hasMetadataFilters = 0;\n int hasDistanceConstraints = 0;\n for(int i = 0; i < argc; i++) {\n int idx = 1 + (i * 4);\n char kind = idxStr[idx + 0];\n if(kind == VEC0_IDXSTR_KIND_METADATA_CONSTRAINT) {\n hasMetadataFilters = 1;\n }\n else if(kind == VEC0_IDXSTR_KIND_KNN_DISTANCE_CONSTRAINT) {\n hasDistanceConstraints = 1;\n }\n }\n\n while (true) {\n rc = sqlite3_step(stmtChunks);\n if (rc == SQLITE_DONE) {\n break;\n }\n if (rc != SQLITE_ROW) {\n vtab_set_error(&p->base, \"chunks iter error\");\n rc = SQLITE_ERROR;\n goto cleanup;\n }\n memset(chunk_distances, 0, p->chunk_size * sizeof(f32));\n memset(chunk_topk_idxs, 0, k * sizeof(i32));\n bitmap_clear(b, p->chunk_size);\n\n i64 chunk_id = sqlite3_column_int64(stmtChunks, 0);\n unsigned char *chunkValidity =\n (unsigned char *)sqlite3_column_blob(stmtChunks, 1);\n i64 validitySize = sqlite3_column_bytes(stmtChunks, 1);\n if (validitySize != p->chunk_size / CHAR_BIT) {\n // IMP: V05271_22109\n vtab_set_error(\n &p->base,\n \"chunk validity size doesn't match - expected %lld, found %lld\",\n p->chunk_size / CHAR_BIT, validitySize);\n rc = SQLITE_ERROR;\n goto cleanup;\n }\n\n i64 *chunkRowids = (i64 *)sqlite3_column_blob(stmtChunks, 2);\n i64 rowidsSize = sqlite3_column_bytes(stmtChunks, 2);\n if (rowidsSize != p->chunk_size * sizeof(i64)) {\n // IMP: V02796_19635\n vtab_set_error(&p->base, \"rowids size doesn't match\");\n vtab_set_error(\n &p->base,\n \"chunk rowids size doesn't match - expected %lld, found %lld\",\n p->chunk_size * sizeof(i64), rowidsSize);\n rc = SQLITE_ERROR;\n goto cleanup;\n }\n\n // open the vector chunk blob for the current chunk\n rc = sqlite3_blob_open(p->db, p->schemaName,\n p->shadowVectorChunksNames[vectorColumnIdx],\n \"vectors\", chunk_id, 0, &blobVectors);\n if (rc != SQLITE_OK) {\n vtab_set_error(&p->base, \"could not open vectors blob for chunk %lld\",\n chunk_id);\n rc = SQLITE_ERROR;\n goto cleanup;\n }\n\n i64 currentBaseVectorsSize = sqlite3_blob_bytes(blobVectors);\n i64 expectedBaseVectorsSize =\n p->chunk_size * vector_column_byte_size(*vector_column);\n if (currentBaseVectorsSize != expectedBaseVectorsSize) {\n // IMP: V16465_00535\n vtab_set_error(\n &p->base,\n \"vectors blob size doesn't match - expected %lld, found %lld\",\n expectedBaseVectorsSize, currentBaseVectorsSize);\n rc = SQLITE_ERROR;\n goto cleanup;\n }\n rc = sqlite3_blob_read(blobVectors, baseVectors, currentBaseVectorsSize, 0);\n\n if (rc != SQLITE_OK) {\n vtab_set_error(&p->base, \"vectors blob read error for %lld\", chunk_id);\n rc = SQLITE_ERROR;\n goto cleanup;\n }\n\n bitmap_copy(b, chunkValidity, p->chunk_size);\n if (arrayRowidsIn) {\n bitmap_clear(bmRowids, p->chunk_size);\n\n for (int i = 0; i < p->chunk_size; i++) {\n if (!bitmap_get(chunkValidity, i)) {\n continue;\n }\n i64 rowid = chunkRowids[i];\n void *in = bsearch(&rowid, arrayRowidsIn->z, arrayRowidsIn->length,\n sizeof(i64), _cmp);\n bitmap_set(bmRowids, i, in ? 1 : 0);\n }\n bitmap_and_inplace(b, bmRowids, p->chunk_size);\n }\n\n if(hasMetadataFilters) {\n for(int i = 0; i < argc; i++) {\n int idx = 1 + (i * 4);\n char kind = idxStr[idx + 0];\n if(kind != VEC0_IDXSTR_KIND_METADATA_CONSTRAINT) {\n continue;\n }\n int metadata_idx = idxStr[idx + 1] - 'A';\n int operator = idxStr[idx + 2];\n\n if(!metadataBlobs[metadata_idx]) {\n rc = sqlite3_blob_open(p->db, p->schemaName, p->shadowMetadataChunksNames[metadata_idx], \"data\", chunk_id, 0, &metadataBlobs[metadata_idx]);\n vtab_set_error(&p->base, \"Could not open metadata blob\");\n if(rc != SQLITE_OK) {\n goto cleanup;\n }\n }\n\n bitmap_clear(bmMetadata, p->chunk_size);\n rc = vec0_set_metadata_filter_bitmap(p, metadata_idx, operator, argv[i], metadataBlobs[metadata_idx], chunk_id, bmMetadata, p->chunk_size, aMetadataIn, i);\n if(rc != SQLITE_OK) {\n vtab_set_error(&p->base, \"Could not filter metadata fields\");\n if(rc != SQLITE_OK) {\n goto cleanup;\n }\n }\n bitmap_and_inplace(b, bmMetadata, p->chunk_size);\n }\n }\n\n\n for (int i = 0; i < p->chunk_size; i++) {\n if (!bitmap_get(b, i)) {\n continue;\n };\n\n f32 result;\n switch (vector_column->element_type) {\n case SQLITE_VEC_ELEMENT_TYPE_FLOAT32: {\n const f32 *base_i =\n ((f32 *)baseVectors) + (i * vector_column->dimensions);\n switch (vector_column->distance_metric) {\n case VEC0_DISTANCE_METRIC_L2: {\n result = distance_l2_sqr_float(base_i, (f32 *)queryVector,\n &vector_column->dimensions);\n break;\n }\n case VEC0_DISTANCE_METRIC_L1: {\n result = distance_l1_f32(base_i, (f32 *)queryVector,\n &vector_column->dimensions);\n break;\n }\n case VEC0_DISTANCE_METRIC_COSINE: {\n result = distance_cosine_float(base_i, (f32 *)queryVector,\n &vector_column->dimensions);\n break;\n }\n }\n break;\n }\n case SQLITE_VEC_ELEMENT_TYPE_INT8: {\n const i8 *base_i =\n ((i8 *)baseVectors) + (i * vector_column->dimensions);\n switch (vector_column->distance_metric) {\n case VEC0_DISTANCE_METRIC_L2: {\n result = distance_l2_sqr_int8(base_i, (i8 *)queryVector,\n &vector_column->dimensions);\n break;\n }\n case VEC0_DISTANCE_METRIC_L1: {\n result = distance_l1_int8(base_i, (i8 *)queryVector,\n &vector_column->dimensions);\n break;\n }\n case VEC0_DISTANCE_METRIC_COSINE: {\n result = distance_cosine_int8(base_i, (i8 *)queryVector,\n &vector_column->dimensions);\n break;\n }\n }\n\n break;\n }\n case SQLITE_VEC_ELEMENT_TYPE_BIT: {\n const u8 *base_i =\n ((u8 *)baseVectors) + (i * (vector_column->dimensions / CHAR_BIT));\n result = distance_hamming(base_i, (u8 *)queryVector,\n &vector_column->dimensions);\n break;\n }\n }\n\n chunk_distances[i] = result;\n }\n\n if(hasDistanceConstraints) {\n for(int i = 0; i < argc; i++) {\n int idx = 1 + (i * 4);\n char kind = idxStr[idx + 0];\n // TODO casts f64 to f32, is that a problem?\n f32 target = (f32) sqlite3_value_double(argv[i]);\n\n if(kind != VEC0_IDXSTR_KIND_KNN_DISTANCE_CONSTRAINT) {\n continue;\n }\n vec0_distance_constraint_operator op = idxStr[idx + 1];\n\n switch(op) {\n case VEC0_DISTANCE_CONSTRAINT_GE: {\n for(int i = 0; i < p->chunk_size;i++) {\n if(bitmap_get(b, i) && !(chunk_distances[i] >= target)) {\n bitmap_set(b, i, 0);\n }\n }\n break;\n }\n case VEC0_DISTANCE_CONSTRAINT_GT: {\n for(int i = 0; i < p->chunk_size;i++) {\n if(bitmap_get(b, i) && !(chunk_distances[i] > target)) {\n bitmap_set(b, i, 0);\n }\n }\n break;\n }\n case VEC0_DISTANCE_CONSTRAINT_LE: {\n for(int i = 0; i < p->chunk_size;i++) {\n if(bitmap_get(b, i) && !(chunk_distances[i] <= target)) {\n bitmap_set(b, i, 0);\n }\n }\n break;\n }\n case VEC0_DISTANCE_CONSTRAINT_LT: {\n for(int i = 0; i < p->chunk_size;i++) {\n if(bitmap_get(b, i) && !(chunk_distances[i] < target)) {\n bitmap_set(b, i, 0);\n }\n }\n break;\n }\n }\n }\n }\n\n int used1;\n min_idx(chunk_distances, p->chunk_size, b, chunk_topk_idxs,\n min(k, p->chunk_size), bTaken, &used1);\n\n i64 used;\n merge_sorted_lists(topk_distances, topk_rowids, k_used, chunk_distances,\n chunkRowids, chunk_topk_idxs,\n min(min(k, p->chunk_size), used1), tmp_topk_distances,\n tmp_topk_rowids, k, &used);\n\n for (int i = 0; i < used; i++) {\n topk_rowids[i] = tmp_topk_rowids[i];\n topk_distances[i] = tmp_topk_distances[i];\n }\n k_used = used;\n // blobVectors is always opened with read-only permissions, so this never\n // fails.\n sqlite3_blob_close(blobVectors);\n blobVectors = NULL;\n }\n\n *out_topk_rowids = topk_rowids;\n *out_topk_distances = topk_distances;\n *out_used = k_used;\n rc = SQLITE_OK;\n\ncleanup:\n if (rc != SQLITE_OK) {\n sqlite3_free(topk_rowids);\n sqlite3_free(topk_distances);\n }\n sqlite3_free(chunk_topk_idxs);\n sqlite3_free(tmp_topk_rowids);\n sqlite3_free(tmp_topk_distances);\n sqlite3_free(b);\n sqlite3_free(bTaken);\n sqlite3_free(bmRowids);\n sqlite3_free(baseVectors);\n sqlite3_free(chunk_distances);\n sqlite3_free(bmMetadata);\n for(int i = 0; i < VEC0_MAX_METADATA_COLUMNS; i++) {\n sqlite3_blob_close(metadataBlobs[i]);\n }\n // blobVectors is always opened with read-only permissions, so this never\n // fails.\n sqlite3_blob_close(blobVectors);\n return rc;\n}\n\nint vec0Filter_knn(vec0_cursor *pCur, vec0_vtab *p, int idxNum,\n const char *idxStr, int argc, sqlite3_value **argv) {\n assert(argc == (strlen(idxStr)-1) / 4);\n int rc;\n struct vec0_query_knn_data *knn_data;\n\n int vectorColumnIdx = idxNum;\n struct VectorColumnDefinition *vector_column =\n &p->vector_columns[vectorColumnIdx];\n\n struct Array *arrayRowidsIn = NULL;\n sqlite3_stmt *stmtChunks = NULL;\n void *queryVector;\n size_t dimensions;\n enum VectorElementType elementType;\n vector_cleanup queryVectorCleanup = vector_cleanup_noop;\n char *pzError;\n knn_data = sqlite3_malloc(sizeof(*knn_data));\n if (!knn_data) {\n return SQLITE_NOMEM;\n }\n memset(knn_data, 0, sizeof(*knn_data));\n // array of `struct Vec0MetadataIn`, IF there are any `xxx in (...)` metadata constraints\n struct Array * aMetadataIn = NULL;\n\n int query_idx =-1;\n int k_idx = -1;\n int rowid_in_idx = -1;\n for(int i = 0; i < argc; i++) {\n if(idxStr[1 + (i*4)] == VEC0_IDXSTR_KIND_KNN_MATCH) {\n query_idx = i;\n }\n if(idxStr[1 + (i*4)] == VEC0_IDXSTR_KIND_KNN_K) {\n k_idx = i;\n }\n if(idxStr[1 + (i*4)] == VEC0_IDXSTR_KIND_KNN_ROWID_IN) {\n rowid_in_idx = i;\n }\n }\n assert(query_idx >= 0);\n assert(k_idx >= 0);\n\n // make sure the query vector matches the vector column (type dimensions etc.)\n rc = vector_from_value(argv[query_idx], &queryVector, &dimensions, &elementType,\n &queryVectorCleanup, &pzError);\n\n if (rc != SQLITE_OK) {\n vtab_set_error(&p->base,\n \"Query vector on the \\\"%.*s\\\" column is invalid: %z\",\n vector_column->name_length, vector_column->name, pzError);\n rc = SQLITE_ERROR;\n goto cleanup;\n }\n if (elementType != vector_column->element_type) {\n vtab_set_error(\n &p->base,\n \"Query vector for the \\\"%.*s\\\" column is expected to be of type \"\n \"%s, but a %s vector was provided.\",\n vector_column->name_length, vector_column->name,\n vector_subtype_name(vector_column->element_type),\n vector_subtype_name(elementType));\n rc = SQLITE_ERROR;\n goto cleanup;\n }\n if (dimensions != vector_column->dimensions) {\n vtab_set_error(\n &p->base,\n \"Dimension mismatch for query vector for the \\\"%.*s\\\" column. \"\n \"Expected %d dimensions but received %d.\",\n vector_column->name_length, vector_column->name,\n vector_column->dimensions, dimensions);\n rc = SQLITE_ERROR;\n goto cleanup;\n }\n\n i64 k = sqlite3_value_int64(argv[k_idx]);\n if (k < 0) {\n vtab_set_error(\n &p->base, \"k value in knn queries must be greater than or equal to 0.\");\n rc = SQLITE_ERROR;\n goto cleanup;\n }\n#define SQLITE_VEC_VEC0_K_MAX 4096\n if (k > SQLITE_VEC_VEC0_K_MAX) {\n vtab_set_error(\n &p->base,\n \"k value in knn query too large, provided %lld and the limit is %lld\",\n k, SQLITE_VEC_VEC0_K_MAX);\n rc = SQLITE_ERROR;\n goto cleanup;\n }\n\n if (k == 0) {\n knn_data->k = 0;\n pCur->knn_data = knn_data;\n pCur->query_plan = VEC0_QUERY_PLAN_KNN;\n rc = SQLITE_OK;\n goto cleanup;\n }\n\n// handle when a `rowid in (...)` operation was provided\n// Array of all the rowids that appear in any `rowid in (...)` constraint.\n// NULL if none were provided, which means a \"full\" scan.\n#if COMPILER_SUPPORTS_VTAB_IN\n if (rowid_in_idx >= 0) {\n sqlite3_value *item;\n int rc;\n arrayRowidsIn = sqlite3_malloc(sizeof(*arrayRowidsIn));\n if (!arrayRowidsIn) {\n rc = SQLITE_NOMEM;\n goto cleanup;\n }\n memset(arrayRowidsIn, 0, sizeof(*arrayRowidsIn));\n\n rc = array_init(arrayRowidsIn, sizeof(i64), 32);\n if (rc != SQLITE_OK) {\n goto cleanup;\n }\n for (rc = sqlite3_vtab_in_first(argv[rowid_in_idx], &item); rc == SQLITE_OK && item;\n rc = sqlite3_vtab_in_next(argv[rowid_in_idx], &item)) {\n i64 rowid;\n if (p->pkIsText) {\n rc = vec0_rowid_from_id(p, item, &rowid);\n if (rc != SQLITE_OK) {\n goto cleanup;\n }\n } else {\n rowid = sqlite3_value_int64(item);\n }\n rc = array_append(arrayRowidsIn, &rowid);\n if (rc != SQLITE_OK) {\n goto cleanup;\n }\n }\n if (rc != SQLITE_DONE) {\n vtab_set_error(&p->base, \"error processing rowid in (...) array\");\n goto cleanup;\n }\n qsort(arrayRowidsIn->z, arrayRowidsIn->length, arrayRowidsIn->element_size,\n _cmp);\n }\n#endif\n\n #if COMPILER_SUPPORTS_VTAB_IN\n for(int i = 0; i < argc; i++) {\n if(!(idxStr[1 + (i*4)] == VEC0_IDXSTR_KIND_METADATA_CONSTRAINT && idxStr[1 + (i*4) + 2] == VEC0_METADATA_OPERATOR_IN)) {\n continue;\n }\n int metadata_idx = idxStr[1 + (i*4) + 1] - 'A';\n if(!aMetadataIn) {\n aMetadataIn = sqlite3_malloc(sizeof(*aMetadataIn));\n if(!aMetadataIn) {\n rc = SQLITE_NOMEM;\n goto cleanup;\n }\n memset(aMetadataIn, 0, sizeof(*aMetadataIn));\n rc = array_init(aMetadataIn, sizeof(struct Vec0MetadataIn), 8);\n if(rc != SQLITE_OK) {\n goto cleanup;\n }\n }\n\n struct Vec0MetadataIn item;\n memset(&item, 0, sizeof(item));\n item.metadata_idx=metadata_idx;\n item.argv_idx = i;\n\n switch(p->metadata_columns[metadata_idx].kind) {\n case VEC0_METADATA_COLUMN_KIND_INTEGER: {\n rc = array_init(&item.array, sizeof(i64), 16);\n if(rc != SQLITE_OK) {\n goto cleanup;\n }\n sqlite3_value *entry;\n for (rc = sqlite3_vtab_in_first(argv[i], &entry); rc == SQLITE_OK && entry; rc = sqlite3_vtab_in_next(argv[i], &entry)) {\n i64 v = sqlite3_value_int64(entry);\n rc = array_append(&item.array, &v);\n if (rc != SQLITE_OK) {\n goto cleanup;\n }\n }\n\n if (rc != SQLITE_DONE) {\n vtab_set_error(&p->base, \"Error fetching next value in `x in (...)` integer expression\");\n goto cleanup;\n }\n\n break;\n }\n case VEC0_METADATA_COLUMN_KIND_TEXT: {\n rc = array_init(&item.array, sizeof(struct Vec0MetadataInTextEntry), 16);\n if(rc != SQLITE_OK) {\n goto cleanup;\n }\n sqlite3_value *entry;\n for (rc = sqlite3_vtab_in_first(argv[i], &entry); rc == SQLITE_OK && entry; rc = sqlite3_vtab_in_next(argv[i], &entry)) {\n const char * s = (const char *) sqlite3_value_text(entry);\n int n = sqlite3_value_bytes(entry);\n\n struct Vec0MetadataInTextEntry entry;\n entry.zString = sqlite3_mprintf(\"%.*s\", n, s);\n if(!entry.zString) {\n rc = SQLITE_NOMEM;\n goto cleanup;\n }\n entry.n = n;\n rc = array_append(&item.array, &entry);\n if (rc != SQLITE_OK) {\n goto cleanup;\n }\n }\n\n if (rc != SQLITE_DONE) {\n vtab_set_error(&p->base, \"Error fetching next value in `x in (...)` text expression\");\n goto cleanup;\n }\n\n break;\n }\n default: {\n vtab_set_error(&p->base, \"Internal sqlite-vec error\");\n goto cleanup;\n }\n }\n\n rc = array_append(aMetadataIn, &item);\n if(rc != SQLITE_OK) {\n goto cleanup;\n }\n }\n #endif\n\n rc = vec0_chunks_iter(p, idxStr, argc, argv, &stmtChunks);\n if (rc != SQLITE_OK) {\n // IMP: V06942_23781\n vtab_set_error(&p->base, \"Error preparing stmtChunk: %s\",\n sqlite3_errmsg(p->db));\n goto cleanup;\n }\n\n i64 *topk_rowids = NULL;\n f32 *topk_distances = NULL;\n i64 k_used = 0;\n rc = vec0Filter_knn_chunks_iter(p, stmtChunks, vector_column, vectorColumnIdx,\n arrayRowidsIn, aMetadataIn, idxStr, argc, argv, queryVector, k, &topk_rowids,\n &topk_distances, &k_used);\n if (rc != SQLITE_OK) {\n goto cleanup;\n }\n\n knn_data->current_idx = 0;\n knn_data->k = k;\n knn_data->rowids = topk_rowids;\n knn_data->distances = topk_distances;\n knn_data->k_used = k_used;\n\n pCur->knn_data = knn_data;\n pCur->query_plan = VEC0_QUERY_PLAN_KNN;\n rc = SQLITE_OK;\n\ncleanup:\n sqlite3_finalize(stmtChunks);\n array_cleanup(arrayRowidsIn);\n sqlite3_free(arrayRowidsIn);\n queryVectorCleanup(queryVector);\n if(aMetadataIn) {\n for(size_t i = 0; i < aMetadataIn->length; i++) {\n struct Vec0MetadataIn* item = &((struct Vec0MetadataIn *) aMetadataIn->z)[i];\n for(size_t j = 0; j < item->array.length; j++) {\n if(p->metadata_columns[item->metadata_idx].kind == VEC0_METADATA_COLUMN_KIND_TEXT) {\n struct Vec0MetadataInTextEntry entry = ((struct Vec0MetadataInTextEntry*)item->array.z)[j];\n sqlite3_free(entry.zString);\n }\n }\n array_cleanup(&item->array);\n }\n array_cleanup(aMetadataIn);\n }\n\n sqlite3_free(aMetadataIn);\n\n if (rc != SQLITE_OK) {\n sqlite3_free(knn_data);\n }\n\n return rc;\n}\n\nint vec0Filter_fullscan(vec0_vtab *p, vec0_cursor *pCur) {\n int rc;\n char *zSql;\n struct vec0_query_fullscan_data *fullscan_data;\n\n fullscan_data = sqlite3_malloc(sizeof(*fullscan_data));\n if (!fullscan_data) {\n return SQLITE_NOMEM;\n }\n memset(fullscan_data, 0, sizeof(*fullscan_data));\n\n zSql = sqlite3_mprintf(\" SELECT rowid \"\n \" FROM \" VEC0_SHADOW_ROWIDS_NAME\n \" ORDER by chunk_id, chunk_offset \",\n p->schemaName, p->tableName);\n if (!zSql) {\n rc = SQLITE_NOMEM;\n goto error;\n }\n rc = sqlite3_prepare_v2(p->db, zSql, -1, &fullscan_data->rowids_stmt, NULL);\n sqlite3_free(zSql);\n if (rc != SQLITE_OK) {\n // IMP: V09901_26739\n vtab_set_error(&p->base, \"Error preparing rowid scan: %s\",\n sqlite3_errmsg(p->db));\n goto error;\n }\n\n rc = sqlite3_step(fullscan_data->rowids_stmt);\n\n // DONE when there's no rowids, ROW when there are, both \"success\"\n if (!(rc == SQLITE_ROW || rc == SQLITE_DONE)) {\n goto error;\n }\n\n fullscan_data->done = rc == SQLITE_DONE;\n pCur->query_plan = VEC0_QUERY_PLAN_FULLSCAN;\n pCur->fullscan_data = fullscan_data;\n return SQLITE_OK;\n\nerror:\n vec0_query_fullscan_data_clear(fullscan_data);\n sqlite3_free(fullscan_data);\n return rc;\n}\n\nint vec0Filter_point(vec0_cursor *pCur, vec0_vtab *p, int argc,\n sqlite3_value **argv) {\n int rc;\n assert(argc == 1);\n i64 rowid;\n struct vec0_query_point_data *point_data = NULL;\n\n point_data = sqlite3_malloc(sizeof(*point_data));\n if (!point_data) {\n rc = SQLITE_NOMEM;\n goto error;\n }\n memset(point_data, 0, sizeof(*point_data));\n\n if (p->pkIsText) {\n rc = vec0_rowid_from_id(p, argv[0], &rowid);\n if (rc == SQLITE_EMPTY) {\n goto eof;\n }\n if (rc != SQLITE_OK) {\n goto error;\n }\n } else {\n rowid = sqlite3_value_int64(argv[0]);\n }\n\n for (int i = 0; i < p->numVectorColumns; i++) {\n rc = vec0_get_vector_data(p, rowid, i, &point_data->vectors[i], NULL);\n if (rc == SQLITE_EMPTY) {\n goto eof;\n }\n if (rc != SQLITE_OK) {\n goto error;\n }\n }\n\n point_data->rowid = rowid;\n point_data->done = 0;\n pCur->point_data = point_data;\n pCur->query_plan = VEC0_QUERY_PLAN_POINT;\n return SQLITE_OK;\n\neof:\n point_data->rowid = rowid;\n point_data->done = 1;\n pCur->point_data = point_data;\n pCur->query_plan = VEC0_QUERY_PLAN_POINT;\n return SQLITE_OK;\n\nerror:\n vec0_query_point_data_clear(point_data);\n sqlite3_free(point_data);\n return rc;\n}\n\nstatic int vec0Filter(sqlite3_vtab_cursor *pVtabCursor, int idxNum,\n const char *idxStr, int argc, sqlite3_value **argv) {\n vec0_vtab *p = (vec0_vtab *)pVtabCursor->pVtab;\n vec0_cursor *pCur = (vec0_cursor *)pVtabCursor;\n vec0_cursor_clear(pCur);\n\n int idxStrLength = strlen(idxStr);\n if(idxStrLength <= 0) {\n return SQLITE_ERROR;\n }\n if((idxStrLength-1) % 4 != 0) {\n return SQLITE_ERROR;\n }\n int numValueEntries = (idxStrLength-1) / 4;\n if(numValueEntries != argc) {\n return SQLITE_ERROR;\n }\n\n char query_plan = idxStr[0];\n switch(query_plan) {\n case VEC0_QUERY_PLAN_FULLSCAN:\n return vec0Filter_fullscan(p, pCur);\n case VEC0_QUERY_PLAN_KNN:\n return vec0Filter_knn(pCur, p, idxNum, idxStr, argc, argv);\n case VEC0_QUERY_PLAN_POINT:\n return vec0Filter_point(pCur, p, argc, argv);\n default:\n vtab_set_error(pVtabCursor->pVtab, \"unknown idxStr '%s'\", idxStr);\n return SQLITE_ERROR;\n }\n}\n\nstatic int vec0Rowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid) {\n vec0_cursor *pCur = (vec0_cursor *)cur;\n switch (pCur->query_plan) {\n case VEC0_QUERY_PLAN_FULLSCAN: {\n *pRowid = sqlite3_column_int64(pCur->fullscan_data->rowids_stmt, 0);\n return SQLITE_OK;\n }\n case VEC0_QUERY_PLAN_POINT: {\n *pRowid = pCur->point_data->rowid;\n return SQLITE_OK;\n }\n case VEC0_QUERY_PLAN_KNN: {\n vtab_set_error(cur->pVtab,\n \"Internal sqlite-vec error: expected point query plan in \"\n \"vec0Rowid, found %d\",\n pCur->query_plan);\n return SQLITE_ERROR;\n }\n }\n return SQLITE_ERROR;\n}\n\nstatic int vec0Next(sqlite3_vtab_cursor *cur) {\n vec0_cursor *pCur = (vec0_cursor *)cur;\n switch (pCur->query_plan) {\n case VEC0_QUERY_PLAN_FULLSCAN: {\n if (!pCur->fullscan_data) {\n return SQLITE_ERROR;\n }\n int rc = sqlite3_step(pCur->fullscan_data->rowids_stmt);\n if (rc == SQLITE_DONE) {\n pCur->fullscan_data->done = 1;\n return SQLITE_OK;\n }\n if (rc == SQLITE_ROW) {\n return SQLITE_OK;\n }\n return SQLITE_ERROR;\n }\n case VEC0_QUERY_PLAN_KNN: {\n if (!pCur->knn_data) {\n return SQLITE_ERROR;\n }\n\n pCur->knn_data->current_idx++;\n return SQLITE_OK;\n }\n case VEC0_QUERY_PLAN_POINT: {\n if (!pCur->point_data) {\n return SQLITE_ERROR;\n }\n pCur->point_data->done = 1;\n return SQLITE_OK;\n }\n }\n return SQLITE_ERROR;\n}\n\nstatic int vec0Eof(sqlite3_vtab_cursor *cur) {\n vec0_cursor *pCur = (vec0_cursor *)cur;\n switch (pCur->query_plan) {\n case VEC0_QUERY_PLAN_FULLSCAN: {\n if (!pCur->fullscan_data) {\n return 1;\n }\n return pCur->fullscan_data->done;\n }\n case VEC0_QUERY_PLAN_KNN: {\n if (!pCur->knn_data) {\n return 1;\n }\n // return (pCur->knn_data->current_idx >= pCur->knn_data->k) ||\n // (pCur->knn_data->distances[pCur->knn_data->current_idx] == FLT_MAX);\n return (pCur->knn_data->current_idx >= pCur->knn_data->k_used);\n }\n case VEC0_QUERY_PLAN_POINT: {\n if (!pCur->point_data) {\n return 1;\n }\n return pCur->point_data->done;\n }\n }\n return 1;\n}\n\nstatic int vec0Column_fullscan(vec0_vtab *pVtab, vec0_cursor *pCur,\n sqlite3_context *context, int i) {\n if (!pCur->fullscan_data) {\n sqlite3_result_error(\n context, \"Internal sqlite-vec error: fullscan_data is NULL.\", -1);\n return SQLITE_ERROR;\n }\n i64 rowid = sqlite3_column_int64(pCur->fullscan_data->rowids_stmt, 0);\n if (i == VEC0_COLUMN_ID) {\n return vec0_result_id(pVtab, context, rowid);\n }\n else if (vec0_column_idx_is_vector(pVtab, i)) {\n void *v;\n int sz;\n int vector_idx = vec0_column_idx_to_vector_idx(pVtab, i);\n int rc = vec0_get_vector_data(pVtab, rowid, vector_idx, &v, &sz);\n if (rc != SQLITE_OK) {\n return rc;\n }\n sqlite3_result_blob(context, v, sz, sqlite3_free);\n sqlite3_result_subtype(context,\n pVtab->vector_columns[vector_idx].element_type);\n\n }\n else if (i == vec0_column_distance_idx(pVtab)) {\n sqlite3_result_null(context);\n }\n else if(vec0_column_idx_is_partition(pVtab, i)) {\n int partition_idx = vec0_column_idx_to_partition_idx(pVtab, i);\n sqlite3_value * v;\n int rc = vec0_get_partition_value_for_rowid(pVtab, rowid, partition_idx, &v);\n if(rc == SQLITE_OK) {\n sqlite3_result_value(context, v);\n sqlite3_value_free(v);\n }else {\n sqlite3_result_error_code(context, rc);\n }\n }\n else if(vec0_column_idx_is_auxiliary(pVtab, i)) {\n int auxiliary_idx = vec0_column_idx_to_auxiliary_idx(pVtab, i);\n sqlite3_value * v;\n int rc = vec0_get_auxiliary_value_for_rowid(pVtab, rowid, auxiliary_idx, &v);\n if(rc == SQLITE_OK) {\n sqlite3_result_value(context, v);\n sqlite3_value_free(v);\n }else {\n sqlite3_result_error_code(context, rc);\n }\n }\n\n else if(vec0_column_idx_is_metadata(pVtab, i)) {\n if(sqlite3_vtab_nochange(context)) {\n return SQLITE_OK;\n }\n int metadata_idx = vec0_column_idx_to_metadata_idx(pVtab, i);\n int rc = vec0_result_metadata_value_for_rowid(pVtab, rowid, metadata_idx, context);\n if(rc != SQLITE_OK) {\n // IMP: V15466_32305\n const char * zErr = sqlite3_mprintf(\n \"Could not extract metadata value for column %.*s at rowid %lld\",\n pVtab->metadata_columns[metadata_idx].name_length,\n pVtab->metadata_columns[metadata_idx].name, rowid\n );\n if(zErr) {\n sqlite3_result_error(context, zErr, -1);\n sqlite3_free((void *) zErr);\n }else {\n sqlite3_result_error_nomem(context);\n }\n }\n }\n\n return SQLITE_OK;\n}\n\nstatic int vec0Column_point(vec0_vtab *pVtab, vec0_cursor *pCur,\n sqlite3_context *context, int i) {\n if (!pCur->point_data) {\n sqlite3_result_error(context,\n \"Internal sqlite-vec error: point_data is NULL.\", -1);\n return SQLITE_ERROR;\n }\n if (i == VEC0_COLUMN_ID) {\n return vec0_result_id(pVtab, context, pCur->point_data->rowid);\n }\n else if (i == vec0_column_distance_idx(pVtab)) {\n sqlite3_result_null(context);\n return SQLITE_OK;\n }\n else if (vec0_column_idx_is_vector(pVtab, i)) {\n if (sqlite3_vtab_nochange(context)) {\n sqlite3_result_null(context);\n return SQLITE_OK;\n }\n int vector_idx = vec0_column_idx_to_vector_idx(pVtab, i);\n sqlite3_result_blob(\n context, pCur->point_data->vectors[vector_idx],\n vector_column_byte_size(pVtab->vector_columns[vector_idx]),\n SQLITE_TRANSIENT);\n sqlite3_result_subtype(context,\n pVtab->vector_columns[vector_idx].element_type);\n return SQLITE_OK;\n }\n else if(vec0_column_idx_is_partition(pVtab, i)) {\n if(sqlite3_vtab_nochange(context)) {\n return SQLITE_OK;\n }\n int partition_idx = vec0_column_idx_to_partition_idx(pVtab, i);\n i64 rowid = pCur->point_data->rowid;\n sqlite3_value * v;\n int rc = vec0_get_partition_value_for_rowid(pVtab, rowid, partition_idx, &v);\n if(rc == SQLITE_OK) {\n sqlite3_result_value(context, v);\n sqlite3_value_free(v);\n }else {\n sqlite3_result_error_code(context, rc);\n }\n }\n else if(vec0_column_idx_is_auxiliary(pVtab, i)) {\n if(sqlite3_vtab_nochange(context)) {\n return SQLITE_OK;\n }\n i64 rowid = pCur->point_data->rowid;\n int auxiliary_idx = vec0_column_idx_to_auxiliary_idx(pVtab, i);\n sqlite3_value * v;\n int rc = vec0_get_auxiliary_value_for_rowid(pVtab, rowid, auxiliary_idx, &v);\n if(rc == SQLITE_OK) {\n sqlite3_result_value(context, v);\n sqlite3_value_free(v);\n }else {\n sqlite3_result_error_code(context, rc);\n }\n }\n\n else if(vec0_column_idx_is_metadata(pVtab, i)) {\n if(sqlite3_vtab_nochange(context)) {\n return SQLITE_OK;\n }\n i64 rowid = pCur->point_data->rowid;\n int metadata_idx = vec0_column_idx_to_metadata_idx(pVtab, i);\n int rc = vec0_result_metadata_value_for_rowid(pVtab, rowid, metadata_idx, context);\n if(rc != SQLITE_OK) {\n const char * zErr = sqlite3_mprintf(\n \"Could not extract metadata value for column %.*s at rowid %lld\",\n pVtab->metadata_columns[metadata_idx].name_length,\n pVtab->metadata_columns[metadata_idx].name, rowid\n );\n if(zErr) {\n sqlite3_result_error(context, zErr, -1);\n sqlite3_free((void *) zErr);\n }else {\n sqlite3_result_error_nomem(context);\n }\n }\n }\n\n return SQLITE_OK;\n}\n\nstatic int vec0Column_knn(vec0_vtab *pVtab, vec0_cursor *pCur,\n sqlite3_context *context, int i) {\n if (!pCur->knn_data) {\n sqlite3_result_error(context,\n \"Internal sqlite-vec error: knn_data is NULL.\", -1);\n return SQLITE_ERROR;\n }\n if (i == VEC0_COLUMN_ID) {\n i64 rowid = pCur->knn_data->rowids[pCur->knn_data->current_idx];\n return vec0_result_id(pVtab, context, rowid);\n }\n else if (i == vec0_column_distance_idx(pVtab)) {\n sqlite3_result_double(\n context, pCur->knn_data->distances[pCur->knn_data->current_idx]);\n return SQLITE_OK;\n }\n else if (vec0_column_idx_is_vector(pVtab, i)) {\n void *out;\n int sz;\n int vector_idx = vec0_column_idx_to_vector_idx(pVtab, i);\n int rc = vec0_get_vector_data(\n pVtab, pCur->knn_data->rowids[pCur->knn_data->current_idx], vector_idx,\n &out, &sz);\n if (rc != SQLITE_OK) {\n return rc;\n }\n sqlite3_result_blob(context, out, sz, sqlite3_free);\n sqlite3_result_subtype(context,\n pVtab->vector_columns[vector_idx].element_type);\n return SQLITE_OK;\n }\n else if(vec0_column_idx_is_partition(pVtab, i)) {\n int partition_idx = vec0_column_idx_to_partition_idx(pVtab, i);\n i64 rowid = pCur->knn_data->rowids[pCur->knn_data->current_idx];\n sqlite3_value * v;\n int rc = vec0_get_partition_value_for_rowid(pVtab, rowid, partition_idx, &v);\n if(rc == SQLITE_OK) {\n sqlite3_result_value(context, v);\n sqlite3_value_free(v);\n }else {\n sqlite3_result_error_code(context, rc);\n }\n }\n else if(vec0_column_idx_is_auxiliary(pVtab, i)) {\n int auxiliary_idx = vec0_column_idx_to_auxiliary_idx(pVtab, i);\n i64 rowid = pCur->knn_data->rowids[pCur->knn_data->current_idx];\n sqlite3_value * v;\n int rc = vec0_get_auxiliary_value_for_rowid(pVtab, rowid, auxiliary_idx, &v);\n if(rc == SQLITE_OK) {\n sqlite3_result_value(context, v);\n sqlite3_value_free(v);\n }else {\n sqlite3_result_error_code(context, rc);\n }\n }\n\n else if(vec0_column_idx_is_metadata(pVtab, i)) {\n int metadata_idx = vec0_column_idx_to_metadata_idx(pVtab, i);\n i64 rowid = pCur->knn_data->rowids[pCur->knn_data->current_idx];\n int rc = vec0_result_metadata_value_for_rowid(pVtab, rowid, metadata_idx, context);\n if(rc != SQLITE_OK) {\n const char * zErr = sqlite3_mprintf(\n \"Could not extract metadata value for column %.*s at rowid %lld\",\n pVtab->metadata_columns[metadata_idx].name_length,\n pVtab->metadata_columns[metadata_idx].name, rowid\n );\n if(zErr) {\n sqlite3_result_error(context, zErr, -1);\n sqlite3_free((void *) zErr);\n }else {\n sqlite3_result_error_nomem(context);\n }\n }\n }\n\n return SQLITE_OK;\n}\n\nstatic int vec0Column(sqlite3_vtab_cursor *cur, sqlite3_context *context,\n int i) {\n vec0_cursor *pCur = (vec0_cursor *)cur;\n vec0_vtab *pVtab = (vec0_vtab *)cur->pVtab;\n switch (pCur->query_plan) {\n case VEC0_QUERY_PLAN_FULLSCAN: {\n return vec0Column_fullscan(pVtab, pCur, context, i);\n }\n case VEC0_QUERY_PLAN_KNN: {\n return vec0Column_knn(pVtab, pCur, context, i);\n }\n case VEC0_QUERY_PLAN_POINT: {\n return vec0Column_point(pVtab, pCur, context, i);\n }\n }\n return SQLITE_OK;\n}\n\n/**\n * @brief Handles the \"insert rowid\" step of a row insert operation of a vec0\n * table.\n *\n * This function will insert a new row into the _rowids vec0 shadow table.\n *\n * @param p: virtual table\n * @param idValue: Value containing the inserted rowid/id value.\n * @param rowid: Output rowid, will point to the \"real\" i64 rowid\n * value that was inserted\n * @return int SQLITE_OK on success, error code on failure\n */\nint vec0Update_InsertRowidStep(vec0_vtab *p, sqlite3_value *idValue,\n i64 *rowid) {\n\n /**\n * An insert into a vec0 table can happen a few different ways:\n * 1) With default INTEGER primary key: With a supplied i64 rowid\n * 2) With default INTEGER primary key: WITHOUT a supplied rowid\n * 3) With TEXT primary key: supplied text rowid\n */\n\n int rc;\n\n // Option 3: vtab has a user-defined TEXT primary key, so ensure a text value\n // is provided.\n if (p->pkIsText) {\n if (sqlite3_value_type(idValue) != SQLITE_TEXT) {\n // IMP: V04200_21039\n vtab_set_error(&p->base,\n \"The %s virtual table was declared with a TEXT primary \"\n \"key, but a non-TEXT value was provided in an INSERT.\",\n p->tableName);\n return SQLITE_ERROR;\n }\n\n return vec0_rowids_insert_id(p, idValue, rowid);\n }\n\n // Option 1: User supplied a i64 rowid\n if (sqlite3_value_type(idValue) == SQLITE_INTEGER) {\n i64 suppliedRowid = sqlite3_value_int64(idValue);\n rc = vec0_rowids_insert_rowid(p, suppliedRowid);\n if (rc == SQLITE_OK) {\n *rowid = suppliedRowid;\n }\n return rc;\n }\n\n // Option 2: User did not suppled a rowid\n\n if (sqlite3_value_type(idValue) != SQLITE_NULL) {\n // IMP: V30855_14925\n vtab_set_error(&p->base,\n \"Only integers are allows for primary key values on %s\",\n p->tableName);\n return SQLITE_ERROR;\n }\n // NULL to get next auto-incremented value\n return vec0_rowids_insert_id(p, NULL, rowid);\n}\n\n/**\n * @brief Determines the \"next available\" chunk position for a newly inserted\n * vec0 row.\n *\n * This operation may insert a new \"blank\" chunk the _chunks table, if there is\n * no more space in previous chunks.\n *\n * @param p: virtual table\n * @param partitionKeyValues: array of partition key column values, to constrain\n * against any partition key columns.\n * @param chunk_rowid: Output rowid of the chunk in the _chunks virtual table\n * that has the avialabiity.\n * @param chunk_offset: Output the index of the available space insert the\n * chunk, based on the index of the first available validity bit.\n * @param pBlobValidity: Output blob of the validity column of the available\n * chunk. Will be opened with read/write permissions.\n * @param pValidity: Output buffer of the original chunk's validity column.\n * Needs to be cleaned up with sqlite3_free().\n * @return int SQLITE_OK on success, error code on failure\n */\nint vec0Update_InsertNextAvailableStep(\n vec0_vtab *p,\n sqlite3_value ** partitionKeyValues,\n i64 *chunk_rowid, i64 *chunk_offset,\n sqlite3_blob **blobChunksValidity,\n const unsigned char **bufferChunksValidity) {\n\n int rc;\n i64 validitySize;\n *chunk_offset = -1;\n\n rc = vec0_get_latest_chunk_rowid(p, chunk_rowid, partitionKeyValues);\n if(rc == SQLITE_EMPTY) {\n goto done;\n }\n if (rc != SQLITE_OK) {\n goto cleanup;\n }\n\n rc = sqlite3_blob_open(p->db, p->schemaName, p->shadowChunksName, \"validity\",\n *chunk_rowid, 1, blobChunksValidity);\n if (rc != SQLITE_OK) {\n // IMP: V22053_06123\n vtab_set_error(&p->base,\n VEC_INTERAL_ERROR\n \"could not open validity blob on %s.%s.%lld\",\n p->schemaName, p->shadowChunksName, *chunk_rowid);\n goto cleanup;\n }\n\n validitySize = sqlite3_blob_bytes(*blobChunksValidity);\n if (validitySize != p->chunk_size / CHAR_BIT) {\n // IMP: V29362_13432\n vtab_set_error(&p->base,\n VEC_INTERAL_ERROR\n \"validity blob size mismatch on \"\n \"%s.%s.%lld, expected %lld but received %lld.\",\n p->schemaName, p->shadowChunksName, *chunk_rowid,\n (i64)(p->chunk_size / CHAR_BIT), validitySize);\n rc = SQLITE_ERROR;\n goto cleanup;\n }\n\n *bufferChunksValidity = sqlite3_malloc(validitySize);\n if (!(*bufferChunksValidity)) {\n vtab_set_error(&p->base, VEC_INTERAL_ERROR\n \"Could not allocate memory for validity bitmap\");\n rc = SQLITE_NOMEM;\n goto cleanup;\n }\n\n rc = sqlite3_blob_read(*blobChunksValidity, (void *)*bufferChunksValidity,\n validitySize, 0);\n\n if (rc != SQLITE_OK) {\n vtab_set_error(&p->base,\n VEC_INTERAL_ERROR\n \"Could not read validity bitmap for %s.%s.%lld\",\n p->schemaName, p->shadowChunksName, *chunk_rowid);\n goto cleanup;\n }\n\n // find the next available offset, ie first `0` in the bitmap.\n for (int i = 0; i < validitySize; i++) {\n if ((*bufferChunksValidity)[i] == 0b11111111)\n continue;\n for (int j = 0; j < CHAR_BIT; j++) {\n if (((((*bufferChunksValidity)[i] >> j) & 1) == 0)) {\n *chunk_offset = (i * CHAR_BIT) + j;\n goto done;\n }\n }\n }\n\ndone:\n // latest chunk was full, so need to create a new one\n if (*chunk_offset == -1) {\n rc = vec0_new_chunk(p, partitionKeyValues, chunk_rowid);\n if (rc != SQLITE_OK) {\n // IMP: V08441_25279\n vtab_set_error(&p->base,\n VEC_INTERAL_ERROR \"Could not insert a new vector chunk\");\n rc = SQLITE_ERROR; // otherwise raises a DatabaseError and not operational\n // error?\n goto cleanup;\n }\n *chunk_offset = 0;\n\n // blobChunksValidity and pValidity are stale, pointing to the previous\n // (full) chunk. to re-assign them\n rc = sqlite3_blob_close(*blobChunksValidity);\n sqlite3_free((void *)*bufferChunksValidity);\n *blobChunksValidity = NULL;\n *bufferChunksValidity = NULL;\n if (rc != SQLITE_OK) {\n vtab_set_error(&p->base, VEC_INTERAL_ERROR\n \"unknown error, blobChunksValidity could not be closed, \"\n \"please file an issue.\");\n rc = SQLITE_ERROR;\n goto cleanup;\n }\n\n rc = sqlite3_blob_open(p->db, p->schemaName, p->shadowChunksName,\n \"validity\", *chunk_rowid, 1, blobChunksValidity);\n if (rc != SQLITE_OK) {\n vtab_set_error(\n &p->base,\n VEC_INTERAL_ERROR\n \"Could not open validity blob for newly created chunk %s.%s.%lld\",\n p->schemaName, p->shadowChunksName, *chunk_rowid);\n goto cleanup;\n }\n validitySize = sqlite3_blob_bytes(*blobChunksValidity);\n if (validitySize != p->chunk_size / CHAR_BIT) {\n vtab_set_error(&p->base,\n VEC_INTERAL_ERROR\n \"validity blob size mismatch for newly created chunk \"\n \"%s.%s.%lld. Exepcted %lld, got %lld\",\n p->schemaName, p->shadowChunksName, *chunk_rowid,\n p->chunk_size / CHAR_BIT, validitySize);\n goto cleanup;\n }\n *bufferChunksValidity = sqlite3_malloc(validitySize);\n rc = sqlite3_blob_read(*blobChunksValidity, (void *)*bufferChunksValidity,\n validitySize, 0);\n if (rc != SQLITE_OK) {\n vtab_set_error(&p->base,\n VEC_INTERAL_ERROR\n \"could not read validity blob newly created chunk \"\n \"%s.%s.%lld\",\n p->schemaName, p->shadowChunksName, *chunk_rowid);\n goto cleanup;\n }\n }\n\n rc = SQLITE_OK;\n\ncleanup:\n return rc;\n}\n\n/**\n * @brief Write the vector data into the provided vector blob at the given\n * offset\n *\n * @param blobVectors SQLite BLOB to write to\n * @param chunk_offset the \"offset\" (ie validity bitmap position) to write the\n * vector to\n * @param bVector pointer to the vector containing data\n * @param dimensions how many dimensions the vector has\n * @param element_type the vector type\n * @return result of sqlite3_blob_write, SQLITE_OK on success, otherwise failure\n */\nstatic int\nvec0_write_vector_to_vector_blob(sqlite3_blob *blobVectors, i64 chunk_offset,\n const void *bVector, size_t dimensions,\n enum VectorElementType element_type) {\n int n;\n int offset;\n\n switch (element_type) {\n case SQLITE_VEC_ELEMENT_TYPE_FLOAT32:\n n = dimensions * sizeof(f32);\n offset = chunk_offset * dimensions * sizeof(f32);\n break;\n case SQLITE_VEC_ELEMENT_TYPE_INT8:\n n = dimensions * sizeof(i8);\n offset = chunk_offset * dimensions * sizeof(i8);\n break;\n case SQLITE_VEC_ELEMENT_TYPE_BIT:\n n = dimensions / CHAR_BIT;\n offset = chunk_offset * dimensions / CHAR_BIT;\n break;\n }\n\n return sqlite3_blob_write(blobVectors, bVector, n, offset);\n}\n\n/**\n * @brief\n *\n * @param p vec0 virtual table\n * @param chunk_rowid: which chunk to write to\n * @param chunk_offset: the offset inside the chunk to write the vector to.\n * @param rowid: the rowid of the inserting row\n * @param vectorDatas: array of the vector data to insert\n * @param blobValidity: writeable validity blob of the row's assigned chunk.\n * @param validity: snapshot buffer of the valdity column from the row's\n * assigned chunk.\n * @return int SQLITE_OK on success, error code on failure\n */\nint vec0Update_InsertWriteFinalStep(vec0_vtab *p, i64 chunk_rowid,\n i64 chunk_offset, i64 rowid,\n void *vectorDatas[],\n sqlite3_blob *blobChunksValidity,\n const unsigned char *bufferChunksValidity) {\n int rc, brc;\n sqlite3_blob *blobChunksRowids = NULL;\n\n // mark the validity bit for this row in the chunk's validity bitmap\n // Get the byte offset of the bitmap\n char unsigned bx = bufferChunksValidity[chunk_offset / CHAR_BIT];\n // set the bit at the chunk_offset position inside that byte\n bx = bx | (1 << (chunk_offset % CHAR_BIT));\n // write that 1 byte\n rc = sqlite3_blob_write(blobChunksValidity, &bx, 1, chunk_offset / CHAR_BIT);\n if (rc != SQLITE_OK) {\n vtab_set_error(&p->base, VEC_INTERAL_ERROR \"could not mark validity bit \");\n return rc;\n }\n\n // Go insert the vector data into the vector chunk shadow tables\n for (int i = 0; i < p->numVectorColumns; i++) {\n sqlite3_blob *blobVectors;\n rc = sqlite3_blob_open(p->db, p->schemaName, p->shadowVectorChunksNames[i],\n \"vectors\", chunk_rowid, 1, &blobVectors);\n if (rc != SQLITE_OK) {\n vtab_set_error(&p->base, \"Error opening vector blob at %s.%s.%lld\",\n p->schemaName, p->shadowVectorChunksNames[i], chunk_rowid);\n goto cleanup;\n }\n\n i64 expected =\n p->chunk_size * vector_column_byte_size(p->vector_columns[i]);\n i64 actual = sqlite3_blob_bytes(blobVectors);\n\n if (actual != expected) {\n // IMP: V16386_00456\n vtab_set_error(\n &p->base,\n VEC_INTERAL_ERROR\n \"vector blob size mismatch on %s.%s.%lld. Expected %lld, actual %lld\",\n p->schemaName, p->shadowVectorChunksNames[i], chunk_rowid, expected,\n actual);\n rc = SQLITE_ERROR;\n // already error, can ignore result code\n sqlite3_blob_close(blobVectors);\n goto cleanup;\n };\n\n rc = vec0_write_vector_to_vector_blob(\n blobVectors, chunk_offset, vectorDatas[i],\n p->vector_columns[i].dimensions, p->vector_columns[i].element_type);\n if (rc != SQLITE_OK) {\n vtab_set_error(&p->base,\n VEC_INTERAL_ERROR\n \"could not write vector blob on %s.%s.%lld\",\n p->schemaName, p->shadowVectorChunksNames[i], chunk_rowid);\n rc = SQLITE_ERROR;\n // already error, can ignore result code\n sqlite3_blob_close(blobVectors);\n goto cleanup;\n }\n rc = sqlite3_blob_close(blobVectors);\n if (rc != SQLITE_OK) {\n vtab_set_error(&p->base,\n VEC_INTERAL_ERROR\n \"could not close vector blob on %s.%s.%lld\",\n p->schemaName, p->shadowVectorChunksNames[i], chunk_rowid);\n rc = SQLITE_ERROR;\n goto cleanup;\n }\n }\n\n // write the new rowid to the rowids column of the _chunks table\n rc = sqlite3_blob_open(p->db, p->schemaName, p->shadowChunksName, \"rowids\",\n chunk_rowid, 1, &blobChunksRowids);\n if (rc != SQLITE_OK) {\n // IMP: V09221_26060\n vtab_set_error(&p->base,\n VEC_INTERAL_ERROR \"could not open rowids blob on %s.%s.%lld\",\n p->schemaName, p->shadowChunksName, chunk_rowid);\n goto cleanup;\n }\n i64 expected = p->chunk_size * sizeof(i64);\n i64 actual = sqlite3_blob_bytes(blobChunksRowids);\n if (expected != actual) {\n // IMP: V12779_29618\n vtab_set_error(\n &p->base,\n VEC_INTERAL_ERROR\n \"rowids blob size mismatch on %s.%s.%lld. Expected %lld, actual %lld\",\n p->schemaName, p->shadowChunksName, chunk_rowid, expected, actual);\n rc = SQLITE_ERROR;\n goto cleanup;\n }\n rc = sqlite3_blob_write(blobChunksRowids, &rowid, sizeof(i64),\n chunk_offset * sizeof(i64));\n if (rc != SQLITE_OK) {\n vtab_set_error(\n &p->base, VEC_INTERAL_ERROR \"could not write rowids blob on %s.%s.%lld\",\n p->schemaName, p->shadowChunksName, chunk_rowid);\n rc = SQLITE_ERROR;\n goto cleanup;\n }\n\n // Now with all the vectors inserted, go back and update the _rowids table\n // with the new chunk_rowid/chunk_offset values\n rc = vec0_rowids_update_position(p, rowid, chunk_rowid, chunk_offset);\n\ncleanup:\n brc = sqlite3_blob_close(blobChunksRowids);\n if ((rc == SQLITE_OK) && (brc != SQLITE_OK)) {\n vtab_set_error(\n &p->base, VEC_INTERAL_ERROR \"could not close rowids blob on %s.%s.%lld\",\n p->schemaName, p->shadowChunksName, chunk_rowid);\n return brc;\n }\n return rc;\n}\n\nint vec0_write_metadata_value(vec0_vtab *p, int metadata_column_idx, i64 rowid, i64 chunk_id, i64 chunk_offset, sqlite3_value * v, int isupdate) {\n int rc;\n struct Vec0MetadataColumnDefinition * metadata_column = &p->metadata_columns[metadata_column_idx];\n vec0_metadata_column_kind kind = metadata_column->kind;\n\n // verify input value matches column type\n switch(kind) {\n case VEC0_METADATA_COLUMN_KIND_BOOLEAN: {\n if(sqlite3_value_type(v) != SQLITE_INTEGER || ((sqlite3_value_int(v) != 0) && (sqlite3_value_int(v) != 1))) {\n rc = SQLITE_ERROR;\n vtab_set_error(&p->base, \"Expected 0 or 1 for BOOLEAN metadata column %.*s\", metadata_column->name_length, metadata_column->name);\n goto done;\n }\n break;\n }\n case VEC0_METADATA_COLUMN_KIND_INTEGER: {\n if(sqlite3_value_type(v) != SQLITE_INTEGER) {\n rc = SQLITE_ERROR;\n vtab_set_error(&p->base, \"Expected integer for INTEGER metadata column %.*s, received %s\", metadata_column->name_length, metadata_column->name, type_name(sqlite3_value_type(v)));\n goto done;\n }\n break;\n }\n case VEC0_METADATA_COLUMN_KIND_FLOAT: {\n if(sqlite3_value_type(v) != SQLITE_FLOAT) {\n rc = SQLITE_ERROR;\n vtab_set_error(&p->base, \"Expected float for FLOAT metadata column %.*s, received %s\", metadata_column->name_length, metadata_column->name, type_name(sqlite3_value_type(v)));\n goto done;\n }\n break;\n }\n case VEC0_METADATA_COLUMN_KIND_TEXT: {\n if(sqlite3_value_type(v) != SQLITE_TEXT) {\n rc = SQLITE_ERROR;\n vtab_set_error(&p->base, \"Expected text for TEXT metadata column %.*s, received %s\", metadata_column->name_length, metadata_column->name, type_name(sqlite3_value_type(v)));\n goto done;\n }\n break;\n }\n }\n\n sqlite3_blob * blobValue = NULL;\n rc = sqlite3_blob_open(p->db, p->schemaName, p->shadowMetadataChunksNames[metadata_column_idx], \"data\", chunk_id, 1, &blobValue);\n if(rc != SQLITE_OK) {\n goto done;\n }\n\n switch(kind) {\n case VEC0_METADATA_COLUMN_KIND_BOOLEAN: {\n u8 block;\n int value = sqlite3_value_int(v);\n rc = sqlite3_blob_read(blobValue, &block, sizeof(u8), (int) (chunk_offset / CHAR_BIT));\n if(rc != SQLITE_OK) {\n goto done;\n }\n\n if (value) {\n block |= 1 << (chunk_offset % CHAR_BIT);\n } else {\n block &= ~(1 << (chunk_offset % CHAR_BIT));\n }\n\n rc = sqlite3_blob_write(blobValue, &block, sizeof(u8), chunk_offset / CHAR_BIT);\n break;\n }\n case VEC0_METADATA_COLUMN_KIND_INTEGER: {\n i64 value = sqlite3_value_int64(v);\n rc = sqlite3_blob_write(blobValue, &value, sizeof(value), chunk_offset * sizeof(i64));\n break;\n }\n case VEC0_METADATA_COLUMN_KIND_FLOAT: {\n double value = sqlite3_value_double(v);\n rc = sqlite3_blob_write(blobValue, &value, sizeof(value), chunk_offset * sizeof(double));\n break;\n }\n case VEC0_METADATA_COLUMN_KIND_TEXT: {\n int prev_n;\n rc = sqlite3_blob_read(blobValue, &prev_n, sizeof(int), chunk_offset * VEC0_METADATA_TEXT_VIEW_BUFFER_LENGTH);\n if(rc != SQLITE_OK) {\n goto done;\n }\n\n const char * s = (const char *) sqlite3_value_text(v);\n int n = sqlite3_value_bytes(v);\n u8 view[VEC0_METADATA_TEXT_VIEW_BUFFER_LENGTH];\n memset(view, 0, VEC0_METADATA_TEXT_VIEW_BUFFER_LENGTH);\n memcpy(view, &n, sizeof(int));\n memcpy(view+4, s, min(n, VEC0_METADATA_TEXT_VIEW_BUFFER_LENGTH-4));\n\n rc = sqlite3_blob_write(blobValue, &view, VEC0_METADATA_TEXT_VIEW_BUFFER_LENGTH, chunk_offset * VEC0_METADATA_TEXT_VIEW_BUFFER_LENGTH);\n if(n > VEC0_METADATA_TEXT_VIEW_DATA_LENGTH) {\n const char * zSql;\n\n if(isupdate && (prev_n > VEC0_METADATA_TEXT_VIEW_DATA_LENGTH)) {\n zSql = sqlite3_mprintf(\"UPDATE \" VEC0_SHADOW_METADATA_TEXT_DATA_NAME \" SET data = ?2 WHERE rowid = ?1\", p->schemaName, p->tableName, metadata_column_idx);\n }else {\n zSql = sqlite3_mprintf(\"INSERT INTO \" VEC0_SHADOW_METADATA_TEXT_DATA_NAME \" (rowid, data) VALUES (?1, ?2)\", p->schemaName, p->tableName, metadata_column_idx);\n }\n if(!zSql) {\n rc = SQLITE_NOMEM;\n goto done;\n }\n sqlite3_stmt * stmt;\n rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, NULL);\n if(rc != SQLITE_OK) {\n goto done;\n }\n sqlite3_bind_int64(stmt, 1, rowid);\n sqlite3_bind_text(stmt, 2, s, n, SQLITE_STATIC);\n rc = sqlite3_step(stmt);\n sqlite3_finalize(stmt);\n\n if(rc != SQLITE_DONE) {\n rc = SQLITE_ERROR;\n goto done;\n }\n }\n else if(prev_n > VEC0_METADATA_TEXT_VIEW_DATA_LENGTH) {\n const char * zSql = sqlite3_mprintf(\"DELETE FROM \" VEC0_SHADOW_METADATA_TEXT_DATA_NAME \" WHERE rowid = ?\", p->schemaName, p->tableName, metadata_column_idx);\n if(!zSql) {\n rc = SQLITE_NOMEM;\n goto done;\n }\n sqlite3_stmt * stmt;\n rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, NULL);\n if(rc != SQLITE_OK) {\n goto done;\n }\n sqlite3_bind_int64(stmt, 1, rowid);\n rc = sqlite3_step(stmt);\n sqlite3_finalize(stmt);\n\n if(rc != SQLITE_DONE) {\n rc = SQLITE_ERROR;\n goto done;\n }\n }\n break;\n }\n }\n\n if(rc != SQLITE_OK) {\n\n }\n rc = sqlite3_blob_close(blobValue);\n if(rc != SQLITE_OK) {\n goto done;\n }\n\n done:\n return rc;\n}\n\n\n/**\n * @brief Handles INSERT INTO operations on a vec0 table.\n *\n * @return int SQLITE_OK on success, otherwise error code on failure\n */\nint vec0Update_Insert(sqlite3_vtab *pVTab, int argc, sqlite3_value **argv,\n sqlite_int64 *pRowid) {\n UNUSED_PARAMETER(argc);\n vec0_vtab *p = (vec0_vtab *)pVTab;\n int rc;\n // Rowid for the inserted row, deterimined by the inserted ID + _rowids shadow\n // table\n i64 rowid;\n\n // Array to hold the vector data of the inserted row. Individual elements will\n // have a lifetime bound to the argv[..] values.\n void *vectorDatas[VEC0_MAX_VECTOR_COLUMNS];\n // Array to hold cleanup functions for vectorDatas[]\n vector_cleanup cleanups[VEC0_MAX_VECTOR_COLUMNS];\n\n sqlite3_value * partitionKeyValues[VEC0_MAX_PARTITION_COLUMNS];\n\n // Rowid of the chunk in the _chunks shadow table that the row will be a part\n // of.\n i64 chunk_rowid;\n // offset within the chunk where the rowid belongs\n i64 chunk_offset;\n\n // a write-able blob of the validity column for the given chunk. Used to mark\n // validity bit\n sqlite3_blob *blobChunksValidity = NULL;\n // buffer for the valididty column for the given chunk. Maybe not needed here?\n const unsigned char *bufferChunksValidity = NULL;\n int numReadVectors = 0;\n\n // Read all provided partition key values into partitionKeyValues\n for (int i = 0; i < vec0_num_defined_user_columns(p); i++) {\n if(p->user_column_kinds[i] != SQLITE_VEC0_USER_COLUMN_KIND_PARTITION) {\n continue;\n }\n int partition_key_idx = p->user_column_idxs[i];\n partitionKeyValues[partition_key_idx] = argv[2+VEC0_COLUMN_USERN_START + i];\n\n int new_value_type = sqlite3_value_type(partitionKeyValues[partition_key_idx]);\n if((new_value_type != SQLITE_NULL) && (new_value_type != p->paritition_columns[partition_key_idx].type)) {\n // IMP: V11454_28292\n vtab_set_error(\n pVTab,\n \"Parition key type mismatch: The partition key column %.*s has type %s, but %s was provided.\",\n p->paritition_columns[partition_key_idx].name_length,\n p->paritition_columns[partition_key_idx].name,\n type_name(p->paritition_columns[partition_key_idx].type),\n type_name(new_value_type)\n );\n rc = SQLITE_ERROR;\n goto cleanup;\n }\n }\n\n // read all the inserted vectors into vectorDatas, validate their lengths.\n for (int i = 0; i < vec0_num_defined_user_columns(p); i++) {\n if(p->user_column_kinds[i] != SQLITE_VEC0_USER_COLUMN_KIND_VECTOR) {\n continue;\n }\n int vector_column_idx = p->user_column_idxs[i];\n sqlite3_value *valueVector = argv[2 + VEC0_COLUMN_USERN_START + i];\n size_t dimensions;\n\n char *pzError;\n enum VectorElementType elementType;\n rc = vector_from_value(valueVector, &vectorDatas[vector_column_idx], &dimensions,\n &elementType, &cleanups[vector_column_idx], &pzError);\n if (rc != SQLITE_OK) {\n // IMP: V06519_23358\n vtab_set_error(\n pVTab, \"Inserted vector for the \\\"%.*s\\\" column is invalid: %z\",\n p->vector_columns[vector_column_idx].name_length, p->vector_columns[vector_column_idx].name, pzError);\n rc = SQLITE_ERROR;\n goto cleanup;\n }\n\n numReadVectors++;\n if (elementType != p->vector_columns[vector_column_idx].element_type) {\n // IMP: V08221_25059\n vtab_set_error(\n pVTab,\n \"Inserted vector for the \\\"%.*s\\\" column is expected to be of type \"\n \"%s, but a %s vector was provided.\",\n p->vector_columns[i].name_length, p->vector_columns[i].name,\n vector_subtype_name(p->vector_columns[i].element_type),\n vector_subtype_name(elementType));\n rc = SQLITE_ERROR;\n goto cleanup;\n }\n\n if (dimensions != p->vector_columns[vector_column_idx].dimensions) {\n // IMP: V01145_17984\n vtab_set_error(\n pVTab,\n \"Dimension mismatch for inserted vector for the \\\"%.*s\\\" column. \"\n \"Expected %d dimensions but received %d.\",\n p->vector_columns[vector_column_idx].name_length, p->vector_columns[vector_column_idx].name,\n p->vector_columns[vector_column_idx].dimensions, dimensions);\n rc = SQLITE_ERROR;\n goto cleanup;\n }\n }\n\n // Cannot insert a value in the hidden \"distance\" column\n if (sqlite3_value_type(argv[2 + vec0_column_distance_idx(p)]) !=\n SQLITE_NULL) {\n // IMP: V24228_08298\n vtab_set_error(pVTab,\n \"A value was provided for the hidden \\\"distance\\\" column.\");\n rc = SQLITE_ERROR;\n goto cleanup;\n }\n // Cannot insert a value in the hidden \"k\" column\n if (sqlite3_value_type(argv[2 + vec0_column_k_idx(p)]) != SQLITE_NULL) {\n // IMP: V11875_28713\n vtab_set_error(pVTab, \"A value was provided for the hidden \\\"k\\\" column.\");\n rc = SQLITE_ERROR;\n goto cleanup;\n }\n\n // Step #1: Insert/get a rowid for this row, from the _rowids table.\n rc = vec0Update_InsertRowidStep(p, argv[2 + VEC0_COLUMN_ID], &rowid);\n if (rc != SQLITE_OK) {\n goto cleanup;\n }\n\n // Step #2: Find the next \"available\" position in the _chunks table for this\n // row.\n rc = vec0Update_InsertNextAvailableStep(p, partitionKeyValues,\n &chunk_rowid, &chunk_offset,\n &blobChunksValidity,\n &bufferChunksValidity);\n if (rc != SQLITE_OK) {\n goto cleanup;\n }\n\n // Step #3: With the next available chunk position, write out all the vectors\n // to their specified location.\n rc = vec0Update_InsertWriteFinalStep(p, chunk_rowid, chunk_offset, rowid,\n vectorDatas, blobChunksValidity,\n bufferChunksValidity);\n if (rc != SQLITE_OK) {\n goto cleanup;\n }\n\n if(p->numAuxiliaryColumns > 0) {\n sqlite3_stmt *stmt;\n sqlite3_str * s = sqlite3_str_new(NULL);\n sqlite3_str_appendf(s, \"INSERT INTO \" VEC0_SHADOW_AUXILIARY_NAME \"(rowid \", p->schemaName, p->tableName);\n for(int i = 0; i < p->numAuxiliaryColumns; i++) {\n sqlite3_str_appendf(s, \", value%02d\", i);\n }\n sqlite3_str_appendall(s, \") VALUES (? \");\n for(int i = 0; i < p->numAuxiliaryColumns; i++) {\n sqlite3_str_appendall(s, \", ?\");\n }\n sqlite3_str_appendall(s, \")\");\n char * zSql = sqlite3_str_finish(s);\n // TODO double check error handling ehre\n if(!zSql) {\n rc = SQLITE_NOMEM;\n goto cleanup;\n }\n rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, NULL);\n if(rc != SQLITE_OK) {\n goto cleanup;\n }\n sqlite3_bind_int64(stmt, 1, rowid);\n\n for (int i = 0; i < vec0_num_defined_user_columns(p); i++) {\n if(p->user_column_kinds[i] != SQLITE_VEC0_USER_COLUMN_KIND_AUXILIARY) {\n continue;\n }\n int auxiliary_key_idx = p->user_column_idxs[i];\n sqlite3_value * v = argv[2+VEC0_COLUMN_USERN_START + i];\n int v_type = sqlite3_value_type(v);\n if(v_type != SQLITE_NULL && (v_type != p->auxiliary_columns[auxiliary_key_idx].type)) {\n sqlite3_finalize(stmt);\n rc = SQLITE_CONSTRAINT;\n vtab_set_error(\n pVTab,\n \"Auxiliary column type mismatch: The auxiliary column %.*s has type %s, but %s was provided.\",\n p->auxiliary_columns[auxiliary_key_idx].name_length,\n p->auxiliary_columns[auxiliary_key_idx].name,\n type_name(p->auxiliary_columns[auxiliary_key_idx].type),\n type_name(v_type)\n );\n goto cleanup;\n }\n // first 1 is for 1-based indexing on sqlite3_bind_*, second 1 is to account for initial rowid parameter\n sqlite3_bind_value(stmt, 1 + 1 + auxiliary_key_idx, v);\n }\n\n rc = sqlite3_step(stmt);\n if(rc != SQLITE_DONE) {\n sqlite3_finalize(stmt);\n rc = SQLITE_ERROR;\n goto cleanup;\n }\n sqlite3_finalize(stmt);\n }\n\n\n for(int i = 0; i < vec0_num_defined_user_columns(p); i++) {\n if(p->user_column_kinds[i] != SQLITE_VEC0_USER_COLUMN_KIND_METADATA) {\n continue;\n }\n int metadata_idx = p->user_column_idxs[i];\n sqlite3_value *v = argv[2 + VEC0_COLUMN_USERN_START + i];\n rc = vec0_write_metadata_value(p, metadata_idx, rowid, chunk_rowid, chunk_offset, v, 0);\n if(rc != SQLITE_OK) {\n goto cleanup;\n }\n }\n\n *pRowid = rowid;\n rc = SQLITE_OK;\n\ncleanup:\n for (int i = 0; i < numReadVectors; i++) {\n cleanups[i](vectorDatas[i]);\n }\n sqlite3_free((void *)bufferChunksValidity);\n int brc = sqlite3_blob_close(blobChunksValidity);\n if ((rc == SQLITE_OK) && (brc != SQLITE_OK)) {\n vtab_set_error(&p->base,\n VEC_INTERAL_ERROR \"unknown error, blobChunksValidity could \"\n \"not be closed, please file an issue\");\n return brc;\n }\n return rc;\n}\n\nint vec0Update_Delete_ClearValidity(vec0_vtab *p, i64 chunk_id,\n u64 chunk_offset) {\n int rc, brc;\n sqlite3_blob *blobChunksValidity = NULL;\n char unsigned bx;\n int validityOffset = chunk_offset / CHAR_BIT;\n\n // 2. ensure chunks.validity bit is 1, then set to 0\n rc = sqlite3_blob_open(p->db, p->schemaName, p->shadowChunksName, \"validity\",\n chunk_id, 1, &blobChunksValidity);\n if (rc != SQLITE_OK) {\n // IMP: V26002_10073\n vtab_set_error(&p->base, \"could not open validity blob for %s.%s.%lld\",\n p->schemaName, p->shadowChunksName, chunk_id);\n return SQLITE_ERROR;\n }\n // will skip the sqlite3_blob_bytes(blobChunksValidity) check for now,\n // the read below would catch it\n\n rc = sqlite3_blob_read(blobChunksValidity, &bx, sizeof(bx), validityOffset);\n if (rc != SQLITE_OK) {\n // IMP: V21193_05263\n vtab_set_error(\n &p->base, \"could not read validity blob for %s.%s.%lld at %d\",\n p->schemaName, p->shadowChunksName, chunk_id, validityOffset);\n goto cleanup;\n }\n if (!(bx >> (chunk_offset % CHAR_BIT))) {\n // IMP: V21193_05263\n rc = SQLITE_ERROR;\n vtab_set_error(\n &p->base,\n \"vec0 deletion error: validity bit is not set for %s.%s.%lld at %d\",\n p->schemaName, p->shadowChunksName, chunk_id, validityOffset);\n goto cleanup;\n }\n char unsigned mask = ~(1 << (chunk_offset % CHAR_BIT));\n char result = bx & mask;\n rc = sqlite3_blob_write(blobChunksValidity, &result, sizeof(bx),\n validityOffset);\n if (rc != SQLITE_OK) {\n vtab_set_error(\n &p->base, \"could not write to validity blob for %s.%s.%lld at %d\",\n p->schemaName, p->shadowChunksName, chunk_id, validityOffset);\n goto cleanup;\n }\n\ncleanup:\n\n brc = sqlite3_blob_close(blobChunksValidity);\n if (rc != SQLITE_OK)\n return rc;\n if (brc != SQLITE_OK) {\n vtab_set_error(&p->base,\n \"vec0 deletion error: Error commiting validity blob \"\n \"transaction on %s.%s.%lld at %d\",\n p->schemaName, p->shadowChunksName, chunk_id,\n validityOffset);\n return brc;\n }\n return SQLITE_OK;\n}\n\nint vec0Update_Delete_ClearRowid(vec0_vtab *p, i64 chunk_id,\n u64 chunk_offset) {\n int rc, brc;\n sqlite3_blob *blobChunksRowids = NULL;\n i64 zero = 0;\n\n rc = sqlite3_blob_open(p->db, p->schemaName, p->shadowChunksName, \"rowids\",\n chunk_id, 1, &blobChunksRowids);\n if (rc != SQLITE_OK) {\n vtab_set_error(&p->base, \"could not open rowids blob for %s.%s.%lld\",\n p->schemaName, p->shadowChunksName, chunk_id);\n return SQLITE_ERROR;\n }\n\n rc = sqlite3_blob_write(blobChunksRowids, &zero, sizeof(zero),\n chunk_offset * sizeof(i64));\n if (rc != SQLITE_OK) {\n vtab_set_error(&p->base,\n \"could not write to rowids blob for %s.%s.%lld at %llu\",\n p->schemaName, p->shadowChunksName, chunk_id, chunk_offset);\n }\n\n brc = sqlite3_blob_close(blobChunksRowids);\n if (rc != SQLITE_OK)\n return rc;\n if (brc != SQLITE_OK) {\n vtab_set_error(&p->base,\n \"vec0 deletion error: Error commiting rowids blob \"\n \"transaction on %s.%s.%lld at %llu\",\n p->schemaName, p->shadowChunksName, chunk_id, chunk_offset);\n return brc;\n }\n return SQLITE_OK;\n}\n\nint vec0Update_Delete_ClearVectors(vec0_vtab *p, i64 chunk_id,\n u64 chunk_offset) {\n int rc, brc;\n for (int i = 0; i < p->numVectorColumns; i++) {\n sqlite3_blob *blobVectors = NULL;\n size_t n = vector_column_byte_size(p->vector_columns[i]);\n\n rc = sqlite3_blob_open(p->db, p->schemaName,\n p->shadowVectorChunksNames[i], \"vectors\",\n chunk_id, 1, &blobVectors);\n if (rc != SQLITE_OK) {\n vtab_set_error(&p->base,\n \"could not open vector blob for %s.%s.%lld column %d\",\n p->schemaName, p->shadowVectorChunksNames[i], chunk_id, i);\n return SQLITE_ERROR;\n }\n\n void *zeroBuf = sqlite3_malloc(n);\n if (!zeroBuf) {\n sqlite3_blob_close(blobVectors);\n return SQLITE_NOMEM;\n }\n memset(zeroBuf, 0, n);\n\n rc = sqlite3_blob_write(blobVectors, zeroBuf, n, chunk_offset * n);\n sqlite3_free(zeroBuf);\n if (rc != SQLITE_OK) {\n vtab_set_error(\n &p->base,\n \"could not write to vector blob for %s.%s.%lld at %llu column %d\",\n p->schemaName, p->shadowVectorChunksNames[i], chunk_id,\n chunk_offset, i);\n }\n\n brc = sqlite3_blob_close(blobVectors);\n if (rc != SQLITE_OK)\n return rc;\n if (brc != SQLITE_OK) {\n vtab_set_error(&p->base,\n \"vec0 deletion error: Error commiting vector blob \"\n \"transaction on %s.%s.%lld column %d\",\n p->schemaName, p->shadowVectorChunksNames[i], chunk_id, i);\n return brc;\n }\n }\n return SQLITE_OK;\n}\n\nint vec0Update_Delete_DeleteChunkIfEmpty(vec0_vtab *p, i64 chunk_id,\n int *deleted) {\n int rc, brc;\n sqlite3_blob *blobValidity = NULL;\n *deleted = 0;\n\n rc = sqlite3_blob_open(p->db, p->schemaName, p->shadowChunksName, \"validity\",\n chunk_id, 0, &blobValidity);\n if (rc != SQLITE_OK) {\n vtab_set_error(&p->base,\n \"could not open validity blob for chunk %lld\", chunk_id);\n return SQLITE_ERROR;\n }\n\n int validitySize = sqlite3_blob_bytes(blobValidity);\n unsigned char *validityBuf = sqlite3_malloc(validitySize);\n if (!validityBuf) {\n sqlite3_blob_close(blobValidity);\n return SQLITE_NOMEM;\n }\n\n rc = sqlite3_blob_read(blobValidity, validityBuf, validitySize, 0);\n brc = sqlite3_blob_close(blobValidity);\n if (rc != SQLITE_OK) {\n sqlite3_free(validityBuf);\n return rc;\n }\n if (brc != SQLITE_OK) {\n sqlite3_free(validityBuf);\n return brc;\n }\n\n int allZero = 1;\n for (int i = 0; i < validitySize; i++) {\n if (validityBuf[i] != 0) {\n allZero = 0;\n break;\n }\n }\n sqlite3_free(validityBuf);\n\n if (!allZero) {\n return SQLITE_OK;\n }\n\n // All validity bits are zero — delete this chunk and its associated data\n char *zSql;\n sqlite3_stmt *stmt;\n\n // Delete from _chunks\n zSql = sqlite3_mprintf(\n \"DELETE FROM \" VEC0_SHADOW_CHUNKS_NAME \" WHERE rowid = ?\",\n p->schemaName, p->tableName);\n if (!zSql)\n return SQLITE_NOMEM;\n rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, NULL);\n sqlite3_free(zSql);\n if (rc != SQLITE_OK)\n return rc;\n sqlite3_bind_int64(stmt, 1, chunk_id);\n rc = sqlite3_step(stmt);\n sqlite3_finalize(stmt);\n if (rc != SQLITE_DONE)\n return SQLITE_ERROR;\n\n // Delete from each _vector_chunksNN\n for (int i = 0; i < p->numVectorColumns; i++) {\n zSql = sqlite3_mprintf(\n \"DELETE FROM \" VEC0_SHADOW_VECTOR_N_NAME \" WHERE rowid = ?\",\n p->schemaName, p->tableName, i);\n if (!zSql)\n return SQLITE_NOMEM;\n rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, NULL);\n sqlite3_free(zSql);\n if (rc != SQLITE_OK)\n return rc;\n sqlite3_bind_int64(stmt, 1, chunk_id);\n rc = sqlite3_step(stmt);\n sqlite3_finalize(stmt);\n if (rc != SQLITE_DONE)\n return SQLITE_ERROR;\n }\n\n // Delete from each _metadatachunksNN\n for (int i = 0; i < p->numMetadataColumns; i++) {\n zSql = sqlite3_mprintf(\n \"DELETE FROM \" VEC0_SHADOW_METADATA_N_NAME \" WHERE rowid = ?\",\n p->schemaName, p->tableName, i);\n if (!zSql)\n return SQLITE_NOMEM;\n rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, NULL);\n sqlite3_free(zSql);\n if (rc != SQLITE_OK)\n return rc;\n sqlite3_bind_int64(stmt, 1, chunk_id);\n rc = sqlite3_step(stmt);\n sqlite3_finalize(stmt);\n if (rc != SQLITE_DONE)\n return SQLITE_ERROR;\n }\n\n // Invalidate cached stmtLatestChunk so it gets re-prepared on next insert\n if (p->stmtLatestChunk) {\n sqlite3_finalize(p->stmtLatestChunk);\n p->stmtLatestChunk = NULL;\n }\n\n *deleted = 1;\n return SQLITE_OK;\n}\n\nint vec0Update_Delete_DeleteRowids(vec0_vtab *p, i64 rowid) {\n int rc;\n sqlite3_stmt *stmt = NULL;\n\n char *zSql =\n sqlite3_mprintf(\"DELETE FROM \" VEC0_SHADOW_ROWIDS_NAME \" WHERE rowid = ?\",\n p->schemaName, p->tableName);\n if (!zSql) {\n return SQLITE_NOMEM;\n }\n\n rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, NULL);\n sqlite3_free(zSql);\n if (rc != SQLITE_OK) {\n goto cleanup;\n }\n sqlite3_bind_int64(stmt, 1, rowid);\n rc = sqlite3_step(stmt);\n if (rc != SQLITE_DONE) {\n goto cleanup;\n }\n rc = SQLITE_OK;\n\ncleanup:\n sqlite3_finalize(stmt);\n return rc;\n}\n\nint vec0Update_Delete_DeleteAux(vec0_vtab *p, i64 rowid) {\n int rc;\n sqlite3_stmt *stmt = NULL;\n\n char *zSql =\n sqlite3_mprintf(\"DELETE FROM \" VEC0_SHADOW_AUXILIARY_NAME \" WHERE rowid = ?\",\n p->schemaName, p->tableName);\n if (!zSql) {\n return SQLITE_NOMEM;\n }\n\n rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, NULL);\n sqlite3_free(zSql);\n if (rc != SQLITE_OK) {\n goto cleanup;\n }\n sqlite3_bind_int64(stmt, 1, rowid);\n rc = sqlite3_step(stmt);\n if (rc != SQLITE_DONE) {\n goto cleanup;\n }\n rc = SQLITE_OK;\n\ncleanup:\n sqlite3_finalize(stmt);\n return rc;\n}\n\nint vec0Update_Delete_ClearMetadata(vec0_vtab *p, int metadata_idx, i64 rowid, i64 chunk_id,\n u64 chunk_offset) {\n int rc;\n sqlite3_blob * blobValue;\n vec0_metadata_column_kind kind = p->metadata_columns[metadata_idx].kind;\n rc = sqlite3_blob_open(p->db, p->schemaName, p->shadowMetadataChunksNames[metadata_idx], \"data\", chunk_id, 1, &blobValue);\n if(rc != SQLITE_OK) {\n return rc;\n }\n\n switch(kind) {\n case VEC0_METADATA_COLUMN_KIND_BOOLEAN: {\n u8 block;\n rc = sqlite3_blob_read(blobValue, &block, sizeof(u8), (int) (chunk_offset / CHAR_BIT));\n if(rc != SQLITE_OK) {\n goto done;\n }\n\n block &= ~(1 << (chunk_offset % CHAR_BIT));\n rc = sqlite3_blob_write(blobValue, &block, sizeof(u8), chunk_offset / CHAR_BIT);\n break;\n }\n case VEC0_METADATA_COLUMN_KIND_INTEGER: {\n i64 v = 0;\n rc = sqlite3_blob_write(blobValue, &v, sizeof(v), chunk_offset * sizeof(i64));\n break;\n }\n case VEC0_METADATA_COLUMN_KIND_FLOAT: {\n double v = 0;\n rc = sqlite3_blob_write(blobValue, &v, sizeof(v), chunk_offset * sizeof(double));\n break;\n }\n case VEC0_METADATA_COLUMN_KIND_TEXT: {\n int n;\n rc = sqlite3_blob_read(blobValue, &n, sizeof(int), chunk_offset * VEC0_METADATA_TEXT_VIEW_BUFFER_LENGTH);\n if(rc != SQLITE_OK) {\n goto done;\n }\n\n u8 view[VEC0_METADATA_TEXT_VIEW_BUFFER_LENGTH];\n memset(view, 0, VEC0_METADATA_TEXT_VIEW_BUFFER_LENGTH);\n rc = sqlite3_blob_write(blobValue, &view, sizeof(view), chunk_offset * VEC0_METADATA_TEXT_VIEW_BUFFER_LENGTH);\n if(rc != SQLITE_OK) {\n goto done;\n }\n\n if(n > VEC0_METADATA_TEXT_VIEW_DATA_LENGTH) {\n const char * zSql = sqlite3_mprintf(\"DELETE FROM \" VEC0_SHADOW_METADATA_TEXT_DATA_NAME \" WHERE rowid = ?\", p->schemaName, p->tableName, metadata_idx);\n if(!zSql) {\n rc = SQLITE_NOMEM;\n goto done;\n }\n sqlite3_stmt * stmt;\n rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, NULL);\n if(rc != SQLITE_OK) {\n goto done;\n }\n sqlite3_bind_int64(stmt, 1, rowid);\n rc = sqlite3_step(stmt);\n if(rc != SQLITE_DONE) {\n rc = SQLITE_ERROR;\n goto done;\n }\n sqlite3_finalize(stmt);\n }\n break;\n }\n }\n int rc2;\n done:\n rc2 = sqlite3_blob_close(blobValue);\n if(rc == SQLITE_OK) {\n return rc2;\n }\n return rc;\n}\n\nint vec0Update_Delete(sqlite3_vtab *pVTab, sqlite3_value *idValue) {\n vec0_vtab *p = (vec0_vtab *)pVTab;\n int rc;\n i64 rowid;\n i64 chunk_id;\n i64 chunk_offset;\n\n if (p->pkIsText) {\n rc = vec0_rowid_from_id(p, idValue, &rowid);\n if (rc != SQLITE_OK) {\n return rc;\n }\n } else {\n rowid = sqlite3_value_int64(idValue);\n }\n\n // 1. Find chunk position for given rowid\n // 2. Ensure that validity bit for position is 1, then set to 0\n // 3. Zero out rowid in chunks.rowid\n // 4. Zero out vector data in all vector column chunks\n // 5. Delete value in _rowids table\n\n // 1. get chunk_id and chunk_offset from _rowids\n rc = vec0_get_chunk_position(p, rowid, NULL, &chunk_id, &chunk_offset);\n if (rc != SQLITE_OK) {\n return rc;\n }\n\n // 2. clear validity bit\n rc = vec0Update_Delete_ClearValidity(p, chunk_id, chunk_offset);\n if (rc != SQLITE_OK) {\n return rc;\n }\n\n // 3. zero out rowid in chunks.rowids\n rc = vec0Update_Delete_ClearRowid(p, chunk_id, chunk_offset);\n if (rc != SQLITE_OK) {\n return rc;\n }\n\n // 4. zero out any data in vector chunks tables\n rc = vec0Update_Delete_ClearVectors(p, chunk_id, chunk_offset);\n if (rc != SQLITE_OK) {\n return rc;\n }\n\n // 5. delete from _rowids table\n rc = vec0Update_Delete_DeleteRowids(p, rowid);\n if (rc != SQLITE_OK) {\n return rc;\n }\n\n // 6. delete any auxiliary rows\n if(p->numAuxiliaryColumns > 0) {\n rc = vec0Update_Delete_DeleteAux(p, rowid);\n if (rc != SQLITE_OK) {\n return rc;\n }\n }\n\n // 7. delete metadata\n for(int i = 0; i < p->numMetadataColumns; i++) {\n rc = vec0Update_Delete_ClearMetadata(p, i, rowid, chunk_id, chunk_offset);\n if (rc != SQLITE_OK) {\n return rc;\n }\n }\n\n // 8. reclaim chunk if fully empty\n {\n int chunkDeleted;\n rc = vec0Update_Delete_DeleteChunkIfEmpty(p, chunk_id, &chunkDeleted);\n if (rc != SQLITE_OK) {\n return rc;\n }\n }\n\n return SQLITE_OK;\n}\n\nint vec0Update_UpdateAuxColumn(vec0_vtab *p, int auxiliary_column_idx, sqlite3_value * value, i64 rowid) {\n int rc;\n sqlite3_stmt *stmt;\n const char * zSql = sqlite3_mprintf(\"UPDATE \" VEC0_SHADOW_AUXILIARY_NAME \" SET value%02d = ? WHERE rowid = ?\", p->schemaName, p->tableName, auxiliary_column_idx);\n if(!zSql) {\n return SQLITE_NOMEM;\n }\n rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, NULL);\n if(rc != SQLITE_OK) {\n return rc;\n }\n sqlite3_bind_value(stmt, 1, value);\n sqlite3_bind_int64(stmt, 2, rowid);\n rc = sqlite3_step(stmt);\n if(rc != SQLITE_DONE) {\n sqlite3_finalize(stmt);\n return SQLITE_ERROR;\n }\n sqlite3_finalize(stmt);\n return SQLITE_OK;\n}\n\nint vec0Update_UpdateVectorColumn(vec0_vtab *p, i64 chunk_id, i64 chunk_offset,\n int i, sqlite3_value *valueVector) {\n int rc;\n\n sqlite3_blob *blobVectors = NULL;\n\n char *pzError;\n size_t dimensions;\n enum VectorElementType elementType;\n void *vector;\n vector_cleanup cleanup = vector_cleanup_noop;\n // https://github.com/asg017/sqlite-vec/issues/53\n rc = vector_from_value(valueVector, &vector, &dimensions, &elementType,\n &cleanup, &pzError);\n if (rc != SQLITE_OK) {\n // IMP: V15203_32042\n vtab_set_error(\n &p->base, \"Updated vector for the \\\"%.*s\\\" column is invalid: %z\",\n p->vector_columns[i].name_length, p->vector_columns[i].name, pzError);\n rc = SQLITE_ERROR;\n goto cleanup;\n }\n if (elementType != p->vector_columns[i].element_type) {\n // IMP: V03643_20481\n vtab_set_error(\n &p->base,\n \"Updated vector for the \\\"%.*s\\\" column is expected to be of type \"\n \"%s, but a %s vector was provided.\",\n p->vector_columns[i].name_length, p->vector_columns[i].name,\n vector_subtype_name(p->vector_columns[i].element_type),\n vector_subtype_name(elementType));\n rc = SQLITE_ERROR;\n goto cleanup;\n }\n if (dimensions != p->vector_columns[i].dimensions) {\n // IMP: V25739_09810\n vtab_set_error(\n &p->base,\n \"Dimension mismatch for new updated vector for the \\\"%.*s\\\" column. \"\n \"Expected %d dimensions but received %d.\",\n p->vector_columns[i].name_length, p->vector_columns[i].name,\n p->vector_columns[i].dimensions, dimensions);\n rc = SQLITE_ERROR;\n goto cleanup;\n }\n\n rc = sqlite3_blob_open(p->db, p->schemaName, p->shadowVectorChunksNames[i],\n \"vectors\", chunk_id, 1, &blobVectors);\n if (rc != SQLITE_OK) {\n vtab_set_error(&p->base, \"Could not open vectors blob for %s.%s.%lld\",\n p->schemaName, p->shadowVectorChunksNames[i], chunk_id);\n goto cleanup;\n }\n rc = vec0_write_vector_to_vector_blob(blobVectors, chunk_offset, vector,\n p->vector_columns[i].dimensions,\n p->vector_columns[i].element_type);\n if (rc != SQLITE_OK) {\n vtab_set_error(&p->base, \"Could not write to vectors blob for %s.%s.%lld\",\n p->schemaName, p->shadowVectorChunksNames[i], chunk_id);\n goto cleanup;\n }\n\ncleanup:\n cleanup(vector);\n int brc = sqlite3_blob_close(blobVectors);\n if (rc != SQLITE_OK) {\n return rc;\n }\n if (brc != SQLITE_OK) {\n vtab_set_error(\n &p->base,\n \"Could not commit blob transaction for vectors blob for %s.%s.%lld\",\n p->schemaName, p->shadowVectorChunksNames[i], chunk_id);\n return brc;\n }\n return SQLITE_OK;\n}\n\nint vec0Update_Update(sqlite3_vtab *pVTab, int argc, sqlite3_value **argv) {\n UNUSED_PARAMETER(argc);\n vec0_vtab *p = (vec0_vtab *)pVTab;\n int rc;\n i64 chunk_id;\n i64 chunk_offset;\n\n i64 rowid;\n if (p->pkIsText) {\n const char *a = (const char *)sqlite3_value_text(argv[0]);\n const char *b = (const char *)sqlite3_value_text(argv[1]);\n // IMP: V08886_25725\n if ((sqlite3_value_bytes(argv[0]) != sqlite3_value_bytes(argv[1])) ||\n strncmp(a, b, sqlite3_value_bytes(argv[0])) != 0) {\n vtab_set_error(pVTab,\n \"UPDATEs on vec0 primary key values are not allowed.\");\n return SQLITE_ERROR;\n }\n rc = vec0_rowid_from_id(p, argv[0], &rowid);\n if (rc != SQLITE_OK) {\n return rc;\n }\n } else {\n rowid = sqlite3_value_int64(argv[0]);\n }\n\n // 1) get chunk_id and chunk_offset from _rowids\n rc = vec0_get_chunk_position(p, rowid, NULL, &chunk_id, &chunk_offset);\n if (rc != SQLITE_OK) {\n return rc;\n }\n\n // 2) update any partition key values\n for (int i = 0; i < vec0_num_defined_user_columns(p); i++) {\n if(p->user_column_kinds[i] != SQLITE_VEC0_USER_COLUMN_KIND_PARTITION) {\n continue;\n }\n sqlite3_value * value = argv[2+VEC0_COLUMN_USERN_START + i];\n if(sqlite3_value_nochange(value)) {\n continue;\n }\n vtab_set_error(pVTab, \"UPDATE on partition key columns are not supported yet. \");\n return SQLITE_ERROR;\n }\n\n // 3) handle auxiliary column updates\n for (int i = 0; i < vec0_num_defined_user_columns(p); i++) {\n if(p->user_column_kinds[i] != SQLITE_VEC0_USER_COLUMN_KIND_AUXILIARY) {\n continue;\n }\n int auxiliary_column_idx = p->user_column_idxs[i];\n sqlite3_value * value = argv[2+VEC0_COLUMN_USERN_START + i];\n if(sqlite3_value_nochange(value)) {\n continue;\n }\n rc = vec0Update_UpdateAuxColumn(p, auxiliary_column_idx, value, rowid);\n if(rc != SQLITE_OK) {\n return SQLITE_ERROR;\n }\n }\n\n // 4) handle metadata column updates\n for (int i = 0; i < vec0_num_defined_user_columns(p); i++) {\n if(p->user_column_kinds[i] != SQLITE_VEC0_USER_COLUMN_KIND_METADATA) {\n continue;\n }\n int metadata_column_idx = p->user_column_idxs[i];\n sqlite3_value * value = argv[2+VEC0_COLUMN_USERN_START + i];\n if(sqlite3_value_nochange(value)) {\n continue;\n }\n rc = vec0_write_metadata_value(p, metadata_column_idx, rowid, chunk_id, chunk_offset, value, 1);\n if(rc != SQLITE_OK) {\n return rc;\n }\n }\n\n // 5) iterate over all new vectors, update the vectors\n for (int i = 0; i < vec0_num_defined_user_columns(p); i++) {\n if(p->user_column_kinds[i] != SQLITE_VEC0_USER_COLUMN_KIND_VECTOR) {\n continue;\n }\n int vector_idx = p->user_column_idxs[i];\n sqlite3_value *valueVector = argv[2 + VEC0_COLUMN_USERN_START + i];\n // in vec0Column, we check sqlite3_vtab_nochange() on vector columns.\n // If the vector column isn't being changed, we return NULL;\n // That's not great, that means vector columns can never be NULLABLE\n // (bc we cant distinguish if an updated vector is truly NULL or nochange).\n // Also it means that if someone tries to run `UPDATE v SET X = NULL`,\n // we can't effectively detect and raise an error.\n // A better solution would be to use a custom result_type for \"empty\",\n // but subtypes don't appear to survive xColumn -> xUpdate, it's always 0.\n // So for now, we'll just use NULL and warn people to not SET X = NULL\n // in the docs.\n if (sqlite3_value_type(valueVector) == SQLITE_NULL) {\n continue;\n }\n\n rc = vec0Update_UpdateVectorColumn(p, chunk_id, chunk_offset, vector_idx,\n valueVector);\n if (rc != SQLITE_OK) {\n return SQLITE_ERROR;\n }\n }\n\n return SQLITE_OK;\n}\n\nstatic int vec0Update(sqlite3_vtab *pVTab, int argc, sqlite3_value **argv,\n sqlite_int64 *pRowid) {\n // DELETE operation\n if (argc == 1 && sqlite3_value_type(argv[0]) != SQLITE_NULL) {\n return vec0Update_Delete(pVTab, argv[0]);\n }\n // INSERT operation\n else if (argc > 1 && sqlite3_value_type(argv[0]) == SQLITE_NULL) {\n return vec0Update_Insert(pVTab, argc, argv, pRowid);\n }\n // UPDATE operation\n else if (argc > 1 && sqlite3_value_type(argv[0]) != SQLITE_NULL) {\n return vec0Update_Update(pVTab, argc, argv);\n } else {\n vtab_set_error(pVTab, \"Unrecognized xUpdate operation provided for vec0.\");\n return SQLITE_ERROR;\n }\n}\n\nstatic int vec0ShadowName(const char *zName) {\n static const char *azName[] = {\n \"rowids\", \"chunks\", \"auxiliary\", \"info\",\n\n // Up to VEC0_MAX_METADATA_COLUMNS\n // TODO be smarter about this man\n \"metadatachunks00\",\n \"metadatachunks01\",\n \"metadatachunks02\",\n \"metadatachunks03\",\n \"metadatachunks04\",\n \"metadatachunks05\",\n \"metadatachunks06\",\n \"metadatachunks07\",\n \"metadatachunks08\",\n \"metadatachunks09\",\n \"metadatachunks10\",\n \"metadatachunks11\",\n \"metadatachunks12\",\n \"metadatachunks13\",\n \"metadatachunks14\",\n \"metadatachunks15\",\n\n // Up to\n \"metadatatext00\",\n \"metadatatext01\",\n \"metadatatext02\",\n \"metadatatext03\",\n \"metadatatext04\",\n \"metadatatext05\",\n \"metadatatext06\",\n \"metadatatext07\",\n \"metadatatext08\",\n \"metadatatext09\",\n \"metadatatext10\",\n \"metadatatext11\",\n \"metadatatext12\",\n \"metadatatext13\",\n \"metadatatext14\",\n \"metadatatext15\",\n };\n\n for (size_t i = 0; i < sizeof(azName) / sizeof(azName[0]); i++) {\n if (sqlite3_stricmp(zName, azName[i]) == 0)\n return 1;\n }\n //for(size_t i = 0; i < )\"vector_chunks\", \"metadatachunks\"\n return 0;\n}\n\nstatic int vec0Begin(sqlite3_vtab *pVTab) {\n UNUSED_PARAMETER(pVTab);\n return SQLITE_OK;\n}\nstatic int vec0Sync(sqlite3_vtab *pVTab) {\n UNUSED_PARAMETER(pVTab);\n vec0_vtab *p = (vec0_vtab *)pVTab;\n if (p->stmtLatestChunk) {\n sqlite3_finalize(p->stmtLatestChunk);\n p->stmtLatestChunk = NULL;\n }\n if (p->stmtRowidsInsertRowid) {\n sqlite3_finalize(p->stmtRowidsInsertRowid);\n p->stmtRowidsInsertRowid = NULL;\n }\n if (p->stmtRowidsInsertId) {\n sqlite3_finalize(p->stmtRowidsInsertId);\n p->stmtRowidsInsertId = NULL;\n }\n if (p->stmtRowidsUpdatePosition) {\n sqlite3_finalize(p->stmtRowidsUpdatePosition);\n p->stmtRowidsUpdatePosition = NULL;\n }\n if (p->stmtRowidsGetChunkPosition) {\n sqlite3_finalize(p->stmtRowidsGetChunkPosition);\n p->stmtRowidsGetChunkPosition = NULL;\n }\n return SQLITE_OK;\n}\nstatic int vec0Commit(sqlite3_vtab *pVTab) {\n UNUSED_PARAMETER(pVTab);\n return SQLITE_OK;\n}\nstatic int vec0Rollback(sqlite3_vtab *pVTab) {\n UNUSED_PARAMETER(pVTab);\n return SQLITE_OK;\n}\n\nstatic sqlite3_module vec0Module = {\n /* iVersion */ 3,\n /* xCreate */ vec0Create,\n /* xConnect */ vec0Connect,\n /* xBestIndex */ vec0BestIndex,\n /* xDisconnect */ vec0Disconnect,\n /* xDestroy */ vec0Destroy,\n /* xOpen */ vec0Open,\n /* xClose */ vec0Close,\n /* xFilter */ vec0Filter,\n /* xNext */ vec0Next,\n /* xEof */ vec0Eof,\n /* xColumn */ vec0Column,\n /* xRowid */ vec0Rowid,\n /* xUpdate */ vec0Update,\n /* xBegin */ vec0Begin,\n /* xSync */ vec0Sync,\n /* xCommit */ vec0Commit,\n /* xRollback */ vec0Rollback,\n /* xFindFunction */ 0,\n /* xRename */ 0, // https://github.com/asg017/sqlite-vec/issues/43\n /* xSavepoint */ 0,\n /* xRelease */ 0,\n /* xRollbackTo */ 0,\n /* xShadowName */ vec0ShadowName,\n#if SQLITE_VERSION_NUMBER >= 3044000\n /* xIntegrity */ 0, // https://github.com/asg017/sqlite-vec/issues/44\n#endif\n};\n#pragma endregion\n\nstatic char *POINTER_NAME_STATIC_BLOB_DEF = \"vec0-static_blob_def\";\nstruct static_blob_definition {\n void *p;\n size_t dimensions;\n size_t nvectors;\n enum VectorElementType element_type;\n};\nstatic void vec_static_blob_from_raw(sqlite3_context *context, int argc,\n sqlite3_value **argv) {\n\n assert(argc == 4);\n struct static_blob_definition *p;\n p = sqlite3_malloc(sizeof(*p));\n if (!p) {\n sqlite3_result_error_nomem(context);\n return;\n }\n memset(p, 0, sizeof(*p));\n p->p = (void *)sqlite3_value_int64(argv[0]);\n p->element_type = SQLITE_VEC_ELEMENT_TYPE_FLOAT32;\n p->dimensions = sqlite3_value_int64(argv[2]);\n p->nvectors = sqlite3_value_int64(argv[3]);\n sqlite3_result_pointer(context, p, POINTER_NAME_STATIC_BLOB_DEF,\n sqlite3_free);\n}\n#pragma region vec_static_blobs() table function\n\n#define MAX_STATIC_BLOBS 16\n\ntypedef struct static_blob static_blob;\nstruct static_blob {\n char *name;\n void *p;\n size_t dimensions;\n size_t nvectors;\n enum VectorElementType element_type;\n};\n\ntypedef struct vec_static_blob_data vec_static_blob_data;\nstruct vec_static_blob_data {\n static_blob static_blobs[MAX_STATIC_BLOBS];\n};\n\ntypedef struct vec_static_blobs_vtab vec_static_blobs_vtab;\nstruct vec_static_blobs_vtab {\n sqlite3_vtab base;\n vec_static_blob_data *data;\n};\n\ntypedef struct vec_static_blobs_cursor vec_static_blobs_cursor;\nstruct vec_static_blobs_cursor {\n sqlite3_vtab_cursor base;\n sqlite3_int64 iRowid;\n};\n\nstatic int vec_static_blobsConnect(sqlite3 *db, void *pAux, int argc,\n const char *const *argv,\n sqlite3_vtab **ppVtab, char **pzErr) {\n UNUSED_PARAMETER(argc);\n UNUSED_PARAMETER(argv);\n UNUSED_PARAMETER(pzErr);\n\n vec_static_blobs_vtab *pNew;\n#define VEC_STATIC_BLOBS_NAME 0\n#define VEC_STATIC_BLOBS_DATA 1\n#define VEC_STATIC_BLOBS_DIMENSIONS 2\n#define VEC_STATIC_BLOBS_COUNT 3\n int rc = sqlite3_declare_vtab(\n db, \"CREATE TABLE x(name, data, dimensions hidden, count hidden)\");\n if (rc == SQLITE_OK) {\n pNew = sqlite3_malloc(sizeof(*pNew));\n *ppVtab = (sqlite3_vtab *)pNew;\n if (pNew == 0)\n return SQLITE_NOMEM;\n memset(pNew, 0, sizeof(*pNew));\n pNew->data = pAux;\n }\n return rc;\n}\n\nstatic int vec_static_blobsDisconnect(sqlite3_vtab *pVtab) {\n vec_static_blobs_vtab *p = (vec_static_blobs_vtab *)pVtab;\n sqlite3_free(p);\n return SQLITE_OK;\n}\n\nstatic int vec_static_blobsUpdate(sqlite3_vtab *pVTab, int argc,\n sqlite3_value **argv, sqlite_int64 *pRowid) {\n UNUSED_PARAMETER(pRowid);\n vec_static_blobs_vtab *p = (vec_static_blobs_vtab *)pVTab;\n // DELETE operation\n if (argc == 1 && sqlite3_value_type(argv[0]) != SQLITE_NULL) {\n return SQLITE_ERROR;\n }\n // INSERT operation\n else if (argc > 1 && sqlite3_value_type(argv[0]) == SQLITE_NULL) {\n const char *key =\n (const char *)sqlite3_value_text(argv[2 + VEC_STATIC_BLOBS_NAME]);\n int idx = -1;\n for (int i = 0; i < MAX_STATIC_BLOBS; i++) {\n if (!p->data->static_blobs[i].name) {\n p->data->static_blobs[i].name = sqlite3_mprintf(\"%s\", key);\n idx = i;\n break;\n }\n }\n if (idx < 0)\n abort();\n struct static_blob_definition *def = sqlite3_value_pointer(\n argv[2 + VEC_STATIC_BLOBS_DATA], POINTER_NAME_STATIC_BLOB_DEF);\n p->data->static_blobs[idx].p = def->p;\n p->data->static_blobs[idx].dimensions = def->dimensions;\n p->data->static_blobs[idx].nvectors = def->nvectors;\n p->data->static_blobs[idx].element_type = def->element_type;\n\n return SQLITE_OK;\n }\n // UPDATE operation\n else if (argc > 1 && sqlite3_value_type(argv[0]) != SQLITE_NULL) {\n return SQLITE_ERROR;\n }\n return SQLITE_ERROR;\n}\n\nstatic int vec_static_blobsOpen(sqlite3_vtab *p,\n sqlite3_vtab_cursor **ppCursor) {\n UNUSED_PARAMETER(p);\n vec_static_blobs_cursor *pCur;\n pCur = sqlite3_malloc(sizeof(*pCur));\n if (pCur == 0)\n return SQLITE_NOMEM;\n memset(pCur, 0, sizeof(*pCur));\n *ppCursor = &pCur->base;\n return SQLITE_OK;\n}\n\nstatic int vec_static_blobsClose(sqlite3_vtab_cursor *cur) {\n vec_static_blobs_cursor *pCur = (vec_static_blobs_cursor *)cur;\n sqlite3_free(pCur);\n return SQLITE_OK;\n}\n\nstatic int vec_static_blobsBestIndex(sqlite3_vtab *pVTab,\n sqlite3_index_info *pIdxInfo) {\n UNUSED_PARAMETER(pVTab);\n pIdxInfo->idxNum = 1;\n pIdxInfo->estimatedCost = (double)10;\n pIdxInfo->estimatedRows = 10;\n return SQLITE_OK;\n}\n\nstatic int vec_static_blobsNext(sqlite3_vtab_cursor *cur);\nstatic int vec_static_blobsFilter(sqlite3_vtab_cursor *pVtabCursor, int idxNum,\n const char *idxStr, int argc,\n sqlite3_value **argv) {\n UNUSED_PARAMETER(idxNum);\n UNUSED_PARAMETER(idxStr);\n UNUSED_PARAMETER(argc);\n UNUSED_PARAMETER(argv);\n vec_static_blobs_cursor *pCur = (vec_static_blobs_cursor *)pVtabCursor;\n pCur->iRowid = -1;\n vec_static_blobsNext(pVtabCursor);\n return SQLITE_OK;\n}\n\nstatic int vec_static_blobsRowid(sqlite3_vtab_cursor *cur,\n sqlite_int64 *pRowid) {\n vec_static_blobs_cursor *pCur = (vec_static_blobs_cursor *)cur;\n *pRowid = pCur->iRowid;\n return SQLITE_OK;\n}\n\nstatic int vec_static_blobsNext(sqlite3_vtab_cursor *cur) {\n vec_static_blobs_cursor *pCur = (vec_static_blobs_cursor *)cur;\n vec_static_blobs_vtab *p = (vec_static_blobs_vtab *)pCur->base.pVtab;\n pCur->iRowid++;\n while (pCur->iRowid < MAX_STATIC_BLOBS) {\n if (p->data->static_blobs[pCur->iRowid].name) {\n return SQLITE_OK;\n }\n pCur->iRowid++;\n }\n return SQLITE_OK;\n}\n\nstatic int vec_static_blobsEof(sqlite3_vtab_cursor *cur) {\n vec_static_blobs_cursor *pCur = (vec_static_blobs_cursor *)cur;\n return pCur->iRowid >= MAX_STATIC_BLOBS;\n}\n\nstatic int vec_static_blobsColumn(sqlite3_vtab_cursor *cur,\n sqlite3_context *context, int i) {\n vec_static_blobs_cursor *pCur = (vec_static_blobs_cursor *)cur;\n vec_static_blobs_vtab *p = (vec_static_blobs_vtab *)cur->pVtab;\n switch (i) {\n case VEC_STATIC_BLOBS_NAME:\n sqlite3_result_text(context, p->data->static_blobs[pCur->iRowid].name, -1,\n SQLITE_TRANSIENT);\n break;\n case VEC_STATIC_BLOBS_DATA:\n sqlite3_result_null(context);\n break;\n case VEC_STATIC_BLOBS_DIMENSIONS:\n sqlite3_result_int64(context,\n p->data->static_blobs[pCur->iRowid].dimensions);\n break;\n case VEC_STATIC_BLOBS_COUNT:\n sqlite3_result_int64(context, p->data->static_blobs[pCur->iRowid].nvectors);\n break;\n }\n return SQLITE_OK;\n}\n\nstatic sqlite3_module vec_static_blobsModule = {\n /* iVersion */ 3,\n /* xCreate */ 0,\n /* xConnect */ vec_static_blobsConnect,\n /* xBestIndex */ vec_static_blobsBestIndex,\n /* xDisconnect */ vec_static_blobsDisconnect,\n /* xDestroy */ 0,\n /* xOpen */ vec_static_blobsOpen,\n /* xClose */ vec_static_blobsClose,\n /* xFilter */ vec_static_blobsFilter,\n /* xNext */ vec_static_blobsNext,\n /* xEof */ vec_static_blobsEof,\n /* xColumn */ vec_static_blobsColumn,\n /* xRowid */ vec_static_blobsRowid,\n /* xUpdate */ vec_static_blobsUpdate,\n /* xBegin */ 0,\n /* xSync */ 0,\n /* xCommit */ 0,\n /* xRollback */ 0,\n /* xFindMethod */ 0,\n /* xRename */ 0,\n /* xSavepoint */ 0,\n /* xRelease */ 0,\n /* xRollbackTo */ 0,\n /* xShadowName */ 0,\n#if SQLITE_VERSION_NUMBER >= 3044000\n /* xIntegrity */ 0\n#endif\n};\n#pragma endregion\n\n#pragma region vec_static_blob_entries() table function\n\ntypedef struct vec_static_blob_entries_vtab vec_static_blob_entries_vtab;\nstruct vec_static_blob_entries_vtab {\n sqlite3_vtab base;\n static_blob *blob;\n};\ntypedef enum {\n VEC_SBE__QUERYPLAN_FULLSCAN = 1,\n VEC_SBE__QUERYPLAN_KNN = 2\n} vec_sbe_query_plan;\n\nstruct sbe_query_knn_data {\n i64 k;\n i64 k_used;\n // Array of rowids of size k. Must be freed with sqlite3_free().\n i32 *rowids;\n // Array of distances of size k. Must be freed with sqlite3_free().\n f32 *distances;\n i64 current_idx;\n};\nvoid sbe_query_knn_data_clear(struct sbe_query_knn_data *knn_data) {\n if (!knn_data)\n return;\n\n if (knn_data->rowids) {\n sqlite3_free(knn_data->rowids);\n knn_data->rowids = NULL;\n }\n if (knn_data->distances) {\n sqlite3_free(knn_data->distances);\n knn_data->distances = NULL;\n }\n}\n\ntypedef struct vec_static_blob_entries_cursor vec_static_blob_entries_cursor;\nstruct vec_static_blob_entries_cursor {\n sqlite3_vtab_cursor base;\n sqlite3_int64 iRowid;\n vec_sbe_query_plan query_plan;\n struct sbe_query_knn_data *knn_data;\n};\n\nstatic int vec_static_blob_entriesConnect(sqlite3 *db, void *pAux, int argc,\n const char *const *argv,\n sqlite3_vtab **ppVtab, char **pzErr) {\n UNUSED_PARAMETER(argc);\n UNUSED_PARAMETER(argv);\n UNUSED_PARAMETER(pzErr);\n vec_static_blob_data *blob_data = pAux;\n int idx = -1;\n for (int i = 0; i < MAX_STATIC_BLOBS; i++) {\n if (!blob_data->static_blobs[i].name)\n continue;\n if (strncmp(blob_data->static_blobs[i].name, argv[3],\n strlen(blob_data->static_blobs[i].name)) == 0) {\n idx = i;\n break;\n }\n }\n if (idx < 0)\n abort();\n vec_static_blob_entries_vtab *pNew;\n#define VEC_STATIC_BLOB_ENTRIES_VECTOR 0\n#define VEC_STATIC_BLOB_ENTRIES_DISTANCE 1\n#define VEC_STATIC_BLOB_ENTRIES_K 2\n int rc = sqlite3_declare_vtab(\n db, \"CREATE TABLE x(vector, distance hidden, k hidden)\");\n if (rc == SQLITE_OK) {\n pNew = sqlite3_malloc(sizeof(*pNew));\n *ppVtab = (sqlite3_vtab *)pNew;\n if (pNew == 0)\n return SQLITE_NOMEM;\n memset(pNew, 0, sizeof(*pNew));\n pNew->blob = &blob_data->static_blobs[idx];\n }\n return rc;\n}\n\nstatic int vec_static_blob_entriesCreate(sqlite3 *db, void *pAux, int argc,\n const char *const *argv,\n sqlite3_vtab **ppVtab, char **pzErr) {\n return vec_static_blob_entriesConnect(db, pAux, argc, argv, ppVtab, pzErr);\n}\n\nstatic int vec_static_blob_entriesDisconnect(sqlite3_vtab *pVtab) {\n vec_static_blob_entries_vtab *p = (vec_static_blob_entries_vtab *)pVtab;\n sqlite3_free(p);\n return SQLITE_OK;\n}\n\nstatic int vec_static_blob_entriesOpen(sqlite3_vtab *p,\n sqlite3_vtab_cursor **ppCursor) {\n UNUSED_PARAMETER(p);\n vec_static_blob_entries_cursor *pCur;\n pCur = sqlite3_malloc(sizeof(*pCur));\n if (pCur == 0)\n return SQLITE_NOMEM;\n memset(pCur, 0, sizeof(*pCur));\n *ppCursor = &pCur->base;\n return SQLITE_OK;\n}\n\nstatic int vec_static_blob_entriesClose(sqlite3_vtab_cursor *cur) {\n vec_static_blob_entries_cursor *pCur = (vec_static_blob_entries_cursor *)cur;\n sqlite3_free(pCur->knn_data);\n sqlite3_free(pCur);\n return SQLITE_OK;\n}\n\nstatic int vec_static_blob_entriesBestIndex(sqlite3_vtab *pVTab,\n sqlite3_index_info *pIdxInfo) {\n vec_static_blob_entries_vtab *p = (vec_static_blob_entries_vtab *)pVTab;\n int iMatchTerm = -1;\n int iLimitTerm = -1;\n // int iRowidTerm = -1; // https://github.com/asg017/sqlite-vec/issues/47\n int iKTerm = -1;\n\n for (int i = 0; i < pIdxInfo->nConstraint; i++) {\n if (!pIdxInfo->aConstraint[i].usable)\n continue;\n\n int iColumn = pIdxInfo->aConstraint[i].iColumn;\n int op = pIdxInfo->aConstraint[i].op;\n if (op == SQLITE_INDEX_CONSTRAINT_MATCH &&\n iColumn == VEC_STATIC_BLOB_ENTRIES_VECTOR) {\n if (iMatchTerm > -1) {\n // https://github.com/asg017/sqlite-vec/issues/51\n return SQLITE_ERROR;\n }\n iMatchTerm = i;\n }\n if (op == SQLITE_INDEX_CONSTRAINT_LIMIT) {\n iLimitTerm = i;\n }\n if (op == SQLITE_INDEX_CONSTRAINT_EQ &&\n iColumn == VEC_STATIC_BLOB_ENTRIES_K) {\n iKTerm = i;\n }\n }\n if (iMatchTerm >= 0) {\n if (iLimitTerm < 0 && iKTerm < 0) {\n // https://github.com/asg017/sqlite-vec/issues/51\n return SQLITE_ERROR;\n }\n if (iLimitTerm >= 0 && iKTerm >= 0) {\n return SQLITE_ERROR; // limit or k, not both\n }\n if (pIdxInfo->nOrderBy < 1) {\n vtab_set_error(pVTab, \"ORDER BY distance required\");\n return SQLITE_CONSTRAINT;\n }\n if (pIdxInfo->nOrderBy > 1) {\n // https://github.com/asg017/sqlite-vec/issues/51\n vtab_set_error(pVTab, \"more than 1 ORDER BY clause provided\");\n return SQLITE_CONSTRAINT;\n }\n if (pIdxInfo->aOrderBy[0].iColumn != VEC_STATIC_BLOB_ENTRIES_DISTANCE) {\n vtab_set_error(pVTab, \"ORDER BY must be on the distance column\");\n return SQLITE_CONSTRAINT;\n }\n if (pIdxInfo->aOrderBy[0].desc) {\n vtab_set_error(pVTab,\n \"Only ascending in ORDER BY distance clause is supported, \"\n \"DESC is not supported yet.\");\n return SQLITE_CONSTRAINT;\n }\n\n pIdxInfo->idxNum = VEC_SBE__QUERYPLAN_KNN;\n pIdxInfo->estimatedCost = (double)10;\n pIdxInfo->estimatedRows = 10;\n\n pIdxInfo->orderByConsumed = 1;\n pIdxInfo->aConstraintUsage[iMatchTerm].argvIndex = 1;\n pIdxInfo->aConstraintUsage[iMatchTerm].omit = 1;\n if (iLimitTerm >= 0) {\n pIdxInfo->aConstraintUsage[iLimitTerm].argvIndex = 2;\n pIdxInfo->aConstraintUsage[iLimitTerm].omit = 1;\n } else {\n pIdxInfo->aConstraintUsage[iKTerm].argvIndex = 2;\n pIdxInfo->aConstraintUsage[iKTerm].omit = 1;\n }\n\n } else {\n pIdxInfo->idxNum = VEC_SBE__QUERYPLAN_FULLSCAN;\n pIdxInfo->estimatedCost = (double)p->blob->nvectors;\n pIdxInfo->estimatedRows = p->blob->nvectors;\n }\n return SQLITE_OK;\n}\n\nstatic int vec_static_blob_entriesFilter(sqlite3_vtab_cursor *pVtabCursor,\n int idxNum, const char *idxStr,\n int argc, sqlite3_value **argv) {\n UNUSED_PARAMETER(idxStr);\n assert(argc >= 0 && argc <= 3);\n vec_static_blob_entries_cursor *pCur =\n (vec_static_blob_entries_cursor *)pVtabCursor;\n vec_static_blob_entries_vtab *p =\n (vec_static_blob_entries_vtab *)pCur->base.pVtab;\n\n if (idxNum == VEC_SBE__QUERYPLAN_KNN) {\n assert(argc == 2);\n pCur->query_plan = VEC_SBE__QUERYPLAN_KNN;\n struct sbe_query_knn_data *knn_data;\n knn_data = sqlite3_malloc(sizeof(*knn_data));\n if (!knn_data) {\n return SQLITE_NOMEM;\n }\n memset(knn_data, 0, sizeof(*knn_data));\n\n void *queryVector;\n size_t dimensions;\n enum VectorElementType elementType;\n vector_cleanup cleanup;\n char *err;\n int rc = vector_from_value(argv[0], &queryVector, &dimensions, &elementType,\n &cleanup, &err);\n if (rc != SQLITE_OK) {\n return SQLITE_ERROR;\n }\n if (elementType != p->blob->element_type) {\n return SQLITE_ERROR;\n }\n if (dimensions != p->blob->dimensions) {\n return SQLITE_ERROR;\n }\n\n i64 k = min(sqlite3_value_int64(argv[1]), (i64)p->blob->nvectors);\n if (k < 0) {\n // HANDLE https://github.com/asg017/sqlite-vec/issues/55\n return SQLITE_ERROR;\n }\n if (k == 0) {\n knn_data->k = 0;\n pCur->knn_data = knn_data;\n return SQLITE_OK;\n }\n\n size_t bsize = (p->blob->nvectors + 7) & ~7;\n\n i32 *topk_rowids = sqlite3_malloc(k * sizeof(i32));\n if (!topk_rowids) {\n // HANDLE https://github.com/asg017/sqlite-vec/issues/55\n return SQLITE_ERROR;\n }\n f32 *distances = sqlite3_malloc(bsize * sizeof(f32));\n if (!distances) {\n // HANDLE https://github.com/asg017/sqlite-vec/issues/55\n return SQLITE_ERROR;\n }\n\n for (size_t i = 0; i < p->blob->nvectors; i++) {\n // https://github.com/asg017/sqlite-vec/issues/52\n float *v = ((float *)p->blob->p) + (i * p->blob->dimensions);\n distances[i] =\n distance_l2_sqr_float(v, (float *)queryVector, &p->blob->dimensions);\n }\n u8 *candidates = bitmap_new(bsize);\n assert(candidates);\n\n u8 *taken = bitmap_new(bsize);\n assert(taken);\n\n bitmap_fill(candidates, bsize);\n for (size_t i = bsize; i >= p->blob->nvectors; i--) {\n bitmap_set(candidates, i, 0);\n }\n i32 k_used = 0;\n min_idx(distances, bsize, candidates, topk_rowids, k, taken, &k_used);\n knn_data->current_idx = 0;\n knn_data->distances = distances;\n knn_data->k = k;\n knn_data->rowids = topk_rowids;\n\n pCur->knn_data = knn_data;\n } else {\n pCur->query_plan = VEC_SBE__QUERYPLAN_FULLSCAN;\n pCur->iRowid = 0;\n }\n\n return SQLITE_OK;\n}\n\nstatic int vec_static_blob_entriesRowid(sqlite3_vtab_cursor *cur,\n sqlite_int64 *pRowid) {\n vec_static_blob_entries_cursor *pCur = (vec_static_blob_entries_cursor *)cur;\n switch (pCur->query_plan) {\n case VEC_SBE__QUERYPLAN_FULLSCAN: {\n *pRowid = pCur->iRowid;\n return SQLITE_OK;\n }\n case VEC_SBE__QUERYPLAN_KNN: {\n i32 rowid = ((i32 *)pCur->knn_data->rowids)[pCur->knn_data->current_idx];\n *pRowid = (sqlite3_int64)rowid;\n return SQLITE_OK;\n }\n }\n return SQLITE_ERROR;\n}\n\nstatic int vec_static_blob_entriesNext(sqlite3_vtab_cursor *cur) {\n vec_static_blob_entries_cursor *pCur = (vec_static_blob_entries_cursor *)cur;\n switch (pCur->query_plan) {\n case VEC_SBE__QUERYPLAN_FULLSCAN: {\n pCur->iRowid++;\n return SQLITE_OK;\n }\n case VEC_SBE__QUERYPLAN_KNN: {\n pCur->knn_data->current_idx++;\n return SQLITE_OK;\n }\n }\n return SQLITE_ERROR;\n}\n\nstatic int vec_static_blob_entriesEof(sqlite3_vtab_cursor *cur) {\n vec_static_blob_entries_cursor *pCur = (vec_static_blob_entries_cursor *)cur;\n vec_static_blob_entries_vtab *p =\n (vec_static_blob_entries_vtab *)pCur->base.pVtab;\n switch (pCur->query_plan) {\n case VEC_SBE__QUERYPLAN_FULLSCAN: {\n return (size_t)pCur->iRowid >= p->blob->nvectors;\n }\n case VEC_SBE__QUERYPLAN_KNN: {\n return pCur->knn_data->current_idx >= pCur->knn_data->k;\n }\n }\n return SQLITE_ERROR;\n}\n\nstatic int vec_static_blob_entriesColumn(sqlite3_vtab_cursor *cur,\n sqlite3_context *context, int i) {\n vec_static_blob_entries_cursor *pCur = (vec_static_blob_entries_cursor *)cur;\n vec_static_blob_entries_vtab *p = (vec_static_blob_entries_vtab *)cur->pVtab;\n\n switch (pCur->query_plan) {\n case VEC_SBE__QUERYPLAN_FULLSCAN: {\n switch (i) {\n case VEC_STATIC_BLOB_ENTRIES_VECTOR:\n\n sqlite3_result_blob(\n context,\n ((unsigned char *)p->blob->p) +\n (pCur->iRowid * p->blob->dimensions * sizeof(float)),\n p->blob->dimensions * sizeof(float), SQLITE_TRANSIENT);\n sqlite3_result_subtype(context, p->blob->element_type);\n break;\n }\n return SQLITE_OK;\n }\n case VEC_SBE__QUERYPLAN_KNN: {\n switch (i) {\n case VEC_STATIC_BLOB_ENTRIES_VECTOR: {\n i32 rowid = ((i32 *)pCur->knn_data->rowids)[pCur->knn_data->current_idx];\n sqlite3_result_blob(context,\n ((unsigned char *)p->blob->p) +\n (rowid * p->blob->dimensions * sizeof(float)),\n p->blob->dimensions * sizeof(float),\n SQLITE_TRANSIENT);\n sqlite3_result_subtype(context, p->blob->element_type);\n break;\n }\n }\n return SQLITE_OK;\n }\n }\n return SQLITE_ERROR;\n}\n\nstatic sqlite3_module vec_static_blob_entriesModule = {\n /* iVersion */ 3,\n /* xCreate */\n vec_static_blob_entriesCreate, // handle rm?\n // https://github.com/asg017/sqlite-vec/issues/55\n /* xConnect */ vec_static_blob_entriesConnect,\n /* xBestIndex */ vec_static_blob_entriesBestIndex,\n /* xDisconnect */ vec_static_blob_entriesDisconnect,\n /* xDestroy */ vec_static_blob_entriesDisconnect,\n /* xOpen */ vec_static_blob_entriesOpen,\n /* xClose */ vec_static_blob_entriesClose,\n /* xFilter */ vec_static_blob_entriesFilter,\n /* xNext */ vec_static_blob_entriesNext,\n /* xEof */ vec_static_blob_entriesEof,\n /* xColumn */ vec_static_blob_entriesColumn,\n /* xRowid */ vec_static_blob_entriesRowid,\n /* xUpdate */ 0,\n /* xBegin */ 0,\n /* xSync */ 0,\n /* xCommit */ 0,\n /* xRollback */ 0,\n /* xFindMethod */ 0,\n /* xRename */ 0,\n /* xSavepoint */ 0,\n /* xRelease */ 0,\n /* xRollbackTo */ 0,\n /* xShadowName */ 0,\n#if SQLITE_VERSION_NUMBER >= 3044000\n /* xIntegrity */ 0\n#endif\n};\n#pragma endregion\n\n#ifdef SQLITE_VEC_ENABLE_AVX\n#define SQLITE_VEC_DEBUG_BUILD_AVX \"avx\"\n#else\n#define SQLITE_VEC_DEBUG_BUILD_AVX \"\"\n#endif\n#ifdef SQLITE_VEC_ENABLE_NEON\n#define SQLITE_VEC_DEBUG_BUILD_NEON \"neon\"\n#else\n#define SQLITE_VEC_DEBUG_BUILD_NEON \"\"\n#endif\n\n#define SQLITE_VEC_DEBUG_BUILD \\\n SQLITE_VEC_DEBUG_BUILD_AVX \" \" SQLITE_VEC_DEBUG_BUILD_NEON\n\n#define SQLITE_VEC_DEBUG_STRING \\\n \"Version: \" SQLITE_VEC_VERSION \"\\n\" \\\n \"Date: \" SQLITE_VEC_DATE \"\\n\" \\\n \"Commit: \" SQLITE_VEC_SOURCE \"\\n\" \\\n \"Build flags: \" SQLITE_VEC_DEBUG_BUILD\n\nSQLITE_VEC_API int sqlite3_vec_init(sqlite3 *db, char **pzErrMsg,\n const sqlite3_api_routines *pApi) {\n#ifndef SQLITE_CORE\n SQLITE_EXTENSION_INIT2(pApi);\n#endif\n int rc = SQLITE_OK;\n\n#define DEFAULT_FLAGS (SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC)\n\n rc = sqlite3_create_function_v2(db, \"vec_version\", 0, DEFAULT_FLAGS,\n SQLITE_VEC_VERSION, _static_text_func, NULL,\n NULL, NULL);\n if (rc != SQLITE_OK) {\n return rc;\n }\n rc = sqlite3_create_function_v2(db, \"vec_debug\", 0, DEFAULT_FLAGS,\n SQLITE_VEC_DEBUG_STRING, _static_text_func,\n NULL, NULL, NULL);\n if (rc != SQLITE_OK) {\n return rc;\n }\n static struct {\n const char *zFName;\n void (*xFunc)(sqlite3_context *, int, sqlite3_value **);\n int nArg;\n int flags;\n } aFunc[] = {\n // clang-format off\n //{\"vec_version\", _static_text_func, 0, DEFAULT_FLAGS, (void *) SQLITE_VEC_VERSION },\n //{\"vec_debug\", _static_text_func, 0, DEFAULT_FLAGS, (void *) SQLITE_VEC_DEBUG_STRING },\n {\"vec_distance_l2\", vec_distance_l2, 2, DEFAULT_FLAGS | SQLITE_SUBTYPE, },\n {\"vec_distance_l1\", vec_distance_l1, 2, DEFAULT_FLAGS | SQLITE_SUBTYPE, },\n {\"vec_distance_hamming\",vec_distance_hamming, 2, DEFAULT_FLAGS | SQLITE_SUBTYPE, },\n {\"vec_distance_cosine\", vec_distance_cosine, 2, DEFAULT_FLAGS | SQLITE_SUBTYPE, },\n {\"vec_length\", vec_length, 1, DEFAULT_FLAGS | SQLITE_SUBTYPE, },\n {\"vec_type\", vec_type, 1, DEFAULT_FLAGS, },\n {\"vec_to_json\", vec_to_json, 1, DEFAULT_FLAGS | SQLITE_SUBTYPE | SQLITE_RESULT_SUBTYPE, },\n {\"vec_add\", vec_add, 2, DEFAULT_FLAGS | SQLITE_SUBTYPE | SQLITE_RESULT_SUBTYPE, },\n {\"vec_sub\", vec_sub, 2, DEFAULT_FLAGS | SQLITE_SUBTYPE | SQLITE_RESULT_SUBTYPE, },\n {\"vec_slice\", vec_slice, 3, DEFAULT_FLAGS | SQLITE_SUBTYPE | SQLITE_RESULT_SUBTYPE, },\n {\"vec_normalize\", vec_normalize, 1, DEFAULT_FLAGS | SQLITE_SUBTYPE | SQLITE_RESULT_SUBTYPE, },\n {\"vec_f32\", vec_f32, 1, DEFAULT_FLAGS | SQLITE_SUBTYPE | SQLITE_RESULT_SUBTYPE, },\n {\"vec_bit\", vec_bit, 1, DEFAULT_FLAGS | SQLITE_SUBTYPE | SQLITE_RESULT_SUBTYPE, },\n {\"vec_int8\", vec_int8, 1, DEFAULT_FLAGS | SQLITE_SUBTYPE | SQLITE_RESULT_SUBTYPE, },\n {\"vec_quantize_int8\", vec_quantize_int8, 2, DEFAULT_FLAGS | SQLITE_SUBTYPE | SQLITE_RESULT_SUBTYPE, },\n {\"vec_quantize_binary\", vec_quantize_binary, 1, DEFAULT_FLAGS | SQLITE_SUBTYPE | SQLITE_RESULT_SUBTYPE, },\n // clang-format on\n };\n\n static struct {\n char *name;\n const sqlite3_module *module;\n void *p;\n void (*xDestroy)(void *);\n } aMod[] = {\n // clang-format off\n {\"vec0\", &vec0Module, NULL, NULL},\n {\"vec_each\", &vec_eachModule, NULL, NULL},\n // clang-format on\n };\n\n for (unsigned long i = 0; i < countof(aFunc) && rc == SQLITE_OK; i++) {\n rc = sqlite3_create_function_v2(db, aFunc[i].zFName, aFunc[i].nArg,\n aFunc[i].flags, NULL, aFunc[i].xFunc, NULL,\n NULL, NULL);\n if (rc != SQLITE_OK) {\n *pzErrMsg = sqlite3_mprintf(\"Error creating function %s: %s\",\n aFunc[i].zFName, sqlite3_errmsg(db));\n return rc;\n }\n }\n\n for (unsigned long i = 0; i < countof(aMod) && rc == SQLITE_OK; i++) {\n rc = sqlite3_create_module_v2(db, aMod[i].name, aMod[i].module, NULL, NULL);\n if (rc != SQLITE_OK) {\n *pzErrMsg = sqlite3_mprintf(\"Error creating module %s: %s\", aMod[i].name,\n sqlite3_errmsg(db));\n return rc;\n }\n }\n\n return SQLITE_OK;\n}\n\n#ifndef SQLITE_VEC_OMIT_FS\nSQLITE_VEC_API int sqlite3_vec_numpy_init(sqlite3 *db, char **pzErrMsg,\n const sqlite3_api_routines *pApi) {\n UNUSED_PARAMETER(pzErrMsg);\n#ifndef SQLITE_CORE\n SQLITE_EXTENSION_INIT2(pApi);\n#endif\n int rc = SQLITE_OK;\n rc = sqlite3_create_function_v2(db, \"vec_npy_file\", 1, SQLITE_RESULT_SUBTYPE,\n NULL, vec_npy_file, NULL, NULL, NULL);\n if(rc != SQLITE_OK) {\n return rc;\n }\n rc = sqlite3_create_module_v2(db, \"vec_npy_each\", &vec_npy_eachModule, NULL, NULL);\n return rc;\n}\n#endif\n\nSQLITE_VEC_API int\nsqlite3_vec_static_blobs_init(sqlite3 *db, char **pzErrMsg,\n const sqlite3_api_routines *pApi) {\n UNUSED_PARAMETER(pzErrMsg);\n#ifndef SQLITE_CORE\n SQLITE_EXTENSION_INIT2(pApi);\n#endif\n\n int rc = SQLITE_OK;\n vec_static_blob_data *static_blob_data;\n static_blob_data = sqlite3_malloc(sizeof(*static_blob_data));\n if (!static_blob_data) {\n return SQLITE_NOMEM;\n }\n memset(static_blob_data, 0, sizeof(*static_blob_data));\n\n rc = sqlite3_create_function_v2(\n db, \"vec_static_blob_from_raw\", 4,\n DEFAULT_FLAGS | SQLITE_SUBTYPE | SQLITE_RESULT_SUBTYPE, NULL,\n vec_static_blob_from_raw, NULL, NULL, NULL);\n if (rc != SQLITE_OK)\n return rc;\n\n rc = sqlite3_create_module_v2(db, \"vec_static_blobs\", &vec_static_blobsModule,\n static_blob_data, sqlite3_free);\n if (rc != SQLITE_OK)\n return rc;\n rc = sqlite3_create_module_v2(db, \"vec_static_blob_entries\",\n &vec_static_blob_entriesModule,\n static_blob_data, NULL);\n if (rc != SQLITE_OK)\n return rc;\n return rc;\n}\n" }, { "path": "sqlite-vec.h.tmpl", "content": "#ifndef SQLITE_VEC_H\n#define SQLITE_VEC_H\n\n#ifndef SQLITE_CORE\n#include \"sqlite3ext.h\"\n#else\n#include \"sqlite3.h\"\n#endif\n\n#ifdef SQLITE_VEC_STATIC\n #define SQLITE_VEC_API\n#else\n #ifdef _WIN32\n #define SQLITE_VEC_API __declspec(dllexport)\n #else\n #define SQLITE_VEC_API\n #endif\n#endif\n\n#define SQLITE_VEC_VERSION \"v${VERSION}\"\n// TODO rm\n#define SQLITE_VEC_DATE \"${DATE}\"\n#define SQLITE_VEC_SOURCE \"${SOURCE}\"\n\n\n#define SQLITE_VEC_VERSION_MAJOR ${VERSION_MAJOR}\n#define SQLITE_VEC_VERSION_MINOR ${VERSION_MINOR}\n#define SQLITE_VEC_VERSION_PATCH ${VERSION_PATCH}\n\n#ifdef __cplusplus\nextern \"C\" {\n#endif\n\nSQLITE_VEC_API int sqlite3_vec_init(sqlite3 *db, char **pzErrMsg,\n const sqlite3_api_routines *pApi);\n\n#ifdef __cplusplus\n} /* end of the 'extern \"C\"' block */\n#endif\n\n#endif /* ifndef SQLITE_VEC_H */\n" }, { "path": "test.sql", "content": "\n.load dist/vec0main\n.bail on\n\n.mode qbox\n\n\n.load ./memstat\n.echo on\n\nselect name, value from sqlite_memstat where name = 'MEMORY_USED';\n\ncreate virtual table v using vec0(\n vector float[1],\n name1 text,\n name2 text,\n age int,\n chunk_size=8\n);\n\nselect name, value from sqlite_memstat where name = 'MEMORY_USED';\n\ninsert into v(vector, name1, name2, age) values\n ('[1]', 'alex', 'xxxx', 1),\n ('[2]', 'alex', 'aaaa', 2),\n ('[3]', 'alex', 'aaaa', 3),\n ('[4]', 'brian', 'aaaa', 1),\n ('[5]', 'brian', 'aaaa', 2),\n ('[6]', 'brian', 'aaaa', 3),\n ('[7]', 'craig', 'aaaa', 1),\n ('[8]', 'craig', 'xxxx', 2),\n ('[9]', 'craig', 'xxxx', 3),\n ('[10]', '123456789012345', 'xxxx', 3);\n\nselect name, value from sqlite_memstat where name = 'MEMORY_USED';\n\nselect rowid, name1, name2, age, vec_to_json(vector)\nfrom v\nwhere vector match '[0]'\n and k = 5\n and name1 in ('alex', 'brian', 'craig')\n --and name2 in ('aaaa', 'xxxx')\n and age in (1, 2, 3, 2222,3333,4444);\n\nselect name, value from sqlite_memstat where name = 'MEMORY_USED';\n\nselect rowid, name1, name2, age, vec_to_json(vector)\nfrom v\nwhere vector match '[0]'\n and k = 5\n and name1 in ('123456789012345', 'superfluous');\n\n\n.exit\n\ncreate virtual table v using vec0(\n vector float[1],\n +description text\n);\ninsert into v(rowid, vector, description) values (1, '[1]', 'aaa');\nselect * from v;\n\n.exit\n\ncreate virtual table vec_articles using vec0(\n article_id integer primary key,\n year integer partition key,\n headline_embedding float[1],\n +headline text,\n +url text,\n word_count integer,\n print_section text,\n print_page integer,\n pub_date text,\n);\n\ninsert into vec_articles values (1111, 2020, '[1]', 'headline', 'https://...', 200, 'A', 1, '2020-01-01');\n\nselect * from vec_articles;\n\n.exit\n\n\ncreate table movies(movie_id integer primary key, synopsis text);\nINSERT INTO movies(movie_id, synopsis)\nVALUES\n (1, 'A family is haunted by demonic spirits after moving into a new house, requiring the help of paranormal investigators.'),\n (2, 'Two dim-witted friends embark on a cross-country road trip to return a briefcase full of money to its owner.'),\n (3, 'A team of explorers travels through a wormhole in space in an attempt to ensure humanity’s survival.'),\n (4, 'A young hobbit embarks on a journey with a fellowship to destroy a powerful ring and save Middle-earth from darkness.'),\n (5, 'A documentary about the dangers of global warming, featuring former U.S. Vice President Al Gore.'),\n (6, 'After the death of her secretive mother, a woman discovers terrifying secrets about her family lineage.'),\n (7, 'A clueless but charismatic TV anchorman struggles to stay relevant in the world of broadcast journalism.'),\n (8, 'A young blade runner uncovers a long-buried secret that leads him to track down former blade runner Rick Deckard.'),\n (9, 'A young boy discovers he is a wizard and attends a magical school, where he learns about his destiny.'),\n (10, 'A rock climber attempts to scale El Capitan in Yosemite National Park without the use of ropes or safety gear.'),\n (11, 'A young African-American man uncovers a disturbing secret when he visits his white girlfriend''s family estate.'),\n (12, 'Three friends wake up from a bachelor party in Las Vegas with no memory of the previous night and must retrace their steps.'),\n (13, 'A computer hacker learns about the true nature of his reality and his role in the war against its controllers.'),\n (14, 'In post-Civil War Spain, a young girl escapes into an eerie but captivating fantasy world.'),\n (15, 'A documentary that explores racial inequality in the United States, focusing on the prison system and mass incarceration.'),\n (16, 'A young woman is followed by an unknown supernatural force after a sexual encounter.'),\n (17, 'Two immature but well-meaning stepbrothers become instant rivals when their single parents marry.'),\n (18, 'A thief with the ability to enter people''s dreams is tasked with planting an idea into a target''s subconscious.'),\n (19, 'A mute woman forms a unique relationship with a mysterious aquatic creature being held in a secret research facility.'),\n (20, 'A documentary about the life and legacy of Fred Rogers, the beloved host of the children''s TV show \"Mister Rogers'' Neighborhood.\"');\n\n\ncreate virtual table vec_movies using vec0(\n movie_id integer primary key,\n synopsis_embedding float[1],\n +title text,\n genre text,\n num_reviews int,\n mean_rating float,\n chunk_size=8\n);\n\n.schema\n/*\ninsert into vec_movies(movie_id, synopsis_embedding, num_reviews, mean_rating) values\n (1, '[1]', 153, 4.6),\n (2, '[2]', 382, 2.6),\n (3, '[3]', 53, 5.0),\n (4, '[4]', 210, 4.2),\n (5, '[5]', 93, 3.4),\n (6, '[6]', 167, 4.7),\n (7, '[7]', 482, 2.9),\n (8, '[8]', 301, 5.0),\n (9, '[9]', 134, 4.1),\n (10, '[10]', 66, 3.2),\n (11, '[11]', 88, 4.9),\n (12, '[12]', 59, 2.8),\n (13, '[13]', 423, 4.5),\n (14, '[14]', 275, 3.6),\n (15, '[15]', 191, 4.4),\n (16, '[16]', 314, 4.3),\n (17, '[17]', 74, 3.0),\n (18, '[18]', 201, 5.0),\n (19, '[19]', 399, 2.7),\n (20, '[20]', 186, 4.8);\n*/\n\n/*\n\nINSERT INTO vec_movies(movie_id, synopsis_embedding, genre, num_reviews, mean_rating)\nVALUES\n (1, '[1]', 'horror', 153, 4.6),\n (2, '[2]', 'comedy', 382, 2.6),\n (3, '[3]', 'scifi', 53, 5.0),\n (4, '[4]', 'fantasy', 210, 4.2),\n (5, '[5]', 'documentary', 93, 3.4),\n (6, '[6]', 'horror', 167, 4.7),\n (7, '[7]', 'comedy', 482, 2.9),\n (8, '[8]', 'scifi', 301, 5.0),\n (9, '[9]', 'fantasy', 134, 4.1),\n (10, '[10]', 'documentary', 66, 3.2),\n (11, '[11]', 'horror', 88, 4.9),\n (12, '[12]', 'comedy', 59, 2.8),\n (13, '[13]', 'scifi', 423, 4.5),\n (14, '[14]', 'fantasy', 275, 3.6),\n (15, '[15]', 'documentary', 191, 4.4),\n (16, '[16]', 'horror', 314, 4.3),\n (17, '[17]', 'comedy', 74, 3.0),\n (18, '[18]', 'scifi', 201, 5.0),\n (19, '[19]', 'fantasy', 399, 2.7),\n (20, '[20]', 'documentary', 186, 4.8);\n*/\n\nINSERT INTO vec_movies(movie_id, synopsis_embedding, genre, title, num_reviews, mean_rating)\nVALUES\n (1, '[1]', 'horror', 'The Conjuring', 153, 4.6),\n (2, '[2]', 'comedy', 'Dumb and Dumber', 382, 2.6),\n (3, '[3]', 'scifi', 'Interstellar', 53, 5.0),\n (4, '[4]', 'fantasy', 'The Lord of the Rings: The Fellowship of the Ring', 210, 4.2),\n (5, '[5]', 'documentary', 'An Inconvenient Truth', 93, 3.4),\n (6, '[6]', 'horror', 'Hereditary', 167, 4.7),\n (7, '[7]', 'comedy', 'Anchorman: The Legend of Ron Burgundy', 482, 2.9),\n (8, '[8]', 'scifi', 'Blade Runner 2049', 301, 5.0),\n (9, '[9]', 'fantasy', 'Harry Potter and the Sorcerer''s Stone', 134, 4.1),\n (10, '[10]', 'documentary', 'Free Solo', 66, 3.2),\n (11, '[11]', 'horror', 'Get Out', 88, 4.9),\n (12, '[12]', 'comedy', 'The Hangover', 59, 2.8),\n (13, '[13]', 'scifi', 'The Matrix', 423, 4.5),\n (14, '[14]', 'fantasy', 'Pan''s Labyrinth', 275, 3.6),\n (15, '[15]', 'documentary', '13th', 191, 4.4),\n (16, '[16]', 'horror', 'It Follows', 314, 4.3),\n (17, '[17]', 'comedy', 'Step Brothers', 74, 3.0),\n (18, '[18]', 'scifi', 'Inception', 201, 5.0),\n (19, '[19]', 'fantasy', 'The Shape of Water', 399, 2.7),\n (20, '[20]', 'documentary', 'Won''t You Be My Neighbor?', 186, 4.8),\n (21, '[21]', 'scifi', 'Gravity', 342, 4.0),\n (22, '[22]', 'scifi', 'Dune', 451, 4.4),\n (23, '[23]', 'scifi', 'The Martian', 522, 4.6),\n (24, '[24]', 'horror', 'A Quiet Place', 271, 4.3),\n (25, '[25]', 'fantasy', 'The Chronicles of Narnia: The Lion, the Witch and the Wardrobe', 310, 3.9);\n\n--select * from vec_movies;\n--select * from vec_movies_metadata_chunks00;\n\n\ncreate virtual table vec_chunks using vec0(\n user_id integer partition key,\n +contents text,\n contents_embedding float[1],\n);\n\nINSERT INTO vec_chunks (rowid, user_id, contents, contents_embedding) VALUES\n(1, 123, 'Our PTO policy allows employees to take both vacation and sick leave as needed.', '[1]'),\n(2, 123, 'Employees must provide notice at least two weeks in advance for planned vacations.', '[2]'),\n(3, 123, 'Sick leave can be taken without advance notice, but employees must inform their manager.', '[3]'),\n(4, 123, 'Unused PTO can be carried over to the following year, up to a maximum of 40 hours.', '[4]'),\n(5, 123, 'PTO must be used in increments of at least 4 hours.', '[5]'),\n(6, 456, 'New employees are granted 10 days of PTO during their first year of employment.', '[6]'),\n(7, 456, 'After the first year, employees earn an additional day of PTO for each year of service.', '[7]'),\n(8, 789, 'PTO requests will be reviewed by the HR department and are subject to approval.', '[8]'),\n(9, 789, 'The company reserves the right to deny PTO requests during peak operational periods.', '[9]'),\n(10, 456, 'If PTO is denied, the employee will be given an alternative time to take leave.', '[10]'),\n(11, 789, 'Employees who are out of PTO must request unpaid leave for any additional time off.', '[11]'),\n(12, 789, 'In case of a family emergency, employees can request emergency leave.', '[12]'),\n(13, 456, 'Emergency leave may be granted for personal or family illness, or other critical situations.', '[13]'),\n(14, 789, 'The maximum length of emergency leave is subject to company discretion.', '[14]'),\n(15, 123, 'All PTO balances will be displayed on the employee self-service portal.', '[15]'),\n(16, 456, 'Employees who are terminated will be paid for unused PTO, as per state law.', '[16]'),\n(17, 123, 'Part-time employees are eligible for PTO on a pro-rata basis.', '[17]'),\n(18, 789, 'The company encourages employees to use their PTO to maintain work-life balance.', '[18]'),\n(19, 456, 'Employees should not book travel plans until their PTO request has been approved.', '[19]'),\n(20, 123, 'Managers are responsible for tracking their team members'' PTO usage.', '[20]');\n\nselect rowid, user_id, contents, distance\nfrom vec_chunks\nwhere contents_embedding match '[19]'\n and user_id = 123\n and k = 5;\n\n.exit\n\n\n\n\n\n-- PARTITION KEY and auxiliar columns!\ncreate virtual table vec_chunks using vec0(\n -- internally shard the vector index by user\n user_id integer partition key,\n -- store the chunk text pre-embedding as an \"auxiliary column\"\n +contents text,\n contents_embeddings float[1024],\n);\n\nselect rowid, user_id, contents, distance\nfrom vec_chunks\nwhere contents_embedding match '[...]'\n and user_id = 123\n and k = 5;\n/*\n┌───────┬─────────┬──────────────────────────────────────────────────────────────┬──────────┐\n│ rowid │ user_id │ contents │ distance │\n├───────┼─────────┼──────────────────────────────────────────────────────────────┼──────────┤\n│ 20 │ 123 │ 'Managers are responsible for tracking their team members'' │ 1.0 │\n│ │ │ PTO usage.' │ │\n├───────┼─────────┼──────────────────────────────────────────────────────────────┼──────────┤\n│ 17 │ 123 │ 'Part-time employees are eligible for PTO on a pro-rata basi │ 2.0 │\n│ │ │ s.' │ │\n├───────┼─────────┼──────────────────────────────────────────────────────────────┼──────────┤\n│ 15 │ 123 │ 'All PTO balances will be displayed on the employee self-ser │ 4.0 │\n│ │ │ vice portal.' │ │\n├───────┼─────────┼──────────────────────────────────────────────────────────────┼──────────┤\n│ 5 │ 123 │ 'PTO must be used in increments of at least 4 hours.' │ 14.0 │\n├───────┼─────────┼──────────────────────────────────────────────────────────────┼──────────┤\n│ 4 │ 123 │ 'Unused PTO can be carried over to the following year, up to │ 15.0 │\n│ │ │ a maximum of 40 hours.' │ │\n└───────┴─────────┴──────────────────────────────────────────────────────────────┴──────────┘\n*/\n\n\n\n\n\n-- metadata filters!\ncreate virtual table vec_movies using vec0(\n movie_id integer primary key,\n synopsis_embedding float[1024],\n genre text,\n num_reviews int,\n mean_rating float\n);\n\nselect\n movie_id,\n title,\n genre,\n num_reviews,\n mean_rating,\n distance\nfrom vec_movies\nwhere synopsis_embedding match '[15.5]'\n and genre = 'scifi'\n and num_reviews between 100 and 500\n and mean_rating > 3.5\n and k = 5;\n/*\n┌──────────┬─────────────────────┬─────────┬─────────────┬──────────────────┬──────────┐\n│ movie_id │ title │ genre │ num_reviews │ mean_rating │ distance │\n├──────────┼─────────────────────┼─────────┼─────────────┼──────────────────┼──────────┤\n│ 13 │ 'The Matrix' │ 'scifi' │ 423 │ 4.5 │ 2.5 │\n│ 18 │ 'Inception' │ 'scifi' │ 201 │ 5.0 │ 2.5 │\n│ 21 │ 'Gravity' │ 'scifi' │ 342 │ 4.0 │ 5.5 │\n│ 22 │ 'Dune' │ 'scifi' │ 451 │ 4.40000009536743 │ 6.5 │\n│ 8 │ 'Blade Runner 2049' │ 'scifi' │ 301 │ 5.0 │ 7.5 │\n└──────────┴─────────────────────┴─────────┴─────────────┴──────────────────┴──────────┘\n*/\n\n\n\n\n.exit\n\ncreate virtual table vec_movies using vec0(\n movie_id integer primary key,\n synopsis_embedding float[768],\n genre text,\n num_reviews int,\n mean_rating float,\n);\n\n\n.exit\n\n\ncreate virtual table vec_chunks using vec0(\n chunk_id integer primary key,\n contents_embedding float[1],\n +contents text\n);\ninsert into vec_chunks(chunk_id, contents_embedding, contents) values\n (1, '[1]', 'alex'),\n (2, '[2]', 'brian'),\n (3, '[3]', 'craig'),\n (4, '[4]', 'dylan');\n\nselect * from vec_chunks;\n\nselect chunk_id, contents, distance\nfrom vec_chunks\nwhere contents_embedding match '[5]'\nand k = 3;\n\n.exit\n\ncreate virtual table v using vec0(a float[1]);\nselect count(*) from v_chunks;\ninsert into v(a) values ('[1.11]');\nselect * from v;\ndrop table v;\n\ncreate virtual table v using vec0(\n\n v_aaa float[1],\n partk_xxx int partition key,\n v_bbb float[2],\n partk_yyy text partition key,\n chunk_size=32\n);\n\n\ninsert into v(rowid, v_aaa, partk_xxx, v_bbb, partk_yyy) values\n (1, '[.1]', 999, '[.11, .11]', 'alex'),\n (2, '[.2]', 999, '[.22, .22]', 'alex'),\n (3, '[.3]', 999, '[.33, .33]', 'brian');\n\n\nselect rowid, vec_to_json(v_aaa), partk_xxx, vec_to_json(v_bbb), partk_yyy from v;\n\nselect * from v;\nselect * from v where rowid = 2;\nupdate v\nset v_aaa = '[.222]',\n v_bbb = '[.222, .222]'\nwhere rowid = 2;\n\nselect rowid, vec_to_json(v_aaa), partk_xxx, vec_to_json(v_bbb), partk_yyy from v;\n\nselect chunk_id, size, sequence_id, partition00, partition01, (validity), length(rowids) from v_chunks;\n\n--explain query plan\nselect *, distance\nfrom v\nwhere v_aaa match '[.5]'\n and partk_xxx = 999\n and partk_yyy = 'alex'\n --and partk_xxx != 20\n and k = 5;\n" }, { "path": "tests/.gitignore", "content": "target/\n" }, { "path": "tests/.python-version", "content": "3.12\n" }, { "path": "tests/Cargo.toml", "content": "[package]\nname = \"tests\"\nversion = \"0.1.0\"\nedition = \"2021\"\n\n[dependencies]\n\n[build-dependencies]\ncc = \"1.0\"\n\n[[bin]]\nname = \"unittest\"\npath = \"unittest.rs\"\n\n\n" }, { "path": "tests/__snapshots__/test-auxiliary.ambr", "content": "# serializer version: 1\n# name: test_constructor_limit[max 16 auxiliary columns]\n dict({\n 'error': 'OperationalError',\n 'message': 'vec0 constructor error: More than 16 auxiliary columns were provided',\n })\n# ---\n# name: test_deletes\n OrderedDict({\n 'sql': 'select rowid, * from v',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'vector': b'\\x00\\x00\\x80?',\n 'name': 'alex',\n }),\n OrderedDict({\n 'rowid': 2,\n 'vector': b'\\x00\\x00\\x00@',\n 'name': 'brian',\n }),\n OrderedDict({\n 'rowid': 3,\n 'vector': b'\\x00\\x00@@',\n 'name': 'craig',\n }),\n ]),\n })\n# ---\n# name: test_deletes.1\n dict({\n 'v_auxiliary': OrderedDict({\n 'sql': 'select * from v_auxiliary',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'value00': 'alex',\n }),\n OrderedDict({\n 'rowid': 2,\n 'value00': 'brian',\n }),\n OrderedDict({\n 'rowid': 3,\n 'value00': 'craig',\n }),\n ]),\n }),\n 'v_chunks': OrderedDict({\n 'sql': 'select * from v_chunks',\n 'rows': list([\n OrderedDict({\n 'chunk_id': 1,\n 'size': 8,\n 'validity': b'\\x07',\n 'rowids': b'\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x03\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n 'v_rowids': OrderedDict({\n 'sql': 'select * from v_rowids',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 0,\n }),\n OrderedDict({\n 'rowid': 2,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 1,\n }),\n OrderedDict({\n 'rowid': 3,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 2,\n }),\n ]),\n }),\n 'v_vector_chunks00': OrderedDict({\n 'sql': 'select * from v_vector_chunks00',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'vectors': b'\\x00\\x00\\x80?\\x00\\x00\\x00@\\x00\\x00@@\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n })\n# ---\n# name: test_deletes.2\n OrderedDict({\n 'sql': 'delete from v where rowid = 1',\n 'rows': list([\n ]),\n })\n# ---\n# name: test_deletes.3\n OrderedDict({\n 'sql': 'select rowid, * from v',\n 'rows': list([\n OrderedDict({\n 'rowid': 2,\n 'vector': b'\\x00\\x00\\x00@',\n 'name': 'brian',\n }),\n OrderedDict({\n 'rowid': 3,\n 'vector': b'\\x00\\x00@@',\n 'name': 'craig',\n }),\n ]),\n })\n# ---\n# name: test_deletes.4\n dict({\n 'v_auxiliary': OrderedDict({\n 'sql': 'select * from v_auxiliary',\n 'rows': list([\n OrderedDict({\n 'rowid': 2,\n 'value00': 'brian',\n }),\n OrderedDict({\n 'rowid': 3,\n 'value00': 'craig',\n }),\n ]),\n }),\n 'v_chunks': OrderedDict({\n 'sql': 'select * from v_chunks',\n 'rows': list([\n OrderedDict({\n 'chunk_id': 1,\n 'size': 8,\n 'validity': b'\\x06',\n 'rowids': b'\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x03\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n 'v_rowids': OrderedDict({\n 'sql': 'select * from v_rowids',\n 'rows': list([\n OrderedDict({\n 'rowid': 2,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 1,\n }),\n OrderedDict({\n 'rowid': 3,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 2,\n }),\n ]),\n }),\n 'v_vector_chunks00': OrderedDict({\n 'sql': 'select * from v_vector_chunks00',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'vectors': b'\\x00\\x00\\x00\\x00\\x00\\x00\\x00@\\x00\\x00@@\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n })\n# ---\n# name: test_knn\n OrderedDict({\n 'sql': 'select * from v',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'vector': b'\\x00\\x00\\x80?',\n 'name': 'alex',\n }),\n OrderedDict({\n 'rowid': 2,\n 'vector': b'\\x00\\x00\\x00@',\n 'name': 'brian',\n }),\n OrderedDict({\n 'rowid': 3,\n 'vector': b'\\x00\\x00@@',\n 'name': 'craig',\n }),\n ]),\n })\n# ---\n# name: test_knn[illegal KNN w/ aux]\n dict({\n 'error': 'OperationalError',\n 'message': 'An illegal WHERE constraint was provided on a vec0 auxiliary column in a KNN query.',\n })\n# ---\n# name: test_knn[legal KNN w/ aux]\n OrderedDict({\n 'sql': \"select *, distance from v where vector match '[5]' and k = 10\",\n 'rows': list([\n OrderedDict({\n 'rowid': 3,\n 'vector': b'\\x00\\x00@@',\n 'name': 'craig',\n 'distance': 2.0,\n }),\n OrderedDict({\n 'rowid': 2,\n 'vector': b'\\x00\\x00\\x00@',\n 'name': 'brian',\n 'distance': 3.0,\n }),\n OrderedDict({\n 'rowid': 1,\n 'vector': b'\\x00\\x00\\x80?',\n 'name': 'alex',\n 'distance': 4.0,\n }),\n ]),\n })\n# ---\n# name: test_normal.1\n OrderedDict({\n 'sql': 'select * from v',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'a': b'\\x11\\x11\\x11\\x11',\n 'name': 'alex',\n }),\n OrderedDict({\n 'rowid': 2,\n 'a': b'\"\"\"\"',\n 'name': 'brian',\n }),\n OrderedDict({\n 'rowid': 3,\n 'a': b'3333',\n 'name': 'craig',\n }),\n ]),\n })\n# ---\n# name: test_normal.2\n dict({\n 'v_auxiliary': OrderedDict({\n 'sql': 'select * from v_auxiliary',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'value00': 'alex',\n }),\n OrderedDict({\n 'rowid': 2,\n 'value00': 'brian',\n }),\n OrderedDict({\n 'rowid': 3,\n 'value00': 'craig',\n }),\n ]),\n }),\n 'v_chunks': OrderedDict({\n 'sql': 'select * from v_chunks',\n 'rows': list([\n OrderedDict({\n 'chunk_id': 1,\n 'size': 8,\n 'validity': b'\\x07',\n 'rowids': b'\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x03\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n 'v_rowids': OrderedDict({\n 'sql': 'select * from v_rowids',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 0,\n }),\n OrderedDict({\n 'rowid': 2,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 1,\n }),\n OrderedDict({\n 'rowid': 3,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 2,\n }),\n ]),\n }),\n 'v_vector_chunks00': OrderedDict({\n 'sql': 'select * from v_vector_chunks00',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'vectors': b'\\x11\\x11\\x11\\x11\"\"\"\"3333\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n })\n# ---\n# name: test_normal[sqlite_master post drop]\n OrderedDict({\n 'sql': 'select * from sqlite_master order by name',\n 'rows': list([\n OrderedDict({\n 'type': 'table',\n 'name': 'sqlite_sequence',\n 'tbl_name': 'sqlite_sequence',\n 'rootpage': 5,\n 'sql': 'CREATE TABLE sqlite_sequence(name,seq)',\n }),\n ]),\n })\n# ---\n# name: test_normal[sqlite_master]\n OrderedDict({\n 'sql': 'select * from sqlite_master order by name',\n 'rows': list([\n OrderedDict({\n 'type': 'index',\n 'name': 'sqlite_autoindex_v_info_1',\n 'tbl_name': 'v_info',\n 'rootpage': 3,\n 'sql': None,\n }),\n OrderedDict({\n 'type': 'index',\n 'name': 'sqlite_autoindex_v_vector_chunks00_1',\n 'tbl_name': 'v_vector_chunks00',\n 'rootpage': 8,\n 'sql': None,\n }),\n OrderedDict({\n 'type': 'table',\n 'name': 'sqlite_sequence',\n 'tbl_name': 'sqlite_sequence',\n 'rootpage': 5,\n 'sql': 'CREATE TABLE sqlite_sequence(name,seq)',\n }),\n OrderedDict({\n 'type': 'table',\n 'name': 'v',\n 'tbl_name': 'v',\n 'rootpage': 0,\n 'sql': 'CREATE VIRTUAL TABLE v using vec0(a float[1], +name text, chunk_size=8)',\n }),\n OrderedDict({\n 'type': 'table',\n 'name': 'v_auxiliary',\n 'tbl_name': 'v_auxiliary',\n 'rootpage': 9,\n 'sql': 'CREATE TABLE \"v_auxiliary\"( rowid integer PRIMARY KEY , value00)',\n }),\n OrderedDict({\n 'type': 'table',\n 'name': 'v_chunks',\n 'tbl_name': 'v_chunks',\n 'rootpage': 4,\n 'sql': 'CREATE TABLE \"v_chunks\"(chunk_id INTEGER PRIMARY KEY AUTOINCREMENT,size INTEGER NOT NULL,validity BLOB NOT NULL,rowids BLOB NOT NULL)',\n }),\n OrderedDict({\n 'type': 'table',\n 'name': 'v_info',\n 'tbl_name': 'v_info',\n 'rootpage': 2,\n 'sql': 'CREATE TABLE \"v_info\" (key text primary key, value any)',\n }),\n OrderedDict({\n 'type': 'table',\n 'name': 'v_rowids',\n 'tbl_name': 'v_rowids',\n 'rootpage': 6,\n 'sql': 'CREATE TABLE \"v_rowids\"(rowid INTEGER PRIMARY KEY AUTOINCREMENT,id,chunk_id INTEGER,chunk_offset INTEGER)',\n }),\n OrderedDict({\n 'type': 'table',\n 'name': 'v_vector_chunks00',\n 'tbl_name': 'v_vector_chunks00',\n 'rootpage': 7,\n 'sql': 'CREATE TABLE \"v_vector_chunks00\"(rowid PRIMARY KEY,vectors BLOB NOT NULL)',\n }),\n ]),\n })\n# ---\n# name: test_types\n OrderedDict({\n 'sql': 'select * from v',\n 'rows': list([\n ]),\n })\n# ---\n# name: test_types.1\n OrderedDict({\n 'sql': 'insert into v(vector, aux_int, aux_float, aux_text, aux_blob) values (?, ?, ?, ?, ?)',\n 'rows': list([\n ]),\n })\n# ---\n# name: test_types.2\n OrderedDict({\n 'sql': 'select * from v',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'vector': b'\\x11\\x11\\x11\\x11',\n 'aux_int': 1,\n 'aux_float': 1.22,\n 'aux_text': 'text',\n 'aux_blob': b'blob',\n }),\n ]),\n })\n# ---\n# name: test_types.3\n dict({\n 'error': 'IntegrityError',\n 'message': 'Auxiliary column type mismatch: The auxiliary column aux_int has type INTEGER, but TEXT was provided.',\n })\n# ---\n# name: test_types.4\n dict({\n 'error': 'IntegrityError',\n 'message': 'Auxiliary column type mismatch: The auxiliary column aux_float has type FLOAT, but TEXT was provided.',\n })\n# ---\n# name: test_types.5\n dict({\n 'error': 'IntegrityError',\n 'message': 'Auxiliary column type mismatch: The auxiliary column aux_text has type TEXT, but INTEGER was provided.',\n })\n# ---\n# name: test_types.6\n dict({\n 'error': 'IntegrityError',\n 'message': 'Auxiliary column type mismatch: The auxiliary column aux_blob has type BLOB, but INTEGER was provided.',\n })\n# ---\n# name: test_types.7\n OrderedDict({\n 'sql': 'insert into v(vector, aux_int, aux_float, aux_text, aux_blob) values (?, ?, ?, ?, ?)',\n 'rows': list([\n ]),\n })\n# ---\n# name: test_types.8\n OrderedDict({\n 'sql': 'select * from v',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'vector': b'\\x11\\x11\\x11\\x11',\n 'aux_int': 1,\n 'aux_float': 1.22,\n 'aux_text': 'text',\n 'aux_blob': b'blob',\n }),\n OrderedDict({\n 'rowid': 2,\n 'vector': b'\\x11\\x11\\x11\\x11',\n 'aux_int': None,\n 'aux_float': None,\n 'aux_text': None,\n 'aux_blob': None,\n }),\n ]),\n })\n# ---\n# name: test_updates\n OrderedDict({\n 'sql': 'select rowid, * from v',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'vector': b'\\x00\\x00\\x80?',\n 'name': 'alex',\n }),\n OrderedDict({\n 'rowid': 2,\n 'vector': b'\\x00\\x00\\x00@',\n 'name': 'brian',\n }),\n OrderedDict({\n 'rowid': 3,\n 'vector': b'\\x00\\x00@@',\n 'name': 'craig',\n }),\n ]),\n })\n# ---\n# name: test_updates.1\n dict({\n 'v_auxiliary': OrderedDict({\n 'sql': 'select * from v_auxiliary',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'value00': 'alex',\n }),\n OrderedDict({\n 'rowid': 2,\n 'value00': 'brian',\n }),\n OrderedDict({\n 'rowid': 3,\n 'value00': 'craig',\n }),\n ]),\n }),\n 'v_chunks': OrderedDict({\n 'sql': 'select * from v_chunks',\n 'rows': list([\n OrderedDict({\n 'chunk_id': 1,\n 'size': 8,\n 'validity': b'\\x07',\n 'rowids': b'\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x03\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n 'v_rowids': OrderedDict({\n 'sql': 'select * from v_rowids',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 0,\n }),\n OrderedDict({\n 'rowid': 2,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 1,\n }),\n OrderedDict({\n 'rowid': 3,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 2,\n }),\n ]),\n }),\n 'v_vector_chunks00': OrderedDict({\n 'sql': 'select * from v_vector_chunks00',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'vectors': b'\\x00\\x00\\x80?\\x00\\x00\\x00@\\x00\\x00@@\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n })\n# ---\n# name: test_updates.2\n OrderedDict({\n 'sql': \"update v set name = 'ALEX' where rowid = 1\",\n 'rows': list([\n ]),\n })\n# ---\n# name: test_updates.3\n OrderedDict({\n 'sql': 'select rowid, * from v',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'vector': b'\\x00\\x00\\x80?',\n 'name': 'ALEX',\n }),\n OrderedDict({\n 'rowid': 2,\n 'vector': b'\\x00\\x00\\x00@',\n 'name': 'brian',\n }),\n OrderedDict({\n 'rowid': 3,\n 'vector': b'\\x00\\x00@@',\n 'name': 'craig',\n }),\n ]),\n })\n# ---\n# name: test_updates.4\n dict({\n 'v_auxiliary': OrderedDict({\n 'sql': 'select * from v_auxiliary',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'value00': 'ALEX',\n }),\n OrderedDict({\n 'rowid': 2,\n 'value00': 'brian',\n }),\n OrderedDict({\n 'rowid': 3,\n 'value00': 'craig',\n }),\n ]),\n }),\n 'v_chunks': OrderedDict({\n 'sql': 'select * from v_chunks',\n 'rows': list([\n OrderedDict({\n 'chunk_id': 1,\n 'size': 8,\n 'validity': b'\\x07',\n 'rowids': b'\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x03\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n 'v_rowids': OrderedDict({\n 'sql': 'select * from v_rowids',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 0,\n }),\n OrderedDict({\n 'rowid': 2,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 1,\n }),\n OrderedDict({\n 'rowid': 3,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 2,\n }),\n ]),\n }),\n 'v_vector_chunks00': OrderedDict({\n 'sql': 'select * from v_vector_chunks00',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'vectors': b'\\x00\\x00\\x80?\\x00\\x00\\x00@\\x00\\x00@@\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n })\n# ---\n" }, { "path": "tests/__snapshots__/test-general.ambr", "content": "# serializer version: 1\n# name: test_info\n OrderedDict({\n 'sql': 'select key, typeof(value) from v_info order by 1',\n 'rows': list([\n OrderedDict({\n 'key': 'CREATE_VERSION',\n 'typeof(value)': 'text',\n }),\n OrderedDict({\n 'key': 'CREATE_VERSION_MAJOR',\n 'typeof(value)': 'integer',\n }),\n OrderedDict({\n 'key': 'CREATE_VERSION_MINOR',\n 'typeof(value)': 'integer',\n }),\n OrderedDict({\n 'key': 'CREATE_VERSION_PATCH',\n 'typeof(value)': 'integer',\n }),\n ]),\n })\n# ---\n# name: test_shadow\n OrderedDict({\n 'sql': 'select * from sqlite_master order by name',\n 'rows': list([\n OrderedDict({\n 'type': 'index',\n 'name': 'sqlite_autoindex_v_info_1',\n 'tbl_name': 'v_info',\n 'rootpage': 3,\n 'sql': None,\n }),\n OrderedDict({\n 'type': 'index',\n 'name': 'sqlite_autoindex_v_metadatachunks00_1',\n 'tbl_name': 'v_metadatachunks00',\n 'rootpage': 10,\n 'sql': None,\n }),\n OrderedDict({\n 'type': 'index',\n 'name': 'sqlite_autoindex_v_metadatatext00_1',\n 'tbl_name': 'v_metadatatext00',\n 'rootpage': 12,\n 'sql': None,\n }),\n OrderedDict({\n 'type': 'index',\n 'name': 'sqlite_autoindex_v_vector_chunks00_1',\n 'tbl_name': 'v_vector_chunks00',\n 'rootpage': 8,\n 'sql': None,\n }),\n OrderedDict({\n 'type': 'table',\n 'name': 'sqlite_sequence',\n 'tbl_name': 'sqlite_sequence',\n 'rootpage': 5,\n 'sql': 'CREATE TABLE sqlite_sequence(name,seq)',\n }),\n OrderedDict({\n 'type': 'table',\n 'name': 'v',\n 'tbl_name': 'v',\n 'rootpage': 0,\n 'sql': 'CREATE VIRTUAL TABLE v using vec0(a float[1], partition text partition key, metadata text, +name text, chunk_size=8)',\n }),\n OrderedDict({\n 'type': 'table',\n 'name': 'v_auxiliary',\n 'tbl_name': 'v_auxiliary',\n 'rootpage': 13,\n 'sql': 'CREATE TABLE \"v_auxiliary\"( rowid integer PRIMARY KEY , value00)',\n }),\n OrderedDict({\n 'type': 'table',\n 'name': 'v_chunks',\n 'tbl_name': 'v_chunks',\n 'rootpage': 4,\n 'sql': 'CREATE TABLE \"v_chunks\"(chunk_id INTEGER PRIMARY KEY AUTOINCREMENT,size INTEGER NOT NULL,sequence_id integer,partition00,validity BLOB NOT NULL, rowids BLOB NOT NULL)',\n }),\n OrderedDict({\n 'type': 'table',\n 'name': 'v_info',\n 'tbl_name': 'v_info',\n 'rootpage': 2,\n 'sql': 'CREATE TABLE \"v_info\" (key text primary key, value any)',\n }),\n OrderedDict({\n 'type': 'table',\n 'name': 'v_metadatachunks00',\n 'tbl_name': 'v_metadatachunks00',\n 'rootpage': 9,\n 'sql': 'CREATE TABLE \"v_metadatachunks00\"(rowid PRIMARY KEY, data BLOB NOT NULL)',\n }),\n OrderedDict({\n 'type': 'table',\n 'name': 'v_metadatatext00',\n 'tbl_name': 'v_metadatatext00',\n 'rootpage': 11,\n 'sql': 'CREATE TABLE \"v_metadatatext00\"(rowid PRIMARY KEY, data TEXT)',\n }),\n OrderedDict({\n 'type': 'table',\n 'name': 'v_rowids',\n 'tbl_name': 'v_rowids',\n 'rootpage': 6,\n 'sql': 'CREATE TABLE \"v_rowids\"(rowid INTEGER PRIMARY KEY AUTOINCREMENT,id,chunk_id INTEGER,chunk_offset INTEGER)',\n }),\n OrderedDict({\n 'type': 'table',\n 'name': 'v_vector_chunks00',\n 'tbl_name': 'v_vector_chunks00',\n 'rootpage': 7,\n 'sql': 'CREATE TABLE \"v_vector_chunks00\"(rowid PRIMARY KEY,vectors BLOB NOT NULL)',\n }),\n ]),\n })\n# ---\n# name: test_shadow.1\n OrderedDict({\n 'sql': \"select * from pragma_table_list where type = 'shadow' order by name\",\n 'rows': list([\n OrderedDict({\n 'schema': 'main',\n 'name': 'v_auxiliary',\n 'type': 'shadow',\n 'ncol': 2,\n 'wr': 0,\n 'strict': 0,\n }),\n OrderedDict({\n 'schema': 'main',\n 'name': 'v_chunks',\n 'type': 'shadow',\n 'ncol': 6,\n 'wr': 0,\n 'strict': 0,\n }),\n OrderedDict({\n 'schema': 'main',\n 'name': 'v_info',\n 'type': 'shadow',\n 'ncol': 2,\n 'wr': 0,\n 'strict': 0,\n }),\n OrderedDict({\n 'schema': 'main',\n 'name': 'v_metadatachunks00',\n 'type': 'shadow',\n 'ncol': 2,\n 'wr': 0,\n 'strict': 0,\n }),\n OrderedDict({\n 'schema': 'main',\n 'name': 'v_metadatatext00',\n 'type': 'shadow',\n 'ncol': 2,\n 'wr': 0,\n 'strict': 0,\n }),\n OrderedDict({\n 'schema': 'main',\n 'name': 'v_rowids',\n 'type': 'shadow',\n 'ncol': 4,\n 'wr': 0,\n 'strict': 0,\n }),\n ]),\n })\n# ---\n# name: test_shadow.2\n OrderedDict({\n 'sql': \"select * from pragma_table_list where type = 'shadow' order by name\",\n 'rows': list([\n ]),\n })\n# ---\n" }, { "path": "tests/__snapshots__/test-insert-delete.ambr", "content": "# serializer version: 1\n# name: test_info_table_contents\n OrderedDict({\n 'sql': \"select key, value from v_info where key not like 'CREATE_VERSION%' order by key\",\n 'rows': list([\n ]),\n })\n# ---\n# name: test_insert_creates_chunks_and_vectors[rowids_count]\n OrderedDict({\n 'sql': 'select count(*) as cnt from v_rowids',\n 'rows': list([\n OrderedDict({\n 'cnt': 5,\n }),\n ]),\n })\n# ---\n# name: test_insert_creates_chunks_and_vectors[vector_chunks_count]\n OrderedDict({\n 'sql': 'select count(*) as cnt from v_vector_chunks00',\n 'rows': list([\n OrderedDict({\n 'cnt': 1,\n }),\n ]),\n })\n# ---\n# name: test_insert_text_primary_key[rowids]\n OrderedDict({\n 'sql': 'select rowid, id, chunk_id, chunk_offset from v_rowids order by rowid',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'id': 'doc_a',\n 'chunk_id': 1,\n 'chunk_offset': 0,\n }),\n OrderedDict({\n 'rowid': 2,\n 'id': 'doc_b',\n 'chunk_id': 1,\n 'chunk_offset': 1,\n }),\n ]),\n })\n# ---\n" }, { "path": "tests/__snapshots__/test-knn-distance-constraints.ambr", "content": "# serializer version: 1\n# name: test_normal\n OrderedDict({\n 'sql': 'SELECT * FROM v',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'embedding': b'\\x00\\x00\\x80?',\n 'is_odd': 1,\n }),\n OrderedDict({\n 'rowid': 2,\n 'embedding': b'\\x00\\x00\\x00@',\n 'is_odd': 0,\n }),\n OrderedDict({\n 'rowid': 3,\n 'embedding': b'\\x00\\x00@@',\n 'is_odd': 1,\n }),\n OrderedDict({\n 'rowid': 4,\n 'embedding': b'\\x00\\x00\\x80@',\n 'is_odd': 0,\n }),\n OrderedDict({\n 'rowid': 5,\n 'embedding': b'\\x00\\x00\\xa0@',\n 'is_odd': 1,\n }),\n OrderedDict({\n 'rowid': 6,\n 'embedding': b'\\x00\\x00\\xc0@',\n 'is_odd': 0,\n }),\n OrderedDict({\n 'rowid': 7,\n 'embedding': b'\\x00\\x00\\xe0@',\n 'is_odd': 1,\n }),\n OrderedDict({\n 'rowid': 8,\n 'embedding': b'\\x00\\x00\\x00A',\n 'is_odd': 0,\n }),\n OrderedDict({\n 'rowid': 9,\n 'embedding': b'\\x00\\x00\\x10A',\n 'is_odd': 1,\n }),\n OrderedDict({\n 'rowid': 10,\n 'embedding': b'\\x00\\x00 A',\n 'is_odd': 0,\n }),\n OrderedDict({\n 'rowid': 11,\n 'embedding': b'\\x00\\x000A',\n 'is_odd': 1,\n }),\n OrderedDict({\n 'rowid': 12,\n 'embedding': b'\\x00\\x00@A',\n 'is_odd': 0,\n }),\n OrderedDict({\n 'rowid': 13,\n 'embedding': b'\\x00\\x00PA',\n 'is_odd': 1,\n }),\n OrderedDict({\n 'rowid': 14,\n 'embedding': b'\\x00\\x00`A',\n 'is_odd': 0,\n }),\n OrderedDict({\n 'rowid': 15,\n 'embedding': b'\\x00\\x00pA',\n 'is_odd': 1,\n }),\n OrderedDict({\n 'rowid': 16,\n 'embedding': b'\\x00\\x00\\x80A',\n 'is_odd': 0,\n }),\n OrderedDict({\n 'rowid': 17,\n 'embedding': b'\\x00\\x00\\x88A',\n 'is_odd': 1,\n }),\n ]),\n })\n# ---\n# name: test_normal.1\n OrderedDict({\n 'sql': 'select rowid, distance from v where embedding match ? and k = ? ',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'distance': 0.0,\n }),\n OrderedDict({\n 'rowid': 2,\n 'distance': 1.0,\n }),\n OrderedDict({\n 'rowid': 3,\n 'distance': 2.0,\n }),\n OrderedDict({\n 'rowid': 4,\n 'distance': 3.0,\n }),\n OrderedDict({\n 'rowid': 5,\n 'distance': 4.0,\n }),\n ]),\n })\n# ---\n# name: test_normal.2\n OrderedDict({\n 'sql': 'select rowid, distance from v where embedding match ? and k = ? AND distance > 5',\n 'rows': list([\n OrderedDict({\n 'rowid': 7,\n 'distance': 6.0,\n }),\n OrderedDict({\n 'rowid': 8,\n 'distance': 7.0,\n }),\n OrderedDict({\n 'rowid': 9,\n 'distance': 8.0,\n }),\n OrderedDict({\n 'rowid': 10,\n 'distance': 9.0,\n }),\n OrderedDict({\n 'rowid': 11,\n 'distance': 10.0,\n }),\n ]),\n })\n# ---\n# name: test_normal.3\n OrderedDict({\n 'sql': 'select rowid, distance from v where embedding match ? and k = ? AND distance >= 5',\n 'rows': list([\n OrderedDict({\n 'rowid': 6,\n 'distance': 5.0,\n }),\n OrderedDict({\n 'rowid': 7,\n 'distance': 6.0,\n }),\n OrderedDict({\n 'rowid': 8,\n 'distance': 7.0,\n }),\n OrderedDict({\n 'rowid': 9,\n 'distance': 8.0,\n }),\n OrderedDict({\n 'rowid': 10,\n 'distance': 9.0,\n }),\n ]),\n })\n# ---\n# name: test_normal.4\n OrderedDict({\n 'sql': 'select rowid, distance from v where embedding match ? and k = ? AND distance < 3',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'distance': 0.0,\n }),\n OrderedDict({\n 'rowid': 2,\n 'distance': 1.0,\n }),\n OrderedDict({\n 'rowid': 3,\n 'distance': 2.0,\n }),\n ]),\n })\n# ---\n# name: test_normal.5\n OrderedDict({\n 'sql': 'select rowid, distance from v where embedding match ? and k = ? AND distance <= 3',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'distance': 0.0,\n }),\n OrderedDict({\n 'rowid': 2,\n 'distance': 1.0,\n }),\n OrderedDict({\n 'rowid': 3,\n 'distance': 2.0,\n }),\n OrderedDict({\n 'rowid': 4,\n 'distance': 3.0,\n }),\n ]),\n })\n# ---\n# name: test_normal.6\n OrderedDict({\n 'sql': 'select rowid, distance from v where embedding match ? and k = ? AND distance > 7 AND distance <= 10',\n 'rows': list([\n OrderedDict({\n 'rowid': 9,\n 'distance': 8.0,\n }),\n OrderedDict({\n 'rowid': 10,\n 'distance': 9.0,\n }),\n OrderedDict({\n 'rowid': 11,\n 'distance': 10.0,\n }),\n ]),\n })\n# ---\n# name: test_normal.7\n OrderedDict({\n 'sql': 'select rowid, distance from v where embedding match ? and k = ? AND distance BETWEEN 7 AND 10',\n 'rows': list([\n OrderedDict({\n 'rowid': 8,\n 'distance': 7.0,\n }),\n OrderedDict({\n 'rowid': 9,\n 'distance': 8.0,\n }),\n OrderedDict({\n 'rowid': 10,\n 'distance': 9.0,\n }),\n OrderedDict({\n 'rowid': 11,\n 'distance': 10.0,\n }),\n ]),\n })\n# ---\n# name: test_normal.8\n OrderedDict({\n 'sql': 'select rowid, distance from v where embedding match ? and k = ? AND is_odd == TRUE AND distance BETWEEN 7 AND 10',\n 'rows': list([\n OrderedDict({\n 'rowid': 9,\n 'distance': 8.0,\n }),\n OrderedDict({\n 'rowid': 11,\n 'distance': 10.0,\n }),\n ]),\n })\n# ---\n" }, { "path": "tests/__snapshots__/test-metadata.ambr", "content": "# serializer version: 1\n# name: test_constructor_limit[max 16 metadata columns]\n dict({\n 'error': 'OperationalError',\n 'message': 'vec0 constructor error: More than 16 metadata columns were provided',\n })\n# ---\n# name: test_deletes\n OrderedDict({\n 'sql': 'insert into v(rowid, vector, b, n, f, t) values (?, ?, ?, ?, ?, ?)',\n 'rows': list([\n ]),\n })\n# ---\n# name: test_deletes.1\n OrderedDict({\n 'sql': 'insert into v(rowid, vector, b, n, f, t) values (?, ?, ?, ?, ?, ?)',\n 'rows': list([\n ]),\n })\n# ---\n# name: test_deletes.10\n dict({\n 'v_chunks': OrderedDict({\n 'sql': 'select * from v_chunks',\n 'rows': list([\n OrderedDict({\n 'chunk_id': 1,\n 'size': 8,\n 'validity': b'\\x06',\n 'rowids': b'\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x03\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n 'v_metadatachunks00': OrderedDict({\n 'sql': 'select * from v_metadatachunks00',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'data': b'\\x06',\n }),\n ]),\n }),\n 'v_metadatachunks01': OrderedDict({\n 'sql': 'select * from v_metadatachunks01',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'data': b'\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x03\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n 'v_metadatachunks02': OrderedDict({\n 'sql': 'select * from v_metadatachunks02',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'data': b'\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x9a\\x99\\x99\\x99\\x99\\x99\\x01@ffffff\\n@\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n 'v_metadatachunks03': OrderedDict({\n 'sql': 'select * from v_metadatachunks03',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'data': b'\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x05\\x00\\x00\\x00test2\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\r\\x00\\x00\\x00123456789012\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n 'v_metadatatext03': OrderedDict({\n 'sql': 'select * from v_metadatatext03',\n 'rows': list([\n OrderedDict({\n 'rowid': 3,\n 'data': '1234567890123',\n }),\n ]),\n }),\n 'v_rowids': OrderedDict({\n 'sql': 'select * from v_rowids',\n 'rows': list([\n OrderedDict({\n 'rowid': 2,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 1,\n }),\n OrderedDict({\n 'rowid': 3,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 2,\n }),\n ]),\n }),\n 'v_vector_chunks00': OrderedDict({\n 'sql': 'select * from v_vector_chunks00',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'vectors': b'\\x00\\x00\\x00\\x00\"\"\"\"3333\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n })\n# ---\n# name: test_deletes.2\n OrderedDict({\n 'sql': 'insert into v(rowid, vector, b, n, f, t) values (?, ?, ?, ?, ?, ?)',\n 'rows': list([\n ]),\n })\n# ---\n# name: test_deletes.3\n OrderedDict({\n 'sql': 'select * from v',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'vector': b'\\x11\\x11\\x11\\x11',\n 'b': 1,\n 'n': 1,\n 'f': 1.1,\n 't': 'test1',\n }),\n OrderedDict({\n 'rowid': 2,\n 'vector': b'\"\"\"\"',\n 'b': 1,\n 'n': 2,\n 'f': 2.2,\n 't': 'test2',\n }),\n OrderedDict({\n 'rowid': 3,\n 'vector': b'3333',\n 'b': 1,\n 'n': 3,\n 'f': 3.3,\n 't': '1234567890123',\n }),\n ]),\n })\n# ---\n# name: test_deletes.4\n dict({\n 'v_chunks': OrderedDict({\n 'sql': 'select * from v_chunks',\n 'rows': list([\n OrderedDict({\n 'chunk_id': 1,\n 'size': 8,\n 'validity': b'\\x07',\n 'rowids': b'\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x03\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n 'v_metadatachunks00': OrderedDict({\n 'sql': 'select * from v_metadatachunks00',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'data': b'\\x07',\n }),\n ]),\n }),\n 'v_metadatachunks01': OrderedDict({\n 'sql': 'select * from v_metadatachunks01',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'data': b'\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x03\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n 'v_metadatachunks02': OrderedDict({\n 'sql': 'select * from v_metadatachunks02',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'data': b'\\x9a\\x99\\x99\\x99\\x99\\x99\\xf1?\\x9a\\x99\\x99\\x99\\x99\\x99\\x01@ffffff\\n@\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n 'v_metadatachunks03': OrderedDict({\n 'sql': 'select * from v_metadatachunks03',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'data': b'\\x05\\x00\\x00\\x00test1\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x05\\x00\\x00\\x00test2\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\r\\x00\\x00\\x00123456789012\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n 'v_metadatatext03': OrderedDict({\n 'sql': 'select * from v_metadatatext03',\n 'rows': list([\n OrderedDict({\n 'rowid': 3,\n 'data': '1234567890123',\n }),\n ]),\n }),\n 'v_rowids': OrderedDict({\n 'sql': 'select * from v_rowids',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 0,\n }),\n OrderedDict({\n 'rowid': 2,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 1,\n }),\n OrderedDict({\n 'rowid': 3,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 2,\n }),\n ]),\n }),\n 'v_vector_chunks00': OrderedDict({\n 'sql': 'select * from v_vector_chunks00',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'vectors': b'\\x11\\x11\\x11\\x11\"\"\"\"3333\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n })\n# ---\n# name: test_deletes.5\n OrderedDict({\n 'sql': 'DELETE FROM v where rowid = 1',\n 'rows': list([\n ]),\n })\n# ---\n# name: test_deletes.6\n OrderedDict({\n 'sql': 'select * from v',\n 'rows': list([\n OrderedDict({\n 'rowid': 2,\n 'vector': b'\"\"\"\"',\n 'b': 1,\n 'n': 2,\n 'f': 2.2,\n 't': 'test2',\n }),\n OrderedDict({\n 'rowid': 3,\n 'vector': b'3333',\n 'b': 1,\n 'n': 3,\n 'f': 3.3,\n 't': '1234567890123',\n }),\n ]),\n })\n# ---\n# name: test_deletes.7\n dict({\n 'v_chunks': OrderedDict({\n 'sql': 'select * from v_chunks',\n 'rows': list([\n OrderedDict({\n 'chunk_id': 1,\n 'size': 8,\n 'validity': b'\\x06',\n 'rowids': b'\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x03\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n 'v_metadatachunks00': OrderedDict({\n 'sql': 'select * from v_metadatachunks00',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'data': b'\\x06',\n }),\n ]),\n }),\n 'v_metadatachunks01': OrderedDict({\n 'sql': 'select * from v_metadatachunks01',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'data': b'\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x03\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n 'v_metadatachunks02': OrderedDict({\n 'sql': 'select * from v_metadatachunks02',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'data': b'\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x9a\\x99\\x99\\x99\\x99\\x99\\x01@ffffff\\n@\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n 'v_metadatachunks03': OrderedDict({\n 'sql': 'select * from v_metadatachunks03',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'data': b'\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x05\\x00\\x00\\x00test2\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\r\\x00\\x00\\x00123456789012\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n 'v_metadatatext03': OrderedDict({\n 'sql': 'select * from v_metadatatext03',\n 'rows': list([\n OrderedDict({\n 'rowid': 3,\n 'data': '1234567890123',\n }),\n ]),\n }),\n 'v_rowids': OrderedDict({\n 'sql': 'select * from v_rowids',\n 'rows': list([\n OrderedDict({\n 'rowid': 2,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 1,\n }),\n OrderedDict({\n 'rowid': 3,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 2,\n }),\n ]),\n }),\n 'v_vector_chunks00': OrderedDict({\n 'sql': 'select * from v_vector_chunks00',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'vectors': b'\\x00\\x00\\x00\\x00\"\"\"\"3333\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n })\n# ---\n# name: test_deletes.8\n OrderedDict({\n 'sql': 'DELETE FROM v where rowid = 3',\n 'rows': list([\n ]),\n })\n# ---\n# name: test_deletes.9\n OrderedDict({\n 'sql': 'select * from v',\n 'rows': list([\n OrderedDict({\n 'rowid': 2,\n 'vector': b'\"\"\"\"',\n 'b': 1,\n 'n': 2,\n 'f': 2.2,\n 't': 'test2',\n }),\n OrderedDict({\n 'rowid': 3,\n 'vector': b'3333',\n 'b': 1,\n 'n': 3,\n 'f': 3.3,\n 't': '1234567890123',\n }),\n ]),\n })\n# ---\n# name: test_errors\n OrderedDict({\n 'sql': 'select * from v',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'vector': b'\\x00\\x00\\x80?',\n 't': 'aaaaaaaaaaaax',\n }),\n ]),\n })\n# ---\n# name: test_errors.1\n dict({\n 'error': 'OperationalError',\n 'message': 'Could not extract metadata value for column t at rowid 1',\n })\n# ---\n# name: test_idxstr\n OrderedDict({\n 'sql': \"select * from vec_movies where synopsis_embedding match '' and k = 0 and is_favorited = true\",\n 'plan': list([\n dict({\n 'detail': 'SCAN vec_movies VIRTUAL TABLE INDEX 0:3{___}___&Aa_',\n 'id': 2,\n 'parent': 0,\n }),\n ]),\n })\n# ---\n# name: test_idxstr[knn-constraint-float !=]\n OrderedDict({\n 'sql': \"select * from vec_movies where synopsis_embedding match '' and k = 0 and mean_rating != NULL\",\n 'plan': list([\n dict({\n 'detail': 'SCAN vec_movies VIRTUAL TABLE INDEX 0:3{___}___&Df_',\n 'id': 2,\n 'parent': 0,\n }),\n ]),\n })\n# ---\n# name: test_idxstr[knn-constraint-float <=]\n OrderedDict({\n 'sql': \"select * from vec_movies where synopsis_embedding match '' and k = 0 and mean_rating <= NULL\",\n 'plan': list([\n dict({\n 'detail': 'SCAN vec_movies VIRTUAL TABLE INDEX 0:3{___}___&Dc_',\n 'id': 2,\n 'parent': 0,\n }),\n ]),\n })\n# ---\n# name: test_idxstr[knn-constraint-float <]\n OrderedDict({\n 'sql': \"select * from vec_movies where synopsis_embedding match '' and k = 0 and mean_rating < NULL\",\n 'plan': list([\n dict({\n 'detail': 'SCAN vec_movies VIRTUAL TABLE INDEX 0:3{___}___&Dd_',\n 'id': 2,\n 'parent': 0,\n }),\n ]),\n })\n# ---\n# name: test_idxstr[knn-constraint-float >=]\n OrderedDict({\n 'sql': \"select * from vec_movies where synopsis_embedding match '' and k = 0 and mean_rating >= NULL\",\n 'plan': list([\n dict({\n 'detail': 'SCAN vec_movies VIRTUAL TABLE INDEX 0:3{___}___&De_',\n 'id': 2,\n 'parent': 0,\n }),\n ]),\n })\n# ---\n# name: test_idxstr[knn-constraint-float >]\n OrderedDict({\n 'sql': \"select * from vec_movies where synopsis_embedding match '' and k = 0 and mean_rating > NULL\",\n 'plan': list([\n dict({\n 'detail': 'SCAN vec_movies VIRTUAL TABLE INDEX 0:3{___}___&Db_',\n 'id': 2,\n 'parent': 0,\n }),\n ]),\n })\n# ---\n# name: test_idxstr[knn-constraint-int !=]\n OrderedDict({\n 'sql': \"select * from vec_movies where synopsis_embedding match '' and k = 0 and num_reviews != NULL\",\n 'plan': list([\n dict({\n 'detail': 'SCAN vec_movies VIRTUAL TABLE INDEX 0:3{___}___&Cf_',\n 'id': 2,\n 'parent': 0,\n }),\n ]),\n })\n# ---\n# name: test_idxstr[knn-constraint-int <=]\n OrderedDict({\n 'sql': \"select * from vec_movies where synopsis_embedding match '' and k = 0 and num_reviews <= NULL\",\n 'plan': list([\n dict({\n 'detail': 'SCAN vec_movies VIRTUAL TABLE INDEX 0:3{___}___&Cc_',\n 'id': 2,\n 'parent': 0,\n }),\n ]),\n })\n# ---\n# name: test_idxstr[knn-constraint-int <]\n OrderedDict({\n 'sql': \"select * from vec_movies where synopsis_embedding match '' and k = 0 and num_reviews < NULL\",\n 'plan': list([\n dict({\n 'detail': 'SCAN vec_movies VIRTUAL TABLE INDEX 0:3{___}___&Cd_',\n 'id': 2,\n 'parent': 0,\n }),\n ]),\n })\n# ---\n# name: test_idxstr[knn-constraint-int >=]\n OrderedDict({\n 'sql': \"select * from vec_movies where synopsis_embedding match '' and k = 0 and num_reviews >= NULL\",\n 'plan': list([\n dict({\n 'detail': 'SCAN vec_movies VIRTUAL TABLE INDEX 0:3{___}___&Ce_',\n 'id': 2,\n 'parent': 0,\n }),\n ]),\n })\n# ---\n# name: test_idxstr[knn-constraint-int >]\n OrderedDict({\n 'sql': \"select * from vec_movies where synopsis_embedding match '' and k = 0 and num_reviews > NULL\",\n 'plan': list([\n dict({\n 'detail': 'SCAN vec_movies VIRTUAL TABLE INDEX 0:3{___}___&Cb_',\n 'id': 2,\n 'parent': 0,\n }),\n ]),\n })\n# ---\n# name: test_idxstr[knn-constraint-text !=]\n OrderedDict({\n 'sql': \"select * from vec_movies where synopsis_embedding match '' and k = 0 and genre != NULL\",\n 'plan': list([\n dict({\n 'detail': 'SCAN vec_movies VIRTUAL TABLE INDEX 0:3{___}___&Bf_',\n 'id': 2,\n 'parent': 0,\n }),\n ]),\n })\n# ---\n# name: test_idxstr[knn-constraint-text <=]\n OrderedDict({\n 'sql': \"select * from vec_movies where synopsis_embedding match '' and k = 0 and genre <= NULL\",\n 'plan': list([\n dict({\n 'detail': 'SCAN vec_movies VIRTUAL TABLE INDEX 0:3{___}___&Bc_',\n 'id': 2,\n 'parent': 0,\n }),\n ]),\n })\n# ---\n# name: test_idxstr[knn-constraint-text <]\n OrderedDict({\n 'sql': \"select * from vec_movies where synopsis_embedding match '' and k = 0 and genre < NULL\",\n 'plan': list([\n dict({\n 'detail': 'SCAN vec_movies VIRTUAL TABLE INDEX 0:3{___}___&Bd_',\n 'id': 2,\n 'parent': 0,\n }),\n ]),\n })\n# ---\n# name: test_idxstr[knn-constraint-text >=]\n OrderedDict({\n 'sql': \"select * from vec_movies where synopsis_embedding match '' and k = 0 and genre >= NULL\",\n 'plan': list([\n dict({\n 'detail': 'SCAN vec_movies VIRTUAL TABLE INDEX 0:3{___}___&Be_',\n 'id': 2,\n 'parent': 0,\n }),\n ]),\n })\n# ---\n# name: test_idxstr[knn-constraint-text >]\n OrderedDict({\n 'sql': \"select * from vec_movies where synopsis_embedding match '' and k = 0 and genre > NULL\",\n 'plan': list([\n dict({\n 'detail': 'SCAN vec_movies VIRTUAL TABLE INDEX 0:3{___}___&Bb_',\n 'id': 2,\n 'parent': 0,\n }),\n ]),\n })\n# ---\n# name: test_knn.1\n dict({\n 'error': 'OperationalError',\n 'message': 'An illegal WHERE constraint was provided on a vec0 metadata column in a KNN query. Only one of EQUALS, GREATER_THAN, LESS_THAN_OR_EQUAL, LESS_THAN, GREATER_THAN_OR_EQUAL, NOT_EQUALS is allowed.',\n })\n# ---\n# name: test_knn[sqlite_master]\n OrderedDict({\n 'sql': \"select * from sqlite_master where type = 'table' order by name\",\n 'rows': list([\n OrderedDict({\n 'type': 'table',\n 'name': 'sqlite_sequence',\n 'tbl_name': 'sqlite_sequence',\n 'rootpage': 5,\n 'sql': 'CREATE TABLE sqlite_sequence(name,seq)',\n }),\n OrderedDict({\n 'type': 'table',\n 'name': 'v',\n 'tbl_name': 'v',\n 'rootpage': 0,\n 'sql': 'CREATE VIRTUAL TABLE v using vec0(vector float[1], name text, chunk_size=8)',\n }),\n OrderedDict({\n 'type': 'table',\n 'name': 'v_chunks',\n 'tbl_name': 'v_chunks',\n 'rootpage': 4,\n 'sql': 'CREATE TABLE \"v_chunks\"(chunk_id INTEGER PRIMARY KEY AUTOINCREMENT,size INTEGER NOT NULL,validity BLOB NOT NULL,rowids BLOB NOT NULL)',\n }),\n OrderedDict({\n 'type': 'table',\n 'name': 'v_info',\n 'tbl_name': 'v_info',\n 'rootpage': 2,\n 'sql': 'CREATE TABLE \"v_info\" (key text primary key, value any)',\n }),\n OrderedDict({\n 'type': 'table',\n 'name': 'v_metadatachunks00',\n 'tbl_name': 'v_metadatachunks00',\n 'rootpage': 9,\n 'sql': 'CREATE TABLE \"v_metadatachunks00\"(rowid PRIMARY KEY, data BLOB NOT NULL)',\n }),\n OrderedDict({\n 'type': 'table',\n 'name': 'v_metadatatext00',\n 'tbl_name': 'v_metadatatext00',\n 'rootpage': 11,\n 'sql': 'CREATE TABLE \"v_metadatatext00\"(rowid PRIMARY KEY, data TEXT)',\n }),\n OrderedDict({\n 'type': 'table',\n 'name': 'v_rowids',\n 'tbl_name': 'v_rowids',\n 'rootpage': 6,\n 'sql': 'CREATE TABLE \"v_rowids\"(rowid INTEGER PRIMARY KEY AUTOINCREMENT,id,chunk_id INTEGER,chunk_offset INTEGER)',\n }),\n OrderedDict({\n 'type': 'table',\n 'name': 'v_vector_chunks00',\n 'tbl_name': 'v_vector_chunks00',\n 'rootpage': 7,\n 'sql': 'CREATE TABLE \"v_vector_chunks00\"(rowid PRIMARY KEY,vectors BLOB NOT NULL)',\n }),\n ]),\n })\n# ---\n# name: test_long_text_knn[eq-bb]\n OrderedDict({\n 'sql': \"select rowid, name, distance from v where vector match '[100]' and k = 5 and name = ?\",\n 'rows': list([\n ]),\n })\n# ---\n# name: test_long_text_knn[eq-bbbb]\n OrderedDict({\n 'sql': \"select rowid, name, distance from v where vector match '[100]' and k = 5 and name = ?\",\n 'rows': list([\n OrderedDict({\n 'rowid': 3,\n 'name': 'bbbb',\n 'distance': 97.0,\n }),\n ]),\n })\n# ---\n# name: test_long_text_knn[eq-bbbbbb]\n OrderedDict({\n 'sql': \"select rowid, name, distance from v where vector match '[100]' and k = 5 and name = ?\",\n 'rows': list([\n ]),\n })\n# ---\n# name: test_long_text_knn[eq-bbbbbbbbbbbb_aaa]\n OrderedDict({\n 'sql': \"select rowid, name, distance from v where vector match '[100]' and k = 5 and name = ?\",\n 'rows': list([\n ]),\n })\n# ---\n# name: test_long_text_knn[eq-bbbbbbbbbbbb_bbb]\n OrderedDict({\n 'sql': \"select rowid, name, distance from v where vector match '[100]' and k = 5 and name = ?\",\n 'rows': list([\n OrderedDict({\n 'rowid': 4,\n 'name': 'bbbbbbbbbbbb_bbb',\n 'distance': 96.0,\n }),\n ]),\n })\n# ---\n# name: test_long_text_knn[eq-bbbbbbbbbbbb_ccc]\n OrderedDict({\n 'sql': \"select rowid, name, distance from v where vector match '[100]' and k = 5 and name = ?\",\n 'rows': list([\n ]),\n })\n# ---\n# name: test_long_text_knn[eq-longlonglonglonglonglonglong]\n OrderedDict({\n 'sql': \"select rowid, name, distance from v where vector match '[100]' and k = 5 and name = ?\",\n 'rows': list([\n ]),\n })\n# ---\n# name: test_long_text_knn[ge-bb]\n OrderedDict({\n 'sql': \"select rowid, name, distance from v where vector match '[100]' and k = 5 and name >= ?\",\n 'rows': list([\n OrderedDict({\n 'rowid': 6,\n 'name': 'cccccccccccc_ccc',\n 'distance': 94.0,\n }),\n OrderedDict({\n 'rowid': 5,\n 'name': 'cccc',\n 'distance': 95.0,\n }),\n OrderedDict({\n 'rowid': 4,\n 'name': 'bbbbbbbbbbbb_bbb',\n 'distance': 96.0,\n }),\n OrderedDict({\n 'rowid': 3,\n 'name': 'bbbb',\n 'distance': 97.0,\n }),\n ]),\n })\n# ---\n# name: test_long_text_knn[ge-bbbb]\n OrderedDict({\n 'sql': \"select rowid, name, distance from v where vector match '[100]' and k = 5 and name >= ?\",\n 'rows': list([\n OrderedDict({\n 'rowid': 6,\n 'name': 'cccccccccccc_ccc',\n 'distance': 94.0,\n }),\n OrderedDict({\n 'rowid': 5,\n 'name': 'cccc',\n 'distance': 95.0,\n }),\n OrderedDict({\n 'rowid': 4,\n 'name': 'bbbbbbbbbbbb_bbb',\n 'distance': 96.0,\n }),\n OrderedDict({\n 'rowid': 3,\n 'name': 'bbbb',\n 'distance': 97.0,\n }),\n ]),\n })\n# ---\n# name: test_long_text_knn[ge-bbbbbb]\n OrderedDict({\n 'sql': \"select rowid, name, distance from v where vector match '[100]' and k = 5 and name >= ?\",\n 'rows': list([\n OrderedDict({\n 'rowid': 6,\n 'name': 'cccccccccccc_ccc',\n 'distance': 94.0,\n }),\n OrderedDict({\n 'rowid': 5,\n 'name': 'cccc',\n 'distance': 95.0,\n }),\n OrderedDict({\n 'rowid': 4,\n 'name': 'bbbbbbbbbbbb_bbb',\n 'distance': 96.0,\n }),\n ]),\n })\n# ---\n# name: test_long_text_knn[ge-bbbbbbbbbbbb_aaa]\n OrderedDict({\n 'sql': \"select rowid, name, distance from v where vector match '[100]' and k = 5 and name >= ?\",\n 'rows': list([\n OrderedDict({\n 'rowid': 6,\n 'name': 'cccccccccccc_ccc',\n 'distance': 94.0,\n }),\n OrderedDict({\n 'rowid': 5,\n 'name': 'cccc',\n 'distance': 95.0,\n }),\n OrderedDict({\n 'rowid': 4,\n 'name': 'bbbbbbbbbbbb_bbb',\n 'distance': 96.0,\n }),\n ]),\n })\n# ---\n# name: test_long_text_knn[ge-bbbbbbbbbbbb_bbb]\n OrderedDict({\n 'sql': \"select rowid, name, distance from v where vector match '[100]' and k = 5 and name >= ?\",\n 'rows': list([\n OrderedDict({\n 'rowid': 6,\n 'name': 'cccccccccccc_ccc',\n 'distance': 94.0,\n }),\n OrderedDict({\n 'rowid': 5,\n 'name': 'cccc',\n 'distance': 95.0,\n }),\n OrderedDict({\n 'rowid': 4,\n 'name': 'bbbbbbbbbbbb_bbb',\n 'distance': 96.0,\n }),\n ]),\n })\n# ---\n# name: test_long_text_knn[ge-bbbbbbbbbbbb_ccc]\n OrderedDict({\n 'sql': \"select rowid, name, distance from v where vector match '[100]' and k = 5 and name >= ?\",\n 'rows': list([\n OrderedDict({\n 'rowid': 6,\n 'name': 'cccccccccccc_ccc',\n 'distance': 94.0,\n }),\n OrderedDict({\n 'rowid': 5,\n 'name': 'cccc',\n 'distance': 95.0,\n }),\n ]),\n })\n# ---\n# name: test_long_text_knn[ge-longlonglonglonglonglonglong]\n OrderedDict({\n 'sql': \"select rowid, name, distance from v where vector match '[100]' and k = 5 and name >= ?\",\n 'rows': list([\n ]),\n })\n# ---\n# name: test_long_text_knn[gt-bb]\n OrderedDict({\n 'sql': \"select rowid, name, distance from v where vector match '[100]' and k = 5 and name > ?\",\n 'rows': list([\n OrderedDict({\n 'rowid': 6,\n 'name': 'cccccccccccc_ccc',\n 'distance': 94.0,\n }),\n OrderedDict({\n 'rowid': 5,\n 'name': 'cccc',\n 'distance': 95.0,\n }),\n OrderedDict({\n 'rowid': 4,\n 'name': 'bbbbbbbbbbbb_bbb',\n 'distance': 96.0,\n }),\n OrderedDict({\n 'rowid': 3,\n 'name': 'bbbb',\n 'distance': 97.0,\n }),\n ]),\n })\n# ---\n# name: test_long_text_knn[gt-bbbb]\n OrderedDict({\n 'sql': \"select rowid, name, distance from v where vector match '[100]' and k = 5 and name > ?\",\n 'rows': list([\n OrderedDict({\n 'rowid': 6,\n 'name': 'cccccccccccc_ccc',\n 'distance': 94.0,\n }),\n OrderedDict({\n 'rowid': 5,\n 'name': 'cccc',\n 'distance': 95.0,\n }),\n OrderedDict({\n 'rowid': 4,\n 'name': 'bbbbbbbbbbbb_bbb',\n 'distance': 96.0,\n }),\n ]),\n })\n# ---\n# name: test_long_text_knn[gt-bbbbbb]\n OrderedDict({\n 'sql': \"select rowid, name, distance from v where vector match '[100]' and k = 5 and name > ?\",\n 'rows': list([\n OrderedDict({\n 'rowid': 6,\n 'name': 'cccccccccccc_ccc',\n 'distance': 94.0,\n }),\n OrderedDict({\n 'rowid': 5,\n 'name': 'cccc',\n 'distance': 95.0,\n }),\n OrderedDict({\n 'rowid': 4,\n 'name': 'bbbbbbbbbbbb_bbb',\n 'distance': 96.0,\n }),\n ]),\n })\n# ---\n# name: test_long_text_knn[gt-bbbbbbbbbbbb_aaa]\n OrderedDict({\n 'sql': \"select rowid, name, distance from v where vector match '[100]' and k = 5 and name > ?\",\n 'rows': list([\n OrderedDict({\n 'rowid': 6,\n 'name': 'cccccccccccc_ccc',\n 'distance': 94.0,\n }),\n OrderedDict({\n 'rowid': 5,\n 'name': 'cccc',\n 'distance': 95.0,\n }),\n OrderedDict({\n 'rowid': 4,\n 'name': 'bbbbbbbbbbbb_bbb',\n 'distance': 96.0,\n }),\n ]),\n })\n# ---\n# name: test_long_text_knn[gt-bbbbbbbbbbbb_bbb]\n OrderedDict({\n 'sql': \"select rowid, name, distance from v where vector match '[100]' and k = 5 and name > ?\",\n 'rows': list([\n OrderedDict({\n 'rowid': 6,\n 'name': 'cccccccccccc_ccc',\n 'distance': 94.0,\n }),\n OrderedDict({\n 'rowid': 5,\n 'name': 'cccc',\n 'distance': 95.0,\n }),\n ]),\n })\n# ---\n# name: test_long_text_knn[gt-bbbbbbbbbbbb_ccc]\n OrderedDict({\n 'sql': \"select rowid, name, distance from v where vector match '[100]' and k = 5 and name > ?\",\n 'rows': list([\n OrderedDict({\n 'rowid': 6,\n 'name': 'cccccccccccc_ccc',\n 'distance': 94.0,\n }),\n OrderedDict({\n 'rowid': 5,\n 'name': 'cccc',\n 'distance': 95.0,\n }),\n ]),\n })\n# ---\n# name: test_long_text_knn[gt-longlonglonglonglonglonglong]\n OrderedDict({\n 'sql': \"select rowid, name, distance from v where vector match '[100]' and k = 5 and name > ?\",\n 'rows': list([\n ]),\n })\n# ---\n# name: test_long_text_knn[le-bb]\n OrderedDict({\n 'sql': \"select rowid, name, distance from v where vector match '[100]' and k = 5 and name <= ?\",\n 'rows': list([\n OrderedDict({\n 'rowid': 2,\n 'name': 'aaaaaaaaaaaa_aaa',\n 'distance': 98.0,\n }),\n OrderedDict({\n 'rowid': 1,\n 'name': 'aaaa',\n 'distance': 99.0,\n }),\n ]),\n })\n# ---\n# name: test_long_text_knn[le-bbbb]\n OrderedDict({\n 'sql': \"select rowid, name, distance from v where vector match '[100]' and k = 5 and name <= ?\",\n 'rows': list([\n OrderedDict({\n 'rowid': 3,\n 'name': 'bbbb',\n 'distance': 97.0,\n }),\n OrderedDict({\n 'rowid': 2,\n 'name': 'aaaaaaaaaaaa_aaa',\n 'distance': 98.0,\n }),\n OrderedDict({\n 'rowid': 1,\n 'name': 'aaaa',\n 'distance': 99.0,\n }),\n ]),\n })\n# ---\n# name: test_long_text_knn[le-bbbbbb]\n OrderedDict({\n 'sql': \"select rowid, name, distance from v where vector match '[100]' and k = 5 and name <= ?\",\n 'rows': list([\n OrderedDict({\n 'rowid': 3,\n 'name': 'bbbb',\n 'distance': 97.0,\n }),\n OrderedDict({\n 'rowid': 2,\n 'name': 'aaaaaaaaaaaa_aaa',\n 'distance': 98.0,\n }),\n OrderedDict({\n 'rowid': 1,\n 'name': 'aaaa',\n 'distance': 99.0,\n }),\n ]),\n })\n# ---\n# name: test_long_text_knn[le-bbbbbbbbbbbb_aaa]\n OrderedDict({\n 'sql': \"select rowid, name, distance from v where vector match '[100]' and k = 5 and name <= ?\",\n 'rows': list([\n OrderedDict({\n 'rowid': 3,\n 'name': 'bbbb',\n 'distance': 97.0,\n }),\n OrderedDict({\n 'rowid': 2,\n 'name': 'aaaaaaaaaaaa_aaa',\n 'distance': 98.0,\n }),\n OrderedDict({\n 'rowid': 1,\n 'name': 'aaaa',\n 'distance': 99.0,\n }),\n ]),\n })\n# ---\n# name: test_long_text_knn[le-bbbbbbbbbbbb_bbb]\n OrderedDict({\n 'sql': \"select rowid, name, distance from v where vector match '[100]' and k = 5 and name <= ?\",\n 'rows': list([\n OrderedDict({\n 'rowid': 4,\n 'name': 'bbbbbbbbbbbb_bbb',\n 'distance': 96.0,\n }),\n OrderedDict({\n 'rowid': 3,\n 'name': 'bbbb',\n 'distance': 97.0,\n }),\n OrderedDict({\n 'rowid': 2,\n 'name': 'aaaaaaaaaaaa_aaa',\n 'distance': 98.0,\n }),\n OrderedDict({\n 'rowid': 1,\n 'name': 'aaaa',\n 'distance': 99.0,\n }),\n ]),\n })\n# ---\n# name: test_long_text_knn[le-bbbbbbbbbbbb_ccc]\n OrderedDict({\n 'sql': \"select rowid, name, distance from v where vector match '[100]' and k = 5 and name <= ?\",\n 'rows': list([\n OrderedDict({\n 'rowid': 4,\n 'name': 'bbbbbbbbbbbb_bbb',\n 'distance': 96.0,\n }),\n OrderedDict({\n 'rowid': 3,\n 'name': 'bbbb',\n 'distance': 97.0,\n }),\n OrderedDict({\n 'rowid': 2,\n 'name': 'aaaaaaaaaaaa_aaa',\n 'distance': 98.0,\n }),\n OrderedDict({\n 'rowid': 1,\n 'name': 'aaaa',\n 'distance': 99.0,\n }),\n ]),\n })\n# ---\n# name: test_long_text_knn[le-longlonglonglonglonglonglong]\n OrderedDict({\n 'sql': \"select rowid, name, distance from v where vector match '[100]' and k = 5 and name <= ?\",\n 'rows': list([\n OrderedDict({\n 'rowid': 6,\n 'name': 'cccccccccccc_ccc',\n 'distance': 94.0,\n }),\n OrderedDict({\n 'rowid': 5,\n 'name': 'cccc',\n 'distance': 95.0,\n }),\n OrderedDict({\n 'rowid': 4,\n 'name': 'bbbbbbbbbbbb_bbb',\n 'distance': 96.0,\n }),\n OrderedDict({\n 'rowid': 3,\n 'name': 'bbbb',\n 'distance': 97.0,\n }),\n OrderedDict({\n 'rowid': 2,\n 'name': 'aaaaaaaaaaaa_aaa',\n 'distance': 98.0,\n }),\n ]),\n })\n# ---\n# name: test_long_text_knn[lt-bb]\n OrderedDict({\n 'sql': \"select rowid, name, distance from v where vector match '[100]' and k = 5 and name < ?\",\n 'rows': list([\n OrderedDict({\n 'rowid': 2,\n 'name': 'aaaaaaaaaaaa_aaa',\n 'distance': 98.0,\n }),\n OrderedDict({\n 'rowid': 1,\n 'name': 'aaaa',\n 'distance': 99.0,\n }),\n ]),\n })\n# ---\n# name: test_long_text_knn[lt-bbbb]\n OrderedDict({\n 'sql': \"select rowid, name, distance from v where vector match '[100]' and k = 5 and name < ?\",\n 'rows': list([\n OrderedDict({\n 'rowid': 2,\n 'name': 'aaaaaaaaaaaa_aaa',\n 'distance': 98.0,\n }),\n OrderedDict({\n 'rowid': 1,\n 'name': 'aaaa',\n 'distance': 99.0,\n }),\n ]),\n })\n# ---\n# name: test_long_text_knn[lt-bbbbbb]\n OrderedDict({\n 'sql': \"select rowid, name, distance from v where vector match '[100]' and k = 5 and name < ?\",\n 'rows': list([\n OrderedDict({\n 'rowid': 3,\n 'name': 'bbbb',\n 'distance': 97.0,\n }),\n OrderedDict({\n 'rowid': 2,\n 'name': 'aaaaaaaaaaaa_aaa',\n 'distance': 98.0,\n }),\n OrderedDict({\n 'rowid': 1,\n 'name': 'aaaa',\n 'distance': 99.0,\n }),\n ]),\n })\n# ---\n# name: test_long_text_knn[lt-bbbbbbbbbbbb_aaa]\n OrderedDict({\n 'sql': \"select rowid, name, distance from v where vector match '[100]' and k = 5 and name < ?\",\n 'rows': list([\n OrderedDict({\n 'rowid': 3,\n 'name': 'bbbb',\n 'distance': 97.0,\n }),\n OrderedDict({\n 'rowid': 2,\n 'name': 'aaaaaaaaaaaa_aaa',\n 'distance': 98.0,\n }),\n OrderedDict({\n 'rowid': 1,\n 'name': 'aaaa',\n 'distance': 99.0,\n }),\n ]),\n })\n# ---\n# name: test_long_text_knn[lt-bbbbbbbbbbbb_bbb]\n OrderedDict({\n 'sql': \"select rowid, name, distance from v where vector match '[100]' and k = 5 and name < ?\",\n 'rows': list([\n OrderedDict({\n 'rowid': 3,\n 'name': 'bbbb',\n 'distance': 97.0,\n }),\n OrderedDict({\n 'rowid': 2,\n 'name': 'aaaaaaaaaaaa_aaa',\n 'distance': 98.0,\n }),\n OrderedDict({\n 'rowid': 1,\n 'name': 'aaaa',\n 'distance': 99.0,\n }),\n ]),\n })\n# ---\n# name: test_long_text_knn[lt-bbbbbbbbbbbb_ccc]\n OrderedDict({\n 'sql': \"select rowid, name, distance from v where vector match '[100]' and k = 5 and name < ?\",\n 'rows': list([\n OrderedDict({\n 'rowid': 4,\n 'name': 'bbbbbbbbbbbb_bbb',\n 'distance': 96.0,\n }),\n OrderedDict({\n 'rowid': 3,\n 'name': 'bbbb',\n 'distance': 97.0,\n }),\n OrderedDict({\n 'rowid': 2,\n 'name': 'aaaaaaaaaaaa_aaa',\n 'distance': 98.0,\n }),\n OrderedDict({\n 'rowid': 1,\n 'name': 'aaaa',\n 'distance': 99.0,\n }),\n ]),\n })\n# ---\n# name: test_long_text_knn[lt-longlonglonglonglonglonglong]\n OrderedDict({\n 'sql': \"select rowid, name, distance from v where vector match '[100]' and k = 5 and name < ?\",\n 'rows': list([\n OrderedDict({\n 'rowid': 6,\n 'name': 'cccccccccccc_ccc',\n 'distance': 94.0,\n }),\n OrderedDict({\n 'rowid': 5,\n 'name': 'cccc',\n 'distance': 95.0,\n }),\n OrderedDict({\n 'rowid': 4,\n 'name': 'bbbbbbbbbbbb_bbb',\n 'distance': 96.0,\n }),\n OrderedDict({\n 'rowid': 3,\n 'name': 'bbbb',\n 'distance': 97.0,\n }),\n OrderedDict({\n 'rowid': 2,\n 'name': 'aaaaaaaaaaaa_aaa',\n 'distance': 98.0,\n }),\n ]),\n })\n# ---\n# name: test_long_text_knn[ne-bb]\n OrderedDict({\n 'sql': \"select rowid, name, distance from v where vector match '[100]' and k = 5 and name != ?\",\n 'rows': list([\n OrderedDict({\n 'rowid': 6,\n 'name': 'cccccccccccc_ccc',\n 'distance': 94.0,\n }),\n OrderedDict({\n 'rowid': 5,\n 'name': 'cccc',\n 'distance': 95.0,\n }),\n OrderedDict({\n 'rowid': 4,\n 'name': 'bbbbbbbbbbbb_bbb',\n 'distance': 96.0,\n }),\n OrderedDict({\n 'rowid': 3,\n 'name': 'bbbb',\n 'distance': 97.0,\n }),\n OrderedDict({\n 'rowid': 2,\n 'name': 'aaaaaaaaaaaa_aaa',\n 'distance': 98.0,\n }),\n ]),\n })\n# ---\n# name: test_long_text_knn[ne-bbbb]\n OrderedDict({\n 'sql': \"select rowid, name, distance from v where vector match '[100]' and k = 5 and name != ?\",\n 'rows': list([\n OrderedDict({\n 'rowid': 6,\n 'name': 'cccccccccccc_ccc',\n 'distance': 94.0,\n }),\n OrderedDict({\n 'rowid': 5,\n 'name': 'cccc',\n 'distance': 95.0,\n }),\n OrderedDict({\n 'rowid': 4,\n 'name': 'bbbbbbbbbbbb_bbb',\n 'distance': 96.0,\n }),\n OrderedDict({\n 'rowid': 2,\n 'name': 'aaaaaaaaaaaa_aaa',\n 'distance': 98.0,\n }),\n OrderedDict({\n 'rowid': 1,\n 'name': 'aaaa',\n 'distance': 99.0,\n }),\n ]),\n })\n# ---\n# name: test_long_text_knn[ne-bbbbbb]\n OrderedDict({\n 'sql': \"select rowid, name, distance from v where vector match '[100]' and k = 5 and name != ?\",\n 'rows': list([\n OrderedDict({\n 'rowid': 6,\n 'name': 'cccccccccccc_ccc',\n 'distance': 94.0,\n }),\n OrderedDict({\n 'rowid': 5,\n 'name': 'cccc',\n 'distance': 95.0,\n }),\n OrderedDict({\n 'rowid': 4,\n 'name': 'bbbbbbbbbbbb_bbb',\n 'distance': 96.0,\n }),\n OrderedDict({\n 'rowid': 3,\n 'name': 'bbbb',\n 'distance': 97.0,\n }),\n OrderedDict({\n 'rowid': 2,\n 'name': 'aaaaaaaaaaaa_aaa',\n 'distance': 98.0,\n }),\n ]),\n })\n# ---\n# name: test_long_text_knn[ne-bbbbbbbbbbbb_aaa]\n OrderedDict({\n 'sql': \"select rowid, name, distance from v where vector match '[100]' and k = 5 and name != ?\",\n 'rows': list([\n OrderedDict({\n 'rowid': 6,\n 'name': 'cccccccccccc_ccc',\n 'distance': 94.0,\n }),\n OrderedDict({\n 'rowid': 5,\n 'name': 'cccc',\n 'distance': 95.0,\n }),\n OrderedDict({\n 'rowid': 4,\n 'name': 'bbbbbbbbbbbb_bbb',\n 'distance': 96.0,\n }),\n OrderedDict({\n 'rowid': 3,\n 'name': 'bbbb',\n 'distance': 97.0,\n }),\n OrderedDict({\n 'rowid': 2,\n 'name': 'aaaaaaaaaaaa_aaa',\n 'distance': 98.0,\n }),\n ]),\n })\n# ---\n# name: test_long_text_knn[ne-bbbbbbbbbbbb_bbb]\n OrderedDict({\n 'sql': \"select rowid, name, distance from v where vector match '[100]' and k = 5 and name != ?\",\n 'rows': list([\n OrderedDict({\n 'rowid': 6,\n 'name': 'cccccccccccc_ccc',\n 'distance': 94.0,\n }),\n OrderedDict({\n 'rowid': 5,\n 'name': 'cccc',\n 'distance': 95.0,\n }),\n OrderedDict({\n 'rowid': 3,\n 'name': 'bbbb',\n 'distance': 97.0,\n }),\n OrderedDict({\n 'rowid': 2,\n 'name': 'aaaaaaaaaaaa_aaa',\n 'distance': 98.0,\n }),\n OrderedDict({\n 'rowid': 1,\n 'name': 'aaaa',\n 'distance': 99.0,\n }),\n ]),\n })\n# ---\n# name: test_long_text_knn[ne-bbbbbbbbbbbb_ccc]\n OrderedDict({\n 'sql': \"select rowid, name, distance from v where vector match '[100]' and k = 5 and name != ?\",\n 'rows': list([\n OrderedDict({\n 'rowid': 6,\n 'name': 'cccccccccccc_ccc',\n 'distance': 94.0,\n }),\n OrderedDict({\n 'rowid': 5,\n 'name': 'cccc',\n 'distance': 95.0,\n }),\n OrderedDict({\n 'rowid': 4,\n 'name': 'bbbbbbbbbbbb_bbb',\n 'distance': 96.0,\n }),\n OrderedDict({\n 'rowid': 3,\n 'name': 'bbbb',\n 'distance': 97.0,\n }),\n OrderedDict({\n 'rowid': 2,\n 'name': 'aaaaaaaaaaaa_aaa',\n 'distance': 98.0,\n }),\n ]),\n })\n# ---\n# name: test_long_text_knn[ne-longlonglonglonglonglonglong]\n OrderedDict({\n 'sql': \"select rowid, name, distance from v where vector match '[100]' and k = 5 and name != ?\",\n 'rows': list([\n OrderedDict({\n 'rowid': 6,\n 'name': 'cccccccccccc_ccc',\n 'distance': 94.0,\n }),\n OrderedDict({\n 'rowid': 5,\n 'name': 'cccc',\n 'distance': 95.0,\n }),\n OrderedDict({\n 'rowid': 4,\n 'name': 'bbbbbbbbbbbb_bbb',\n 'distance': 96.0,\n }),\n OrderedDict({\n 'rowid': 3,\n 'name': 'bbbb',\n 'distance': 97.0,\n }),\n OrderedDict({\n 'rowid': 2,\n 'name': 'aaaaaaaaaaaa_aaa',\n 'distance': 98.0,\n }),\n ]),\n })\n# ---\n# name: test_long_text_updates\n dict({\n 'v_chunks': OrderedDict({\n 'sql': 'select * from v_chunks',\n 'rows': list([\n ]),\n }),\n 'v_metadatachunks00': OrderedDict({\n 'sql': 'select * from v_metadatachunks00',\n 'rows': list([\n ]),\n }),\n 'v_metadatatext00': OrderedDict({\n 'sql': 'select * from v_metadatatext00',\n 'rows': list([\n ]),\n }),\n 'v_rowids': OrderedDict({\n 'sql': 'select * from v_rowids',\n 'rows': list([\n ]),\n }),\n 'v_vector_chunks00': OrderedDict({\n 'sql': 'select * from v_vector_chunks00',\n 'rows': list([\n ]),\n }),\n })\n# ---\n# name: test_long_text_updates.1\n OrderedDict({\n 'sql': 'select * from v',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'vector': b'\\x11\\x11\\x11\\x11',\n 'name': '123456789a12',\n }),\n OrderedDict({\n 'rowid': 2,\n 'vector': b'\\x11\\x11\\x11\\x11',\n 'name': '123456789a123',\n }),\n ]),\n })\n# ---\n# name: test_long_text_updates.2\n dict({\n 'v_chunks': OrderedDict({\n 'sql': 'select * from v_chunks',\n 'rows': list([\n OrderedDict({\n 'chunk_id': 1,\n 'size': 8,\n 'validity': b'\\x03',\n 'rowids': b'\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n 'v_metadatachunks00': OrderedDict({\n 'sql': 'select * from v_metadatachunks00',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'data': b'\\x0c\\x00\\x00\\x00123456789a12\\r\\x00\\x00\\x00123456789a12\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n 'v_metadatatext00': OrderedDict({\n 'sql': 'select * from v_metadatatext00',\n 'rows': list([\n OrderedDict({\n 'rowid': 2,\n 'data': '123456789a123',\n }),\n ]),\n }),\n 'v_rowids': OrderedDict({\n 'sql': 'select * from v_rowids',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 0,\n }),\n OrderedDict({\n 'rowid': 2,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 1,\n }),\n ]),\n }),\n 'v_vector_chunks00': OrderedDict({\n 'sql': 'select * from v_vector_chunks00',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'vectors': b'\\x11\\x11\\x11\\x11\\x11\\x11\\x11\\x11\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n })\n# ---\n# name: test_normal.1\n dict({\n 'v_chunks': OrderedDict({\n 'sql': 'select * from v_chunks',\n 'rows': list([\n ]),\n }),\n 'v_metadatachunks00': OrderedDict({\n 'sql': 'select * from v_metadatachunks00',\n 'rows': list([\n ]),\n }),\n 'v_metadatachunks01': OrderedDict({\n 'sql': 'select * from v_metadatachunks01',\n 'rows': list([\n ]),\n }),\n 'v_metadatachunks02': OrderedDict({\n 'sql': 'select * from v_metadatachunks02',\n 'rows': list([\n ]),\n }),\n 'v_metadatachunks03': OrderedDict({\n 'sql': 'select * from v_metadatachunks03',\n 'rows': list([\n ]),\n }),\n 'v_metadatatext03': OrderedDict({\n 'sql': 'select * from v_metadatatext03',\n 'rows': list([\n ]),\n }),\n 'v_rowids': OrderedDict({\n 'sql': 'select * from v_rowids',\n 'rows': list([\n ]),\n }),\n 'v_vector_chunks00': OrderedDict({\n 'sql': 'select * from v_vector_chunks00',\n 'rows': list([\n ]),\n }),\n })\n# ---\n# name: test_normal.2\n OrderedDict({\n 'sql': 'insert into v(vector, b, n, f, t) values (?, ?, ?, ?, ?)',\n 'rows': list([\n ]),\n })\n# ---\n# name: test_normal.3\n OrderedDict({\n 'sql': 'insert into v(vector, b, n, f, t) values (?, ?, ?, ?, ?)',\n 'rows': list([\n ]),\n })\n# ---\n# name: test_normal.4\n OrderedDict({\n 'sql': 'insert into v(vector, b, n, f, t) values (?, ?, ?, ?, ?)',\n 'rows': list([\n ]),\n })\n# ---\n# name: test_normal.5\n OrderedDict({\n 'sql': 'select * from v',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'vector': b'\\x11\\x11\\x11\\x11',\n 'b': 1,\n 'n': 1,\n 'f': 1.1,\n 't': 'one',\n }),\n OrderedDict({\n 'rowid': 2,\n 'vector': b'\"\"\"\"',\n 'b': 1,\n 'n': 2,\n 'f': 2.2,\n 't': 'two',\n }),\n OrderedDict({\n 'rowid': 3,\n 'vector': b'3333',\n 'b': 1,\n 'n': 3,\n 'f': 3.3,\n 't': 'three',\n }),\n ]),\n })\n# ---\n# name: test_normal.6\n dict({\n 'v_chunks': OrderedDict({\n 'sql': 'select * from v_chunks',\n 'rows': list([\n OrderedDict({\n 'chunk_id': 1,\n 'size': 8,\n 'validity': b'\\x07',\n 'rowids': b'\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x03\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n 'v_metadatachunks00': OrderedDict({\n 'sql': 'select * from v_metadatachunks00',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'data': b'\\x07',\n }),\n ]),\n }),\n 'v_metadatachunks01': OrderedDict({\n 'sql': 'select * from v_metadatachunks01',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'data': b'\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x03\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n 'v_metadatachunks02': OrderedDict({\n 'sql': 'select * from v_metadatachunks02',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'data': b'\\x9a\\x99\\x99\\x99\\x99\\x99\\xf1?\\x9a\\x99\\x99\\x99\\x99\\x99\\x01@ffffff\\n@\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n 'v_metadatachunks03': OrderedDict({\n 'sql': 'select * from v_metadatachunks03',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'data': b'\\x03\\x00\\x00\\x00one\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x03\\x00\\x00\\x00two\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x05\\x00\\x00\\x00three\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n 'v_metadatatext03': OrderedDict({\n 'sql': 'select * from v_metadatatext03',\n 'rows': list([\n ]),\n }),\n 'v_rowids': OrderedDict({\n 'sql': 'select * from v_rowids',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 0,\n }),\n OrderedDict({\n 'rowid': 2,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 1,\n }),\n OrderedDict({\n 'rowid': 3,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 2,\n }),\n ]),\n }),\n 'v_vector_chunks00': OrderedDict({\n 'sql': 'select * from v_vector_chunks00',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'vectors': b'\\x11\\x11\\x11\\x11\"\"\"\"3333\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n })\n# ---\n# name: test_normal.7\n OrderedDict({\n 'sql': 'drop table v',\n 'rows': list([\n ]),\n })\n# ---\n# name: test_normal.8\n OrderedDict({\n 'sql': 'select * from sqlite_master',\n 'rows': list([\n OrderedDict({\n 'type': 'table',\n 'name': 'sqlite_sequence',\n 'tbl_name': 'sqlite_sequence',\n 'rootpage': 5,\n 'sql': 'CREATE TABLE sqlite_sequence(name,seq)',\n }),\n ]),\n })\n# ---\n# name: test_normal[sqlite_master]\n OrderedDict({\n 'sql': \"select * from sqlite_master where type = 'table' order by name\",\n 'rows': list([\n OrderedDict({\n 'type': 'table',\n 'name': 'sqlite_sequence',\n 'tbl_name': 'sqlite_sequence',\n 'rootpage': 5,\n 'sql': 'CREATE TABLE sqlite_sequence(name,seq)',\n }),\n OrderedDict({\n 'type': 'table',\n 'name': 'v',\n 'tbl_name': 'v',\n 'rootpage': 0,\n 'sql': 'CREATE VIRTUAL TABLE v using vec0(vector float[1], b boolean, n int, f float, t text, chunk_size=8)',\n }),\n OrderedDict({\n 'type': 'table',\n 'name': 'v_chunks',\n 'tbl_name': 'v_chunks',\n 'rootpage': 4,\n 'sql': 'CREATE TABLE \"v_chunks\"(chunk_id INTEGER PRIMARY KEY AUTOINCREMENT,size INTEGER NOT NULL,validity BLOB NOT NULL,rowids BLOB NOT NULL)',\n }),\n OrderedDict({\n 'type': 'table',\n 'name': 'v_info',\n 'tbl_name': 'v_info',\n 'rootpage': 2,\n 'sql': 'CREATE TABLE \"v_info\" (key text primary key, value any)',\n }),\n OrderedDict({\n 'type': 'table',\n 'name': 'v_metadatachunks00',\n 'tbl_name': 'v_metadatachunks00',\n 'rootpage': 9,\n 'sql': 'CREATE TABLE \"v_metadatachunks00\"(rowid PRIMARY KEY, data BLOB NOT NULL)',\n }),\n OrderedDict({\n 'type': 'table',\n 'name': 'v_metadatachunks01',\n 'tbl_name': 'v_metadatachunks01',\n 'rootpage': 11,\n 'sql': 'CREATE TABLE \"v_metadatachunks01\"(rowid PRIMARY KEY, data BLOB NOT NULL)',\n }),\n OrderedDict({\n 'type': 'table',\n 'name': 'v_metadatachunks02',\n 'tbl_name': 'v_metadatachunks02',\n 'rootpage': 13,\n 'sql': 'CREATE TABLE \"v_metadatachunks02\"(rowid PRIMARY KEY, data BLOB NOT NULL)',\n }),\n OrderedDict({\n 'type': 'table',\n 'name': 'v_metadatachunks03',\n 'tbl_name': 'v_metadatachunks03',\n 'rootpage': 15,\n 'sql': 'CREATE TABLE \"v_metadatachunks03\"(rowid PRIMARY KEY, data BLOB NOT NULL)',\n }),\n OrderedDict({\n 'type': 'table',\n 'name': 'v_metadatatext03',\n 'tbl_name': 'v_metadatatext03',\n 'rootpage': 17,\n 'sql': 'CREATE TABLE \"v_metadatatext03\"(rowid PRIMARY KEY, data TEXT)',\n }),\n OrderedDict({\n 'type': 'table',\n 'name': 'v_rowids',\n 'tbl_name': 'v_rowids',\n 'rootpage': 6,\n 'sql': 'CREATE TABLE \"v_rowids\"(rowid INTEGER PRIMARY KEY AUTOINCREMENT,id,chunk_id INTEGER,chunk_offset INTEGER)',\n }),\n OrderedDict({\n 'type': 'table',\n 'name': 'v_vector_chunks00',\n 'tbl_name': 'v_vector_chunks00',\n 'rootpage': 7,\n 'sql': 'CREATE TABLE \"v_vector_chunks00\"(rowid PRIMARY KEY,vectors BLOB NOT NULL)',\n }),\n ]),\n })\n# ---\n# name: test_stress\n dict({\n 'vec_movies_auxiliary': OrderedDict({\n 'sql': 'select * from vec_movies_auxiliary',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'value00': 'The Conjuring',\n }),\n OrderedDict({\n 'rowid': 2,\n 'value00': 'Dumb and Dumber',\n }),\n OrderedDict({\n 'rowid': 3,\n 'value00': 'Interstellar',\n }),\n OrderedDict({\n 'rowid': 4,\n 'value00': 'The Lord of the Rings: The Fellowship of the Ring',\n }),\n OrderedDict({\n 'rowid': 5,\n 'value00': 'An Inconvenient Truth',\n }),\n OrderedDict({\n 'rowid': 6,\n 'value00': 'Hereditary',\n }),\n OrderedDict({\n 'rowid': 7,\n 'value00': 'Anchorman: The Legend of Ron Burgundy',\n }),\n OrderedDict({\n 'rowid': 8,\n 'value00': 'Blade Runner 2049',\n }),\n OrderedDict({\n 'rowid': 9,\n 'value00': \"Harry Potter and the Sorcerer's Stone\",\n }),\n OrderedDict({\n 'rowid': 10,\n 'value00': 'Free Solo',\n }),\n OrderedDict({\n 'rowid': 11,\n 'value00': 'Get Out',\n }),\n OrderedDict({\n 'rowid': 12,\n 'value00': 'The Hangover',\n }),\n OrderedDict({\n 'rowid': 13,\n 'value00': 'The Matrix',\n }),\n OrderedDict({\n 'rowid': 14,\n 'value00': \"Pan's Labyrinth\",\n }),\n OrderedDict({\n 'rowid': 15,\n 'value00': '13th',\n }),\n OrderedDict({\n 'rowid': 16,\n 'value00': 'It Follows',\n }),\n OrderedDict({\n 'rowid': 17,\n 'value00': 'Step Brothers',\n }),\n OrderedDict({\n 'rowid': 18,\n 'value00': 'Inception',\n }),\n OrderedDict({\n 'rowid': 19,\n 'value00': 'The Shape of Water',\n }),\n OrderedDict({\n 'rowid': 20,\n 'value00': \"Won't You Be My Neighbor?\",\n }),\n OrderedDict({\n 'rowid': 21,\n 'value00': 'Gravity',\n }),\n OrderedDict({\n 'rowid': 22,\n 'value00': 'Dune',\n }),\n OrderedDict({\n 'rowid': 23,\n 'value00': 'The Martian',\n }),\n OrderedDict({\n 'rowid': 24,\n 'value00': 'A Quiet Place',\n }),\n OrderedDict({\n 'rowid': 25,\n 'value00': 'The Chronicles of Narnia: The Lion, the Witch and the Wardrobe',\n }),\n ]),\n }),\n 'vec_movies_chunks': OrderedDict({\n 'sql': 'select * from vec_movies_chunks',\n 'rows': list([\n OrderedDict({\n 'chunk_id': 1,\n 'size': 8,\n 'validity': b'\\xff',\n 'rowids': b'\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x03\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x04\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x05\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x06\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x07\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x08\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n OrderedDict({\n 'chunk_id': 2,\n 'size': 8,\n 'validity': b'\\xff',\n 'rowids': b'\\t\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\n\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x0b\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x0c\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\r\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x0e\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x0f\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x10\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n OrderedDict({\n 'chunk_id': 3,\n 'size': 8,\n 'validity': b'\\xff',\n 'rowids': b'\\x11\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x12\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x13\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x14\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x15\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x16\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x17\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x18\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n OrderedDict({\n 'chunk_id': 4,\n 'size': 8,\n 'validity': b'\\x01',\n 'rowids': b'\\x19\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n 'vec_movies_metadatachunks00': OrderedDict({\n 'sql': 'select * from vec_movies_metadatachunks00',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'data': b'p',\n }),\n OrderedDict({\n 'rowid': 2,\n 'data': b'U',\n }),\n OrderedDict({\n 'rowid': 3,\n 'data': b'\\xff',\n }),\n OrderedDict({\n 'rowid': 4,\n 'data': b'\\x01',\n }),\n ]),\n }),\n 'vec_movies_metadatachunks01': OrderedDict({\n 'sql': 'select * from vec_movies_metadatachunks01',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'data': b'\\x06\\x00\\x00\\x00horror\\x00\\x00\\x00\\x00\\x00\\x00\\x06\\x00\\x00\\x00comedy\\x00\\x00\\x00\\x00\\x00\\x00\\x05\\x00\\x00\\x00scifi\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x07\\x00\\x00\\x00fantasy\\x00\\x00\\x00\\x00\\x00\\x0b\\x00\\x00\\x00documentary\\x00\\x06\\x00\\x00\\x00horror\\x00\\x00\\x00\\x00\\x00\\x00\\x06\\x00\\x00\\x00comedy\\x00\\x00\\x00\\x00\\x00\\x00\\x05\\x00\\x00\\x00scifi\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n OrderedDict({\n 'rowid': 2,\n 'data': b'\\x07\\x00\\x00\\x00fantasy\\x00\\x00\\x00\\x00\\x00\\x0b\\x00\\x00\\x00documentary\\x00\\x06\\x00\\x00\\x00horror\\x00\\x00\\x00\\x00\\x00\\x00\\x06\\x00\\x00\\x00comedy\\x00\\x00\\x00\\x00\\x00\\x00\\x05\\x00\\x00\\x00scifi\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x07\\x00\\x00\\x00fantasy\\x00\\x00\\x00\\x00\\x00\\x0b\\x00\\x00\\x00documentary\\x00\\x06\\x00\\x00\\x00horror\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n OrderedDict({\n 'rowid': 3,\n 'data': b'\\x06\\x00\\x00\\x00comedy\\x00\\x00\\x00\\x00\\x00\\x00\\x05\\x00\\x00\\x00scifi\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x07\\x00\\x00\\x00fantasy\\x00\\x00\\x00\\x00\\x00\\x0b\\x00\\x00\\x00documentary\\x00\\x05\\x00\\x00\\x00scifi\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x05\\x00\\x00\\x00scifi\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x05\\x00\\x00\\x00scifi\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x06\\x00\\x00\\x00horror\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n OrderedDict({\n 'rowid': 4,\n 'data': b'\\x07\\x00\\x00\\x00fantasy\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n 'vec_movies_metadatachunks02': OrderedDict({\n 'sql': 'select * from vec_movies_metadatachunks02',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'data': b'\\x99\\x00\\x00\\x00\\x00\\x00\\x00\\x00~\\x01\\x00\\x00\\x00\\x00\\x00\\x005\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\xd2\\x00\\x00\\x00\\x00\\x00\\x00\\x00]\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\xa7\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\xe2\\x01\\x00\\x00\\x00\\x00\\x00\\x00-\\x01\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n OrderedDict({\n 'rowid': 2,\n 'data': b'\\x86\\x00\\x00\\x00\\x00\\x00\\x00\\x00B\\x00\\x00\\x00\\x00\\x00\\x00\\x00X\\x00\\x00\\x00\\x00\\x00\\x00\\x00;\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\xa7\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x13\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\xbf\\x00\\x00\\x00\\x00\\x00\\x00\\x00:\\x01\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n OrderedDict({\n 'rowid': 3,\n 'data': b'J\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\xc9\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x8f\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\xba\\x00\\x00\\x00\\x00\\x00\\x00\\x00V\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\xc3\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\n\\x02\\x00\\x00\\x00\\x00\\x00\\x00\\x0f\\x01\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n OrderedDict({\n 'rowid': 4,\n 'data': b'6\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n 'vec_movies_metadatachunks03': OrderedDict({\n 'sql': 'select * from vec_movies_metadatachunks03',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'data': b'ffffff\\x12@\\xcd\\xcc\\xcc\\xcc\\xcc\\xcc\\x04@\\x00\\x00\\x00\\x00\\x00\\x00\\x14@\\xcd\\xcc\\xcc\\xcc\\xcc\\xcc\\x10@333333\\x0b@\\xcd\\xcc\\xcc\\xcc\\xcc\\xcc\\x12@333333\\x07@\\x00\\x00\\x00\\x00\\x00\\x00\\x14@',\n }),\n OrderedDict({\n 'rowid': 2,\n 'data': b'ffffff\\x10@\\x9a\\x99\\x99\\x99\\x99\\x99\\t@\\x9a\\x99\\x99\\x99\\x99\\x99\\x13@ffffff\\x06@\\x00\\x00\\x00\\x00\\x00\\x00\\x12@\\xcd\\xcc\\xcc\\xcc\\xcc\\xcc\\x0c@\\x9a\\x99\\x99\\x99\\x99\\x99\\x11@333333\\x11@',\n }),\n OrderedDict({\n 'rowid': 3,\n 'data': b'\\x00\\x00\\x00\\x00\\x00\\x00\\x08@\\x00\\x00\\x00\\x00\\x00\\x00\\x14@\\x9a\\x99\\x99\\x99\\x99\\x99\\x05@333333\\x13@\\x00\\x00\\x00\\x00\\x00\\x00\\x10@\\x9a\\x99\\x99\\x99\\x99\\x99\\x11@ffffff\\x12@333333\\x11@',\n }),\n OrderedDict({\n 'rowid': 4,\n 'data': b'333333\\x0f@\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n 'vec_movies_metadatatext01': OrderedDict({\n 'sql': 'select * from vec_movies_metadatatext01',\n 'rows': list([\n ]),\n }),\n 'vec_movies_rowids': OrderedDict({\n 'sql': 'select * from vec_movies_rowids',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 0,\n }),\n OrderedDict({\n 'rowid': 2,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 1,\n }),\n OrderedDict({\n 'rowid': 3,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 2,\n }),\n OrderedDict({\n 'rowid': 4,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 3,\n }),\n OrderedDict({\n 'rowid': 5,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 4,\n }),\n OrderedDict({\n 'rowid': 6,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 5,\n }),\n OrderedDict({\n 'rowid': 7,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 6,\n }),\n OrderedDict({\n 'rowid': 8,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 7,\n }),\n OrderedDict({\n 'rowid': 9,\n 'id': None,\n 'chunk_id': 2,\n 'chunk_offset': 0,\n }),\n OrderedDict({\n 'rowid': 10,\n 'id': None,\n 'chunk_id': 2,\n 'chunk_offset': 1,\n }),\n OrderedDict({\n 'rowid': 11,\n 'id': None,\n 'chunk_id': 2,\n 'chunk_offset': 2,\n }),\n OrderedDict({\n 'rowid': 12,\n 'id': None,\n 'chunk_id': 2,\n 'chunk_offset': 3,\n }),\n OrderedDict({\n 'rowid': 13,\n 'id': None,\n 'chunk_id': 2,\n 'chunk_offset': 4,\n }),\n OrderedDict({\n 'rowid': 14,\n 'id': None,\n 'chunk_id': 2,\n 'chunk_offset': 5,\n }),\n OrderedDict({\n 'rowid': 15,\n 'id': None,\n 'chunk_id': 2,\n 'chunk_offset': 6,\n }),\n OrderedDict({\n 'rowid': 16,\n 'id': None,\n 'chunk_id': 2,\n 'chunk_offset': 7,\n }),\n OrderedDict({\n 'rowid': 17,\n 'id': None,\n 'chunk_id': 3,\n 'chunk_offset': 0,\n }),\n OrderedDict({\n 'rowid': 18,\n 'id': None,\n 'chunk_id': 3,\n 'chunk_offset': 1,\n }),\n OrderedDict({\n 'rowid': 19,\n 'id': None,\n 'chunk_id': 3,\n 'chunk_offset': 2,\n }),\n OrderedDict({\n 'rowid': 20,\n 'id': None,\n 'chunk_id': 3,\n 'chunk_offset': 3,\n }),\n OrderedDict({\n 'rowid': 21,\n 'id': None,\n 'chunk_id': 3,\n 'chunk_offset': 4,\n }),\n OrderedDict({\n 'rowid': 22,\n 'id': None,\n 'chunk_id': 3,\n 'chunk_offset': 5,\n }),\n OrderedDict({\n 'rowid': 23,\n 'id': None,\n 'chunk_id': 3,\n 'chunk_offset': 6,\n }),\n OrderedDict({\n 'rowid': 24,\n 'id': None,\n 'chunk_id': 3,\n 'chunk_offset': 7,\n }),\n OrderedDict({\n 'rowid': 25,\n 'id': None,\n 'chunk_id': 4,\n 'chunk_offset': 0,\n }),\n ]),\n }),\n 'vec_movies_vector_chunks00': OrderedDict({\n 'sql': 'select * from vec_movies_vector_chunks00',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'vectors': b'\\x00\\x00\\x80?\\x00\\x00\\x00@\\x00\\x00@@\\x00\\x00\\x80@\\x00\\x00\\xa0@\\x00\\x00\\xc0@\\x00\\x00\\xe0@\\x00\\x00\\x00A',\n }),\n OrderedDict({\n 'rowid': 2,\n 'vectors': b'\\x00\\x00\\x10A\\x00\\x00 A\\x00\\x000A\\x00\\x00@A\\x00\\x00PA\\x00\\x00`A\\x00\\x00pA\\x00\\x00\\x80A',\n }),\n OrderedDict({\n 'rowid': 3,\n 'vectors': b'\\x00\\x00\\x88A\\x00\\x00\\x90A\\x00\\x00\\x98A\\x00\\x00\\xa0A\\x00\\x00\\xa8A\\x00\\x00\\xb0A\\x00\\x00\\xb8A\\x00\\x00\\xc0A',\n }),\n OrderedDict({\n 'rowid': 4,\n 'vectors': b'\\x00\\x00\\xc8A\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n })\n# ---\n# name: test_stress.1\n OrderedDict({\n 'sql': '''\n \n select\n movie_id,\n title,\n genre,\n num_reviews,\n mean_rating,\n is_favorited,\n distance\n from vec_movies\n where synopsis_embedding match '[15.5]'\n and genre = 'scifi'\n and num_reviews between 100 and 500\n and mean_rating > 3.5\n and k = 5;\n \n ''',\n 'rows': list([\n OrderedDict({\n 'movie_id': 13,\n 'title': 'The Matrix',\n 'genre': 'scifi',\n 'num_reviews': 423,\n 'mean_rating': 4.5,\n 'is_favorited': 1,\n 'distance': 2.5,\n }),\n OrderedDict({\n 'movie_id': 18,\n 'title': 'Inception',\n 'genre': 'scifi',\n 'num_reviews': 201,\n 'mean_rating': 5.0,\n 'is_favorited': 1,\n 'distance': 2.5,\n }),\n OrderedDict({\n 'movie_id': 21,\n 'title': 'Gravity',\n 'genre': 'scifi',\n 'num_reviews': 342,\n 'mean_rating': 4.0,\n 'is_favorited': 1,\n 'distance': 5.5,\n }),\n OrderedDict({\n 'movie_id': 22,\n 'title': 'Dune',\n 'genre': 'scifi',\n 'num_reviews': 451,\n 'mean_rating': 4.4,\n 'is_favorited': 1,\n 'distance': 6.5,\n }),\n OrderedDict({\n 'movie_id': 8,\n 'title': 'Blade Runner 2049',\n 'genre': 'scifi',\n 'num_reviews': 301,\n 'mean_rating': 5.0,\n 'is_favorited': 0,\n 'distance': 7.5,\n }),\n ]),\n })\n# ---\n# name: test_stress.2\n OrderedDict({\n 'sql': \"select movie_id, genre, distance from vec_movies where synopsis_embedding match '[100]' and k = 5 and genre = 'horror'\",\n 'rows': list([\n OrderedDict({\n 'movie_id': 24,\n 'genre': 'horror',\n 'distance': 76.0,\n }),\n OrderedDict({\n 'movie_id': 16,\n 'genre': 'horror',\n 'distance': 84.0,\n }),\n OrderedDict({\n 'movie_id': 11,\n 'genre': 'horror',\n 'distance': 89.0,\n }),\n OrderedDict({\n 'movie_id': 6,\n 'genre': 'horror',\n 'distance': 94.0,\n }),\n OrderedDict({\n 'movie_id': 1,\n 'genre': 'horror',\n 'distance': 99.0,\n }),\n ]),\n })\n# ---\n# name: test_stress.3\n OrderedDict({\n 'sql': \"select movie_id, genre, distance from vec_movies where synopsis_embedding match '[100]' and k = 5 and genre = 'comedy'\",\n 'rows': list([\n OrderedDict({\n 'movie_id': 17,\n 'genre': 'comedy',\n 'distance': 83.0,\n }),\n OrderedDict({\n 'movie_id': 12,\n 'genre': 'comedy',\n 'distance': 88.0,\n }),\n OrderedDict({\n 'movie_id': 7,\n 'genre': 'comedy',\n 'distance': 93.0,\n }),\n OrderedDict({\n 'movie_id': 2,\n 'genre': 'comedy',\n 'distance': 98.0,\n }),\n ]),\n })\n# ---\n# name: test_stress.4\n OrderedDict({\n 'sql': \"select movie_id, num_reviews, distance from vec_movies where synopsis_embedding match '[100]' and k = 5 and num_reviews between 100 and 500\",\n 'rows': list([\n OrderedDict({\n 'movie_id': 25,\n 'num_reviews': 310,\n 'distance': 75.0,\n }),\n OrderedDict({\n 'movie_id': 24,\n 'num_reviews': 271,\n 'distance': 76.0,\n }),\n OrderedDict({\n 'movie_id': 22,\n 'num_reviews': 451,\n 'distance': 78.0,\n }),\n OrderedDict({\n 'movie_id': 21,\n 'num_reviews': 342,\n 'distance': 79.0,\n }),\n OrderedDict({\n 'movie_id': 20,\n 'num_reviews': 186,\n 'distance': 80.0,\n }),\n ]),\n })\n# ---\n# name: test_stress.5\n OrderedDict({\n 'sql': \"select movie_id, num_reviews, distance from vec_movies where synopsis_embedding match '[100]' and k = 5 and num_reviews >= 500\",\n 'rows': list([\n OrderedDict({\n 'movie_id': 23,\n 'num_reviews': 522,\n 'distance': 77.0,\n }),\n ]),\n })\n# ---\n# name: test_stress.6\n OrderedDict({\n 'sql': \"select movie_id, mean_rating, distance from vec_movies where synopsis_embedding match '[100]' and k = 5 and mean_rating < 3.0\",\n 'rows': list([\n OrderedDict({\n 'movie_id': 19,\n 'mean_rating': 2.7,\n 'distance': 81.0,\n }),\n OrderedDict({\n 'movie_id': 12,\n 'mean_rating': 2.8,\n 'distance': 88.0,\n }),\n OrderedDict({\n 'movie_id': 7,\n 'mean_rating': 2.9,\n 'distance': 93.0,\n }),\n OrderedDict({\n 'movie_id': 2,\n 'mean_rating': 2.6,\n 'distance': 98.0,\n }),\n ]),\n })\n# ---\n# name: test_stress.7\n OrderedDict({\n 'sql': \"select movie_id, mean_rating, distance from vec_movies where synopsis_embedding match '[100]' and k = 5 and mean_rating between 4.0 and 5.0\",\n 'rows': list([\n OrderedDict({\n 'movie_id': 24,\n 'mean_rating': 4.3,\n 'distance': 76.0,\n }),\n OrderedDict({\n 'movie_id': 23,\n 'mean_rating': 4.6,\n 'distance': 77.0,\n }),\n OrderedDict({\n 'movie_id': 22,\n 'mean_rating': 4.4,\n 'distance': 78.0,\n }),\n OrderedDict({\n 'movie_id': 21,\n 'mean_rating': 4.0,\n 'distance': 79.0,\n }),\n OrderedDict({\n 'movie_id': 20,\n 'mean_rating': 4.8,\n 'distance': 80.0,\n }),\n ]),\n })\n# ---\n# name: test_stress[bool-eq-false]\n OrderedDict({\n 'sql': \"select movie_id, is_favorited, distance from vec_movies where synopsis_embedding match '[100]' and k = 5 and is_favorited = FALSE\",\n 'rows': list([\n OrderedDict({\n 'movie_id': 16,\n 'is_favorited': 0,\n 'distance': 84.0,\n }),\n OrderedDict({\n 'movie_id': 14,\n 'is_favorited': 0,\n 'distance': 86.0,\n }),\n OrderedDict({\n 'movie_id': 12,\n 'is_favorited': 0,\n 'distance': 88.0,\n }),\n OrderedDict({\n 'movie_id': 10,\n 'is_favorited': 0,\n 'distance': 90.0,\n }),\n OrderedDict({\n 'movie_id': 8,\n 'is_favorited': 0,\n 'distance': 92.0,\n }),\n ]),\n })\n# ---\n# name: test_stress[bool-eq-true]\n OrderedDict({\n 'sql': \"select movie_id, is_favorited, distance from vec_movies where synopsis_embedding match '[100]' and k = 5 and is_favorited = TRUE\",\n 'rows': list([\n OrderedDict({\n 'movie_id': 25,\n 'is_favorited': 1,\n 'distance': 75.0,\n }),\n OrderedDict({\n 'movie_id': 24,\n 'is_favorited': 1,\n 'distance': 76.0,\n }),\n OrderedDict({\n 'movie_id': 23,\n 'is_favorited': 1,\n 'distance': 77.0,\n }),\n OrderedDict({\n 'movie_id': 22,\n 'is_favorited': 1,\n 'distance': 78.0,\n }),\n OrderedDict({\n 'movie_id': 21,\n 'is_favorited': 1,\n 'distance': 79.0,\n }),\n ]),\n })\n# ---\n# name: test_stress[bool-ne-false]\n OrderedDict({\n 'sql': \"select movie_id, is_favorited, distance from vec_movies where synopsis_embedding match '[100]' and k = 5 and is_favorited != FALSE\",\n 'rows': list([\n OrderedDict({\n 'movie_id': 25,\n 'is_favorited': 1,\n 'distance': 75.0,\n }),\n OrderedDict({\n 'movie_id': 24,\n 'is_favorited': 1,\n 'distance': 76.0,\n }),\n OrderedDict({\n 'movie_id': 23,\n 'is_favorited': 1,\n 'distance': 77.0,\n }),\n OrderedDict({\n 'movie_id': 22,\n 'is_favorited': 1,\n 'distance': 78.0,\n }),\n OrderedDict({\n 'movie_id': 21,\n 'is_favorited': 1,\n 'distance': 79.0,\n }),\n ]),\n })\n# ---\n# name: test_stress[bool-ne-true]\n OrderedDict({\n 'sql': \"select movie_id, is_favorited, distance from vec_movies where synopsis_embedding match '[100]' and k = 5 and is_favorited != TRUE\",\n 'rows': list([\n OrderedDict({\n 'movie_id': 16,\n 'is_favorited': 0,\n 'distance': 84.0,\n }),\n OrderedDict({\n 'movie_id': 14,\n 'is_favorited': 0,\n 'distance': 86.0,\n }),\n OrderedDict({\n 'movie_id': 12,\n 'is_favorited': 0,\n 'distance': 88.0,\n }),\n OrderedDict({\n 'movie_id': 10,\n 'is_favorited': 0,\n 'distance': 90.0,\n }),\n OrderedDict({\n 'movie_id': 8,\n 'is_favorited': 0,\n 'distance': 92.0,\n }),\n ]),\n })\n# ---\n# name: test_stress[bool-other-op]\n dict({\n 'error': 'OperationalError',\n 'message': 'ONLY EQUALS (=) or NOT_EQUALS (!=) operators are allowed on boolean metadata columns.',\n })\n# ---\n# name: test_text_knn\n dict({\n 'v_chunks': OrderedDict({\n 'sql': 'select * from v_chunks',\n 'rows': list([\n ]),\n }),\n 'v_metadatachunks00': OrderedDict({\n 'sql': 'select * from v_metadatachunks00',\n 'rows': list([\n ]),\n }),\n 'v_metadatatext00': OrderedDict({\n 'sql': 'select * from v_metadatatext00',\n 'rows': list([\n ]),\n }),\n 'v_rowids': OrderedDict({\n 'sql': 'select * from v_rowids',\n 'rows': list([\n ]),\n }),\n 'v_vector_chunks00': OrderedDict({\n 'sql': 'select * from v_vector_chunks00',\n 'rows': list([\n ]),\n }),\n })\n# ---\n# name: test_text_knn.1\n OrderedDict({\n 'sql': 'select * from v',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'vector': b'\\xaeG\\xe1=',\n 'name': 'aaa',\n }),\n OrderedDict({\n 'rowid': 2,\n 'vector': b'\\xaeGa>',\n 'name': 'bbb',\n }),\n OrderedDict({\n 'rowid': 3,\n 'vector': b'\\xc3\\xf5\\xa8>',\n 'name': 'ccc',\n }),\n OrderedDict({\n 'rowid': 4,\n 'vector': b'\\xaeG\\xe1>',\n 'name': 'ddd',\n }),\n OrderedDict({\n 'rowid': 5,\n 'vector': b'\\xcd\\xcc\\x0c?',\n 'name': 'eee',\n }),\n OrderedDict({\n 'rowid': 6,\n 'vector': b'\\xc3\\xf5(?',\n 'name': 'fff',\n }),\n OrderedDict({\n 'rowid': 7,\n 'vector': b'\\xb8\\x1eE?',\n 'name': 'ggg',\n }),\n OrderedDict({\n 'rowid': 8,\n 'vector': b'\\xaeGa?',\n 'name': 'hhh',\n }),\n OrderedDict({\n 'rowid': 9,\n 'vector': b'\\xa4p}?',\n 'name': 'iii',\n }),\n ]),\n })\n# ---\n# name: test_text_knn.2\n dict({\n 'v_chunks': OrderedDict({\n 'sql': 'select * from v_chunks',\n 'rows': list([\n OrderedDict({\n 'chunk_id': 1,\n 'size': 8,\n 'validity': b'\\xff',\n 'rowids': b'\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x03\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x04\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x05\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x06\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x07\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x08\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n OrderedDict({\n 'chunk_id': 2,\n 'size': 8,\n 'validity': b'\\x01',\n 'rowids': b'\\t\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n 'v_metadatachunks00': OrderedDict({\n 'sql': 'select * from v_metadatachunks00',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'data': b'\\x03\\x00\\x00\\x00aaa\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x03\\x00\\x00\\x00bbb\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x03\\x00\\x00\\x00ccc\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x03\\x00\\x00\\x00ddd\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x03\\x00\\x00\\x00eee\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x03\\x00\\x00\\x00fff\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x03\\x00\\x00\\x00ggg\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x03\\x00\\x00\\x00hhh\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n OrderedDict({\n 'rowid': 2,\n 'data': b'\\x03\\x00\\x00\\x00iii\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n 'v_metadatatext00': OrderedDict({\n 'sql': 'select * from v_metadatatext00',\n 'rows': list([\n ]),\n }),\n 'v_rowids': OrderedDict({\n 'sql': 'select * from v_rowids',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 0,\n }),\n OrderedDict({\n 'rowid': 2,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 1,\n }),\n OrderedDict({\n 'rowid': 3,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 2,\n }),\n OrderedDict({\n 'rowid': 4,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 3,\n }),\n OrderedDict({\n 'rowid': 5,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 4,\n }),\n OrderedDict({\n 'rowid': 6,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 5,\n }),\n OrderedDict({\n 'rowid': 7,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 6,\n }),\n OrderedDict({\n 'rowid': 8,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 7,\n }),\n OrderedDict({\n 'rowid': 9,\n 'id': None,\n 'chunk_id': 2,\n 'chunk_offset': 0,\n }),\n ]),\n }),\n 'v_vector_chunks00': OrderedDict({\n 'sql': 'select * from v_vector_chunks00',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'vectors': b'\\xaeG\\xe1=\\xaeGa>\\xc3\\xf5\\xa8>\\xaeG\\xe1>\\xcd\\xcc\\x0c?\\xc3\\xf5(?\\xb8\\x1eE?\\xaeGa?',\n }),\n OrderedDict({\n 'rowid': 2,\n 'vectors': b'\\xa4p}?\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n })\n# ---\n# name: test_text_knn.3\n OrderedDict({\n 'sql': \"select rowid, name, distance from v where vector match '[1]' and k = 5\",\n 'rows': list([\n OrderedDict({\n 'rowid': 9,\n 'name': 'iii',\n 'distance': 0.009999990463256836,\n }),\n OrderedDict({\n 'rowid': 8,\n 'name': 'hhh',\n 'distance': 0.12000000476837158,\n }),\n OrderedDict({\n 'rowid': 7,\n 'name': 'ggg',\n 'distance': 0.23000001907348633,\n }),\n OrderedDict({\n 'rowid': 6,\n 'name': 'fff',\n 'distance': 0.3399999737739563,\n }),\n OrderedDict({\n 'rowid': 5,\n 'name': 'eee',\n 'distance': 0.44999998807907104,\n }),\n ]),\n })\n# ---\n# name: test_text_knn.4\n OrderedDict({\n 'sql': \"select rowid, name, distance from v where vector match '[1]' and k = 5 and name < 'ddd'\",\n 'rows': list([\n OrderedDict({\n 'rowid': 3,\n 'name': 'ccc',\n 'distance': 0.6699999570846558,\n }),\n OrderedDict({\n 'rowid': 2,\n 'name': 'bbb',\n 'distance': 0.7799999713897705,\n }),\n OrderedDict({\n 'rowid': 1,\n 'name': 'aaa',\n 'distance': 0.8899999856948853,\n }),\n ]),\n })\n# ---\n# name: test_text_knn.5\n OrderedDict({\n 'sql': \"select rowid, name, distance from v where vector match '[1]' and k = 5 and name <= 'ddd'\",\n 'rows': list([\n OrderedDict({\n 'rowid': 4,\n 'name': 'ddd',\n 'distance': 0.5600000023841858,\n }),\n OrderedDict({\n 'rowid': 3,\n 'name': 'ccc',\n 'distance': 0.6699999570846558,\n }),\n OrderedDict({\n 'rowid': 2,\n 'name': 'bbb',\n 'distance': 0.7799999713897705,\n }),\n OrderedDict({\n 'rowid': 1,\n 'name': 'aaa',\n 'distance': 0.8899999856948853,\n }),\n ]),\n })\n# ---\n# name: test_text_knn.6\n OrderedDict({\n 'sql': \"select rowid, name, distance from v where vector match '[1]' and k = 5 and name > 'fff'\",\n 'rows': list([\n OrderedDict({\n 'rowid': 9,\n 'name': 'iii',\n 'distance': 0.009999990463256836,\n }),\n OrderedDict({\n 'rowid': 8,\n 'name': 'hhh',\n 'distance': 0.12000000476837158,\n }),\n OrderedDict({\n 'rowid': 7,\n 'name': 'ggg',\n 'distance': 0.23000001907348633,\n }),\n ]),\n })\n# ---\n# name: test_text_knn.7\n OrderedDict({\n 'sql': \"select rowid, name, distance from v where vector match '[1]' and k = 5 and name >= 'fff'\",\n 'rows': list([\n OrderedDict({\n 'rowid': 9,\n 'name': 'iii',\n 'distance': 0.009999990463256836,\n }),\n OrderedDict({\n 'rowid': 8,\n 'name': 'hhh',\n 'distance': 0.12000000476837158,\n }),\n OrderedDict({\n 'rowid': 7,\n 'name': 'ggg',\n 'distance': 0.23000001907348633,\n }),\n OrderedDict({\n 'rowid': 6,\n 'name': 'fff',\n 'distance': 0.3399999737739563,\n }),\n ]),\n })\n# ---\n# name: test_text_knn.8\n OrderedDict({\n 'sql': \"select rowid, name, distance from v where vector match '[1]' and k = 5 and name = 'aaa'\",\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'name': 'aaa',\n 'distance': 0.8899999856948853,\n }),\n ]),\n })\n# ---\n# name: test_text_knn.9\n OrderedDict({\n 'sql': \"select rowid, name, distance from v where vector match '[.01]' and k = 5 and name != 'aaa'\",\n 'rows': list([\n OrderedDict({\n 'rowid': 2,\n 'name': 'bbb',\n 'distance': 0.20999999344348907,\n }),\n OrderedDict({\n 'rowid': 3,\n 'name': 'ccc',\n 'distance': 0.320000022649765,\n }),\n OrderedDict({\n 'rowid': 4,\n 'name': 'ddd',\n 'distance': 0.4300000071525574,\n }),\n OrderedDict({\n 'rowid': 5,\n 'name': 'eee',\n 'distance': 0.5400000214576721,\n }),\n OrderedDict({\n 'rowid': 6,\n 'name': 'fff',\n 'distance': 0.6500000357627869,\n }),\n ]),\n })\n# ---\n# name: test_types[illegal-boolean]\n dict({\n 'error': 'OperationalError',\n 'message': 'Expected 0 or 1 for BOOLEAN metadata column b',\n })\n# ---\n# name: test_types[illegal-type-boolean]\n dict({\n 'error': 'OperationalError',\n 'message': 'Expected 0 or 1 for BOOLEAN metadata column b',\n })\n# ---\n# name: test_types[illegal-type-float]\n dict({\n 'error': 'OperationalError',\n 'message': 'Expected float for FLOAT metadata column f, received TEXT',\n })\n# ---\n# name: test_types[illegal-type-int]\n dict({\n 'error': 'OperationalError',\n 'message': 'Expected integer for INTEGER metadata column n, received TEXT',\n })\n# ---\n# name: test_types[illegal-type-text]\n dict({\n 'error': 'OperationalError',\n 'message': 'Expected text for TEXT metadata column t, received INTEGER',\n })\n# ---\n# name: test_types[legal]\n OrderedDict({\n 'sql': 'insert into v(vector, b, n, f, t) values (?, ?, ?, ?, ?)',\n 'rows': list([\n ]),\n })\n# ---\n# name: test_updates[1-init-contents]\n OrderedDict({\n 'sql': 'select * from v',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'vector': b'\\x11\\x11\\x11\\x11',\n 'b': 1,\n 'n': 1,\n 'f': 1.1,\n 't': 'test1',\n }),\n OrderedDict({\n 'rowid': 2,\n 'vector': b'\"\"\"\"',\n 'b': 1,\n 'n': 2,\n 'f': 2.2,\n 't': 'test2',\n }),\n OrderedDict({\n 'rowid': 3,\n 'vector': b'3333',\n 'b': 1,\n 'n': 3,\n 'f': 3.3,\n 't': '1234567890123',\n }),\n ]),\n })\n# ---\n# name: test_updates[1-init-shadow]\n dict({\n 'v_chunks': OrderedDict({\n 'sql': 'select * from v_chunks',\n 'rows': list([\n OrderedDict({\n 'chunk_id': 1,\n 'size': 8,\n 'validity': b'\\x07',\n 'rowids': b'\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x03\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n 'v_metadatachunks00': OrderedDict({\n 'sql': 'select * from v_metadatachunks00',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'data': b'\\x07',\n }),\n ]),\n }),\n 'v_metadatachunks01': OrderedDict({\n 'sql': 'select * from v_metadatachunks01',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'data': b'\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x03\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n 'v_metadatachunks02': OrderedDict({\n 'sql': 'select * from v_metadatachunks02',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'data': b'\\x9a\\x99\\x99\\x99\\x99\\x99\\xf1?\\x9a\\x99\\x99\\x99\\x99\\x99\\x01@ffffff\\n@\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n 'v_metadatachunks03': OrderedDict({\n 'sql': 'select * from v_metadatachunks03',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'data': b'\\x05\\x00\\x00\\x00test1\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x05\\x00\\x00\\x00test2\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\r\\x00\\x00\\x00123456789012\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n 'v_metadatatext03': OrderedDict({\n 'sql': 'select * from v_metadatatext03',\n 'rows': list([\n OrderedDict({\n 'rowid': 3,\n 'data': '1234567890123',\n }),\n ]),\n }),\n 'v_rowids': OrderedDict({\n 'sql': 'select * from v_rowids',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 0,\n }),\n OrderedDict({\n 'rowid': 2,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 1,\n }),\n OrderedDict({\n 'rowid': 3,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 2,\n }),\n ]),\n }),\n 'v_vector_chunks00': OrderedDict({\n 'sql': 'select * from v_vector_chunks00',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'vectors': b'\\x11\\x11\\x11\\x11\"\"\"\"3333\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n })\n# ---\n# name: test_updates[general-update-contents]\n OrderedDict({\n 'sql': 'select * from v',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'vector': b'\\x11\\x11\\x11\\x11',\n 'b': 0,\n 'n': 11,\n 'f': 11.11,\n 't': 'newtest1',\n }),\n OrderedDict({\n 'rowid': 2,\n 'vector': b'\"\"\"\"',\n 'b': 1,\n 'n': 2,\n 'f': 2.2,\n 't': 'test2',\n }),\n OrderedDict({\n 'rowid': 3,\n 'vector': b'3333',\n 'b': 1,\n 'n': 3,\n 'f': 3.3,\n 't': '1234567890123',\n }),\n ]),\n })\n# ---\n# name: test_updates[general-update-shaodnw]\n dict({\n 'v_chunks': OrderedDict({\n 'sql': 'select * from v_chunks',\n 'rows': list([\n OrderedDict({\n 'chunk_id': 1,\n 'size': 8,\n 'validity': b'\\x07',\n 'rowids': b'\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x03\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n 'v_metadatachunks00': OrderedDict({\n 'sql': 'select * from v_metadatachunks00',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'data': b'\\x06',\n }),\n ]),\n }),\n 'v_metadatachunks01': OrderedDict({\n 'sql': 'select * from v_metadatachunks01',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'data': b'\\x0b\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x03\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n 'v_metadatachunks02': OrderedDict({\n 'sql': 'select * from v_metadatachunks02',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'data': b'\\xb8\\x1e\\x85\\xebQ8&@\\x9a\\x99\\x99\\x99\\x99\\x99\\x01@ffffff\\n@\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n 'v_metadatachunks03': OrderedDict({\n 'sql': 'select * from v_metadatachunks03',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'data': b'\\x08\\x00\\x00\\x00newtest1\\x00\\x00\\x00\\x00\\x05\\x00\\x00\\x00test2\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\r\\x00\\x00\\x00123456789012\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n 'v_metadatatext03': OrderedDict({\n 'sql': 'select * from v_metadatatext03',\n 'rows': list([\n OrderedDict({\n 'rowid': 3,\n 'data': '1234567890123',\n }),\n ]),\n }),\n 'v_rowids': OrderedDict({\n 'sql': 'select * from v_rowids',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 0,\n }),\n OrderedDict({\n 'rowid': 2,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 1,\n }),\n OrderedDict({\n 'rowid': 3,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 2,\n }),\n ]),\n }),\n 'v_vector_chunks00': OrderedDict({\n 'sql': 'select * from v_vector_chunks00',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'vectors': b'\\x11\\x11\\x11\\x11\"\"\"\"3333\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n })\n# ---\n# name: test_updates[string-update-1-contents]\n OrderedDict({\n 'sql': 'select * from v',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'vector': b'\\x11\\x11\\x11\\x11',\n 'b': 0,\n 'n': 11,\n 'f': 11.11,\n 't': 'newtest1',\n }),\n OrderedDict({\n 'rowid': 2,\n 'vector': b'\"\"\"\"',\n 'b': 1,\n 'n': 2,\n 'f': 2.2,\n 't': 'test2',\n }),\n OrderedDict({\n 'rowid': 3,\n 'vector': b'3333',\n 'b': 1,\n 'n': 3,\n 'f': 3.3,\n 't': '1234567890123-updated',\n }),\n ]),\n })\n# ---\n# name: test_updates[string-update-1-shadow]\n dict({\n 'v_chunks': OrderedDict({\n 'sql': 'select * from v_chunks',\n 'rows': list([\n OrderedDict({\n 'chunk_id': 1,\n 'size': 8,\n 'validity': b'\\x07',\n 'rowids': b'\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x03\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n 'v_metadatachunks00': OrderedDict({\n 'sql': 'select * from v_metadatachunks00',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'data': b'\\x06',\n }),\n ]),\n }),\n 'v_metadatachunks01': OrderedDict({\n 'sql': 'select * from v_metadatachunks01',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'data': b'\\x0b\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x03\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n 'v_metadatachunks02': OrderedDict({\n 'sql': 'select * from v_metadatachunks02',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'data': b'\\xb8\\x1e\\x85\\xebQ8&@\\x9a\\x99\\x99\\x99\\x99\\x99\\x01@ffffff\\n@\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n 'v_metadatachunks03': OrderedDict({\n 'sql': 'select * from v_metadatachunks03',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'data': b'\\x08\\x00\\x00\\x00newtest1\\x00\\x00\\x00\\x00\\x05\\x00\\x00\\x00test2\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x15\\x00\\x00\\x00123456789012\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n 'v_metadatatext03': OrderedDict({\n 'sql': 'select * from v_metadatatext03',\n 'rows': list([\n OrderedDict({\n 'rowid': 3,\n 'data': '1234567890123-updated',\n }),\n ]),\n }),\n 'v_rowids': OrderedDict({\n 'sql': 'select * from v_rowids',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 0,\n }),\n OrderedDict({\n 'rowid': 2,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 1,\n }),\n OrderedDict({\n 'rowid': 3,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 2,\n }),\n ]),\n }),\n 'v_vector_chunks00': OrderedDict({\n 'sql': 'select * from v_vector_chunks00',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'vectors': b'\\x11\\x11\\x11\\x11\"\"\"\"3333\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n })\n# ---\n# name: test_updates[string-update-2-contents]\n OrderedDict({\n 'sql': 'select * from v',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'vector': b'\\x11\\x11\\x11\\x11',\n 'b': 0,\n 'n': 11,\n 'f': 11.11,\n 't': 'newtest1',\n }),\n OrderedDict({\n 'rowid': 2,\n 'vector': b'\"\"\"\"',\n 'b': 1,\n 'n': 2,\n 'f': 2.2,\n 't': 'test2-short',\n }),\n OrderedDict({\n 'rowid': 3,\n 'vector': b'3333',\n 'b': 1,\n 'n': 3,\n 'f': 3.3,\n 't': '1234567890123-updated',\n }),\n ]),\n })\n# ---\n# name: test_updates[string-update-2-shadow]\n dict({\n 'v_chunks': OrderedDict({\n 'sql': 'select * from v_chunks',\n 'rows': list([\n OrderedDict({\n 'chunk_id': 1,\n 'size': 8,\n 'validity': b'\\x07',\n 'rowids': b'\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x03\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n 'v_metadatachunks00': OrderedDict({\n 'sql': 'select * from v_metadatachunks00',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'data': b'\\x06',\n }),\n ]),\n }),\n 'v_metadatachunks01': OrderedDict({\n 'sql': 'select * from v_metadatachunks01',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'data': b'\\x0b\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x03\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n 'v_metadatachunks02': OrderedDict({\n 'sql': 'select * from v_metadatachunks02',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'data': b'\\xb8\\x1e\\x85\\xebQ8&@\\x9a\\x99\\x99\\x99\\x99\\x99\\x01@ffffff\\n@\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n 'v_metadatachunks03': OrderedDict({\n 'sql': 'select * from v_metadatachunks03',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'data': b'\\x08\\x00\\x00\\x00newtest1\\x00\\x00\\x00\\x00\\x0b\\x00\\x00\\x00test2-short\\x00\\x15\\x00\\x00\\x00123456789012\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n 'v_metadatatext03': OrderedDict({\n 'sql': 'select * from v_metadatatext03',\n 'rows': list([\n OrderedDict({\n 'rowid': 3,\n 'data': '1234567890123-updated',\n }),\n ]),\n }),\n 'v_rowids': OrderedDict({\n 'sql': 'select * from v_rowids',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 0,\n }),\n OrderedDict({\n 'rowid': 2,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 1,\n }),\n OrderedDict({\n 'rowid': 3,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 2,\n }),\n ]),\n }),\n 'v_vector_chunks00': OrderedDict({\n 'sql': 'select * from v_vector_chunks00',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'vectors': b'\\x11\\x11\\x11\\x11\"\"\"\"3333\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n })\n# ---\n# name: test_updates[string-update-3-contents]\n OrderedDict({\n 'sql': 'select * from v',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'vector': b'\\x11\\x11\\x11\\x11',\n 'b': 0,\n 'n': 11,\n 'f': 11.11,\n 't': 'newtest1',\n }),\n OrderedDict({\n 'rowid': 2,\n 'vector': b'\"\"\"\"',\n 'b': 1,\n 'n': 2,\n 'f': 2.2,\n 't': 'test2-long-long-long',\n }),\n OrderedDict({\n 'rowid': 3,\n 'vector': b'3333',\n 'b': 1,\n 'n': 3,\n 'f': 3.3,\n 't': '1234567890123-updated',\n }),\n ]),\n })\n# ---\n# name: test_updates[string-update-3-shadow]\n dict({\n 'v_chunks': OrderedDict({\n 'sql': 'select * from v_chunks',\n 'rows': list([\n OrderedDict({\n 'chunk_id': 1,\n 'size': 8,\n 'validity': b'\\x07',\n 'rowids': b'\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x03\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n 'v_metadatachunks00': OrderedDict({\n 'sql': 'select * from v_metadatachunks00',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'data': b'\\x06',\n }),\n ]),\n }),\n 'v_metadatachunks01': OrderedDict({\n 'sql': 'select * from v_metadatachunks01',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'data': b'\\x0b\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x03\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n 'v_metadatachunks02': OrderedDict({\n 'sql': 'select * from v_metadatachunks02',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'data': b'\\xb8\\x1e\\x85\\xebQ8&@\\x9a\\x99\\x99\\x99\\x99\\x99\\x01@ffffff\\n@\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n 'v_metadatachunks03': OrderedDict({\n 'sql': 'select * from v_metadatachunks03',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'data': b'\\x08\\x00\\x00\\x00newtest1\\x00\\x00\\x00\\x00\\x14\\x00\\x00\\x00test2-long-l\\x15\\x00\\x00\\x00123456789012\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n 'v_metadatatext03': OrderedDict({\n 'sql': 'select * from v_metadatatext03',\n 'rows': list([\n OrderedDict({\n 'rowid': 3,\n 'data': '1234567890123-updated',\n }),\n OrderedDict({\n 'rowid': 2,\n 'data': 'test2-long-long-long',\n }),\n ]),\n }),\n 'v_rowids': OrderedDict({\n 'sql': 'select * from v_rowids',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 0,\n }),\n OrderedDict({\n 'rowid': 2,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 1,\n }),\n OrderedDict({\n 'rowid': 3,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 2,\n }),\n ]),\n }),\n 'v_vector_chunks00': OrderedDict({\n 'sql': 'select * from v_vector_chunks00',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'vectors': b'\\x11\\x11\\x11\\x11\"\"\"\"3333\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n })\n# ---\n# name: test_updates[string-update-4-contents]\n OrderedDict({\n 'sql': 'select * from v',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'vector': b'\\x11\\x11\\x11\\x11',\n 'b': 0,\n 'n': 11,\n 'f': 11.11,\n 't': 'newtest1',\n }),\n OrderedDict({\n 'rowid': 2,\n 'vector': b'\"\"\"\"',\n 'b': 1,\n 'n': 2,\n 'f': 2.2,\n 't': 'test2-shortx',\n }),\n OrderedDict({\n 'rowid': 3,\n 'vector': b'3333',\n 'b': 1,\n 'n': 3,\n 'f': 3.3,\n 't': '1234567890123-updated',\n }),\n ]),\n })\n# ---\n# name: test_updates[string-update-4-shadow]\n dict({\n 'v_chunks': OrderedDict({\n 'sql': 'select * from v_chunks',\n 'rows': list([\n OrderedDict({\n 'chunk_id': 1,\n 'size': 8,\n 'validity': b'\\x07',\n 'rowids': b'\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x03\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n 'v_metadatachunks00': OrderedDict({\n 'sql': 'select * from v_metadatachunks00',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'data': b'\\x06',\n }),\n ]),\n }),\n 'v_metadatachunks01': OrderedDict({\n 'sql': 'select * from v_metadatachunks01',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'data': b'\\x0b\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x03\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n 'v_metadatachunks02': OrderedDict({\n 'sql': 'select * from v_metadatachunks02',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'data': b'\\xb8\\x1e\\x85\\xebQ8&@\\x9a\\x99\\x99\\x99\\x99\\x99\\x01@ffffff\\n@\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n 'v_metadatachunks03': OrderedDict({\n 'sql': 'select * from v_metadatachunks03',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'data': b'\\x08\\x00\\x00\\x00newtest1\\x00\\x00\\x00\\x00\\x0c\\x00\\x00\\x00test2-shortx\\x15\\x00\\x00\\x00123456789012\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n 'v_metadatatext03': OrderedDict({\n 'sql': 'select * from v_metadatatext03',\n 'rows': list([\n OrderedDict({\n 'rowid': 3,\n 'data': '1234567890123-updated',\n }),\n ]),\n }),\n 'v_rowids': OrderedDict({\n 'sql': 'select * from v_rowids',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 0,\n }),\n OrderedDict({\n 'rowid': 2,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 1,\n }),\n OrderedDict({\n 'rowid': 3,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 2,\n }),\n ]),\n }),\n 'v_vector_chunks00': OrderedDict({\n 'sql': 'select * from v_vector_chunks00',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'vectors': b'\\x11\\x11\\x11\\x11\"\"\"\"3333\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n })\n# ---\n# name: test_vtab_in[allow-int-all]\n OrderedDict({\n 'sql': \"select rowid, n, distance from v where vector match '[0]' and k = 8 and n in (555, 999)\",\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'n': 999,\n 'distance': 1.0,\n }),\n OrderedDict({\n 'rowid': 2,\n 'n': 555,\n 'distance': 2.0,\n }),\n OrderedDict({\n 'rowid': 3,\n 'n': 999,\n 'distance': 3.0,\n }),\n OrderedDict({\n 'rowid': 4,\n 'n': 555,\n 'distance': 4.0,\n }),\n OrderedDict({\n 'rowid': 5,\n 'n': 999,\n 'distance': 5.0,\n }),\n OrderedDict({\n 'rowid': 6,\n 'n': 555,\n 'distance': 6.0,\n }),\n OrderedDict({\n 'rowid': 7,\n 'n': 999,\n 'distance': 7.0,\n }),\n OrderedDict({\n 'rowid': 8,\n 'n': 555,\n 'distance': 8.0,\n }),\n ]),\n })\n# ---\n# name: test_vtab_in[allow-int-superfluous]\n OrderedDict({\n 'sql': \"select rowid, n, distance from v where vector match '[0]' and k = 8 and n in (555, -1, -2)\",\n 'rows': list([\n OrderedDict({\n 'rowid': 2,\n 'n': 555,\n 'distance': 2.0,\n }),\n OrderedDict({\n 'rowid': 4,\n 'n': 555,\n 'distance': 4.0,\n }),\n OrderedDict({\n 'rowid': 6,\n 'n': 555,\n 'distance': 6.0,\n }),\n OrderedDict({\n 'rowid': 8,\n 'n': 555,\n 'distance': 8.0,\n }),\n ]),\n })\n# ---\n# name: test_vtab_in[allow-text-all]\n OrderedDict({\n 'sql': \"select rowid, t, distance from v where vector match '[0]' and k = 8 and t in ('aaaa', 'zzzz')\",\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 't': 'aaaa',\n 'distance': 1.0,\n }),\n OrderedDict({\n 'rowid': 2,\n 't': 'aaaa',\n 'distance': 2.0,\n }),\n OrderedDict({\n 'rowid': 3,\n 't': 'aaaa',\n 'distance': 3.0,\n }),\n OrderedDict({\n 'rowid': 4,\n 't': 'aaaa',\n 'distance': 4.0,\n }),\n OrderedDict({\n 'rowid': 5,\n 't': 'zzzz',\n 'distance': 5.0,\n }),\n OrderedDict({\n 'rowid': 6,\n 't': 'zzzz',\n 'distance': 6.0,\n }),\n OrderedDict({\n 'rowid': 7,\n 't': 'zzzz',\n 'distance': 7.0,\n }),\n OrderedDict({\n 'rowid': 8,\n 't': 'zzzz',\n 'distance': 8.0,\n }),\n ]),\n })\n# ---\n# name: test_vtab_in[allow-text-superfluous]\n OrderedDict({\n 'sql': \"select rowid, t, distance from v where vector match '[0]' and k = 8 and t in ('aaaa', 'foo', 'bar')\",\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 't': 'aaaa',\n 'distance': 1.0,\n }),\n OrderedDict({\n 'rowid': 2,\n 't': 'aaaa',\n 'distance': 2.0,\n }),\n OrderedDict({\n 'rowid': 3,\n 't': 'aaaa',\n 'distance': 3.0,\n }),\n OrderedDict({\n 'rowid': 4,\n 't': 'aaaa',\n 'distance': 4.0,\n }),\n ]),\n })\n# ---\n# name: test_vtab_in[block-bool]\n dict({\n 'error': 'OperationalError',\n 'message': \"'xxx in (...)' is only available on INTEGER or TEXT metadata columns.\",\n })\n# ---\n# name: test_vtab_in[block-float]\n dict({\n 'error': 'OperationalError',\n 'message': \"'xxx in (...)' is only available on INTEGER or TEXT metadata columns.\",\n })\n# ---\n# name: test_vtab_in_long_text[all]\n OrderedDict({\n 'sql': \"select rowid, t from v where vector match '[0]' and k = 10 and t in (select value from json_each(?))\",\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 't': 'aaaa',\n }),\n OrderedDict({\n 'rowid': 2,\n 't': 'aaaaaaaaaaaa_aaa',\n }),\n OrderedDict({\n 'rowid': 3,\n 't': 'bbbb',\n }),\n OrderedDict({\n 'rowid': 4,\n 't': 'bbbbbbbbbbbb_bbb',\n }),\n OrderedDict({\n 'rowid': 5,\n 't': 'cccc',\n }),\n OrderedDict({\n 'rowid': 6,\n 't': 'cccccccccccc_ccc',\n }),\n ]),\n })\n# ---\n# name: test_vtab_in_long_text[individual-aaaa]\n OrderedDict({\n 'sql': \"select rowid, t from v where vector match '[0]' and k = 10 and t in (?, 'nonsense')\",\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 't': 'aaaa',\n }),\n ]),\n })\n# ---\n# name: test_vtab_in_long_text[individual-aaaaaaaaaaaa_aaa]\n OrderedDict({\n 'sql': \"select rowid, t from v where vector match '[0]' and k = 10 and t in (?, 'nonsense')\",\n 'rows': list([\n OrderedDict({\n 'rowid': 2,\n 't': 'aaaaaaaaaaaa_aaa',\n }),\n ]),\n })\n# ---\n# name: test_vtab_in_long_text[individual-bbbb]\n OrderedDict({\n 'sql': \"select rowid, t from v where vector match '[0]' and k = 10 and t in (?, 'nonsense')\",\n 'rows': list([\n OrderedDict({\n 'rowid': 3,\n 't': 'bbbb',\n }),\n ]),\n })\n# ---\n# name: test_vtab_in_long_text[individual-bbbbbbbbbbbb_bbb]\n OrderedDict({\n 'sql': \"select rowid, t from v where vector match '[0]' and k = 10 and t in (?, 'nonsense')\",\n 'rows': list([\n OrderedDict({\n 'rowid': 4,\n 't': 'bbbbbbbbbbbb_bbb',\n }),\n ]),\n })\n# ---\n# name: test_vtab_in_long_text[individual-cccc]\n OrderedDict({\n 'sql': \"select rowid, t from v where vector match '[0]' and k = 10 and t in (?, 'nonsense')\",\n 'rows': list([\n OrderedDict({\n 'rowid': 5,\n 't': 'cccc',\n }),\n ]),\n })\n# ---\n# name: test_vtab_in_long_text[individual-cccccccccccc_ccc]\n OrderedDict({\n 'sql': \"select rowid, t from v where vector match '[0]' and k = 10 and t in (?, 'nonsense')\",\n 'rows': list([\n OrderedDict({\n 'rowid': 6,\n 't': 'cccccccccccc_ccc',\n }),\n ]),\n })\n# ---\n" }, { "path": "tests/__snapshots__/test-partition-keys.ambr", "content": "# serializer version: 1\n# name: test_constructor_limit[max 4 partition keys]\n dict({\n 'error': 'OperationalError',\n 'message': 'vec0 constructor error: More than 4 partition key columns were provided',\n })\n# ---\n# name: test_normal[1 row]\n dict({\n 'v_chunks': OrderedDict({\n 'sql': 'select * from v_chunks',\n 'rows': list([\n OrderedDict({\n 'chunk_id': 1,\n 'size': 8,\n 'sequence_id': None,\n 'partition00': 100,\n 'validity': b'\\x01',\n 'rowids': b'\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n 'v_rowids': OrderedDict({\n 'sql': 'select * from v_rowids',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 0,\n }),\n ]),\n }),\n 'v_vector_chunks00': OrderedDict({\n 'sql': 'select * from v_vector_chunks00',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'vectors': b'\\x11\"3D\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n })\n# ---\n# name: test_normal[2 rows, same parition]\n dict({\n 'v_chunks': OrderedDict({\n 'sql': 'select * from v_chunks',\n 'rows': list([\n OrderedDict({\n 'chunk_id': 1,\n 'size': 8,\n 'sequence_id': None,\n 'partition00': 100,\n 'validity': b'\\x03',\n 'rowids': b'\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n 'v_rowids': OrderedDict({\n 'sql': 'select * from v_rowids',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 0,\n }),\n OrderedDict({\n 'rowid': 2,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 1,\n }),\n ]),\n }),\n 'v_vector_chunks00': OrderedDict({\n 'sql': 'select * from v_vector_chunks00',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'vectors': b'\\x11\"3DDUfw\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n })\n# ---\n# name: test_normal[3 rows, 2 partitions]\n dict({\n 'v_chunks': OrderedDict({\n 'sql': 'select * from v_chunks',\n 'rows': list([\n OrderedDict({\n 'chunk_id': 1,\n 'size': 8,\n 'sequence_id': None,\n 'partition00': 100,\n 'validity': b'\\x03',\n 'rowids': b'\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n OrderedDict({\n 'chunk_id': 2,\n 'size': 8,\n 'sequence_id': None,\n 'partition00': 200,\n 'validity': b'\\x01',\n 'rowids': b'\\x03\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n 'v_rowids': OrderedDict({\n 'sql': 'select * from v_rowids',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 0,\n }),\n OrderedDict({\n 'rowid': 2,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 1,\n }),\n OrderedDict({\n 'rowid': 3,\n 'id': None,\n 'chunk_id': 2,\n 'chunk_offset': 0,\n }),\n ]),\n }),\n 'v_vector_chunks00': OrderedDict({\n 'sql': 'select * from v_vector_chunks00',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'vectors': b'\\x11\"3DDUfw\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n OrderedDict({\n 'rowid': 2,\n 'vectors': b'\\x88\\x99\\xaa\\xbb\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n })\n# ---\n# name: test_types[1. raises type error]\n dict({\n 'error': 'OperationalError',\n 'message': 'Parition key type mismatch: The partition key column p1 has type INTEGER, but TEXT was provided.',\n })\n# ---\n# name: test_types[2. empty DB]\n dict({\n 'v_chunks': OrderedDict({\n 'sql': 'select * from v_chunks',\n 'rows': list([\n ]),\n }),\n 'v_rowids': OrderedDict({\n 'sql': 'select * from v_rowids',\n 'rows': list([\n ]),\n }),\n 'v_vector_chunks00': OrderedDict({\n 'sql': 'select * from v_vector_chunks00',\n 'rows': list([\n ]),\n }),\n })\n# ---\n# name: test_types[3. allow nulls]\n OrderedDict({\n 'sql': 'insert into v(p1, a) values(?, ?)',\n 'rows': list([\n ]),\n })\n# ---\n# name: test_types[4. show NULL partition key]\n dict({\n 'v_chunks': OrderedDict({\n 'sql': 'select * from v_chunks',\n 'rows': list([\n OrderedDict({\n 'chunk_id': 1,\n 'size': 8,\n 'sequence_id': None,\n 'partition00': None,\n 'validity': b'\\x01',\n 'rowids': b'\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n 'v_rowids': OrderedDict({\n 'sql': 'select * from v_rowids',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'id': None,\n 'chunk_id': 1,\n 'chunk_offset': 0,\n }),\n ]),\n }),\n 'v_vector_chunks00': OrderedDict({\n 'sql': 'select * from v_vector_chunks00',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'vectors': b'\\x11\"3D\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00',\n }),\n ]),\n }),\n })\n# ---\n# name: test_updates[1. Initial dataset]\n OrderedDict({\n 'sql': 'select * from v',\n 'rows': list([\n OrderedDict({\n 'rowid': 1,\n 'p': 'a',\n 'a': b'\\x11\\x11\\x11\\x11',\n }),\n OrderedDict({\n 'rowid': 2,\n 'p': 'a',\n 'a': b'\"\"\"\"',\n }),\n OrderedDict({\n 'rowid': 3,\n 'p': 'a',\n 'a': b'3333',\n }),\n ]),\n })\n# ---\n# name: test_updates[2. update #1]\n dict({\n 'error': 'OperationalError',\n 'message': 'UPDATE on partition key columns are not supported yet. ',\n })\n# ---\n" }, { "path": "tests/afbd/.gitignore", "content": "*.tgz\n" }, { "path": "tests/afbd/.python-version", "content": "3.12\n" }, { "path": "tests/afbd/Makefile", "content": "random_ints_1m.tgz:\n\tcurl -o $@ https://storage.googleapis.com/ann-filtered-benchmark/datasets/random_ints_1m.tgz\n\nrandom_float_1m.tgz:\n\tcurl -o $@ https://storage.googleapis.com/ann-filtered-benchmark/datasets/random_float_1m.tgz\n\nrandom_keywords_1m.tgz:\n\tcurl -o $@ https://storage.googleapis.com/ann-filtered-benchmark/datasets/random_keywords_1m.tgz\nall: random_ints_1m.tgz random_float_1m.tgz random_keywords_1m.tgz\n" }, { "path": "tests/afbd/README.md", "content": "\n# hnm\n\n```\ntar -xOzf hnm.tgz ./tests.jsonl > tests.jsonl\nsolite q \"select group_concat(distinct key) from lines_read('tests.jsonl'), json_each(line -> '$.conditions.and[0]')\"\n```\n\n\n```\n> python test-afbd.py build hnm.tgz --metadata product_group_name,colour_group_name,index_group_name,perceived_colour_value_name,section_name,product_type_name,department_name,graphical_appearance_name,garment_group_name,perceived_colour_master_name\n```\n" }, { "path": "tests/afbd/test-afbd.py", "content": "import numpy as np\nfrom tqdm import tqdm\nfrom deepdiff import DeepDiff\n\nimport tarfile\nimport json\nfrom io import BytesIO\nimport sqlite3\nfrom typing import List\nfrom struct import pack\nimport time\nfrom pathlib import Path\nimport argparse\n\n\ndef serialize_float32(vector: List[float]) -> bytes:\n \"\"\"Serializes a list of floats into the \"raw bytes\" format sqlite-vec expects\"\"\"\n return pack(\"%sf\" % len(vector), *vector)\n\n\ndef build_command(file_path, metadata_set=None):\n if metadata_set:\n metadata_set = set(metadata_set.split(\",\"))\n\n file_path = Path(file_path)\n print(f\"reading {file_path}...\")\n t0 = time.time()\n with tarfile.open(file_path, \"r:gz\") as archive:\n for file in archive:\n if file.name == \"./payloads.jsonl\":\n payloads = [\n json.loads(line)\n for line in archive.extractfile(file.name).readlines()\n ]\n if file.name == \"./tests.jsonl\":\n tests = [\n json.loads(line)\n for line in archive.extractfile(file.name).readlines()\n ]\n if file.name == \"./vectors.npy\":\n f = BytesIO()\n f.write(archive.extractfile(file.name).read())\n f.seek(0)\n vectors = np.load(f)\n\n assert payloads is not None\n assert tests is not None\n assert vectors is not None\n dimensions = vectors.shape[1]\n metadata_columns = sorted(list(payloads[0].keys()))\n\n def col_type(v):\n if isinstance(v, int):\n return \"integer\"\n if isinstance(v, float):\n return \"float\"\n if isinstance(v, str):\n return \"text\"\n raise Exception(f\"Unknown column type: {v}\")\n\n metadata_columns_types = [col_type(payloads[0][col]) for col in metadata_columns]\n\n print(time.time() - t0)\n t0 = time.time()\n print(\"seeding...\")\n\n db = sqlite3.connect(f\"{file_path.stem}.db\")\n db.execute(\"PRAGMA page_size = 16384\")\n db.row_factory = sqlite3.Row\n db.enable_load_extension(True)\n db.load_extension(\"../../dist/vec0\")\n db.enable_load_extension(False)\n\n with db:\n db.execute(\"create table tests(data)\")\n\n for test in tests:\n db.execute(\"insert into tests values (?)\", [json.dumps(test)])\n\n with db:\n create_sql = f\"create virtual table v using vec0(vector float[{dimensions}] distance_metric=cosine\"\n insert_sql = \"insert into v(rowid, vector\"\n for name, type in zip(metadata_columns, metadata_columns_types):\n if metadata_set:\n if name in metadata_set:\n create_sql += f\", {name} {type}\"\n else:\n create_sql += f\", +{name} {type}\"\n else:\n create_sql += f\", {name} {type}\"\n\n insert_sql += f\", {name}\"\n create_sql += \")\"\n insert_sql += \") values (\" + \",\".join(\"?\" * (2 + len(metadata_columns))) + \")\"\n print(create_sql)\n print(insert_sql)\n\n db.execute(create_sql)\n\n for idx, (payload, vector) in enumerate(\n tqdm(zip(payloads, vectors), total=len(payloads))\n ):\n params = [idx, vector]\n for c in metadata_columns:\n params.append(payload[c])\n db.execute(insert_sql, params)\n\n print(time.time() - t0)\n\n\ndef tests_command(file_path):\n file_path = Path(file_path)\n db = sqlite3.connect(f\"{file_path.stem}.db\")\n db.execute(\"PRAGMA cache_size = -100000000\")\n db.row_factory = sqlite3.Row\n db.enable_load_extension(True)\n db.load_extension(\"../../dist/vec0\")\n db.enable_load_extension(False)\n\n tests = [\n json.loads(row[\"data\"])\n for row in db.execute(\"select data from tests\").fetchall()\n ]\n\n num_or_skips = 0\n num_1off_errors = 0\n\n t0 = time.time()\n print(\"testing...\")\n for idx, test in enumerate(tqdm(tests)):\n query = test[\"query\"]\n conditions = test[\"conditions\"]\n expected_closest_ids = test[\"closest_ids\"]\n expected_closest_scores = test[\"closest_scores\"]\n\n sql = \"select rowid, 1 - distance as similarity from v where vector match ? and k = ?\"\n params = [serialize_float32(query), len(expected_closest_ids)]\n\n if \"and\" in conditions:\n for condition in conditions[\"and\"]:\n assert len(condition.keys()) == 1\n column = list(condition.keys())[0]\n assert len(list(condition[column].keys())) == 1\n condition_type = list(condition[column].keys())[0]\n if condition_type == \"match\":\n value = condition[column][\"match\"][\"value\"]\n sql += f\" and {column} = ?\"\n params.append(value)\n elif condition_type == \"range\":\n sql += f\" and {column} between ? and ?\"\n params.append(condition[column][\"range\"][\"gt\"])\n params.append(condition[column][\"range\"][\"lt\"])\n else:\n raise Exception(f\"Unknown condition type: {condition_type}\")\n elif \"or\" in conditions:\n column = list(conditions[\"or\"][0].keys())[0]\n condition_type = list(conditions[\"or\"][0][column].keys())[0]\n assert condition_type == \"match\"\n sql += f\" and {column} in (\"\n for idx, condition in enumerate(conditions[\"or\"]):\n if condition_type == \"match\":\n value = condition[column][\"match\"][\"value\"]\n if idx != 0:\n sql += \",\"\n sql += \"?\"\n params.append(value)\n elif condition_type == \"range\":\n breakpoint()\n else:\n raise Exception(f\"Unknown condition type: {condition_type}\")\n sql += \")\"\n\n # print(sql, params[1:])\n rows = db.execute(sql, params).fetchall()\n actual_closest_ids = [row[\"rowid\"] for row in rows]\n matches = expected_closest_ids == actual_closest_ids\n if not matches:\n diff = DeepDiff(\n expected_closest_ids, actual_closest_ids, ignore_order=False\n )\n assert len(list(diff.keys())) == 1\n assert \"values_changed\" in diff.keys()\n keys_changed = list(diff[\"values_changed\"].keys())\n if len(keys_changed) == 2:\n akey, bkey = keys_changed\n a = int(akey.lstrip(\"root[\").rstrip(\"]\"))\n b = int(bkey.lstrip(\"root[\").rstrip(\"]\"))\n assert abs(a - b) == 1\n assert (\n diff[\"values_changed\"][akey][\"new_value\"]\n == diff[\"values_changed\"][bkey][\"old_value\"]\n )\n assert (\n diff[\"values_changed\"][akey][\"old_value\"]\n == diff[\"values_changed\"][bkey][\"new_value\"]\n )\n elif len(keys_changed) == 1:\n v = int(keys_changed[0].lstrip(\"root[\").rstrip(\"]\"))\n assert (v + 1) == len(expected_closest_ids)\n else:\n raise Exception(\"fuck\")\n num_1off_errors += 1\n # print(closest_scores)\n # print([row[\"similarity\"] for row in rows])\n # assert closest_scores == [row[\"similarity\"] for row in rows]\n print(\"Number skipped: \", num_or_skips)\n print(\"Num 1 off errors: \", num_1off_errors)\n print(\"1 off error rate: \", num_1off_errors / (len(tests) - num_or_skips))\n print(time.time() - t0)\n print(\"done\")\n\n\ndef main():\n parser = argparse.ArgumentParser(description=\"CLI tool\")\n subparsers = parser.add_subparsers(dest=\"command\", required=True)\n\n build_parser = subparsers.add_parser(\"build\")\n build_parser.add_argument(\"file\", type=str, help=\"Path to input file\")\n build_parser.add_argument(\"--metadata\", type=str, help=\"Metadata columns\")\n build_parser.set_defaults(func=lambda args: build_command(args.file, args.metadata))\n\n tests_parser = subparsers.add_parser(\"test\")\n tests_parser.add_argument(\"file\", type=str, help=\"Path to input file\")\n tests_parser.set_defaults(func=lambda args: tests_command(args.file))\n\n args = parser.parse_args()\n args.func(args)\n\n\nif __name__ == \"__main__\":\n main()\n" }, { "path": "tests/build.rs", "content": "fn main() {\n cc::Build::new()\n .file(\"../sqlite-vec.c\")\n .file(\"../vendor/sqlite3.c\")\n .define(\"SQLITE_CORE\", None)\n .include(\"../vendor\")\n .include(\"..\")\n .static_flag(true)\n .compile(\"sqlite-vec-internal\");\n println!(\"cargo:rerun-if-changed=build.rs\");\n println!(\"cargo:rerun-if-changed=../sqlite-vec.c\");\n}\n" }, { "path": "tests/conftest.py", "content": "import pytest\nimport sqlite3\n\n\n@pytest.fixture()\ndef db():\n db = sqlite3.connect(\":memory:\")\n db.row_factory = sqlite3.Row\n db.enable_load_extension(True)\n db.load_extension(\"dist/vec0\")\n db.enable_load_extension(False)\n return db\n" }, { "path": "tests/correctness/build.py", "content": "import numpy as np\nimport duckdb\ndb = duckdb.connect(\":memory:\")\n\nresult = db.execute(\n\"\"\"\n select\n -- _id,\n -- title,\n -- text as contents,\n embedding::float[] as embeddings\n from \"hf://datasets/Supabase/dbpedia-openai-3-large-1M/dbpedia_openai_3_large_00.parquet\"\n\"\"\"\n).fetchnumpy()['embeddings']\n\nnp.save(\"dbpedia_openai_3_large_00.npy\", np.vstack(result))\n" }, { "path": "tests/correctness/test-correctness.py", "content": "import numpy as np\nimport numpy.typing as npt\nimport time\nimport tqdm\nimport pytest\n\ndef cosine_similarity(\n vec: npt.NDArray[np.float32], mat: npt.NDArray[np.float32], do_norm: bool = True\n) -> npt.NDArray[np.float32]:\n sim = vec @ mat.T\n if do_norm:\n sim /= np.linalg.norm(vec) * np.linalg.norm(mat, axis=1)\n return sim\n\ndef distance_l2(\n vec: npt.NDArray[np.float32], mat: npt.NDArray[np.float32]\n) -> npt.NDArray[np.float32]:\n return np.sqrt(np.sum((mat - vec) ** 2, axis=1))\n\n\ndef topk(\n vec: npt.NDArray[np.float32],\n mat: npt.NDArray[np.float32],\n k: int = 5,\n) -> tuple[npt.NDArray[np.int32], npt.NDArray[np.float32]]:\n distances = distance_l2(vec, mat)\n # Rather than sorting all similarities and taking the top K, it's faster to\n # argpartition and then just sort the top K.\n # The difference is O(N logN) vs O(N + k logk)\n indices = np.argpartition(distances, kth=k)[:k]\n top_indices = indices[np.argsort(distances[indices])]\n return top_indices, distances[top_indices]\n\n\n\nvec = np.array([1.0, 2.0, 3.0], dtype=np.float32)\nmat = np.array([\n [4.0, 5.0, 6.0],\n [1.0, 2.0, 1.0],\n [7.0, 8.0, 9.0]\n], dtype=np.float32)\nindices, distances = topk(vec, mat, k=2)\nprint(indices)\nprint(distances)\n\nimport sqlite3\nimport json\ndb = sqlite3.connect(\":memory:\")\ndb.enable_load_extension(True)\ndb.load_extension(\"../../dist/vec0\")\ndb.execute(\"select load_extension('../../dist/vec0', 'sqlite3_vec_fs_read_init')\")\ndb.enable_load_extension(False)\n\nresults = db.execute(\n '''\n select\n key,\n --value,\n vec_distance_l2(:q, value) as distance\n from json_each(:base)\n order by distance\n limit 2\n ''',\n {\n 'base': json.dumps(mat.tolist()),\n 'q': '[1.0, 2.0, 3.0]'\n }).fetchall()\na = [row[0] for row in results]\nb = [row[1] for row in results]\nprint(a)\nprint(b)\n\n\n#import sys; sys.exit()\n\ndb.execute('PRAGMA page_size=16384')\n\nprint(\"Loading into sqlite-vec vec0 table...\")\nt0 = time.time()\ndb.execute(\"create virtual table v using vec0(a float[3072], chunk_size=16)\")\ndb.execute('insert into v select rowid, vector from vec_npy_each(vec_npy_file(\"dbpedia_openai_3_large_00.npy\"))')\nprint(time.time() - t0)\n\nprint(\"loading numpy array...\")\nt0 = time.time()\nbase = np.load('dbpedia_openai_3_large_00.npy')\nprint(time.time() - t0)\n\nnp.random.seed(1)\nqueries = base[np.random.choice(base.shape[0], 20, replace=False), :]\n\nnp_durations = []\nvec_durations = []\nfrom random import randrange\n\ndef test_all():\n for idx, query in tqdm.tqdm(enumerate(queries)):\n #k = randrange(20, 1000)\n #k = 500\n k = 10\n\n t0 = time.time()\n np_ids, np_distances = topk(query, base, k=k)\n np_durations.append(time.time() - t0)\n\n t0 = time.time()\n rows = db.execute('select rowid, distance from v where a match ? and k = ?', [query, k]).fetchall()\n vec_durations.append(time.time() - t0)\n\n vec_ids = [row[0] for row in rows]\n vec_distances = [row[1] for row in rows]\n\n assert vec_distances == np_distances.tolist()\n #assert vec_ids == np_ids.tolist()\n #if (vec_ids != np_ids).any():\n # print('idx', idx)\n # print('query', query)\n # print('np_ids', np_ids)\n # print('np_distances', np_distances)\n # print('vec_ids', vec_ids)\n # print('vec_distances', vec_distances)\n # raise Exception(idx)\n\n print('final', 'np' ,np.mean(np_durations), 'vec', np.mean(vec_durations))\n" }, { "path": "tests/fuzz/.gitignore", "content": "*.dSYM\ntargets/\n" }, { "path": "tests/fuzz/Makefile", "content": "# Auto-detect clang with libFuzzer support.\n# Priority: Homebrew LLVM (macOS ARM) → Homebrew LLVM (macOS Intel) →\n# versioned clang (Linux) → system clang\nFUZZ_CC ?= $(shell \\\n if [ -x /opt/homebrew/opt/llvm/bin/clang ]; then \\\n echo \"/opt/homebrew/opt/llvm/bin/clang\"; \\\n elif [ -x /usr/local/opt/llvm/bin/clang ]; then \\\n echo \"/usr/local/opt/llvm/bin/clang\"; \\\n elif command -v clang-18 >/dev/null 2>&1; then \\\n echo \"clang-18\"; \\\n elif command -v clang-17 >/dev/null 2>&1; then \\\n echo \"clang-17\"; \\\n elif command -v clang >/dev/null 2>&1; then \\\n echo \"clang\"; \\\n else \\\n echo \"FUZZ_CC_NOT_FOUND\"; \\\n fi)\n\n# AddressSanitizer + UndefinedBehaviorSanitizer + libFuzzer.\n# Override FUZZ_SANITIZERS to change (e.g., drop ubsan on Windows).\nFUZZ_SANITIZERS ?= -fsanitize=address,undefined,fuzzer\n\n# On macOS, Homebrew LLVM may need -Wl,-ld_classic to work with the system linker.\nFUZZ_LDFLAGS ?= $(shell \\\n if [ \"$$(uname -s)\" = \"Darwin\" ]; then \\\n echo \"-Wl,-ld_classic\"; \\\n fi)\n\nFUZZ_CFLAGS = $(FUZZ_SANITIZERS) -I ../../ -I ../../vendor -DSQLITE_CORE -g $(FUZZ_LDFLAGS)\nFUZZ_SRCS = ../../vendor/sqlite3.c ../../sqlite-vec.c\n\nTARGET_DIR = ./targets\n\n$(TARGET_DIR):\n\tmkdir -p $@\n\n# Existing targets (filename uses -, Makefile target uses _)\n$(TARGET_DIR)/vec0_create: vec0-create.c $(FUZZ_SRCS) | $(TARGET_DIR)\n\t$(FUZZ_CC) $(FUZZ_CFLAGS) $(FUZZ_SRCS) $< -o $@\n\n$(TARGET_DIR)/exec: exec.c $(FUZZ_SRCS) | $(TARGET_DIR)\n\t$(FUZZ_CC) $(FUZZ_CFLAGS) $(FUZZ_SRCS) $< -o $@\n\n$(TARGET_DIR)/json: json.c $(FUZZ_SRCS) | $(TARGET_DIR)\n\t$(FUZZ_CC) $(FUZZ_CFLAGS) $(FUZZ_SRCS) $< -o $@\n\n$(TARGET_DIR)/numpy: numpy.c $(FUZZ_SRCS) | $(TARGET_DIR)\n\t$(FUZZ_CC) $(FUZZ_CFLAGS) $(FUZZ_SRCS) $< -o $@\n\n# New targets\n$(TARGET_DIR)/shadow_corrupt: shadow-corrupt.c $(FUZZ_SRCS) | $(TARGET_DIR)\n\t$(FUZZ_CC) $(FUZZ_CFLAGS) $(FUZZ_SRCS) $< -o $@\n\n$(TARGET_DIR)/vec0_operations: vec0-operations.c $(FUZZ_SRCS) | $(TARGET_DIR)\n\t$(FUZZ_CC) $(FUZZ_CFLAGS) $(FUZZ_SRCS) $< -o $@\n\n$(TARGET_DIR)/scalar_functions: scalar-functions.c $(FUZZ_SRCS) | $(TARGET_DIR)\n\t$(FUZZ_CC) $(FUZZ_CFLAGS) $(FUZZ_SRCS) $< -o $@\n\n$(TARGET_DIR)/vec0_create_full: vec0-create-full.c $(FUZZ_SRCS) | $(TARGET_DIR)\n\t$(FUZZ_CC) $(FUZZ_CFLAGS) $(FUZZ_SRCS) $< -o $@\n\n$(TARGET_DIR)/metadata_columns: metadata-columns.c $(FUZZ_SRCS) | $(TARGET_DIR)\n\t$(FUZZ_CC) $(FUZZ_CFLAGS) $(FUZZ_SRCS) $< -o $@\n\n$(TARGET_DIR)/vec_each: vec-each.c $(FUZZ_SRCS) | $(TARGET_DIR)\n\t$(FUZZ_CC) $(FUZZ_CFLAGS) $(FUZZ_SRCS) $< -o $@\n\n$(TARGET_DIR)/vec_mismatch: vec-mismatch.c $(FUZZ_SRCS) | $(TARGET_DIR)\n\t$(FUZZ_CC) $(FUZZ_CFLAGS) $(FUZZ_SRCS) $< -o $@\n\n$(TARGET_DIR)/vec0_delete_completeness: vec0-delete-completeness.c $(FUZZ_SRCS) | $(TARGET_DIR)\n\t$(FUZZ_CC) $(FUZZ_CFLAGS) $(FUZZ_SRCS) $< -o $@\n\nFUZZ_TARGETS = vec0_create exec json numpy \\\n shadow_corrupt vec0_operations scalar_functions \\\n vec0_create_full metadata_columns vec_each vec_mismatch \\\n vec0_delete_completeness\n\nall: $(addprefix $(TARGET_DIR)/,$(FUZZ_TARGETS))\n\nclean:\n\trm -rf $(TARGET_DIR)/*\n\n.PHONY: all clean\n" }, { "path": "tests/fuzz/README.md", "content": "```\nASAN_OPTIONS=detect_leaks=1 ./targets/vec0_create \\\n -dict=./vec0-create.dict -max_total_time=5 \\\n ./corpus/vec0-create\n```\n\n```\nexport PATH=\"/opt/homebrew/opt/llvm/bin:$PATH\"\nexport LDFLAGS=\"-L/opt/homebrew/opt/llvm/lib\"\nexport CPPFLAGS=\"-I/opt/homebrew/opt/llvm/include\"\n\n\nLDFLAGS=\"-L/opt/homebrew/opt/llvm/lib/c++ -Wl,-rpath,/opt/homebrew/opt/llvm/lib/c++\"\n```\n" }, { "path": "tests/fuzz/TODO.md", "content": "# Fuzz Testing TODO: Undefined Behavior Findings\n\nUBSAN findings from fuzz targets. None are crash-level bugs, but all are\nformally undefined behavior per the C standard.\n\n## Class 1: Function pointer type mismatch (~20 sites)\n\n`fvec_cleanup_noop` is defined as `void (f32 *)` but called through\n`vector_cleanup` which is `void (*)(void *)`. Two cleanup typedefs exist\nwith incompatible signatures:\n\n```c\ntypedef void (*vector_cleanup)(void *p); // line 597\ntypedef void (*fvec_cleanup)(f32 *vector); // line 695\n```\n\nAffected lines: 1031, 1049, 1050, 1160, 1200, 1201, 1241, 1242, 1282,\n1283, 1324, 1325, 1356, 1424, 1524, 1525, 1582, 1583, 1699, 1749, 1798,\n2520, 7236, 8501, and sqlite3.c:82930 (via sqlite3_result_blob destructor).\n\nLow practical severity — calling conventions on all real platforms pass\n`f32 *` and `void *` identically — but flags on every UBSAN run.\n\nFix: change `fvec_cleanup_noop` to take `void *`, or unify the typedefs.\n\n## Class 2: Misaligned f32 reads (~10 sites)\n\n`f32` (4-byte alignment required) read from potentially unaligned addresses.\nHappens when a blob from SQLite's internal storage is cast to `f32 *` and\ndereferenced. The blob pointer may not be 4-byte aligned.\n\nAffected lines: 369, 446, 473-475, 1401, 1461, 1501, 1559, 1653, 1726,\n1789, 1793.\n\nMedium severity — silent on x86/ARM64 (hardware supports unaligned float\naccess) but UB on strict-alignment architectures.\n\nFix: use `memcpy` to load floats from potentially-unaligned memory, or\nensure blob pointers are aligned before use.\n\n## Class 3: Float-to-integer overflow (1 site)\n\n`vec_quantize_int8` at line 1461 — when `srcVector[i]` is a large float,\nthe expression `((srcVector[i] - (-1.0)) / step) - 128` overflows\n`signed char` range. Assigning this to `i8 out[i]` is UB.\n\nLow-medium severity — silent truncation in practice.\n\nFix: clamp the result before cast.\n" }, { "path": "tests/fuzz/corpus/exec/select1", "content": "SELECT 1" }, { "path": "tests/fuzz/corpus/exec/vec_version", "content": "SELECT vec_version()" }, { "path": "tests/fuzz/corpus/json/empty", "content": "[]" }, { "path": "tests/fuzz/corpus/json/valid_2d", "content": "[0.5, -0.5]" }, { "path": "tests/fuzz/corpus/json/valid_4d", "content": "[1.0, 2.0, 3.0, 4.0]" }, { "path": "tests/fuzz/corpus/vec-mismatch/json_1d_blob_5byte", "content": "\fABCDE" }, { "path": "tests/fuzz/corpus/vec-mismatch/json_2d_blob_3byte", "content": "\u0006ABC" }, { "path": "tests/fuzz/corpus/vec0-create/normal1", "content": "aaa float[12]\n" }, { "path": "tests/fuzz/corpus/vec0-create/normal2", "content": "aaa float[12], bbb int8[6]\n" }, { "path": "tests/fuzz/exec.c", "content": "#include \n#include \n\n#include \n#include \n#include \n#include \"sqlite-vec.h\"\n#include \"sqlite3.h\"\n#include \n\nint LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {\n int rc = SQLITE_OK;\n sqlite3 *db;\n sqlite3_stmt *stmt;\n if(size < 1) return 0;\n\n rc = sqlite3_open(\":memory:\", &db);\n assert(rc == SQLITE_OK);\n rc = sqlite3_vec_init(db, NULL, NULL);\n assert(rc == SQLITE_OK);\n\n const char * zSrc = sqlite3_mprintf(\"%.*s\", size, data);\n assert(zSrc);\n\n sqlite3_exec(db, zSrc, NULL, NULL, NULL);\n sqlite3_free(zSrc);\n\n sqlite3_close(db);\n return 0;\n}\n" }, { "path": "tests/fuzz/exec.dict", "content": "select=\"select\"\nfrom=\"from\"\ncname1=\"aaa\"\ncname1=\"bbb\"\ncname1=\"ccc\"\ntype1=\"float\"\ntype2=\"int8\"\ntype3=\"bit\"\nlparen=\"[\"\nrparen=\"]\"\npk=\"primary key\"\ntext=\"text\"\ndistance_metric=\"distance_metric\"\neq=\"=\"\nl1=\"l1\"\nl2=\"l2\"\ncosine=\"cosine\"\nhamming=\"hamming\"\nvec_distance_l2=\"vec_distance_l2\"\nvec_distance_l1=\"vec_distance_l1\"\ncomma=\",\"\n" }, { "path": "tests/fuzz/json.c", "content": "#include \n#include \n\n#include \n#include \n#include \n#include \"sqlite-vec.h\"\n#include \"sqlite3.h\"\n#include \n\nint LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {\n int rc = SQLITE_OK;\n sqlite3 *db;\n sqlite3_stmt *stmt;\n\n //rc = sqlite3_auto_extension((void (*)())sqlite3_vec_init);\n //assert(rc == SQLITE_OK);\n\n rc = sqlite3_open(\":memory:\", &db);\n assert(rc == SQLITE_OK);\n rc = sqlite3_vec_init(db, NULL, NULL);\n assert(rc == SQLITE_OK);\n\n rc = sqlite3_prepare_v2(db, \"SELECT vec_f32(cast(? as text))\", -1, &stmt, NULL);\n assert(rc == SQLITE_OK);\n\n sqlite3_bind_blob(stmt, 1, data, size, SQLITE_STATIC);\n sqlite3_step(stmt);\n\n sqlite3_finalize(stmt);\n sqlite3_close(db);\n return 0;\n\n}\n" }, { "path": "tests/fuzz/metadata-columns.c", "content": "#include \n#include \n#include \n#include \n#include \n#include \"sqlite-vec.h\"\n#include \"sqlite3.h\"\n#include \n\nint LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {\n if (size < 8) return 0;\n\n int rc;\n sqlite3 *db;\n\n rc = sqlite3_open(\":memory:\", &db);\n assert(rc == SQLITE_OK);\n rc = sqlite3_vec_init(db, NULL, NULL);\n assert(rc == SQLITE_OK);\n\n rc = sqlite3_exec(db,\n \"CREATE VIRTUAL TABLE v USING vec0(\"\n \" emb float[4],\"\n \" flag boolean metadata,\"\n \" count integer metadata,\"\n \" score float metadata,\"\n \" label text metadata,\"\n \" aux_data text auxiliary\"\n \")\", NULL, NULL, NULL);\n if (rc != SQLITE_OK) { sqlite3_close(db); return 0; }\n\n // Prepare statements for insert and query\n sqlite3_stmt *stmtInsert = NULL;\n sqlite3_stmt *stmtKnn = NULL;\n sqlite3_stmt *stmtKnnFilter = NULL;\n sqlite3_stmt *stmtDelete = NULL;\n\n sqlite3_prepare_v2(db,\n \"INSERT INTO v(rowid, emb, flag, count, score, label, aux_data) \"\n \"VALUES (?, ?, ?, ?, ?, ?, ?)\", -1, &stmtInsert, NULL);\n sqlite3_prepare_v2(db,\n \"SELECT rowid, distance FROM v WHERE emb MATCH ? LIMIT 3\",\n -1, &stmtKnn, NULL);\n sqlite3_prepare_v2(db,\n \"SELECT rowid, distance FROM v WHERE emb MATCH ? AND flag = 1 LIMIT 3\",\n -1, &stmtKnnFilter, NULL);\n sqlite3_prepare_v2(db,\n \"DELETE FROM v WHERE rowid = ?\", -1, &stmtDelete, NULL);\n\n if (!stmtInsert || !stmtKnn || !stmtKnnFilter || !stmtDelete) goto cleanup;\n\n size_t i = 0;\n while (i + 6 <= size) {\n uint8_t op = data[i++] % 4;\n uint8_t rowid_byte = data[i++];\n int64_t rowid = (int64_t)(rowid_byte % 50) + 1;\n\n switch (op) {\n case 0: {\n // INSERT with fuzz-derived vector and metadata\n float vec[4];\n for (int j = 0; j < 4 && i < size; j++, i++) {\n vec[j] = (float)((int8_t)data[i]) / 10.0f;\n }\n int flag_val = (i < size) ? data[i++] % 2 : 0;\n int count_val = (i < size) ? (int)((int8_t)data[i++]) : 0;\n float score_val = (i < size) ? (float)((int8_t)data[i++]) / 10.0f : 0.0f;\n\n sqlite3_reset(stmtInsert);\n sqlite3_bind_int64(stmtInsert, 1, rowid);\n sqlite3_bind_blob(stmtInsert, 2, vec, sizeof(vec), SQLITE_TRANSIENT);\n sqlite3_bind_int(stmtInsert, 3, flag_val);\n sqlite3_bind_int(stmtInsert, 4, count_val);\n sqlite3_bind_double(stmtInsert, 5, (double)score_val);\n sqlite3_bind_text(stmtInsert, 6, \"label\", -1, SQLITE_STATIC);\n sqlite3_bind_text(stmtInsert, 7, \"aux\", -1, SQLITE_STATIC);\n sqlite3_step(stmtInsert);\n break;\n }\n case 1: {\n // KNN query (no filter)\n float qvec[4] = {1.0f, 0.0f, 0.0f, 0.0f};\n sqlite3_reset(stmtKnn);\n sqlite3_bind_blob(stmtKnn, 1, qvec, sizeof(qvec), SQLITE_STATIC);\n while (sqlite3_step(stmtKnn) == SQLITE_ROW) {}\n break;\n }\n case 2: {\n // KNN query WITH metadata filter\n float qvec[4] = {0.0f, 1.0f, 0.0f, 0.0f};\n sqlite3_reset(stmtKnnFilter);\n sqlite3_bind_blob(stmtKnnFilter, 1, qvec, sizeof(qvec), SQLITE_STATIC);\n while (sqlite3_step(stmtKnnFilter) == SQLITE_ROW) {}\n break;\n }\n case 3: {\n // DELETE\n sqlite3_reset(stmtDelete);\n sqlite3_bind_int64(stmtDelete, 1, rowid);\n sqlite3_step(stmtDelete);\n break;\n }\n }\n }\n\n sqlite3_exec(db, \"SELECT * FROM v\", NULL, NULL, NULL);\n\ncleanup:\n sqlite3_finalize(stmtInsert);\n sqlite3_finalize(stmtKnn);\n sqlite3_finalize(stmtKnnFilter);\n sqlite3_finalize(stmtDelete);\n sqlite3_close(db);\n return 0;\n}\n" }, { "path": "tests/fuzz/numpy.c", "content": "#include \n#include \n\n#include \n#include \n#include \n#include \"sqlite-vec.h\"\n#include \"sqlite3.h\"\n#include \n\nextern int sqlite3_vec_numpy_init(sqlite3 *db, char **pzErrMsg,\n const sqlite3_api_routines *pApi);\n\nint LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {\n int rc = SQLITE_OK;\n sqlite3 *db;\n sqlite3_stmt *stmt;\n\n rc = sqlite3_open(\":memory:\", &db);\n assert(rc == SQLITE_OK);\n rc = sqlite3_vec_init(db, NULL, NULL);\n assert(rc == SQLITE_OK);\n rc = sqlite3_vec_numpy_init(db, NULL, NULL);\n assert(rc == SQLITE_OK);\n\n rc = sqlite3_prepare_v2(db, \"select * from vec_npy_each(?)\", -1, &stmt, NULL);\n assert(rc == SQLITE_OK);\n sqlite3_bind_blob(stmt, 1, data, size, SQLITE_STATIC);\n rc = sqlite3_step(stmt);\n while (rc == SQLITE_ROW) {\n rc = sqlite3_step(stmt);\n }\n\n sqlite3_finalize(stmt);\n sqlite3_close(db);\n return 0;\n}\n" }, { "path": "tests/fuzz/numpy.dict", "content": "magic=\"\\x93NUMPY\"\nlparen=\"(\"\nrparen=\")\"\nlbrace=\"{\"\nrbrace=\"}\"\nsq1=\"\\\"\"\nsq2=\"'\"\n" }, { "path": "tests/fuzz/scalar-functions.c", "content": "#include \n#include \n#include \n#include \n#include \n#include \"sqlite-vec.h\"\n#include \"sqlite3.h\"\n#include \n\nint LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {\n if (size < 2) return 0;\n\n int rc;\n sqlite3 *db;\n sqlite3_stmt *stmt = NULL;\n\n rc = sqlite3_open(\":memory:\", &db);\n assert(rc == SQLITE_OK);\n rc = sqlite3_vec_init(db, NULL, NULL);\n assert(rc == SQLITE_OK);\n\n static const char *queries[] = {\n \"SELECT vec_f32(cast(? as text))\", /* 0: JSON text -> f32 */\n \"SELECT vec_f32(?)\", /* 1: blob -> f32 */\n \"SELECT vec_int8(?)\", /* 2: blob -> int8 */\n \"SELECT vec_bit(?)\", /* 3: blob -> bit */\n \"SELECT vec_length(?)\", /* 4: vector length */\n \"SELECT vec_type(?)\", /* 5: vector type string */\n \"SELECT vec_to_json(?)\", /* 6: vector -> JSON */\n \"SELECT vec_normalize(?)\", /* 7: normalize */\n \"SELECT vec_quantize_binary(?)\", /* 8: quantize to binary */\n \"SELECT vec_quantize_int8(?, 'unit')\", /* 9: quantize to int8 */\n \"SELECT vec_distance_l2(?, ?)\", /* 10: L2 distance */\n \"SELECT vec_distance_cosine(?, ?)\", /* 11: cosine distance */\n \"SELECT vec_distance_l1(?, ?)\", /* 12: L1 distance */\n \"SELECT vec_distance_hamming(?, ?)\", /* 13: hamming distance */\n \"SELECT vec_add(?, ?)\", /* 14: vector add */\n \"SELECT vec_sub(?, ?)\", /* 15: vector subtract */\n \"SELECT vec_slice(?, 0, ?)\", /* 16: vector slice */\n };\n static const int nQueries = sizeof(queries) / sizeof(queries[0]);\n\n int qIdx = data[0] % nQueries;\n const uint8_t *payload = data + 1;\n int payload_size = (int)(size - 1);\n\n rc = sqlite3_prepare_v2(db, queries[qIdx], -1, &stmt, NULL);\n if (rc != SQLITE_OK) {\n sqlite3_close(db);\n return 0;\n }\n\n int nParams = sqlite3_bind_parameter_count(stmt);\n\n // Bind param 1: fuzz payload as blob\n sqlite3_bind_blob(stmt, 1, payload, payload_size, SQLITE_STATIC);\n\n if (nParams >= 2) {\n if (qIdx == 16) {\n // vec_slice 3rd param is integer (end index)\n int end_idx = (payload_size > 0) ? (payload[0] % 64) : 0;\n sqlite3_bind_int(stmt, 2, end_idx);\n } else {\n // For 2-param functions (distance, add, sub): split payload in half\n int half = payload_size / 2;\n sqlite3_bind_blob(stmt, 2, payload + half,\n payload_size - half, SQLITE_STATIC);\n }\n }\n\n if (nParams >= 3) {\n // vec_slice: param 3 is the end index\n int end_idx = (payload_size > 1) ? (payload[1] % 64) : 0;\n sqlite3_bind_int(stmt, 3, end_idx);\n }\n\n sqlite3_step(stmt);\n sqlite3_finalize(stmt);\n sqlite3_close(db);\n return 0;\n}\n" }, { "path": "tests/fuzz/scalar-functions.dict", "content": "json_vec1=\"[1.0, 2.0, 3.0, 4.0]\"\njson_vec2=\"[0.5, -0.5]\"\njson_empty=\"[]\"\njson_nan=\"[NaN]\"\njson_inf=\"[Infinity]\"\njson_large=\"[1e38, -1e38]\"\nunit=\"unit\"\nnull=\"null\"\n" }, { "path": "tests/fuzz/shadow-corrupt.c", "content": "#include \n#include \n#include \n#include \n#include \n#include \"sqlite-vec.h\"\n#include \"sqlite3.h\"\n#include \n\nint LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {\n if (size < 2) return 0;\n\n int rc;\n sqlite3 *db;\n\n rc = sqlite3_open(\":memory:\", &db);\n assert(rc == SQLITE_OK);\n rc = sqlite3_vec_init(db, NULL, NULL);\n assert(rc == SQLITE_OK);\n\n // Build a valid table with 3 vectors (float[4] = 16 bytes each)\n // [1,0,0,0], [0,-1,0,1], [1,1,0,1] as little-endian float32 hex\n rc = sqlite3_exec(db,\n \"CREATE VIRTUAL TABLE v USING vec0(emb float[4]);\"\n \"INSERT INTO v(rowid, emb) VALUES (1, X'0000803f000000000000000000000000');\"\n \"INSERT INTO v(rowid, emb) VALUES (2, X'00000000000080bf000000000000803f');\"\n \"INSERT INTO v(rowid, emb) VALUES (3, X'0000803f0000803f000000000000803f');\",\n NULL, NULL, NULL);\n if (rc != SQLITE_OK) {\n sqlite3_close(db);\n return 0;\n }\n\n // Use first byte to select corruption strategy\n int target = data[0] % 6;\n const uint8_t *payload = data + 1;\n int payload_size = (int)(size - 1);\n\n sqlite3_stmt *stmt = NULL;\n\n switch (target) {\n case 0: {\n // Corrupt _chunks validity blob with fuzz data\n rc = sqlite3_prepare_v2(db,\n \"UPDATE v_chunks SET validity = ? WHERE rowid = 1\", -1, &stmt, NULL);\n if (rc == SQLITE_OK) {\n sqlite3_bind_blob(stmt, 1, payload, payload_size, SQLITE_STATIC);\n sqlite3_step(stmt);\n sqlite3_finalize(stmt);\n }\n break;\n }\n case 1: {\n // Corrupt _chunks rowids blob with fuzz data\n rc = sqlite3_prepare_v2(db,\n \"UPDATE v_chunks SET rowids = ? WHERE rowid = 1\", -1, &stmt, NULL);\n if (rc == SQLITE_OK) {\n sqlite3_bind_blob(stmt, 1, payload, payload_size, SQLITE_STATIC);\n sqlite3_step(stmt);\n sqlite3_finalize(stmt);\n }\n break;\n }\n case 2: {\n // Corrupt _vector_chunks00 vectors blob with fuzz data\n rc = sqlite3_prepare_v2(db,\n \"UPDATE v_vector_chunks00 SET vectors = ? WHERE rowid = 1\", -1, &stmt, NULL);\n if (rc == SQLITE_OK) {\n sqlite3_bind_blob(stmt, 1, payload, payload_size, SQLITE_STATIC);\n sqlite3_step(stmt);\n sqlite3_finalize(stmt);\n }\n break;\n }\n case 3: {\n // Set validity to NULL (violates NOT NULL but shadow tables are writable)\n sqlite3_exec(db,\n \"UPDATE v_chunks SET validity = NULL WHERE rowid = 1\",\n NULL, NULL, NULL);\n break;\n }\n case 4: {\n // Set rowids to NULL\n sqlite3_exec(db,\n \"UPDATE v_chunks SET rowids = NULL WHERE rowid = 1\",\n NULL, NULL, NULL);\n break;\n }\n case 5: {\n // Delete shadow table rows entirely (orphan the virtual table data)\n sqlite3_exec(db,\n \"DELETE FROM v_vector_chunks00 WHERE rowid = 1\",\n NULL, NULL, NULL);\n break;\n }\n }\n\n // Exercise all read paths — NONE should crash\n sqlite3_exec(db, \"SELECT * FROM v\", NULL, NULL, NULL);\n sqlite3_exec(db, \"SELECT * FROM v WHERE rowid = 1\", NULL, NULL, NULL);\n sqlite3_exec(db, \"SELECT * FROM v WHERE rowid = 2\", NULL, NULL, NULL);\n sqlite3_exec(db,\n \"SELECT rowid, distance FROM v \"\n \"WHERE emb MATCH X'0000803f000000000000000000000000' LIMIT 3\",\n NULL, NULL, NULL);\n sqlite3_exec(db, \"DELETE FROM v WHERE rowid = 2\", NULL, NULL, NULL);\n sqlite3_exec(db,\n \"INSERT INTO v(rowid, emb) VALUES (4, X'0000803f000000000000000000000000')\",\n NULL, NULL, NULL);\n sqlite3_exec(db, \"DROP TABLE v\", NULL, NULL, NULL);\n\n sqlite3_close(db);\n return 0;\n}\n" }, { "path": "tests/fuzz/targets/.gitignore", "content": "*\n!.gitignore\n" }, { "path": "tests/fuzz/vec-each.c", "content": "#include \n#include \n#include \n#include \n#include \n#include \"sqlite-vec.h\"\n#include \"sqlite3.h\"\n#include \n\nint LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {\n int rc;\n sqlite3 *db;\n sqlite3_stmt *stmt = NULL;\n\n rc = sqlite3_open(\":memory:\", &db);\n assert(rc == SQLITE_OK);\n rc = sqlite3_vec_init(db, NULL, NULL);\n assert(rc == SQLITE_OK);\n\n rc = sqlite3_prepare_v2(db,\n \"SELECT * FROM vec_each(?)\", -1, &stmt, NULL);\n assert(rc == SQLITE_OK);\n\n sqlite3_bind_blob(stmt, 1, data, (int)size, SQLITE_STATIC);\n while (sqlite3_step(stmt) == SQLITE_ROW) {\n // Consume all rows — just exercise the iteration path\n }\n\n sqlite3_finalize(stmt);\n sqlite3_close(db);\n return 0;\n}\n" }, { "path": "tests/fuzz/vec-mismatch.c", "content": "#include \n#include \n#include \n#include \n#include \n#include \"sqlite-vec.h\"\n#include \"sqlite3.h\"\n#include \n\n/*\n * Fuzz target that exercises error-path cleanup in vector functions.\n *\n * The key insight: when a vector is parsed from JSON TEXT, the cleanup\n * function is sqlite3_free (heap allocator). When parsed from BLOB,\n * cleanup is a no-op. Bugs in cleanup code (wrong pointer, missing\n * cleanup, double-free) are only observable with the sqlite3_free path.\n *\n * This fuzzer systematically covers:\n * 1. Valid JSON arg1 + invalid fuzz arg2 (parse failure → cleanup arg1)\n * 2. Valid JSON arg1 + valid JSON arg2 with different dimensions\n * (dimension mismatch → cleanup both)\n * 3. Valid JSON arg1 + int8/bit blob arg2 with mismatched type\n * (type mismatch → cleanup both)\n * 4. Fuzz arg1 + valid JSON arg2 (parse failure of arg1, no cleanup)\n * 5. Single-arg functions with JSON text (normal cleanup path)\n * 6. Single-arg functions with fuzz text (parse failure path)\n */\n\n/* Helper: bind a valid vector to a statement parameter.\n * mode selects the vector type and format. */\nstatic void bind_valid_vector(sqlite3_stmt *stmt, int param, int mode) {\n /* JSON text vectors — cleanup = sqlite3_free */\n static const char *json_f32_4d = \"[1.0, 0.0, 0.0, 0.0]\";\n static const char *json_f32_2d = \"[1.0, 2.0]\";\n static const char *json_f32_1d = \"[1.0]\";\n\n /* Blob vectors — cleanup = noop */\n static const float blob_f32_4d[] = {1.0f, 0.0f, 0.0f, 0.0f};\n static const float blob_f32_2d[] = {1.0f, 2.0f};\n\n /* int8 blob — 4 bytes = 4 dimensions */\n static const int8_t blob_int8_4d[] = {10, 20, 30, 40};\n\n /* bit blob — 1 byte = 8 bits */\n static const uint8_t blob_bit_1b[] = {0xAA};\n\n switch (mode % 7) {\n case 0: sqlite3_bind_text(stmt, param, json_f32_4d, -1, SQLITE_STATIC); break;\n case 1: sqlite3_bind_text(stmt, param, json_f32_2d, -1, SQLITE_STATIC); break;\n case 2: sqlite3_bind_text(stmt, param, json_f32_1d, -1, SQLITE_STATIC); break;\n case 3: sqlite3_bind_blob(stmt, param, blob_f32_4d, sizeof(blob_f32_4d), SQLITE_STATIC); break;\n case 4: sqlite3_bind_blob(stmt, param, blob_f32_2d, sizeof(blob_f32_2d), SQLITE_STATIC); break;\n case 5: /* int8 — must set subtype */\n sqlite3_bind_blob(stmt, param, blob_int8_4d, sizeof(blob_int8_4d), SQLITE_STATIC);\n break;\n case 6: /* bit — must set subtype */\n sqlite3_bind_blob(stmt, param, blob_bit_1b, sizeof(blob_bit_1b), SQLITE_STATIC);\n break;\n }\n}\n\nstatic void run_query(sqlite3 *db, const char *sql,\n int arg1_mode, int arg2_mode,\n const uint8_t *fuzz, int fuzz_len,\n int fuzz_arg, int fuzz_as_text) {\n sqlite3_stmt *stmt = NULL;\n int rc = sqlite3_prepare_v2(db, sql, -1, &stmt, NULL);\n if (rc != SQLITE_OK) return;\n\n int nParams = sqlite3_bind_parameter_count(stmt);\n\n for (int p = 1; p <= nParams; p++) {\n if (p == fuzz_arg) {\n /* Bind fuzz data */\n if (fuzz_as_text)\n sqlite3_bind_text(stmt, p, (const char *)fuzz, fuzz_len, SQLITE_STATIC);\n else\n sqlite3_bind_blob(stmt, p, fuzz, fuzz_len, SQLITE_STATIC);\n } else if (p == 1) {\n bind_valid_vector(stmt, p, arg1_mode);\n } else {\n bind_valid_vector(stmt, p, arg2_mode);\n }\n }\n\n sqlite3_step(stmt);\n sqlite3_finalize(stmt);\n}\n\nint LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {\n if (size < 3) return 0;\n\n int rc;\n sqlite3 *db;\n\n rc = sqlite3_open(\":memory:\", &db);\n assert(rc == SQLITE_OK);\n rc = sqlite3_vec_init(db, NULL, NULL);\n assert(rc == SQLITE_OK);\n\n /* --- Decode fuzz control bytes --- */\n uint8_t b0 = data[0];\n uint8_t b1 = data[1];\n uint8_t b2 = data[2];\n const uint8_t *payload = data + 3;\n int payload_size = (int)(size - 3);\n\n /* Two-argument vector functions */\n static const char *two_arg[] = {\n \"SELECT vec_distance_l2(?, ?)\",\n \"SELECT vec_distance_cosine(?, ?)\",\n \"SELECT vec_distance_l1(?, ?)\",\n \"SELECT vec_distance_hamming(?, ?)\",\n \"SELECT vec_add(?, ?)\",\n \"SELECT vec_sub(?, ?)\",\n };\n\n /* Single-argument vector functions that call cleanup */\n static const char *one_arg[] = {\n \"SELECT vec_f32(?)\",\n \"SELECT vec_int8(?)\",\n \"SELECT vec_bit(?)\",\n \"SELECT vec_length(?)\",\n \"SELECT vec_type(?)\",\n \"SELECT vec_to_json(?)\",\n \"SELECT vec_normalize(?)\",\n \"SELECT vec_quantize_binary(?)\",\n };\n\n int qIdx2 = b0 % 6;\n int qIdx1 = b0 % 8;\n int arg1_mode = b1 % 7;\n int arg2_mode = b2 % 7;\n\n /*\n * Phase 1: Two-arg functions — fuzz arg2, valid arg1\n * Exercises: parse-failure cleanup of arg1 (the fixed bug),\n * type mismatch cleanup, dimension mismatch cleanup.\n */\n /* arg2 as fuzz blob */\n run_query(db, two_arg[qIdx2], arg1_mode, 0,\n payload, payload_size, /*fuzz_arg=*/2, /*as_text=*/0);\n /* arg2 as fuzz text */\n run_query(db, two_arg[qIdx2], arg1_mode, 0,\n payload, payload_size, /*fuzz_arg=*/2, /*as_text=*/1);\n\n /*\n * Phase 2: Two-arg functions — fuzz arg1, valid arg2\n * Exercises: parse-failure of arg1 (no cleanup needed), and\n * type/dimension mismatch when arg1 parses to unexpected type.\n */\n run_query(db, two_arg[qIdx2], 0, arg2_mode,\n payload, payload_size, /*fuzz_arg=*/1, /*as_text=*/0);\n run_query(db, two_arg[qIdx2], 0, arg2_mode,\n payload, payload_size, /*fuzz_arg=*/1, /*as_text=*/1);\n\n /*\n * Phase 3: Two-arg — both valid but deliberately mismatched types/dims.\n * arg1_mode and arg2_mode often produce different types or dimensions.\n * Exercises: type mismatch (lines 1035-1042) and dimension mismatch\n * (lines 1044-1051) with sqlite3_free cleanup on both sides.\n */\n run_query(db, two_arg[qIdx2], arg1_mode, arg2_mode,\n NULL, 0, /*fuzz_arg=*/0, /*as_text=*/0);\n\n /*\n * Phase 4: Single-arg functions — fuzz as blob and text.\n * Exercises: parse failure paths in vec_f32, vec_int8, vec_bit, etc.\n * Also exercises normal cleanup when fuzz data happens to be valid.\n */\n run_query(db, one_arg[qIdx1], 0, 0,\n payload, payload_size, /*fuzz_arg=*/1, /*as_text=*/0);\n run_query(db, one_arg[qIdx1], 0, 0,\n payload, payload_size, /*fuzz_arg=*/1, /*as_text=*/1);\n\n sqlite3_close(db);\n return 0;\n}\n" }, { "path": "tests/fuzz/vec0-create-full.c", "content": "#include \n#include \n#include \n#include \n#include \n#include \"sqlite-vec.h\"\n#include \"sqlite3.h\"\n#include \n\nint LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {\n int rc;\n sqlite3 *db;\n\n rc = sqlite3_open(\":memory:\", &db);\n assert(rc == SQLITE_OK);\n rc = sqlite3_vec_init(db, NULL, NULL);\n assert(rc == SQLITE_OK);\n\n // Create table with fuzz input as column definitions\n sqlite3_str *s = sqlite3_str_new(NULL);\n assert(s);\n sqlite3_str_appendall(s, \"CREATE VIRTUAL TABLE v USING vec0(\");\n sqlite3_str_appendf(s, \"%.*s\", (int)size, data);\n sqlite3_str_appendall(s, \")\");\n char *zSql = sqlite3_str_finish(s);\n assert(zSql);\n\n rc = sqlite3_exec(db, zSql, NULL, NULL, NULL);\n sqlite3_free(zSql);\n\n if (rc == SQLITE_OK) {\n // Table was created — try to use it. These may fail (errors are fine),\n // but must never crash.\n sqlite3_exec(db, \"INSERT INTO v(rowid) VALUES (1)\", NULL, NULL, NULL);\n sqlite3_exec(db, \"SELECT * FROM v\", NULL, NULL, NULL);\n sqlite3_exec(db, \"SELECT * FROM v WHERE rowid = 1\", NULL, NULL, NULL);\n sqlite3_exec(db, \"DELETE FROM v WHERE rowid = 1\", NULL, NULL, NULL);\n sqlite3_exec(db, \"DROP TABLE v\", NULL, NULL, NULL);\n }\n\n sqlite3_close(db);\n return 0;\n}\n" }, { "path": "tests/fuzz/vec0-create.c", "content": "#include \n#include \n\n#include \n#include \n#include \n#include \"sqlite-vec.h\"\n#include \"sqlite3.h\"\n#include \n\nint LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {\n int rc = SQLITE_OK;\n sqlite3 *db;\n sqlite3_stmt *stmt;\n\n rc = sqlite3_open(\":memory:\", &db);\n assert(rc == SQLITE_OK);\n rc = sqlite3_vec_init(db, NULL, NULL);\n assert(rc == SQLITE_OK);\n\n sqlite3_str * s = sqlite3_str_new(NULL);\n assert(s);\n sqlite3_str_appendall(s, \"CREATE VIRTUAL TABLE v USING vec0(\");\n sqlite3_str_appendf(s, \"%.*s\", size, data);\n sqlite3_str_appendall(s, \")\");\n const char * zSql = sqlite3_str_finish(s);\n assert(zSql);\n\n rc = sqlite3_prepare_v2(db, zSql, -1, &stmt, NULL);\n sqlite3_free(zSql);\n if(rc == SQLITE_OK) {\n sqlite3_step(stmt);\n }\n sqlite3_finalize(stmt);\n sqlite3_close(db);\n return 0;\n}\n" }, { "path": "tests/fuzz/vec0-create.dict", "content": "cname1=\"aaa\"\ncname1=\"bbb\"\ncname1=\"ccc\"\ntype1=\"float\"\ntype2=\"int8\"\ntype3=\"bit\"\nlparen=\"[\"\nrparen=\"]\"\npk=\"primary key\"\ntext=\"text\"\ndistance_metric=\"distance_metric\"\neq=\"=\"\nl1=\"l1\"\nl2=\"l2\"\ncosine=\"cosine\"\nhamming=\"hamming\"\n" }, { "path": "tests/fuzz/vec0-delete-completeness.c", "content": "#include \n#include \n#include \n#include \n#include \n#include \"sqlite-vec.h\"\n#include \"sqlite3.h\"\n#include \n\nint LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {\n if (size < 6) return 0;\n\n int rc;\n sqlite3 *db;\n sqlite3_stmt *stmtInsert = NULL;\n sqlite3_stmt *stmtDelete = NULL;\n sqlite3_stmt *stmtScan = NULL;\n sqlite3_stmt *stmtCount = NULL;\n\n rc = sqlite3_open(\":memory:\", &db);\n assert(rc == SQLITE_OK);\n rc = sqlite3_vec_init(db, NULL, NULL);\n assert(rc == SQLITE_OK);\n\n rc = sqlite3_exec(db,\n \"CREATE VIRTUAL TABLE v USING vec0(emb float[4], chunk_size=4)\",\n NULL, NULL, NULL);\n if (rc != SQLITE_OK) { sqlite3_close(db); return 0; }\n\n sqlite3_prepare_v2(db,\n \"INSERT INTO v(rowid, emb) VALUES (?, ?)\", -1, &stmtInsert, NULL);\n sqlite3_prepare_v2(db,\n \"DELETE FROM v WHERE rowid = ?\", -1, &stmtDelete, NULL);\n sqlite3_prepare_v2(db,\n \"SELECT rowid FROM v\", -1, &stmtScan, NULL);\n\n if (!stmtInsert || !stmtDelete || !stmtScan) goto cleanup;\n\n size_t i = 0;\n while (i + 2 <= size) {\n uint8_t op = data[i++] % 3;\n uint8_t rowid_byte = data[i++];\n int64_t rowid = (int64_t)(rowid_byte % 16) + 1;\n\n switch (op) {\n case 0: {\n // INSERT\n float vec[4] = {0.0f, 0.0f, 0.0f, 0.0f};\n for (int j = 0; j < 4 && i < size; j++, i++) {\n vec[j] = (float)((int8_t)data[i]) / 10.0f;\n }\n sqlite3_reset(stmtInsert);\n sqlite3_bind_int64(stmtInsert, 1, rowid);\n sqlite3_bind_blob(stmtInsert, 2, vec, sizeof(vec), SQLITE_TRANSIENT);\n sqlite3_step(stmtInsert);\n break;\n }\n case 1: {\n // DELETE\n sqlite3_reset(stmtDelete);\n sqlite3_bind_int64(stmtDelete, 1, rowid);\n sqlite3_step(stmtDelete);\n break;\n }\n case 2: {\n // Full scan\n sqlite3_reset(stmtScan);\n while (sqlite3_step(stmtScan) == SQLITE_ROW) {}\n break;\n }\n }\n }\n\n // Delete all remaining rows\n sqlite3_exec(db, \"DELETE FROM v\", NULL, NULL, NULL);\n\n // Assert all shadow tables are empty after full deletion\n sqlite3_prepare_v2(db,\n \"SELECT count(*) FROM v_rowids\", -1, &stmtCount, NULL);\n if (stmtCount) {\n rc = sqlite3_step(stmtCount);\n assert(rc == SQLITE_ROW);\n assert(sqlite3_column_int(stmtCount, 0) == 0);\n sqlite3_finalize(stmtCount);\n stmtCount = NULL;\n }\n\n sqlite3_prepare_v2(db,\n \"SELECT count(*) FROM v_chunks\", -1, &stmtCount, NULL);\n if (stmtCount) {\n rc = sqlite3_step(stmtCount);\n assert(rc == SQLITE_ROW);\n assert(sqlite3_column_int(stmtCount, 0) == 0);\n sqlite3_finalize(stmtCount);\n stmtCount = NULL;\n }\n\n sqlite3_prepare_v2(db,\n \"SELECT count(*) FROM v_vector_chunks00\", -1, &stmtCount, NULL);\n if (stmtCount) {\n rc = sqlite3_step(stmtCount);\n assert(rc == SQLITE_ROW);\n assert(sqlite3_column_int(stmtCount, 0) == 0);\n sqlite3_finalize(stmtCount);\n stmtCount = NULL;\n }\n\ncleanup:\n sqlite3_finalize(stmtInsert);\n sqlite3_finalize(stmtDelete);\n sqlite3_finalize(stmtScan);\n sqlite3_close(db);\n return 0;\n}\n" }, { "path": "tests/fuzz/vec0-operations.c", "content": "#include \n#include \n#include \n#include \n#include \n#include \"sqlite-vec.h\"\n#include \"sqlite3.h\"\n#include \n\nint LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {\n if (size < 6) return 0;\n\n int rc;\n sqlite3 *db;\n sqlite3_stmt *stmtInsert = NULL;\n sqlite3_stmt *stmtDelete = NULL;\n sqlite3_stmt *stmtKnn = NULL;\n sqlite3_stmt *stmtScan = NULL;\n\n rc = sqlite3_open(\":memory:\", &db);\n assert(rc == SQLITE_OK);\n rc = sqlite3_vec_init(db, NULL, NULL);\n assert(rc == SQLITE_OK);\n\n rc = sqlite3_exec(db,\n \"CREATE VIRTUAL TABLE v USING vec0(emb float[4])\",\n NULL, NULL, NULL);\n if (rc != SQLITE_OK) { sqlite3_close(db); return 0; }\n\n sqlite3_prepare_v2(db,\n \"INSERT INTO v(rowid, emb) VALUES (?, ?)\", -1, &stmtInsert, NULL);\n sqlite3_prepare_v2(db,\n \"DELETE FROM v WHERE rowid = ?\", -1, &stmtDelete, NULL);\n sqlite3_prepare_v2(db,\n \"SELECT rowid, distance FROM v WHERE emb MATCH ? LIMIT 3\",\n -1, &stmtKnn, NULL);\n sqlite3_prepare_v2(db,\n \"SELECT rowid FROM v\", -1, &stmtScan, NULL);\n\n if (!stmtInsert || !stmtDelete || !stmtKnn || !stmtScan) goto cleanup;\n\n size_t i = 0;\n while (i + 2 <= size) {\n uint8_t op = data[i++] % 4;\n uint8_t rowid_byte = data[i++];\n int64_t rowid = (int64_t)(rowid_byte % 32) + 1;\n\n switch (op) {\n case 0: {\n // INSERT: consume 16 bytes for 4 floats, or use what's left\n float vec[4] = {0.0f, 0.0f, 0.0f, 0.0f};\n for (int j = 0; j < 4 && i < size; j++, i++) {\n vec[j] = (float)((int8_t)data[i]) / 10.0f;\n }\n sqlite3_reset(stmtInsert);\n sqlite3_bind_int64(stmtInsert, 1, rowid);\n sqlite3_bind_blob(stmtInsert, 2, vec, sizeof(vec), SQLITE_TRANSIENT);\n sqlite3_step(stmtInsert);\n break;\n }\n case 1: {\n // DELETE\n sqlite3_reset(stmtDelete);\n sqlite3_bind_int64(stmtDelete, 1, rowid);\n sqlite3_step(stmtDelete);\n break;\n }\n case 2: {\n // KNN query with a fixed query vector\n float qvec[4] = {1.0f, 0.0f, 0.0f, 0.0f};\n sqlite3_reset(stmtKnn);\n sqlite3_bind_blob(stmtKnn, 1, qvec, sizeof(qvec), SQLITE_STATIC);\n while (sqlite3_step(stmtKnn) == SQLITE_ROW) {}\n break;\n }\n case 3: {\n // Full scan\n sqlite3_reset(stmtScan);\n while (sqlite3_step(stmtScan) == SQLITE_ROW) {}\n break;\n }\n }\n }\n\n // Final operations — must not crash regardless of prior state\n sqlite3_exec(db, \"SELECT * FROM v\", NULL, NULL, NULL);\n\ncleanup:\n sqlite3_finalize(stmtInsert);\n sqlite3_finalize(stmtDelete);\n sqlite3_finalize(stmtKnn);\n sqlite3_finalize(stmtScan);\n sqlite3_close(db);\n return 0;\n}\n" }, { "path": "tests/fuzz.py", "content": "import sqlite3\nEXT_PATH = \"dist/vec0\"\ndb = sqlite3.connect(\":memory:\")\n\ndb.execute(\n \"create temp table base_functions as select name from pragma_function_list\"\n)\ndb.execute(\"create temp table base_modules as select name from pragma_module_list\")\n\ndb.enable_load_extension(True)\ndb.load_extension(EXT_PATH)\ndb.row_factory = sqlite3.Row\nloaded_functions = db.execute(\n \"select name, narg from pragma_function_list where name not in (select name from base_functions) order by name\"\n).fetchall()\n\ndb.execute(\n \"create temp table loaded_modules as select name from pragma_module_list where name not in (select name from base_modules) order by name\"\n)\n\ndb.row_factory = sqlite3.Row\n\ndef trace(sql):\n print(sql)\ndb.set_trace_callback(trace)\n\ndef spread_args(n):\n return \",\".join([\"?\"] * n)\n\nfor f in loaded_functions:\n v = [None, 1, 1.2, b\"\", '', \"asdf\", b\"\\xff\", b\"\\x00\", \"\\0\\0\\0\\0\"]\n for x in v:\n try:\n db.execute(\"select {}({}); \".format(f['name'],spread_args(f['narg'])), [x] * f['narg'])\n except sqlite3.OperationalError:\n pass\n" }, { "path": "tests/helpers.py", "content": "import sqlite3\nimport struct\nfrom collections import OrderedDict\n\n\ndef _f32(list):\n return struct.pack(\"%sf\" % len(list), *list)\n\n\ndef _i64(list):\n return struct.pack(\"%sq\" % len(list), *list)\n\n\ndef _int8(list):\n return struct.pack(\"%sb\" % len(list), *list)\n\n\ndef exec(db, sql, parameters=[]):\n try:\n rows = db.execute(sql, parameters).fetchall()\n except (sqlite3.OperationalError, sqlite3.DatabaseError) as e:\n return {\n \"error\": e.__class__.__name__,\n \"message\": str(e),\n }\n a = []\n for row in rows:\n o = OrderedDict()\n for k in row.keys():\n o[k] = row[k]\n a.append(o)\n result = OrderedDict()\n result[\"sql\"] = sql\n result[\"rows\"] = a\n return result\n\n\ndef vec0_shadow_table_contents(db, v, skip_info=True):\n shadow_tables = [\n row[0]\n for row in db.execute(\n \"select name from sqlite_master where name like ? order by 1\", [f\"{v}_%\"]\n ).fetchall()\n ]\n o = {}\n for shadow_table in shadow_tables:\n if skip_info and shadow_table.endswith(\"_info\"):\n continue\n o[shadow_table] = exec(db, f\"select * from {shadow_table}\")\n return o\n" }, { "path": "tests/leak-fixtures/each.sql", "content": ".load dist/vec0\n.mode box\n.header on\n.eqp on\n.echo on\n\nselect sqlite_version(), vec_version();\n\nselect * from vec_each('[1,2,3]');\n\nselect *\nfrom json_each('[\n [1,2,3,4],\n [1,2,3,4]\n]')\njoin vec_each(json_each.value);\n" }, { "path": "tests/leak-fixtures/knn.sql", "content": ".load dist/vec0\n.mode box\n.header on\n.eqp on\n.echo on\n\nselect sqlite_version(), vec_version();\n\ncreate virtual table v using vec0(a float[1], chunk_size=8);\n\ninsert into v\n select value, format('[%f]', value / 100.0)\n from generate_series(1, 100);\n\nselect\n rowid,\n vec_to_json(a)\nfrom v\nwhere a match '[.3]'\n and k = 2;\n\nselect\n rowid,\n vec_to_json(a)\nfrom v\nwhere a match '[.3]'\n and k = 0;\n\n\nselect\n rowid,\n vec_to_json(a)\nfrom v\nwhere a match '[2.0]'\n and k = 2\n and rowid in (1,2,3,4,5);\n\n\n\nwith queries as (\n select\n rowid as query_id,\n json_array(value / 100.0) as value\n from generate_series(24, 39)\n)\nselect\n query_id,\n rowid,\n distance,\n vec_to_json(a)\nfrom queries, v\nwhere a match queries.value\n and k =5;\n\n\nselect *\nfrom v\nwhere rowid in (1,2,3,4);\n\ndrop table v;\n\n" }, { "path": "tests/leak-fixtures/vec0-create.sql", "content": ".load dist/vec0\n.mode box\n.header on\n.eqp on\n.echo on\n\ncreate virtual table v using vec0(y);\n" }, { "path": "tests/minimum/.gitignore", "content": "dist/\n" }, { "path": "tests/minimum/Makefile", "content": "dist/.stammp:\n\tmkdir -p dist\n\ttouch $@\n\ndist/sqlite-amalgamation-3310100/.stamp: dist/.stammp\n\trm -rf dist/sqlite-amalgamation-3310100/ || true\n\tcurl -q -o sqlite-amalgamation-3310100.zip https://www.sqlite.org/2020/sqlite-amalgamation-3310100.zip\n\tunzip -d dist/ sqlite-amalgamation-3310100.zip\n\trm sqlite-amalgamation-3310100.zip\n\ttouch $@\n\ndist/t3310100: demo.c dist/sqlite-amalgamation-3310100/.stamp ../../sqlite-vec.c\n\tgcc \\\n\t\t-lm \\\n\t\t-DSQLITE_CORE -DSQLITE_ENABLE_JSON1 \\\n\t\t-I dist/sqlite-amalgamation-3310100 \\\n\t\t-I ../../ \\\n\t\t$< dist/sqlite-amalgamation-3310100/sqlite3.c \\\n\t\t../../sqlite-vec.c \\\n\t\t-o $@\n\ndist/t3310100-threadsafe: demo.c dist/sqlite-amalgamation-3310100/.stamp ../../sqlite-vec.c\n\tgcc \\\n\t\t-lm \\\n\t\t-DSQLITE_CORE -DSQLITE_ENABLE_JSON1 -DSQLITE_THREADSAFE=0 \\\n\t\t-I dist/sqlite-amalgamation-3310100 \\\n\t\t-I ../../ \\\n\t\t$< dist/sqlite-amalgamation-3310100/sqlite3.c \\\n\t\t../../sqlite-vec.c \\\n\t\t-o $@\n\ntest: dist/t3310100 dist/t3310100-threadsafe\n\t./dist/t3310100\n\t./dist/t3310100-threadsafe\n\nclean:\n\trm -rf dist/\n" }, { "path": "tests/minimum/demo.c", "content": "#include \"sqlite3.h\"\n#include \"sqlite-vec.h\"\n#include \n#include \n#include \n\nint main(int argc, char *argv[]) {\n int rc = SQLITE_OK;\n sqlite3 *db;\n sqlite3_stmt *stmt;\n\n rc = sqlite3_open(\":memory:\", &db);\n assert(rc == SQLITE_OK);\n\n rc = sqlite3_vec_init(db, NULL, NULL);\n assert(rc == SQLITE_OK);\n\n\n rc = sqlite3_prepare_v2(db, \"SELECT sqlite_version(), vec_version(), (select json_group_array(compile_options) from pragma_compile_options)\", -1, &stmt, NULL);\n assert(rc == SQLITE_OK);\n\n rc = sqlite3_step(stmt);\n printf(\"sqlite_version=%s, vec_version=%s %s\\n\", sqlite3_column_text(stmt, 0), sqlite3_column_text(stmt, 1), sqlite3_column_text(stmt, 2));\n sqlite3_finalize(stmt);\n sqlite3_close(db);\n return 0;\n}\n" }, { "path": "tests/pyproject.toml", "content": "[project]\nname = \"tests\"\nversion = \"0.1.0\"\ndescription = \"Add your description here\"\nreadme = \"README.md\"\nrequires-python = \">=3.12\"\ndependencies = [\n \"pytest\", \"numpy\", \"syrupy\"\n]\n" }, { "path": "tests/skip.test-correctness.py", "content": "import sqlite3\nimport json\n\ndb = sqlite3.connect(\"test2.db\")\ndb.enable_load_extension(True)\ndb.load_extension(\"dist/vec0\")\ndb.enable_load_extension(False)\ndb.row_factory = sqlite3.Row\ndb.execute('attach database \"sift1m-base.db\" as sift1m')\n\n\n#def test_sift1m():\nrows = db.execute(\n '''\n with q as (\n select rowid, vector, k100 from sift1m.sift1m_query limit 10\n ),\n results as (\n select\n q.rowid as query_rowid,\n vec_sift1m.rowid as vec_rowid,\n distance,\n k100 as k100_groundtruth\n from q\n join vec_sift1m\n where\n vec_sift1m.vector match q.vector\n and k = 100\n order by distance\n )\n select\n query_rowid,\n json_group_array(vec_rowid order by distance) as topk,\n k100_groundtruth,\n json_group_array(vec_rowid order by distance) == k100_groundtruth\n from results\n group by 1;\n ''').fetchall()\n\nresults = []\nfor row in rows:\n actual = json.loads(row[\"topk\"])\n expected = json.loads(row[\"k100_groundtruth\"])\n\n ncorrect = sum([x in expected for x in actual])\n results.append(ncorrect / 100.0)\n\nfrom statistics import mean\nprint(mean(results))\n" }, { "path": "tests/sqlite-vec-internal.h", "content": "#ifndef SQLITE_VEC_INTERNAL_H\n#define SQLITE_VEC_INTERNAL_H\n\n#include \n#include \n\nint min_idx(\n const float *distances,\n int32_t n,\n uint8_t *candidates,\n int32_t *out,\n int32_t k,\n uint8_t *bTaken,\n int32_t *k_used\n);\n\n// Scanner / tokenizer types and functions\n\nenum Vec0TokenType {\n TOKEN_TYPE_IDENTIFIER = 0,\n TOKEN_TYPE_DIGIT = 1,\n TOKEN_TYPE_LBRACKET = 2,\n TOKEN_TYPE_RBRACKET = 3,\n TOKEN_TYPE_PLUS = 4,\n TOKEN_TYPE_EQ = 5,\n TOKEN_TYPE_LPAREN = 6,\n TOKEN_TYPE_RPAREN = 7,\n TOKEN_TYPE_COMMA = 8,\n};\n\n#define VEC0_TOKEN_RESULT_EOF 1\n#define VEC0_TOKEN_RESULT_SOME 2\n#define VEC0_TOKEN_RESULT_ERROR 3\n\nstruct Vec0Token {\n enum Vec0TokenType token_type;\n char *start;\n char *end;\n};\n\nstruct Vec0Scanner {\n char *start;\n char *end;\n char *ptr;\n};\n\nvoid vec0_scanner_init(struct Vec0Scanner *scanner, const char *source, int source_length);\nint vec0_scanner_next(struct Vec0Scanner *scanner, struct Vec0Token *out);\nint vec0_token_next(char *start, char *end, struct Vec0Token *out);\n\n// Vector column definition types and parser\n\nenum VectorElementType {\n SQLITE_VEC_ELEMENT_TYPE_FLOAT32 = 223 + 0,\n SQLITE_VEC_ELEMENT_TYPE_BIT = 223 + 1,\n SQLITE_VEC_ELEMENT_TYPE_INT8 = 223 + 2,\n};\n\nenum Vec0DistanceMetrics {\n VEC0_DISTANCE_METRIC_L2 = 1,\n VEC0_DISTANCE_METRIC_COSINE = 2,\n VEC0_DISTANCE_METRIC_L1 = 3,\n};\n\nstruct VectorColumnDefinition {\n char *name;\n int name_length;\n size_t dimensions;\n enum VectorElementType element_type;\n enum Vec0DistanceMetrics distance_metric;\n};\n\nint vec0_parse_vector_column(const char *source, int source_length,\n struct VectorColumnDefinition *outColumn);\n\nint vec0_parse_partition_key_definition(const char *source, int source_length,\n char **out_column_name,\n int *out_column_name_length,\n int *out_column_type);\n\n#ifdef SQLITE_VEC_TEST\nfloat _test_distance_l2_sqr_float(const float *a, const float *b, size_t dims);\nfloat _test_distance_cosine_float(const float *a, const float *b, size_t dims);\nfloat _test_distance_hamming(const unsigned char *a, const unsigned char *b, size_t dims);\n#endif\n\n#endif /* SQLITE_VEC_INTERNAL_H */\n" }, { "path": "tests/test-auxiliary.py", "content": "import sqlite3\nfrom helpers import exec, vec0_shadow_table_contents\n\n\ndef test_constructor_limit(db, snapshot):\n assert exec(\n db,\n f\"\"\"\n create virtual table v using vec0(\n {\",\".join([f\"+aux{x} integer\" for x in range(17)])}\n v float[1]\n )\n \"\"\",\n ) == snapshot(name=\"max 16 auxiliary columns\")\n\n\ndef test_normal(db, snapshot):\n db.execute(\n \"create virtual table v using vec0(a float[1], +name text, chunk_size=8)\"\n )\n assert exec(db, \"select * from sqlite_master order by name\") == snapshot(\n name=\"sqlite_master\"\n )\n\n db.execute(\"insert into v(a, name) values (?, ?)\", [b\"\\x11\\x11\\x11\\x11\", \"alex\"])\n db.execute(\"insert into v(a, name) values (?, ?)\", [b\"\\x22\\x22\\x22\\x22\", \"brian\"])\n db.execute(\"insert into v(a, name) values (?, ?)\", [b\"\\x33\\x33\\x33\\x33\", \"craig\"])\n\n assert exec(db, \"select * from v\") == snapshot()\n assert vec0_shadow_table_contents(db, \"v\") == snapshot()\n\n db.execute(\"drop table v;\")\n assert exec(db, \"select * from sqlite_master order by name\") == snapshot(\n name=\"sqlite_master post drop\"\n )\n\n\ndef test_types(db, snapshot):\n db.execute(\n \"\"\"\n create virtual table v using vec0(\n vector float[1],\n +aux_int integer,\n +aux_float float,\n +aux_text text,\n +aux_blob blob\n )\n \"\"\"\n )\n assert exec(db, \"select * from v\") == snapshot()\n INSERT = \"insert into v(vector, aux_int, aux_float, aux_text, aux_blob) values (?, ?, ?, ?, ?)\"\n\n assert (\n exec(db, INSERT, [b\"\\x11\\x11\\x11\\x11\", 1, 1.22, \"text\", b\"blob\"]) == snapshot()\n )\n assert exec(db, \"select * from v\") == snapshot()\n\n # TODO: integrity test transaction failures in shadow tables\n db.commit()\n # bad types\n db.execute(\"BEGIN\")\n assert (\n exec(db, INSERT, [b\"\\x11\\x11\\x11\\x11\", \"not int\", 1.2, \"text\", b\"blob\"])\n == snapshot()\n )\n assert (\n exec(db, INSERT, [b\"\\x11\\x11\\x11\\x11\", 1, \"not float\", \"text\", b\"blob\"])\n == snapshot()\n )\n assert exec(db, INSERT, [b\"\\x11\\x11\\x11\\x11\", 1, 1.2, 1, b\"blob\"]) == snapshot()\n assert exec(db, INSERT, [b\"\\x11\\x11\\x11\\x11\", 1, 1.2, \"text\", 1]) == snapshot()\n db.execute(\"ROLLBACK\")\n\n # NULLs are totally chill\n assert exec(db, INSERT, [b\"\\x11\\x11\\x11\\x11\", None, None, None, None]) == snapshot()\n assert exec(db, \"select * from v\") == snapshot()\n\n\ndef test_updates(db, snapshot):\n db.execute(\n \"create virtual table v using vec0(vector float[1], +name text, chunk_size=8)\"\n )\n db.executemany(\n \"insert into v(vector, name) values (?, ?)\",\n [(\"[1]\", \"alex\"), (\"[2]\", \"brian\"), (\"[3]\", \"craig\")],\n )\n assert exec(db, \"select rowid, * from v\") == snapshot()\n assert vec0_shadow_table_contents(db, \"v\") == snapshot()\n\n assert exec(db, \"update v set name = 'ALEX' where rowid = 1\") == snapshot()\n assert exec(db, \"select rowid, * from v\") == snapshot()\n assert vec0_shadow_table_contents(db, \"v\") == snapshot()\n\n\ndef test_deletes(db, snapshot):\n db.execute(\n \"create virtual table v using vec0(vector float[1], +name text, chunk_size=8)\"\n )\n db.executemany(\n \"insert into v(vector, name) values (?, ?)\",\n [(\"[1]\", \"alex\"), (\"[2]\", \"brian\"), (\"[3]\", \"craig\")],\n )\n assert exec(db, \"select rowid, * from v\") == snapshot()\n assert vec0_shadow_table_contents(db, \"v\") == snapshot()\n\n assert exec(db, \"delete from v where rowid = 1\") == snapshot()\n assert exec(db, \"select rowid, * from v\") == snapshot()\n assert vec0_shadow_table_contents(db, \"v\") == snapshot()\n\n\ndef test_knn(db, snapshot):\n db.execute(\"create virtual table v using vec0(vector float[1], +name text)\")\n db.executemany(\n \"insert into v(vector, name) values (?, ?)\",\n [(\"[1]\", \"alex\"), (\"[2]\", \"brian\"), (\"[3]\", \"craig\")],\n )\n assert exec(db, \"select * from v\") == snapshot()\n assert exec(\n db, \"select *, distance from v where vector match '[5]' and k = 10\"\n ) == snapshot(name=\"legal KNN w/ aux\")\n\n # EVIDENCE-OF: V25623_09693 No aux constraint allowed on KNN queries\n assert exec(\n db,\n \"select *, distance from v where vector match '[5]' and k = 10 and name = 'alex'\",\n ) == snapshot(name=\"illegal KNN w/ aux\")\n\n\n" }, { "path": "tests/test-general.py", "content": "import sqlite3\nimport pytest\nfrom helpers import exec\n\n\n@pytest.mark.skipif(\n sqlite3.sqlite_version_info[1] < 37,\n reason=\"pragma_table_list was added in SQLite 3.37\",\n)\ndef test_shadow(db, snapshot):\n db.execute(\n \"create virtual table v using vec0(a float[1], partition text partition key, metadata text, +name text, chunk_size=8)\"\n )\n assert exec(db, \"select * from sqlite_master order by name\") == snapshot()\n assert (\n exec(db, \"select * from pragma_table_list where type = 'shadow' order by name\") == snapshot()\n )\n\n db.execute(\"drop table v;\")\n assert (\n exec(db, \"select * from pragma_table_list where type = 'shadow' order by name\") == snapshot()\n )\n\n\ndef test_info(db, snapshot):\n db.execute(\"create virtual table v using vec0(a float[1])\")\n assert exec(db, \"select key, typeof(value) from v_info order by 1\") == snapshot()\n\n\n" }, { "path": "tests/test-insert-delete.py", "content": "import sqlite3\nimport struct\nimport pytest\nfrom helpers import _f32, _i64, _int8, exec\n\n\ndef test_insert_creates_chunks_and_vectors(db, snapshot):\n db.execute(\"create virtual table v using vec0(emb float[4], chunk_size=8)\")\n\n vecs = [\n [1.0, 2.0, 3.0, 4.0],\n [5.0, 6.0, 7.0, 8.0],\n [0.1, 0.2, 0.3, 0.4],\n [10.0, 20.0, 30.0, 40.0],\n [0.5, 0.5, 0.5, 0.5],\n ]\n for i, v in enumerate(vecs, start=1):\n db.execute(\"insert into v(rowid, emb) values (?, ?)\", [i, _f32(v)])\n\n assert exec(db, \"select count(*) as cnt from v_rowids\") == snapshot(\n name=\"rowids_count\"\n )\n assert exec(db, \"select count(*) as cnt from v_vector_chunks00\") == snapshot(\n name=\"vector_chunks_count\"\n )\n\n # Verify round-trip: each inserted vector comes back identical\n for i, v in enumerate(vecs, start=1):\n rows = db.execute(\"select emb from v where rowid = ?\", [i]).fetchall()\n assert len(rows) == 1\n assert rows[0][0] == _f32(v)\n\n\ndef test_insert_auto_rowid(db):\n db.execute(\"create virtual table v using vec0(emb float[4], chunk_size=8)\")\n\n vecs = [[1.0, 0.0, 0.0, 0.0], [0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0]]\n for v in vecs:\n db.execute(\"insert into v(emb) values (?)\", [_f32(v)])\n\n rows = db.execute(\"select rowid from v order by rowid\").fetchall()\n rowids = [r[0] for r in rows]\n assert rowids == [1, 2, 3]\n\n for i, v in enumerate(vecs, start=1):\n row = db.execute(\"select emb from v where rowid = ?\", [i]).fetchone()\n assert row[0] == _f32(v)\n\n\ndef test_insert_text_primary_key(db, snapshot):\n db.execute(\n \"create virtual table v using vec0(id text primary key, emb float[4], chunk_size=8)\"\n )\n\n db.execute(\n \"insert into v(id, emb) values ('doc_a', ?)\", [_f32([1.0, 2.0, 3.0, 4.0])]\n )\n db.execute(\n \"insert into v(id, emb) values ('doc_b', ?)\", [_f32([5.0, 6.0, 7.0, 8.0])]\n )\n\n assert exec(db, \"select rowid, id, chunk_id, chunk_offset from v_rowids order by rowid\") == snapshot(\n name=\"rowids\"\n )\n\n row = db.execute(\"select emb from v where id = 'doc_a'\").fetchone()\n assert row[0] == _f32([1.0, 2.0, 3.0, 4.0])\n\n row = db.execute(\"select emb from v where id = 'doc_b'\").fetchone()\n assert row[0] == _f32([5.0, 6.0, 7.0, 8.0])\n\n\ndef test_delete_clears_validity(db):\n db.execute(\"create virtual table v using vec0(emb float[4], chunk_size=8)\")\n\n for i, v in enumerate(\n [[1.0, 0.0, 0.0, 0.0], [0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0]],\n start=1,\n ):\n db.execute(\"insert into v(rowid, emb) values (?, ?)\", [i, _f32(v)])\n\n db.execute(\"delete from v where rowid = 2\")\n\n rows = db.execute(\"select rowid from v order by rowid\").fetchall()\n assert [r[0] for r in rows] == [1, 3]\n\n rowid_rows = db.execute(\"select rowid from v_rowids order by rowid\").fetchall()\n assert 2 not in [r[0] for r in rowid_rows]\n\n # Inserting a new vector after deletion still works\n db.execute(\"insert into v(rowid, emb) values (4, ?)\", [_f32([0.0, 0.0, 0.0, 1.0])])\n row = db.execute(\"select emb from v where rowid = 4\").fetchone()\n assert row[0] == _f32([0.0, 0.0, 0.0, 1.0])\n\n\ndef test_insert_delete_reinsert(db):\n db.execute(\"create virtual table v using vec0(emb float[4], chunk_size=8)\")\n\n db.execute(\"insert into v(rowid, emb) values (1, ?)\", [_f32([1.0, 1.0, 1.0, 1.0])])\n db.execute(\"delete from v where rowid = 1\")\n db.execute(\"insert into v(rowid, emb) values (2, ?)\", [_f32([2.0, 2.0, 2.0, 2.0])])\n\n rows = db.execute(\"select rowid from v order by rowid\").fetchall()\n assert [r[0] for r in rows] == [2]\n\n # KNN query works and returns rowid 2\n knn = db.execute(\n \"select rowid, distance from v where emb match ? and k = 1\",\n [_f32([2.0, 2.0, 2.0, 2.0])],\n ).fetchall()\n assert len(knn) == 1\n assert knn[0][0] == 2\n\n\ndef test_insert_validates_dimensions(db):\n db.execute(\"create virtual table v using vec0(emb float[4], chunk_size=8)\")\n\n result = exec(db, \"insert into v(rowid, emb) values (1, ?)\", [_f32([1.0, 2.0, 3.0])])\n assert result[\"error\"] == \"OperationalError\"\n assert \"Dimension mismatch\" in result[\"message\"]\n assert \"Expected 4\" in result[\"message\"]\n assert \"3\" in result[\"message\"]\n\n result = exec(\n db, \"insert into v(rowid, emb) values (1, ?)\", [_f32([1.0, 2.0, 3.0, 4.0, 5.0])]\n )\n assert result[\"error\"] == \"OperationalError\"\n assert \"Dimension mismatch\" in result[\"message\"]\n assert \"Expected 4\" in result[\"message\"]\n assert \"5\" in result[\"message\"]\n\n\ndef test_insert_validates_type(db):\n db.execute(\"create virtual table v using vec0(emb float[4], chunk_size=8)\")\n\n int8_vec = struct.pack(\"4b\", 1, 2, 3, 4)\n result = exec(\n db,\n \"insert into v(rowid, emb) values (1, vec_int8(?))\",\n [int8_vec],\n )\n assert \"error\" in result\n assert \"float32\" in result[\"message\"]\n assert \"int8\" in result[\"message\"]\n\n\ndef test_info_table_contents(db, snapshot):\n db.execute(\"create virtual table v using vec0(emb float[4], chunk_size=8)\")\n assert exec(db, \"select key, value from v_info where key not like 'CREATE_VERSION%' order by key\") == snapshot()\n # Smoke-check that version keys exist without pinning exact values\n version_rows = exec(db, \"select key from v_info where key like 'CREATE_VERSION%' order by key\")\n keys = [r[\"key\"] for r in version_rows[\"rows\"]]\n assert keys == [\"CREATE_VERSION\", \"CREATE_VERSION_MAJOR\", \"CREATE_VERSION_MINOR\", \"CREATE_VERSION_PATCH\"]\n\n\ndef test_delete_zeroes_rowid_blob(db):\n db.execute(\"create virtual table v using vec0(emb float[4], chunk_size=8)\")\n\n for i in range(1, 4):\n db.execute(\n \"insert into v(rowid, emb) values (?, ?)\",\n [i, _f32([float(i)] * 4)],\n )\n\n db.execute(\"delete from v where rowid = 2\")\n\n blob = db.execute(\"select rowids from v_chunks where rowid = 1\").fetchone()[0]\n rowids = struct.unpack(\"<8q\", blob)\n assert rowids[0] == 1 # slot 0 intact\n assert rowids[1] == 0 # slot 1 zeroed (was rowid 2)\n assert rowids[2] == 3 # slot 2 intact\n\n\ndef test_delete_zeroes_vector_blob(db):\n db.execute(\"create virtual table v using vec0(emb float[4], chunk_size=8)\")\n\n db.execute(\n \"insert into v(rowid, emb) values (1, ?)\", [_f32([1.0, 2.0, 3.0, 4.0])]\n )\n db.execute(\n \"insert into v(rowid, emb) values (2, ?)\", [_f32([5.0, 6.0, 7.0, 8.0])]\n )\n\n db.execute(\"delete from v where rowid = 1\")\n\n blob = db.execute(\n \"select vectors from v_vector_chunks00 where rowid = 1\"\n ).fetchone()[0]\n # First slot (4 floats = 16 bytes) should be zeroed\n first_slot = struct.unpack(\"<4f\", blob[:16])\n assert first_slot == (0.0, 0.0, 0.0, 0.0)\n # Second slot should be unchanged\n second_slot = struct.unpack(\"<4f\", blob[16:32])\n assert second_slot == (5.0, 6.0, 7.0, 8.0)\n\n\ndef test_delete_all_rows_deletes_chunk(db):\n db.execute(\"create virtual table v using vec0(emb float[4], chunk_size=8)\")\n\n for i in range(1, 9):\n db.execute(\n \"insert into v(rowid, emb) values (?, ?)\",\n [i, _f32([float(i)] * 4)],\n )\n\n for i in range(1, 9):\n db.execute(\"delete from v where rowid = ?\", [i])\n\n assert (\n db.execute(\"select count(*) from v_chunks\").fetchone()[0] == 0\n )\n assert (\n db.execute(\"select count(*) from v_vector_chunks00\").fetchone()[0] == 0\n )\n\n # Inserting after full deletion still works\n db.execute(\n \"insert into v(rowid, emb) values (100, ?)\", [_f32([9.0, 9.0, 9.0, 9.0])]\n )\n row = db.execute(\"select emb from v where rowid = 100\").fetchone()\n assert row[0] == _f32([9.0, 9.0, 9.0, 9.0])\n\n\ndef test_delete_chunk_multiple_chunks(db):\n db.execute(\"create virtual table v using vec0(emb float[4], chunk_size=8)\")\n\n for i in range(1, 17):\n db.execute(\n \"insert into v(rowid, emb) values (?, ?)\",\n [i, _f32([float(i)] * 4)],\n )\n\n # Delete all rows from the first chunk (rows 1-8)\n for i in range(1, 9):\n db.execute(\"delete from v where rowid = ?\", [i])\n\n # Only 1 chunk should remain\n assert db.execute(\"select count(*) from v_chunks\").fetchone()[0] == 1\n\n # Rows 9-16 still queryable\n for i in range(9, 17):\n row = db.execute(\"select emb from v where rowid = ?\", [i]).fetchone()\n assert row[0] == _f32([float(i)] * 4)\n\n\ndef test_delete_with_metadata_columns(db):\n db.execute(\n \"create virtual table v using vec0(\"\n \"emb float[4], \"\n \"m_bool boolean, \"\n \"m_int integer, \"\n \"m_float float, \"\n \"m_text text, \"\n \"chunk_size=8\"\n \")\"\n )\n\n for i in range(1, 9):\n db.execute(\n \"insert into v(rowid, emb, m_bool, m_int, m_float, m_text) \"\n \"values (?, ?, ?, ?, ?, ?)\",\n [i, _f32([float(i)] * 4), i % 2 == 0, i * 10, float(i) / 2.0, f\"text_{i}\"],\n )\n\n for i in range(1, 9):\n db.execute(\"delete from v where rowid = ?\", [i])\n\n assert db.execute(\"select count(*) from v_chunks\").fetchone()[0] == 0\n assert db.execute(\"select count(*) from v_vector_chunks00\").fetchone()[0] == 0\n assert db.execute(\"select count(*) from v_metadatachunks00\").fetchone()[0] == 0\n assert db.execute(\"select count(*) from v_metadatachunks01\").fetchone()[0] == 0\n assert db.execute(\"select count(*) from v_metadatachunks02\").fetchone()[0] == 0\n assert db.execute(\"select count(*) from v_metadatachunks03\").fetchone()[0] == 0\n\n\ndef test_delete_with_auxiliary_columns(db):\n db.execute(\n \"create virtual table v using vec0(\"\n \"emb float[4], \"\n \"+aux_text text, \"\n \"chunk_size=8\"\n \")\"\n )\n\n for i in range(1, 9):\n db.execute(\n \"insert into v(rowid, emb, aux_text) values (?, ?, ?)\",\n [i, _f32([float(i)] * 4), f\"aux_{i}\"],\n )\n\n for i in range(1, 9):\n db.execute(\"delete from v where rowid = ?\", [i])\n\n assert db.execute(\"select count(*) from v_chunks\").fetchone()[0] == 0\n assert db.execute(\"select count(*) from v_auxiliary\").fetchone()[0] == 0\n\n\ndef test_delete_with_text_primary_key(db):\n db.execute(\n \"create virtual table v using vec0(\"\n \"id text primary key, emb float[4], chunk_size=8\"\n \")\"\n )\n\n db.execute(\n \"insert into v(id, emb) values ('a', ?)\", [_f32([1.0, 2.0, 3.0, 4.0])]\n )\n db.execute(\n \"insert into v(id, emb) values ('b', ?)\", [_f32([5.0, 6.0, 7.0, 8.0])]\n )\n\n db.execute(\"delete from v where id = 'a'\")\n\n # Vector blob slot 0 should be zeroed\n blob = db.execute(\n \"select vectors from v_vector_chunks00 where rowid = 1\"\n ).fetchone()[0]\n first_slot = struct.unpack(\"<4f\", blob[:16])\n assert first_slot == (0.0, 0.0, 0.0, 0.0)\n\n # Remaining row still queryable\n row = db.execute(\"select emb from v where id = 'b'\").fetchone()\n assert row[0] == _f32([5.0, 6.0, 7.0, 8.0])\n\n\ndef test_delete_with_partition_keys(db):\n db.execute(\n \"create virtual table v using vec0(\"\n \"part text partition key, emb float[4], chunk_size=8\"\n \")\"\n )\n\n for i in range(1, 9):\n db.execute(\n \"insert into v(rowid, part, emb) values (?, 'A', ?)\",\n [i, _f32([float(i)] * 4)],\n )\n for i in range(9, 17):\n db.execute(\n \"insert into v(rowid, part, emb) values (?, 'B', ?)\",\n [i, _f32([float(i)] * 4)],\n )\n\n # Delete all from partition A\n for i in range(1, 9):\n db.execute(\"delete from v where rowid = ?\", [i])\n\n # 1 chunk should remain (partition B's)\n assert db.execute(\"select count(*) from v_chunks\").fetchone()[0] == 1\n\n # Partition B rows intact\n for i in range(9, 17):\n row = db.execute(\"select emb from v where rowid = ?\", [i]).fetchone()\n assert row[0] == _f32([float(i)] * 4)\n\n # Re-insert into partition A works\n db.execute(\n \"insert into v(rowid, part, emb) values (100, 'A', ?)\",\n [_f32([99.0, 99.0, 99.0, 99.0])],\n )\n row = db.execute(\"select emb from v where rowid = 100\").fetchone()\n assert row[0] == _f32([99.0, 99.0, 99.0, 99.0])\n\n\ndef test_delete_int8_vectors(db):\n db.execute(\"create virtual table v using vec0(emb int8[4], chunk_size=8)\")\n\n db.execute(\n \"insert into v(rowid, emb) values (1, vec_int8(?))\",\n [_int8([1, 2, 3, 4])],\n )\n db.execute(\n \"insert into v(rowid, emb) values (2, vec_int8(?))\",\n [_int8([5, 6, 7, 8])],\n )\n\n db.execute(\"delete from v where rowid = 1\")\n\n blob = db.execute(\n \"select vectors from v_vector_chunks00 where rowid = 1\"\n ).fetchone()[0]\n # int8[4] = 4 bytes per slot\n first_slot = struct.unpack(\"<4b\", blob[:4])\n assert first_slot == (0, 0, 0, 0)\n second_slot = struct.unpack(\"<4b\", blob[4:8])\n assert second_slot == (5, 6, 7, 8)\n\n\ndef test_delete_bit_vectors(db):\n db.execute(\"create virtual table v using vec0(emb bit[8], chunk_size=8)\")\n\n db.execute(\n \"insert into v(rowid, emb) values (1, vec_bit(?))\",\n [bytes([0xFF])],\n )\n db.execute(\n \"insert into v(rowid, emb) values (2, vec_bit(?))\",\n [bytes([0xAA])],\n )\n\n db.execute(\"delete from v where rowid = 1\")\n\n blob = db.execute(\n \"select vectors from v_vector_chunks00 where rowid = 1\"\n ).fetchone()[0]\n # bit[8] = 1 byte per slot\n assert blob[0:1] == bytes([0x00])\n assert blob[1:2] == bytes([0xAA])\n\n\ndef _file_db(tmp_path):\n \"\"\"Open a file-backed DB (required for page_count to shrink after VACUUM).\"\"\"\n db = sqlite3.connect(str(tmp_path / \"test.db\"))\n db.row_factory = sqlite3.Row\n db.enable_load_extension(True)\n db.load_extension(\"dist/vec0\")\n db.enable_load_extension(False)\n return db\n\n\ndef test_delete_chunk_shrinks_pages(tmp_path):\n \"\"\"Use large vectors (float[256]) so each chunk blob spans multiple pages,\n making the page_count difference measurable after VACUUM.\"\"\"\n dims = 256\n db = _file_db(tmp_path)\n db.execute(f\"create virtual table v using vec0(emb float[{dims}], chunk_size=8)\")\n\n for i in range(1, 25): # 3 full chunks of 8\n db.execute(\n \"insert into v(rowid, emb) values (?, ?)\",\n [i, _f32([float(i)] * dims)],\n )\n db.commit()\n pages_before = db.execute(\"pragma page_count\").fetchone()[0]\n\n # Delete all rows\n for i in range(1, 25):\n db.execute(\"delete from v where rowid = ?\", [i])\n db.commit()\n\n assert db.execute(\"select count(*) from v_chunks\").fetchone()[0] == 0\n\n db.execute(\"vacuum\")\n pages_after = db.execute(\"pragma page_count\").fetchone()[0]\n assert pages_after < pages_before, (\n f\"page_count should shrink after deleting all chunks and vacuum: \"\n f\"{pages_before} -> {pages_after}\"\n )\n db.close()\n\n\ndef test_delete_one_chunk_of_two_shrinks_pages(tmp_path):\n \"\"\"Use large vectors (float[256]) so each chunk blob spans multiple pages,\n making the page_count difference measurable after VACUUM.\"\"\"\n dims = 256\n db = _file_db(tmp_path)\n db.execute(f\"create virtual table v using vec0(emb float[{dims}], chunk_size=8)\")\n\n for i in range(1, 17): # 2 full chunks of 8\n db.execute(\n \"insert into v(rowid, emb) values (?, ?)\",\n [i, _f32([float(i)] * dims)],\n )\n db.commit()\n pages_before = db.execute(\"pragma page_count\").fetchone()[0]\n\n # Delete all rows from the first chunk (rows 1-8)\n for i in range(1, 9):\n db.execute(\"delete from v where rowid = ?\", [i])\n db.commit()\n\n assert db.execute(\"select count(*) from v_chunks\").fetchone()[0] == 1\n\n db.execute(\"vacuum\")\n pages_after = db.execute(\"pragma page_count\").fetchone()[0]\n assert pages_after < pages_before, (\n f\"page_count should shrink after deleting one chunk and vacuum: \"\n f\"{pages_before} -> {pages_after}\"\n )\n\n # Remaining rows still queryable after vacuum\n for i in range(9, 17):\n row = db.execute(\"select emb from v where rowid = ?\", [i]).fetchone()\n assert row[0] == _f32([float(i)] * dims)\n db.close()\n" }, { "path": "tests/test-knn-distance-constraints.py", "content": "import sqlite3\nfrom helpers import exec\n\n\ndef test_normal(db, snapshot):\n db.execute(\"create virtual table v using vec0(embedding float[1], is_odd boolean, chunk_size=8)\")\n db.executemany(\n \"insert into v(rowid, is_odd, embedding) values (?1, ?1 % 2, ?2)\",\n [\n [1, \"[1]\"],\n [2, \"[2]\"],\n [3, \"[3]\"],\n [4, \"[4]\"],\n [5, \"[5]\"],\n [6, \"[6]\"],\n [7, \"[7]\"],\n [8, \"[8]\"],\n [9, \"[9]\"],\n [10, \"[10]\"],\n [11, \"[11]\"],\n [12, \"[12]\"],\n [13, \"[13]\"],\n [14, \"[14]\"],\n [15, \"[15]\"],\n [16, \"[16]\"],\n [17, \"[17]\"],\n ],\n )\n assert exec(db,\"SELECT * FROM v\") == snapshot()\n\n BASE_KNN = \"select rowid, distance from v where embedding match ? and k = ? \"\n assert exec(db, BASE_KNN, [\"[1]\", 5]) == snapshot()\n assert exec(db, BASE_KNN + \"AND distance > 5\", [\"[1]\", 5]) == snapshot()\n assert exec(db, BASE_KNN + \"AND distance >= 5\", [\"[1]\", 5]) == snapshot()\n assert exec(db, BASE_KNN + \"AND distance < 3\", [\"[1]\", 5]) == snapshot()\n assert exec(db, BASE_KNN + \"AND distance <= 3\", [\"[1]\", 5]) == snapshot()\n assert exec(db, BASE_KNN + \"AND distance > 7 AND distance <= 10\", [\"[1]\", 5]) == snapshot()\n assert exec(db, BASE_KNN + \"AND distance BETWEEN 7 AND 10\", [\"[1]\", 5]) == snapshot()\n assert exec(db, BASE_KNN + \"AND is_odd == TRUE AND distance BETWEEN 7 AND 10\", [\"[1]\", 5]) == snapshot()\n\n\nclass Row:\n def __init__(self):\n pass\n\n def __repr__(self) -> str:\n return repr()\n\n\n" }, { "path": "tests/test-loadable.py", "content": "# ruff: noqa: E731\n\nimport re\nfrom typing import List\nimport sqlite3\nimport unittest\nfrom random import random\nimport struct\nimport inspect\nimport pytest\nimport json\nimport numpy as np\nfrom math import isclose\n\nEXT_PATH = \"./dist/vec0\"\n\nSUPPORTS_SUBTYPE = sqlite3.sqlite_version_info[1] > 38\nSUPPORTS_DROP_COLUMN = sqlite3.sqlite_version_info[1] >= 35\nSUPPORTS_VTAB_IN = sqlite3.sqlite_version_info[1] >= 38\nSUPPORTS_VTAB_LIMIT = sqlite3.sqlite_version_info[1] >= 41\n\n\ndef bitmap_full(n: int) -> bytearray:\n assert (n % 8) == 0\n return bytes([0xFF] * int(n / 8))\n\n\ndef bitmap_zerod(n: int) -> bytearray:\n assert (n % 8) == 0\n return bytes([0x00] * int(n / 8))\n\n\ndef f32_zerod(n: int) -> bytearray:\n return bytes([0x00, 0x00, 0x00, 0x00] * int(n))\n\n\nCHAR_BIT = 8\n\n\ndef _f32(list):\n return struct.pack(\"%sf\" % len(list), *list)\n\n\ndef _i64(list):\n return struct.pack(\"%sq\" % len(list), *list)\n\n\ndef _int8(list):\n return struct.pack(\"%sb\" % len(list), *list)\n\n\ndef bitmap(bitstring):\n return bytes([int(bitstring, 2)])\n\n\ndef connect(ext, path=\":memory:\", extra_entrypoint=None):\n db = sqlite3.connect(path)\n\n db.execute(\n \"create temp table base_functions as select name from pragma_function_list\"\n )\n db.execute(\"create temp table base_modules as select name from pragma_module_list\")\n\n db.enable_load_extension(True)\n db.load_extension(ext)\n\n if extra_entrypoint:\n db.execute(\"select load_extension(?, ?)\", [ext, extra_entrypoint])\n\n db.execute(\n \"create temp table loaded_functions as select name from pragma_function_list where name not in (select name from base_functions) order by name\"\n )\n db.execute(\n \"create temp table loaded_modules as select name from pragma_module_list where name not in (select name from base_modules) order by name\"\n )\n\n db.row_factory = sqlite3.Row\n return db\n\n\ndb = connect(EXT_PATH)\n\n\ndef explain_query_plan(sql, db=db):\n return db.execute(\"explain query plan \" + sql).fetchone()[\"detail\"]\n\n\ndef execute_all(cursor, sql, args=None):\n if args is None:\n args = []\n results = cursor.execute(sql, args).fetchall()\n return list(map(lambda x: dict(x), results))\n\n\ndef spread_args(args):\n return \",\".join([\"?\"] * len(args))\n\n\nFUNCTIONS = [\n \"vec_add\",\n \"vec_bit\",\n \"vec_debug\",\n \"vec_distance_cosine\",\n \"vec_distance_hamming\",\n \"vec_distance_l1\",\n \"vec_distance_l2\",\n \"vec_f32\",\n \"vec_int8\",\n \"vec_length\",\n \"vec_normalize\",\n \"vec_quantize_binary\",\n \"vec_quantize_int8\",\n \"vec_slice\",\n \"vec_sub\",\n \"vec_to_json\",\n \"vec_type\",\n \"vec_version\",\n]\nMODULES = [\n \"vec0\",\n \"vec_each\",\n # \"vec_static_blob_entries\",\n # \"vec_static_blobs\",\n]\n\n\ndef register_numpy(db, name: str, array):\n ptr = array.__array_interface__[\"data\"][0]\n nvectors, dimensions = array.__array_interface__[\"shape\"]\n element_type = array.__array_interface__[\"typestr\"]\n\n assert element_type == \"\\x9a\\x99\\x99>\",\n },\n {\n \"vector\": b\"fff?\\xcd\\xccL?\",\n },\n ]\n assert execute_all(db, \"select rowid, (vector) from z\") == [\n {\n \"rowid\": 0,\n \"vector\": b\"\\xcd\\xcc\\xcc=\\xcd\\xcc\\xcc=\\xcd\\xcc\\xcc=\\xcd\\xcc\\xcc=\",\n },\n {\n \"rowid\": 1,\n \"vector\": b\"\\xcd\\xccL>\\xcd\\xccL>\\xcd\\xccL>\\xcd\\xccL>\",\n },\n {\n \"rowid\": 2,\n \"vector\": b\"\\x9a\\x99\\x99>\\x9a\\x99\\x99>\\x9a\\x99\\x99>\\x9a\\x99\\x99>\",\n },\n {\n \"rowid\": 3,\n \"vector\": b\"\\xcd\\xcc\\xcc>\\xcd\\xcc\\xcc>\\xcd\\xcc\\xcc>\\xcd\\xcc\\xcc>\",\n },\n {\n \"rowid\": 4,\n \"vector\": b\"\\x00\\x00\\x00?\\x00\\x00\\x00?\\x00\\x00\\x00?\\x00\\x00\\x00?\",\n },\n ]\n assert execute_all(\n db,\n \"select rowid, vec_to_json(vector) as v from z where vector match ? and k = 3 order by distance;\",\n [np.array([0.3, 0.3, 0.3, 0.3], dtype=np.float32)],\n ) == [\n {\n \"rowid\": 2,\n \"v\": \"[0.300000,0.300000,0.300000,0.300000]\",\n },\n {\n \"rowid\": 3,\n \"v\": \"[0.400000,0.400000,0.400000,0.400000]\",\n },\n {\n \"rowid\": 1,\n \"v\": \"[0.200000,0.200000,0.200000,0.200000]\",\n },\n ]\n assert execute_all(\n db,\n \"select rowid, vec_to_json(vector) as v from z where vector match ? and k = 3 order by distance;\",\n [np.array([0.6, 0.6, 0.6, 0.6], dtype=np.float32)],\n ) == [\n {\n \"rowid\": 4,\n \"v\": \"[0.500000,0.500000,0.500000,0.500000]\",\n },\n {\n \"rowid\": 3,\n \"v\": \"[0.400000,0.400000,0.400000,0.400000]\",\n },\n {\n \"rowid\": 2,\n \"v\": \"[0.300000,0.300000,0.300000,0.300000]\",\n },\n ]\n\n\ndef test_limits():\n db = connect(EXT_PATH)\n with _raises(\n \"vec0 constructor error: Dimension on vector column too large, provided 8193, maximum 8192\"\n ):\n db.execute(\"create virtual table v using vec0(a float[8193])\")\n with _raises(\"vec0 constructor error: chunk_size too large\"):\n db.execute(\"create virtual table v using vec0(a float[4], chunk_size=8200)\")\n db.execute(\"create virtual table v using vec0(a float[1])\")\n\n with _raises(\"k value in knn query too large, provided 8193 and the limit is 4096\"):\n db.execute(\"select * from v where a match '[0.1]' and k = 8193\")\n\n\ndef test_funcs():\n funcs = list(\n map(\n lambda a: a[0],\n db.execute(\"select name from loaded_functions\").fetchall(),\n )\n )\n assert funcs == FUNCTIONS\n\n\ndef test_modules():\n modules = list(\n map(lambda a: a[0], db.execute(\"select name from loaded_modules\").fetchall())\n )\n assert modules == MODULES\n\n\ndef test_vec_version():\n vec_version = lambda *args: db.execute(\"select vec_version()\", args).fetchone()[0]\n assert vec_version()[0] == \"v\"\n\n\ndef test_vec_debug():\n vec_debug = lambda *args: db.execute(\"select vec_debug()\", args).fetchone()[0]\n d = vec_debug().split(\"\\n\")\n assert len(d) == 4\n\n\ndef test_vec_bit():\n vec_bit = lambda *args: db.execute(\"select vec_bit(?)\", args).fetchone()[0]\n assert vec_bit(b\"\\xff\") == b\"\\xff\"\n\n if SUPPORTS_SUBTYPE:\n assert db.execute(\"select subtype(vec_bit(X'FF'))\").fetchone()[0] == 224\n\n with pytest.raises(\n sqlite3.OperationalError, match=\"zero-length vectors are not supported.\"\n ):\n db.execute(\"select vec_bit(X'')\").fetchone()\n\n for x in [None, \"text\", 1, 1.999]:\n with pytest.raises(\n sqlite3.OperationalError, match=\"Unknown type for bitvector.\"\n ):\n db.execute(\"select vec_bit(?)\", [x]).fetchone()\n\n\ndef test_vec_f32():\n vec_f32 = lambda *args: db.execute(\"select vec_f32(?)\", args).fetchone()[0]\n assert vec_f32(b\"\\x00\\x00\\x00\\x00\") == b\"\\x00\\x00\\x00\\x00\"\n assert vec_f32(\"[0.0000]\") == b\"\\x00\\x00\\x00\\x00\"\n # fmt: off\n tests = [\n [0],\n [0, 0, 0, 0],\n [1, -1, 10, -10],\n [-0, 0, .0001, -.0001],\n ]\n # fmt: on\n for test in tests:\n assert vec_f32(json.dumps(test)) == _f32(test)\n\n if SUPPORTS_SUBTYPE:\n assert db.execute(\"select subtype(vec_f32(X'00000000'))\").fetchone()[0] == 223\n\n with pytest.raises(\n sqlite3.OperationalError, match=\"zero-length vectors are not supported.\"\n ):\n vec_f32(b\"\")\n\n for invalid in [None, 1, 1.2]:\n with pytest.raises(\n sqlite3.OperationalError,\n match=re.escape(\n \"Input must have type BLOB (compact format) or TEXT (JSON)\",\n ),\n ):\n vec_f32(invalid)\n\n with pytest.raises(\n sqlite3.OperationalError,\n match=\"invalid float32 vector BLOB length. Must be divisible by 4, found 5\",\n ):\n vec_f32(b\"aaaaa\")\n\n with pytest.raises(\n sqlite3.OperationalError,\n match=re.escape(\"JSON array parsing error: Input does not start with '['\"),\n ):\n vec_f32(\"1]\")\n # TODO mas tests\n\n # TODO different error message\n with _raises(\"zero-length vectors are not supported.\"):\n vec_f32(\"[\")\n\n with _raises(\"zero-length vectors are not supported.\"):\n vec_f32(\"[]\")\n # with _raises(\"zero-length vectors are not supported.\"):\n # vec_f32(\"[1.2\")\n\n # vec_f32(\"[]\")\n\n\ndef test_vec_int8():\n vec_int8 = lambda *args: db.execute(\"select vec_int8(?)\", args).fetchone()[0]\n assert vec_int8(b\"\\x00\") == _int8([0])\n assert vec_int8(b\"\\x00\\x0f\") == _int8([0, 15])\n assert vec_int8(\"[0]\") == _int8([0])\n assert vec_int8(\"[1, 2, 3]\") == _int8([1, 2, 3])\n\n if SUPPORTS_SUBTYPE:\n assert db.execute(\"select subtype(vec_int8(?))\", [b\"\\x00\"]).fetchone()[0] == 225\n\n\ndef npy_cosine(a, b):\n return 1 - (np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b)))\n\n\ndef npy_l2(a, b):\n return np.linalg.norm(a - b)\n\n\ndef test_vec_distance_cosine():\n vec_distance_cosine = lambda *args, a=\"?\", b=\"?\": db.execute(\n f\"select vec_distance_cosine({a}, {b})\", args\n ).fetchone()[0]\n\n def check(a, b, dtype=np.float32):\n if dtype == np.float32:\n transform = \"?\"\n elif dtype == np.int8:\n transform = \"vec_int8(?)\"\n a = np.array(a, dtype=dtype)\n b = np.array(b, dtype=dtype)\n\n x = vec_distance_cosine(a, b, a=transform, b=transform)\n y = npy_cosine(a, b)\n assert isclose(x, y, abs_tol=1e-6)\n\n check([1.2, 0.1], [0.4, -0.4])\n check([-1.2, -0.1], [-0.4, 0.4])\n check([1, 2, 3], [-9, -8, -7], dtype=np.int8)\n assert vec_distance_cosine(\"[1.1, 1.0]\", \"[1.2, 1.2]\") == 0.001131898257881403\n\n\ndef test_vec_distance_hamming():\n vec_distance_hamming = lambda *args: db.execute(\n \"select vec_distance_hamming(vec_bit(?), vec_bit(?))\", args\n ).fetchone()[0]\n assert vec_distance_hamming(b\"\\xff\", b\"\\x00\") == 8\n assert vec_distance_hamming(b\"\\xff\", b\"\\x01\") == 7\n assert vec_distance_hamming(b\"\\xab\", b\"\\xab\") == 0\n\n with pytest.raises(\n sqlite3.OperationalError,\n match=\"Cannot calculate hamming distance between two float32 vectors.\",\n ):\n db.execute(\"select vec_distance_hamming(vec_f32('[1.0]'), vec_f32('[1.0]'))\")\n\n with pytest.raises(\n sqlite3.OperationalError,\n match=\"Cannot calculate hamming distance between two int8 vectors.\",\n ):\n db.execute(\"select vec_distance_hamming(vec_int8(X'FF'), vec_int8(X'FF'))\")\n\n\ndef test_vec_distance_l1():\n vec_distance_l1 = lambda *args, a=\"?\", b=\"?\": db.execute(\n f\"select vec_distance_l1({a}, {b})\", args\n ).fetchone()[0]\n\n def check(a, b, dtype=np.float32):\n if dtype == np.float32:\n transform = \"?\"\n elif dtype == np.int8:\n transform = \"vec_int8(?)\"\n\n a_sql_t = np.array(a, dtype=dtype)\n b_sql_t = np.array(b, dtype=dtype)\n\n x = vec_distance_l1(a_sql_t, b_sql_t, a=transform, b=transform)\n # dont use dtype here bc overflow\n y = np.sum(np.abs(np.array(a) - np.array(b)))\n assert isclose(x, y, abs_tol=1e-6)\n\n check([1, 2, 3], [-9, -8, -7], dtype=np.int8)\n # check overflow\n check([127] * 20, [-128] * 20, dtype=np.int8)\n check([-128, 127], [127, -128], dtype=np.int8)\n check(\n [1, 2, 3, 4, 5, 6, 7, 8, 1, 1, 2, 3, 4, 5, 6, 7, 8, 1],\n [1, 20, 38, 23, 29, 4, 10, 9, 3, 1, 20, 38, 23, 29, 4, 10, 9, 3],\n dtype=np.int8,\n )\n check([0] * 20, [0] * 20, dtype=np.int8)\n check(\n [5, 15, -20, 5, 15, -20, 5, 15, -20, 5, 15, -20, 5, 15, -20, 5, 15, -20],\n [5, 15, -20, 5, 15, -20, 5, 15, -20, 5, 15, -20, 5, 15, -20, 5, 15, -20],\n dtype=np.int8,\n )\n check([100] * 20, [-100] * 20, dtype=np.int8)\n check([127] * 1000000, [-128] * 1000000, dtype=np.int8)\n\n check(\n [1.2, 0.1, 0.5, 0.9, 1.4, 4.5],\n [0.4, -0.4, 0.1, 0.1, 0.5, 0.9],\n dtype=np.float32,\n )\n check([1.0, 2.0, 3.0], [-1.0, -2.0, -3.0], dtype=np.float32)\n check(\n [1e10, 2e10, np.finfo(np.float32).max],\n [-1e10, -2e10, np.finfo(np.float32).min],\n dtype=np.float32,\n )\n # overflow in leftover elements\n check(\n [1e10, 2e10, 1e10, 2e10, np.finfo(np.float32).max],\n [-1e10, -2e10, -1e10, -2e10, np.finfo(np.float32).min],\n dtype=np.float32,\n )\n # overflow in neon elements\n check(\n [np.finfo(np.float32).max, 1e10, 2e10, 1e10, 2e10],\n [np.finfo(np.float32).min, -1e10, -2e10, -1e10, -2e10],\n dtype=np.float32,\n )\n\n\ndef test_vec_distance_l2():\n vec_distance_l2 = lambda *args, a=\"?\", b=\"?\": db.execute(\n f\"select vec_distance_l2({a}, {b})\", args\n ).fetchone()[0]\n\n def check(a, b, dtype=np.float32):\n if dtype == np.float32:\n transform = \"?\"\n elif dtype == np.int8:\n transform = \"vec_int8(?)\"\n\n a_sql_t = np.array(a, dtype=dtype)\n b_sql_t = np.array(b, dtype=dtype)\n\n x = vec_distance_l2(a_sql_t, b_sql_t, a=transform, b=transform)\n y = npy_l2(np.array(a), np.array(b))\n assert isclose(x, y, abs_tol=1e-6)\n\n check([1.2, 0.1], [0.4, -0.4])\n check([-1.2, -0.1], [-0.4, 0.4])\n check([1, 2, 3], [-9, -8, -7], dtype=np.int8)\n\n\ndef test_vec_length():\n def test_f32():\n vec_length = lambda *args: db.execute(\"select vec_length(?)\", args).fetchone()[\n 0\n ]\n assert vec_length(b\"\\xAA\\xBB\\xCC\\xDD\") == 1\n assert vec_length(b\"\\xAA\\xBB\\xCC\\xDD\\x01\\x02\\x03\\x04\") == 2\n assert vec_length(f32_zerod(1024)) == 1024\n\n with pytest.raises(\n sqlite3.OperationalError, match=\"zero-length vectors are not supported.\"\n ):\n assert vec_length(b\"\") == 0\n with pytest.raises(\n sqlite3.OperationalError, match=\"zero-length vectors are not supported.\"\n ):\n vec_length(\"[]\")\n\n def test_int8():\n vec_length_int8 = lambda *args: db.execute(\n \"select vec_length(vec_int8(?))\", args\n ).fetchone()[0]\n assert vec_length_int8(b\"\\xAA\") == 1\n assert vec_length_int8(b\"\\xAA\\xBB\\xCC\\xDD\") == 4\n assert vec_length_int8(b\"\\xAA\\xBB\\xCC\\xDD\\x01\\x02\\x03\\x04\") == 8\n\n with pytest.raises(\n sqlite3.OperationalError, match=\"zero-length vectors are not supported.\"\n ):\n assert vec_length_int8(b\"\") == 0\n\n def test_bit():\n vec_length_bit = lambda *args: db.execute(\n \"select vec_length(vec_bit(?))\", args\n ).fetchone()[0]\n assert vec_length_bit(b\"\\xAA\") == 8\n assert vec_length_bit(b\"\\xAA\\xBB\\xCC\\xDD\") == 8 * 4\n assert vec_length_bit(b\"\\xAA\\xBB\\xCC\\xDD\\x01\\x02\\x03\\x04\") == 8 * 8\n\n with pytest.raises(\n sqlite3.OperationalError, match=\"zero-length vectors are not supported.\"\n ):\n assert vec_length_bit(b\"\") == 0\n\n test_f32()\n test_int8()\n test_bit()\n\n\ndef test_vec_normalize():\n vec_normalize = lambda *args: db.execute(\n \"select vec_normalize(?)\", args\n ).fetchone()[0]\n assert list(struct.unpack_from(\"4f\", vec_normalize(_f32([1, 2, -1, -2])))) == [\n 0.3162277638912201,\n 0.6324555277824402,\n -0.3162277638912201,\n -0.6324555277824402,\n ]\n\n\ndef test_vec_slice():\n vec_slice = lambda *args, f=\"?\": db.execute(\n f\"select vec_slice({f}, ?, ?)\", args\n ).fetchone()[0]\n assert vec_slice(_f32([1.1, 2.2, 3.3]), 0, 3) == _f32([1.1, 2.2, 3.3])\n assert vec_slice(_f32([1.1, 2.2, 3.3]), 0, 2) == _f32([1.1, 2.2])\n assert vec_slice(_f32([1.1, 2.2, 3.3]), 0, 1) == _f32([1.1])\n assert vec_slice(_int8([1, 2, 3]), 0, 3, f=\"vec_int8(?)\") == _int8([1, 2, 3])\n assert vec_slice(_int8([1, 2, 3]), 0, 2, f=\"vec_int8(?)\") == _int8([1, 2])\n assert vec_slice(_int8([1, 2, 3]), 0, 1, f=\"vec_int8(?)\") == _int8([1])\n assert vec_slice(b\"\\xAA\\xBB\\xCC\\xDD\", 0, 8, f=\"vec_bit(?)\") == b\"\\xAA\"\n assert vec_slice(b\"\\xAA\\xBB\\xCC\\xDD\", 8, 16, f=\"vec_bit(?)\") == b\"\\xBB\"\n assert vec_slice(b\"\\xAA\\xBB\\xCC\\xDD\", 8, 24, f=\"vec_bit(?)\") == b\"\\xBB\\xCC\"\n assert vec_slice(b\"\\xAA\\xBB\\xCC\\xDD\", 0, 32, f=\"vec_bit(?)\") == b\"\\xAA\\xBB\\xCC\\xDD\"\n\n with pytest.raises(\n sqlite3.OperationalError, match=\"start index must be divisible by 8.\"\n ):\n vec_slice(b\"\\xAA\\xBB\\xCC\\xDD\", 2, 32, f=\"vec_bit(?)\")\n\n with pytest.raises(\n sqlite3.OperationalError, match=\"end index must be divisible by 8.\"\n ):\n vec_slice(b\"\\xAA\\xBB\\xCC\\xDD\", 0, 31, f=\"vec_bit(?)\")\n\n with pytest.raises(\n sqlite3.OperationalError, match=\"slice 'start' index must be a postive number.\"\n ):\n vec_slice(b\"\\xab\\xab\\xab\\xab\", -1, 1)\n\n with pytest.raises(\n sqlite3.OperationalError, match=\"slice 'end' index must be a postive number.\"\n ):\n vec_slice(b\"\\xab\\xab\\xab\\xab\", 0, -3)\n with pytest.raises(\n sqlite3.OperationalError,\n match=\"slice 'start' index is greater than the number of dimensions\",\n ):\n vec_slice(b\"\\xab\\xab\\xab\\xab\", 2, 3)\n with pytest.raises(\n sqlite3.OperationalError,\n match=\"slice 'end' index is greater than the number of dimensions\",\n ):\n vec_slice(b\"\\xab\\xab\\xab\\xab\", 0, 2)\n with pytest.raises(\n sqlite3.OperationalError,\n match=\"slice 'start' index is greater than 'end' index\",\n ):\n vec_slice(b\"\\xab\\xab\\xab\\xab\", 1, 0)\n\n with _raises(\n \"slice 'start' index is equal to the 'end' index, vectors must have non-zero length\"\n ):\n vec_slice(b\"\\xab\\xab\\xab\\xab\", 0, 0)\n\n\ndef test_vec_type():\n vec_type = lambda *args, a=\"?\": db.execute(\n f\"select vec_type({a})\", args\n ).fetchone()[0]\n assert vec_type(\"[1]\") == \"float32\"\n assert vec_type(b\"\\xaa\\xbb\\xcc\\xdd\") == \"float32\"\n assert vec_type(\"[1]\", a=\"vec_f32(?)\") == \"float32\"\n assert vec_type(\"[1]\", a=\"vec_int8(?)\") == \"int8\"\n assert vec_type(b\"\\xaa\", a=\"vec_bit(?)\") == \"bit\"\n\n with _raises(\"invalid float32 vector\"):\n vec_type(b\"\\xaa\")\n with _raises(\"found NULL\"):\n vec_type(None)\n\n\ndef test_vec_add():\n vec_add = lambda *args, a=\"?\", b=\"?\": db.execute(\n f\"select vec_add({a}, {b})\", args\n ).fetchone()[0]\n assert vec_add(\"[1]\", \"[2]\") == _f32([3])\n assert vec_add(\"[.1]\", \"[.2]\") == _f32([0.3])\n assert vec_add(_int8([1]), _int8([2]), a=\"vec_int8(?)\", b=\"vec_int8(?)\") == _int8(\n [3]\n )\n\n with pytest.raises(\n sqlite3.OperationalError,\n match=\"Cannot add two bitvectors together.\",\n ):\n vec_add(b\"0xff\", b\"0xff\", a=\"vec_bit(?)\", b=\"vec_bit(?)\")\n\n with pytest.raises(\n sqlite3.OperationalError,\n match=\"Vector type mistmatch. First vector has type float32, while the second has type int8.\",\n ):\n vec_add(_f32([1]), _int8([2]), b=\"vec_int8(?)\")\n with pytest.raises(\n sqlite3.OperationalError,\n match=\"Vector type mistmatch. First vector has type int8, while the second has type float32.\",\n ):\n vec_add(_int8([2]), _f32([1]), a=\"vec_int8(?)\")\n\n\ndef test_vec_sub():\n vec_sub = lambda *args, a=\"?\", b=\"?\": db.execute(\n f\"select vec_sub({a}, {b})\", args\n ).fetchone()[0]\n assert vec_sub(\"[1]\", \"[2]\") == _f32([-1])\n assert vec_sub(\"[.1]\", \"[.2]\") == _f32([-0.1])\n assert vec_sub(_int8([11]), _int8([2]), a=\"vec_int8(?)\", b=\"vec_int8(?)\") == _int8(\n [9]\n )\n\n with pytest.raises(\n sqlite3.OperationalError,\n match=\"Cannot subtract two bitvectors together.\",\n ):\n vec_sub(b\"0xff\", b\"0xff\", a=\"vec_bit(?)\", b=\"vec_bit(?)\")\n\n with pytest.raises(\n sqlite3.OperationalError,\n match=\"Vector type mistmatch. First vector has type float32, while the second has type int8.\",\n ):\n vec_sub(_f32([1]), _int8([2]), b=\"vec_int8(?)\")\n with pytest.raises(\n sqlite3.OperationalError,\n match=\"Vector type mistmatch. First vector has type int8, while the second has type float32.\",\n ):\n vec_sub(_int8([2]), _f32([1]), a=\"vec_int8(?)\")\n\n\ndef test_vec_to_json():\n vec_to_json = lambda *args, input=\"?\": db.execute(\n f\"select vec_to_json({input})\", args\n ).fetchone()[0]\n assert vec_to_json(\"[1, 2, 3]\") == \"[1.000000,2.000000,3.000000]\"\n assert vec_to_json(b\"\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\xbf\") == \"[0.000000,-1.000000]\"\n assert vec_to_json(b\"\\x04\", input=\"vec_int8(?)\") == \"[4]\"\n assert vec_to_json(b\"\\x04\\xff\", input=\"vec_int8(?)\") == \"[4,-1]\"\n assert vec_to_json(b\"\\xff\", input=\"vec_bit(?)\") == \"[1,1,1,1,1,1,1,1]\"\n assert vec_to_json(b\"\\x0f\", input=\"vec_bit(?)\") == \"[1,1,1,1,0,0,0,0]\"\n\n\n@pytest.mark.skip(reason=\"TODO\")\ndef test_vec_quantize_int8():\n vec_quantize_int8 = lambda *args: db.execute(\n \"select vec_quantize_int8()\", args\n ).fetchone()[0]\n assert vec_quantize_int8() == 111\n\n\ndef test_vec_quantize_binary():\n vec_quantize_binary = lambda *args, input=\"?\": db.execute(\n f\"select vec_quantize_binary({input})\", args\n ).fetchone()[0]\n assert vec_quantize_binary(\"[-1, -1, -1, -1, 1, 1, 1, 1]\") == b\"\\xf0\"\n\n\n@pytest.mark.skip(reason=\"TODO\")\ndef test_vec0():\n pass\n\n\ndef test_vec0_inserts():\n db = connect(EXT_PATH)\n db.execute(\n \"\"\"\n create virtual table t using vec0(\n aaa float[128],\n bbb int8[128],\n ccc bit[128]\n );\n \"\"\"\n )\n\n db.execute(\n \"insert into t values (?, ?, vec_int8(?), vec_bit(?))\",\n [\n 1,\n np.full((128,), 0.0001, dtype=\"float32\"),\n np.full((128,), 4, dtype=\"int8\"),\n bitmap_full(128),\n ],\n )\n\n assert execute_all(db, \"select * from t\") == [\n {\n \"rowid\": 1,\n \"aaa\": _f32([0.0001] * 128),\n \"bbb\": _int8([4] * 128),\n \"ccc\": bitmap_full(128),\n }\n ]\n # db.execute(\n # \"update t set aaa = ? where rowid = ?\",\n # [np.full((128,), 0.00011, dtype=\"float32\"), 1],\n # )\n # assert execute_all(db, \"select * from t\") == [\n # {\n # \"rowid\": 1,\n # \"aaa\": _f32([0.00011] * 128),\n # \"bbb\": _int8([4] * 128),\n # \"ccc\": bitmap_full(128),\n # }\n # ]\n\n db.execute(\"create virtual table t1 using vec0(aaa float[4], chunk_size=8)\")\n db.execute(\n \"create virtual table txt_pk using vec0( txt_id text primary key, aaa float[4])\"\n )\n\n # EVIDENCE-OF: V06519_23358 vec0 INSERT validates vector\n with _raises(\n 'Inserted vector for the \"aaa\" column is invalid: Input must have type BLOB (compact format) or TEXT (JSON)'\n ):\n db.execute(\"insert into t1 values (1, ?)\", [None])\n\n # EVIDENCE-OF: V08221_25059 vec0 INSERT validates vector type\n with _raises(\n 'Inserted vector for the \"aaa\" column is expected to be of type float32, but a bit vector was provided.'\n ):\n db.execute(\"insert into t1 values (1, vec_bit(?))\", [b\"\\xff\\xff\\xff\\xff\"])\n\n # EVIDENCE-OF: V01145_17984 vec0 INSERT validates vector dimension match\n with _raises(\n 'Dimension mismatch for inserted vector for the \"aaa\" column. Expected 4 dimensions but received 3.'\n ):\n db.execute(\"insert into t1 values (1, ?)\", [\"[1,2,3]\"])\n\n # EVIDENCE-OF: V24228_08298 vec0 INSERT ensure no value provided for \"distance\" hidden column.\n with _raises('A value was provided for the hidden \"distance\" column.'):\n db.execute(\"insert into t1(rowid, aaa, distance) values (1, '[1,2,3,4]', 1)\")\n\n # EVIDENCE-OF: V11875_28713 vec0 INSERT ensure no value provided for \"distance\" hidden column.\n with _raises('A value was provided for the hidden \"k\" column.'):\n db.execute(\"insert into t1(rowid, aaa, k) values (1, '[1,2,3,4]', 1)\")\n\n # EVIDENCE-OF: V17090_01160 vec0 INSERT duplicated int primary key raises uniqueness error\n db.execute(\"insert into t1 values (1, '[1,1,1,1]')\")\n with _raises(\"UNIQUE constraint failed on t1 primary key\"):\n db.execute(\"insert into t1 values (1, '[2,2,2,2]')\")\n\n # similate error on rowids shadow table\n db.commit()\n db.set_authorizer(authorizer_deny_on(sqlite3.SQLITE_INSERT, \"t1_rowids\"))\n # EVIDENCE-OF: V04679_21517 vec0 INSERT failed on _rowid shadow insert raises error\n with _raises(\n \"Internal sqlite-vec error: could not initialize 'insert rowids' statement\",\n sqlite3.DatabaseError,\n ):\n db.execute(\"insert into t1 values (2, '[2,2,2,2]')\")\n db.set_authorizer(None)\n db.rollback()\n db.execute(\"insert into t1 values (2, '[2,2,2,2]')\")\n\n # test inserts where no rowid is provided\n db.execute(\"insert into t1(aaa) values ('[3,3,3,3]')\")\n\n # EVIDENCE-OF: V30855_14925 vec0 INSERT non-integer/text primary key value rauses error\n with _raises(\"Only integers are allows for primary key values on t1\"):\n db.execute(\"insert into t1 values (1.2, '[4,4,4,4]')\")\n\n # similate error on rowids shadow table, when rowid is not provided\n # EVIDENCE-OF: V15177_32015 vec0 INSERT error on _rowids shadow insert raises error\n db.set_authorizer(authorizer_deny_on(sqlite3.SQLITE_INSERT, \"t1_rowids\"))\n with _raises(\"Error inserting id into rowids shadow table: not authorized\"):\n db.execute(\"insert into t1(aaa) values ('[2,2,2,2]')\")\n db.set_authorizer(None)\n\n # EVIDENCE-OF: V31559_15629 vec0 INSERT error on _chunks shadow insert raises error\n db.set_authorizer(authorizer_deny_on(sqlite3.SQLITE_READ, \"t1_chunks\", \"chunk_id\"))\n with _raises(\"Internal sqlite-vec error: Could not find latest chunk\"):\n db.execute(\"insert into t1 values (999, '[2,2,2,2]')\")\n db.set_authorizer(None)\n\n # EVIDENCE-OF: V22053_06123 vec0 INSERT error on reading validity blob\n if SUPPORTS_DROP_COLUMN:\n db.commit()\n db.execute(\"begin\")\n db.execute(\"ALTER TABLE t1_chunks DROP COLUMN validity\")\n with _raises(\n \"Internal sqlite-vec error: could not open validity blob on main.t1_chunks.1\"\n ):\n db.execute(\"insert into t1 values (9999, '[2,2,2,2]')\")\n db.rollback()\n\n # EVIDENCE-OF: V29362_13432 vec0 INSERT validity blob size mismatch with chunk_size\n db.commit()\n db.execute(\"begin\")\n db.execute(\"UPDATE t1_chunks SET validity = zeroblob(101)\")\n with _raises(\n \"Internal sqlite-vec error: validity blob size mismatch on main.t1_chunks.1, expected 1 but received 101.\"\n ):\n db.execute(\"insert into t1 values (9999, '[2,2,2,2]')\")\n db.rollback()\n\n # EVIDENCE-OF: V16386_00456 vec0 INSERT valdates vector blob column sizes\n db.commit()\n db.execute(\"begin\")\n db.execute(\"UPDATE t1_vector_chunks00 SET vectors = zeroblob(101)\")\n with _raises(\n \"Internal sqlite-vec error: vector blob size mismatch on main.t1_vector_chunks00.1. Expected 128, actual 101\"\n ):\n db.execute(\"insert into t1 values (9999, '[2,2,2,2]')\")\n db.rollback()\n\n # EVIDENCE-OF: V09221_26060 vec0 INSERT rowids blob open error\n if SUPPORTS_DROP_COLUMN:\n db.commit()\n db.execute(\"begin\")\n db.execute(\"ALTER TABLE t1_chunks DROP COLUMN rowids\")\n with _raises(\n \"Internal sqlite-vec error: could not open rowids blob on main.t1_chunks.1\"\n ):\n db.execute(\"insert into t1 values (9999, '[2,2,2,2]')\")\n db.rollback()\n\n # EVIDENCE-OF: V12779_29618 vec0 INSERT rowids blob validates size\n db.commit()\n db.execute(\"begin\")\n db.execute(\"UPDATE t1_chunks SET rowids = zeroblob(101)\")\n with _raises(\n \"Internal sqlite-vec error: rowids blob size mismatch on main.t1_chunks.1. Expected 64, actual 101\"\n ):\n db.execute(\"insert into t1 values (9999, '[2,2,2,2]')\")\n db.rollback()\n\n # EVIDENCE-OF: V21925_05995 vec0 INSERT error on \"rowids update position\" raises error\n db.commit()\n db.execute(\"begin\")\n db.execute(\"insert into t1 values (9998, '[2,2,2,2]')\")\n db.set_authorizer(\n authorizer_deny_on(sqlite3.SQLITE_UPDATE, \"t1_rowids\", \"chunk_id\")\n )\n with _raises(\n \"Internal sqlite-vec error: could not update rowids position for rowid=9999, chunk_rowid=1, chunk_offset=4\"\n ):\n db.execute(\"insert into t1 values (9999, '[2,2,2,2]')\")\n db.set_authorizer(None)\n db.rollback()\n\n ########## testing inserts on text primary key tables ##########\n\n # EVIDENCE-OF: V04200_21039 vec0 table with text primary key ensure text values\n with _raises(\n \"The txt_pk virtual table was declared with a TEXT primary key, but a non-TEXT value was provided in an INSERT.\"\n ):\n db.execute(\"insert into txt_pk(txt_id, aaa) values (1, '[1,2,3,4]')\")\n\n db.execute(\"insert into txt_pk(txt_id, aaa) values ('a', '[1,2,3,4]')\")\n\n # EVIDENCE-OF: V20497_04568 vec0 table with text primary key raises uniqueness error on duplicate values\n with _raises(\"UNIQUE constraint failed on txt_pk primary key\"):\n db.execute(\"insert into txt_pk(txt_id, aaa) values ('a', '[5,6,7,8]')\")\n\n # EVIDENCE-OF: V24016_08086 vec0 table with text primary key raises error on rowid write error\n db.set_authorizer(authorizer_deny_on(sqlite3.SQLITE_INSERT, \"txt_pk_rowids\"))\n with _raises(\"Error inserting id into rowids shadow table: not authorized\"):\n db.execute(\"insert into txt_pk(txt_id, aaa) values ('b', '[2,2,2,2]')\")\n db.set_authorizer(None)\n db.execute(\"insert into txt_pk(txt_id, aaa) values ('b', '[2,2,2,2]')\")\n\n\ndef test_vec0_insert_errors2():\n db = connect(EXT_PATH)\n db.execute(\"create virtual table t1 using vec0(aaa float[4], chunk_size=8)\")\n db.execute(\n \"\"\"\n insert into t1(aaa) values\n ('[1,1,1,1]'),\n ('[2,1,1,1]'),\n ('[3,1,1,1]'),\n ('[4,1,1,1]'),\n ('[5,1,1,1]'),\n ('[6,1,1,1]')\n \"\"\"\n )\n assert execute_all(db, \"select * from t1_chunks\") == [\n {\n \"chunk_id\": 1,\n \"rowids\": b\"\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\"\n + b\"\\x02\\x00\\x00\\x00\\x00\\x00\\x00\\x00\"\n + b\"\\x03\\x00\\x00\\x00\\x00\\x00\\x00\\x00\"\n + b\"\\x04\\x00\\x00\\x00\\x00\\x00\\x00\\x00\"\n + b\"\\x05\\x00\\x00\\x00\\x00\\x00\\x00\\x00\"\n + b\"\\x06\\x00\\x00\\x00\\x00\\x00\\x00\\x00\"\n + b\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\"\n + b\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\",\n \"size\": 8,\n \"validity\": b\"?\", # 0b00111111\n }\n ]\n db.execute(\n \"\"\"\n insert into t1(aaa) values\n ('[7,1,1,1]'),\n ('[8,1,1,1]')\n \"\"\"\n )\n assert execute_all(db, \"select * from t1_chunks\") == [\n {\n \"chunk_id\": 1,\n \"rowids\": b\"\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\"\n + b\"\\x02\\x00\\x00\\x00\\x00\\x00\\x00\\x00\"\n + b\"\\x03\\x00\\x00\\x00\\x00\\x00\\x00\\x00\"\n + b\"\\x04\\x00\\x00\\x00\\x00\\x00\\x00\\x00\"\n + b\"\\x05\\x00\\x00\\x00\\x00\\x00\\x00\\x00\"\n + b\"\\x06\\x00\\x00\\x00\\x00\\x00\\x00\\x00\"\n + b\"\\x07\\x00\\x00\\x00\\x00\\x00\\x00\\x00\"\n + b\"\\x08\\x00\\x00\\x00\\x00\\x00\\x00\\x00\",\n \"size\": 8,\n \"validity\": b\"\\xff\", # 0b11111111\n }\n ]\n # EVIDENCE-OF: V08441_25279 vec0 INSERT error on new chunk creation raises error\n db.set_authorizer(authorizer_deny_on(sqlite3.SQLITE_INSERT, \"t1_chunks\"))\n with _raises(\"Internal sqlite-vec error: Could not insert a new vector chunk\"):\n db.execute(\"insert into t1(aaa) values ('[9,1,1,1]')\")\n db.set_authorizer(None)\n\n\ndef test_vec0_drops():\n db = connect(EXT_PATH)\n db.execute(\n \"create virtual table t1 using vec0(aaa float[4], bbb float[4], chunk_size=8)\"\n )\n assert [\n row[\"name\"]\n for row in execute_all(\n db, \"select name from sqlite_master where name like 't1%' order by 1\"\n )\n ] == [\n \"t1\",\n \"t1_chunks\",\n \"t1_info\",\n \"t1_rowids\",\n \"t1_vector_chunks00\",\n \"t1_vector_chunks01\",\n ]\n\n db.execute(\"drop table t1\")\n assert [\n row[\"name\"]\n for row in execute_all(\n db, \"select name from sqlite_master where name like 't1%' order by 1\"\n )\n ] == []\n\n\ndef test_vec0_delete():\n db = connect(EXT_PATH)\n db.execute(\"create virtual table t1 using vec0(aaa float[4], chunk_size=8)\")\n db.execute(\n \"\"\"\n insert into t1(aaa) values\n ('[1,1,1,1]'),\n ('[2,1,1,1]'),\n ('[3,1,1,1]'),\n ('[4,1,1,1]'),\n ('[5,1,1,1]'),\n ('[6,1,1,1]')\n \"\"\"\n )\n assert execute_all(db, \"select * from t1_rowids\") == [\n {\n \"chunk_id\": 1,\n \"chunk_offset\": 0,\n \"id\": None,\n \"rowid\": 1,\n },\n {\n \"chunk_id\": 1,\n \"chunk_offset\": 1,\n \"id\": None,\n \"rowid\": 2,\n },\n {\n \"chunk_id\": 1,\n \"chunk_offset\": 2,\n \"id\": None,\n \"rowid\": 3,\n },\n {\n \"chunk_id\": 1,\n \"chunk_offset\": 3,\n \"id\": None,\n \"rowid\": 4,\n },\n {\n \"chunk_id\": 1,\n \"chunk_offset\": 4,\n \"id\": None,\n \"rowid\": 5,\n },\n {\n \"chunk_id\": 1,\n \"chunk_offset\": 5,\n \"id\": None,\n \"rowid\": 6,\n },\n ]\n assert execute_all(db, \"select * from t1_chunks\") == [\n {\n \"chunk_id\": 1,\n \"rowids\": _i64([1, 2, 3, 4, 5, 6, 0, 0]),\n \"size\": 8,\n \"validity\": bitmap(\"00111111\"),\n }\n ]\n assert execute_all(db, \"select * from t1_vector_chunks00\") == [\n {\n \"rowid\": 1,\n \"vectors\": _f32([1, 1, 1, 1])\n + _f32([2, 1, 1, 1])\n + _f32([3, 1, 1, 1])\n + _f32([4, 1, 1, 1])\n + _f32([5, 1, 1, 1])\n + _f32([6, 1, 1, 1])\n + _f32([0, 0, 0, 0])\n + _f32([0, 0, 0, 0]),\n }\n ]\n\n db.execute(\"DELETE FROM t1 WHERE rowid = 1\")\n assert execute_all(db, \"select * from t1_rowids\") == [\n {\n \"chunk_id\": 1,\n \"chunk_offset\": 1,\n \"id\": None,\n \"rowid\": 2,\n },\n {\n \"chunk_id\": 1,\n \"chunk_offset\": 2,\n \"id\": None,\n \"rowid\": 3,\n },\n {\n \"chunk_id\": 1,\n \"chunk_offset\": 3,\n \"id\": None,\n \"rowid\": 4,\n },\n {\n \"chunk_id\": 1,\n \"chunk_offset\": 4,\n \"id\": None,\n \"rowid\": 5,\n },\n {\n \"chunk_id\": 1,\n \"chunk_offset\": 5,\n \"id\": None,\n \"rowid\": 6,\n },\n ]\n # TODO finish delete support\n # assert execute_all(db, \"select * from t1_chunks\") == [\n # {\n # 'chunk_id': 1,\n # 'rowids': _i64([0,2,3,4,5,6,0,0]),\n # 'size': 8,\n # 'validity': bitmap(\"00111110\"),\n # }\n # ]\n # assert execute_all(db, \"select * from t1_vector_chunks00\") == [\n # {\n # 'rowid': 1,\n # 'vectors': _f32([0,0,0,0])\n # +_f32([2,1,1,1])\n # +_f32([3,1,1,1])\n # +_f32([4,1,1,1])\n # +_f32([5,1,1,1])\n # +_f32([6,1,1,1])\n # +_f32([0,0,0,0])\n # +_f32([0,0,0,0])\n # }\n # ]\n\n # TODO test with text primary keys\n\n\ndef test_vec0_delete_errors():\n db = connect(EXT_PATH)\n db.execute(\"create virtual table t1 using vec0(aaa float[4], chunk_size=8)\")\n db.execute(\n \"\"\"\n insert into t1(aaa) values\n ('[1,1,1,1]'),\n ('[2,1,1,1]'),\n ('[3,1,1,1]'),\n ('[4,1,1,1]'),\n ('[5,1,1,1]'),\n ('[6,1,1,1]')\n \"\"\"\n )\n\n # db.commit()\n # db.execute(\"begin\")\n # db.execute(\"DELETE FROM t1_rowids WHERE rowid = 1\")\n # with _raises(\"XXX\"):\n # db.execute(\"DELETE FROM t1 where rowid = 1\")\n # db.rollback()\n\n # EVIDENCE-OF: V26002_10073 vec0 DELETE error on reading validity blob\n if SUPPORTS_DROP_COLUMN:\n db.commit()\n db.execute(\"begin\")\n db.execute(\"ALTER TABLE t1_chunks DROP COLUMN validity\")\n with _raises(\"could not open validity blob for main.t1_chunks.1\"):\n db.execute(\"delete from t1 where rowid = 1\")\n db.rollback()\n\n # EVIDENCE-OF: V21193_05263 vec0 DELETE verifies that the validity bit is 1 before clearing\n db.commit()\n db.execute(\"begin\")\n db.execute(\"UPDATE t1_chunks SET validity = zeroblob(1)\")\n with _raises(\n \"vec0 deletion error: validity bit is not set for main.t1_chunks.1 at 0\"\n ):\n db.execute(\"delete from t1 where rowid = 1\")\n db.rollback()\n\n # EVIDENCE-OF: V21193_05263 vec0 DELETE raises error on validity blob error\n db.commit()\n db.execute(\"begin\")\n db.execute(\"UPDATE t1_chunks SET validity = zeroblob(0)\")\n with _raises(\"could not read validity blob for main.t1_chunks.1 at 0\"):\n db.execute(\"delete from t1 where rowid = 1\")\n db.rollback()\n\n if False: # TODO\n with _raises(\"XXX\"):\n db.execute(\"DELETE FROM t1 WHERE rowid = 999\")\n if False: # TODO\n db.commit()\n db.execute(\"begin\")\n db.execute(\"DELETE FROM t1_rowids WHERE rowid = 1\")\n with _raises(\"XXX\"):\n db.execute(\"DELETE FROM t1 where rowid = 1\")\n db.rollback()\n\n\ndef test_vec0_updates():\n db = connect(EXT_PATH)\n db.execute(\n \"\"\"\n create virtual table t3 using vec0(\n aaa float[8],\n bbb int8[8],\n ccc bit[8]\n );\n \"\"\"\n )\n db.execute(\n \"\"\"\n INSERT INTO t3 VALUES\n (1, :x, vec_quantize_int8(:x, 'unit') ,vec_quantize_binary(:x)),\n (2, :y, vec_quantize_int8(:y, 'unit') ,vec_quantize_binary(:y)),\n (3, :z, vec_quantize_int8(:z, 'unit') ,vec_quantize_binary(:z));\n \"\"\",\n {\n \"x\": \"[.1, .1, .1, .1, -.1, -.1, -.1, -.1]\",\n \"y\": \"[-.2, .2, .2, .2, .2, .2, -.2, .2]\",\n \"z\": \"[.3, .3, .3, .3, .3, .3, .3, .3]\",\n },\n )\n assert execute_all(db, \"select * from t3\") == [\n {\n \"rowid\": 1,\n \"aaa\": _f32([0.1, 0.1, 0.1, 0.1, -0.1, -0.1, -0.1, -0.1]),\n \"bbb\": _int8([12, 12, 12, 12, -13, -13, -13, -13]),\n \"ccc\": bitmap(\"00001111\"),\n },\n {\n \"rowid\": 2,\n \"aaa\": _f32([-0.2, 0.2, 0.2, 0.2, 0.2, 0.2, -0.2, 0.2]),\n \"bbb\": _int8([-26, 24, 24, 24, 24, 24, -26, 24]),\n \"ccc\": bitmap(\"10111110\"),\n },\n {\n \"rowid\": 3,\n \"aaa\": _f32([0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3]),\n \"bbb\": _int8(\n [\n 37,\n 37,\n 37,\n 37,\n 37,\n 37,\n 37,\n 37,\n ]\n ),\n \"ccc\": bitmap(\"11111111\"),\n },\n ]\n\n db.execute(\"UPDATE t3 SET aaa = ? WHERE rowid = 1\", [\"[.9,.9,.9,.9,.9,.9,.9,.9]\"])\n assert execute_all(db, \"select * from t3\") == [\n {\n \"rowid\": 1,\n \"aaa\": _f32([0.9, 0.9, 0.9, 0.9, 0.9, 0.9, 0.9, 0.9]),\n \"bbb\": _int8([12, 12, 12, 12, -13, -13, -13, -13]),\n \"ccc\": bitmap(\"00001111\"),\n },\n {\n \"rowid\": 2,\n \"aaa\": _f32([-0.2, 0.2, 0.2, 0.2, 0.2, 0.2, -0.2, 0.2]),\n \"bbb\": _int8([-26, 24, 24, 24, 24, 24, -26, 24]),\n \"ccc\": bitmap(\"10111110\"),\n },\n {\n \"rowid\": 3,\n \"aaa\": _f32([0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3]),\n \"bbb\": _int8(\n [\n 37,\n 37,\n 37,\n 37,\n 37,\n 37,\n 37,\n 37,\n ]\n ),\n \"ccc\": bitmap(\"11111111\"),\n },\n ]\n\n # EVIDENCE-OF: V15203_32042 vec0 UPDATE validates vector\n with _raises(\n 'Updated vector for the \"aaa\" column is invalid: invalid float32 vector BLOB length. Must be divisible by 4, found 1'\n ):\n db.execute(\"UPDATE t3 SET aaa = X'AB' WHERE rowid = 1\")\n\n # EVIDENCE-OF: V25739_09810 vec0 UPDATE validates dimension length\n with _raises(\n 'Dimension mismatch for new updated vector for the \"aaa\" column. Expected 8 dimensions but received 1.'\n ):\n db.execute(\"UPDATE t3 SET aaa = vec_bit(X'AABBCCDD') WHERE rowid = 1\")\n\n # EVIDENCE-OF: V03643_20481 vec0 UPDATE validates vector column type\n with _raises(\n 'Updated vector for the \"bbb\" column is expected to be of type int8, but a float32 vector was provided.'\n ):\n db.execute(\"UPDATE t3 SET bbb = X'ABABABAB' WHERE rowid = 1\")\n\n db.execute(\"CREATE VIRTUAL TABLE t2 USING vec0(a float[2], b float[2])\")\n db.execute(\"INSERT INTO t2(rowid, a, b) VALUES (1, '[.1, .1]', '[.2, .2]')\")\n assert execute_all(db, \"select * from t2\") == [\n {\n \"rowid\": 1,\n \"a\": _f32([0.1, 0.1]),\n \"b\": _f32([0.2, 0.2]),\n }\n ]\n # sanity check: the 1st column UPDATE \"works\", but since the 2nd one fails,\n # then aaa should remain unchanged.\n with _raises(\n 'Dimension mismatch for new updated vector for the \"b\" column. Expected 2 dimensions but received 3.'\n ):\n db.execute(\n \"UPDATE t2 SET a = '[.11, .11]', b = '[.22, .22, .22]' WHERE rowid = 1\"\n )\n assert execute_all(db, \"select * from t2\") == [\n {\n \"rowid\": 1,\n \"a\": _f32([0.1, 0.1]),\n \"b\": _f32([0.2, 0.2]),\n }\n ]\n # TODO: set UPDATEs on int8/bit columns\n\n # db.execute(\"UPDATE t3 SET ccc = vec_bit(?) WHERE rowid = 3\", [bitmap('01010101')])\n # assert execute_all(db, \"select * from t3\") == [\n # {\n # \"rowid\": 1,\n # \"aaa\": _f32([0.9, 0.9, 0.9, 0.9, 0.9, 0.9, 0.9, 0.9]),\n # \"bbb\": _int8([12, 12, 12, 12, -13, -13, -13, -13]),\n # \"ccc\": bitmap(\"00001111\"),\n # },\n # {\n # \"rowid\": 2,\n # \"aaa\": _f32([-0.2, 0.2, 0.2, 0.2, 0.2, 0.2, -0.2, 0.2]),\n # \"bbb\": _int8([-26, 24, 24, 24, 24, 24, -26, 24]),\n # \"ccc\": bitmap(\"10111110\"),\n # },\n # {\n # \"rowid\": 3,\n # \"aaa\": _f32([0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3]),\n # \"bbb\": _int8([37, 37, 37, 37, 37, 37, 37, 37, ]),\n # \"ccc\": bitmap(\"11111111\"),\n # },\n # ]\n\n\ndef test_vec0_point():\n db = connect(EXT_PATH)\n db.execute(\"CREATE VIRTUAL TABLE t USING vec0(a float[1], b float[1])\")\n db.execute(\n \"INSERT INTO t VALUES (1, X'AABBCCDD', X'00112233'), (2, X'AABBCCDD', X'99887766');\"\n )\n\n assert execute_all(db, \"select * from t where rowid = 1\") == [\n {\n \"a\": b\"\\xaa\\xbb\\xcc\\xdd\",\n \"b\": b'\\x00\\x11\"3',\n \"rowid\": 1,\n }\n ]\n assert execute_all(db, \"select * from t where rowid = 999\") == []\n\n db.execute(\n \"CREATE VIRTUAL TABLE t2 USING vec0(id text primary key, a float[1], b float[1])\"\n )\n db.execute(\n \"INSERT INTO t2 VALUES ('A', X'AABBCCDD', X'00112233'), ('B', X'AABBCCDD', X'99887766');\"\n )\n\n assert execute_all(db, \"select * from t2 where id = 'A'\") == [\n {\n \"a\": b\"\\xaa\\xbb\\xcc\\xdd\",\n \"b\": b'\\x00\\x11\"3',\n \"id\": \"A\",\n }\n ]\n\n assert execute_all(db, \"select * from t2 where id = 'xxx'\") == []\n\n\ndef test_vec0_text_pk():\n db = connect(EXT_PATH)\n db.execute(\n \"\"\"\n create virtual table t using vec0(\n t_id text primary key,\n aaa float[1],\n bbb float8[1],\n chunk_size=8\n );\n \"\"\"\n )\n assert execute_all(db, \"select * from t\") == []\n\n with _raises(\n \"The t virtual table was declared with a TEXT primary key, but a non-TEXT value was provided in an INSERT.\"\n ):\n db.execute(\"INSERT INTO t VALUES (1, X'AABBCCDD', X'AABBCCDD')\")\n\n db.executemany(\n \"INSERT INTO t VALUES (:t_id, :aaa, :bbb)\",\n [\n {\n \"t_id\": \"t_1\",\n \"aaa\": \"[.1]\",\n \"bbb\": \"[-.1]\",\n },\n {\n \"t_id\": \"t_2\",\n \"aaa\": \"[.2]\",\n \"bbb\": \"[-.2]\",\n },\n {\n \"t_id\": \"t_3\",\n \"aaa\": \"[.3]\",\n \"bbb\": \"[-.3]\",\n },\n ],\n )\n assert execute_all(db, \"select t_id from t\") == [\n {\"t_id\": \"t_1\"},\n {\"t_id\": \"t_2\"},\n {\"t_id\": \"t_3\"},\n ]\n assert execute_all(db, \"select * from t\") == [\n {\"t_id\": \"t_1\", \"aaa\": _f32([0.1]), \"bbb\": _f32([-0.1])},\n {\"t_id\": \"t_2\", \"aaa\": _f32([0.2]), \"bbb\": _f32([-0.2])},\n {\"t_id\": \"t_3\", \"aaa\": _f32([0.3]), \"bbb\": _f32([-0.3])},\n ]\n\n # EVIDENCE-OF: V09901_26739 vec0 full scan catches _rowid prep error\n db.set_authorizer(authorizer_deny_on(sqlite3.SQLITE_READ, \"t_rowids\", \"rowid\"))\n with _raises(\n \"Error preparing rowid scan: access to t_rowids.rowid is prohibited\",\n sqlite3.DatabaseError,\n ):\n db.execute(\"select * from t\")\n db.set_authorizer(None)\n\n assert execute_all(\n db, \"select t_id, distance from t where aaa match ? and k = 3\", [\"[.01]\"]\n ) == [\n {\n \"t_id\": \"t_1\",\n \"distance\": 0.09000000357627869,\n },\n {\n \"t_id\": \"t_2\",\n \"distance\": 0.1899999976158142,\n },\n {\n \"t_id\": \"t_3\",\n \"distance\": 0.2900000214576721,\n },\n ]\n\n if SUPPORTS_VTAB_IN:\n assert re.match(\n (\"SCAN (TABLE )?t VIRTUAL TABLE INDEX 0:3{___}___\\[___\"),\n explain_query_plan(\n \"select t_id, distance from t where aaa match '' and k = 3 and t_id in ('t_2', 't_3')\",\n db=db,\n ),\n )\n assert execute_all(\n db,\n \"select t_id, distance from t where aaa match ? and k = 3 and t_id in ('t_2', 't_3')\",\n [\"[.01]\"],\n ) == [\n {\n \"t_id\": \"t_2\",\n \"distance\": 0.1899999976158142,\n },\n {\n \"t_id\": \"t_3\",\n \"distance\": 0.2900000214576721,\n },\n ]\n\n # test deletes on text primary keys\n db.execute(\"delete from t where t_id = 't_1'\")\n assert execute_all(db, \"select * from t\") == [\n {\"t_id\": \"t_2\", \"aaa\": _f32([0.2]), \"bbb\": _f32([-0.2])},\n {\"t_id\": \"t_3\", \"aaa\": _f32([0.3]), \"bbb\": _f32([-0.3])},\n ]\n\n # test updates on text primary keys\n db.execute(\"update t set aaa = '[999]' where t_id = 't_2'\")\n assert execute_all(db, \"select * from t\") == [\n {\"t_id\": \"t_2\", \"aaa\": _f32([999]), \"bbb\": _f32([-0.2])},\n {\"t_id\": \"t_3\", \"aaa\": _f32([0.3]), \"bbb\": _f32([-0.3])},\n ]\n\n # EVIDENCE-OF: V08886_25725 vec0 primary keys don't allow updates on PKs\n with _raises(\"UPDATEs on vec0 primary key values are not allowed.\"):\n db.execute(\"update t set t_id = 'xxx' where t_id = 't_2'\")\n\n\ndef test_vec0_best_index():\n db = connect(EXT_PATH)\n db.execute(\n \"\"\"\n create virtual table t using vec0(\n aaa float[1],\n bbb float8[1]\n );\n \"\"\"\n )\n\n with _raises(\"only 1 MATCH operator is allowed in a single vec0 query\"):\n db.execute(\"select * from t where aaa match NULL and bbb match NULL\")\n\n if SUPPORTS_VTAB_IN:\n with _raises(\n \"only 1 'rowid in (..)' operator is allowed in a single vec0 query\"\n ):\n db.execute(\"select * from t where rowid in(4,5,6) and rowid in (1, 2,3)\")\n\n with _raises(\"A LIMIT or 'k = ?' constraint is required on vec0 knn queries.\"):\n db.execute(\"select * from t where aaa MATCH ?\")\n\n if SUPPORTS_VTAB_LIMIT:\n with _raises(\"Only LIMIT or 'k =?' can be provided, not both\"):\n db.execute(\"select * from t where aaa MATCH ? and k = 10 limit 20\")\n\n with _raises(\n \"Only a single 'ORDER BY distance' clause is allowed on vec0 KNN queries\"\n ):\n db.execute(\n \"select * from t where aaa MATCH NULL and k = 10 order by distance, distance\"\n )\n\n with _raises(\n \"Only ascending in ORDER BY distance clause is supported, DESC is not supported yet.\"\n ):\n db.execute(\n \"select * from t where aaa MATCH NULL and k = 10 order by distance desc\"\n )\n\n\ndef authorizer_deny_on(operation, x1, x2=None):\n def _auth(op, p1, p2, p3, p4):\n if op == operation and p1 == x1 and p2 == x2:\n return sqlite3.SQLITE_DENY\n return sqlite3.SQLITE_OK\n\n return _auth\n\n\ndef authorizer_debug(op, p1, p2, p3, p4):\n print(op, p1, p2, p3, p4)\n return sqlite3.SQLITE_OK\n\n\nfrom contextlib import contextmanager\n\n\n@contextmanager\ndef _raises(message, error=sqlite3.OperationalError):\n with pytest.raises(error, match=re.escape(message)):\n yield\n\n\ndef test_vec_each():\n vec_each_f32 = lambda *args: execute_all(\n db, \"select rowid, * from vec_each(vec_f32(?))\", args\n )\n assert vec_each_f32(_f32([1.0, 2.0, 3.0])) == [\n {\"rowid\": 0, \"value\": 1.0},\n {\"rowid\": 1, \"value\": 2.0},\n {\"rowid\": 2, \"value\": 3.0},\n ]\n\n with _raises(\"Input must have type BLOB (compact format) or TEXT (JSON), found NULL\"):\n vec_each_f32(None)\n\n\nimport io\n\n\ndef to_npy(arr):\n buf = io.BytesIO()\n np.save(buf, arr)\n buf.seek(0)\n return buf.read()\n\n\ndef test_vec_npy_each():\n db = connect(EXT_PATH, extra_entrypoint=\"sqlite3_vec_numpy_init\")\n vec_npy_each = lambda *args: execute_all(\n db, \"select rowid, * from vec_npy_each(?)\", args\n )\n assert vec_npy_each(to_npy(np.array([1.1, 2.2, 3.3], dtype=np.float32))) == [\n {\n \"rowid\": 0,\n \"vector\": _f32([1.1, 2.2, 3.3]),\n },\n ]\n assert vec_npy_each(to_npy(np.array([[1.1, 2.2, 3.3]], dtype=np.float32))) == [\n {\n \"rowid\": 0,\n \"vector\": _f32([1.1, 2.2, 3.3]),\n },\n ]\n assert vec_npy_each(\n to_npy(np.array([[1.1, 2.2, 3.3], [9.9, 8.8, 7.7]], dtype=np.float32))\n ) == [\n {\n \"rowid\": 0,\n \"vector\": _f32([1.1, 2.2, 3.3]),\n },\n {\n \"rowid\": 1,\n \"vector\": _f32([9.9, 8.8, 7.7]),\n },\n ]\n\n assert vec_npy_each(to_npy(np.array([], dtype=np.float32))) == []\n\n\ndef test_vec_npy_each_errors():\n db = connect(EXT_PATH, extra_entrypoint=\"sqlite3_vec_numpy_init\")\n vec_npy_each = lambda *args: execute_all(\n db, \"select rowid, * from vec_npy_each(?)\", args\n )\n\n full = b\"\\x93NUMPY\\x01\\x00v\\x00{'descr': ' npt.NDArray[np.float32]:\n return np.sqrt(np.sum((mat - vec) ** 2, axis=1))\n\n\ndef np_topk(\n vec: npt.NDArray[np.float32],\n mat: npt.NDArray[np.float32],\n k: int = 5,\n) -> tuple[npt.NDArray[np.int32], npt.NDArray[np.float32]]:\n distances = np_distance_l2(vec, mat)\n # Rather than sorting all similarities and taking the top K, it's faster to\n # argpartition and then just sort the top K.\n # The difference is O(N logN) vs O(N + k logk)\n indices = np.argpartition(distances, kth=k)[:k]\n top_indices = indices[np.argsort(distances[indices])]\n return top_indices, distances[top_indices]\n\n\n# import faiss\n@pytest.mark.skip(reason=\"TODO\")\ndef test_correctness_npy():\n db = connect(EXT_PATH)\n np.random.seed(420 + 1 + 2)\n mat = np.random.uniform(low=-1.0, high=1.0, size=(10000, 24)).astype(np.float32)\n queries = np.random.uniform(low=-1.0, high=1.0, size=(1000, 24)).astype(np.float32)\n\n # sqlite-vec with vec0\n db.execute(\"create virtual table v using vec0(a float[24], chunk_size=8)\")\n for v in mat:\n db.execute(\"insert into v(a) values (?)\", [v])\n\n # sqlite-vec with scalar functions\n db.execute(\"create table t(a float[24])\")\n for v in mat:\n db.execute(\"insert into t(a) values (?)\", [v])\n\n faiss_index = faiss.IndexFlatL2(24)\n faiss_index.add(mat)\n\n k = 10000 - 1\n for idx, q in enumerate(queries):\n print(idx)\n result = execute_all(\n db,\n \"select rowid - 1 as idx, distance from v where a match ? and k = ?\",\n [q, k],\n )\n vec_vtab_rowids = [row[\"idx\"] for row in result]\n vec_vtab_distances = [row[\"distance\"] for row in result]\n\n result = execute_all(\n db,\n \"select rowid - 1 as idx, vec_distance_l2(a, ?) as distance from t order by 2 limit ?\",\n [q, k],\n )\n vec_scalar_rowids = [row[\"idx\"] for row in result]\n vec_scalar_distances = [row[\"distance\"] for row in result]\n assert vec_scalar_rowids == vec_vtab_rowids\n assert vec_scalar_distances == vec_vtab_distances\n\n faiss_distances, faiss_rowids = faiss_index.search(np.array([q]), k)\n faiss_distances = np.sqrt(faiss_distances)\n assert faiss_rowids[0].tolist() == vec_scalar_rowids\n assert faiss_distances[0].tolist() == vec_scalar_distances\n\n assert faiss_distances[0].tolist() == vec_vtab_distances\n assert faiss_rowids[0].tolist() == vec_vtab_rowids\n\n np_rowids, np_distances = np_topk(mat, q, k=k)\n # assert vec_vtab_rowids == np_rowids.tolist()\n # assert vec_vtab_distances == np_distances.tolist()\n\n\ndef test_smoke():\n db.execute(\"drop table if exists vec_xyz\")\n db.execute(\"create virtual table vec_xyz using vec0( a float[2] )\")\n assert execute_all(\n db,\n \"select name from sqlite_master where name like 'vec_xyz%' order by name;\",\n ) == [\n {\n \"name\": \"vec_xyz\",\n },\n {\n \"name\": \"vec_xyz_chunks\",\n },\n {\n \"name\": \"vec_xyz_info\",\n },\n {\n \"name\": \"vec_xyz_rowids\",\n },\n {\n \"name\": \"vec_xyz_vector_chunks00\",\n },\n ]\n chunk = db.execute(\"select * from vec_xyz_chunks\").fetchone()\n # as of TODO, no initial row is inside the chunks table\n assert chunk is None\n # assert chunk[\"chunk_id\"] == 1\n # assert chunk[\"validity\"] == bytearray(int(1024 / 8))\n # assert chunk[\"rowids\"] == bytearray(int(1024 * 8))\n # vchunk = db.execute(\"select * from vec_xyz_vector_chunks00\").fetchone()\n # assert vchunk[\"rowid\"] == 1\n # assert vchunk[\"vectors\"] == bytearray(int(1024 * 4 * 2))\n\n assert re.match(\n \"SCAN (TABLE )?vec_xyz VIRTUAL TABLE INDEX 0:3{___}___\",\n explain_query_plan(\n \"select * from vec_xyz where a match X'' and k = 10 order by distance\"\n ),\n )\n if SUPPORTS_VTAB_LIMIT:\n assert re.match(\n \"SCAN (TABLE )?vec_xyz VIRTUAL TABLE INDEX 0:3{___}___\",\n explain_query_plan(\n \"select * from vec_xyz where a match X'' order by distance limit 10\"\n ),\n )\n assert re.match(\n \"SCAN (TABLE )?vec_xyz VIRTUAL TABLE INDEX 0:1\",\n explain_query_plan(\"select * from vec_xyz\"),\n )\n assert re.match(\n \"SCAN (TABLE )?vec_xyz VIRTUAL TABLE INDEX 3:2\",\n explain_query_plan(\"select * from vec_xyz where rowid = 4\"),\n )\n\n db.execute(\"insert into vec_xyz(rowid, a) select 1, X'000000000000803f'\")\n chunk = db.execute(\"select * from vec_xyz_chunks\").fetchone()\n assert chunk[\"chunk_id\"] == 1\n assert chunk[\"validity\"] == b\"\\x01\" + bytearray(int(1024 / 8) - 1)\n assert chunk[\"rowids\"] == b\"\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\" + bytearray(\n int(1024 * 8) - 8\n )\n vchunk = db.execute(\"select * from vec_xyz_vector_chunks00\").fetchone()\n assert vchunk[\"rowid\"] == 1\n assert vchunk[\"vectors\"] == b\"\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x3f\" + bytearray(\n int(1024 * 4 * 2) - (2 * 4)\n )\n\n db.execute(\"insert into vec_xyz(rowid, a) select 2, X'0000000000000040'\")\n chunk = db.execute(\"select * from vec_xyz_chunks\").fetchone()\n assert (\n chunk[\"rowids\"]\n == b\"\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\"\n + b\"\\x02\\x00\\x00\\x00\\x00\\x00\\x00\\x00\"\n + bytearray(int(1024 * 8) - 8 * 2)\n )\n assert chunk[\"chunk_id\"] == 1\n assert chunk[\"validity\"] == b\"\\x03\" + bytearray(int(1024 / 8) - 1)\n vchunk = db.execute(\"select * from vec_xyz_vector_chunks00\").fetchone()\n assert vchunk[\"rowid\"] == 1\n assert (\n vchunk[\"vectors\"]\n == b\"\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x3f\"\n + b\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x40\"\n + bytearray(int(1024 * 4 * 2) - (2 * 4 * 2))\n )\n\n db.execute(\"insert into vec_xyz(rowid, a) select 3, X'00000000000080bf'\")\n chunk = db.execute(\"select * from vec_xyz_chunks\").fetchone()\n assert chunk[\"chunk_id\"] == 1\n assert chunk[\"validity\"] == b\"\\x07\" + bytearray(int(1024 / 8) - 1)\n assert (\n chunk[\"rowids\"]\n == b\"\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\"\n + b\"\\x02\\x00\\x00\\x00\\x00\\x00\\x00\\x00\"\n + b\"\\x03\\x00\\x00\\x00\\x00\\x00\\x00\\x00\"\n + bytearray(int(1024 * 8) - 8 * 3)\n )\n vchunk = db.execute(\"select * from vec_xyz_vector_chunks00\").fetchone()\n assert vchunk[\"rowid\"] == 1\n assert (\n vchunk[\"vectors\"]\n == b\"\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x3f\"\n + b\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x40\"\n + b\"\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\xbf\"\n + bytearray(int(1024 * 4 * 2) - (2 * 4 * 3))\n )\n\n # db.execute(\"select * from vec_xyz\")\n assert execute_all(db, \"select * from vec_xyz\") == [\n {\"rowid\": 1, \"a\": b\"\\x00\\x00\\x00\\x00\\x00\\x00\\x80?\"},\n {\"rowid\": 2, \"a\": b\"\\x00\\x00\\x00\\x00\\x00\\x00\\x00@\"},\n {\"rowid\": 3, \"a\": b\"\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\xbf\"},\n ]\n\n\ndef test_vec0_stress_small_chunks():\n data = np.zeros((1000, 8), dtype=np.float32)\n for i in range(1000):\n data[i] = np.array([(i + 1) * 0.1] * 8)\n db.execute(\"drop table if exists vec_small\")\n db.execute(\"create virtual table vec_small using vec0(chunk_size=8, a float[8])\")\n assert execute_all(db, \"select rowid, * from vec_small\") == []\n with db:\n for row in data:\n db.execute(\"insert into vec_small(a) values (?) \", [row])\n assert execute_all(db, \"select rowid, * from vec_small limit 8\") == [\n {\"rowid\": 1, \"a\": _f32([0.1] * 8)},\n {\"rowid\": 2, \"a\": _f32([0.2] * 8)},\n {\"rowid\": 3, \"a\": _f32([0.3] * 8)},\n {\"rowid\": 4, \"a\": _f32([0.4] * 8)},\n {\"rowid\": 5, \"a\": _f32([0.5] * 8)},\n {\"rowid\": 6, \"a\": _f32([0.6] * 8)},\n {\"rowid\": 7, \"a\": _f32([0.7] * 8)},\n {\"rowid\": 8, \"a\": _f32([0.8] * 8)},\n ]\n assert db.execute(\"select count(*) from vec_small\").fetchone()[0] == 1000\n assert execute_all(\n db, \"select rowid, * from vec_small order by rowid desc limit 8\"\n ) == [\n {\"rowid\": 1000, \"a\": _f32([100.0] * 8)},\n {\"rowid\": 999, \"a\": _f32([99.9] * 8)},\n {\"rowid\": 998, \"a\": _f32([99.8] * 8)},\n {\"rowid\": 997, \"a\": _f32([99.7] * 8)},\n {\"rowid\": 996, \"a\": _f32([99.6] * 8)},\n {\"rowid\": 995, \"a\": _f32([99.5] * 8)},\n {\"rowid\": 994, \"a\": _f32([99.4] * 8)},\n {\"rowid\": 993, \"a\": _f32([99.3] * 8)},\n ]\n assert execute_all(\n db,\n \"\"\"\n select rowid, a, distance\n from vec_small\n where a match ?\n and k = 9\n order by distance\n \"\"\",\n [_f32([50.0] * 8)],\n ) == [\n {\n \"a\": _f32([500 * 0.1] * 8),\n \"distance\": 0.0,\n \"rowid\": 500,\n },\n {\n \"a\": _f32([501 * 0.1] * 8),\n \"distance\": 0.2828384041786194,\n \"rowid\": 501,\n },\n {\n \"a\": _f32([499 * 0.1] * 8),\n \"distance\": 0.2828384041786194,\n \"rowid\": 499,\n },\n {\n \"a\": _f32([502 * 0.1] * 8),\n \"distance\": 0.5656875967979431,\n \"rowid\": 502,\n },\n {\n \"a\": _f32([498 * 0.1] * 8),\n \"distance\": 0.5656875967979431,\n \"rowid\": 498,\n },\n {\n \"a\": _f32([503 * 0.1] * 8),\n \"distance\": 0.8485260009765625,\n \"rowid\": 503,\n },\n {\n \"a\": _f32([497 * 0.1] * 8),\n \"distance\": 0.8485260009765625,\n \"rowid\": 497,\n },\n {\n \"a\": _f32([496 * 0.1] * 8),\n \"distance\": 1.1313751935958862,\n \"rowid\": 496,\n },\n {\n \"a\": _f32([504 * 0.1] * 8),\n \"distance\": 1.1313751935958862,\n \"rowid\": 504,\n },\n ]\n\n\ndef test_vec0_distance_metric():\n base = \"('[1, 2]'), ('[3, 4]'), ('[5, 6]')\"\n q = \"[-1, -2]\"\n\n db = connect(EXT_PATH)\n db.execute(\"create virtual table v1 using vec0( a float[2])\")\n db.execute(f\"insert into v1(a) values {base}\")\n\n db.execute(\"create virtual table v2 using vec0( a float[2] distance_metric=l2)\")\n db.execute(f\"insert into v2(a) values {base}\")\n\n db.execute(\"create virtual table v3 using vec0( a float[2] distance_metric=l1)\")\n db.execute(f\"insert into v3(a) values {base}\")\n\n db.execute(\"create virtual table v4 using vec0( a float[2] distance_metric=cosine)\")\n db.execute(f\"insert into v4(a) values {base}\")\n\n # default (L2)\n assert execute_all(\n db, \"select rowid, distance from v1 where a match ? and k = 3\", [q]\n ) == [\n {\"rowid\": 1, \"distance\": 4.4721360206604},\n {\"rowid\": 2, \"distance\": 7.211102485656738},\n {\"rowid\": 3, \"distance\": 10.0},\n ]\n\n # l2\n assert execute_all(\n db, \"select rowid, distance from v2 where a match ? and k = 3\", [q]\n ) == [\n {\"rowid\": 1, \"distance\": 4.4721360206604},\n {\"rowid\": 2, \"distance\": 7.211102485656738},\n {\"rowid\": 3, \"distance\": 10.0},\n ]\n # l1\n assert execute_all(\n db, \"select rowid, distance from v3 where a match ? and k = 3\", [q]\n ) == [\n {\"rowid\": 1, \"distance\": 6},\n {\"rowid\": 2, \"distance\": 10},\n {\"rowid\": 3, \"distance\": 14},\n ]\n # consine\n assert execute_all(\n db, \"select rowid, distance from v4 where a match ? and k = 3\", [q]\n ) == [\n {\"rowid\": 3, \"distance\": 1.9734171628952026},\n {\"rowid\": 2, \"distance\": 1.9838699102401733},\n {\"rowid\": 1, \"distance\": 2},\n ]\n\n\ndef test_vec0_vacuum():\n db = connect(EXT_PATH)\n db.execute(\"create virtual table vec_t using vec0(a float[1]);\")\n db.execute(\"begin\")\n db.execute(\"insert into vec_t(a) values (X'AABBCCDD')\")\n db.commit()\n db.execute(\"vacuum\")\n\n\ndef rowids_value(buffer: bytearray) -> List[int]:\n assert (len(buffer) % 8) == 0\n n = int(len(buffer) / 8)\n return list(struct.unpack_from(f\"<{n}q\", buffer))\n\n\nimport numpy.typing as npt\n\n\ndef cosine_similarity(\n vec: npt.NDArray[np.float32], mat: npt.NDArray[np.float32], do_norm: bool = True\n) -> npt.NDArray[np.float32]:\n sim = vec @ mat.T\n if do_norm:\n sim /= np.linalg.norm(vec) * np.linalg.norm(mat, axis=1)\n return sim\n\n\ndef topk(\n vec: npt.NDArray[np.float32],\n mat: npt.NDArray[np.float32],\n k: int = 5,\n do_norm: bool = True,\n) -> tuple[npt.NDArray[np.int32], npt.NDArray[np.float32]]:\n sim = cosine_similarity(vec, mat, do_norm=do_norm)\n # Rather than sorting all similarities and taking the top K, it's faster to\n # argpartition and then just sort the top K.\n # The difference is O(N logN) vs O(N + k logk)\n indices = np.argpartition(-sim, kth=k)[:k]\n top_indices = np.argsort(-sim[indices])\n return indices[top_indices], sim[top_indices]\n\n\ndef test_stress1():\n np.random.seed(1234)\n data = np.random.uniform(-1.0, 1.0, (8000, 128)).astype(np.float32)\n db.execute(\"drop table if exists vec_stress1\")\n db.execute(\n \"create virtual table vec_stress1 using vec0( a float[128] distance_metric=cosine)\"\n )\n with db:\n for idx, row in enumerate(data):\n db.execute(\"insert into vec_stress1 values (?, ?)\", [idx, row])\n queries = np.random.uniform(-1.0, 1.0, (100, 128)).astype(np.float32)\n for q in queries:\n ids, distances = topk(q, data, k=10)\n rows = db.execute(\n \"\"\"\n select rowid, distance\n from vec_stress1\n where a match ? and k = ?\n order by distance\n \"\"\",\n [q, 10],\n ).fetchall()\n assert len(ids) == 10\n assert len(rows) == 10\n vec_ids = [row[0] for row in rows]\n assert ids.tolist() == vec_ids\n\n\n@pytest.mark.skip(reason=\"slow\")\ndef test_stress():\n db.execute(\"create virtual table vec_t1 using vec0( a float[1536])\")\n\n def rand_vec(n):\n return struct.pack(\"%sf\" % n, *list(map(lambda x: random(), range(n))))\n\n for i in range(1025):\n db.execute(\"insert into vec_t1(a) values (?)\", [rand_vec(1536)])\n rows = db.execute(\"select validity, rowids from vec_t1_chunks\").fetchall()\n assert len(rows) == 2\n\n assert len(rows[0][\"validity\"]) == 1024 / CHAR_BIT\n assert len(rows[0][\"rowids\"]) == 1024 * CHAR_BIT\n assert rows[0][\"validity\"] == bitmap_full(1024)\n assert rowids_value(rows[0][\"rowids\"]) == [x + 1 for x in range(1024)]\n\n assert len(rows[1][\"validity\"]) == 1024 / CHAR_BIT\n assert len(rows[1][\"rowids\"]) == 1024 * CHAR_BIT\n assert rows[1][\"validity\"] == bytes([0b0000_0001]) + bitmap_zerod(1024)[1:]\n assert rowids_value(rows[1][\"rowids\"])[0] == 1025\n\n\ndef test_coverage():\n current_module = inspect.getmodule(inspect.currentframe())\n test_methods = [\n member[0]\n for member in inspect.getmembers(current_module)\n if member[0].startswith(\"test_\")\n ]\n funcs_with_tests = set([x.replace(\"test_\", \"\") for x in test_methods])\n for func in [*FUNCTIONS, *MODULES]:\n assert func in funcs_with_tests, f\"{func} is not tested\"\n\n\nif __name__ == \"__main__\":\n unittest.main()\n" }, { "path": "tests/test-metadata.py", "content": "import pytest\nimport sqlite3\nfrom collections import OrderedDict\nimport json\nfrom helpers import exec, vec0_shadow_table_contents\n\n\ndef test_constructor_limit(db, snapshot):\n assert exec(\n db,\n f\"\"\"\n create virtual table v using vec0(\n {\",\".join([f\"metadata{x} integer\" for x in range(17)])}\n v float[1]\n )\n \"\"\",\n ) == snapshot(name=\"max 16 metadata columns\")\n\n\ndef test_normal(db, snapshot):\n db.execute(\n \"create virtual table v using vec0(vector float[1], b boolean, n int, f float, t text, chunk_size=8)\"\n )\n assert exec(\n db, \"select * from sqlite_master where type = 'table' order by name\"\n ) == snapshot(name=\"sqlite_master\")\n\n assert vec0_shadow_table_contents(db, \"v\") == snapshot()\n\n INSERT = \"insert into v(vector, b, n, f, t) values (?, ?, ?, ?, ?)\"\n assert exec(db, INSERT, [b\"\\x11\\x11\\x11\\x11\", 1, 1, 1.1, \"one\"]) == snapshot()\n assert exec(db, INSERT, [b\"\\x22\\x22\\x22\\x22\", 1, 2, 2.2, \"two\"]) == snapshot()\n assert exec(db, INSERT, [b\"\\x33\\x33\\x33\\x33\", 1, 3, 3.3, \"three\"]) == snapshot()\n\n assert exec(db, \"select * from v\") == snapshot()\n assert vec0_shadow_table_contents(db, \"v\") == snapshot()\n\n assert exec(db, \"drop table v\") == snapshot()\n assert exec(db, \"select * from sqlite_master\") == snapshot()\n\n\n#\n# assert exec(db, \"select * from v\") == snapshot()\n# assert vec0_shadow_table_contents(db, \"v\") == snapshot()\n#\n# db.execute(\"drop table v;\")\n# assert exec(db, \"select * from sqlite_master order by name\") == snapshot(\n# name=\"sqlite_master post drop\"\n# )\n\n\ndef test_text_knn(db, snapshot):\n db.execute(\n \"create virtual table v using vec0(vector float[1], name text, chunk_size=8)\"\n )\n assert vec0_shadow_table_contents(db, \"v\") == snapshot()\n INSERT = \"insert into v(vector, name) values (?, ?)\"\n db.execute(\n \"\"\"\n INSERT INTO v(vector, name) VALUES\n ('[.11]', 'aaa'),\n ('[.22]', 'bbb'),\n ('[.33]', 'ccc'),\n ('[.44]', 'ddd'),\n ('[.55]', 'eee'),\n ('[.66]', 'fff'),\n ('[.77]', 'ggg'),\n ('[.88]', 'hhh'),\n ('[.99]', 'iii');\n \"\"\"\n )\n assert exec(db, \"select * from v\") == snapshot()\n assert vec0_shadow_table_contents(db, \"v\") == snapshot()\n\n assert (\n exec(\n db,\n \"select rowid, name, distance from v where vector match '[1]' and k = 5\",\n )\n == snapshot()\n )\n\n assert (\n exec(\n db,\n \"select rowid, name, distance from v where vector match '[1]' and k = 5 and name < 'ddd'\",\n )\n == snapshot()\n )\n assert (\n exec(\n db,\n \"select rowid, name, distance from v where vector match '[1]' and k = 5 and name <= 'ddd'\",\n )\n == snapshot()\n )\n assert (\n exec(\n db,\n \"select rowid, name, distance from v where vector match '[1]' and k = 5 and name > 'fff'\",\n )\n == snapshot()\n )\n assert (\n exec(\n db,\n \"select rowid, name, distance from v where vector match '[1]' and k = 5 and name >= 'fff'\",\n )\n == snapshot()\n )\n assert (\n exec(\n db,\n \"select rowid, name, distance from v where vector match '[1]' and k = 5 and name = 'aaa'\",\n )\n == snapshot()\n )\n assert (\n exec(\n db,\n \"select rowid, name, distance from v where vector match '[.01]' and k = 5 and name != 'aaa'\",\n )\n == snapshot()\n )\n\n\ndef test_long_text_updates(db, snapshot):\n db.execute(\n \"create virtual table v using vec0(vector float[1], name text, chunk_size=8)\"\n )\n assert vec0_shadow_table_contents(db, \"v\") == snapshot()\n INSERT = \"insert into v(vector, name) values (?, ?)\"\n exec(db, INSERT, [b\"\\x11\\x11\\x11\\x11\", \"123456789a12\"])\n exec(db, INSERT, [b\"\\x11\\x11\\x11\\x11\", \"123456789a123\"])\n assert exec(db, \"select * from v\") == snapshot()\n assert vec0_shadow_table_contents(db, \"v\") == snapshot()\n\n\ndef test_long_text_knn(db, snapshot):\n db.execute(\n \"create virtual table v using vec0(vector float[1], name text, chunk_size=8)\"\n )\n INSERT = \"insert into v(vector, name) values (?, ?)\"\n exec(db, INSERT, [\"[1]\", \"aaaa\"])\n exec(db, INSERT, [\"[2]\", \"aaaaaaaaaaaa_aaa\"])\n exec(db, INSERT, [\"[3]\", \"bbbb\"])\n exec(db, INSERT, [\"[4]\", \"bbbbbbbbbbbb_bbb\"])\n exec(db, INSERT, [\"[5]\", \"cccc\"])\n exec(db, INSERT, [\"[6]\", \"cccccccccccc_ccc\"])\n\n tests = [\n \"bbbb\",\n \"bb\",\n \"bbbbbb\",\n \"bbbbbbbbbbbb_bbb\",\n \"bbbbbbbbbbbb_aaa\",\n \"bbbbbbbbbbbb_ccc\",\n \"longlonglonglonglonglonglong\",\n ]\n ops = [\"=\", \"!=\", \"<\", \"<=\", \">\", \">=\"]\n op_names = [\"eq\", \"ne\", \"lt\", \"le\", \"gt\", \"ge\"]\n\n for test in tests:\n for op, op_name in zip(ops, op_names):\n assert exec(\n db,\n f\"select rowid, name, distance from v where vector match '[100]' and k = 5 and name {op} ?\",\n [test],\n ) == snapshot(name=f\"{op_name}-{test}\")\n\n\ndef test_types(db, snapshot):\n db.execute(\n \"create virtual table v using vec0(vector float[1], b boolean, n int, f float, t text, chunk_size=8)\"\n )\n INSERT = \"insert into v(vector, b, n, f, t) values (?, ?, ?, ?, ?)\"\n\n assert exec(db, INSERT, [b\"\\x11\\x11\\x11\\x11\", 1, 1, 1.1, \"test\"]) == snapshot(\n name=\"legal\"\n )\n\n # fmt: off\n assert exec(db, INSERT, [b\"\\x11\\x11\\x11\\x11\", 'illegal', 1, 1.1, 'test']) == snapshot(name=\"illegal-type-boolean\")\n assert exec(db, INSERT, [b\"\\x11\\x11\\x11\\x11\", 1, 'illegal', 1.1, 'test']) == snapshot(name=\"illegal-type-int\")\n assert exec(db, INSERT, [b\"\\x11\\x11\\x11\\x11\", 1, 1, 'illegal', 'test']) == snapshot(name=\"illegal-type-float\")\n assert exec(db, INSERT, [b\"\\x11\\x11\\x11\\x11\", 1, 1, 1.1, 420]) == snapshot(name=\"illegal-type-text\")\n # fmt: on\n\n assert exec(db, INSERT, [b\"\\x11\\x11\\x11\\x11\", 44, 1, 1.1, \"test\"]) == snapshot(\n name=\"illegal-boolean\"\n )\n\n\ndef test_updates(db, snapshot):\n db.execute(\n \"create virtual table v using vec0(vector float[1], b boolean, n int, f float, t text, chunk_size=8)\"\n )\n INSERT = \"insert into v(rowid, vector, b, n, f, t) values (?, ?, ?, ?, ?, ?)\"\n\n exec(db, INSERT, [1, b\"\\x11\\x11\\x11\\x11\", 1, 1, 1.1, \"test1\"])\n exec(db, INSERT, [2, b\"\\x22\\x22\\x22\\x22\", 1, 2, 2.2, \"test2\"])\n exec(db, INSERT, [3, b\"\\x33\\x33\\x33\\x33\", 1, 3, 3.3, \"1234567890123\"])\n assert exec(db, \"select * from v\") == snapshot(name=\"1-init-contents\")\n assert vec0_shadow_table_contents(db, \"v\") == snapshot(name=\"1-init-shadow\")\n\n assert exec(\n db, \"UPDATE v SET b = 0, n = 11, f = 11.11, t = 'newtest1' where rowid = 1\"\n )\n assert exec(db, \"select * from v\") == snapshot(name=\"general-update-contents\")\n assert vec0_shadow_table_contents(db, \"v\") == snapshot(\n name=\"general-update-shaodnw\"\n )\n\n # string update #1: long string updated to long string\n exec(db, \"UPDATE v SET t = '1234567890123-updated' where rowid = 3\")\n assert exec(db, \"select * from v\") == snapshot(name=\"string-update-1-contents\")\n assert vec0_shadow_table_contents(db, \"v\") == snapshot(\n name=\"string-update-1-shadow\"\n )\n\n # string update #2: short string updated to short string\n exec(db, \"UPDATE v SET t = 'test2-short' where rowid = 2\")\n assert exec(db, \"select * from v\") == snapshot(name=\"string-update-2-contents\")\n assert vec0_shadow_table_contents(db, \"v\") == snapshot(\n name=\"string-update-2-shadow\"\n )\n\n # string update #3: short string updated to long string\n exec(db, \"UPDATE v SET t = 'test2-long-long-long' where rowid = 2\")\n assert exec(db, \"select * from v\") == snapshot(name=\"string-update-3-contents\")\n assert vec0_shadow_table_contents(db, \"v\") == snapshot(\n name=\"string-update-3-shadow\"\n )\n\n # string update #4: long string updated to short string\n exec(db, \"UPDATE v SET t = 'test2-shortx' where rowid = 2\")\n assert exec(db, \"select * from v\") == snapshot(name=\"string-update-4-contents\")\n assert vec0_shadow_table_contents(db, \"v\") == snapshot(\n name=\"string-update-4-shadow\"\n )\n\n\ndef test_deletes(db, snapshot):\n db.execute(\n \"create virtual table v using vec0(vector float[1], b boolean, n int, f float, t text, chunk_size=8)\"\n )\n INSERT = \"insert into v(rowid, vector, b, n, f, t) values (?, ?, ?, ?, ?, ?)\"\n\n assert exec(db, INSERT, [1, b\"\\x11\\x11\\x11\\x11\", 1, 1, 1.1, \"test1\"]) == snapshot()\n assert exec(db, INSERT, [2, b\"\\x22\\x22\\x22\\x22\", 1, 2, 2.2, \"test2\"]) == snapshot()\n assert (\n exec(db, INSERT, [3, b\"\\x33\\x33\\x33\\x33\", 1, 3, 3.3, \"1234567890123\"])\n == snapshot()\n )\n\n assert exec(db, \"select * from v\") == snapshot()\n assert vec0_shadow_table_contents(db, \"v\") == snapshot()\n\n assert exec(db, \"DELETE FROM v where rowid = 1\") == snapshot()\n assert exec(db, \"select * from v\") == snapshot()\n assert vec0_shadow_table_contents(db, \"v\") == snapshot()\n\n assert exec(db, \"DELETE FROM v where rowid = 3\") == snapshot()\n assert exec(db, \"select * from v\") == snapshot()\n assert vec0_shadow_table_contents(db, \"v\") == snapshot()\n\n\ndef test_knn(db, snapshot):\n db.execute(\n \"create virtual table v using vec0(vector float[1], name text, chunk_size=8)\"\n )\n assert exec(\n db, \"select * from sqlite_master where type = 'table' order by name\"\n ) == snapshot(name=\"sqlite_master\")\n db.executemany(\n \"insert into v(vector, name) values (?, ?)\",\n [(\"[1]\", \"alex\"), (\"[2]\", \"brian\"), (\"[3]\", \"craig\")],\n )\n\n # EVIDENCE-OF: V16511_00582 catches \"illegal\" constraints on metadata columns\n assert (\n exec(\n db,\n \"select *, distance from v where vector match '[5]' and k = 3 and name like 'illegal'\",\n )\n == snapshot()\n )\n\n\nSUPPORTS_VTAB_IN = sqlite3.sqlite_version_info[1] >= 38\n\n\n@pytest.mark.skipif(\n not SUPPORTS_VTAB_IN, reason=\"requires vtab `x in (...)` support in SQLite >=3.38\"\n)\ndef test_vtab_in(db, snapshot):\n db.execute(\n \"create virtual table v using vec0(vector float[1], n int, t text, b boolean, f float, chunk_size=8)\"\n )\n db.executemany(\n \"insert into v(rowid, vector, n, t, b, f) values (?, ?, ?, ?, ?, ?)\",\n [\n (1, \"[1]\", 999, \"aaaa\", 0, 1.1),\n (2, \"[2]\", 555, \"aaaa\", 0, 1.1),\n (3, \"[3]\", 999, \"aaaa\", 0, 1.1),\n (4, \"[4]\", 555, \"aaaa\", 0, 1.1),\n (5, \"[5]\", 999, \"zzzz\", 0, 1.1),\n (6, \"[6]\", 555, \"zzzz\", 0, 1.1),\n (7, \"[7]\", 999, \"zzzz\", 0, 1.1),\n (8, \"[8]\", 555, \"zzzz\", 0, 1.1),\n ],\n )\n\n # EVIDENCE-OF: V15248_32086\n assert exec(\n db, \"select * from v where vector match '[0]' and k = 8 and b in (1, 0)\"\n ) == snapshot(name=\"block-bool\")\n\n assert exec(\n db, \"select * from v where vector match '[0]' and k = 8 and f in (1.1, 0.0)\"\n ) == snapshot(name=\"block-float\")\n\n assert exec(\n db,\n \"select rowid, n, distance from v where vector match '[0]' and k = 8 and n in (555, 999)\",\n ) == snapshot(name=\"allow-int-all\")\n assert exec(\n db,\n \"select rowid, n, distance from v where vector match '[0]' and k = 8 and n in (555, -1, -2)\",\n ) == snapshot(name=\"allow-int-superfluous\")\n\n assert exec(\n db,\n \"select rowid, t, distance from v where vector match '[0]' and k = 8 and t in ('aaaa', 'zzzz')\",\n ) == snapshot(name=\"allow-text-all\")\n assert exec(\n db,\n \"select rowid, t, distance from v where vector match '[0]' and k = 8 and t in ('aaaa', 'foo', 'bar')\",\n ) == snapshot(name=\"allow-text-superfluous\")\n\n\ndef test_vtab_in_long_text(db, snapshot):\n db.execute(\n \"create virtual table v using vec0(vector float[1], t text, chunk_size=8)\"\n )\n data = [\n (1, \"aaaa\"),\n (2, \"aaaaaaaaaaaa_aaa\"),\n (3, \"bbbb\"),\n (4, \"bbbbbbbbbbbb_bbb\"),\n (5, \"cccc\"),\n (6, \"cccccccccccc_ccc\"),\n ]\n db.executemany(\n \"insert into v(rowid, vector, t) values (:rowid, printf('[%d]', :rowid), :vector)\",\n [{\"rowid\": row[0], \"vector\": row[1]} for row in data],\n )\n\n for _, lookup in data:\n assert exec(\n db,\n \"select rowid, t from v where vector match '[0]' and k = 10 and t in (?, 'nonsense')\",\n [lookup],\n ) == snapshot(name=f\"individual-{lookup}\")\n\n assert exec(\n db,\n \"select rowid, t from v where vector match '[0]' and k = 10 and t in (select value from json_each(?))\",\n [json.dumps([row[1] for row in data])],\n ) == snapshot(name=\"all\")\n\n\ndef test_idxstr(db, snapshot):\n db.execute(\n \"\"\"\n create virtual table vec_movies using vec0(\n movie_id integer primary key,\n synopsis_embedding float[1],\n +title text,\n is_favorited boolean,\n genre text,\n num_reviews int,\n mean_rating float,\n chunk_size=8\n );\n \"\"\"\n )\n\n assert (\n eqp(\n db,\n \"select * from vec_movies where synopsis_embedding match '' and k = 0 and is_favorited = true\",\n )\n == snapshot()\n )\n\n ops = [\"<\", \">\", \"<=\", \">=\", \"!=\"]\n\n for op in ops:\n assert eqp(\n db,\n f\"select * from vec_movies where synopsis_embedding match '' and k = 0 and genre {op} NULL\",\n ) == snapshot(name=f\"knn-constraint-text {op}\")\n\n for op in ops:\n assert eqp(\n db,\n f\"select * from vec_movies where synopsis_embedding match '' and k = 0 and num_reviews {op} NULL\",\n ) == snapshot(name=f\"knn-constraint-int {op}\")\n\n for op in ops:\n assert eqp(\n db,\n f\"select * from vec_movies where synopsis_embedding match '' and k = 0 and mean_rating {op} NULL\",\n ) == snapshot(name=f\"knn-constraint-float {op}\")\n\n # for op in ops:\n # assert eqp(\n # db,\n # f\"select * from vec_movies where synopsis_embedding match '' and k = 0 and is_favorited {op} NULL\",\n # ) == snapshot(name=f\"knn-constraint-boolean {op}\")\n\n\ndef eqp(db, sql):\n o = OrderedDict()\n o[\"sql\"] = sql\n o[\"plan\"] = [\n dict(row) for row in db.execute(f\"explain query plan {sql}\").fetchall()\n ]\n for p in o[\"plan\"]:\n # value is different on macos-aarch64 in github actions, not sure why\n del p[\"notused\"]\n return o\n\n\ndef test_stress(db, snapshot):\n db.execute(\n \"\"\"\n create virtual table vec_movies using vec0(\n movie_id integer primary key,\n synopsis_embedding float[1],\n +title text,\n is_favorited boolean,\n genre text,\n num_reviews int,\n mean_rating float,\n chunk_size=8\n );\n \"\"\"\n )\n\n db.execute(\n \"\"\"\n INSERT INTO vec_movies(movie_id, synopsis_embedding, is_favorited, genre, title, num_reviews, mean_rating)\n VALUES\n (1, '[1]', 0, 'horror', 'The Conjuring', 153, 4.6),\n (2, '[2]', 0, 'comedy', 'Dumb and Dumber', 382, 2.6),\n (3, '[3]', 0, 'scifi', 'Interstellar', 53, 5.0),\n (4, '[4]', 0, 'fantasy', 'The Lord of the Rings: The Fellowship of the Ring', 210, 4.2),\n (5, '[5]', 1, 'documentary', 'An Inconvenient Truth', 93, 3.4),\n (6, '[6]', 1, 'horror', 'Hereditary', 167, 4.7),\n (7, '[7]', 1, 'comedy', 'Anchorman: The Legend of Ron Burgundy', 482, 2.9),\n (8, '[8]', 0, 'scifi', 'Blade Runner 2049', 301, 5.0),\n (9, '[9]', 1, 'fantasy', 'Harry Potter and the Sorcerer''s Stone', 134, 4.1),\n (10, '[10]', 0, 'documentary', 'Free Solo', 66, 3.2),\n (11, '[11]', 1, 'horror', 'Get Out', 88, 4.9),\n (12, '[12]', 0, 'comedy', 'The Hangover', 59, 2.8),\n (13, '[13]', 1, 'scifi', 'The Matrix', 423, 4.5),\n (14, '[14]', 0, 'fantasy', 'Pan''s Labyrinth', 275, 3.6),\n (15, '[15]', 1, 'documentary', '13th', 191, 4.4),\n (16, '[16]', 0, 'horror', 'It Follows', 314, 4.3),\n (17, '[17]', 1, 'comedy', 'Step Brothers', 74, 3.0),\n (18, '[18]', 1, 'scifi', 'Inception', 201, 5.0),\n (19, '[19]', 1, 'fantasy', 'The Shape of Water', 399, 2.7),\n (20, '[20]', 1, 'documentary', 'Won''t You Be My Neighbor?', 186, 4.8),\n (21, '[21]', 1, 'scifi', 'Gravity', 342, 4.0),\n (22, '[22]', 1, 'scifi', 'Dune', 451, 4.4),\n (23, '[23]', 1, 'scifi', 'The Martian', 522, 4.6),\n (24, '[24]', 1, 'horror', 'A Quiet Place', 271, 4.3),\n (25, '[25]', 1, 'fantasy', 'The Chronicles of Narnia: The Lion, the Witch and the Wardrobe', 310, 3.9);\n\n \"\"\"\n )\n\n assert vec0_shadow_table_contents(db, \"vec_movies\") == snapshot()\n assert (\n exec(\n db,\n \"\"\"\n select\n movie_id,\n title,\n genre,\n num_reviews,\n mean_rating,\n is_favorited,\n distance\n from vec_movies\n where synopsis_embedding match '[15.5]'\n and genre = 'scifi'\n and num_reviews between 100 and 500\n and mean_rating > 3.5\n and k = 5;\n \"\"\",\n )\n == snapshot()\n )\n\n assert (\n exec(\n db,\n \"select movie_id, genre, distance from vec_movies where synopsis_embedding match '[100]' and k = 5 and genre = 'horror'\",\n )\n == snapshot()\n )\n assert (\n exec(\n db,\n \"select movie_id, genre, distance from vec_movies where synopsis_embedding match '[100]' and k = 5 and genre = 'comedy'\",\n )\n == snapshot()\n )\n assert (\n exec(\n db,\n \"select movie_id, num_reviews, distance from vec_movies where synopsis_embedding match '[100]' and k = 5 and num_reviews between 100 and 500\",\n )\n == snapshot()\n )\n assert (\n exec(\n db,\n \"select movie_id, num_reviews, distance from vec_movies where synopsis_embedding match '[100]' and k = 5 and num_reviews >= 500\",\n )\n == snapshot()\n )\n assert (\n exec(\n db,\n \"select movie_id, mean_rating, distance from vec_movies where synopsis_embedding match '[100]' and k = 5 and mean_rating < 3.0\",\n )\n == snapshot()\n )\n assert (\n exec(\n db,\n \"select movie_id, mean_rating, distance from vec_movies where synopsis_embedding match '[100]' and k = 5 and mean_rating between 4.0 and 5.0\",\n )\n == snapshot()\n )\n\n assert exec(\n db,\n \"select movie_id, is_favorited, distance from vec_movies where synopsis_embedding match '[100]' and k = 5 and is_favorited = TRUE\",\n ) == snapshot(name=\"bool-eq-true\")\n assert exec(\n db,\n \"select movie_id, is_favorited, distance from vec_movies where synopsis_embedding match '[100]' and k = 5 and is_favorited != TRUE\",\n ) == snapshot(name=\"bool-ne-true\")\n assert exec(\n db,\n \"select movie_id, is_favorited, distance from vec_movies where synopsis_embedding match '[100]' and k = 5 and is_favorited = FALSE\",\n ) == snapshot(name=\"bool-eq-false\")\n assert exec(\n db,\n \"select movie_id, is_favorited, distance from vec_movies where synopsis_embedding match '[100]' and k = 5 and is_favorited != FALSE\",\n ) == snapshot(name=\"bool-ne-false\")\n\n # EVIDENCE-OF: V10145_26984\n assert exec(\n db,\n \"select movie_id, is_favorited, distance from vec_movies where synopsis_embedding match '[100]' and k = 5 and is_favorited >= 999\",\n ) == snapshot(name=\"bool-other-op\")\n\n\ndef test_errors(db, snapshot):\n db.execute(\"create virtual table v using vec0(vector float[1], t text)\")\n db.execute(\"insert into v(vector, t) values ('[1]', 'aaaaaaaaaaaax')\")\n\n assert exec(db, \"select * from v\") == snapshot()\n\n # EVIDENCE-OF: V15466_32305\n db.set_authorizer(\n authorizer_deny_on(sqlite3.SQLITE_READ, \"v_metadatatext00\", \"data\")\n )\n assert exec(db, \"select * from v\") == snapshot()\n\n\ndef authorizer_deny_on(operation, x1, x2=None):\n def _auth(op, p1, p2, p3, p4):\n if op == operation and p1 == x1 and p2 == x2:\n return sqlite3.SQLITE_DENY\n return sqlite3.SQLITE_OK\n\n return _auth\n\n\n" }, { "path": "tests/test-partition-keys.py", "content": "import sqlite3\nfrom helpers import exec, vec0_shadow_table_contents\n\n\ndef test_constructor_limit(db, snapshot):\n assert exec(\n db,\n \"\"\"\n create virtual table v using vec0(\n p1 int partition key,\n p2 int partition key,\n p3 int partition key,\n p4 int partition key,\n p5 int partition key,\n v float[1]\n )\n \"\"\",\n ) == snapshot(name=\"max 4 partition keys\")\n\n\ndef test_normal(db, snapshot):\n db.execute(\n \"create virtual table v using vec0(p1 int partition key, a float[1], chunk_size=8)\"\n )\n\n db.execute(\"insert into v(rowid, p1, a) values (1, 100, X'11223344')\")\n assert vec0_shadow_table_contents(db, \"v\") == snapshot(name=\"1 row\")\n db.execute(\"insert into v(rowid, p1, a) values (2, 100, X'44556677')\")\n assert vec0_shadow_table_contents(db, \"v\") == snapshot(name=\"2 rows, same parition\")\n db.execute(\"insert into v(rowid, p1, a) values (3, 200, X'8899aabb')\")\n assert vec0_shadow_table_contents(db, \"v\") == snapshot(name=\"3 rows, 2 partitions\")\n\n\ndef test_types(db, snapshot):\n db.execute(\n \"create virtual table v using vec0(p1 int partition key, a float[1], chunk_size=8)\"\n )\n\n # EVIDENCE-OF: V11454_28292\n assert exec(\n db, \"insert into v(p1, a) values(?, ?)\", [\"not int\", b\"\\x11\\x22\\x33\\x44\"]\n ) == snapshot(name=\"1. raises type error\")\n\n assert vec0_shadow_table_contents(db, \"v\") == snapshot(name=\"2. empty DB\")\n\n # but allow NULLs\n assert exec(\n db, \"insert into v(p1, a) values(?, ?)\", [None, b\"\\x11\\x22\\x33\\x44\"]\n ) == snapshot(name=\"3. allow nulls\")\n\n assert vec0_shadow_table_contents(db, \"v\") == snapshot(\n name=\"4. show NULL partition key\"\n )\n\n\ndef test_updates(db, snapshot):\n db.execute(\n \"create virtual table v using vec0(p text partition key, a float[1], chunk_size=8)\"\n )\n\n db.execute(\n \"insert into v(rowid, p, a) values (?, ?, ?)\", [1, \"a\", b\"\\x11\\x11\\x11\\x11\"]\n )\n db.execute(\n \"insert into v(rowid, p, a) values (?, ?, ?)\", [2, \"a\", b\"\\x22\\x22\\x22\\x22\"]\n )\n db.execute(\n \"insert into v(rowid, p, a) values (?, ?, ?)\", [3, \"a\", b\"\\x33\\x33\\x33\\x33\"]\n )\n\n assert exec(db, \"select * from v\") == snapshot(name=\"1. Initial dataset\")\n assert exec(db, \"update v set p = ? where rowid = ?\", [\"new\", 1]) == snapshot(\n name=\"2. update #1\"\n )\n\n\nclass Row:\n def __init__(self):\n pass\n\n def __repr__(self) -> str:\n return repr()\n\n\n" }, { "path": "tests/test-unit.c", "content": "#include \"../sqlite-vec.h\"\n#include \"sqlite-vec-internal.h\"\n#include \n#include \n#include \n#include \n\n#define countof(x) (sizeof(x) / sizeof((x)[0]))\n\n// Tests vec0_token_next(), the low-level tokenizer that extracts the next\n// token from a raw char range. Covers every token type (identifier, digit,\n// brackets, plus, equals), whitespace skipping, EOF on empty/whitespace-only\n// input, error on unrecognised characters, and boundary behaviour where\n// identifiers and digits stop at the next non-matching character.\nvoid test_vec0_token_next() {\n printf(\"Starting %s...\\n\", __func__);\n struct Vec0Token token;\n int rc;\n char *input;\n\n // Single-character tokens\n input = \"+\";\n rc = vec0_token_next(input, input + 1, &token);\n assert(rc == VEC0_TOKEN_RESULT_SOME);\n assert(token.token_type == TOKEN_TYPE_PLUS);\n\n input = \"[\";\n rc = vec0_token_next(input, input + 1, &token);\n assert(rc == VEC0_TOKEN_RESULT_SOME);\n assert(token.token_type == TOKEN_TYPE_LBRACKET);\n\n input = \"]\";\n rc = vec0_token_next(input, input + 1, &token);\n assert(rc == VEC0_TOKEN_RESULT_SOME);\n assert(token.token_type == TOKEN_TYPE_RBRACKET);\n\n input = \"=\";\n rc = vec0_token_next(input, input + 1, &token);\n assert(rc == VEC0_TOKEN_RESULT_SOME);\n assert(token.token_type == TOKEN_TYPE_EQ);\n\n // Identifier\n input = \"hello\";\n rc = vec0_token_next(input, input + 5, &token);\n assert(rc == VEC0_TOKEN_RESULT_SOME);\n assert(token.token_type == TOKEN_TYPE_IDENTIFIER);\n assert(token.start == input);\n assert(token.end == input + 5);\n\n // Identifier with underscores and digits\n input = \"col_1a\";\n rc = vec0_token_next(input, input + 6, &token);\n assert(rc == VEC0_TOKEN_RESULT_SOME);\n assert(token.token_type == TOKEN_TYPE_IDENTIFIER);\n assert(token.end - token.start == 6);\n\n // Digit sequence\n input = \"1234\";\n rc = vec0_token_next(input, input + 4, &token);\n assert(rc == VEC0_TOKEN_RESULT_SOME);\n assert(token.token_type == TOKEN_TYPE_DIGIT);\n assert(token.start == input);\n assert(token.end == input + 4);\n\n // Leading whitespace is skipped\n input = \" abc\";\n rc = vec0_token_next(input, input + 5, &token);\n assert(rc == VEC0_TOKEN_RESULT_SOME);\n assert(token.token_type == TOKEN_TYPE_IDENTIFIER);\n assert(token.end - token.start == 3);\n\n // Tab/newline whitespace\n input = \"\\t\\n\\r X\";\n rc = vec0_token_next(input, input + 5, &token);\n assert(rc == VEC0_TOKEN_RESULT_SOME);\n assert(token.token_type == TOKEN_TYPE_IDENTIFIER);\n\n // Empty input\n input = \"\";\n rc = vec0_token_next(input, input, &token);\n assert(rc == VEC0_TOKEN_RESULT_EOF);\n\n // Only whitespace\n input = \" \";\n rc = vec0_token_next(input, input + 3, &token);\n assert(rc == VEC0_TOKEN_RESULT_EOF);\n\n // Unrecognized character\n input = \"@\";\n rc = vec0_token_next(input, input + 1, &token);\n assert(rc == VEC0_TOKEN_RESULT_ERROR);\n\n input = \"!\";\n rc = vec0_token_next(input, input + 1, &token);\n assert(rc == VEC0_TOKEN_RESULT_ERROR);\n\n // Identifier stops at bracket\n input = \"foo[\";\n rc = vec0_token_next(input, input + 4, &token);\n assert(rc == VEC0_TOKEN_RESULT_SOME);\n assert(token.token_type == TOKEN_TYPE_IDENTIFIER);\n assert(token.end - token.start == 3);\n\n // Digit stops at non-digit\n input = \"42abc\";\n rc = vec0_token_next(input, input + 5, &token);\n assert(rc == VEC0_TOKEN_RESULT_SOME);\n assert(token.token_type == TOKEN_TYPE_DIGIT);\n assert(token.end - token.start == 2);\n\n // Left paren\n input = \"(\";\n rc = vec0_token_next(input, input + 1, &token);\n assert(rc == VEC0_TOKEN_RESULT_SOME);\n assert(token.token_type == TOKEN_TYPE_LPAREN);\n\n // Right paren\n input = \")\";\n rc = vec0_token_next(input, input + 1, &token);\n assert(rc == VEC0_TOKEN_RESULT_SOME);\n assert(token.token_type == TOKEN_TYPE_RPAREN);\n\n // Comma\n input = \",\";\n rc = vec0_token_next(input, input + 1, &token);\n assert(rc == VEC0_TOKEN_RESULT_SOME);\n assert(token.token_type == TOKEN_TYPE_COMMA);\n\n printf(\" All vec0_token_next tests passed.\\n\");\n}\n\n// Tests Vec0Scanner, the stateful wrapper around vec0_token_next() that\n// tracks position and yields successive tokens. Verifies correct tokenisation\n// of full sequences like \"abc float[128]\" and \"key=value\", empty input,\n// whitespace-heavy input, and expressions with operators (\"a+b\").\nvoid test_vec0_scanner() {\n printf(\"Starting %s...\\n\", __func__);\n struct Vec0Scanner scanner;\n struct Vec0Token token;\n int rc;\n\n // Scan \"abc float[128]\"\n {\n const char *input = \"abc float[128]\";\n vec0_scanner_init(&scanner, input, (int)strlen(input));\n\n rc = vec0_scanner_next(&scanner, &token);\n assert(rc == VEC0_TOKEN_RESULT_SOME);\n assert(token.token_type == TOKEN_TYPE_IDENTIFIER);\n assert(token.end - token.start == 3);\n assert(strncmp(token.start, \"abc\", 3) == 0);\n\n rc = vec0_scanner_next(&scanner, &token);\n assert(rc == VEC0_TOKEN_RESULT_SOME);\n assert(token.token_type == TOKEN_TYPE_IDENTIFIER);\n assert(token.end - token.start == 5);\n assert(strncmp(token.start, \"float\", 5) == 0);\n\n rc = vec0_scanner_next(&scanner, &token);\n assert(rc == VEC0_TOKEN_RESULT_SOME);\n assert(token.token_type == TOKEN_TYPE_LBRACKET);\n\n rc = vec0_scanner_next(&scanner, &token);\n assert(rc == VEC0_TOKEN_RESULT_SOME);\n assert(token.token_type == TOKEN_TYPE_DIGIT);\n assert(strncmp(token.start, \"128\", 3) == 0);\n\n rc = vec0_scanner_next(&scanner, &token);\n assert(rc == VEC0_TOKEN_RESULT_SOME);\n assert(token.token_type == TOKEN_TYPE_RBRACKET);\n\n rc = vec0_scanner_next(&scanner, &token);\n assert(rc == VEC0_TOKEN_RESULT_EOF);\n }\n\n // Scan \"key=value\"\n {\n const char *input = \"key=value\";\n vec0_scanner_init(&scanner, input, (int)strlen(input));\n\n rc = vec0_scanner_next(&scanner, &token);\n assert(rc == VEC0_TOKEN_RESULT_SOME);\n assert(token.token_type == TOKEN_TYPE_IDENTIFIER);\n assert(strncmp(token.start, \"key\", 3) == 0);\n\n rc = vec0_scanner_next(&scanner, &token);\n assert(rc == VEC0_TOKEN_RESULT_SOME);\n assert(token.token_type == TOKEN_TYPE_EQ);\n\n rc = vec0_scanner_next(&scanner, &token);\n assert(rc == VEC0_TOKEN_RESULT_SOME);\n assert(token.token_type == TOKEN_TYPE_IDENTIFIER);\n assert(strncmp(token.start, \"value\", 5) == 0);\n\n rc = vec0_scanner_next(&scanner, &token);\n assert(rc == VEC0_TOKEN_RESULT_EOF);\n }\n\n // Scan empty string\n {\n const char *input = \"\";\n vec0_scanner_init(&scanner, input, 0);\n\n rc = vec0_scanner_next(&scanner, &token);\n assert(rc == VEC0_TOKEN_RESULT_EOF);\n }\n\n // Scan with lots of whitespace\n {\n const char *input = \" a b \";\n vec0_scanner_init(&scanner, input, (int)strlen(input));\n\n rc = vec0_scanner_next(&scanner, &token);\n assert(rc == VEC0_TOKEN_RESULT_SOME);\n assert(token.token_type == TOKEN_TYPE_IDENTIFIER);\n assert(token.end - token.start == 1);\n assert(*token.start == 'a');\n\n rc = vec0_scanner_next(&scanner, &token);\n assert(rc == VEC0_TOKEN_RESULT_SOME);\n assert(token.token_type == TOKEN_TYPE_IDENTIFIER);\n assert(token.end - token.start == 1);\n assert(*token.start == 'b');\n\n rc = vec0_scanner_next(&scanner, &token);\n assert(rc == VEC0_TOKEN_RESULT_EOF);\n }\n\n // Scan \"a+b\"\n {\n const char *input = \"a+b\";\n vec0_scanner_init(&scanner, input, (int)strlen(input));\n\n rc = vec0_scanner_next(&scanner, &token);\n assert(rc == VEC0_TOKEN_RESULT_SOME);\n assert(token.token_type == TOKEN_TYPE_IDENTIFIER);\n\n rc = vec0_scanner_next(&scanner, &token);\n assert(rc == VEC0_TOKEN_RESULT_SOME);\n assert(token.token_type == TOKEN_TYPE_PLUS);\n\n rc = vec0_scanner_next(&scanner, &token);\n assert(rc == VEC0_TOKEN_RESULT_SOME);\n assert(token.token_type == TOKEN_TYPE_IDENTIFIER);\n\n rc = vec0_scanner_next(&scanner, &token);\n assert(rc == VEC0_TOKEN_RESULT_EOF);\n }\n\n // Scan \"diskann(k=v, k2=v2)\"\n {\n const char *input = \"diskann(k=v, k2=v2)\";\n vec0_scanner_init(&scanner, input, (int)strlen(input));\n\n rc = vec0_scanner_next(&scanner, &token);\n assert(rc == VEC0_TOKEN_RESULT_SOME);\n assert(token.token_type == TOKEN_TYPE_IDENTIFIER);\n assert(strncmp(token.start, \"diskann\", 7) == 0);\n\n rc = vec0_scanner_next(&scanner, &token);\n assert(rc == VEC0_TOKEN_RESULT_SOME);\n assert(token.token_type == TOKEN_TYPE_LPAREN);\n\n rc = vec0_scanner_next(&scanner, &token);\n assert(rc == VEC0_TOKEN_RESULT_SOME);\n assert(token.token_type == TOKEN_TYPE_IDENTIFIER);\n assert(strncmp(token.start, \"k\", 1) == 0);\n\n rc = vec0_scanner_next(&scanner, &token);\n assert(rc == VEC0_TOKEN_RESULT_SOME);\n assert(token.token_type == TOKEN_TYPE_EQ);\n\n rc = vec0_scanner_next(&scanner, &token);\n assert(rc == VEC0_TOKEN_RESULT_SOME);\n assert(token.token_type == TOKEN_TYPE_IDENTIFIER);\n assert(strncmp(token.start, \"v\", 1) == 0);\n\n rc = vec0_scanner_next(&scanner, &token);\n assert(rc == VEC0_TOKEN_RESULT_SOME);\n assert(token.token_type == TOKEN_TYPE_COMMA);\n\n rc = vec0_scanner_next(&scanner, &token);\n assert(rc == VEC0_TOKEN_RESULT_SOME);\n assert(token.token_type == TOKEN_TYPE_IDENTIFIER);\n assert(strncmp(token.start, \"k2\", 2) == 0);\n\n rc = vec0_scanner_next(&scanner, &token);\n assert(rc == VEC0_TOKEN_RESULT_SOME);\n assert(token.token_type == TOKEN_TYPE_EQ);\n\n rc = vec0_scanner_next(&scanner, &token);\n assert(rc == VEC0_TOKEN_RESULT_SOME);\n assert(token.token_type == TOKEN_TYPE_IDENTIFIER);\n assert(strncmp(token.start, \"v2\", 2) == 0);\n\n rc = vec0_scanner_next(&scanner, &token);\n assert(rc == VEC0_TOKEN_RESULT_SOME);\n assert(token.token_type == TOKEN_TYPE_RPAREN);\n\n rc = vec0_scanner_next(&scanner, &token);\n assert(rc == VEC0_TOKEN_RESULT_EOF);\n }\n\n printf(\" All vec0_scanner tests passed.\\n\");\n}\n\n// Tests vec0_parse_vector_column(), which parses a vec0 column definition\n// string like \"embedding float[768] distance_metric=cosine\" into a\n// VectorColumnDefinition struct. Covers all element types (float/f32, int8/i8,\n// bit), column names with underscores/digits, all distance metrics (L2, L1,\n// cosine), the default metric, and error cases: empty input, missing type,\n// unknown type, missing dimensions, unknown metric, unknown option key, and\n// distance_metric on bit columns.\nvoid test_vec0_parse_vector_column() {\n printf(\"Starting %s...\\n\", __func__);\n struct VectorColumnDefinition col;\n int rc;\n\n // Basic float column\n {\n const char *input = \"embedding float[768]\";\n rc = vec0_parse_vector_column(input, (int)strlen(input), &col);\n assert(rc == SQLITE_OK);\n assert(col.name_length == 9);\n assert(strncmp(col.name, \"embedding\", 9) == 0);\n assert(col.element_type == SQLITE_VEC_ELEMENT_TYPE_FLOAT32);\n assert(col.dimensions == 768);\n assert(col.distance_metric == VEC0_DISTANCE_METRIC_L2);\n sqlite3_free(col.name);\n }\n\n // f32 alias\n {\n const char *input = \"v f32[3]\";\n rc = vec0_parse_vector_column(input, (int)strlen(input), &col);\n assert(rc == SQLITE_OK);\n assert(col.element_type == SQLITE_VEC_ELEMENT_TYPE_FLOAT32);\n assert(col.dimensions == 3);\n sqlite3_free(col.name);\n }\n\n // int8 column\n {\n const char *input = \"quantized int8[256]\";\n rc = vec0_parse_vector_column(input, (int)strlen(input), &col);\n assert(rc == SQLITE_OK);\n assert(col.element_type == SQLITE_VEC_ELEMENT_TYPE_INT8);\n assert(col.dimensions == 256);\n assert(col.name_length == 9);\n assert(strncmp(col.name, \"quantized\", 9) == 0);\n sqlite3_free(col.name);\n }\n\n // i8 alias\n {\n const char *input = \"q i8[64]\";\n rc = vec0_parse_vector_column(input, (int)strlen(input), &col);\n assert(rc == SQLITE_OK);\n assert(col.element_type == SQLITE_VEC_ELEMENT_TYPE_INT8);\n assert(col.dimensions == 64);\n sqlite3_free(col.name);\n }\n\n // bit column\n {\n const char *input = \"bvec bit[1024]\";\n rc = vec0_parse_vector_column(input, (int)strlen(input), &col);\n assert(rc == SQLITE_OK);\n assert(col.element_type == SQLITE_VEC_ELEMENT_TYPE_BIT);\n assert(col.dimensions == 1024);\n sqlite3_free(col.name);\n }\n\n // Column name with underscores and digits\n {\n const char *input = \"col_name_2 float[10]\";\n rc = vec0_parse_vector_column(input, (int)strlen(input), &col);\n assert(rc == SQLITE_OK);\n assert(col.name_length == 10);\n assert(strncmp(col.name, \"col_name_2\", 10) == 0);\n sqlite3_free(col.name);\n }\n\n // distance_metric=cosine\n {\n const char *input = \"emb float[128] distance_metric=cosine\";\n rc = vec0_parse_vector_column(input, (int)strlen(input), &col);\n assert(rc == SQLITE_OK);\n assert(col.distance_metric == VEC0_DISTANCE_METRIC_COSINE);\n assert(col.dimensions == 128);\n sqlite3_free(col.name);\n }\n\n // distance_metric=L2 (explicit)\n {\n const char *input = \"emb float[128] distance_metric=L2\";\n rc = vec0_parse_vector_column(input, (int)strlen(input), &col);\n assert(rc == SQLITE_OK);\n assert(col.distance_metric == VEC0_DISTANCE_METRIC_L2);\n sqlite3_free(col.name);\n }\n\n // distance_metric=L1\n {\n const char *input = \"emb float[128] distance_metric=l1\";\n rc = vec0_parse_vector_column(input, (int)strlen(input), &col);\n assert(rc == SQLITE_OK);\n assert(col.distance_metric == VEC0_DISTANCE_METRIC_L1);\n sqlite3_free(col.name);\n }\n\n // SQLITE_EMPTY: empty string\n {\n const char *input = \"\";\n rc = vec0_parse_vector_column(input, 0, &col);\n assert(rc == SQLITE_EMPTY);\n }\n\n // SQLITE_EMPTY: non-vector column (text primary key)\n {\n const char *input = \"document_id text primary key\";\n rc = vec0_parse_vector_column(input, (int)strlen(input), &col);\n assert(rc == SQLITE_EMPTY);\n }\n\n // SQLITE_EMPTY: non-vector column (partition key)\n {\n const char *input = \"user_id integer partition key\";\n rc = vec0_parse_vector_column(input, (int)strlen(input), &col);\n assert(rc == SQLITE_EMPTY);\n }\n\n // SQLITE_EMPTY: no type (single identifier)\n {\n const char *input = \"emb\";\n rc = vec0_parse_vector_column(input, (int)strlen(input), &col);\n assert(rc == SQLITE_EMPTY);\n }\n\n // SQLITE_EMPTY: unknown type\n {\n const char *input = \"emb double[128]\";\n rc = vec0_parse_vector_column(input, (int)strlen(input), &col);\n assert(rc == SQLITE_EMPTY);\n }\n\n // SQLITE_EMPTY: unknown type (unknowntype)\n {\n const char *input = \"v unknowntype[128]\";\n rc = vec0_parse_vector_column(input, (int)strlen(input), &col);\n assert(rc == SQLITE_EMPTY);\n }\n\n // SQLITE_EMPTY: missing brackets entirely\n {\n const char *input = \"emb float\";\n rc = vec0_parse_vector_column(input, (int)strlen(input), &col);\n assert(rc == SQLITE_EMPTY);\n }\n\n // Error: zero dimensions\n {\n const char *input = \"v float[0]\";\n rc = vec0_parse_vector_column(input, (int)strlen(input), &col);\n assert(rc == SQLITE_ERROR);\n }\n\n // Error: empty brackets (no dimensions)\n {\n const char *input = \"v float[]\";\n rc = vec0_parse_vector_column(input, (int)strlen(input), &col);\n assert(rc == SQLITE_ERROR);\n }\n\n // Error: unknown distance metric\n {\n const char *input = \"emb float[128] distance_metric=hamming\";\n rc = vec0_parse_vector_column(input, (int)strlen(input), &col);\n assert(rc == SQLITE_ERROR);\n }\n\n // Error: unknown distance metric (foo)\n {\n const char *input = \"v float[128] distance_metric=foo\";\n rc = vec0_parse_vector_column(input, (int)strlen(input), &col);\n assert(rc == SQLITE_ERROR);\n }\n\n // Error: unknown option key\n {\n const char *input = \"emb float[128] foobar=baz\";\n rc = vec0_parse_vector_column(input, (int)strlen(input), &col);\n assert(rc == SQLITE_ERROR);\n }\n\n // Error: distance_metric on bit type\n {\n const char *input = \"emb bit[64] distance_metric=cosine\";\n rc = vec0_parse_vector_column(input, (int)strlen(input), &col);\n assert(rc == SQLITE_ERROR);\n }\n\n printf(\" All vec0_parse_vector_column tests passed.\\n\");\n}\n\n// Tests vec0_parse_partition_key_definition(), which parses a vec0 partition\n// key column definition like \"user_id integer partition key\". Verifies correct\n// parsing of integer and text partition keys, column name extraction, and\n// rejection of invalid inputs: empty strings, non-partition-key definitions\n// (\"primary key\"), and misspelled keywords.\nvoid test_vec0_parse_partition_key_definition() {\n printf(\"Starting %s...\\n\", __func__);\n typedef struct {\n char * test;\n int expected_rc;\n const char *expected_column_name;\n int expected_column_type;\n } TestCase;\n\n TestCase suite[] = {\n {\"user_id integer partition key\", SQLITE_OK, \"user_id\", SQLITE_INTEGER},\n {\"USER_id int partition key\", SQLITE_OK, \"USER_id\", SQLITE_INTEGER},\n {\"category text partition key\", SQLITE_OK, \"category\", SQLITE_TEXT},\n\n {\"\", SQLITE_EMPTY, \"\", 0},\n {\"document_id text primary key\", SQLITE_EMPTY, \"\", 0},\n {\"document_id text partition keyy\", SQLITE_EMPTY, \"\", 0},\n };\n for(int i = 0; i < countof(suite); i++) {\n char * out_column_name;\n int out_column_name_length;\n int out_column_type;\n int rc;\n rc = vec0_parse_partition_key_definition(\n suite[i].test,\n strlen(suite[i].test),\n &out_column_name,\n &out_column_name_length,\n &out_column_type\n );\n assert(rc == suite[i].expected_rc);\n\n if(rc == SQLITE_OK) {\n assert(out_column_name_length == strlen(suite[i].expected_column_name));\n assert(strncmp(out_column_name, suite[i].expected_column_name, out_column_name_length) == 0);\n assert(out_column_type == suite[i].expected_column_type);\n }\n\n printf(\" Passed: \\\"%s\\\"\\n\", suite[i].test);\n }\n}\n\nvoid test_distance_l2_sqr_float() {\n printf(\"Starting %s...\\n\", __func__);\n float d;\n\n // Identical vectors: distance = 0\n {\n float a[] = {1.0f, 2.0f, 3.0f};\n float b[] = {1.0f, 2.0f, 3.0f};\n d = _test_distance_l2_sqr_float(a, b, 3);\n assert(d == 0.0f);\n }\n\n // Orthogonal unit vectors: sqrt(1+1) = sqrt(2)\n {\n float a[] = {1.0f, 0.0f, 0.0f};\n float b[] = {0.0f, 1.0f, 0.0f};\n d = _test_distance_l2_sqr_float(a, b, 3);\n assert(fabsf(d - sqrtf(2.0f)) < 1e-6f);\n }\n\n // Known computation: [1,2,3] vs [4,5,6] = sqrt(9+9+9) = sqrt(27)\n {\n float a[] = {1.0f, 2.0f, 3.0f};\n float b[] = {4.0f, 5.0f, 6.0f};\n d = _test_distance_l2_sqr_float(a, b, 3);\n assert(fabsf(d - sqrtf(27.0f)) < 1e-5f);\n }\n\n // Single dimension: sqrt(16) = 4.0\n {\n float a[] = {3.0f};\n float b[] = {7.0f};\n d = _test_distance_l2_sqr_float(a, b, 1);\n assert(d == 4.0f);\n }\n\n printf(\" All distance_l2_sqr_float tests passed.\\n\");\n}\n\nvoid test_distance_cosine_float() {\n printf(\"Starting %s...\\n\", __func__);\n float d;\n\n // Identical direction: distance = 0.0\n {\n float a[] = {1.0f, 0.0f};\n float b[] = {2.0f, 0.0f};\n d = _test_distance_cosine_float(a, b, 2);\n assert(fabsf(d - 0.0f) < 1e-6f);\n }\n\n // Orthogonal: distance = 1.0\n {\n float a[] = {1.0f, 0.0f};\n float b[] = {0.0f, 1.0f};\n d = _test_distance_cosine_float(a, b, 2);\n assert(fabsf(d - 1.0f) < 1e-6f);\n }\n\n // Opposite direction: distance = 2.0\n {\n float a[] = {1.0f, 0.0f};\n float b[] = {-1.0f, 0.0f};\n d = _test_distance_cosine_float(a, b, 2);\n assert(fabsf(d - 2.0f) < 1e-6f);\n }\n\n printf(\" All distance_cosine_float tests passed.\\n\");\n}\n\nvoid test_distance_hamming() {\n printf(\"Starting %s...\\n\", __func__);\n float d;\n\n // Identical bitmaps: distance = 0\n {\n unsigned char a[] = {0xFF};\n unsigned char b[] = {0xFF};\n d = _test_distance_hamming(a, b, 8);\n assert(d == 0.0f);\n }\n\n // All different: distance = 8\n {\n unsigned char a[] = {0xFF};\n unsigned char b[] = {0x00};\n d = _test_distance_hamming(a, b, 8);\n assert(d == 8.0f);\n }\n\n // Half different: 0xFF vs 0x0F = 4 bits differ\n {\n unsigned char a[] = {0xFF};\n unsigned char b[] = {0x0F};\n d = _test_distance_hamming(a, b, 8);\n assert(d == 4.0f);\n }\n\n // Multi-byte: [0xFF, 0x00] vs [0x00, 0xFF] = 16 bits differ\n {\n unsigned char a[] = {0xFF, 0x00};\n unsigned char b[] = {0x00, 0xFF};\n d = _test_distance_hamming(a, b, 16);\n assert(d == 16.0f);\n }\n\n printf(\" All distance_hamming tests passed.\\n\");\n}\n\nint main() {\n printf(\"Starting unit tests...\\n\");\n#ifdef SQLITE_VEC_ENABLE_AVX\n printf(\"SQLITE_VEC_ENABLE_AVX=1\\n\");\n#endif\n#ifdef SQLITE_VEC_ENABLE_NEON\n printf(\"SQLITE_VEC_ENABLE_NEON=1\\n\");\n#endif\n#if !defined(SQLITE_VEC_ENABLE_AVX) && !defined(SQLITE_VEC_ENABLE_NEON)\n printf(\"SIMD: none\\n\");\n#endif\n test_vec0_token_next();\n test_vec0_scanner();\n test_vec0_parse_vector_column();\n test_vec0_parse_partition_key_definition();\n test_distance_l2_sqr_float();\n test_distance_cosine_float();\n test_distance_hamming();\n printf(\"All unit tests passed.\\n\");\n}\n" }, { "path": "tests/test-wasm.mjs", "content": "async function main() {\n const { default: init } = await import(\"../dist/.wasm/sqlite3.mjs\");\n const sqlite3 = await init();\n const vec_version = new sqlite3.oo1.DB(\":memory:\").selectValue(\n \"select vec_version()\",\n );\n console.log(vec_version);\n}\n\nmain();\n" }, { "path": "tests/unittest.rs", "content": "fn main() {\n println!(\"Hello, world!\");\n println!(\"{:?}\", _min_idx(vec![3.0, 2.0, 1.0, f32::MAX, f32::MAX, f32::MAX, f32::MAX, f32::MAX], 2));\n}\n\nfn _min_idx(distances: Vec, k: i32) -> Vec {\n let n = distances.len();\n assert!(n % 8 == 0, \"distances.len() must be a multiple of 8\");\n\n let mut out: Vec = vec![0; k as usize];\n let bitmap_bytes = n / 8;\n let mut candidates: Vec = vec![0xFF; bitmap_bytes];\n let mut b_taken: Vec = vec![0; bitmap_bytes];\n let mut k_used: i32 = 0;\n\n unsafe {\n min_idx(\n distances.as_ptr(),\n n as i32,\n candidates.as_mut_ptr(),\n out.as_mut_ptr(),\n k,\n b_taken.as_mut_ptr(),\n &mut k_used,\n );\n }\n out.truncate(k_used as usize);\n out\n}\n\nfn _merge_sorted_lists(\n a: &Vec,\n a_rowids: &Vec,\n b: &Vec,\n b_rowids: &Vec,\n b_top_idx: &Vec,\n n: usize,\n) -> (Vec, Vec) {\n let mut out_used: i64 = 0;\n let mut out: Vec = Vec::with_capacity(n);\n let mut out_rowids: Vec = Vec::with_capacity(n);\n unsafe {\n merge_sorted_lists(\n a.as_ptr().cast(),\n a_rowids.as_ptr().cast(),\n a.len() as i64,\n b.as_ptr().cast(),\n b_rowids.as_ptr().cast(),\n b_top_idx.as_ptr().cast(),\n b.len() as i64,\n out.as_ptr().cast(),\n out_rowids.as_ptr().cast(),\n n as i64,\n &mut out_used,\n );\n out.set_len(out_used as usize);\n out_rowids.set_len(out_used as usize);\n }\n\n (out_rowids, out)\n}\n\n#[link(name = \"sqlite-vec-internal\")]\nextern \"C\" {\n fn min_idx(\n distances: *const f32,\n n: i32,\n candidates: *mut u8,\n out: *mut i32,\n k: i32,\n b_taken: *mut u8,\n k_used: *mut i32,\n ) -> i32;\n\n fn merge_sorted_lists(\n a: *const f32,\n a_rowids: *const i64,\n a_length: i64,\n b: *const f32,\n b_rowids: *const i64,\n b_top_idx: *const i32,\n b_length: i64,\n out: *const f32,\n out_rowids: *const i64,\n out_length: i64,\n out_used: *mut i64,\n );\n}\n\n#[cfg(test)]\nmod tests {\n use super::*;\n\n #[test]\n fn test_basic() {\n let pad = |v: &[f32]| -> Vec {\n let mut r = v.to_vec();\n r.resize(8, f32::MAX);\n r\n };\n\n assert_eq!(_min_idx(pad(&[1.0, 2.0, 3.0]), 3), vec![0, 1, 2]);\n assert_eq!(_min_idx(pad(&[3.0, 2.0, 1.0]), 3), vec![2, 1, 0]);\n\n assert_eq!(_min_idx(pad(&[1.0, 2.0, 3.0]), 2), vec![0, 1]);\n assert_eq!(_min_idx(pad(&[3.0, 2.0, 1.0]), 2), vec![2, 1]);\n }\n\n #[test]\n fn test_merge_sorted_lists() {\n let a = &vec![0.01, 0.02, 0.03];\n let a_rowids = &vec![1, 2, 3];\n\n //let b = &vec![0.1, 0.2, 0.3, 0.4];\n //let b_rowids = &vec![4, 5, 6, 7];\n let b = &vec![0.4, 0.2, 0.3, 0.1];\n let b_rowids = &vec![7, 5, 6, 4];\n let b_top_idx = &vec![3, 1, 2, 0];\n\n assert_eq!(\n _merge_sorted_lists(a, a_rowids, b, b_rowids, b_top_idx, 0),\n (vec![], vec![])\n );\n assert_eq!(\n _merge_sorted_lists(a, a_rowids, b, b_rowids, b_top_idx, 1),\n (vec![1], vec![0.01])\n );\n assert_eq!(\n _merge_sorted_lists(a, a_rowids, b, b_rowids, b_top_idx, 2),\n (vec![1, 2], vec![0.01, 0.02])\n );\n assert_eq!(\n _merge_sorted_lists(a, a_rowids, b, b_rowids, b_top_idx, 3),\n (vec![1, 2, 3], vec![0.01, 0.02, 0.03])\n );\n assert_eq!(\n _merge_sorted_lists(a, a_rowids, b, b_rowids, b_top_idx, 4),\n (vec![1, 2, 3, 4], vec![0.01, 0.02, 0.03, 0.1])\n );\n assert_eq!(\n _merge_sorted_lists(a, a_rowids, b, b_rowids, b_top_idx, 5),\n (vec![1, 2, 3, 4, 5], vec![0.01, 0.02, 0.03, 0.1, 0.2])\n );\n assert_eq!(\n _merge_sorted_lists(a, a_rowids, b, b_rowids, b_top_idx, 6),\n (\n vec![1, 2, 3, 4, 5, 6],\n vec![0.01, 0.02, 0.03, 0.1, 0.2, 0.3]\n )\n );\n assert_eq!(\n _merge_sorted_lists(a, a_rowids, b, b_rowids, b_top_idx, 7),\n (\n vec![1, 2, 3, 4, 5, 6, 7],\n vec![0.01, 0.02, 0.03, 0.1, 0.2, 0.3, 0.4]\n )\n );\n\n assert_eq!(\n _merge_sorted_lists(a, a_rowids, b, b_rowids, b_top_idx, 8),\n (\n vec![1, 2, 3, 4, 5, 6, 7],\n vec![0.01, 0.02, 0.03, 0.1, 0.2, 0.3, 0.4]\n )\n );\n }\n /*\n #[test]\n fn test_merge_sorted_lists_empty() {\n let x = vec![0.1, 0.2, 0.3];\n let x_rowids = vec![666, 888, 777];\n assert_eq!(\n _merge_sorted_lists(&x, &x_rowids, &vec![], &vec![], 3),\n (vec![666, 888, 777], vec![0.1, 0.2, 0.3])\n );\n assert_eq!(\n _merge_sorted_lists(&vec![], &vec![], &x, &x_rowids, 3),\n (vec![666, 888, 777], vec![0.1, 0.2, 0.3])\n );\n assert_eq!(\n _merge_sorted_lists(&vec![], &vec![], &x, &x_rowids, 4),\n (vec![666, 888, 777], vec![0.1, 0.2, 0.3])\n );\n assert_eq!(\n _merge_sorted_lists(&vec![], &vec![], &x, &x_rowids, 2),\n (vec![666, 888], vec![0.1, 0.2])\n );\n }*/\n}\n" }, { "path": "tests/utils.py", "content": "import numpy as np\nfrom io import BytesIO\n\n\ndef to_npy(arr):\n buf = BytesIO()\n np.save(buf, arr)\n buf.seek(0)\n return buf.read()\n\n\nto_npy(np.array([[1.0, 2.0, 3.0], [2.0, 3.0, 4.0]], dtype=np.float32))\n\nprint(to_npy(np.array([[1.0, 2.0]], dtype=np.float32)))\nprint(to_npy(np.array([1.0, 2.0], dtype=np.float32)))\n\nto_npy(\n np.array(\n [np.zeros(10), np.zeros(10), np.zeros(10), np.zeros(10), np.zeros(10)],\n dtype=np.float32,\n )\n)\n" }, { "path": "tmp-static.py", "content": "import sqlite3\nimport numpy as np\n\ndb = sqlite3.connect(\":memory:\")\n\ndb.enable_load_extension(True)\ndb.load_extension(\"./dist/vec0\")\ndb.execute(\"select load_extension('./dist/vec0', 'sqlite3_vec_raw_init')\")\ndb.enable_load_extension(False)\n\nx = np.array([[0.1, 0.2, 0.3, 0.4], [0.9, 0.8, 0.7, 0.6]], dtype=np.float32)\ny = np.array([[0.2, 0.3], [0.9, 0.8], [0.6, 0.5]], dtype=np.float32)\nz = np.array(\n [\n [0.1, 0.1, 0.1, 0.1],\n [0.2, 0.2, 0.2, 0.2],\n [0.3, 0.3, 0.3, 0.3],\n [0.4, 0.4, 0.4, 0.4],\n [0.5, 0.5, 0.5, 0.5],\n ],\n dtype=np.float32,\n)\n\n\ndef register_np(array, name):\n ptr = array.__array_interface__[\"data\"][0]\n nvectors, dimensions = array.__array_interface__[\"shape\"]\n element_type = array.__array_interface__[\"typestr\"]\n\n assert element_type == \"