Showing preview only (2,979K chars total). Download the full file or copy to clipboard to get everything.
Repository: tobi/qmd
Branch: main
Commit: 2b8f329d7e44
Files: 111
Total size: 2.8 MB
Directory structure:
gitextract_5sd6wqxn/
├── .claude-plugin/
│ └── marketplace.json
├── .gitattributes
├── .github/
│ └── workflows/
│ ├── ci.yml
│ └── publish.yml
├── .gitignore
├── .pi/
│ └── settings.json
├── CHANGELOG.md
├── CLAUDE.md
├── LICENSE
├── README.md
├── bin/
│ └── qmd
├── docs/
│ └── SYNTAX.md
├── example-index.yml
├── finetune/
│ ├── .gitignore
│ ├── CLAUDE.md
│ ├── Justfile
│ ├── Modelfile
│ ├── README.md
│ ├── SCORING.md
│ ├── configs/
│ │ ├── accelerate_multi_gpu.yaml
│ │ ├── sft.yaml
│ │ └── sft_local.yaml
│ ├── convert_gguf.py
│ ├── convert_onnx.py
│ ├── data/
│ │ ├── qmd_expansion_balanced_deduped.jsonl
│ │ ├── qmd_expansion_diverse_addon.jsonl
│ │ ├── qmd_expansion_handcrafted.jsonl
│ │ ├── qmd_expansion_handcrafted_only.jsonl
│ │ ├── qmd_expansion_lex_phrases_negation.jsonl
│ │ ├── qmd_expansion_locations.jsonl
│ │ ├── qmd_expansion_people.jsonl
│ │ ├── qmd_expansion_personal_entities.jsonl
│ │ ├── qmd_expansion_short_nontech.jsonl
│ │ ├── qmd_expansion_sports.jsonl
│ │ ├── qmd_expansion_v3_structured.jsonl
│ │ └── qmd_only_sampled.jsonl
│ ├── dataset/
│ │ ├── analyze_data.py
│ │ ├── prepare_data.py
│ │ ├── schema.py
│ │ ├── score_data.py
│ │ └── validate_schema.py
│ ├── eval.py
│ ├── evals/
│ │ ├── .gitignore
│ │ └── queries.txt
│ ├── experiments/
│ │ ├── gepa/
│ │ │ ├── __init__.py
│ │ │ ├── best_prompt.txt
│ │ │ ├── best_prompt_glm.txt
│ │ │ ├── dspy_gepa.py
│ │ │ ├── example.py
│ │ │ ├── generate.py
│ │ │ ├── gepa_outputs.jsonl
│ │ │ ├── gepa_outputs_glm.jsonl
│ │ │ ├── model.json
│ │ │ ├── optimizer.py
│ │ │ └── score.py
│ │ ├── grpo/
│ │ │ ├── README.md
│ │ │ ├── grpo.py
│ │ │ └── grpo.yaml
│ │ └── lfm2/
│ │ ├── sft_lfm2.py
│ │ └── sft_lfm2.yaml
│ ├── jobs/
│ │ ├── eval.py
│ │ ├── eval_common.py
│ │ └── sft.py
│ ├── pyproject.toml
│ ├── reward.py
│ └── train.py
├── flake.nix
├── migrate-schema.ts
├── package.json
├── scripts/
│ ├── extract-changelog.sh
│ ├── install-hooks.sh
│ ├── pre-push
│ └── release.sh
├── skills/
│ ├── qmd/
│ │ ├── SKILL.md
│ │ └── references/
│ │ └── mcp-setup.md
│ └── release/
│ ├── SKILL.md
│ └── scripts/
│ └── install-hooks.sh
├── src/
│ ├── bench-rerank.ts
│ ├── cli/
│ │ ├── formatter.ts
│ │ └── qmd.ts
│ ├── collections.ts
│ ├── db.ts
│ ├── embedded-skills.ts
│ ├── index.ts
│ ├── llm.ts
│ ├── maintenance.ts
│ ├── mcp/
│ │ └── server.ts
│ ├── store.ts
│ └── test-preload.ts
├── test/
│ ├── Containerfile
│ ├── cli.test.ts
│ ├── collections-config.test.ts
│ ├── eval-bm25.test.ts
│ ├── eval-harness.ts
│ ├── eval.test.ts
│ ├── formatter.test.ts
│ ├── intent.test.ts
│ ├── launcher-detection.test.sh
│ ├── llm.test.ts
│ ├── mcp.test.ts
│ ├── multi-collection-filter.test.ts
│ ├── rrf-trace.test.ts
│ ├── sdk.test.ts
│ ├── smoke-install.sh
│ ├── store-paths.test.ts
│ ├── store.helpers.unit.test.ts
│ ├── store.test.ts
│ └── structured-search.test.ts
├── tsconfig.build.json
├── tsconfig.json
└── vitest.config.ts
================================================
FILE CONTENTS
================================================
================================================
FILE: .claude-plugin/marketplace.json
================================================
{
"name": "qmd",
"owner": {
"name": "tobi",
"email": "tobi@lutke.com"
},
"plugins": [
{
"name": "qmd",
"source": "./",
"description": "Search and retrieve documents from local markdown files.",
"version": "0.1.0",
"author": {
"name": "tobi",
"email": "tobi@lutke.com"
},
"repository": "https://github.com/tobi/qmd",
"license": "MIT",
"keywords": ["markdown", "search", "qmd"],
"skills": ["./skills/"],
"mcpServers": {
"qmd": {
"command": "qmd",
"args": ["mcp"]
}
}
}
]
}
================================================
FILE: .gitattributes
================================================
# Use bd merge for beads JSONL files
.beads/issues.jsonl merge=beads
================================================
FILE: .github/workflows/ci.yml
================================================
name: CI
on:
push:
branches: [main]
pull_request:
branches: [main]
jobs:
test-node:
name: Node ${{ matrix.node-version }} (${{ matrix.os }})
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest]
node-version: ["22", "23"]
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: ${{ matrix.node-version }}
- name: Install SQLite (Ubuntu)
if: runner.os == 'Linux'
run: sudo apt-get update && sudo apt-get install -y libsqlite3-dev
- name: Install SQLite (macOS)
if: runner.os == 'macOS'
run: brew install sqlite
- run: npm install
- name: Tests
run: npx vitest run --reporter=verbose --testTimeout 60000 test/
env:
CI: true
test-bun:
name: Bun (${{ matrix.os }})
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest]
steps:
- uses: actions/checkout@v4
- uses: oven-sh/setup-bun@v2
with:
bun-version: latest
- name: Install SQLite (Ubuntu)
if: runner.os == 'Linux'
run: sudo apt-get update && sudo apt-get install -y libsqlite3-dev
- name: Install SQLite (macOS)
if: runner.os == 'macOS'
run: brew install sqlite
- name: Verify lockfile is up-to-date
run: bun install --frozen-lockfile
- name: Tests
run: bun test --timeout 60000 --preload ./src/test-preload.ts test/
env:
CI: true
DYLD_LIBRARY_PATH: /opt/homebrew/opt/sqlite/lib
LD_LIBRARY_PATH: /usr/lib/x86_64-linux-gnu
================================================
FILE: .github/workflows/publish.yml
================================================
name: Publish
on:
push:
tags: ["v*"]
jobs:
publish:
runs-on: ubuntu-latest
permissions:
contents: write
id-token: write
steps:
- uses: actions/checkout@v4
- uses: oven-sh/setup-bun@v2
with:
bun-version: latest
- name: Install SQLite
run: sudo apt-get update && sudo apt-get install -y libsqlite3-dev
- name: Verify lockfile is up-to-date
run: bun install --frozen-lockfile
- run: bun test --timeout 60000 --preload ./src/test-preload.ts test/
env:
CI: true
LD_LIBRARY_PATH: /usr/lib/x86_64-linux-gnu
- uses: actions/setup-node@v4
with:
node-version: 22
registry-url: https://registry.npmjs.org
- run: npm run build
- run: npm publish --provenance --access public
env:
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
- name: Extract release notes
id: notes
run: |
VERSION="${GITHUB_REF_NAME#v}"
NOTES=$(./scripts/extract-changelog.sh "$VERSION")
# Write to file for gh release (avoids quoting issues)
echo "$NOTES" > /tmp/release-notes.md
- name: Create GitHub release
run: |
gh release create "$GITHUB_REF_NAME" \
--title "$GITHUB_REF_NAME" \
--notes-file /tmp/release-notes.md
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
================================================
FILE: .gitignore
================================================
node_modules/
dist/
package-lock.json
.npmrc
*.sqlite
.DS_Store
archive/
texts/
.cursor/
.github/copilot/
*.md
!README.md
!CLAUDE.md
!CHANGELOG.md
!skills/**/*.md
!finetune/*.md
!docs/*.md
finetune/outputs/
finetune/data/train/
.claude/
================================================
FILE: .pi/settings.json
================================================
{
"skills": ["skills"]
}
================================================
FILE: CHANGELOG.md
================================================
# Changelog
## [Unreleased]
### Fixes
- Sync stale `bun.lock` (`better-sqlite3` 11.x → 12.x). CI and release
script now use `--frozen-lockfile` to prevent recurrence. #386
(thanks @Mic92)
## [2.0.1] - 2026-03-10
### Changes
- `qmd skill install` copies the packaged QMD skill into
`~/.claude/commands/` for one-command setup. #355 (thanks @nibzard)
### Fixes
- Fix Qwen3-Embedding GGUF filename case — HuggingFace filenames are
case-sensitive, the lowercase variant returned 404. #349 (thanks @byheaven)
- Resolve symlinked global launcher path so `qmd` works correctly when
installed via `npm i -g`. #352 (thanks @nibzard)
## [2.0.0] - 2026-03-10
QMD 2.0 declares a stable library API. The SDK is now the primary interface —
the MCP server is a clean consumer of it, and the source is organized into
`src/cli/` and `src/mcp/`. Also: Node 25 support and a runtime-aware bin wrapper
for bun installs.
### Changes
- Stable SDK API with `QMDStore` interface — search, retrieval, collection/context
management, indexing, lifecycle
- Unified `search()`: pass `query` for auto-expansion or `queries` for
pre-expanded lex/vec/hyde — replaces the old query/search/structuredSearch split
- New `getDocumentBody()`, `getDefaultCollectionNames()`, `Maintenance` class
- MCP server rewritten as a clean SDK consumer — zero internal store access
- CLI and MCP organized into `src/cli/` and `src/mcp/` subdirectories
- Runtime-aware `bin/qmd` wrapper detects bun vs node to avoid ABI mismatches.
Closes #319
- `better-sqlite3` bumped to ^12.4.5 for Node 25 support. Closes #257
- Utility exports: `extractSnippet`, `addLineNumbers`, `DEFAULT_MULTI_GET_MAX_BYTES`
### Fixes
- Remove unused `import { resolve }` in store.ts that shadowed local export
## [1.1.6] - 2026-03-09
QMD can now be used as a library. `import { createStore } from '@tobilu/qmd'`
gives you the full search and indexing API — hybrid query, BM25, structured
search, collection/context management — without shelling out to the CLI.
### Changes
- **SDK / library mode**: `createStore({ dbPath, config })` returns a
`QMDStore` with `query()`, `search()`, `structuredSearch()`, `get()`,
`multiGet()`, and collection/context management methods. Supports inline
config (no files needed) or a YAML config path.
- **Package exports**: `package.json` now declares `main`, `types`, and
`exports` so bundlers and TypeScript resolve `@tobilu/qmd` correctly.
## [1.1.5] - 2026-03-07
Ambiguous queries like "performance" now produce dramatically better results
when the caller knows what they mean. The new `intent` parameter steers all
five pipeline stages — expansion, strong-signal bypass, chunk selection,
reranking, and snippet extraction — without searching on its own. Design and
original implementation by Ilya Grigorik (@vyalamar) in #180.
### Changes
- **Intent parameter**: optional `intent` string disambiguates queries across
the entire search pipeline. Available via CLI (`--intent` flag or `intent:`
line in query documents), MCP (`intent` field on the query tool), and
programmatic API. Adapted from PR #180 (thanks @vyalamar).
- **Query expansion**: when intent is provided, the expansion LLM prompt
includes `Query intent: {intent}`, matching the finetune training data
format for better-aligned expansions.
- **Reranking**: intent is prepended to the rerank query so Qwen3-Reranker
scores with domain context.
- **Chunk selection**: intent terms scored at 0.5× weight alongside query
terms (1.0×) when selecting the best chunk per document for reranking.
- **Snippet extraction**: intent terms scored at 0.3× weight to nudge
snippets toward intent-relevant lines without overriding query anchoring.
- **Strong-signal bypass disabled with intent**: when intent is provided, the
BM25 strong-signal shortcut is skipped — the obvious keyword match may not
be what the caller wants.
- **MCP instructions**: callers are now guided to provide `intent` on every
search call for disambiguation.
- **Query document syntax**: `intent:` recognized as a line type. At most one
per document, cannot appear alone. Grammar updated in `docs/SYNTAX.md`.
## [1.1.2] - 2026-03-07
13 community PRs merged. GPU initialization replaced with node-llama-cpp's
built-in `autoAttempt` — deleting ~220 lines of manual fallback code and
fixing GPU issues reported across 10+ PRs in one shot. Reranking is faster
through chunk deduplication and a parallelism cap that prevents VRAM
exhaustion.
### Changes
- **GPU init**: use node-llama-cpp's `build: "autoAttempt"` instead of manual
GPU backend detection. Automatically tries Metal/CUDA/Vulkan and falls back
gracefully. #310 (thanks @giladgd — the node-llama-cpp author)
- **Query `--explain`**: `qmd query --explain` exposes retrieval score traces
— backend scores, per-list RRF contributions, top-rank bonus, reranker
score, and final blended score. Works in JSON and CLI output. #242
(thanks @vyalamar)
- **Collection ignore patterns**: `ignore: ["Sessions/**", "*.tmp"]` in
collection config to exclude files from indexing. #304 (thanks @sebkouba)
- **Multilingual embeddings**: `QMD_EMBED_MODEL` env var lets you swap in
models like Qwen3-Embedding for non-English collections. #273 (thanks
@daocoding)
- **Configurable expansion context**: `QMD_EXPAND_CONTEXT_SIZE` env var
(default 2048) — previously used the model's full 40960-token window,
wasting VRAM. #313 (thanks @0xble)
- **`candidateLimit` exposed**: `-C` / `--candidate-limit` flag and MCP
parameter to tune how many candidates reach the reranker. #255 (thanks
@pandysp)
- **MCP multi-session**: HTTP transport now supports multiple concurrent
client sessions, each with its own server instance. #286 (thanks @joelev)
### Fixes
- **Reranking performance**: cap parallel rerank contexts at 4 to prevent
VRAM exhaustion on high-core machines. Deduplicate identical chunk texts
before reranking — same content from different files now shares a single
reranker call. Cache scores by content hash instead of file path.
- Deactivate stale docs when all files are removed from a collection and
`qmd update` is run. #312 (thanks @0xble)
- Handle emoji-only filenames (`🐘.md` → `1f418.md`) instead of crashing.
#308 (thanks @debugerman)
- Skip unreadable files during indexing (e.g. iCloud-evicted files returning
EAGAIN) instead of crashing. #253 (thanks @jimmynail)
- Suppress progress bar escape sequences when stderr is not a TTY. #230
(thanks @dgilperez)
- Emit format-appropriate empty output (`[]` for JSON, CSV header for CSV,
etc.) instead of plain text "No results." #228 (thanks @amsminn)
- Correct Windows sqlite-vec package name (`sqlite-vec-windows-x64`) and add
`sqlite-vec-linux-arm64`. #225 (thanks @ilepn)
- Fix claude plugin setup CLI commands in README. #311 (thanks @gi11es)
## [1.1.1] - 2026-03-06
### Fixes
- Reranker: truncate documents exceeding the 2048-token context window
instead of silently producing garbage scores. Long chunks (e.g. from
PDF ingestion) now get a fair ranking.
- Nix: add python3 and cctools to build dependencies. #214 (thanks
@pcasaretto)
## [1.1.0] - 2026-02-20
QMD now speaks in **query documents** — structured multi-line queries where every line is typed (`lex:`, `vec:`, `hyde:`), combining keyword precision with semantic recall. A single plain query still works exactly as before (it's treated as an implicit `expand:` and auto-expanded by the LLM). Lex now supports quoted phrases and negation (`"C++ performance" -sports -athlete`), making intent-aware disambiguation practical. The formal query grammar is documented in `docs/SYNTAX.md`.
The npm package now uses the standard `#!/usr/bin/env node` bin convention, replacing the custom bash wrapper. This fixes native module ABI mismatches when installed via bun and works on any platform with node >= 22 on PATH.
### Changes
- **Query document format**: multi-line queries with typed sub-queries (`lex:`, `vec:`, `hyde:`). Plain queries remain the default (`expand:` implicit, but not written inside the document). First sub-query gets 2× fusion weight — put your strongest signal first. Formal grammar in `docs/SYNTAX.md`.
- **Lex syntax**: full BM25 operator support. `"exact phrase"` for verbatim matching; `-term` and `-"phrase"` for exclusions. Essential for disambiguation when a term is overloaded across domains (e.g. `performance -sports -athlete`).
- **`expand:` shortcut**: send a single plain query (or start the document with `expand:` on its only line) to auto-expand via the local LLM. Query documents themselves are limited to `lex`, `vec`, and `hyde` lines.
- **MCP `query` tool** (renamed from `structured_search`): rewrote the tool description to fully teach AI agents the query document format, lex syntax, and combination strategy. Includes worked examples with intent-aware lex.
- **HTTP `/query` endpoint** (renamed from `/search`; `/search` kept as silent alias).
- **`collections` array filter**: filter by multiple collections in a single query (`collections: ["notes", "brain"]`). Removed the single `collection` string param — array only.
- **Collection `include`/`exclude`**: `includeByDefault: false` hides a collection from all queries unless explicitly named via `collections`. CLI: `qmd collection exclude <name>` / `qmd collection include <name>`.
- **Collection `update-cmd`**: attach a shell command that runs before every `qmd update` (e.g. `git stash && git pull --rebase --ff-only && git stash pop`). CLI: `qmd collection update-cmd <name> '<cmd>'`.
- **`qmd status` tips**: shows actionable tips when collections lack context descriptions or update commands.
- **`qmd collection` subcommands**: `show`, `update-cmd`, `include`, `exclude`. Bare `qmd collection` now prints help.
- **Packaging**: replaced custom bash wrapper with standard `#!/usr/bin/env node` shebang on `dist/qmd.js`. Fixes native module ABI mismatches when installed via bun, and works on any platform where node >= 22 is on PATH.
- **Removed MCP tools** `search`, `vector_search`, `deep_search` — all superseded by `query`.
- **Removed** `qmd context check` command.
- **CLI timing**: each LLM step (expand, embed, rerank) prints elapsed time inline (`Expanding query... (4.2s)`).
### Fixes
- `qmd collection list` shows `[excluded]` tag for collections with `includeByDefault: false`.
- Default searches now respect `includeByDefault` — excluded collections are skipped unless explicitly named.
- Fix main module detection when installed globally via npm/bun (symlink resolution).
## [1.0.7] - 2026-02-18
### Changes
- LLM: add LiquidAI LFM2-1.2B as an alternative base model for query
expansion fine-tuning. LFM2's hybrid architecture (convolutions + attention)
is 2x faster at decode/prefill vs standard transformers — good fit for
on-device inference.
- CLI: support multiple `-c` flags to search across several collections at
once (e.g. `qmd search -c notes -c journals "query"`). #191 (thanks
@openclaw)
### Fixes
- Return empty JSON array `[]` instead of no output when `--json` search
finds no results.
- Resolve relative paths passed to `--index` so they don't produce malformed
config entries.
- Respect `XDG_CONFIG_HOME` for collection config path instead of always
using `~/.config`. #190 (thanks @openclaw)
- CLI: empty-collection hint now shows the correct `collection add` command.
#200 (thanks @vincentkoc)
## [1.0.6] - 2026-02-16
### Changes
- CLI: `qmd status` now shows models with full HuggingFace links instead of
static names in `--help`. Model info is derived from the actual configured
URIs so it stays accurate if models change.
- Release tooling: pre-push hook handles non-interactive shells (CI, editors)
gracefully — warnings auto-proceed instead of hanging on a tty prompt.
Annotated tags now resolve correctly for CI checks.
## [1.0.5] - 2026-02-16
The npm package now ships compiled JavaScript instead of raw TypeScript,
removing the `tsx` runtime dependency. A new `/release` skill automates the
full release workflow with changelog validation and git hook enforcement.
### Changes
- Build: compile TypeScript to `dist/` via `tsc` so the npm package no longer
requires `tsx` at runtime. The `qmd` shell wrapper now runs `dist/qmd.js`
directly.
- Release tooling: new `/release` skill that manages the full release
lifecycle — validates changelog, installs git hooks, previews release notes,
and cuts the release. Auto-populates `[Unreleased]` from git history when
empty.
- Release tooling: `scripts/extract-changelog.sh` extracts cumulative notes
for the full minor series (e.g. 1.0.0 through 1.0.5) for GitHub releases.
Includes `[Unreleased]` content in previews.
- Release tooling: `scripts/release.sh` renames `[Unreleased]` to a versioned
heading and inserts a fresh empty `[Unreleased]` section automatically.
- Release tooling: pre-push git hook blocks `v*` tag pushes unless
`package.json` version matches the tag, a changelog entry exists, and CI
passed on GitHub.
- Publish workflow: GitHub Actions now builds TypeScript, creates a GitHub
release with cumulative notes extracted from the changelog, and publishes
to npm with provenance.
## [1.0.0] - 2026-02-15
QMD now runs on both Node.js and Bun, with up to 2.7x faster reranking
through parallel GPU contexts. GPU auto-detection replaces the unreliable
`gpu: "auto"` with explicit CUDA/Metal/Vulkan probing.
### Changes
- Runtime: support Node.js (>=22) alongside Bun via a cross-runtime SQLite
abstraction layer (`src/db.ts`). `bun:sqlite` on Bun, `better-sqlite3` on
Node. The `qmd` wrapper auto-detects a suitable Node.js install via PATH,
then falls back to mise, asdf, nvm, and Homebrew locations.
- Performance: parallel embedding & reranking via multiple LlamaContext
instances — up to 2.7x faster on multi-core machines.
- Performance: flash attention for ~20% less VRAM per reranking context,
enabling more parallel contexts on GPU.
- Performance: right-sized reranker context (40960 → 2048 tokens, 17x less
memory) since chunks are capped at ~900 tokens.
- Performance: adaptive parallelism — context count computed from available
VRAM (GPU) or CPU math cores rather than hardcoded.
- GPU: probe for CUDA, Metal, Vulkan explicitly at startup instead of
relying on node-llama-cpp's `gpu: "auto"`. `qmd status` shows device info.
- Tests: reorganized into flat `test/` directory with vitest for Node.js and
bun test for Bun. New `eval-bm25` and `store.helpers.unit` suites.
### Fixes
- Prevent VRAM waste from duplicate context creation during concurrent
`embedBatch` calls — initialization lock now covers the full path.
- Collection-aware FTS filtering so scoped keyword search actually restricts
results to the requested collection.
## [0.9.0] - 2026-02-15
First published release on npm as `@tobilu/qmd`. MCP HTTP transport with
daemon mode cuts warm query latency from ~16s to ~10s by keeping models
loaded between requests.
### Changes
- MCP: HTTP transport with daemon lifecycle — `qmd mcp --http --daemon`
starts a background server, `qmd mcp stop` shuts it down. Models stay warm
in VRAM between queries. #149 (thanks @igrigorik)
- Search: type-routed query expansion preserves lex/vec/hyde type info and
routes to the appropriate backend. Eliminates ~4 wasted backend calls per
query (10.0 → 6.0 calls, 1278ms → 549ms). #149 (thanks @igrigorik)
- Search: unified pipeline — extracted `hybridQuery()` and
`vectorSearchQuery()` to `store.ts` so CLI and MCP share identical logic.
Fixes a class of bugs where results differed between the two. #149 (thanks
@igrigorik)
- MCP: dynamic instructions generated at startup from actual index state —
LLMs see collection names, doc counts, and content descriptions. #149
(thanks @igrigorik)
- MCP: tool renames (vsearch → vector_search, query → deep_search) with
rewritten descriptions for better tool selection. #149 (thanks @igrigorik)
- Integration: Claude Code plugin with inline status checks and MCP
integration. #99 (thanks @galligan)
### Fixes
- BM25 score normalization — formula was inverted (`1/(1+|x|)` instead of
`|x|/(1+|x|)`), so strong matches scored *lowest*. Broke `--min-score`
filtering and made the "strong signal" short-circuit dead code. #76 (thanks
@dgilperez)
- Normalize Unicode paths to NFC for macOS compatibility. #82 (thanks
@c-stoeckl)
- Handle dense content (code) that tokenizes beyond expected chunk size.
- Proper cleanup of Metal GPU resources on process exit.
- SQLite-vec readiness verification after extension load.
- Reactivate deactivated documents on re-index instead of creating duplicates.
- Bun UTF-8 path corruption workaround for non-ASCII filenames.
- Disable following symlinks in glob.scan to avoid infinite loops.
## [0.8.0] - 2026-01-28
Fine-tuned query expansion model trained with GRPO replaces the stock Qwen3
0.6B. The training pipeline scores expansions on named entity preservation,
format compliance, and diversity — producing noticeably better lexical
variations and HyDE documents.
### Changes
- LLM: deploy GRPO-trained (Group Relative Policy Optimization) query
expansion model, hosted on HuggingFace and auto-downloaded on first use.
Better preservation of proper nouns and technical terms in expansions.
- LLM: `/only:lex` mode for single-type expansions — useful when you know
which search backend will help.
- LLM: HyDE output moved to first position so vector search can start
embedding while other expansions generate.
- LLM: session lifecycle management via `withLLMSession()` pattern — ensures
cleanup even on failure, similar to database transactions.
- Integration: org-mode title extraction support. #50 (thanks @sh54)
- Integration: SQLite extension loading in Nix devshell. #48 (thanks @sh54)
- Integration: AI agent discovery via skills.sh. #64 (thanks @Algiras)
### Fixes
- Use sequential embedding on CPU-only systems — parallel contexts caused a
race condition where contexts competed for CPU cores, making things slower.
#54 (thanks @freeman-jiang)
- Fix `collectionName` column in vector search SQL (was still using old
`collectionId` from before YAML migration). #61 (thanks @jdvmi00)
- Fix Qwen3 sampling params to prevent repetition loops — stock
temperature/top-p caused occasional infinite repeat patterns.
- Add `--index` option to CLI argument parser (was documented but not wired
up). #84 (thanks @Tritlo)
- Fix DisposedError during slow batch embedding. #41 (thanks @wuhup)
## [0.7.0] - 2026-01-09
First community contributions. The project gained external contributors,
surfacing bugs that only appear in diverse environments — Homebrew sqlite-vec
paths, case-sensitive model filenames, and sqlite-vec JOIN incompatibilities.
### Changes
- Indexing: native `realpathSync()` replaces `readlink -f` subprocess spawn
per file. On a 5000-file collection this eliminates 5000 shell spawns,
~15% faster. #8 (thanks @burke)
- Indexing: single-pass tokenization — chunking algorithm tokenized each
document twice (count then split); now tokenizes once and reuses. #9
(thanks @burke)
### Fixes
- Fix `vsearch` and `query` hanging — sqlite-vec's virtual table doesn't
support the JOIN pattern used; rewrote to subquery. #23 (thanks @mbrendan)
- Fix MCP server exiting immediately after startup — process had no active
handles keeping the event loop alive. #29 (thanks @mostlydev)
- Fix collection filter SQL to properly restrict vector search results.
- Support non-ASCII filenames in collection filter.
- Skip empty files during indexing instead of crashing on zero-length content.
- Fix case sensitivity in Qwen3 model filename resolution. #15 (thanks
@gavrix)
- Fix sqlite-vec loading on macOS with Homebrew (`BREW_PREFIX` detection).
#42 (thanks @komsit37)
- Fix Nix flake to use correct `src/qmd.ts` path. #7 (thanks @burke)
- Fix docid lookup with quotes support in get command. #36 (thanks
@JoshuaLelon)
- Fix query expansion model size in documentation. #38 (thanks @odysseus0)
## [0.6.0] - 2025-12-28
Replaced Ollama HTTP API with node-llama-cpp for all LLM operations. Ollama
adds convenience but also a running server dependency. node-llama-cpp loads
GGUF models directly in-process — zero external dependencies. Models
auto-download from HuggingFace on first use.
### Changes
- LLM: structured query expansion via JSON schema grammar constraints.
Model produces typed expansions — **lexical** (BM25 keywords), **vector**
(semantic rephrasings), **HyDE** (hypothetical document excerpts) — so each
routes to the right backend instead of sending everything everywhere.
- LLM: lazy model loading with 2-minute inactivity auto-unload. Keeps memory
low when idle while avoiding ~3s model load on every query.
- Search: conditional query expansion — when BM25 returns strong results, the
expensive LLM expansion is skipped entirely.
- Search: multi-chunk reranking — documents with multiple relevant chunks
scored by aggregating across all chunks rather than best single chunk.
- Search: cosine distance for vector search (was L2).
- Search: embeddinggemma nomic-style prompt formatting.
- Testing: evaluation harness with synthetic test documents and Hit@K metrics
for BM25, vector, and hybrid RRF.
## [0.5.0] - 2025-12-13
Collections and contexts moved from SQLite tables to YAML at
`~/.config/qmd/index.yml`. SQLite was overkill for config — you can't share
it, and it's opaque. YAML is human-readable and version-controllable. The
migration was extensive (35+ commits) because every part of the system that
touched collections or contexts had to be updated.
### Changes
- Config: YAML-based collections and contexts replace SQLite tables.
`collections` and `path_contexts` tables dropped from schema. Collections
support an optional `update:` command (e.g., `git pull`) before re-index.
- CLI: `qmd collection add/list/remove/rename` commands with `--name` and
`--mask` glob pattern support.
- CLI: `qmd ls` virtual file tree — list collections, files in a collection,
or files under a path prefix.
- CLI: `qmd context add/list/check/rm` with hierarchical context inheritance.
A query to `qmd://notes/2024/jan/` inherits context from `notes/`,
`notes/2024/`, and `notes/2024/jan/`.
- CLI: `qmd context add / "text"` for global context across all collections.
- CLI: `qmd context check` audit command to find paths without context.
- Paths: `qmd://` virtual URI scheme for portable document references.
`qmd://notes/ideas.md` works regardless of where the collection lives on
disk. Works in `get`, `multi-get`, `ls`, and context commands.
- CLI: document IDs (docid) — first 6 chars of content hash for stable
references. Shown as `#abc123` in search results, usable with `get` and
`multi-get`.
- CLI: `--line-numbers` flag for get command output.
## [0.4.0] - 2025-12-10
MCP server for AI agent integration. Without it, agents had to shell out to
`qmd search` and parse CLI output. The monolithic `qmd.ts` (1840 lines) was
split into focused modules with the project's first test suite (215 tests).
### Changes
- MCP: stdio server with tools for search, vector search, hybrid query,
document retrieval, and status. Runs over stdio transport for Claude
Desktop and MCP clients.
- MCP: spec-compliant with June 2025 MCP specification — removed non-spec
`mimeType`, added `isError: true` to errors, `structuredContent` for
machine-readable results, proper URI encoding.
- MCP: simplified tool naming (`qmd_search` → `search`) since MCP already
namespaces by server.
- Architecture: extract `store.ts` (1221 LOC), `llm.ts` (539 LOC),
`formatter.ts` (359 LOC), `mcp.ts` (503 LOC) from monolithic `qmd.ts`.
- Testing: 215 tests (store: 96, llm: 60, mcp: 59) with mocked Ollama for
fast, deterministic runs. Before this: zero tests.
## [0.3.0] - 2025-12-08
Document chunking for vector search. A 5000-word document about many topics
gets a single embedding that averages everything together, matching poorly for
specific queries. Chunking produces one embedding per ~900-token section with
focused semantic signal.
### Changes
- Search: markdown-aware chunking — prefers heading boundaries, then paragraph
breaks, then sentence boundaries. 15% overlap between chunks ensures
cross-boundary queries still match.
- Search: multi-chunk scoring bonus (+0.02 per additional chunk, capped at
+0.1 for 5+ chunks). Documents relevant in multiple sections rank higher.
- CLI: display paths show collection-relative paths and extracted titles
(from H1 headings or YAML frontmatter) instead of raw filesystem paths.
- CLI: `--all` flag returns all matches (use with `--min-score` to filter).
- CLI: byte-based progress bar with ETA for `embed` command.
- CLI: human-readable time formatting ("15m 4s" instead of "904.2s").
- CLI: documents >64KB truncated with warning during embedding.
## [0.2.0] - 2025-12-08
### Changes
- CLI: `--json`, `--csv`, `--files`, `--md`, `--xml` output format flags.
`--json` for programmatic access, `--files` for piping, `--md`/`--xml` for
LLM consumption, `--csv` for spreadsheets.
- CLI: `qmd status` shows index health — document count, size, embedding
coverage, time since last update.
- Search: weighted RRF — original query gets 2x weight relative to expanded
queries since the user's actual words are a more reliable signal.
## [0.1.0] - 2025-12-07
Initial implementation. Built in a single day for searching personal markdown
notes, journals, and meeting transcripts.
### Changes
- Search: SQLite FTS5 with BM25 ranking. Chose SQLite over Elasticsearch
because QMD is a personal tool — single binary, no server dependencies.
- Search: sqlite-vec for vector similarity. Same rationale: in-process, no
external vector database.
- Search: Reciprocal Rank Fusion to combine BM25 and vector results. RRF is
parameter-free and handles missing signals gracefully.
- LLM: Ollama for embeddings, reranking, and query expansion. Later replaced
with node-llama-cpp in 0.6.0.
- CLI: `qmd add`, `qmd embed`, `qmd search`, `qmd vsearch`, `qmd query`,
`qmd get`. ~1800 lines of TypeScript in a single `qmd.ts` file.
[Unreleased]: https://github.com/tobi/qmd/compare/v1.0.0...HEAD
[1.0.0]: https://github.com/tobi/qmd/releases/tag/v1.0.0
[0.9.0]: https://github.com/tobi/qmd/compare/v0.8.0...v0.9.0
================================================
FILE: CLAUDE.md
================================================
# QMD - Query Markup Documents
Use Bun instead of Node.js (`bun` not `node`, `bun install` not `npm install`).
## Commands
```sh
qmd collection add . --name <n> # Create/index collection
qmd collection list # List all collections with details
qmd collection remove <name> # Remove a collection by name
qmd collection rename <old> <new> # Rename a collection
qmd ls [collection[/path]] # List collections or files in a collection
qmd context add [path] "text" # Add context for path (defaults to current dir)
qmd context list # List all contexts
qmd context check # Check for collections/paths missing context
qmd context rm <path> # Remove context
qmd get <file> # Get document by path or docid (#abc123)
qmd multi-get <pattern> # Get multiple docs by glob or comma-separated list
qmd status # Show index status and collections
qmd update [--pull] # Re-index all collections (--pull: git pull first)
qmd embed # Generate vector embeddings (uses node-llama-cpp)
qmd query <query> # Search with query expansion + reranking (recommended)
qmd search <query> # Full-text keyword search (BM25, no LLM)
qmd vsearch <query> # Vector similarity search (no reranking)
qmd mcp # Start MCP server (stdio transport)
qmd mcp --http [--port N] # Start MCP server (HTTP, default port 8181)
qmd mcp --http --daemon # Start as background daemon
qmd mcp stop # Stop background MCP daemon
```
## Collection Management
```sh
# List all collections
qmd collection list
# Create a collection with explicit name
qmd collection add ~/Documents/notes --name mynotes --mask '**/*.md'
# Remove a collection
qmd collection remove mynotes
# Rename a collection
qmd collection rename mynotes my-notes
# List all files in a collection
qmd ls mynotes
# List files with a path prefix
qmd ls journals/2025
qmd ls qmd://journals/2025
```
## Context Management
```sh
# Add context to current directory (auto-detects collection)
qmd context add "Description of these files"
# Add context to a specific path
qmd context add /subfolder "Description for subfolder"
# Add global context to all collections (system message)
qmd context add / "Always include this context"
# Add context using virtual paths
qmd context add qmd://journals/ "Context for entire journals collection"
qmd context add qmd://journals/2024 "Journal entries from 2024"
# List all contexts
qmd context list
# Check for collections or paths without context
qmd context check
# Remove context
qmd context rm qmd://journals/2024
qmd context rm / # Remove global context
```
## Document IDs (docid)
Each document has a unique short ID (docid) - the first 6 characters of its content hash.
Docids are shown in search results as `#abc123` and can be used with `get` and `multi-get`:
```sh
# Search returns docid in results
qmd search "query" --json
# Output: [{"docid": "#abc123", "score": 0.85, "file": "docs/readme.md", ...}]
# Get document by docid
qmd get "#abc123"
qmd get abc123 # Leading # is optional
# Docids also work in multi-get comma-separated lists
qmd multi-get "#abc123, #def456"
```
## Options
```sh
# Search & retrieval
-c, --collection <name> # Restrict search to a collection (matches pwd suffix)
-n <num> # Number of results
--all # Return all matches
--min-score <num> # Minimum score threshold
--full # Show full document content
--line-numbers # Add line numbers to output
# Multi-get specific
-l <num> # Maximum lines per file
--max-bytes <num> # Skip files larger than this (default 10KB)
# Output formats (search and multi-get)
--json, --csv, --md, --xml, --files
```
## Development
```sh
bun src/cli/qmd.ts <command> # Run from source
bun link # Install globally as 'qmd'
```
## Tests
All tests live in `test/`. Run everything:
```sh
npx vitest run --reporter=verbose test/
bun test --preload ./src/test-preload.ts test/
```
## Architecture
- SQLite FTS5 for full-text search (BM25)
- sqlite-vec for vector similarity search
- node-llama-cpp for embeddings (embeddinggemma), reranking (qwen3-reranker), and query expansion (Qwen3)
- Reciprocal Rank Fusion (RRF) for combining results
- Smart chunking: 900 tokens/chunk with 15% overlap, prefers markdown headings as boundaries
## Important: Do NOT run automatically
- Never run `qmd collection add`, `qmd embed`, or `qmd update` automatically
- Never modify the SQLite database directly
- Write out example commands for the user to run manually
- Index is stored at `~/.cache/qmd/index.sqlite`
## Do NOT compile
- Never run `bun build --compile` - it overwrites the shell wrapper and breaks sqlite-vec
- The `qmd` file is a shell script that runs compiled JS from `dist/` - do not replace it
- `npm run build` compiles TypeScript to `dist/` via `tsc -p tsconfig.build.json`
## Releasing
Use `/release <version>` to cut a release. Full changelog standards,
release workflow, and git hook setup are documented in the
[release skill](skills/release/SKILL.md).
Key points:
- Add changelog entries under `## [Unreleased]` **as you make changes**
- The release script renames `[Unreleased]` → `[X.Y.Z] - date` at release time
- Credit external PRs with `#NNN (thanks @username)`
- GitHub releases roll up the full minor series (e.g. 1.2.0 through 1.2.3)
================================================
FILE: LICENSE
================================================
MIT License
Copyright (c) 2024-2026 Tobi Lutke
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
================================================
FILE: README.md
================================================
# QMD - Query Markup Documents
An on-device search engine for everything you need to remember. Index your markdown notes, meeting transcripts, documentation, and knowledge bases. Search with keywords or natural language. Ideal for your agentic flows.
QMD combines BM25 full-text search, vector semantic search, and LLM re-ranking—all running locally via node-llama-cpp with GGUF models.
You can read more about QMD's progress in the [CHANGELOG](CHANGELOG.md).
## Quick Start
```sh
# Install globally (Node or Bun)
npm install -g @tobilu/qmd
# or
bun install -g @tobilu/qmd
# Or run directly
npx @tobilu/qmd ...
bunx @tobilu/qmd ...
# Create collections for your notes, docs, and meeting transcripts
qmd collection add ~/notes --name notes
qmd collection add ~/Documents/meetings --name meetings
qmd collection add ~/work/docs --name docs
# Add context to help with search results, each piece of context will be returned when matching sub documents are returned. This works as a tree. This is the key feature of QMD as it allows LLMs to make much better contextual choices when selecting documents. Don't sleep on it!
qmd context add qmd://notes "Personal notes and ideas"
qmd context add qmd://meetings "Meeting transcripts and notes"
qmd context add qmd://docs "Work documentation"
# Generate embeddings for semantic search
qmd embed
# Search across everything
qmd search "project timeline" # Fast keyword search
qmd vsearch "how to deploy" # Semantic search
qmd query "quarterly planning process" # Hybrid + reranking (best quality)
# Get a specific document
qmd get "meetings/2024-01-15.md"
# Get a document by docid (shown in search results)
qmd get "#abc123"
# Get multiple documents by glob pattern
qmd multi-get "journals/2025-05*.md"
# Search within a specific collection
qmd search "API" -c notes
# Export all matches for an agent
qmd search "API" --all --files --min-score 0.3
```
### Using with AI Agents
QMD's `--json` and `--files` output formats are designed for agentic workflows:
```sh
# Get structured results for an LLM
qmd search "authentication" --json -n 10
# List all relevant files above a threshold
qmd query "error handling" --all --files --min-score 0.4
# Retrieve full document content
qmd get "docs/api-reference.md" --full
```
### MCP Server
Although the tool works perfectly fine when you just tell your agent to use it on the command line, it also exposes an MCP (Model Context Protocol) server for tighter integration.
**Tools exposed:**
- `query` — Search with typed sub-queries (`lex`/`vec`/`hyde`), combined via RRF + reranking
- `get` — Retrieve a document by path or docid (with fuzzy matching suggestions)
- `multi_get` — Batch retrieve by glob pattern, comma-separated list, or docids
- `status` — Index health and collection info
**Claude Desktop configuration** (`~/Library/Application Support/Claude/claude_desktop_config.json`):
```json
{
"mcpServers": {
"qmd": {
"command": "qmd",
"args": ["mcp"]
}
}
}
```
**Claude Code** — Install the plugin (recommended):
```bash
claude plugin marketplace add tobi/qmd
claude plugin install qmd@qmd
```
Or configure MCP manually in `~/.claude/settings.json`:
```json
{
"mcpServers": {
"qmd": {
"command": "qmd",
"args": ["mcp"]
}
}
}
```
#### HTTP Transport
By default, QMD's MCP server uses stdio (launched as a subprocess by each client). For a shared, long-lived server that avoids repeated model loading, use the HTTP transport:
```sh
# Foreground (Ctrl-C to stop)
qmd mcp --http # localhost:8181
qmd mcp --http --port 8080 # custom port
# Background daemon
qmd mcp --http --daemon # start, writes PID to ~/.cache/qmd/mcp.pid
qmd mcp stop # stop via PID file
qmd status # shows "MCP: running (PID ...)" when active
```
The HTTP server exposes two endpoints:
- `POST /mcp` — MCP Streamable HTTP (JSON responses, stateless)
- `GET /health` — liveness check with uptime
LLM models stay loaded in VRAM across requests. Embedding/reranking contexts are disposed after 5 min idle and transparently recreated on the next request (~1s penalty, models remain loaded).
Point any MCP client at `http://localhost:8181/mcp` to connect.
### SDK / Library Usage
Use QMD as a library in your own Node.js or Bun applications.
#### Installation
```sh
npm install @tobilu/qmd
```
#### Quick Start
```typescript
import { createStore } from '@tobilu/qmd'
const store = await createStore({
dbPath: './my-index.sqlite',
config: {
collections: {
docs: { path: '/path/to/docs', pattern: '**/*.md' },
},
},
})
const results = await store.search({ query: "authentication flow" })
console.log(results.map(r => `${r.title} (${Math.round(r.score * 100)}%)`))
await store.close()
```
#### Store Creation
`createStore()` accepts three modes:
```typescript
import { createStore } from '@tobilu/qmd'
// 1. Inline config — no files needed besides the DB
const store = await createStore({
dbPath: './index.sqlite',
config: {
collections: {
docs: { path: '/path/to/docs', pattern: '**/*.md' },
notes: { path: '/path/to/notes' },
},
},
})
// 2. YAML config file — collections defined in a file
const store2 = await createStore({
dbPath: './index.sqlite',
configPath: './qmd.yml',
})
// 3. DB-only — reopen a previously configured store
const store3 = await createStore({ dbPath: './index.sqlite' })
```
#### Search
The unified `search()` method handles both simple queries and pre-expanded structured queries:
```typescript
// Simple query — auto-expanded via LLM, then BM25 + vector + reranking
const results = await store.search({ query: "authentication flow" })
// With options
const results2 = await store.search({
query: "rate limiting",
intent: "API throttling and abuse prevention",
collection: "docs",
limit: 5,
minScore: 0.3,
explain: true,
})
// Pre-expanded queries — skip auto-expansion, control each sub-query
const results3 = await store.search({
queries: [
{ type: 'lex', query: '"connection pool" timeout -redis' },
{ type: 'vec', query: 'why do database connections time out under load' },
],
collections: ["docs", "notes"],
})
// Skip reranking for faster results
const fast = await store.search({ query: "auth", rerank: false })
```
For direct backend access:
```typescript
// BM25 keyword search (fast, no LLM)
const lexResults = await store.searchLex("auth middleware", { limit: 10 })
// Vector similarity search (embedding model, no reranking)
const vecResults = await store.searchVector("how users log in", { limit: 10 })
// Manual query expansion for full control
const expanded = await store.expandQuery("auth flow", { intent: "user login" })
const results4 = await store.search({ queries: expanded })
```
#### Retrieval
```typescript
// Get a document by path or docid
const doc = await store.get("docs/readme.md")
const byId = await store.get("#abc123")
if (!("error" in doc)) {
console.log(doc.title, doc.displayPath, doc.context)
}
// Get document body with line range
const body = await store.getDocumentBody("docs/readme.md", {
fromLine: 50,
maxLines: 100,
})
// Batch retrieve by glob or comma-separated list
const { docs, errors } = await store.multiGet("docs/**/*.md", {
maxBytes: 20480,
})
```
#### Collections
```typescript
// Add a collection
await store.addCollection("myapp", {
path: "/src/myapp",
pattern: "**/*.ts",
ignore: ["node_modules/**", "*.test.ts"],
})
// List collections with document stats
const collections = await store.listCollections()
// => [{ name, pwd, glob_pattern, doc_count, active_count, last_modified, includeByDefault }]
// Get names of collections included in queries by default
const defaults = await store.getDefaultCollectionNames()
// Remove / rename
await store.removeCollection("myapp")
await store.renameCollection("old-name", "new-name")
```
#### Context
Context adds descriptive metadata that improves search relevance and is returned alongside results:
```typescript
// Add context for a path within a collection
await store.addContext("docs", "/api", "REST API reference documentation")
// Set global context (applies to all collections)
await store.setGlobalContext("Internal engineering documentation")
// List all contexts
const contexts = await store.listContexts()
// => [{ collection, path, context }]
// Remove context
await store.removeContext("docs", "/api")
await store.setGlobalContext(undefined) // clear global
```
#### Indexing
```typescript
// Re-index collections by scanning the filesystem
const result = await store.update({
collections: ["docs"], // optional — defaults to all
onProgress: ({ collection, file, current, total }) => {
console.log(`[${collection}] ${current}/${total} ${file}`)
},
})
// => { collections, indexed, updated, unchanged, removed, needsEmbedding }
// Generate vector embeddings
const embedResult = await store.embed({
force: false, // true to re-embed everything
onProgress: ({ current, total, collection }) => {
console.log(`Embedding ${current}/${total}`)
},
})
```
#### Types
Key types exported for SDK consumers:
```typescript
import type {
QMDStore, // The store interface
SearchOptions, // Options for search()
LexSearchOptions, // Options for searchLex()
VectorSearchOptions, // Options for searchVector()
HybridQueryResult, // Search result with score, snippet, context
SearchResult, // Result from searchLex/searchVector
ExpandedQuery, // Typed sub-query { type: 'lex'|'vec'|'hyde', query }
DocumentResult, // Document metadata + body
DocumentNotFound, // Error with similarFiles suggestions
MultiGetResult, // Batch retrieval result
UpdateProgress, // Progress callback info for update()
UpdateResult, // Aggregated update result
EmbedProgress, // Progress callback info for embed()
EmbedResult, // Embedding result
StoreOptions, // createStore() options
CollectionConfig, // Inline config shape
IndexStatus, // From getStatus()
IndexHealthInfo, // From getIndexHealth()
} from '@tobilu/qmd'
```
Utility exports:
```typescript
import {
extractSnippet, // Extract a relevant snippet from text
addLineNumbers, // Add line numbers to text
DEFAULT_MULTI_GET_MAX_BYTES, // Default max file size for multiGet (10KB)
Maintenance, // Database maintenance operations
} from '@tobilu/qmd'
```
#### Lifecycle
```typescript
// Close the store — disposes LLM models and DB connection
await store.close()
```
The SDK requires explicit `dbPath` — no defaults are assumed. This makes it safe to embed in any application without side effects.
## Architecture
```
┌─────────────────────────────────────────────────────────────────────────────┐
│ QMD Hybrid Search Pipeline │
└─────────────────────────────────────────────────────────────────────────────┘
┌─────────────────┐
│ User Query │
└────────┬────────┘
│
┌──────────────┴──────────────┐
▼ ▼
┌────────────────┐ ┌────────────────┐
│ Query Expansion│ │ Original Query│
│ (fine-tuned) │ │ (×2 weight) │
└───────┬────────┘ └───────┬────────┘
│ │
│ 2 alternative queries │
└──────────────┬──────────────┘
│
┌───────────────────────┼───────────────────────┐
▼ ▼ ▼
┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐
│ Original Query │ │ Expanded Query 1│ │ Expanded Query 2│
└────────┬────────┘ └────────┬────────┘ └────────┬────────┘
│ │ │
┌───────┴───────┐ ┌───────┴───────┐ ┌───────┴───────┐
▼ ▼ ▼ ▼ ▼ ▼
┌───────┐ ┌───────┐ ┌───────┐ ┌───────┐ ┌───────┐ ┌───────┐
│ BM25 │ │Vector │ │ BM25 │ │Vector │ │ BM25 │ │Vector │
│(FTS5) │ │Search │ │(FTS5) │ │Search │ │(FTS5) │ │Search │
└───┬───┘ └───┬───┘ └───┬───┘ └───┬───┘ └───┬───┘ └───┬───┘
│ │ │ │ │ │
└───────┬───────┘ └──────┬──────┘ └──────┬──────┘
│ │ │
└────────────────────────┼───────────────────────┘
│
▼
┌───────────────────────┐
│ RRF Fusion + Bonus │
│ Original query: ×2 │
│ Top-rank bonus: +0.05│
│ Top 30 Kept │
└───────────┬───────────┘
│
▼
┌───────────────────────┐
│ LLM Re-ranking │
│ (qwen3-reranker) │
│ Yes/No + logprobs │
└───────────┬───────────┘
│
▼
┌───────────────────────┐
│ Position-Aware Blend │
│ Top 1-3: 75% RRF │
│ Top 4-10: 60% RRF │
│ Top 11+: 40% RRF │
└───────────────────────┘
```
## Score Normalization & Fusion
### Search Backends
| Backend | Raw Score | Conversion | Range |
|---------|-----------|------------|-------|
| **FTS (BM25)** | SQLite FTS5 BM25 | `Math.abs(score)` | 0 to ~25+ |
| **Vector** | Cosine distance | `1 / (1 + distance)` | 0.0 to 1.0 |
| **Reranker** | LLM 0-10 rating | `score / 10` | 0.0 to 1.0 |
### Fusion Strategy
The `query` command uses **Reciprocal Rank Fusion (RRF)** with position-aware blending:
1. **Query Expansion**: Original query (×2 for weighting) + 1 LLM variation
2. **Parallel Retrieval**: Each query searches both FTS and vector indexes
3. **RRF Fusion**: Combine all result lists using `score = Σ(1/(k+rank+1))` where k=60
4. **Top-Rank Bonus**: Documents ranking #1 in any list get +0.05, #2-3 get +0.02
5. **Top-K Selection**: Take top 30 candidates for reranking
6. **Re-ranking**: LLM scores each document (yes/no with logprobs confidence)
7. **Position-Aware Blending**:
- RRF rank 1-3: 75% retrieval, 25% reranker (preserves exact matches)
- RRF rank 4-10: 60% retrieval, 40% reranker
- RRF rank 11+: 40% retrieval, 60% reranker (trust reranker more)
**Why this approach**: Pure RRF can dilute exact matches when expanded queries don't match. The top-rank bonus preserves documents that score #1 for the original query. Position-aware blending prevents the reranker from destroying high-confidence retrieval results.
### Score Interpretation
| Score | Meaning |
|-------|---------|
| 0.8 - 1.0 | Highly relevant |
| 0.5 - 0.8 | Moderately relevant |
| 0.2 - 0.5 | Somewhat relevant |
| 0.0 - 0.2 | Low relevance |
## Requirements
### System Requirements
- **Node.js** >= 22
- **Bun** >= 1.0.0
- **macOS**: Homebrew SQLite (for extension support)
```sh
brew install sqlite
```
### GGUF Models (via node-llama-cpp)
QMD uses three local GGUF models (auto-downloaded on first use):
| Model | Purpose | Size |
|-------|---------|------|
| `embeddinggemma-300M-Q8_0` | Vector embeddings (default) | ~300MB |
| `qwen3-reranker-0.6b-q8_0` | Re-ranking | ~640MB |
| `qmd-query-expansion-1.7B-q4_k_m` | Query expansion (fine-tuned) | ~1.1GB |
Models are downloaded from HuggingFace and cached in `~/.cache/qmd/models/`.
### Custom Embedding Model
Override the default embedding model via the `QMD_EMBED_MODEL` environment variable.
This is useful for multilingual corpora (e.g. Chinese, Japanese, Korean) where
`embeddinggemma-300M` has limited coverage.
```sh
# Use Qwen3-Embedding-0.6B for better multilingual (CJK) support
export QMD_EMBED_MODEL="hf:Qwen/Qwen3-Embedding-0.6B-GGUF/Qwen3-Embedding-0.6B-Q8_0.gguf"
# After changing the model, re-embed all collections:
qmd embed -f
```
Supported model families:
- **embeddinggemma** (default) — English-optimized, small footprint
- **Qwen3-Embedding** — Multilingual (119 languages including CJK), MTEB top-ranked
> **Note:** When switching embedding models, you must re-index with `qmd embed -f`
> since vectors are not cross-compatible between models. The prompt format is
> automatically adjusted for each model family.
## Installation
```sh
npm install -g @tobilu/qmd
# or
bun install -g @tobilu/qmd
```
### Development
```sh
git clone https://github.com/tobi/qmd
cd qmd
npm install
npm link
```
## Usage
### Collection Management
```sh
# Create a collection from current directory
qmd collection add . --name myproject
# Create a collection with explicit path and custom glob mask
qmd collection add ~/Documents/notes --name notes --mask "**/*.md"
# List all collections
qmd collection list
# Remove a collection
qmd collection remove myproject
# Rename a collection
qmd collection rename myproject my-project
# List files in a collection
qmd ls notes
qmd ls notes/subfolder
```
### Generate Vector Embeddings
```sh
# Embed all indexed documents (900 tokens/chunk, 15% overlap)
qmd embed
# Force re-embed everything
qmd embed -f
```
### Context Management
Context adds descriptive metadata to collections and paths, helping search understand your content.
```sh
# Add context to a collection (using qmd:// virtual paths)
qmd context add qmd://notes "Personal notes and ideas"
qmd context add qmd://docs/api "API documentation"
# Add context from within a collection directory
cd ~/notes && qmd context add "Personal notes and ideas"
cd ~/notes/work && qmd context add "Work-related notes"
# Add global context (applies to all collections)
qmd context add / "Knowledge base for my projects"
# List all contexts
qmd context list
# Remove context
qmd context rm qmd://notes/old
```
### Search Commands
```
┌──────────────────────────────────────────────────────────────────┐
│ Search Modes │
├──────────┬───────────────────────────────────────────────────────┤
│ search │ BM25 full-text search only │
│ vsearch │ Vector semantic search only │
│ query │ Hybrid: FTS + Vector + Query Expansion + Re-ranking │
└──────────┴───────────────────────────────────────────────────────┘
```
```sh
# Full-text search (fast, keyword-based)
qmd search "authentication flow"
# Vector search (semantic similarity)
qmd vsearch "how to login"
# Hybrid search with re-ranking (best quality)
qmd query "user authentication"
```
### Options
```sh
# Search options
-n <num> # Number of results (default: 5, or 20 for --files/--json)
-c, --collection # Restrict search to a specific collection
--all # Return all matches (use with --min-score to filter)
--min-score <num> # Minimum score threshold (default: 0)
--full # Show full document content
--line-numbers # Add line numbers to output
--explain # Include retrieval score traces (query, JSON/CLI output)
--index <name> # Use named index
# Output formats (for search and multi-get)
--files # Output: docid,score,filepath,context
--json # JSON output with snippets
--csv # CSV output
--md # Markdown output
--xml # XML output
# Get options
qmd get <file>[:line] # Get document, optionally starting at line
-l <num> # Maximum lines to return
--from <num> # Start from line number
# Multi-get options
-l <num> # Maximum lines per file
--max-bytes <num> # Skip files larger than N bytes (default: 10KB)
```
### Output Format
Default output is colorized CLI format (respects `NO_COLOR` env):
```
docs/guide.md:42 #a1b2c3
Title: Software Craftsmanship
Context: Work documentation
Score: 93%
This section covers the **craftsmanship** of building
quality software with attention to detail.
See also: engineering principles
notes/meeting.md:15 #d4e5f6
Title: Q4 Planning
Context: Personal notes and ideas
Score: 67%
Discussion about code quality and craftsmanship
in the development process.
```
- **Path**: Collection-relative path (e.g., `docs/guide.md`)
- **Docid**: Short hash identifier (e.g., `#a1b2c3`) - use with `qmd get #a1b2c3`
- **Title**: Extracted from document (first heading or filename)
- **Context**: Path context if configured via `qmd context add`
- **Score**: Color-coded (green >70%, yellow >40%, dim otherwise)
- **Snippet**: Context around match with query terms highlighted
### Examples
```sh
# Get 10 results with minimum score 0.3
qmd query -n 10 --min-score 0.3 "API design patterns"
# Output as markdown for LLM context
qmd search --md --full "error handling"
# JSON output for scripting
qmd query --json "quarterly reports"
# Inspect how each result was scored (RRF + rerank blend)
qmd query --json --explain "quarterly reports"
# Use separate index for different knowledge base
qmd --index work search "quarterly reports"
```
### Index Maintenance
```sh
# Show index status and collections with contexts
qmd status
# Re-index all collections
qmd update
# Re-index with git pull first (for remote repos)
qmd update --pull
# Get document by filepath (with fuzzy matching suggestions)
qmd get notes/meeting.md
# Get document by docid (from search results)
qmd get "#abc123"
# Get document starting at line 50, max 100 lines
qmd get notes/meeting.md:50 -l 100
# Get multiple documents by glob pattern
qmd multi-get "journals/2025-05*.md"
# Get multiple documents by comma-separated list (supports docids)
qmd multi-get "doc1.md, doc2.md, #abc123"
# Limit multi-get to files under 20KB
qmd multi-get "docs/*.md" --max-bytes 20480
# Output multi-get as JSON for agent processing
qmd multi-get "docs/*.md" --json
# Clean up cache and orphaned data
qmd cleanup
```
## Data Storage
Index stored in: `~/.cache/qmd/index.sqlite`
### Schema
```sql
collections -- Indexed directories with name and glob patterns
path_contexts -- Context descriptions by virtual path (qmd://...)
documents -- Markdown content with metadata and docid (6-char hash)
documents_fts -- FTS5 full-text index
content_vectors -- Embedding chunks (hash, seq, pos, 900 tokens each)
vectors_vec -- sqlite-vec vector index (hash_seq key)
llm_cache -- Cached LLM responses (query expansion, rerank scores)
```
## Environment Variables
| Variable | Default | Description |
|----------|---------|-------------|
| `XDG_CACHE_HOME` | `~/.cache` | Cache directory location |
## How It Works
### Indexing Flow
```
Collection ──► Glob Pattern ──► Markdown Files ──► Parse Title ──► Hash Content
│ │ │
│ │ ▼
│ │ Generate docid
│ │ (6-char hash)
│ │ │
└──────────────────────────────────────────────────►└──► Store in SQLite
│
▼
FTS5 Index
```
### Embedding Flow
Documents are chunked into ~900-token pieces with 15% overlap using smart boundary detection:
```
Document ──► Smart Chunk (~900 tokens) ──► Format each chunk ──► node-llama-cpp ──► Store Vectors
│ "title | text" embedBatch()
│
└─► Chunks stored with:
- hash: document hash
- seq: chunk sequence (0, 1, 2...)
- pos: character position in original
```
### Smart Chunking
Instead of cutting at hard token boundaries, QMD uses a scoring algorithm to find natural markdown break points. This keeps semantic units (sections, paragraphs, code blocks) together.
**Break Point Scores:**
| Pattern | Score | Description |
|---------|-------|-------------|
| `# Heading` | 100 | H1 - major section |
| `## Heading` | 90 | H2 - subsection |
| `### Heading` | 80 | H3 |
| `#### Heading` | 70 | H4 |
| `##### Heading` | 60 | H5 |
| `###### Heading` | 50 | H6 |
| ` ``` ` | 80 | Code block boundary |
| `---` / `***` | 60 | Horizontal rule |
| Blank line | 20 | Paragraph boundary |
| `- item` / `1. item` | 5 | List item |
| Line break | 1 | Minimal break |
**Algorithm:**
1. Scan document for all break points with scores
2. When approaching the 900-token target, search a 200-token window before the cutoff
3. Score each break point: `finalScore = baseScore × (1 - (distance/window)² × 0.7)`
4. Cut at the highest-scoring break point
The squared distance decay means a heading 200 tokens back (score ~30) still beats a simple line break at the target (score 1), but a closer heading wins over a distant one.
**Code Fence Protection:** Break points inside code blocks are ignored—code stays together. If a code block exceeds the chunk size, it's kept whole when possible.
### Query Flow (Hybrid)
```
Query ──► LLM Expansion ──► [Original, Variant 1, Variant 2]
│
┌─────────┴─────────┐
▼ ▼
For each query: FTS (BM25)
│ │
▼ ▼
Vector Search Ranked List
│
▼
Ranked List
│
└─────────┬─────────┘
▼
RRF Fusion (k=60)
Original query ×2 weight
Top-rank bonus: +0.05/#1, +0.02/#2-3
│
▼
Top 30 candidates
│
▼
LLM Re-ranking
(yes/no + logprob confidence)
│
▼
Position-Aware Blend
Rank 1-3: 75% RRF / 25% reranker
Rank 4-10: 60% RRF / 40% reranker
Rank 11+: 40% RRF / 60% reranker
│
▼
Final Results
```
## Model Configuration
Models are configured in `src/llm.ts` as HuggingFace URIs:
```typescript
const DEFAULT_EMBED_MODEL = "hf:ggml-org/embeddinggemma-300M-GGUF/embeddinggemma-300M-Q8_0.gguf";
const DEFAULT_RERANK_MODEL = "hf:ggml-org/Qwen3-Reranker-0.6B-Q8_0-GGUF/qwen3-reranker-0.6b-q8_0.gguf";
const DEFAULT_GENERATE_MODEL = "hf:tobil/qmd-query-expansion-1.7B-gguf/qmd-query-expansion-1.7B-q4_k_m.gguf";
```
### EmbeddingGemma Prompt Format
```
// For queries
"task: search result | query: {query}"
// For documents
"title: {title} | text: {content}"
```
### Qwen3-Reranker
Uses node-llama-cpp's `createRankingContext()` and `rankAndSort()` API for cross-encoder reranking. Returns documents sorted by relevance score (0.0 - 1.0).
### Qwen3 (Query Expansion)
Used for generating query variations via `LlamaChatSession`.
## License
MIT
================================================
FILE: bin/qmd
================================================
#!/bin/sh
# Resolve symlinks so global installs (npm link / npm install -g) can find the
# actual package directory instead of the global bin directory.
SOURCE="$0"
while [ -L "$SOURCE" ]; do
SOURCE_DIR="$(cd -P "$(dirname "$SOURCE")" && pwd)"
TARGET="$(readlink "$SOURCE")"
case "$TARGET" in
/*) SOURCE="$TARGET" ;;
*) SOURCE="$SOURCE_DIR/$TARGET" ;;
esac
done
# Detect the runtime used to install this package and use the matching one
# to avoid native module ABI mismatches (e.g., better-sqlite3 compiled for bun vs node)
DIR="$(cd -P "$(dirname "$SOURCE")/.." && pwd)"
# Detect the package manager that installed dependencies by checking lockfiles.
# $BUN_INSTALL is intentionally NOT checked — it only indicates that bun exists
# on the system, not that it was used to install this package (see #361).
#
# package-lock.json takes priority: if it exists, npm installed the native
# modules for Node. The repo ships bun.lock, so without this check, source
# builds that use npm would be incorrectly routed to bun, causing ABI
# mismatches with better-sqlite3 / sqlite-vec (see #381).
if [ -f "$DIR/package-lock.json" ]; then
exec node "$DIR/dist/cli/qmd.js" "$@"
elif [ -f "$DIR/bun.lock" ] || [ -f "$DIR/bun.lockb" ]; then
exec bun "$DIR/dist/cli/qmd.js" "$@"
else
exec node "$DIR/dist/cli/qmd.js" "$@"
fi
================================================
FILE: docs/SYNTAX.md
================================================
# QMD Query Syntax
QMD queries are structured documents with typed sub-queries. Each line specifies a search type and query text.
## Grammar
```ebnf
query = expand_query | query_document ;
expand_query = text | explicit_expand ;
explicit_expand= "expand:" text ;
query_document = [ intent_line ] { typed_line } ;
intent_line = "intent:" text newline ;
typed_line = type ":" text newline ;
type = "lex" | "vec" | "hyde" ;
text = quoted_phrase | plain_text ;
quoted_phrase = '"' { character } '"' ;
plain_text = { character } ;
newline = "\n" ;
```
## Query Types
| Type | Method | Description |
|------|--------|-------------|
| `lex` | BM25 | Keyword search with exact matching |
| `vec` | Vector | Semantic similarity search |
| `hyde` | Vector | Hypothetical document embedding |
## Default Behavior
A QMD query is either a single expand query or a multi-line query document. Any single-line query with no prefix is treated as an expand query and passed to the expansion model, which emits lex, vec, and hyde variants automatically.
```
# These are equivalent and cannot be combined with typed lines:
how does authentication work
expand: how does authentication work
```
## Lex Query Syntax
Lex queries support special syntax for precise keyword matching:
```ebnf
lex_query = { lex_term } ;
lex_term = negation | phrase | word ;
negation = "-" ( phrase | word ) ;
phrase = '"' { character } '"' ;
word = { letter | digit | "'" } ;
```
| Syntax | Meaning | Example |
|--------|---------|---------|
| `word` | Prefix match | `perf` matches "performance" |
| `"phrase"` | Exact phrase | `"rate limiter"` |
| `-word` | Exclude term | `-sports` |
| `-"phrase"` | Exclude phrase | `-"test data"` |
### Examples
```
lex: CAP theorem consistency
lex: "machine learning" -"deep learning"
lex: auth -oauth -saml
```
## Vec Query Syntax
Vec queries are natural language questions. No special syntax — just write what you're looking for.
```
vec: how does the rate limiter handle burst traffic
vec: what is the tradeoff between consistency and availability
```
## Hyde Query Syntax
Hyde queries are hypothetical answer passages (50-100 words). Write what you expect the answer to look like.
```
hyde: The rate limiter uses a sliding window algorithm with a 60-second window. When a client exceeds 100 requests per minute, subsequent requests return 429 Too Many Requests.
```
## Multi-Line Queries
Combine multiple query types for best results. First query gets 2x weight in fusion.
```
lex: rate limiter algorithm
vec: how does rate limiting work in the API
hyde: The API implements rate limiting using a token bucket algorithm...
```
## Expand Queries
An expand query stands alone; it's not mixed with typed lines. You can either rely on the default untyped form or add the explicit `expand:` prefix:
```
expand: error handling best practices
# equivalent
error handling best practices
```
Both forms call the local query expansion model, which generates lex, vec, and hyde variations automatically.
## Intent
An optional `intent:` line provides background context to disambiguate ambiguous queries. It steers query expansion, reranking, and snippet extraction but does not search on its own.
- At most one `intent:` line per query document
- `intent:` cannot appear alone — at least one `lex:`, `vec:`, or `hyde:` line is required
- Intent is also available via the `--intent` CLI flag or MCP `intent` parameter
```
intent: web page load times and Core Web Vitals
lex: performance
vec: how to improve performance
```
Without intent, "performance" is ambiguous (web-perf? team health? fitness?). With intent, the search pipeline preferentially selects and ranks web-performance content.
## Constraints
- Top-level query must be either a standalone expand query or a multi-line document
- Query documents allow only `lex`, `vec`, `hyde`, and `intent` typed lines (no `expand:` inside)
- `lex` syntax (`-term`, `"phrase"`) only works in lex queries
- At most one `intent:` line per query document; cannot appear alone
- Empty lines are ignored
- Leading/trailing whitespace is trimmed
## MCP/HTTP API
The `query` tool accepts a query document:
```json
{
"q": "lex: CAP theorem\nvec: consistency vs availability",
"collections": ["docs"],
"limit": 10
}
```
Or structured format:
```json
{
"searches": [
{ "type": "lex", "query": "CAP theorem" },
{ "type": "vec", "query": "consistency vs availability" }
]
}
```
With intent:
```json
{
"searches": [
{ "type": "lex", "query": "performance" }
],
"intent": "web page load times and Core Web Vitals"
}
```
## CLI
```bash
# Single line (implicit expand)
qmd query "how does auth work"
# Multi-line with types
qmd query $'lex: auth token\nvec: how does authentication work'
# Structured
qmd query $'lex: keywords\nvec: question\nhyde: hypothetical answer...'
# With intent (inline)
qmd query $'intent: web performance and latency\nlex: performance\nvec: how to improve performance'
# With intent (flag)
qmd query --intent "web performance and latency" "performance"
```
================================================
FILE: example-index.yml
================================================
# QMD Collections Configuration
# Location: ~/.config/qmd/index.yml
#
# This file defines all collections and their contexts.
# You can manually edit this file - changes take effect immediately.
# Global context applied to all collections
# Use this for universal search instructions or patterns
global_context: "If you see a relevant [[WikiWord]], you can search for that WikiWord to get more context."
# Collection definitions
collections:
# Meeting notes
Meetings:
path: ~/Documents/Meetings
pattern: "**/*.md"
context:
"/": "Meeting notes and summaries"
# Daily journal entries
journals:
path: ~/Documents/Notes
pattern: "**/*.md"
context:
"/journal/2024": "Daily notes from 2024"
"/journal/2025": "Daily notes from 2025"
"/": "Notes vault"
codex:
path: ~/Documents/Codex
pattern: "**/*.md"
context:
"/": "Thematic collections of important concepts and discussions"
================================================
FILE: finetune/.gitignore
================================================
# Training outputs (run eval before pushing to HuggingFace)
outputs/
# Model checkpoints
*.pt
*.safetensors
# Processed data files (regenerated by prepare_data.py)
data/train/
data/train_*/
data/qmd_combined.jsonl
data/qmd_cleaned.jsonl
data/qmd_expansion_cleaned.jsonl
data/quality_report.txt
# Eval results
evals/results_*.jsonl
# Scripts (temporary/local)
scripts/
# Python cache
__pycache__/
*.pyc
================================================
FILE: finetune/CLAUDE.md
================================================
# QMD Query Expansion Fine-Tuning
## Overview
Train Qwen3-1.7B to expand search queries into structured `hyde:/lex:/vec:` output for QMD's hybrid retrieval pipeline.
## Output Format
```
hyde: A hypothetical document passage that would answer the query.
lex: keyword1
lex: keyword2
vec: semantic query reformulation
vec: another semantic variation
```
- `hyde:` always comes FIRST (one line max)
- `lex:` lines for BM25 keyword search (1-3 lines, short keywords)
- `vec:` lines for vector similarity search (1-3 lines, natural language)
## Training Data Format
**There is exactly one JSONL format.** Every file in `data/*.jsonl` must match the strict Pydantic schema in `dataset/schema.py`:
```json
{"query": "auth config", "output": [["hyde", "..."], ["lex", "..."], ["vec", "..."]]}
```
- `query`: non-empty string
- `output`: list of `[type, text]` pairs where type is `"lex"`, `"vec"`, or `"hyde"`
- Extra metadata fields (`category`, `intent`, `is_short`) are allowed but ignored
The schema is enforced by `dataset/schema.py:TrainingExample` (Pydantic model). All data loading goes through `load_examples()` which fails loudly on invalid data. No format alternatives, no legacy fallbacks.
**All `.jsonl` files in `data/` are concatenated and deduplicated for training runs.** The prepared train/val files in `data/train/` are ephemeral build artifacts.
## HuggingFace Repositories
| Repository | Purpose |
|------------|---------|
| `tobil/qmd-query-expansion-1.7B` | Final merged model (SFT baseline) |
| `tobil/qmd-query-expansion-1.7B-gguf` | GGUF quantized versions for deployment |
| `tobil/qmd-query-expansion-1.7B-sft` | SFT adapter checkpoint (intermediate) |
| `tobil/qmd-query-expansion-train` | Prepared training dataset |
| `tobil/qmd-query-expansion-1.7B-grpo` | Experimental GRPO adapter (optional) |
**Rules:**
- No versioned repos (`-v1`, `-v2`, `-v4`, etc.) - update in place
- Only push when eval scores improve over current deployed model
- Always include eval results in model card when pushing
## Dataset Tools
| Script | Purpose |
|--------|---------|
| `dataset/schema.py` | Pydantic `TrainingExample` model + `load_examples()` |
| `dataset/prepare_data.py` | Load via schema, apply Qwen3 chat template, dedup, split |
| `dataset/validate_schema.py` | Validate all JSONL files against schema |
| `dataset/score_data.py` | Score all examples using reward.py |
| `dataset/analyze_data.py` | Analyze distribution and quality |
## Training Pipeline
Always use **Qwen3-1.7B** as the base model unless explicitly stated otherwise.
### Stage 0: Prepare Data
```bash
uv run dataset/prepare_data.py
# Creates: data/train/train.jsonl, data/train/val.jsonl (ephemeral)
```
### Stage 1: SFT
```bash
# Local (requires CUDA)
uv run train.py sft --config configs/sft.yaml
# Cloud (HuggingFace Jobs)
hf jobs uv run --flavor a10g-large --secrets HF_TOKEN --timeout 2h jobs/sft.py
```
### Stage 2: (Experimental) GRPO
```bash
# Experimental script
cd finetune && HF_TOKEN=${HF_TOKEN} uv run python experiments/grpo/grpo.py
```
### HuggingFace Jobs
```bash
hf jobs ps # List running jobs
hf jobs logs <job-id> # Stream logs
hf jobs inspect <job-id> # Check status
hf jobs cancel <job-id> # Cancel a job
```
### Evaluation
```bash
uv run eval.py ./outputs/sft
uv run eval.py tobil/qmd-query-expansion-1.7B
uv run eval.py ./outputs/sft -o eval_results.json
```
## Quality Scoring
`reward.py` is the single source of truth for scoring:
```bash
uv run reward.py # Self-test
```
See `SCORING.md` for the full rubric.
## Experiments
Experimental training configurations live in `experiments/`:
```
experiments/
├── lfm2/ # LiquidAI LFM2-1.2B (hybrid architecture, faster inference)
│ ├── sft_lfm2.yaml
│ └── sft_lfm2.py
├── grpo/ # Experimental GRPO recipe and config
│ ├── grpo.py
│ └── grpo.yaml
└── gepa/ # DSPy-based prompt optimization (GEPA)
├── dspy_gepa.py
└── ...
```
These are not part of the main training pipeline.
## Key Files
```
finetune/
├── reward.py # Scoring function (single source of truth)
├── train.py # SFT training entrypoint
├── eval.py # Generate and score expansions
├── convert_gguf.py # GGUF conversion
├── SCORING.md # Detailed scoring rubric
├── CLAUDE.md # This file
├── Justfile # Common commands
├── data/ # All training JSONL files (strict schema)
├── dataset/ # Schema + data tools (Pydantic-based)
├── jobs/ # Self-contained HuggingFace Jobs scripts
├── configs/ # Training configs (sft.yaml)
├── evals/ # Test queries
├── experiments/ # Experimental configs (LFM2, GEPA, GRPO)
└── outputs/ # Local training outputs (gitignored)
```
================================================
FILE: finetune/Justfile
================================================
set shell := ["bash", "-uc"]
validate:
uv run dataset/validate_schema.py
uv run dataset/score_data.py
for f in data/*.jsonl; do \
uv run dataset/analyze_data.py --input "$f" --show-examples 0; \
done
score:
uv run dataset/score_data.py
schema:
uv run dataset/validate_schema.py
analyze:
for f in data/*.jsonl; do \
uv run dataset/analyze_data.py --input "$f" --show-examples 0; \
done
prepare:
QMD_BASE_MODEL=Qwen/Qwen3-1.7B uv run dataset/prepare_data.py --seed 42
convert-onnx size="1.7B":
uv run convert_onnx.py --size {{size}}
convert-gguf size="1.7B":
uv run convert_gguf.py --size {{size}}
train-local:
just prepare
HF_TOKEN=${HF_TOKEN} uv run torchrun --standalone --nproc_per_node auto \
train.py sft --config configs/sft_local.yaml |& tee /tmp/qmd-sft-train.log
# Experimental GRPO training is in finetune/experiments/grpo and not part of
# the default pipeline.
#
# grpo-local:
# HF_TOKEN=${HF_TOKEN} uv run train.py grpo --config experiments/grpo/grpo.yaml |& tee /tmp/qmd-grpo-train.log
================================================
FILE: finetune/Modelfile
================================================
FROM /home/tobi/src/github.com/tobi/qmd/finetune/outputs/sft/gguf/sft-q4_k_m.gguf
PARAMETER temperature 0.0
PARAMETER top_p 1.0
PARAMETER top_k 0
PARAMETER repeat_penalty 1.1
PARAMETER num_ctx 4096
TEMPLATE """<|im_start|>system
You are a helpful assistant.
<|im_end|>
<|im_start|>user
/no_think Expand this search query: {{ .Prompt }}
<|im_end|>
<|im_start|>assistant
"""
================================================
FILE: finetune/README.md
================================================
---
license: mit
language:
- en
base_model: Qwen/Qwen3-1.7B
tags:
- query-expansion
- search
- gguf
- qwen3
pipeline_tag: text-generation
---
# QMD Query Expansion Fine-Tuning
Train small language models to expand search queries for [QMD](https://github.com/tobi/qmd)'s hybrid retrieval pipeline.
## What This Does
Given a raw search query like `"auth config"`, the trained model produces structured expansions:
```
hyde: Authentication can be configured by setting the AUTH_SECRET environment variable.
lex: authentication configuration
lex: auth settings setup
vec: how to configure authentication settings
vec: authentication configuration options
```
These feed into QMD's three search backends:
- **`lex:`** lines go to BM25 full-text search (short, keyword-focused)
- **`vec:`** lines go to vector similarity search (natural language phrases)
- **`hyde:`** is a hypothetical document passage for embedding-based retrieval ([HyDE](https://arxiv.org/abs/2212.10496) technique)
## Quick Start
### Cloud training via HuggingFace Jobs (no GPU needed)
```bash
# 1. SFT: teach the model the output format (~45 min on A10G, ~$1.50)
hf jobs uv run --flavor a10g-large --secrets HF_TOKEN --timeout 2h jobs/sft.py
# 2. Evaluate against test queries (needs local GPU or use eval job)
uv run eval.py tobil/qmd-query-expansion-1.7B
# 3. Convert to GGUF for local deployment (Ollama, llama.cpp)
uv run convert_gguf.py --size 1.7B
# NOTE: GRPO is currently experimental and moved to finetune/experiments/grpo
# if you want to run it manually, use:
# cd finetune && uv run python experiments/grpo/grpo.py
```
### Local training (if you have a GPU)
```bash
uv run train.py sft --config configs/sft.yaml
# Experimental GRPO
cd finetune && uv run python experiments/grpo/grpo.py
```
### Monitoring HF Jobs
```bash
hf jobs ps # list running jobs
hf jobs inspect <job-id> # check status
hf jobs logs <job-id> # stream logs
hf jobs cancel <job-id> # cancel a job
```
## Prompt Format
All tools use the same prompt — **Qwen3 chat template with `/no_think`**:
```
<|im_start|>user
/no_think Expand this search query: {query}<|im_end|>
<|im_start|>assistant
```
The `/no_think` directive suppresses Qwen3's chain-of-thought mode, producing
direct `lex:/vec:/hyde:` output without `<think>` blocks.
## File Structure
```
finetune/
├── reward.py # Scoring/reward function (single source of truth)
├── train.py # SFT training entrypoint
├── eval.py # Generate expansions and score them
├── convert_gguf.py # GGUF conversion for Ollama/llama.cpp
├── jobs/
│ ├── sft.py # Self-contained SFT for HuggingFace Jobs
│ ├── eval.py # Self-contained eval for HuggingFace Jobs
│ └── eval_common.py # Shared eval utilities
├── configs/
│ └── sft.yaml # SFT hyperparameters for Qwen3-1.7B
├── evals/
│ └── queries.txt # 31 test queries across 8 categories
├── experiments/
│ └── grpo/ # Experimental GRPO configuration and script (optional)
├── data/ # Training JSONL files (all concatenated for training)
├── dataset/
│ ├── prepare_data.py # Format for Qwen3 chat template, dedup, split
│ ├── schema.py # Parse/normalize output format
│ ├── validate_schema.py # Validate JSONL against schema
│ ├── score_data.py # Score all examples using reward.py
│ └── analyze_data.py # Analyze distribution and quality
├── SCORING.md # Detailed scoring rubric reference
└── README.md # This file
```
## Training Pipeline
### Stage 1: SFT (Supervised Fine-Tuning)
Teaches the model the `lex:/vec:/hyde:` output format from labeled examples.
| Parameter | Value |
|-----------|-------|
| Base model | `Qwen/Qwen3-1.7B` |
| Method | LoRA (rank 16, alpha 32) |
| Target modules | All projection layers (q/k/v/o/gate/up/down) |
| Dataset | ~2,290 examples (train split) |
| Effective batch size | 16 (4 x 4 gradient accumulation) |
| Epochs | 5 |
| Learning rate | 2e-4 (cosine schedule) |
```bash
uv run train.py sft --config configs/sft.yaml
uv run train.py sft --config configs/sft.yaml --dry-run # preview config
```
### Stage 2: (Experimental) GRPO
GRPO is currently treated as experimental and kept under `experiments/grpo/`.
It is not part of the default production path for this repository.
```bash
# Optional experimental GRPO run
cd finetune && uv run python experiments/grpo/grpo.py
```
## Evaluation
`eval.py` generates expansions from a model and scores them against test queries:
```bash
# Evaluate a SFT model
uv run eval.py --model tobil/qmd-query-expansion-1.7B-sft
# Evaluate an SFT output dir
uv run eval.py outputs/sft
# Verbose output with deduction details
uv run eval.py tobil/qmd-query-expansion-1.7B -v
# Optional: evaluate GRPO experimental output (if run)
uv run eval.py outputs/grpo
# Save detailed scores to JSON
uv run eval.py tobil/qmd-query-expansion-1.7B -o scores.json
```
## Reward Function
`reward.py` is the single source of truth for scoring. It is used for evaluation
and (optionally) as the GRPO reward signal in the experimental path.
Five scoring dimensions (max 120 without hyde, 140 with):
| Dimension | Points | What It Measures |
|-----------|--------|------------------|
| **Format** | 0-30 | Has lex/vec lines, no invalid lines |
| **Diversity** | 0-30 | Multiple expansion types, diverse content, no query echoes |
| **HyDE** | 0-20 | Present, 50-200 chars, single line, not repetitive |
| **Quality** | 0-20 | Lex shorter than vec, natural language, preserves key terms |
| **Entity** | -45 to +20 | Named entities preserved in lex and vec lines |
| **Think bonus** | 0-20 | Reward for NOT using `<think>` mode |
**Hard failures** (instant 0.0):
- Chat template leakage (`<|im_start|>`, `<|im_end|>`, etc.)
- Any line without a valid `lex:`, `vec:`, or `hyde:` prefix
```bash
# Self-test the reward function
uv run reward.py
```
## GGUF Conversion
Merges base + SFT and (optionally) GRPO adapters into a single model, then
produces quantized GGUF files for deployment:
```bash
# Use preset for 1.7B
uv run convert_gguf.py --size 1.7B
# Custom models
uv run convert_gguf.py --base Qwen/Qwen3-1.7B \
--sft tobil/qmd-query-expansion-1.7B-sft \
--grpo tobil/qmd-query-expansion-1.7B-grpo \
--output tobil/qmd-query-expansion-1.7B-gguf
```
### Using with Ollama
```bash
huggingface-cli download tobil/qmd-query-expansion-1.7B-gguf \
qmd-query-expansion-1.7B-q4_k_m.gguf --local-dir .
echo 'FROM ./qmd-query-expansion-1.7B-q4_k_m.gguf' > Modelfile
ollama create qmd-expand -f Modelfile
ollama run qmd-expand
```
## Data Pipeline
All JSONL files in `data/` are concatenated for training. To prepare for training:
```bash
# Format for Qwen3 chat template, deduplicate, split train/val
uv run dataset/prepare_data.py
# Validate data quality
just validate
```
## Architecture Notes
The production training approach is currently **SFT-only**:
1. **SFT** establishes format compliance and basic query understanding. It uses
a large LoRA (rank 16, all projection layers) because it needs to learn a
new output format from scratch.
2. **GRPO** exists as an optional experimental path under `experiments/grpo/`
and is not in the production training pipeline.
The reward function is entirely rule-based (no LLM judge) which makes it fast,
deterministic, and suitable as an RL signal. See `SCORING.md` for the full rubric.
## Training Results (Qwen3-1.7B, v2)
### SFT
| Metric | Value |
|--------|-------|
| Final train loss | 0.472 |
| Final eval loss | 0.304 |
| Token accuracy (train) | 97.4% |
| Token accuracy (eval) | 93.8% |
| Epochs | 5 |
| Hardware | A10G (24 GB VRAM) |
### Evaluation Scores
| Model | Average Score | Excellent (30) |
|-------|--------------|-----------------|
| SFT | 92.0% | 30/30 |
> GRPO scores are not tracked in this branch; see `experiments/grpo/` for historical
> experimental results.
================================================
FILE: finetune/SCORING.md
================================================
# QMD Query Expansion Scoring
## Goal
Transform a random typed query into a great set of retrieval-optimized expansions.
**Input:** `"auth config"`
**Output:**
```
hyde: Authentication can be configured by setting the AUTH_SECRET environment variable and enabling the auth middleware in your application's config file.
lex: authentication configuration
lex: auth settings setup
vec: how to configure authentication settings
vec: authentication configuration options
```
## Output Format
| Prefix | Purpose | Required | Count |
|--------|---------|----------|-------|
| `lex:` | BM25 keyword variations (shorter, keyword-focused) | Yes | 1-3 |
| `vec:` | Semantic reformulations (natural language) | Yes | 1-3 |
| `hyde:` | Hypothetical document passage | Optional | 0-1 |
## Scoring Criteria
### 1. Format Compliance (0-30 points)
| Criterion | Points | Deduction |
|-----------|--------|-----------|
| Has at least one `lex:` line | +10 | -10 if missing |
| Has at least one `vec:` line | +10 | -10 if missing |
| All lines have valid prefix (`lex:`, `vec:`, `hyde:`) | +10 | -5 per invalid line |
| No garbage/prose outside of prefixed lines | - | -10 if present |
### 2. Diversity & Coverage (0-30 points)
| Criterion | Points | Deduction |
|-----------|--------|-----------|
| 2+ different types present (lex + vec) | +10 | -10 if only one type |
| 2+ total expansions | +5 | -5 if only one |
| Multiple lex: lines are diverse (edit distance > 3) | +5 | -2 per duplicate pair |
| Multiple vec: lines are diverse (edit distance > 5) | +5 | -2 per duplicate pair |
| lex/vec not identical to original query | +5 | -5 per line that equals query |
### 3. Hyde Quality (0-20 points, optional bonus)
| Criterion | Points | Deduction |
|-----------|--------|-----------|
| Hyde present and well-formed | +5 | - |
| Hyde is concise (50-200 chars) | +5 | -3 if too short, -5 if too long |
| Hyde has no newlines | +5 | -5 if contains newlines |
| Hyde has no excessive repetition | +5 | -3 if word repeats 3+ times |
### 4. Content Quality (0-20 points)
| Criterion | Points | Deduction |
|-----------|--------|-----------|
| Base relevance | +5 | Subjective |
| Lex lines preserve key terms from query | +5 | -5 if lex is generic |
| Lex lines are keyword-focused (shorter) | +5 | -2 if lex is longer than vec |
| Vec lines are natural language (complete phrases) | +5 | -2 if vec is just keywords |
### 5. Named Entity Preservation (-65 to +20 points, CRITICAL)
Named entities are proper nouns, brand names, personal names, technical terms, and acronyms that MUST appear in lex queries. This prevents generic expansions that lose the specific topic.
**Two-level checking:**
| Criterion | Points | Deduction |
|-----------|--------|-----------|
| **Per-line**: All lex lines contain at least one entity | +15 | - |
| **Per-line**: Some lex lines contain entities | +5 | - |
| **Per-line**: NO lex lines contain entities | - | **-30 HEAVY PENALTY** |
| **Per-entity**: Entity completely absent from all lex+vec | - | **-20 per dropped entity** |
| Generic filler phrases in lex | - | -15 per phrase |
| Entities also in vec lines | +5 | - |
**Named Entity Detection:**
- All-caps acronyms: `TDS`, `API`, `GPU`, `AWS`
- Capitalized proper nouns (any position): `React`, `Docker`, `Bob`, `Sarah`
- Personal names at query start: `Bob asked about deploy` → `Bob` is an entity
- Technical terms: `node.js`, `C++`, `.NET`
- CamelCase: `JavaScript`, `TypeScript`
- Compound names: `TDS motorsports` → both words are entities
- Project names: `Project Atlas`, `Horizon team`
**Generic Filler Phrases (BANNED in lex):**
- "find information about"
- "search for", "look up"
- "get information", "learn about"
- "details about", "guide to"
**Examples:**
| Query | Bad Lex | Good Lex |
|-------|---------|----------|
| `who is TDS motorsports` | `lex: find information about` | `lex: TDS motorsports history` |
| | `lex: company details` | `lex: TDS motorsports founders` |
| `meeting with Bob about C++` | `lex: c++ meetings` | `lex: Bob "C++" meeting` |
| | `vec: programming meeting notes` | `vec: meeting notes with Bob about C++` |
| `how to use React hooks` | `lex: programming tutorial` | `lex: React hooks tutorial` |
| | `lex: how to code` | `lex: useEffect useState hooks` |
**Key Rule**: If a query mentions a specific entity (person, brand, product, technology, project name), that entity MUST appear somewhere in the lex+vec output. Dropping a person's name is especially costly.
### 6. Lex Phrase Quoting (bonus, +3 points)
When a query contains multi-word technical terms or proper nouns, lex output should use quoted phrases for exact matching in BM25.
| Criterion | Points |
|-----------|--------|
| Uses `"quoted phrases"` in lex when query has multi-word entities | +3 |
**When to quote:**
- Multi-word proper nouns: `"New York"`, `"Monte Carlo"`
- Specific technical terms: `"machine learning"`, `"rate limit"`
- Exact compound terms: `"connection pool"`, `"merge conflict"`
**When to use negation (`-term`):**
- Disambiguating terms: `rust -corrosion`, `java -coffee`, `apple -fruit`
- Excluding related-but-wrong topics: `"machine learning" -"deep learning"`
- Narrowing scope: `docker -kubernetes`, `python -snake`
**Example:**
```
Query: python memory leak debugging
Good lex: "memory leak" python -java -javascript
Good lex: tracemalloc "garbage collector" profiler
```
## Score Calculation
```
Total Score = Format + Diversity + Hyde + Quality + Entity
Max Score = 120 (100 without hyde)
Normalized = Total / Max (0.0 - 1.0)
```
**Rating:**
- 0.80-1.00: Excellent
- 0.60-0.79: Good
- 0.40-0.59: Acceptable
- 0.20-0.39: Poor
- 0.00-0.19: Failed
**Note:** Entity score can go negative, heavily penalizing outputs that miss named entities.
## Examples
### Good Example (Score: 85/100)
**Query:** `"react hooks"`
**Output:**
```
hyde: React Hooks allow you to use state and lifecycle features in functional components without writing a class.
lex: react hooks tutorial
lex: usestate useeffect
vec: how to use react hooks in functional components
vec: react hooks best practices guide
```
**Scoring:**
- Format: 30/30 (all prefixes valid)
- Diversity: 28/30 (2 types, 4 expansions, good diversity)
- Hyde: 17/20 (present, good length, no repetition)
- Quality: 10/20 (relevant, good structure)
### Bad Example (Score: 15/100)
**Query:** `"auth"`
**Output:**
```
auth is an important concept that relates to authentication.
The answer should be in Chinese.
The answer should be in Chinese.
```
**Scoring:**
- Format: 0/30 (no valid prefixes)
- Diversity: 0/30 (no valid expansions)
- Hyde: 0/20 (N/A)
- Quality: 15/20 (some relevance but wrong format)
### Medium Example (Score: 55/100)
**Query:** `"docker networking"`
**Output:**
```
hyde: Docker networking is an important concept. Docker networking is used for container communication. Docker networking configuration is essential.
lex: docker networking
vec: docker networking
```
**Scoring:**
- Format: 30/30 (valid prefixes)
- Diversity: 10/30 (lex=vec=query, no diversity)
- Hyde: 5/20 (too repetitive - "docker networking" 3x)
- Quality: 10/20 (relevant but low effort)
## Heuristics
### Repetition Detection
```python
def word_repetition_score(text):
words = text.lower().split()
counts = Counter(words)
# Deduct for words appearing 3+ times (excluding stopwords)
stopwords = {'the', 'a', 'an', 'is', 'are', 'to', 'for', 'of', 'in', 'and', 'or'}
repeated = sum(1 for w, c in counts.items() if c >= 3 and w not in stopwords)
return max(0, 5 - repeated * 2)
```
### Diversity Check (Simple)
```python
def is_diverse(a, b, min_distance=3):
"""Check if two strings are sufficiently different."""
a, b = a.lower().strip(), b.lower().strip()
if a == b:
return False
# Simple: check if one is not a substring of the other
if a in b or b in a:
return False
# Check edit distance (simplified)
return len(set(a.split()) ^ set(b.split())) >= min_distance
```
### Query Echo Detection
```python
def echoes_query(expansion, query):
"""Check if expansion is just echoing the query."""
exp = expansion.lower().strip()
q = query.lower().strip()
return exp == q or exp in q or q in exp
```
### Named Entity Extraction
```python
KEY_TERM_STOPWORDS = {'what', 'is', 'how', 'to', 'the', 'a', 'an', 'in', 'on', 'for', 'of',
'and', 'or', 'with', 'my', 'your', 'do', 'does', 'can', 'i', 'me', 'we',
'who', 'where', 'when', 'why', 'which', 'find', 'get', 'show', 'tell'}
def extract_named_entities(query: str) -> set:
"""Extract named entities using simple heuristics."""
entities = set()
words = query.split()
prev_was_entity = False
for i, word in enumerate(words):
clean = word.strip('.,!?:;()[]"\'')
if not clean:
prev_was_entity = False
continue
is_entity = False
# All-caps acronyms: TDS, API, GPU
if clean.isupper() and len(clean) >= 2:
entities.add(clean.lower())
is_entity = True
# Capitalized proper nouns (not first word)
elif i > 0 and clean[0].isupper() and clean.lower() not in KEY_TERM_STOPWORDS:
entities.add(clean.lower())
is_entity = True
# Technical terms: node.js, C++
elif any(c in clean for c in '.+-#@') and len(clean) >= 2:
entities.add(clean.lower())
is_entity = True
# CamelCase: JavaScript
elif len(clean) > 1 and any(c.isupper() for c in clean[1:]) and clean[0].isupper():
entities.add(clean.lower())
is_entity = True
# Word following an entity (compound names: TDS motorsports)
elif prev_was_entity and clean.lower() not in KEY_TERM_STOPWORDS:
entities.add(clean.lower())
is_entity = True
prev_was_entity = is_entity
return entities
```
### Generic Phrase Detection
```python
GENERIC_LEX_PHRASES = {
'find information about', 'search for', 'look up', 'get information',
'learn about', 'information on', 'details about', 'find out about',
'what is', 'how to', 'guide to', 'help with'
}
def lex_is_generic(lex_line: str) -> bool:
"""Check if lex line is a useless generic filler."""
lex_lower = lex_line.lower().strip()
for phrase in GENERIC_LEX_PHRASES:
if phrase in lex_lower:
# Check if there's specific content beyond the generic phrase
remaining = lex_lower
for word in phrase.split():
remaining = remaining.replace(word, '', 1).strip()
if len(remaining) < 3: # Nothing specific left
return True
return False
```
## Training Data Requirements
1. **EOM tokens**: Ensure training examples end with proper end-of-message tokens
2. **Diverse examples**: Include varied query types (short, long, technical, casual)
3. **Quality hyde**: Hyde passages should be informative, not template-y
4. **No repetition**: Avoid "This is important. This is very important." patterns
================================================
FILE: finetune/configs/accelerate_multi_gpu.yaml
================================================
compute_environment: LOCAL_MACHINE
debug: false
distributed_type: MULTI_GPU
downcast_bf16: 'no'
enable_cpu_affinity: false
gpu_ids: all
machine_rank: 0
main_training_function: main
mixed_precision: bf16
num_machines: 1
num_processes: 4
rdzv_backend: static
same_network: true
tpu_env: []
tpu_use_cluster: false
tpu_use_sudo: false
use_cpu: false
================================================
FILE: finetune/configs/sft.yaml
================================================
# SFT Training Config for QMD Query Expansion
# Target: Qwen3-1.7B with LoRA
#
# Usage: uv run train.py sft --config configs/sft.yaml
model:
base: "Qwen/Qwen3-1.7B"
output: "outputs/sft" # Local training output (push to HF manually after eval)
dataset:
# Local: run `uv run dataset/prepare_data.py` first, then use "data/train/"
# HuggingFace: use "tobil/qmd-query-expansion-train" (already prepared)
name: "data/train/"
text_field: "text"
split: "train"
eval_split: 0.1
training:
epochs: 5
batch_size: 4
gradient_accumulation_steps: 4
learning_rate: 2e-4
max_length: 512
warmup_ratio: 0.03
lr_scheduler: "cosine"
# Save checkpoints every 30 minutes
save_interval_minutes: 30
# Fallback time-step save cadence if needed (not used for wall-clock mode)
save_steps: 200
save_total_limit: 3
lora:
rank: 16
alpha: 32
dropout: 0.0
target_modules:
- "q_proj"
- "k_proj"
- "v_proj"
- "o_proj"
- "gate_proj"
- "up_proj"
- "down_proj"
tracking:
project: "qmd-query-expansion"
run_name: "sft-1.7B"
================================================
FILE: finetune/configs/sft_local.yaml
================================================
# SFT Training Config - Local Data, Multi-GPU
# Usage: accelerate launch --config_file configs/accelerate_multi_gpu.yaml train.py sft --config configs/sft_local.yaml
model:
base: "Qwen/Qwen3-1.7B"
output: "outputs/sft" # Local output
push_to_hub: false
dataset:
name: "data/train" # Local path
text_field: "text"
split: "train"
eval_split: 0.1
training:
epochs: 5
batch_size: 2 # Per GPU, effective batch = 2 * 4 GPUs * 4 accum = 32
gradient_accumulation_steps: 4
learning_rate: 0.0002 # 2e-4 as float
max_length: 512
warmup_ratio: 0.03
lr_scheduler: "cosine"
ddp_find_unused_parameters: false
# Save checkpoints every 30 minutes
save_interval_minutes: 30
# Fallback time-step save cadence if needed (not used for wall-clock mode)
save_steps: 200
lora:
rank: 16
alpha: 32
dropout: 0.05
target_modules:
- "q_proj"
- "k_proj"
- "v_proj"
- "o_proj"
- "gate_proj"
- "up_proj"
- "down_proj"
tracking:
project: "qmd-query-expansion"
run_name: "{day} {time}"
================================================
FILE: finetune/convert_gguf.py
================================================
#!/usr/bin/env python3
# /// script
# requires-python = ">=3.10"
# dependencies = [
# "transformers>=4.36.0",
# "peft>=0.7.0",
# "torch>=2.0.0",
# "accelerate>=0.24.0",
# "huggingface_hub>=0.20.0",
# "sentencepiece>=0.1.99",
# "protobuf>=3.20.0",
# "numpy",
# "gguf",
# ]
# ///
"""
Convert QMD query expansion model to GGUF format.
Loads the base model, merges SFT and GRPO adapters, then converts to
GGUF with multiple quantizations for use with Ollama/llama.cpp/LM Studio.
Usage:
uv run convert_gguf.py --size 1.7B
uv run convert_gguf.py --size 4B --skip-quantize
uv run convert_gguf.py --base Qwen/Qwen3-1.7B \
--sft tobil/qmd-query-expansion-1.7B-sft \
--grpo tobil/qmd-query-expansion-1.7B-grpo \
--output tobil/qmd-query-expansion-1.7B-gguf
"""
import argparse
import os
import subprocess
import sys
import torch
from huggingface_hub import HfApi, login
from peft import PeftModel
from transformers import AutoModelForCausalLM, AutoTokenizer
# Preset configurations for each model size
PRESETS = {
"1.7B": {
"base": "Qwen/Qwen3-1.7B",
"sft": "tobil/qmd-query-expansion-1.7B-sft",
"grpo": "tobil/qmd-query-expansion-1.7B-grpo",
"output": "tobil/qmd-query-expansion-1.7B-gguf",
"ollama_name": "qmd-expand",
},
"4B": {
"base": "Qwen/Qwen3-4B",
"sft": "tobil/qmd-query-expansion-4B-sft",
"grpo": "tobil/qmd-query-expansion-4B-grpo",
"output": "tobil/qmd-query-expansion-4B-gguf",
"ollama_name": "qmd-expand-4b",
},
}
def run_cmd(cmd, description):
"""Run a shell command with error handling."""
print(f" {description}...")
try:
subprocess.run(cmd, check=True, capture_output=True, text=True)
return True
except subprocess.CalledProcessError as e:
print(f" FAILED: {' '.join(cmd)}")
if e.stderr:
print(f" {e.stderr[:500]}")
return False
except FileNotFoundError:
print(f" Command not found: {cmd[0]}")
return False
def main():
parser = argparse.ArgumentParser(description="Convert QMD model to GGUF")
parser.add_argument("--size", choices=PRESETS.keys(), help="Use preset config for model size")
parser.add_argument("--base", help="Base model (overrides preset)")
parser.add_argument("--sft", help="SFT adapter (overrides preset)")
parser.add_argument("--grpo", help="GRPO adapter (overrides preset)")
parser.add_argument("--output", help="Output HF repo (overrides preset)")
parser.add_argument("--skip-quantize", action="store_true", help="Only produce FP16 GGUF")
parser.add_argument("--no-upload", action="store_true", help="Don't upload to HF Hub")
args = parser.parse_args()
# Resolve config
if args.size:
preset = PRESETS[args.size]
base_model = args.base or preset["base"]
sft_model = args.sft or preset["sft"]
grpo_model = args.grpo or preset["grpo"]
output_repo = args.output or preset["output"]
elif args.base and args.sft and args.grpo and args.output:
base_model = args.base
sft_model = args.sft
grpo_model = args.grpo
output_repo = args.output
else:
parser.error("Either --size or all of --base/--sft/--grpo/--output are required")
model_name = output_repo.split("/")[-1].replace("-gguf", "")
print(f"QMD GGUF Conversion: {model_name}")
print("=" * 60)
# Install build tools (for Colab/cloud environments)
print("\nInstalling build dependencies...")
subprocess.run(["apt-get", "update", "-qq"], capture_output=True)
subprocess.run(["apt-get", "install", "-y", "-qq", "build-essential", "cmake", "git"], capture_output=True)
# Login
hf_token = os.environ.get("HF_TOKEN")
if hf_token:
print("Logging in to HuggingFace...")
login(token=hf_token)
# Step 1: Load and merge
print(f"\nStep 1: Loading base model {base_model}...")
model = AutoModelForCausalLM.from_pretrained(
base_model, torch_dtype=torch.bfloat16, device_map="auto", trust_remote_code=True,
)
print(f"Step 2: Merging SFT adapter {sft_model}...")
model = PeftModel.from_pretrained(model, sft_model)
model = model.merge_and_unload()
print(f"Step 3: Merging GRPO adapter {grpo_model}...")
model = PeftModel.from_pretrained(model, grpo_model)
model = model.merge_and_unload()
tokenizer = AutoTokenizer.from_pretrained(base_model, trust_remote_code=True)
# Step 2: Save merged model
merged_dir = "/tmp/merged_model"
print(f"\nStep 4: Saving merged model to {merged_dir}...")
model.save_pretrained(merged_dir, safe_serialization=True)
tokenizer.save_pretrained(merged_dir)
# Step 3: Setup llama.cpp
print("\nStep 5: Setting up llama.cpp...")
if not os.path.exists("/tmp/llama.cpp"):
run_cmd(["git", "clone", "--depth", "1", "https://github.com/ggerganov/llama.cpp.git", "/tmp/llama.cpp"],
"Cloning llama.cpp")
subprocess.run([sys.executable, "-m", "pip", "install", "-q", "-r", "/tmp/llama.cpp/requirements.txt"],
capture_output=True)
# Step 4: Convert to FP16 GGUF
gguf_dir = "/tmp/gguf_output"
os.makedirs(gguf_dir, exist_ok=True)
gguf_file = f"{gguf_dir}/{model_name}-f16.gguf"
print(f"\nStep 6: Converting to FP16 GGUF...")
if not run_cmd([sys.executable, "/tmp/llama.cpp/convert_hf_to_gguf.py",
merged_dir, "--outfile", gguf_file, "--outtype", "f16"],
"Converting"):
sys.exit(1)
size_mb = os.path.getsize(gguf_file) / (1024 * 1024)
print(f" FP16: {size_mb:.1f} MB")
# Step 5: Quantize
quantized_files = []
if not args.skip_quantize:
print("\nStep 7: Building quantize tool...")
os.makedirs("/tmp/llama.cpp/build", exist_ok=True)
run_cmd(["cmake", "-B", "/tmp/llama.cpp/build", "-S", "/tmp/llama.cpp", "-DGGML_CUDA=OFF"],
"CMake configure")
run_cmd(["cmake", "--build", "/tmp/llama.cpp/build", "--target", "llama-quantize", "-j", "4"],
"Building llama-quantize")
quantize_bin = "/tmp/llama.cpp/build/bin/llama-quantize"
print("\nStep 8: Quantizing...")
for quant_type, desc in [("Q4_K_M", "4-bit"), ("Q5_K_M", "5-bit"), ("Q8_0", "8-bit")]:
qfile = f"{gguf_dir}/{model_name}-{quant_type.lower()}.gguf"
if run_cmd([quantize_bin, gguf_file, qfile, quant_type], f"{quant_type} ({desc})"):
qsize = os.path.getsize(qfile) / (1024 * 1024)
print(f" {quant_type}: {qsize:.1f} MB")
quantized_files.append((qfile, quant_type))
# Step 6: Upload
if not args.no_upload:
print(f"\nStep 9: Uploading to {output_repo}...")
api = HfApi()
api.create_repo(repo_id=output_repo, repo_type="model", exist_ok=True)
api.upload_file(path_or_fileobj=gguf_file,
path_in_repo=f"{model_name}-f16.gguf", repo_id=output_repo)
for qfile, qtype in quantized_files:
api.upload_file(path_or_fileobj=qfile,
path_in_repo=f"{model_name}-{qtype.lower()}.gguf", repo_id=output_repo)
# Upload README
readme = f"""---
base_model: {base_model}
tags: [gguf, llama.cpp, quantized, query-expansion, qmd]
---
# {model_name} (GGUF)
GGUF conversion of the QMD Query Expansion model.
## Details
- **Base:** {base_model}
- **SFT:** {sft_model}
- **GRPO:** {grpo_model}
- **Task:** Query expansion (lex/vec/hyde format)
## Prompt Format
```
<|im_start|>user
/no_think Expand this search query: your query here<|im_end|>
<|im_start|>assistant
```
"""
api.upload_file(path_or_fileobj=readme.encode(),
path_in_repo="README.md", repo_id=output_repo)
print(f"\nDone! Repository: https://huggingface.co/{output_repo}")
if __name__ == "__main__":
main()
================================================
FILE: finetune/convert_onnx.py
================================================
#!/usr/bin/env python3
# /// script
# requires-python = ">=3.10"
# dependencies = [
# "transformers>=4.36.0",
# "peft>=0.7.0",
# "torch>=2.0.0",
# "accelerate>=0.24.0",
# "huggingface_hub>=0.20.0",
# "sentencepiece>=0.1.99",
# "protobuf>=3.20.0",
# "numpy",
# "optimum[onnxruntime]",
# "onnx>=1.15.0",
# "onnxruntime>=1.17.0",
# "onnxconverter-common>=1.14.0",
# ]
# ///
"""
Convert QMD query expansion model to ONNX format for Transformers.js.
Loads the base model, merges SFT and GRPO adapters, then exports to ONNX
with quantization for browser deployment via Transformers.js + WebGPU.
Usage:
uv run convert_onnx.py --size 1.7B
uv run convert_onnx.py --size 1.7B --no-upload
uv run convert_onnx.py --base Qwen/Qwen3-1.7B \
--sft tobil/qmd-query-expansion-1.7B-sft \
--grpo tobil/qmd-query-expansion-1.7B-grpo \
--output tobil/qmd-query-expansion-1.7B-ONNX
Quantization options:
--quantize q4 MatMulNBits 4-bit (default, smallest)
--quantize q8 8-bit dynamic quantization
--quantize fp16 FP16 (requires GPU export)
--quantize none No quantization (FP32, ~7GB)
"""
import argparse
import json
import os
import shutil
import subprocess
import sys
from pathlib import Path
import torch
from huggingface_hub import HfApi, login
from peft import PeftModel
from transformers import AutoModelForCausalLM, AutoTokenizer
PRESETS = {
"1.7B": {
"base": "Qwen/Qwen3-1.7B",
"sft": "tobil/qmd-query-expansion-1.7B-sft",
"grpo": "tobil/qmd-query-expansion-1.7B-grpo",
"output": "tobil/qmd-query-expansion-1.7B-ONNX",
},
"4B": {
"base": "Qwen/Qwen3-4B",
"sft": "tobil/qmd-query-expansion-4B-sft",
"grpo": "tobil/qmd-query-expansion-4B-grpo",
"output": "tobil/qmd-query-expansion-4B-ONNX",
},
}
def merge_adapters(base_model: str, sft_model: str, grpo_model: str) -> tuple:
"""Load base model, merge SFT + GRPO adapters, return (model, tokenizer)."""
print(f"\nStep 1: Loading base model {base_model}...")
model = AutoModelForCausalLM.from_pretrained(
base_model, dtype=torch.float32, trust_remote_code=True,
)
print(f"Step 2: Merging SFT adapter {sft_model}...")
model = PeftModel.from_pretrained(model, sft_model)
model = model.merge_and_unload()
print(f"Step 3: Merging GRPO adapter {grpo_model}...")
model = PeftModel.from_pretrained(model, grpo_model)
model = model.merge_and_unload()
tokenizer = AutoTokenizer.from_pretrained(base_model, trust_remote_code=True)
return model, tokenizer
def export_onnx(model, tokenizer, output_dir: str):
"""Export merged model to ONNX using Optimum."""
from optimum.exporters.onnx import main_export
# Save merged model to temp dir first (Optimum needs HF format on disk)
merged_dir = "/tmp/merged_model_onnx"
print(f"\nStep 4: Saving merged model to {merged_dir}...")
model.save_pretrained(merged_dir, safe_serialization=True)
tokenizer.save_pretrained(merged_dir)
print(f"\nStep 5: Exporting to ONNX at {output_dir}...")
# no_post_process=True avoids the 2GB protobuf serialization limit
# that occurs during tied-weight deduplication on large FP32 models.
# The exported model still works correctly — the tied weights just
# aren't deduplicated in the graph, which is fine since we quantize next.
main_export(
model_name_or_path=merged_dir,
output=output_dir,
task="text-generation-with-past",
device="cpu",
fp16=False,
no_post_process=True,
)
# Clean up temp merged dir
shutil.rmtree(merged_dir, ignore_errors=True)
def _find_onnx_model(onnx_dir: str) -> Path:
"""Find the main ONNX model file in the output directory."""
model_path = Path(onnx_dir) / "model.onnx"
if model_path.exists():
return model_path
candidates = list(Path(onnx_dir).glob("*.onnx"))
if not candidates:
raise FileNotFoundError(f"No .onnx files found in {onnx_dir}")
return candidates[0]
def quantize_onnx(onnx_dir: str, quantize_type: str):
"""Quantize the exported ONNX model."""
if quantize_type == "none":
print("\nSkipping quantization (FP32).")
return
model_path = _find_onnx_model(onnx_dir)
print(f"\nStep 6: Quantizing {model_path.name} ({quantize_type})...")
if quantize_type == "q4":
_quantize_q4(model_path)
elif quantize_type == "q8":
_quantize_q8(model_path)
elif quantize_type == "fp16":
_convert_fp16(model_path)
def _quantize_q4(model_path: Path):
"""4-bit MatMulNBits quantization via onnxruntime. Needs ~16GB RAM for 1.7B models."""
from onnxruntime.quantization import matmul_nbits_quantizer
q_path = model_path.with_name(model_path.stem + "_q4" + model_path.suffix)
quant = matmul_nbits_quantizer.MatMulNBitsQuantizer(
model=str(model_path),
block_size=32,
is_symmetric=True,
bits=4,
)
quant.process()
quant.model.save(str(q_path))
# Remove original FP32 files, keep only quantized
if q_path.exists():
_report_size(q_path)
model_path.unlink(missing_ok=True)
data_path = model_path.with_name(model_path.name + "_data")
data_path.unlink(missing_ok=True)
# Rename quantized to model.onnx for Transformers.js compatibility
q_path.rename(model_path)
print(f" Renamed {q_path.name} -> {model_path.name}")
def _quantize_q8(model_path: Path):
"""8-bit dynamic quantization via onnxruntime."""
from onnxruntime.quantization import quantize_dynamic, QuantType
q_path = model_path.with_name(model_path.stem + "_q8" + model_path.suffix)
quantize_dynamic(
model_input=str(model_path),
model_output=str(q_path),
weight_type=QuantType.QUInt8,
)
if q_path.exists():
_report_size(q_path)
model_path.unlink(missing_ok=True)
data_path = model_path.with_name(model_path.name + "_data")
data_path.unlink(missing_ok=True)
q_path.rename(model_path)
print(f" Renamed {q_path.name} -> {model_path.name}")
def _convert_fp16(model_path: Path):
"""Convert ONNX model weights to FP16."""
from onnxconverter_common import float16
import onnx
print(" Converting to FP16...")
model = onnx.load(str(model_path), load_external_data=True)
model_fp16 = float16.convert_float_to_float16(model, keep_io_types=True)
fp16_path = model_path.with_name(model_path.stem + "_fp16" + model_path.suffix)
onnx.save(model_fp16, str(fp16_path))
if fp16_path.exists():
_report_size(fp16_path)
model_path.unlink(missing_ok=True)
data_path = model_path.with_name(model_path.name + "_data")
data_path.unlink(missing_ok=True)
fp16_path.rename(model_path)
print(f" Renamed {fp16_path.name} -> {model_path.name}")
def _report_size(path: Path):
"""Print file size in MB."""
size_mb = path.stat().st_size / (1024 * 1024)
print(f" {path.name}: {size_mb:.1f} MB")
def validate_onnx(onnx_dir: str, base_model: str):
"""Run a sample inference through the ONNX model to verify it works."""
import onnxruntime as ort
import numpy as np
model_path = _find_onnx_model(onnx_dir)
print(f"\nValidation: loading {model_path.name}...")
tokenizer = AutoTokenizer.from_pretrained(onnx_dir, trust_remote_code=True)
session = ort.InferenceSession(
str(model_path),
providers=["CPUExecutionProvider"],
)
# Tokenize a test prompt
test_query = "/no_think Expand this search query: distributed consensus"
chat_prompt = tokenizer.apply_chat_template(
[{"role": "user", "content": test_query}],
add_generation_prompt=True,
tokenize=False,
)
inputs = tokenizer(chat_prompt, return_tensors="np")
input_ids = inputs["input_ids"].astype(np.int64)
attention_mask = inputs["attention_mask"].astype(np.int64)
# Build feed dict with all required inputs
seq_len = input_ids.shape[1]
feed = {"input_ids": input_ids, "attention_mask": attention_mask}
# Add position_ids if needed
all_inputs = {inp.name: inp for inp in session.get_inputs()}
if "position_ids" in all_inputs:
feed["position_ids"] = np.arange(seq_len, dtype=np.int64).reshape(1, -1)
# Initialize past_key_values to zeros if the model expects them
for name, inp in sorted(all_inputs.items()):
if name.startswith("past_key_values"):
shape = []
for dim in inp.shape:
shape.append(dim if isinstance(dim, int) else 0)
# batch dim = 1
if shape and shape[0] == 0:
shape[0] = 1
feed[name] = np.zeros(shape, dtype=np.float32)
# Run inference
output_names = [o.name for o in session.get_outputs()]
results = session.run(output_names, feed)
# Check logits shape
logits = results[0]
print(f" Input tokens: {input_ids.shape[1]}")
print(f" Output logits shape: {logits.shape}")
print(f" Logits range: [{logits.min():.2f}, {logits.max():.2f}]")
# Greedy decode next token
next_token_id = int(np.argmax(logits[0, -1, :]))
next_token = tokenizer.decode([next_token_id])
print(f" Next token: {repr(next_token)} (id={next_token_id})")
# Check KV cache outputs exist
kv_outputs = [n for n in output_names if n.startswith("present")]
if kv_outputs:
print(f" KV cache outputs: {len(kv_outputs)} tensors (generation-ready)")
else:
print(" WARNING: No KV cache outputs — model may not support efficient generation")
# Sanity checks
assert logits.shape[0] == 1, "Batch size mismatch"
assert logits.shape[1] == input_ids.shape[1], "Sequence length mismatch"
assert logits.max() > logits.min(), "Logits are constant (broken model)"
assert not np.isnan(logits).any(), "Logits contain NaN"
assert not np.isinf(logits).any(), "Logits contain Inf"
print(" Validation PASSED")
def write_transformers_js_config(onnx_dir: str, quantize_type: str = "q4"):
"""Write Transformers.js compatibility config."""
config_path = Path(onnx_dir) / "transformers_js_config.json"
config = {
"model_type": "text-generation",
"quantized": quantize_type != "none",
}
config_path.write_text(json.dumps(config, indent=2) + "\n")
print(f" Wrote {config_path.name}")
def upload_to_hub(
onnx_dir: str,
output_repo: str,
base_model: str,
sft_model: str,
grpo_model: str,
quantize_type: str = "q4",
):
"""Upload ONNX model to HuggingFace Hub."""
print(f"\nStep 7: Uploading to {output_repo}...")
api = HfApi()
api.create_repo(repo_id=output_repo, repo_type="model", exist_ok=True)
api.upload_folder(
folder_path=onnx_dir,
repo_id=output_repo,
commit_message="Upload ONNX model",
)
# Map quantize_type to Transformers.js dtype values
dtype_map = {"q4": "q4", "q8": "q8", "fp16": "fp16", "none": "fp32"}
tj_dtype = dtype_map.get(quantize_type, "fp32")
format_desc = "FP32 (no quantization)" if quantize_type == "none" else f"{quantize_type.upper()} quantization"
repo_name = output_repo.split("/")[-1]
readme = f"""---
base_model: {base_model}
tags: [onnx, transformers.js, webgpu, query-expansion, qmd]
library_name: transformers.js
---
# {repo_name}
ONNX conversion of the QMD Query Expansion model for use with
[Transformers.js](https://huggingface.co/docs/transformers.js) and WebGPU.
## Details
- **Base:** {base_model}
- **SFT:** {sft_model}
- **GRPO:** {grpo_model}
- **Task:** Query expansion (lex/vec/hyde format)
- **Format:** ONNX with {format_desc}
## Usage with Transformers.js
```javascript
import {{ AutoTokenizer, AutoModelForCausalLM }} from "@huggingface/transformers";
const tokenizer = await AutoTokenizer.from_pretrained("{output_repo}");
const model = await AutoModelForCausalLM.from_pretrained("{output_repo}", {{
dtype: "{tj_dtype}",
device: "webgpu",
}});
```
## Prompt Format
```
<|im_start|>user
/no_think Expand this search query: your query here<|im_end|>
<|im_start|>assistant
```
"""
api.upload_file(
path_or_fileobj=readme.encode(),
path_in_repo="README.md",
repo_id=output_repo,
)
def main():
parser = argparse.ArgumentParser(description="Convert QMD model to ONNX")
parser.add_argument(
"--size", choices=PRESETS.keys(), help="Use preset config for model size",
)
parser.add_argument("--base", help="Base model (overrides preset)")
parser.add_argument("--sft", help="SFT adapter (overrides preset)")
parser.add_argument("--grpo", help="GRPO adapter (overrides preset)")
parser.add_argument("--output", help="Output HF repo (overrides preset)")
parser.add_argument(
"--quantize",
choices=["q4", "q8", "fp16", "none"],
default="q4",
help="Quantization type (default: q4)",
)
parser.add_argument(
"--no-upload", action="store_true", help="Don't upload to HF Hub",
)
parser.add_argument(
"--validate", action="store_true",
help="Run inference validation on exported model",
)
parser.add_argument(
"--validate-only", metavar="DIR",
help="Skip export, only validate an existing ONNX dir",
)
args = parser.parse_args()
# Validate-only mode: skip export, just run validation
if args.validate_only:
validate_onnx(args.validate_only, "")
return
# Resolve config
if args.size:
preset = PRESETS[args.size]
base_model = args.base or preset["base"]
sft_model = args.sft or preset["sft"]
grpo_model = args.grpo or preset["grpo"]
output_repo = args.output or preset["output"]
elif args.base and args.sft and args.grpo and args.output:
base_model = args.base
sft_model = args.sft
grpo_model = args.grpo
output_repo = args.output
else:
parser.error(
"Either --size or all of --base/--sft/--grpo/--output are required",
)
model_name = output_repo.split("/")[-1]
print(f"QMD ONNX Conversion: {model_name}")
print("=" * 60)
# Login
hf_token = os.environ.get("HF_TOKEN")
if hf_token:
print("Logging in to HuggingFace...")
login(token=hf_token)
# Merge adapters
model, tokenizer = merge_adapters(base_model, sft_model, grpo_model)
# Export to ONNX
onnx_dir = f"/tmp/onnx_output/{model_name}"
os.makedirs(onnx_dir, exist_ok=True)
export_onnx(model, tokenizer, onnx_dir)
# Quantize
quantize_onnx(onnx_dir, args.quantize)
# Write Transformers.js config
write_transformers_js_config(onnx_dir, args.quantize)
# Validate
if args.validate:
validate_onnx(onnx_dir, base_model)
# Upload
if not args.no_upload:
upload_to_hub(onnx_dir, output_repo, base_model, sft_model, grpo_model, args.quantize)
print(f"\nDone! ONNX files at: {onnx_dir}")
if not args.no_upload:
print(f"Repository: https://huggingface.co/{output_repo}")
if __name__ == "__main__":
main()
================================================
FILE: finetune/data/qmd_expansion_balanced_deduped.jsonl
================================================
{"output": [["lex", "orchestra overview instruments guide tutorial"], ["lex", "orchestra overview instruments guide guide"], ["lex", "orchestra overview instruments guide tips"], ["vec", "learn orchestra instruments guide step by step"], ["vec", "best way to orchestra instruments guide"], ["hyde", "Learning orchestra instruments guide requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "arts_culture", "is_short": false, "query": "orchestra instruments guide"}
{"output": [["lex", "Asian overview cooking techniques advice"], ["lex", "Asian overview cooking techniques tips"], ["lex", "Asian overview cooking techniques tutorial"], ["vec", "learn Asian cooking techniques step by step"], ["vec", "tips for Asian cooking techniques success"], ["hyde", "Learning Asian cooking techniques requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "food_cooking", "is_short": false, "query": "Asian cooking techniques"}
{"output": [["lex", "hike how to overview"], ["lex", "hike guide overview"], ["lex", "hike advice overview"], ["vec", "learn hike step by step"], ["vec", "how to hike effectively"], ["hyde", "Learning hike requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "food_cooking", "is_short": true, "query": "hike"}
{"output": [["lex", "how overview to metal smithing tutorial"], ["lex", "how overview to metal smithing how to"], ["lex", "how overview to metal smithing tips"], ["vec", "learn how to metal smithing step by step"], ["vec", "how to how to metal smithing effectively"], ["hyde", "This comprehensive guide to how to metal smithing covers all the essential information you need to get started. Follow the steps carefully for best results."]], "category": "hobbies_crafts", "is_short": false, "query": "how to metal smithing"}
{"output": [["lex", "international overview cuisine recipe tips"], ["lex", "international overview cuisine recipe guide"], ["lex", "international overview cuisine recipe how to"], ["vec", "learn international cuisine recipe step by step"], ["vec", "best way to international cuisine recipe"], ["hyde", "This comprehensive guide to international cuisine recipe covers all the essential information you need to get started. Follow the steps carefully for best results."]], "category": "food_cooking", "is_short": false, "query": "international cuisine recipe"}
{"output": [["lex", "gallery guide overview"], ["lex", "gallery how to overview"], ["lex", "gallery tutorial overview"], ["vec", "best way to gallery"], ["vec", "how to gallery effectively"], ["hyde", "This comprehensive guide to gallery covers all the essential information you need to get started. Follow the steps carefully for best results."]], "category": "hobbies_crafts", "is_short": true, "query": "gallery"}
{"output": [["lex", "how overview to luggage selection advice"], ["lex", "how overview to luggage selection guide"], ["lex", "how overview to luggage selection how to"], ["vec", "learn how to luggage selection step by step"], ["vec", "complete guide to how to luggage selection"], ["hyde", "This comprehensive guide to how to luggage selection covers all the essential information you need to get started. Follow the steps carefully for best results."]], "category": "travel_geography", "is_short": false, "query": "how to luggage selection"}
{"output": [["lex", "best overview choose paint color tutorial"], ["lex", "best overview choose paint color tips"], ["lex", "best overview choose paint color how to"], ["vec", "tips for best choose paint color success"], ["vec", "learn best choose paint color step by step"], ["hyde", "Whether you're a beginner or looking to improve, this guide to best choose paint color offers practical advice, common pitfalls to avoid, and proven strategies for success."]], "category": "home_garden", "is_short": false, "query": "best choose paint color"}
{"output": [["lex", "best overview posture correction how to"], ["lex", "best overview posture correction tips"], ["lex", "best overview posture correction guide"], ["vec", "best way to best posture correction"], ["vec", "complete guide to best posture correction"], ["hyde", "This comprehensive guide to best posture correction covers all the essential information you need to get started. Follow the steps carefully for best results."]], "category": "health_wellness", "is_short": false, "query": "best posture correction"}
{"output": [["lex", "gluten overview free techniques tips"], ["lex", "gluten overview free techniques advice"], ["lex", "gluten overview free techniques guide"], ["vec", "how to gluten free techniques effectively"], ["vec", "tips for gluten free techniques success"], ["hyde", "Whether you're a beginner or looking to improve, this guide to gluten free techniques offers practical advice, common pitfalls to avoid, and proven strategies for success."]], "category": "food_cooking", "is_short": false, "query": "gluten free techniques"}
{"output": [["lex", "pruning overview trees tutorial tips"], ["lex", "pruning overview trees tutorial advice"], ["lex", "pruning overview trees tutorial how to"], ["vec", "tips for pruning trees tutorial success"], ["vec", "how to pruning trees tutorial effectively"], ["hyde", "Learning pruning trees tutorial requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "home_garden", "is_short": false, "query": "pruning trees tutorial"}
{"output": [["lex", "understand overview modern art advice"], ["lex", "understand overview modern art how to"], ["lex", "understand overview modern art guide"], ["vec", "complete guide to understand modern art"], ["vec", "best way to understand modern art"], ["hyde", "This comprehensive guide to understand modern art covers all the essential information you need to get started. Follow the steps carefully for best results."]], "category": "arts_culture", "is_short": false, "query": "understand modern art"}
{"output": [["lex", "best overview travel phrases tutorial"], ["lex", "best overview travel phrases tips"], ["lex", "best overview travel phrases guide"], ["vec", "tips for best travel phrases success"], ["vec", "best way to best travel phrases"], ["hyde", "Whether you're a beginner or looking to improve, this guide to best travel phrases offers practical advice, common pitfalls to avoid, and proven strategies for success."]], "category": "travel_geography", "is_short": false, "query": "best travel phrases"}
{"output": [["lex", "how overview to translation apps guide"], ["lex", "how overview to translation apps tips"], ["lex", "how overview to translation apps how to"], ["vec", "how to how to translation apps effectively"], ["vec", "tips for how to translation apps success"], ["hyde", "This comprehensive guide to how to translation apps covers all the essential information you need to get started. Follow the steps carefully for best results."]], "category": "travel_geography", "is_short": false, "query": "how to translation apps"}
{"output": [["lex", "how overview to networking tips"], ["lex", "how overview to networking guide"], ["lex", "how overview to networking tutorial"], ["vec", "tips for how to networking success"], ["vec", "how to how to networking effectively"], ["hyde", "Learning how to networking requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "lifestyle_relationships", "is_short": false, "query": "how to networking"}
{"output": [["lex", "repair advice overview"], ["lex", "repair guide overview"], ["lex", "repair tips overview"], ["vec", "learn repair step by step"], ["vec", "tips for repair success"], ["hyde", "Learning repair requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "hobbies_crafts", "is_short": true, "query": "repair"}
{"output": [["lex", "improve overview acupuncture tutorial"], ["lex", "improve overview acupuncture tips"], ["lex", "improve overview acupuncture advice"], ["vec", "tips for improve acupuncture success"], ["vec", "how to improve acupuncture effectively"], ["hyde", "This comprehensive guide to improve acupuncture covers all the essential information you need to get started. Follow the steps carefully for best results."]], "category": "health_wellness", "is_short": false, "query": "improve acupuncture"}
{"output": [["lex", "home overview workout for beginners tutorial"], ["lex", "home overview workout for beginners advice"], ["lex", "home overview workout for beginners guide"], ["vec", "complete guide to home workout for beginners"], ["vec", "best way to home workout for beginners"], ["hyde", "This comprehensive guide to home workout for beginners covers all the essential information you need to get started. Follow the steps carefully for best results."]], "category": "health_wellness", "is_short": false, "query": "home workout for beginners"}
{"output": [["lex", "online overview privacy setup advice"], ["lex", "online overview privacy setup how to"], ["lex", "online overview privacy setup tips"], ["vec", "tips for online privacy setup success"], ["vec", "complete guide to online privacy setup"], ["hyde", "Learning online privacy setup requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "technology", "is_short": false, "query": "online privacy setup"}
{"output": [["lex", "gift overview giving tips tips"], ["lex", "gift overview giving tips how to"], ["lex", "gift overview giving tips tutorial"], ["vec", "learn gift giving tips step by step"], ["vec", "best way to gift giving tips"], ["hyde", "Learning gift giving tips requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "lifestyle_relationships", "is_short": false, "query": "gift giving tips"}
{"output": [["lex", "visa tips overview"], ["lex", "visa tutorial overview"], ["lex", "visa how to overview"], ["vec", "complete guide to visa"], ["vec", "how to visa effectively"], ["hyde", "Whether you're a beginner or looking to improve, this guide to visa offers practical advice, common pitfalls to avoid, and proven strategies for success."]], "category": "health_wellness", "is_short": true, "query": "visa"}
{"output": [["lex", "start overview real estate investing advice"], ["lex", "start overview real estate investing tips"], ["lex", "start overview real estate investing tutorial"], ["vec", "tips for start real estate investing success"], ["vec", "best way to start real estate investing"], ["hyde", "Learning start real estate investing requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "finance_business", "is_short": false, "query": "start real estate investing"}
{"output": [["lex", "design overview principles history guide"], ["lex", "design overview principles history advice"], ["lex", "design overview principles history how to"], ["vec", "tips for design principles history success"], ["vec", "learn design principles history step by step"], ["hyde", "Whether you're a beginner or looking to improve, this guide to design principles history offers practical advice, common pitfalls to avoid, and proven strategies for success."]], "category": "arts_culture", "is_short": false, "query": "design principles history"}
{"output": [["lex", "DIY overview choose paint color tips"], ["lex", "DIY overview choose paint color how to"], ["lex", "DIY overview choose paint color guide"], ["vec", "complete guide to DIY choose paint color"], ["vec", "how to DIY choose paint color effectively"], ["hyde", "This comprehensive guide to DIY choose paint color covers all the essential information you need to get started. Follow the steps carefully for best results."]], "category": "home_garden", "is_short": false, "query": "DIY choose paint color"}
{"output": [["lex", "understand overview dance styles tips"], ["lex", "understand overview dance styles guide"], ["lex", "understand overview dance styles tutorial"], ["vec", "learn understand dance styles step by step"], ["vec", "best way to understand dance styles"], ["hyde", "Whether you're a beginner or looking to improve, this guide to understand dance styles offers practical advice, common pitfalls to avoid, and proven strategies for success."]], "category": "arts_culture", "is_short": false, "query": "understand dance styles"}
{"output": [["lex", "401k overview tips guide"], ["lex", "401k overview tips how to"], ["lex", "401k overview tips tutorial"], ["vec", "best way to 401k tips"], ["vec", "tips for 401k tips success"], ["hyde", "This comprehensive guide to 401k tips covers all the essential information you need to get started. Follow the steps carefully for best results."]], "category": "finance_business", "is_short": false, "query": "401k tips"}
{"output": [["lex", "stress overview management guide tips"], ["lex", "stress overview management guide advice"], ["lex", "stress overview management guide guide"], ["vec", "tips for stress management guide success"], ["vec", "how to stress management guide effectively"], ["hyde", "Learning stress management guide requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "health_wellness", "is_short": false, "query": "stress management guide"}
{"output": [["lex", "how overview to ceramics how to"], ["lex", "how overview to ceramics guide"], ["lex", "how overview to ceramics tutorial"], ["vec", "tips for how to ceramics success"], ["vec", "complete guide to how to ceramics"], ["hyde", "Whether you're a beginner or looking to improve, this guide to how to ceramics offers practical advice, common pitfalls to avoid, and proven strategies for success."]], "category": "hobbies_crafts", "is_short": false, "query": "how to ceramics"}
{"output": [["lex", "laundry overview tips tutorial guide"], ["lex", "laundry overview tips tutorial how to"], ["lex", "laundry overview tips tutorial tutorial"], ["vec", "tips for laundry tips tutorial success"], ["vec", "complete guide to laundry tips tutorial"], ["hyde", "Learning laundry tips tutorial requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "home_garden", "is_short": false, "query": "laundry tips tutorial"}
{"output": [["lex", "wood overview carving basics tutorial"], ["lex", "wood overview carving basics guide"], ["lex", "wood overview carving basics advice"], ["vec", "best way to wood carving basics"], ["vec", "complete guide to wood carving basics"], ["hyde", "Learning wood carving basics requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "hobbies_crafts", "is_short": false, "query": "wood carving basics"}
{"output": [["lex", "start overview retirement planning tips"], ["lex", "start overview retirement planning guide"], ["lex", "start overview retirement planning advice"], ["vec", "best way to start retirement planning"], ["vec", "complete guide to start retirement planning"], ["hyde", "Whether you're a beginner or looking to improve, this guide to start retirement planning offers practical advice, common pitfalls to avoid, and proven strategies for success."]], "category": "finance_business", "is_short": false, "query": "start retirement planning"}
{"output": [["lex", "stock overview market tips how to"], ["lex", "stock overview market tips tips"], ["lex", "stock overview market tips guide"], ["vec", "complete guide to stock market tips"], ["vec", "tips for stock market tips success"], ["hyde", "Learning stock market tips requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "finance_business", "is_short": false, "query": "stock market tips"}
{"output": [["lex", "acupuncture overview for beginners guide"], ["lex", "acupuncture overview for beginners tutorial"], ["lex", "acupuncture overview for beginners how to"], ["vec", "complete guide to acupuncture for beginners"], ["vec", "learn acupuncture for beginners step by step"], ["hyde", "This comprehensive guide to acupuncture for beginners covers all the essential information you need to get started. Follow the steps carefully for best results."]], "category": "health_wellness", "is_short": false, "query": "acupuncture for beginners"}
{"output": [["lex", "fix overview two factor auth how to"], ["lex", "fix overview two factor auth advice"], ["lex", "fix overview two factor auth guide"], ["vec", "tips for fix two factor auth success"], ["vec", "best way to fix two factor auth"], ["hyde", "Whether you're a beginner or looking to improve, this guide to fix two factor auth offers practical advice, common pitfalls to avoid, and proven strategies for success."]], "category": "technology", "is_short": false, "query": "fix two factor auth"}
{"output": [["lex", "woodwork tips overview"], ["lex", "woodwork advice overview"], ["lex", "woodwork guide overview"], ["vec", "how to woodwork effectively"], ["vec", "tips for woodwork success"], ["hyde", "Learning woodwork requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "lifestyle_relationships", "is_short": true, "query": "woodwork"}
{"output": [["lex", "visual overview learner guide how to"], ["lex", "visual overview learner guide guide"], ["lex", "visual overview learner guide tips"], ["vec", "how to visual learner guide effectively"], ["vec", "best way to visual learner guide"], ["hyde", "Learning visual learner guide requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "education_learning", "is_short": false, "query": "visual learner guide"}
{"output": [["lex", "video overview calls setup advice"], ["lex", "video overview calls setup guide"], ["lex", "video overview calls setup tips"], ["vec", "complete guide to video calls setup"], ["vec", "how to video calls setup effectively"], ["hyde", "Whether you're a beginner or looking to improve, this guide to video calls setup offers practical advice, common pitfalls to avoid, and proven strategies for success."]], "category": "technology", "is_short": false, "query": "video calls setup"}
{"output": [["lex", "best overview sibling relationships advice"], ["lex", "best overview sibling relationships how to"], ["lex", "best overview sibling relationships tutorial"], ["vec", "learn best sibling relationships step by step"], ["vec", "complete guide to best sibling relationships"], ["hyde", "This comprehensive guide to best sibling relationships covers all the essential information you need to get started. Follow the steps carefully for best results."]], "category": "lifestyle_relationships", "is_short": false, "query": "best sibling relationships"}
{"output": [["lex", "how overview to boost immunity tutorial"], ["lex", "how overview to boost immunity guide"], ["lex", "how overview to boost immunity how to"], ["vec", "complete guide to how to boost immunity"], ["vec", "how to how to boost immunity effectively"], ["hyde", "Learning how to boost immunity requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "health_wellness", "is_short": false, "query": "how to boost immunity"}
{"output": [["lex", "best overview budget travel how to"], ["lex", "best overview budget travel tutorial"], ["lex", "best overview budget travel tips"], ["vec", "learn best budget travel step by step"], ["vec", "best way to best budget travel"], ["hyde", "Learning best budget travel requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "travel_geography", "is_short": false, "query": "best budget travel"}
{"output": [["lex", "flower overview arranging tutorial tutorial"], ["lex", "flower overview arranging tutorial advice"], ["lex", "flower overview arranging tutorial tips"], ["vec", "learn flower arranging tutorial step by step"], ["vec", "tips for flower arranging tutorial success"], ["hyde", "Learning flower arranging tutorial requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "home_garden", "is_short": false, "query": "flower arranging tutorial"}
{"output": [["lex", "organic overview gardening tutorial advice"], ["lex", "organic overview gardening tutorial how to"], ["lex", "organic overview gardening tutorial tutorial"], ["vec", "best way to organic gardening tutorial"], ["vec", "complete guide to organic gardening tutorial"], ["hyde", "Whether you're a beginner or looking to improve, this guide to organic gardening tutorial offers practical advice, common pitfalls to avoid, and proven strategies for success."]], "category": "home_garden", "is_short": false, "query": "organic gardening tutorial"}
{"output": [["lex", "best overview trail finding tutorial"], ["lex", "best overview trail finding how to"], ["lex", "best overview trail finding tips"], ["vec", "how to best trail finding effectively"], ["vec", "best way to best trail finding"], ["hyde", "Whether you're a beginner or looking to improve, this guide to best trail finding offers practical advice, common pitfalls to avoid, and proven strategies for success."]], "category": "hobbies_crafts", "is_short": false, "query": "best trail finding"}
{"output": [["lex", "meal how to overview"], ["lex", "meal advice overview"], ["lex", "meal tutorial overview"], ["vec", "tips for meal success"], ["vec", "how to meal effectively"], ["hyde", "Whether you're a beginner or looking to improve, this guide to meal offers practical advice, common pitfalls to avoid, and proven strategies for success."]], "category": "education_learning", "is_short": true, "query": "meal"}
{"output": [["lex", "best overview tent setup how to"], ["lex", "best overview tent setup tips"], ["lex", "best overview tent setup tutorial"], ["vec", "best way to best tent setup"], ["vec", "how to best tent setup effectively"], ["hyde", "Whether you're a beginner or looking to improve, this guide to best tent setup offers practical advice, common pitfalls to avoid, and proven strategies for success."]], "category": "hobbies_crafts", "is_short": false, "query": "best tent setup"}
{"output": [["lex", "unclog overview drain tutorial guide"], ["lex", "unclog overview drain tutorial tutorial"], ["lex", "unclog overview drain tutorial how to"], ["vec", "best way to unclog drain tutorial"], ["vec", "tips for unclog drain tutorial success"], ["hyde", "Learning unclog drain tutorial requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "home_garden", "is_short": false, "query": "unclog drain tutorial"}
{"output": [["lex", "networking overview tips how to"], ["lex", "networking overview tips advice"], ["lex", "networking overview tips tutorial"], ["vec", "how to networking tips effectively"], ["vec", "tips for networking tips success"], ["hyde", "Whether you're a beginner or looking to improve, this guide to networking tips offers practical advice, common pitfalls to avoid, and proven strategies for success."]], "category": "lifestyle_relationships", "is_short": false, "query": "networking tips"}
{"output": [["lex", "learn overview furniture making tips"], ["lex", "learn overview furniture making advice"], ["lex", "learn overview furniture making how to"], ["vec", "tips for learn furniture making success"], ["vec", "learn learn furniture making step by step"], ["hyde", "Whether you're a beginner or looking to improve, this guide to learn furniture making offers practical advice, common pitfalls to avoid, and proven strategies for success."]], "category": "hobbies_crafts", "is_short": false, "query": "learn furniture making"}
{"output": [["lex", "passive overview income tips guide"], ["lex", "passive overview income tips tips"], ["lex", "passive overview income tips how to"], ["vec", "learn passive income tips step by step"], ["vec", "complete guide to passive income tips"], ["hyde", "Whether you're a beginner or looking to improve, this guide to passive income tips offers practical advice, common pitfalls to avoid, and proven strategies for success."]], "category": "finance_business", "is_short": false, "query": "passive income tips"}
{"output": [["lex", "travel guide overview"], ["lex", "travel advice overview"], ["lex", "travel how to overview"], ["vec", "learn travel step by step"], ["vec", "best way to travel"], ["hyde", "Whether you're a beginner or looking to improve, this guide to travel offers practical advice, common pitfalls to avoid, and proven strategies for success."]], "category": "education_learning", "is_short": true, "query": "travel"}
{"output": [["lex", "small overview space tutorial advice"], ["lex", "small overview space tutorial guide"], ["lex", "small overview space tutorial tips"], ["vec", "best way to small space tutorial"], ["vec", "how to small space tutorial effectively"], ["hyde", "Whether you're a beginner or looking to improve, this guide to small space tutorial offers practical advice, common pitfalls to avoid, and proven strategies for success."]], "category": "home_garden", "is_short": false, "query": "small space tutorial"}
{"output": [["lex", "learn overview vegan cooking advice"], ["lex", "learn overview vegan cooking how to"], ["lex", "learn overview vegan cooking tips"], ["vec", "best way to learn vegan cooking"], ["vec", "how to learn vegan cooking effectively"], ["hyde", "This comprehensive guide to learn vegan cooking covers all the essential information you need to get started. Follow the steps carefully for best results."]], "category": "food_cooking", "is_short": false, "query": "learn vegan cooking"}
{"output": [["lex", "earth overview science guide tips"], ["lex", "earth overview science guide advice"], ["lex", "earth overview science guide tutorial"], ["vec", "learn earth science guide step by step"], ["vec", "tips for earth science guide success"], ["hyde", "Learning earth science guide requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "education_learning", "is_short": false, "query": "earth science guide"}
{"output": [["lex", "learn overview classical music tutorial"], ["lex", "learn overview classical music guide"], ["lex", "learn overview classical music advice"], ["vec", "best way to learn classical music"], ["vec", "tips for learn classical music success"], ["hyde", "Whether you're a beginner or looking to improve, this guide to learn classical music offers practical advice, common pitfalls to avoid, and proven strategies for success."]], "category": "arts_culture", "is_short": false, "query": "learn classical music"}
{"output": [["lex", "best overview upcycling how to"], ["lex", "best overview upcycling guide"], ["lex", "best overview upcycling tutorial"], ["vec", "complete guide to best upcycling"], ["vec", "learn best upcycling step by step"], ["hyde", "Learning best upcycling requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "hobbies_crafts", "is_short": false, "query": "best upcycling"}
{"output": [["lex", "improve overview sibling relationships guide"], ["lex", "improve overview sibling relationships how to"], ["lex", "improve overview sibling relationships tips"], ["vec", "learn improve sibling relationships step by step"], ["vec", "best way to improve sibling relationships"], ["hyde", "Learning improve sibling relationships requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "lifestyle_relationships", "is_short": false, "query": "improve sibling relationships"}
{"output": [["lex", "DIY overview fix leaky faucet how to"], ["lex", "DIY overview fix leaky faucet tips"], ["lex", "DIY overview fix leaky faucet guide"], ["vec", "how to DIY fix leaky faucet effectively"], ["vec", "tips for DIY fix leaky faucet success"], ["hyde", "Learning DIY fix leaky faucet requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "home_garden", "is_short": false, "query": "DIY fix leaky faucet"}
{"output": [["lex", "improve overview biology tips"], ["lex", "improve overview biology how to"], ["lex", "improve overview biology advice"], ["vec", "complete guide to improve biology"], ["vec", "learn improve biology step by step"], ["hyde", "Learning improve biology requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "education_learning", "is_short": false, "query": "improve biology"}
{"output": [["lex", "paint guide overview"], ["lex", "paint advice overview"], ["lex", "paint tips overview"], ["vec", "complete guide to paint"], ["vec", "best way to paint"], ["hyde", "This comprehensive guide to paint covers all the essential information you need to get started. Follow the steps carefully for best results."]], "category": "finance_business", "is_short": true, "query": "paint"}
{"output": [["lex", "how overview to adjust timezone guide"], ["lex", "how overview to adjust timezone tips"], ["lex", "how overview to adjust timezone tutorial"], ["vec", "complete guide to how to adjust timezone"], ["vec", "learn how to adjust timezone step by step"], ["hyde", "Whether you're a beginner or looking to improve, this guide to how to adjust timezone offers practical advice, common pitfalls to avoid, and proven strategies for success."]], "category": "travel_geography", "is_short": false, "query": "how to adjust timezone"}
{"output": [["lex", "best overview backpacking how to"], ["lex", "best overview backpacking tutorial"], ["lex", "best overview backpacking guide"], ["vec", "learn best backpacking step by step"], ["vec", "best way to best backpacking"], ["hyde", "Whether you're a beginner or looking to improve, this guide to best backpacking offers practical advice, common pitfalls to avoid, and proven strategies for success."]], "category": "travel_geography", "is_short": false, "query": "best backpacking"}
{"output": [["lex", "contemporary overview dance history tutorial"], ["lex", "contemporary overview dance history how to"], ["lex", "contemporary overview dance history tips"], ["vec", "complete guide to contemporary dance history"], ["vec", "tips for contemporary dance history success"], ["hyde", "Learning contemporary dance history requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "arts_culture", "is_short": false, "query": "contemporary dance history"}
{"output": [["lex", "how overview to fix leaky faucet tutorial"], ["lex", "how overview to fix leaky faucet tips"], ["lex", "how overview to fix leaky faucet how to"], ["vec", "how to how to fix leaky faucet effectively"], ["vec", "best way to how to fix leaky faucet"], ["hyde", "This comprehensive guide to how to fix leaky faucet covers all the essential information you need to get started. Follow the steps carefully for best results."]], "category": "home_garden", "is_short": false, "query": "how to fix leaky faucet"}
{"output": [["lex", "speed overview reading guide how to"], ["lex", "speed overview reading guide guide"], ["lex", "speed overview reading guide tips"], ["vec", "learn speed reading guide step by step"], ["vec", "complete guide to speed reading guide"], ["hyde", "Learning speed reading guide requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "education_learning", "is_short": false, "query": "speed reading guide"}
{"output": [["lex", "best overview organic gardening how to"], ["lex", "best overview organic gardening guide"], ["lex", "best overview organic gardening tutorial"], ["vec", "complete guide to best organic gardening"], ["vec", "best way to best organic gardening"], ["hyde", "Learning best organic gardening requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "home_garden", "is_short": false, "query": "best organic gardening"}
{"output": [["lex", "best overview hydration tips how to"], ["lex", "best overview hydration tips tips"], ["lex", "best overview hydration tips advice"], ["vec", "how to best hydration tips effectively"], ["vec", "tips for best hydration tips success"], ["hyde", "Learning best hydration tips requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "health_wellness", "is_short": false, "query": "best hydration tips"}
{"output": [["lex", "best overview social skills advice"], ["lex", "best overview social skills tips"], ["lex", "best overview social skills how to"], ["vec", "tips for best social skills success"], ["vec", "learn best social skills step by step"], ["hyde", "Whether you're a beginner or looking to improve, this guide to best social skills offers practical advice, common pitfalls to avoid, and proven strategies for success."]], "category": "lifestyle_relationships", "is_short": false, "query": "best social skills"}
{"output": [["lex", "improve overview reading list advice"], ["lex", "improve overview reading list tips"], ["lex", "improve overview reading list guide"], ["vec", "best way to improve reading list"], ["vec", "how to improve reading list effectively"], ["hyde", "Whether you're a beginner or looking to improve, this guide to improve reading list offers practical advice, common pitfalls to avoid, and proven strategies for success."]], "category": "education_learning", "is_short": false, "query": "improve reading list"}
{"output": [["lex", "digital overview detox tips tutorial"], ["lex", "digital overview detox tips guide"], ["lex", "digital overview detox tips how to"], ["vec", "tips for digital detox tips success"], ["vec", "how to digital detox tips effectively"], ["hyde", "Learning digital detox tips requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "lifestyle_relationships", "is_short": false, "query": "digital detox tips"}
{"output": [["lex", "MOOC overview platforms guide tips"], ["lex", "MOOC overview platforms guide guide"], ["lex", "MOOC overview platforms guide tutorial"], ["vec", "tips for MOOC platforms guide success"], ["vec", "best way to MOOC platforms guide"], ["hyde", "Learning MOOC platforms guide requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "education_learning", "is_short": false, "query": "MOOC platforms guide"}
{"output": [["lex", "dividend overview stocks basics guide"], ["lex", "dividend overview stocks basics tips"], ["lex", "dividend overview stocks basics how to"], ["vec", "learn dividend stocks basics step by step"], ["vec", "best way to dividend stocks basics"], ["hyde", "This comprehensive guide to dividend stocks basics covers all the essential information you need to get started. Follow the steps carefully for best results."]], "category": "finance_business", "is_short": false, "query": "dividend stocks basics"}
{"output": [["lex", "fix overview leaky faucet ideas how to"], ["lex", "fix overview leaky faucet ideas advice"], ["lex", "fix overview leaky faucet ideas guide"], ["vec", "how to fix leaky faucet ideas effectively"], ["vec", "best way to fix leaky faucet ideas"], ["hyde", "Learning fix leaky faucet ideas requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "home_garden", "is_short": false, "query": "fix leaky faucet ideas"}
{"output": [["lex", "learn overview dietary restrictions tutorial"], ["lex", "learn overview dietary restrictions how to"], ["lex", "learn overview dietary restrictions advice"], ["vec", "learn learn dietary restrictions step by step"], ["vec", "how to learn dietary restrictions effectively"], ["hyde", "Learning learn dietary restrictions requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "food_cooking", "is_short": false, "query": "learn dietary restrictions"}
{"output": [["lex", "travel overview gear guide tips"], ["lex", "travel overview gear guide guide"], ["lex", "travel overview gear guide advice"], ["vec", "learn travel gear guide step by step"], ["vec", "best way to travel gear guide"], ["hyde", "This comprehensive guide to travel gear guide covers all the essential information you need to get started. Follow the steps carefully for best results."]], "category": "travel_geography", "is_short": false, "query": "travel gear guide"}
{"output": [["lex", "video overview calls tips tutorial"], ["lex", "video overview calls tips how to"], ["lex", "video overview calls tips tips"], ["vec", "complete guide to video calls tips"], ["vec", "learn video calls tips step by step"], ["hyde", "This comprehensive guide to video calls tips covers all the essential information you need to get started. Follow the steps carefully for best results."]], "category": "technology", "is_short": false, "query": "video calls tips"}
{"output": [["lex", "how overview to landscaping advice"], ["lex", "how overview to landscaping how to"], ["lex", "how overview to landscaping tutorial"], ["vec", "complete guide to how to landscaping"], ["vec", "tips for how to landscaping success"], ["hyde", "Whether you're a beginner or looking to improve, this guide to how to landscaping offers practical advice, common pitfalls to avoid, and proven strategies for success."]], "category": "home_garden", "is_short": false, "query": "how to landscaping"}
{"output": [["lex", "world overview wars guide guide"], ["lex", "world overview wars guide tutorial"], ["lex", "world overview wars guide how to"], ["vec", "how to world wars guide effectively"], ["vec", "complete guide to world wars guide"], ["hyde", "This comprehensive guide to world wars guide covers all the essential information you need to get started. Follow the steps carefully for best results."]], "category": "education_learning", "is_short": false, "query": "world wars guide"}
{"output": [["lex", "how overview to laundry tips tutorial"], ["lex", "how overview to laundry tips advice"], ["lex", "how overview to laundry tips guide"], ["vec", "tips for how to laundry tips success"], ["vec", "complete guide to how to laundry tips"], ["hyde", "Whether you're a beginner or looking to improve, this guide to how to laundry tips offers practical advice, common pitfalls to avoid, and proven strategies for success."]], "category": "home_garden", "is_short": false, "query": "how to laundry tips"}
{"output": [["lex", "best overview teenager advice guide"], ["lex", "best overview teenager advice tips"], ["lex", "best overview teenager advice advice"], ["vec", "learn best teenager advice step by step"], ["vec", "tips for best teenager advice success"], ["hyde", "Whether you're a beginner or looking to improve, this guide to best teenager advice offers practical advice, common pitfalls to avoid, and proven strategies for success."]], "category": "lifestyle_relationships", "is_short": false, "query": "best teenager advice"}
{"output": [["lex", "understand overview holiday traditions tips"], ["lex", "understand overview holiday traditions how to"], ["lex", "understand overview holiday traditions advice"], ["vec", "best way to understand holiday traditions"], ["vec", "learn understand holiday traditions step by step"], ["hyde", "This comprehensive guide to understand holiday traditions covers all the essential information you need to get started. Follow the steps carefully for best results."]], "category": "arts_culture", "is_short": false, "query": "understand holiday traditions"}
{"output": [["lex", "bake how to overview"], ["lex", "bake advice overview"], ["lex", "bake tips overview"], ["vec", "best way to bake"], ["vec", "complete guide to bake"], ["hyde", "Learning bake requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "education_learning", "is_short": true, "query": "bake"}
{"output": [["lex", "kinesthetic overview guide tips"], ["lex", "kinesthetic overview guide advice"], ["lex", "kinesthetic overview guide how to"], ["vec", "learn kinesthetic guide step by step"], ["vec", "tips for kinesthetic guide success"], ["hyde", "This comprehensive guide to kinesthetic guide covers all the essential information you need to get started. Follow the steps carefully for best results."]], "category": "education_learning", "is_short": false, "query": "kinesthetic guide"}
{"output": [["lex", "batch overview cooking techniques guide"], ["lex", "batch overview cooking techniques tips"], ["lex", "batch overview cooking techniques advice"], ["vec", "best way to batch cooking techniques"], ["vec", "tips for batch cooking techniques success"], ["hyde", "Learning batch cooking techniques requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "food_cooking", "is_short": false, "query": "batch cooking techniques"}
{"output": [["lex", "nutrition how to overview"], ["lex", "nutrition guide overview"], ["lex", "nutrition tips overview"], ["vec", "complete guide to nutrition"], ["vec", "best way to nutrition"], ["hyde", "Whether you're a beginner or looking to improve, this guide to nutrition offers practical advice, common pitfalls to avoid, and proven strategies for success."]], "category": "technology", "is_short": true, "query": "nutrition"}
{"output": [["lex", "habit overview formation tips how to"], ["lex", "habit overview formation tips advice"], ["lex", "habit overview formation tips guide"], ["vec", "complete guide to habit formation tips"], ["vec", "best way to habit formation tips"], ["hyde", "This comprehensive guide to habit formation tips covers all the essential information you need to get started. Follow the steps carefully for best results."]], "category": "lifestyle_relationships", "is_short": false, "query": "habit formation tips"}
{"output": [["lex", "plan overview local experiences tutorial"], ["lex", "plan overview local experiences tips"], ["lex", "plan overview local experiences advice"], ["vec", "learn plan local experiences step by step"], ["vec", "complete guide to plan local experiences"], ["hyde", "Learning plan local experiences requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "travel_geography", "is_short": false, "query": "plan local experiences"}
{"output": [["lex", "diet guide overview"], ["lex", "diet tutorial overview"], ["lex", "diet how to overview"], ["vec", "how to diet effectively"], ["vec", "tips for diet success"], ["hyde", "This comprehensive guide to diet covers all the essential information you need to get started. Follow the steps carefully for best results."]], "category": "arts_culture", "is_short": true, "query": "diet"}
{"output": [["lex", "best overview unclog drain advice"], ["lex", "best overview unclog drain how to"], ["lex", "best overview unclog drain guide"], ["vec", "learn best unclog drain step by step"], ["vec", "tips for best unclog drain success"], ["hyde", "Whether you're a beginner or looking to improve, this guide to best unclog drain offers practical advice, common pitfalls to avoid, and proven strategies for success."]], "category": "home_garden", "is_short": false, "query": "best unclog drain"}
{"output": [["lex", "study tutorial overview"], ["lex", "study tips overview"], ["lex", "study how to overview"], ["vec", "complete guide to study"], ["vec", "how to study effectively"], ["hyde", "This comprehensive guide to study covers all the essential information you need to get started. Follow the steps carefully for best results."]], "category": "education_learning", "is_short": true, "query": "study"}
{"output": [["lex", "how overview to coffee brewing guide"], ["lex", "how overview to coffee brewing advice"], ["lex", "how overview to coffee brewing how to"], ["vec", "how to how to coffee brewing effectively"], ["vec", "learn how to coffee brewing step by step"], ["hyde", "Learning how to coffee brewing requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "food_cooking", "is_short": false, "query": "how to coffee brewing"}
{"output": [["lex", "understand overview film genres guide"], ["lex", "understand overview film genres advice"], ["lex", "understand overview film genres how to"], ["vec", "complete guide to understand film genres"], ["vec", "learn understand film genres step by step"], ["hyde", "Whether you're a beginner or looking to improve, this guide to understand film genres offers practical advice, common pitfalls to avoid, and proven strategies for success."]], "category": "arts_culture", "is_short": false, "query": "understand film genres"}
{"output": [["lex", "fix overview email etiquette tips"], ["lex", "fix overview email etiquette advice"], ["lex", "fix overview email etiquette guide"], ["vec", "best way to fix email etiquette"], ["vec", "complete guide to fix email etiquette"], ["hyde", "This comprehensive guide to fix email etiquette covers all the essential information you need to get started. Follow the steps carefully for best results."]], "category": "technology", "is_short": false, "query": "fix email etiquette"}
{"output": [["lex", "best overview travel gear tips"], ["lex", "best overview travel gear advice"], ["lex", "best overview travel gear how to"], ["vec", "complete guide to best travel gear"], ["vec", "best way to best travel gear"], ["hyde", "This comprehensive guide to best travel gear covers all the essential information you need to get started. Follow the steps carefully for best results."]], "category": "travel_geography", "is_short": false, "query": "best travel gear"}
{"output": [["lex", "Baroque overview guide advice"], ["lex", "Baroque overview guide how to"], ["lex", "Baroque overview guide guide"], ["vec", "best way to Baroque guide"], ["vec", "tips for Baroque guide success"], ["hyde", "Learning Baroque guide requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "arts_culture", "is_short": false, "query": "Baroque guide"}
{"output": [["lex", "appreciate overview dance styles tutorial"], ["lex", "appreciate overview dance styles how to"], ["lex", "appreciate overview dance styles guide"], ["vec", "how to appreciate dance styles effectively"], ["vec", "tips for appreciate dance styles success"], ["hyde", "Learning appreciate dance styles requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "arts_culture", "is_short": false, "query": "appreciate dance styles"}
{"output": [["lex", "travel overview safety guide how to"], ["lex", "travel overview safety guide tips"], ["lex", "travel overview safety guide guide"], ["vec", "how to travel safety guide effectively"], ["vec", "complete guide to travel safety guide"], ["hyde", "Whether you're a beginner or looking to improve, this guide to travel safety guide offers practical advice, common pitfalls to avoid, and proven strategies for success."]], "category": "travel_geography", "is_short": false, "query": "travel safety guide"}
{"output": [["lex", "BBQ overview techniques recipe tutorial"], ["lex", "BBQ overview techniques recipe how to"], ["lex", "BBQ overview techniques recipe advice"], ["vec", "complete guide to BBQ techniques recipe"], ["vec", "tips for BBQ techniques recipe success"], ["hyde", "Learning BBQ techniques recipe requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "food_cooking", "is_short": false, "query": "BBQ techniques recipe"}
{"output": [["lex", "how overview to indoor plants tips"], ["lex", "how overview to indoor plants how to"], ["lex", "how overview to indoor plants advice"], ["vec", "learn how to indoor plants step by step"], ["vec", "best way to how to indoor plants"], ["hyde", "Learning how to indoor plants requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "home_garden", "is_short": false, "query": "how to indoor plants"}
{"output": [["lex", "best overview common cold remedies tutorial"], ["lex", "best overview common cold remedies how to"], ["lex", "best overview common cold remedies advice"], ["vec", "best way to best common cold remedies"], ["vec", "how to best common cold remedies effectively"], ["hyde", "Whether you're a beginner or looking to improve, this guide to best common cold remedies offers practical advice, common pitfalls to avoid, and proven strategies for success."]], "category": "health_wellness", "is_short": false, "query": "best common cold remedies"}
{"output": [["lex", "retirement overview planning tips advice"], ["lex", "retirement overview planning tips tips"], ["lex", "retirement overview planning tips tutorial"], ["vec", "best way to retirement planning tips"], ["vec", "how to retirement planning tips effectively"], ["hyde", "Whether you're a beginner or looking to improve, this guide to retirement planning tips offers practical advice, common pitfalls to avoid, and proven strategies for success."]], "category": "finance_business", "is_short": false, "query": "retirement planning tips"}
{"output": [["lex", "problem overview solving techniques tutorial"], ["lex", "problem overview solving techniques how to"], ["lex", "problem overview solving techniques advice"], ["vec", "learn problem solving techniques step by step"], ["vec", "how to problem solving techniques effectively"], ["hyde", "Whether you're a beginner or looking to improve, this guide to problem solving techniques offers practical advice, common pitfalls to avoid, and proven strategies for success."]], "category": "education_learning", "is_short": false, "query": "problem solving techniques"}
{"output": [["lex", "garden guide overview"], ["lex", "garden how to overview"], ["lex", "garden tutorial overview"], ["vec", "tips for garden success"], ["vec", "best way to garden"], ["hyde", "This comprehensive guide to garden covers all the essential information you need to get started. Follow the steps carefully for best results."]], "category": "finance_business", "is_short": true, "query": "garden"}
{"output": [["lex", "plan overview carry on essentials advice"], ["lex", "plan overview carry on essentials guide"], ["lex", "plan overview carry on essentials tutorial"], ["vec", "learn plan carry on essentials step by step"], ["vec", "tips for plan carry on essentials success"], ["hyde", "Learning plan carry on essentials requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "travel_geography", "is_short": false, "query": "plan carry on essentials"}
{"output": [["lex", "best overview lunch prep advice"], ["lex", "best overview lunch prep guide"], ["lex", "best overview lunch prep how to"], ["vec", "how to best lunch prep effectively"], ["vec", "complete guide to best lunch prep"], ["hyde", "Learning best lunch prep requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "food_cooking", "is_short": false, "query": "best lunch prep"}
{"output": [["lex", "book overview recommendations techniques tutorial"], ["lex", "book overview recommendations techniques advice"], ["lex", "book overview recommendations techniques tips"], ["vec", "best way to book recommendations techniques"], ["vec", "complete guide to book recommendations techniques"], ["hyde", "Whether you're a beginner or looking to improve, this guide to book recommendations techniques offers practical advice, common pitfalls to avoid, and proven strategies for success."]], "category": "education_learning", "is_short": false, "query": "book recommendations techniques"}
{"output": [["lex", "lawn overview care tutorial tips"], ["lex", "lawn overview care tutorial tutorial"], ["lex", "lawn overview care tutorial how to"], ["vec", "complete guide to lawn care tutorial"], ["vec", "best way to lawn care tutorial"], ["hyde", "Learning lawn care tutorial requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "home_garden", "is_short": false, "query": "lawn care tutorial"}
{"output": [["lex", "plan overview road trip planning guide"], ["lex", "plan overview road trip planning how to"], ["lex", "plan overview road trip planning tutorial"], ["vec", "tips for plan road trip planning success"], ["vec", "learn plan road trip planning step by step"], ["hyde", "Whether you're a beginner or looking to improve, this guide to plan road trip planning offers practical advice, common pitfalls to avoid, and proven strategies for success."]], "category": "travel_geography", "is_short": false, "query": "plan road trip planning"}
{"output": [["lex", "degree advice overview"], ["lex", "degree tips overview"], ["lex", "degree tutorial overview"], ["vec", "best way to degree"], ["vec", "how to degree effectively"], ["hyde", "Learning degree requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "home_garden", "is_short": true, "query": "degree"}
{"output": [["lex", "how overview to basic troubleshooting tutorial"], ["lex", "how overview to basic troubleshooting how to"], ["lex", "how overview to basic troubleshooting advice"], ["vec", "tips for how to basic troubleshooting success"], ["vec", "learn how to basic troubleshooting step by step"], ["hyde", "This comprehensive guide to how to basic troubleshooting covers all the essential information you need to get started. Follow the steps carefully for best results."]], "category": "technology", "is_short": false, "query": "how to basic troubleshooting"}
{"output": [["lex", "study overview ancient civilizations tips"], ["lex", "study overview ancient civilizations advice"], ["lex", "study overview ancient civilizations tutorial"], ["vec", "best way to study ancient civilizations"], ["vec", "complete guide to study ancient civilizations"], ["hyde", "Learning study ancient civilizations requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "education_learning", "is_short": false, "query": "study ancient civilizations"}
{"output": [["lex", "in-laws overview advice advice"], ["lex", "in-laws overview advice tutorial"], ["lex", "in-laws overview advice tips"], ["vec", "tips for in-laws advice success"], ["vec", "best way to in-laws advice"], ["hyde", "This comprehensive guide to in-laws advice covers all the essential information you need to get started. Follow the steps carefully for best results."]], "category": "lifestyle_relationships", "is_short": false, "query": "in-laws advice"}
{"output": [["lex", "how overview to app recommendations guide"], ["lex", "how overview to app recommendations advice"], ["lex", "how overview to app recommendations tips"], ["vec", "complete guide to how to app recommendations"], ["vec", "best way to how to app recommendations"], ["hyde", "Learning how to app recommendations requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "technology", "is_short": false, "query": "how to app recommendations"}
{"output": [["lex", "best overview carry on essentials how to"], ["lex", "best overview carry on essentials tips"], ["lex", "best overview carry on essentials advice"], ["vec", "tips for best carry on essentials success"], ["vec", "best way to best carry on essentials"], ["hyde", "Learning best carry on essentials requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "travel_geography", "is_short": false, "query": "best carry on essentials"}
{"output": [["lex", "best overview playing guitar guide"], ["lex", "best overview playing guitar how to"], ["lex", "best overview playing guitar tutorial"], ["vec", "learn best playing guitar step by step"], ["vec", "how to best playing guitar effectively"], ["hyde", "Whether you're a beginner or looking to improve, this guide to best playing guitar offers practical advice, common pitfalls to avoid, and proven strategies for success."]], "category": "hobbies_crafts", "is_short": false, "query": "best playing guitar"}
{"output": [["lex", "best overview camping tips"], ["lex", "best overview camping guide"], ["lex", "best overview camping how to"], ["vec", "tips for best camping success"], ["vec", "best way to best camping"], ["hyde", "Learning best camping requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "hobbies_crafts", "is_short": false, "query": "best camping"}
{"output": [["lex", "improve overview breathing exercises how to"], ["lex", "improve overview breathing exercises tips"], ["lex", "improve overview breathing exercises guide"], ["vec", "how to improve breathing exercises effectively"], ["vec", "complete guide to improve breathing exercises"], ["hyde", "Learning improve breathing exercises requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "health_wellness", "is_short": false, "query": "improve breathing exercises"}
{"output": [["lex", "learn overview meal ideas guide"], ["lex", "learn overview meal ideas advice"], ["lex", "learn overview meal ideas tutorial"], ["vec", "best way to learn meal ideas"], ["vec", "complete guide to learn meal ideas"], ["hyde", "Whether you're a beginner or looking to improve, this guide to learn meal ideas offers practical advice, common pitfalls to avoid, and proven strategies for success."]], "category": "food_cooking", "is_short": false, "query": "learn meal ideas"}
{"output": [["lex", "how overview to slow living tutorial"], ["lex", "how overview to slow living advice"], ["lex", "how overview to slow living how to"], ["vec", "tips for how to slow living success"], ["vec", "how to how to slow living effectively"], ["hyde", "Whether you're a beginner or looking to improve, this guide to how to slow living offers practical advice, common pitfalls to avoid, and proven strategies for success."]], "category": "lifestyle_relationships", "is_short": false, "query": "how to slow living"}
{"output": [["lex", "compost how to overview"], ["lex", "compost tips overview"], ["lex", "compost advice overview"], ["vec", "tips for compost success"], ["vec", "how to compost effectively"], ["hyde", "Whether you're a beginner or looking to improve, this guide to compost offers practical advice, common pitfalls to avoid, and proven strategies for success."]], "category": "technology", "is_short": true, "query": "compost"}
{"output": [["lex", "protein advice overview"], ["lex", "protein tips overview"], ["lex", "protein guide overview"], ["vec", "best way to protein"], ["vec", "how to protein effectively"], ["hyde", "This comprehensive guide to protein covers all the essential information you need to get started. Follow the steps carefully for best results."]], "category": "hobbies_crafts", "is_short": true, "query": "protein"}
{"output": [["lex", "learn overview reading list how to"], ["lex", "learn overview reading list guide"], ["lex", "learn overview reading list tutorial"], ["vec", "learn learn reading list step by step"], ["vec", "how to learn reading list effectively"], ["hyde", "Whether you're a beginner or looking to improve, this guide to learn reading list offers practical advice, common pitfalls to avoid, and proven strategies for success."]], "category": "education_learning", "is_short": false, "query": "learn reading list"}
{"output": [["lex", "how overview to bucket list destinations tips"], ["lex", "how overview to bucket list destinations guide"], ["lex", "how overview to bucket list destinations tutorial"], ["vec", "learn how to bucket list destinations step by step"], ["vec", "tips for how to bucket list destinations success"], ["hyde", "This comprehensive guide to how to bucket list destinations covers all the essential information you need to get started. Follow the steps carefully for best results."]], "category": "travel_geography", "is_short": false, "query": "how to bucket list destinations"}
{"output": [["lex", "best overview knife skills tutorial"], ["lex", "best overview knife skills advice"], ["lex", "best overview knife skills tips"], ["vec", "best way to best knife skills"], ["vec", "how to best knife skills effectively"], ["hyde", "Learning best knife skills requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "food_cooking", "is_short": false, "query": "best knife skills"}
{"output": [["lex", "hydration overview tips guide tips"], ["lex", "hydration overview tips guide advice"], ["lex", "hydration overview tips guide how to"], ["vec", "tips for hydration tips guide success"], ["vec", "complete guide to hydration tips guide"], ["hyde", "Learning hydration tips guide requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "health_wellness", "is_short": false, "query": "hydration tips guide"}
{"output": [["lex", "learn overview problem solving advice"], ["lex", "learn overview problem solving guide"], ["lex", "learn overview problem solving how to"], ["vec", "best way to learn problem solving"], ["vec", "complete guide to learn problem solving"], ["hyde", "This comprehensive guide to learn problem solving covers all the essential information you need to get started. Follow the steps carefully for best results."]], "category": "education_learning", "is_short": false, "query": "learn problem solving"}
{"output": [["lex", "social overview media privacy setup advice"], ["lex", "social overview media privacy setup tips"], ["lex", "social overview media privacy setup how to"], ["vec", "best way to social media privacy setup"], ["vec", "tips for social media privacy setup success"], ["hyde", "Learning social media privacy setup requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "technology", "is_short": false, "query": "social media privacy setup"}
{"output": [["lex", "passport tips overview"], ["lex", "passport advice overview"], ["lex", "passport guide overview"], ["vec", "tips for passport success"], ["vec", "best way to passport"], ["hyde", "Learning passport requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "home_garden", "is_short": true, "query": "passport"}
{"output": [["lex", "breakfast overview ideas recipe guide"], ["lex", "breakfast overview ideas recipe how to"], ["lex", "breakfast overview ideas recipe tips"], ["vec", "how to breakfast ideas recipe effectively"], ["vec", "tips for breakfast ideas recipe success"], ["hyde", "This comprehensive guide to breakfast ideas recipe covers all the essential information you need to get started. Follow the steps carefully for best results."]], "category": "food_cooking", "is_short": false, "query": "breakfast ideas recipe"}
{"output": [["lex", "start overview LLC setup advice"], ["lex", "start overview LLC setup tips"], ["lex", "start overview LLC setup tutorial"], ["vec", "how to start LLC setup effectively"], ["vec", "learn start LLC setup step by step"], ["hyde", "This comprehensive guide to start LLC setup covers all the essential information you need to get started. Follow the steps carefully for best results."]], "category": "finance_business", "is_short": false, "query": "start LLC setup"}
{"output": [["lex", "improve overview physical therapy tips"], ["lex", "improve overview physical therapy tutorial"], ["lex", "improve overview physical therapy guide"], ["vec", "complete guide to improve physical therapy"], ["vec", "best way to improve physical therapy"], ["hyde", "Learning improve physical therapy requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "health_wellness", "is_short": false, "query": "improve physical therapy"}
{"output": [["lex", "eye overview health guide advice"], ["lex", "eye overview health guide how to"], ["lex", "eye overview health guide tutorial"], ["vec", "complete guide to eye health guide"], ["vec", "best way to eye health guide"], ["hyde", "This comprehensive guide to eye health guide covers all the essential information you need to get started. Follow the steps carefully for best results."]], "category": "health_wellness", "is_short": false, "query": "eye health guide"}
{"output": [["lex", "side overview hustle tips tips"], ["lex", "side overview hustle tips tutorial"], ["lex", "side overview hustle tips how to"], ["vec", "how to side hustle tips effectively"], ["vec", "complete guide to side hustle tips"], ["hyde", "Whether you're a beginner or looking to improve, this guide to side hustle tips offers practical advice, common pitfalls to avoid, and proven strategies for success."]], "category": "finance_business", "is_short": false, "query": "side hustle tips"}
{"output": [["lex", "trail overview finding for beginners tutorial"], ["lex", "trail overview finding for beginners tips"], ["lex", "trail overview finding for beginners guide"], ["vec", "complete guide to trail finding for beginners"], ["vec", "learn trail finding for beginners step by step"], ["hyde", "This comprehensive guide to trail finding for beginners covers all the essential information you need to get started. Follow the steps carefully for best results."]], "category": "hobbies_crafts", "is_short": false, "query": "trail finding for beginners"}
{"output": [["lex", "fix overview basic troubleshooting tips"], ["lex", "fix overview basic troubleshooting guide"], ["lex", "fix overview basic troubleshooting how to"], ["vec", "learn fix basic troubleshooting step by step"], ["vec", "how to fix basic troubleshooting effectively"], ["hyde", "Learning fix basic troubleshooting requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "technology", "is_short": false, "query": "fix basic troubleshooting"}
{"output": [["lex", "understand overview world celebrations tips"], ["lex", "understand overview world celebrations tutorial"], ["lex", "understand overview world celebrations how to"], ["vec", "best way to understand world celebrations"], ["vec", "learn understand world celebrations step by step"], ["hyde", "Learning understand world celebrations requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "arts_culture", "is_short": false, "query": "understand world celebrations"}
{"output": [["lex", "wheel overview throwing basics tutorial"], ["lex", "wheel overview throwing basics guide"], ["lex", "wheel overview throwing basics tips"], ["vec", "how to wheel throwing basics effectively"], ["vec", "learn wheel throwing basics step by step"], ["hyde", "Whether you're a beginner or looking to improve, this guide to wheel throwing basics offers practical advice, common pitfalls to avoid, and proven strategies for success."]], "category": "hobbies_crafts", "is_short": false, "query": "wheel throwing basics"}
{"output": [["lex", "breakfast overview ideas techniques how to"], ["lex", "breakfast overview ideas techniques tips"], ["lex", "breakfast overview ideas techniques tutorial"], ["vec", "complete guide to breakfast ideas techniques"], ["vec", "best way to breakfast ideas techniques"], ["hyde", "Learning breakfast ideas techniques requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "food_cooking", "is_short": false, "query": "breakfast ideas techniques"}
{"output": [["lex", "recipe tutorial overview"], ["lex", "recipe how to overview"], ["lex", "recipe advice overview"], ["vec", "best way to recipe"], ["vec", "learn recipe step by step"], ["hyde", "Learning recipe requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "education_learning", "is_short": true, "query": "recipe"}
{"output": [["lex", "improve overview child development advice"], ["lex", "improve overview child development tutorial"], ["lex", "improve overview child development tips"], ["vec", "learn improve child development step by step"], ["vec", "tips for improve child development success"], ["hyde", "This comprehensive guide to improve child development covers all the essential information you need to get started. Follow the steps carefully for best results."]], "category": "lifestyle_relationships", "is_short": false, "query": "improve child development"}
{"output": [["lex", "learn overview cooking techniques how to"], ["lex", "learn overview cooking techniques tips"], ["lex", "learn overview cooking techniques advice"], ["vec", "best way to learn cooking techniques"], ["vec", "how to learn cooking techniques effectively"], ["hyde", "Whether you're a beginner or looking to improve, this guide to learn cooking techniques offers practical advice, common pitfalls to avoid, and proven strategies for success."]], "category": "food_cooking", "is_short": false, "query": "learn cooking techniques"}
{"output": [["lex", "flight tutorial overview"], ["lex", "flight advice overview"], ["lex", "flight how to overview"], ["vec", "learn flight step by step"], ["vec", "how to flight effectively"], ["hyde", "Learning flight requires practice and patience. This resource provides detailed instructions, examples, and tips to help you master the basics quickly."]], "category": "home_garden", "is_short": true, "query": "flight"}
{"output": [["lex", "DIY overview lawn care how to"], ["lex", "DIY overview lawn care guide"], ["lex", "DIY overview lawn care advice"], ["vec", "complete guide to DIY lawn care"], ["vec", "how to DIY lawn care effectively"], ["hyde", "This comprehensive guide to DIY lawn care covers all the essential information you need to get started. Follow the steps carefully for best results."]], "category": "home_garden", "is_short": false, "
gitextract_5sd6wqxn/ ├── .claude-plugin/ │ └── marketplace.json ├── .gitattributes ├── .github/ │ └── workflows/ │ ├── ci.yml │ └── publish.yml ├── .gitignore ├── .pi/ │ └── settings.json ├── CHANGELOG.md ├── CLAUDE.md ├── LICENSE ├── README.md ├── bin/ │ └── qmd ├── docs/ │ └── SYNTAX.md ├── example-index.yml ├── finetune/ │ ├── .gitignore │ ├── CLAUDE.md │ ├── Justfile │ ├── Modelfile │ ├── README.md │ ├── SCORING.md │ ├── configs/ │ │ ├── accelerate_multi_gpu.yaml │ │ ├── sft.yaml │ │ └── sft_local.yaml │ ├── convert_gguf.py │ ├── convert_onnx.py │ ├── data/ │ │ ├── qmd_expansion_balanced_deduped.jsonl │ │ ├── qmd_expansion_diverse_addon.jsonl │ │ ├── qmd_expansion_handcrafted.jsonl │ │ ├── qmd_expansion_handcrafted_only.jsonl │ │ ├── qmd_expansion_lex_phrases_negation.jsonl │ │ ├── qmd_expansion_locations.jsonl │ │ ├── qmd_expansion_people.jsonl │ │ ├── qmd_expansion_personal_entities.jsonl │ │ ├── qmd_expansion_short_nontech.jsonl │ │ ├── qmd_expansion_sports.jsonl │ │ ├── qmd_expansion_v3_structured.jsonl │ │ └── qmd_only_sampled.jsonl │ ├── dataset/ │ │ ├── analyze_data.py │ │ ├── prepare_data.py │ │ ├── schema.py │ │ ├── score_data.py │ │ └── validate_schema.py │ ├── eval.py │ ├── evals/ │ │ ├── .gitignore │ │ └── queries.txt │ ├── experiments/ │ │ ├── gepa/ │ │ │ ├── __init__.py │ │ │ ├── best_prompt.txt │ │ │ ├── best_prompt_glm.txt │ │ │ ├── dspy_gepa.py │ │ │ ├── example.py │ │ │ ├── generate.py │ │ │ ├── gepa_outputs.jsonl │ │ │ ├── gepa_outputs_glm.jsonl │ │ │ ├── model.json │ │ │ ├── optimizer.py │ │ │ └── score.py │ │ ├── grpo/ │ │ │ ├── README.md │ │ │ ├── grpo.py │ │ │ └── grpo.yaml │ │ └── lfm2/ │ │ ├── sft_lfm2.py │ │ └── sft_lfm2.yaml │ ├── jobs/ │ │ ├── eval.py │ │ ├── eval_common.py │ │ └── sft.py │ ├── pyproject.toml │ ├── reward.py │ └── train.py ├── flake.nix ├── migrate-schema.ts ├── package.json ├── scripts/ │ ├── extract-changelog.sh │ ├── install-hooks.sh │ ├── pre-push │ └── release.sh ├── skills/ │ ├── qmd/ │ │ ├── SKILL.md │ │ └── references/ │ │ └── mcp-setup.md │ └── release/ │ ├── SKILL.md │ └── scripts/ │ └── install-hooks.sh ├── src/ │ ├── bench-rerank.ts │ ├── cli/ │ │ ├── formatter.ts │ │ └── qmd.ts │ ├── collections.ts │ ├── db.ts │ ├── embedded-skills.ts │ ├── index.ts │ ├── llm.ts │ ├── maintenance.ts │ ├── mcp/ │ │ └── server.ts │ ├── store.ts │ └── test-preload.ts ├── test/ │ ├── Containerfile │ ├── cli.test.ts │ ├── collections-config.test.ts │ ├── eval-bm25.test.ts │ ├── eval-harness.ts │ ├── eval.test.ts │ ├── formatter.test.ts │ ├── intent.test.ts │ ├── launcher-detection.test.sh │ ├── llm.test.ts │ ├── mcp.test.ts │ ├── multi-collection-filter.test.ts │ ├── rrf-trace.test.ts │ ├── sdk.test.ts │ ├── smoke-install.sh │ ├── store-paths.test.ts │ ├── store.helpers.unit.test.ts │ ├── store.test.ts │ └── structured-search.test.ts ├── tsconfig.build.json ├── tsconfig.json └── vitest.config.ts
SYMBOL INDEX (621 symbols across 41 files)
FILE: finetune/convert_gguf.py
function run_cmd (line 60) | def run_cmd(cmd, description):
function main (line 76) | def main():
FILE: finetune/convert_onnx.py
function merge_adapters (line 69) | def merge_adapters(base_model: str, sft_model: str, grpo_model: str) -> ...
function export_onnx (line 88) | def export_onnx(model, tokenizer, output_dir: str):
function _find_onnx_model (line 116) | def _find_onnx_model(onnx_dir: str) -> Path:
function quantize_onnx (line 127) | def quantize_onnx(onnx_dir: str, quantize_type: str):
function _quantize_q4 (line 144) | def _quantize_q4(model_path: Path):
function _quantize_q8 (line 169) | def _quantize_q8(model_path: Path):
function _convert_fp16 (line 189) | def _convert_fp16(model_path: Path):
function _report_size (line 210) | def _report_size(path: Path):
function validate_onnx (line 217) | def validate_onnx(onnx_dir: str, base_model: str):
function write_transformers_js_config (line 294) | def write_transformers_js_config(onnx_dir: str, quantize_type: str = "q4"):
function upload_to_hub (line 305) | def upload_to_hub(
function main (line 373) | def main():
FILE: finetune/dataset/analyze_data.py
class DatasetStats (line 28) | class DatasetStats:
function categorize_query (line 43) | def categorize_query(query: str) -> str:
function extract_named_entities (line 71) | def extract_named_entities(query: str) -> list:
function analyze_examples (line 84) | def analyze_examples(examples: list[TrainingExample]) -> tuple[DatasetSt...
function print_report (line 131) | def print_report(stats: DatasetStats, categories: dict, category_example...
function main (line 196) | def main():
FILE: finetune/dataset/prepare_data.py
function get_tokenizer (line 37) | def get_tokenizer():
function format_for_training (line 46) | def format_for_training(ex: TrainingExample) -> dict:
function main (line 81) | def main():
FILE: finetune/dataset/schema.py
class OutputType (line 35) | class OutputType(str, Enum):
class OutputPair (line 44) | class OutputPair(BaseModel):
method text_not_empty (line 54) | def text_not_empty(cls, v: str) -> str:
method to_list (line 59) | def to_list(self) -> list[str]:
function _coerce_output_pairs (line 63) | def _coerce_output_pairs(v: list) -> list[OutputPair]:
class TrainingExample (line 82) | class TrainingExample(BaseModel):
method query_not_empty (line 97) | def query_not_empty(cls, v: str) -> str:
method output_not_empty (line 104) | def output_not_empty(cls, v: list[OutputPair]) -> list[OutputPair]:
method output_as_lists (line 109) | def output_as_lists(self) -> list[list[str]]:
function load_examples (line 118) | def load_examples(path: str | Path) -> list[TrainingExample]:
function parse_output_text (line 142) | def parse_output_text(text: str) -> list[list[str]]:
function reorder_hyde_first (line 162) | def reorder_hyde_first(items: list[list[str]]) -> list[list[str]]:
function output_items_to_text (line 170) | def output_items_to_text(
function normalize_output_items (line 204) | def normalize_output_items(
function has_type (line 237) | def has_type(items: Iterable, kind: str) -> bool:
FILE: finetune/dataset/score_data.py
function score_file (line 20) | def score_file(path: Path) -> tuple[int, int, list[float], dict]:
function main (line 48) | def main() -> int:
FILE: finetune/dataset/validate_schema.py
function validate_file (line 19) | def validate_file(path: Path) -> tuple[int, int]:
function main (line 45) | def main() -> int:
FILE: finetune/eval.py
function load_model (line 35) | def load_model(model_path: str):
function generate_batch (line 77) | def generate_batch(
function main (line 118) | def main():
FILE: finetune/experiments/gepa/dspy_gepa.py
function _import_dspy (line 13) | def _import_dspy():
class ExpandSignature (line 34) | class ExpandSignature(dspy.Signature):
class Expander (line 49) | class Expander(dspy.Module):
method __init__ (line 50) | def __init__(self):
method forward (line 54) | def forward(self, query: str):
function reward_metric (line 58) | def reward_metric(gold, pred, trace=None, pred_name=None, pred_trace=None):
function load_queries (line 66) | def load_queries(path: Path) -> list[str]:
function to_examples (line 80) | def to_examples(queries: list[str]) -> list[dspy.Example]:
function _coerce_output_items (line 84) | def _coerce_output_items(pred) -> list[list[str]]:
function write_jsonl (line 106) | def write_jsonl(path: Path, queries: list[str], outputs: list[list[list[...
function main (line 112) | def main() -> int:
FILE: finetune/experiments/gepa/example.py
class SearchType (line 13) | class SearchType(str, Enum):
class OutputItem (line 27) | class OutputItem:
method __post_init__ (line 37) | def __post_init__(self) -> None:
method to_pair (line 48) | def to_pair(self) -> list[str]:
class Example (line 53) | class Example:
method __post_init__ (line 59) | def __post_init__(self) -> None:
method to_json (line 66) | def to_json(self) -> dict:
method to_jsonl (line 72) | def to_jsonl(self) -> str:
function parse_output_items (line 76) | def parse_output_items(raw_output: Iterable[Iterable[str]]) -> list[Outp...
function example_from_json (line 95) | def example_from_json(obj: dict) -> Example:
function load_jsonl (line 104) | def load_jsonl(path: str | Path) -> list[Example]:
FILE: finetune/experiments/gepa/generate.py
function _import_dspy (line 13) | def _import_dspy():
function load_topics (line 32) | def load_topics(path: Path) -> list[str]:
function write_jsonl_line (line 53) | def write_jsonl_line(handle, query: str, output_text: str) -> None:
function parse_queries (line 58) | def parse_queries(text: str) -> list[str]:
function main (line 68) | def main() -> int:
FILE: finetune/experiments/gepa/optimizer.py
function build_prompt (line 13) | def build_prompt() -> str:
function write_model_json (line 34) | def write_model_json(path: Path) -> None:
function main (line 52) | def main() -> int:
FILE: finetune/experiments/gepa/score.py
function score_file (line 17) | def score_file(path: Path) -> tuple[int, int, list[float], dict]:
function main (line 50) | def main() -> int:
FILE: finetune/experiments/grpo/grpo.py
function main (line 54) | def main():
FILE: finetune/jobs/eval.py
function parse_expansion (line 108) | def parse_expansion(text):
function clean_model_output (line 125) | def clean_model_output(text):
function extract_named_entities (line 133) | def extract_named_entities(query):
function get_key_terms (line 157) | def get_key_terms(query):
function lex_preserves_key_terms (line 161) | def lex_preserves_key_terms(lex_line, query):
function lex_preserves_entities (line 166) | def lex_preserves_entities(line, entities):
function lex_is_generic (line 171) | def lex_is_generic(lex_line):
function word_set_distance (line 183) | def word_set_distance(a, b):
function is_diverse (line 187) | def is_diverse(a, b, min_distance=2):
function echoes_query (line 193) | def echoes_query(expansion, query):
function word_repetition_penalty (line 198) | def word_repetition_penalty(text):
function score_expansion_detailed (line 204) | def score_expansion_detailed(query, expansion):
function load_model (line 312) | def load_model(base, sft=None, grpo=None):
function generate_expansion (line 336) | def generate_expansion(model, tokenizer, query, max_new_tokens=200):
function results_to_csv (line 366) | def results_to_csv(results, label):
function upload_csv (line 386) | def upload_csv(results, label, repo_id, api):
function evaluate_model (line 401) | def evaluate_model(model, tokenizer, label):
function main (line 437) | def main():
FILE: finetune/jobs/eval_common.py
function parse_expansion (line 42) | def parse_expansion(text):
function clean_model_output (line 59) | def clean_model_output(text):
function extract_named_entities (line 67) | def extract_named_entities(query):
function get_key_terms (line 91) | def get_key_terms(query):
function lex_preserves_key_terms (line 95) | def lex_preserves_key_terms(lex_line, query):
function lex_preserves_entities (line 100) | def lex_preserves_entities(line, entities):
function lex_is_generic (line 106) | def lex_is_generic(lex_line):
function word_set_distance (line 118) | def word_set_distance(a, b):
function is_diverse (line 122) | def is_diverse(a, b, min_distance=2):
function echoes_query (line 129) | def echoes_query(expansion, query):
function word_repetition_penalty (line 134) | def word_repetition_penalty(text):
function score_expansion (line 140) | def score_expansion(query, expansion):
function extract_query_from_prompt (line 218) | def extract_query_from_prompt(prompt):
class QMDRewardFunction (line 228) | class QMDRewardFunction:
method __call__ (line 232) | def __call__(self, completions, prompts=None, **kwargs):
function generate_expansion (line 285) | def generate_expansion(model, tokenizer, query, max_new_tokens=200):
function run_eval (line 305) | def run_eval(model, tokenizer, label, upload_repo="tobil/qmd-query-expan...
FILE: finetune/reward.py
function parse_expansion (line 90) | def parse_expansion(text: str) -> dict:
function detect_only_mode (line 108) | def detect_only_mode(query: str) -> tuple[str | None, str]:
function clean_model_output (line 121) | def clean_model_output(text: str) -> tuple[str, bool]:
function extract_named_entities (line 140) | def extract_named_entities(query: str) -> set:
function get_key_terms (line 196) | def get_key_terms(query: str) -> set:
function lex_preserves_key_terms (line 201) | def lex_preserves_key_terms(lex_line: str, query: str) -> bool:
function lex_preserves_entities (line 209) | def lex_preserves_entities(line: str, entities: set) -> bool:
function lex_has_filler (line 217) | def lex_has_filler(lex_line: str, query: str) -> bool:
function lex_is_generic (line 224) | def lex_is_generic(lex_line: str) -> bool:
function word_set_distance (line 237) | def word_set_distance(a: str, b: str) -> int:
function is_diverse (line 242) | def is_diverse(a: str, b: str, min_distance: int = 2) -> bool:
function echoes_query (line 250) | def echoes_query(expansion: str, query: str) -> bool:
function word_repetition_penalty (line 256) | def word_repetition_penalty(text: str) -> int:
function _score_only_mode (line 267) | def _score_only_mode(query: str, base_query: str, text: str, used_thinki...
function score_expansion_detailed (line 408) | def score_expansion_detailed(query: str, expansion: str) -> dict:
function score_expansion (line 628) | def score_expansion(query: str, expansion: str) -> float:
function extract_query_from_prompt (line 634) | def extract_query_from_prompt(prompt: str) -> str:
class QMDRewardFunction (line 648) | class QMDRewardFunction:
method __call__ (line 652) | def __call__(self, completions: list[str], prompts: list[str] = None, ...
FILE: finetune/train.py
function export_gguf (line 42) | def export_gguf(model, tokenizer, output_dir: str, model_name: str):
class TimedSaveCallback (line 161) | class TimedSaveCallback(TrainerCallback):
method __init__ (line 164) | def __init__(self, interval_minutes: float):
method on_step_end (line 168) | def on_step_end(self, args, state, control, **kwargs):
function run_eval (line 179) | def run_eval(model_path: str) -> float | None:
function cmd_sft (line 208) | def cmd_sft(args):
function cmd_grpo (line 413) | def cmd_grpo(args):
function main (line 648) | def main():
FILE: src/bench-rerank.ts
constant RERANK_MODEL (line 29) | const RERANK_MODEL = "hf:ggml-org/Qwen3-Reranker-0.6B-Q8_0-GGUF/qwen3-re...
constant MODEL_CACHE (line 30) | const MODEL_CACHE = join(homedir(), ".cache", "qmd", "models");
constant CONTEXT_SIZE (line 31) | const CONTEXT_SIZE = 2048;
constant DOC_COUNT (line 36) | const DOC_COUNT = docsIdx >= 0 ? parseInt(args[docsIdx + 1]!) : (quick ?...
constant ITERATIONS (line 37) | const ITERATIONS = quick ? 1 : 3;
constant PARALLEL_CONFIGS (line 38) | const PARALLEL_CONFIGS = quick ? [1, 4] : [1, 2, 4, 8];
constant QUERY (line 44) | const QUERY = "How do AI agents work and what are their limitations?";
function generateDocs (line 46) | function generateDocs(n: number): string[] {
function formatBytes (line 66) | function formatBytes(bytes: number): string {
function getMemUsage (line 72) | function getMemUsage(): { rss: number; heapUsed: number } {
function median (line 77) | function median(arr: number[]): number {
type BenchResult (line 87) | interface BenchResult {
function benchmarkConfig (line 99) | async function benchmarkConfig(
function main (line 189) | async function main() {
FILE: src/cli/formatter.ts
type MultiGetFile (line 19) | type MultiGetFile = {
type OutputFormat (line 36) | type OutputFormat = "cli" | "csv" | "md" | "xml" | "files" | "json";
type FormatOptions (line 38) | type FormatOptions = {
function addLineNumbers (line 56) | function addLineNumbers(text: string, startLine: number = 1): string {
function getDocid (line 64) | function getDocid(hash: string): string {
function escapeCSV (line 72) | function escapeCSV(value: string | null | number): string {
function escapeXml (line 81) | function escapeXml(str: string): string {
function searchResultsToJson (line 97) | function searchResultsToJson(
function searchResultsToCsv (line 128) | function searchResultsToCsv(
function searchResultsToFiles (line 157) | function searchResultsToFiles(results: SearchResult[]): string {
function searchResultsToMarkdown (line 167) | function searchResultsToMarkdown(
function searchResultsToXml (line 192) | function searchResultsToXml(
function searchResultsToMcpCsv (line 213) | function searchResultsToMcpCsv(
function documentsToJson (line 230) | function documentsToJson(results: MultiGetFile[]): string {
function documentsToCsv (line 243) | function documentsToCsv(results: MultiGetFile[]): string {
function documentsToFiles (line 260) | function documentsToFiles(results: MultiGetFile[]): string {
function documentsToMarkdown (line 271) | function documentsToMarkdown(results: MultiGetFile[]): string {
function documentsToXml (line 288) | function documentsToXml(results: MultiGetFile[]): string {
function documentToJson (line 313) | function documentToJson(doc: DocumentResult): string {
function documentToMarkdown (line 328) | function documentToMarkdown(doc: DocumentResult): string {
function documentToXml (line 342) | function documentToXml(doc: DocumentResult): string {
function formatDocument (line 360) | function formatDocument(doc: DocumentResult, format: OutputFormat): stri...
function formatSearchResults (line 381) | function formatSearchResults(
function formatDocuments (line 409) | function formatDocuments(
FILE: src/cli/qmd.ts
function getStore (line 113) | function getStore(): ReturnType<typeof createStore> {
function getDb (line 127) | function getDb(): Database {
function resyncConfig (line 132) | function resyncConfig(): void {
function closeDb (line 144) | function closeDb(): void {
function getDbPath (line 151) | function getDbPath(): string {
function setIndexName (line 155) | function setIndexName(name: string | null): void {
function ensureVecTable (line 170) | function ensureVecTable(_db: Database, dimensions: number): void {
method hide (line 190) | hide() { process.stderr.write('\x1b[?25l'); }
method show (line 191) | show() { process.stderr.write('\x1b[?25h'); }
method set (line 201) | set(percent: number) {
method clear (line 204) | clear() {
method indeterminate (line 207) | indeterminate() {
method error (line 210) | error() {
function formatETA (line 216) | function formatETA(seconds: number): string {
function checkIndexHealth (line 224) | function checkIndexHealth(db: Database): void {
function computeDisplayPath (line 245) | function computeDisplayPath(
function formatTimeAgo (line 280) | function formatTimeAgo(date: Date): string {
function formatMs (line 291) | function formatMs(ms: number): string {
function formatBytes (line 296) | function formatBytes(bytes: number): string {
function showStatus (line 303) | async function showStatus(): Promise<void> {
function updateCollections (line 497) | async function updateCollections(): Promise<void> {
function detectCollectionFromPath (line 594) | function detectCollectionFromPath(db: Database, fsPath: string): { colle...
function contextAdd (line 626) | async function contextAdd(pathArg: string | undefined, contextText: stri...
function contextList (line 692) | function contextList(): void {
function contextRemove (line 720) | function contextRemove(pathArg: string): void {
function getDocument (line 786) | function getDocument(filename: string, fromLine?: number, maxLines?: num...
function multiGet (line 967) | function multiGet(pattern: string, maxLines?: number, maxBytes: number =...
function listFiles (line 1201) | function listFiles(pathArg?: string): void {
function formatLsTime (line 1323) | function formatLsTime(date: Date): string {
function collectionList (line 1343) | function collectionList(): void {
function collectionAdd (line 1377) | async function collectionAdd(pwd: string, globPattern: string, name?: st...
function collectionRemove (line 1417) | function collectionRemove(name: string): void {
function collectionRename (line 1439) | function collectionRename(oldName: string, newName: string): void {
function indexFiles (line 1466) | async function indexFiles(pwd?: string, globPattern: string = DEFAULT_GL...
function renderProgressBar (line 1604) | function renderProgressBar(percent: number, width: number = 30): string {
function parseEmbedBatchOption (line 1611) | function parseEmbedBatchOption(name: string, value: unknown): number | u...
function vectorIndex (line 1620) | async function vectorIndex(
function sanitizeFTS5Term (line 1695) | function sanitizeFTS5Term(term: string): string {
function buildFTS5Query (line 1701) | function buildFTS5Query(query: string): string {
function normalizeBM25 (line 1727) | function normalizeBM25(score: number): number {
type OutputOptions (line 1736) | type OutputOptions = {
function highlightTerms (line 1752) | function highlightTerms(text: string, query: string): string {
function formatScore (line 1764) | function formatScore(score: number): string {
function formatExplainNumber (line 1772) | function formatExplainNumber(value: number): string {
function shortPath (line 1777) | function shortPath(dirpath: string): string {
type EmptySearchReason (line 1785) | type EmptySearchReason = "no_results" | "min_score";
function printEmptySearchResults (line 1788) | function printEmptySearchResults(format: OutputFormat, reason: EmptySear...
type OutputRow (line 1812) | type OutputRow = {
function outputResults (line 1825) | function outputResults(results: OutputRow[], query: string, opts: Output...
function resolveCollectionFilter (line 1969) | function resolveCollectionFilter(raw: string | string[] | undefined, use...
function filterByCollections (line 1990) | function filterByCollections<T extends { filepath?: string; file?: strin...
type ParsedStructuredQuery (line 2014) | interface ParsedStructuredQuery {
function parseStructuredQuery (line 2019) | function parseStructuredQuery(query: string): ParsedStructuredQuery | nu...
function search (line 2089) | function search(query: string, opts: OutputOptions): void {
function logExpansionTree (line 2126) | function logExpansionTree(originalQuery: string, expanded: ExpandedQuery...
function vectorSearch (line 2140) | async function vectorSearch(query: string, opts: OutputOptions, _model: ...
function querySearch (line 2191) | async function querySearch(query: string, opts: OutputOptions, _embedMod...
function parseCLI (line 2328) | function parseCLI() {
function getSkillInstallDir (line 2427) | function getSkillInstallDir(globalInstall: boolean): string {
function getClaudeSkillLinkPath (line 2433) | function getClaudeSkillLinkPath(globalInstall: boolean): string {
function pathExists (line 2439) | function pathExists(path: string): boolean {
function removePath (line 2448) | function removePath(path: string): void {
function showSkill (line 2457) | function showSkill(): void {
function writeEmbeddedSkill (line 2464) | function writeEmbeddedSkill(targetDir: string, force: boolean): void {
function ensureClaudeSymlink (line 2480) | function ensureClaudeSymlink(linkPath: string, targetDir: string, force:...
function shouldCreateClaudeSymlink (line 2512) | async function shouldCreateClaudeSymlink(linkPath: string, autoYes: bool...
function installSkill (line 2535) | async function installSkill(globalInstall: boolean, force: boolean, auto...
function showHelp (line 2553) | function showHelp(): void {
function showVersion (line 2646) | async function showVersion(): Promise<void> {
FILE: src/collections.ts
type ContextMap (line 22) | type ContextMap = Record<string, string>;
type Collection (line 27) | interface Collection {
type CollectionConfig (line 39) | interface CollectionConfig {
type NamedCollection (line 47) | interface NamedCollection extends Collection {
function setConfigSource (line 67) | function setConfigSource(source?: { configPath?: string; config?: Collec...
function setConfigIndexName (line 89) | function setConfigIndexName(name: string): void {
function getConfigDir (line 102) | function getConfigDir(): string {
function getConfigFilePath (line 114) | function getConfigFilePath(): string {
function ensureConfigDir (line 121) | function ensureConfigDir(): void {
function loadConfig (line 138) | function loadConfig(): CollectionConfig {
function saveConfig (line 170) | function saveConfig(config: CollectionConfig): void {
function getCollection (line 198) | function getCollection(name: string): NamedCollection | null {
function listCollections (line 212) | function listCollections(): NamedCollection[] {
function getDefaultCollections (line 223) | function getDefaultCollections(): NamedCollection[] {
function getDefaultCollectionNames (line 230) | function getDefaultCollectionNames(): string[] {
function updateCollectionSettings (line 237) | function updateCollectionSettings(
function addCollection (line 269) | function addCollection(
function removeCollection (line 288) | function removeCollection(name: string): boolean {
function renameCollection (line 303) | function renameCollection(oldName: string, newName: string): boolean {
function getGlobalContext (line 327) | function getGlobalContext(): string | undefined {
function setGlobalContext (line 335) | function setGlobalContext(context: string | undefined): void {
function getContexts (line 344) | function getContexts(collectionName: string): ContextMap | undefined {
function addContext (line 352) | function addContext(
function removeContext (line 376) | function removeContext(
function listAllContexts (line 401) | function listAllContexts(): Array<{
function findContextForPath (line 438) | function findContextForPath(
function getConfigPath (line 479) | function getConfigPath(): string {
function configExists (line 487) | function configExists(): boolean {
function isValidCollectionName (line 497) | function isValidCollectionName(name: string): boolean {
FILE: src/db.ts
function openDatabase (line 61) | function openDatabase(path: string): Database {
type Database (line 68) | interface Database {
type Statement (line 75) | interface Statement {
function loadSqliteVec (line 87) | function loadSqliteVec(db: Database): void {
FILE: src/embedded-skills.ts
type EmbeddedSkillFile (line 3) | type EmbeddedSkillFile = {
constant EMBEDDED_QMD_SKILL_BASE64 (line 8) | const EMBEDDED_QMD_SKILL_BASE64: Record<string, string> = {
function getEmbeddedQmdSkillFiles (line 13) | function getEmbeddedQmdSkillFiles(): EmbeddedSkillFile[] {
function getEmbeddedQmdSkillContent (line 20) | function getEmbeddedQmdSkillContent(): string {
FILE: src/index.ts
type UpdateProgress (line 123) | type UpdateProgress = {
type UpdateResult (line 133) | type UpdateResult = {
type SearchOptions (line 145) | interface SearchOptions {
type LexSearchOptions (line 169) | interface LexSearchOptions {
type VectorSearchOptions (line 177) | interface VectorSearchOptions {
type ExpandQueryOptions (line 185) | interface ExpandQueryOptions {
type StoreOptions (line 196) | interface StoreOptions {
type QMDStore (line 212) | interface QMDStore {
function createStore (line 333) | async function createStore(options: StoreOptions): Promise<QMDStore> {
FILE: src/llm.ts
function isQwen3EmbeddingModel (line 29) | function isQwen3EmbeddingModel(modelUri: string): boolean {
function formatQueryForEmbedding (line 38) | function formatQueryForEmbedding(query: string, modelUri?: string): stri...
function formatDocForEmbedding (line 51) | function formatDocForEmbedding(text: string, title?: string, modelUri?: ...
type TokenLogProb (line 67) | type TokenLogProb = {
type EmbeddingResult (line 75) | type EmbeddingResult = {
type GenerateResult (line 83) | type GenerateResult = {
type RerankDocumentResult (line 93) | type RerankDocumentResult = {
type RerankResult (line 102) | type RerankResult = {
type ModelInfo (line 110) | type ModelInfo = {
type EmbedOptions (line 119) | type EmbedOptions = {
type GenerateOptions (line 128) | type GenerateOptions = {
type RerankOptions (line 137) | type RerankOptions = {
type LLMSessionOptions (line 144) | type LLMSessionOptions = {
type ILLMSession (line 156) | interface ILLMSession {
type QueryType (line 170) | type QueryType = 'lex' | 'vec' | 'hyde';
type Queryable (line 175) | type Queryable = {
type RerankDocument (line 183) | type RerankDocument = {
constant DEFAULT_EMBED_MODEL (line 196) | const DEFAULT_EMBED_MODEL = process.env.QMD_EMBED_MODEL ?? "hf:ggml-org/...
constant DEFAULT_RERANK_MODEL (line 197) | const DEFAULT_RERANK_MODEL = "hf:ggml-org/Qwen3-Reranker-0.6B-Q8_0-GGUF/...
constant DEFAULT_GENERATE_MODEL (line 199) | const DEFAULT_GENERATE_MODEL = "hf:tobil/qmd-query-expansion-1.7B-gguf/q...
constant LFM2_GENERATE_MODEL (line 204) | const LFM2_GENERATE_MODEL = "hf:LiquidAI/LFM2-1.2B-GGUF/LFM2-1.2B-Q4_K_M...
constant LFM2_INSTRUCT_MODEL (line 205) | const LFM2_INSTRUCT_MODEL = "hf:LiquidAI/LFM2.5-1.2B-Instruct-GGUF/LFM2....
constant DEFAULT_EMBED_MODEL_URI (line 207) | const DEFAULT_EMBED_MODEL_URI = DEFAULT_EMBED_MODEL;
constant DEFAULT_RERANK_MODEL_URI (line 208) | const DEFAULT_RERANK_MODEL_URI = DEFAULT_RERANK_MODEL;
constant DEFAULT_GENERATE_MODEL_URI (line 209) | const DEFAULT_GENERATE_MODEL_URI = DEFAULT_GENERATE_MODEL;
constant MODEL_CACHE_DIR (line 212) | const MODEL_CACHE_DIR = join(homedir(), ".cache", "qmd", "models");
constant DEFAULT_MODEL_CACHE_DIR (line 213) | const DEFAULT_MODEL_CACHE_DIR = MODEL_CACHE_DIR;
type PullResult (line 215) | type PullResult = {
type HfRef (line 222) | type HfRef = {
function parseHfUri (line 227) | function parseHfUri(model: string): HfRef | null {
function getRemoteEtag (line 237) | async function getRemoteEtag(ref: HfRef): Promise<string | null> {
function pullModels (line 249) | async function pullModels(
type LLM (line 314) | interface LLM {
type LlamaCppConfig (line 352) | type LlamaCppConfig = {
constant DEFAULT_INACTIVITY_TIMEOUT_MS (line 383) | const DEFAULT_INACTIVITY_TIMEOUT_MS = 5 * 60 * 1000;
constant DEFAULT_EXPAND_CONTEXT_SIZE (line 384) | const DEFAULT_EXPAND_CONTEXT_SIZE = 2048;
function resolveExpandContextSize (line 386) | function resolveExpandContextSize(configValue?: number): number {
class LlamaCpp (line 407) | class LlamaCpp implements LLM {
method constructor (line 436) | constructor(config: LlamaCppConfig = {}) {
method touchActivity (line 450) | private touchActivity(): void {
method hasLoadedContexts (line 480) | private hasLoadedContexts(): boolean {
method unloadIdleResources (line 490) | async unloadIdleResources(): Promise<void> {
method ensureModelCacheDir (line 538) | private ensureModelCacheDir(): void {
method ensureLlama (line 547) | private async ensureLlama(): Promise<Llama> {
method resolveModel (line 568) | private async resolveModel(modelUri: string): Promise<string> {
method ensureEmbedModel (line 577) | private async ensureEmbedModel(): Promise<LlamaModel> {
method computeParallelism (line 611) | private async computeParallelism(perContextMB: number): Promise<number> {
method threadsPerContext (line 635) | private async threadsPerContext(parallelism: number): Promise<number> {
method ensureEmbedContexts (line 648) | private async ensureEmbedContexts(): Promise<LlamaEmbeddingContext[]> {
method ensureEmbedContext (line 687) | private async ensureEmbedContext(): Promise<LlamaEmbeddingContext> {
method ensureGenerateModel (line 695) | private async ensureGenerateModel(): Promise<LlamaModel> {
method ensureRerankModel (line 725) | private async ensureRerankModel(): Promise<LlamaModel> {
method ensureRerankContexts (line 763) | private async ensureRerankContexts(): Promise<Awaited<ReturnType<Llama...
method tokenize (line 804) | async tokenize(text: string): Promise<readonly LlamaToken[]> {
method countTokens (line 815) | async countTokens(text: string): Promise<number> {
method detokenize (line 823) | async detokenize(tokens: readonly LlamaToken[]): Promise<string> {
method truncateToContextSize (line 841) | private async truncateToContextSize(text: string): Promise<{ text: str...
method embed (line 857) | async embed(text: string, options: EmbedOptions = {}): Promise<Embeddi...
method embedBatch (line 886) | async embedBatch(texts: string[]): Promise<(EmbeddingResult | null)[]> {
method generate (line 953) | async generate(prompt: string, options: GenerateOptions = {}): Promise...
method modelExists (line 994) | async modelExists(modelUri: string): Promise<ModelInfo> {
method expandQuery (line 1013) | async expandQuery(query: string, options: { context?: string, includeL...
method rerank (line 1106) | async rerank(
method getDeviceInfo (line 1205) | async getDeviceInfo(): Promise<{
method dispose (line 1230) | async dispose(): Promise<void> {
class LLMSessionManager (line 1276) | class LLMSessionManager {
method constructor (line 1281) | constructor(llm: LlamaCpp) {
method activeSessionCount (line 1285) | get activeSessionCount(): number {
method inFlightOperations (line 1289) | get inFlightOperations(): number {
method canUnload (line 1297) | canUnload(): boolean {
method acquire (line 1301) | acquire(): void {
method release (line 1305) | release(): void {
method operationStart (line 1309) | operationStart(): void {
method operationEnd (line 1313) | operationEnd(): void {
method getLlamaCpp (line 1317) | getLlamaCpp(): LlamaCpp {
class SessionReleasedError (line 1325) | class SessionReleasedError extends Error {
method constructor (line 1326) | constructor(message = "LLM session has been released or aborted") {
class LLMSession (line 1336) | class LLMSession implements ILLMSession {
method constructor (line 1343) | constructor(manager: LLMSessionManager, options: LLMSessionOptions = {...
method isValid (line 1372) | get isValid(): boolean {
method signal (line 1376) | get signal(): AbortSignal {
method release (line 1384) | release(): void {
method withOperation (line 1400) | private async withOperation<T>(fn: () => Promise<T>): Promise<T> {
method embed (line 1419) | async embed(text: string, options?: EmbedOptions): Promise<EmbeddingRe...
method embedBatch (line 1423) | async embedBatch(texts: string[]): Promise<(EmbeddingResult | null)[]> {
method expandQuery (line 1427) | async expandQuery(
method rerank (line 1434) | async rerank(
function getSessionManager (line 1449) | function getSessionManager(): LLMSessionManager {
function withLLMSession (line 1471) | async function withLLMSession<T>(
function withLLMSessionForLlm (line 1489) | async function withLLMSessionForLlm<T>(
function canUnloadLLM (line 1508) | function canUnloadLLM(): boolean {
function getDefaultLlamaCpp (line 1522) | function getDefaultLlamaCpp(): LlamaCpp {
function setDefaultLlamaCpp (line 1533) | function setDefaultLlamaCpp(llm: LlamaCpp | null): void {
function disposeDefaultLlamaCpp (line 1541) | async function disposeDefaultLlamaCpp(): Promise<void> {
FILE: src/maintenance.ts
class Maintenance (line 18) | class Maintenance {
method constructor (line 21) | constructor(store: Store) {
method vacuum (line 26) | vacuum(): void {
method cleanupOrphanedContent (line 31) | cleanupOrphanedContent(): number {
method cleanupOrphanedVectors (line 36) | cleanupOrphanedVectors(): number {
method clearLLMCache (line 41) | clearLLMCache(): number {
method deleteInactiveDocs (line 46) | deleteInactiveDocs(): number {
method clearEmbeddings (line 51) | clearEmbeddings(): void {
FILE: src/mcp/server.ts
type SearchResultItem (line 34) | type SearchResultItem = {
type StatusResult (line 43) | type StatusResult = {
function encodeQmdPath (line 64) | function encodeQmdPath(path: string): string {
function formatSearchSummary (line 72) | function formatSearchSummary(results: SearchResultItem[], query: string)...
function buildInstructions (line 92) | async function buildInstructions(store: QMDStore): Promise<string> {
function createMcpServer (line 158) | async function createMcpServer(store: QMDStore): Promise<McpServer> {
function startMcpServer (line 522) | async function startMcpServer(): Promise<void> {
type HttpServerHandle (line 533) | type HttpServerHandle = {
function startMcpHttpServer (line 543) | async function startMcpHttpServer(port: number, options?: { quiet?: bool...
FILE: src/store.ts
constant HOME (line 41) | const HOME = process.env.HOME || "/tmp";
constant DEFAULT_EMBED_MODEL (line 42) | const DEFAULT_EMBED_MODEL = "embeddinggemma";
constant DEFAULT_RERANK_MODEL (line 43) | const DEFAULT_RERANK_MODEL = "ExpedientFalcon/qwen3-reranker:0.6b-q8_0";
constant DEFAULT_QUERY_MODEL (line 44) | const DEFAULT_QUERY_MODEL = "Qwen/Qwen3-1.7B";
constant DEFAULT_GLOB (line 45) | const DEFAULT_GLOB = "**/*.md";
constant DEFAULT_MULTI_GET_MAX_BYTES (line 46) | const DEFAULT_MULTI_GET_MAX_BYTES = 10 * 1024;
constant DEFAULT_EMBED_MAX_DOCS_PER_BATCH (line 47) | const DEFAULT_EMBED_MAX_DOCS_PER_BATCH = 64;
constant DEFAULT_EMBED_MAX_BATCH_BYTES (line 48) | const DEFAULT_EMBED_MAX_BATCH_BYTES = 64 * 1024 * 1024;
constant CHUNK_SIZE_TOKENS (line 52) | const CHUNK_SIZE_TOKENS = 900;
constant CHUNK_OVERLAP_TOKENS (line 53) | const CHUNK_OVERLAP_TOKENS = Math.floor(CHUNK_SIZE_TOKENS * 0.15);
constant CHUNK_SIZE_CHARS (line 55) | const CHUNK_SIZE_CHARS = CHUNK_SIZE_TOKENS * 4;
constant CHUNK_OVERLAP_CHARS (line 56) | const CHUNK_OVERLAP_CHARS = CHUNK_OVERLAP_TOKENS * 4;
constant CHUNK_WINDOW_TOKENS (line 58) | const CHUNK_WINDOW_TOKENS = 200;
constant CHUNK_WINDOW_CHARS (line 59) | const CHUNK_WINDOW_CHARS = CHUNK_WINDOW_TOKENS * 4;
function getLlm (line 65) | function getLlm(store: Store): LlamaCpp {
type BreakPoint (line 76) | interface BreakPoint {
type CodeFenceRegion (line 86) | interface CodeFenceRegion {
constant BREAK_PATTERNS (line 97) | const BREAK_PATTERNS: [RegExp, number, string][] = [
function scanBreakPoints (line 117) | function scanBreakPoints(text: string): BreakPoint[] {
function findCodeFences (line 144) | function findCodeFences(text: string): CodeFenceRegion[] {
function isInsideCodeFence (line 171) | function isInsideCodeFence(pos: number, fences: CodeFenceRegion[]): bool...
function findBestCutoff (line 188) | function findBestCutoff(
constant STRONG_SIGNAL_MIN_SCORE (line 228) | const STRONG_SIGNAL_MIN_SCORE = 0.85;
constant STRONG_SIGNAL_MIN_GAP (line 229) | const STRONG_SIGNAL_MIN_GAP = 0.15;
constant RERANK_CANDIDATE_LIMIT (line 232) | const RERANK_CANDIDATE_LIMIT = 40;
type ExpandedQuery (line 242) | type ExpandedQuery = {
function homedir (line 253) | function homedir(): string {
function isAbsolutePath (line 267) | function isAbsolutePath(path: string): boolean {
function normalizePathSeparators (line 297) | function normalizePathSeparators(path: string): string {
function isWSL (line 305) | function isWSL(): boolean {
function getRelativePathFromPrefix (line 314) | function getRelativePathFromPrefix(path: string, prefix: string): string...
function resolve (line 341) | function resolve(...paths: string[]): string {
function enableProductionMode (line 438) | function enableProductionMode(): void {
function getDefaultDbPath (line 442) | function getDefaultDbPath(indexName: string = "index"): string {
function getPwd (line 462) | function getPwd(): string {
function getRealPath (line 466) | function getRealPath(path: string): string {
type VirtualPath (line 478) | type VirtualPath = {
function normalizeVirtualPath (line 494) | function normalizeVirtualPath(input: string): string {
function parseVirtualPath (line 521) | function parseVirtualPath(virtualPath: string): VirtualPath | null {
function buildVirtualPath (line 538) | function buildVirtualPath(collectionName: string, path: string): string {
function isVirtualPath (line 551) | function isVirtualPath(path: string): boolean {
function resolveVirtualPath (line 566) | function resolveVirtualPath(db: Database, virtualPath: string): string |...
function toVirtualPath (line 580) | function toVirtualPath(db: Database, absolutePath: string): string | null {
function createSqliteVecUnavailableError (line 614) | function createSqliteVecUnavailableError(reason: string): Error {
function getErrorMessage (line 623) | function getErrorMessage(err: unknown): string {
function verifySqliteVecLoaded (line 627) | function verifySqliteVecLoaded(db: Database): void {
function initializeDatabase (line 641) | function initializeDatabase(db: Database): void {
type StoreCollectionRow (line 787) | type StoreCollectionRow = {
function rowToNamedCollection (line 797) | function rowToNamedCollection(row: StoreCollectionRow): NamedCollection {
function getStoreCollections (line 809) | function getStoreCollections(db: Database): NamedCollection[] {
function getStoreCollection (line 814) | function getStoreCollection(db: Database, name: string): NamedCollection...
function getStoreGlobalContext (line 820) | function getStoreGlobalContext(db: Database): string | undefined {
function getStoreContexts (line 826) | function getStoreContexts(db: Database): Array<{ collection: string; pat...
function upsertStoreCollection (line 847) | function upsertStoreCollection(db: Database, name: string, collection: O...
function deleteStoreCollection (line 869) | function deleteStoreCollection(db: Database, name: string): boolean {
function renameStoreCollection (line 874) | function renameStoreCollection(db: Database, oldName: string, newName: s...
function updateStoreContext (line 885) | function updateStoreContext(db: Database, collectionName: string, path: ...
function removeStoreContext (line 895) | function removeStoreContext(db: Database, collectionName: string, path: ...
function setStoreGlobalContext (line 909) | function setStoreGlobalContext(db: Database, value: string | undefined):...
function syncConfigToDb (line 921) | function syncConfigToDb(db: Database, config: CollectionConfig): void {
function isSqliteVecAvailable (line 958) | function isSqliteVecAvailable(): boolean {
function ensureVecTableInternal (line 962) | function ensureVecTableInternal(db: Database, dimensions: number): void {
type Store (line 983) | type Store = {
type ReindexProgress (line 1060) | type ReindexProgress = {
type ReindexResult (line 1066) | type ReindexResult = {
function reindexCollection (line 1078) | async function reindexCollection(
type EmbedProgress (line 1180) | type EmbedProgress = {
type EmbedResult (line 1188) | type EmbedResult = {
type EmbedOptions (line 1195) | type EmbedOptions = {
type PendingEmbeddingDoc (line 1203) | type PendingEmbeddingDoc = {
type EmbeddingDoc (line 1209) | type EmbeddingDoc = PendingEmbeddingDoc & {
type ChunkItem (line 1213) | type ChunkItem = {
function validatePositiveIntegerOption (line 1223) | function validatePositiveIntegerOption(name: string, value: number | und...
function resolveEmbedOptions (line 1231) | function resolveEmbedOptions(options?: EmbedOptions): Required<Pick<Embe...
function getPendingEmbeddingDocs (line 1238) | function getPendingEmbeddingDocs(db: Database): PendingEmbeddingDoc[] {
function buildEmbeddingBatches (line 1250) | function buildEmbeddingBatches(
function getEmbeddingDocsForBatch (line 1281) | function getEmbeddingDocsForBatch(db: Database, batch: PendingEmbeddingD...
function generateEmbeddings (line 1303) | async function generateEmbeddings(
function createStore (line 1456) | function createStore(dbPath?: string): Store {
type DocumentResult (line 1543) | type DocumentResult = {
function getDocid (line 1559) | function getDocid(hash: string): string {
function emojiToHex (line 1573) | function emojiToHex(str: string): string {
function handelize (line 1581) | function handelize(path: string): string {
type SearchResult (line 1637) | type SearchResult = DocumentResult & {
type RankedResult (line 1646) | type RankedResult = {
type RRFContributionTrace (line 1654) | type RRFContributionTrace = {
type RRFScoreTrace (line 1665) | type RRFScoreTrace = {
type HybridQueryExplain (line 1673) | type HybridQueryExplain = {
type DocumentNotFound (line 1692) | type DocumentNotFound = {
type MultiGetResult (line 1701) | type MultiGetResult = {
type CollectionInfo (line 1710) | type CollectionInfo = {
type IndexStatus (line 1718) | type IndexStatus = {
function getHashesNeedingEmbedding (line 1729) | function getHashesNeedingEmbedding(db: Database): number {
type IndexHealthInfo (line 1739) | type IndexHealthInfo = {
function getIndexHealth (line 1745) | function getIndexHealth(db: Database): IndexHealthInfo {
function getCacheKey (line 1763) | function getCacheKey(url: string, body: object): string {
function getCachedResult (line 1770) | function getCachedResult(db: Database, cacheKey: string): string | null {
function setCachedResult (line 1775) | function setCachedResult(db: Database, cacheKey: string, result: string)...
function clearCache (line 1783) | function clearCache(db: Database): void {
function deleteLLMCache (line 1795) | function deleteLLMCache(db: Database): number {
function deleteInactiveDocuments (line 1804) | function deleteInactiveDocuments(db: Database): number {
function cleanupOrphanedContent (line 1813) | function cleanupOrphanedContent(db: Database): number {
function cleanupOrphanedVectors (line 1825) | function cleanupOrphanedVectors(db: Database): number {
function vacuumDatabase (line 1879) | function vacuumDatabase(db: Database): void {
function hashContent (line 1887) | async function hashContent(content: string): Promise<string> {
function extractTitle (line 1915) | function extractTitle(content: string, filename: string): string {
function insertContent (line 1933) | function insertContent(db: Database, hash: string, content: string, crea...
function insertDocument (line 1941) | function insertDocument(
function findActiveDocument (line 1964) | function findActiveDocument(
function updateDocumentTitle (line 1979) | function updateDocumentTitle(
function updateDocument (line 1993) | function updateDocument(
function deactivateDocument (line 2007) | function deactivateDocument(db: Database, collectionName: string, path: ...
function getActiveDocumentPaths (line 2015) | function getActiveDocumentPaths(db: Database, collectionName: string): s...
function chunkDocument (line 2024) | function chunkDocument(
function chunkDocumentByTokens (line 2091) | async function chunkDocumentByTokens(
function levenshtein (line 2143) | function levenshtein(a: string, b: string): number {
function normalizeDocid (line 2168) | function normalizeDocid(docid: string): string {
function isDocid (line 2190) | function isDocid(input: string): boolean {
function findDocumentByDocid (line 2203) | function findDocumentByDocid(db: Database, docid: string): { filepath: s...
function findSimilarFiles (line 2219) | function findSimilarFiles(db: Database, query: string, maxDistance: numb...
function matchFilesByGlob (line 2234) | function matchFilesByGlob(db: Database, pattern: string): { filepath: st...
function getContextForPath (line 2270) | function getContextForPath(db: Database, collectionName: string, path: s...
function getContextForFile (line 2314) | function getContextForFile(db: Database, filepath: string): string | null {
function getCollectionByName (line 2400) | function getCollectionByName(db: Database, name: string): { name: string...
function listCollections (line 2415) | function listCollections(db: Database): { name: string; pwd: string; glo...
function removeCollection (line 2447) | function removeCollection(db: Database, collectionName: string): { delet...
function renameCollection (line 2470) | function renameCollection(db: Database, oldName: string, newName: string...
function insertContext (line 2486) | function insertContext(db: Database, collectionId: number, pathPrefix: s...
function deleteContext (line 2501) | function deleteContext(db: Database, collectionName: string, pathPrefix:...
function deleteGlobalContexts (line 2511) | function deleteGlobalContexts(db: Database): number {
function listPathContexts (line 2534) | function listPathContexts(db: Database): { collection_name: string; path...
function getAllCollections (line 2559) | function getAllCollections(db: Database): { name: string }[] {
function getCollectionsWithoutContext (line 2568) | function getCollectionsWithoutContext(db: Database): { name: string; pwd...
function getTopLevelPathsWithoutContext (line 2600) | function getTopLevelPathsWithoutContext(db: Database, collectionName: st...
function sanitizeFTS5Term (line 2653) | function sanitizeFTS5Term(term: string): string {
function buildFTS5Query (line 2672) | function buildFTS5Query(query: string): string | null {
function validateSemanticQuery (line 2744) | function validateSemanticQuery(query: string): string | null {
function validateLexQuery (line 2752) | function validateLexQuery(query: string): string | null {
function searchFTS (line 2763) | function searchFTS(db: Database, query: string, limit: number = 20, coll...
function searchVec (line 2820) | async function searchVec(db: Database, query: string, model: string, lim...
function getEmbedding (line 2910) | async function getEmbedding(text: string, model: string, isQuery: boolea...
function getHashesForEmbedding (line 2923) | function getHashesForEmbedding(db: Database): { hash: string; body: stri...
function clearAllEmbeddings (line 2938) | function clearAllEmbeddings(db: Database): void {
function insertEmbedding (line 2947) | function insertEmbedding(
function expandQuery (line 2968) | async function expandQuery(query: string, model: string = DEFAULT_QUERY_...
function rerank (line 3007) | async function rerank(query: string, documents: { file: string; text: st...
function reciprocalRankFusion (line 3056) | function reciprocalRankFusion(
function buildRrfTrace (line 3104) | function buildRrfTrace(
type DbDocRow (line 3171) | type DbDocRow = {
function findDocument (line 3193) | function findDocument(db: Database, filename: string, options: { include...
function getDocumentBody (line 3302) | function getDocumentBody(db: Database, doc: DocumentResult | { filepath:...
function findDocuments (line 3352) | function findDocuments(
function getStatus (line 3462) | function getStatus(db: Database): IndexStatus {
type SnippetResult (line 3512) | type SnippetResult = {
constant INTENT_WEIGHT_SNIPPET (line 3521) | const INTENT_WEIGHT_SNIPPET = 0.3;
constant INTENT_WEIGHT_CHUNK (line 3524) | const INTENT_WEIGHT_CHUNK = 0.5;
constant INTENT_STOP_WORDS (line 3530) | const INTENT_STOP_WORDS = new Set([
function extractIntentTerms (line 3552) | function extractIntentTerms(intent: string): string[] {
function extractSnippet (line 3558) | function extractSnippet(body: string, query: string, maxLen = 500, chunk...
function addLineNumbers (line 3634) | function addLineNumbers(text: string, startLine: number = 1): string {
type SearchHooks (line 3652) | interface SearchHooks {
type HybridQueryOptions (line 3669) | interface HybridQueryOptions {
type HybridQueryResult (line 3680) | interface HybridQueryResult {
type RankedListMeta (line 3693) | type RankedListMeta = {
function hybridQuery (line 3712) | async function hybridQuery(
type VectorSearchOptions (line 3992) | interface VectorSearchOptions {
type VectorSearchResult (line 4000) | interface VectorSearchResult {
function vectorSearchQuery (line 4019) | async function vectorSearchQuery(
type StructuredSearchOptions (line 4075) | interface StructuredSearchOptions {
function structuredSearch (line 4106) | async function structuredSearch(
FILE: test/cli.test.ts
function runQmd (line 38) | async function runQmd(
function getFreshDbPath (line 80) | function getFreshDbPath(): string {
function createIsolatedTestEnv (line 86) | async function createIsolatedTestEnv(prefix: string): Promise<{ dbPath: ...
function pidPath (line 1310) | function pidPath(): string {
function runDaemonQmd (line 1315) | async function runDaemonQmd(
function spawnHttpServer (line 1326) | function spawnHttpServer(port: number): import("child_process").ChildPro...
function waitForServer (line 1341) | async function waitForServer(port: number, timeoutMs = 5000): Promise<bo...
function randomPort (line 1354) | function randomPort(): number {
FILE: test/eval-bm25.test.ts
function matchesExpected (line 69) | function matchesExpected(filepath: string, expectedDoc: string): boolean {
function calcHitRate (line 73) | function calcHitRate(
FILE: test/eval-harness.ts
type SearchResult (line 132) | interface SearchResult {
function runSearch (line 138) | function runSearch(query: string): SearchResult[] {
function runQuery (line 150) | function runQuery(query: string): SearchResult[] {
function evaluate (line 162) | function evaluate(mode: "search" | "query") {
FILE: test/eval.test.ts
function matchesExpected (line 82) | function matchesExpected(filepath: string, expectedDoc: string): boolean {
function calcHitRate (line 87) | function calcHitRate(
function hybridSearch (line 294) | async function hybridSearch(query: string, limit: number = 10): Promise<...
FILE: test/formatter.test.ts
constant TEST_CONTEXT (line 37) | const TEST_CONTEXT = "Internal engineering keynotes from company summit ...
function makeSearchResult (line 39) | function makeSearchResult(overrides: Partial<SearchResult> = {}): Search...
function makeDocumentResult (line 57) | function makeDocumentResult(overrides: Partial<DocumentResult> = {}): Do...
function makeMultiGetFile (line 73) | function makeMultiGetFile(overrides: Partial<MultiGetFile & { skipped: f...
FILE: test/intent.test.ts
type ParsedStructuredQuery (line 29) | interface ParsedStructuredQuery {
function parseStructuredQuery (line 34) | function parseStructuredQuery(query: string): ParsedStructuredQuery | nu...
function scoreChunk (line 303) | function scoreChunk(text: string, query: string, intent?: string): number {
function hasStrongSignal (line 368) | function hasStrongSignal(topScore: number, secondScore: number, intent?:...
FILE: test/mcp.test.ts
function initTestDatabase (line 36) | function initTestDatabase(db: Database): void {
function seedTestData (line 130) | function seedTestData(db: Database): void {
function mcpRequest (line 986) | async function mcpRequest(body: object): Promise<{ status: number; json:...
FILE: test/multi-collection-filter.test.ts
function filterByCollections (line 13) | function filterByCollections<T extends { filepath?: string; file?: strin...
function normalizeCollectionInput (line 87) | function normalizeCollectionInput(raw: string | string[] | undefined): s...
FILE: test/sdk.test.ts
function freshDbPath (line 60) | function freshDbPath(): string {
function createFakeTokenizer (line 929) | function createFakeTokenizer() {
function createFakeEmbedLlm (line 937) | function createFakeEmbedLlm() {
FILE: test/store-paths.test.ts
function mockPWD (line 48) | function mockPWD(path: string): void {
FILE: test/store.test.ts
function createTestStore (line 74) | async function createTestStore(): Promise<Store> {
function cleanupTestDb (line 96) | async function cleanupTestDb(store: Store): Promise<void> {
function insertTestDocument (line 122) | async function insertTestDocument(
function syncTestConfig (line 173) | async function syncTestConfig(): Promise<void> {
function createTestCollection (line 185) | async function createTestCollection(
function addPathContext (line 211) | async function addPathContext(collectionName: string, pathPrefix: string...
function addGlobalContext (line 235) | async function addGlobalContext(contextText: string): Promise<void> {
function addNoiseDocuments (line 1272) | async function addNoiseDocuments(db: Database, collectionName: string, c...
function createFakeTokenizer (line 2594) | function createFakeTokenizer() {
function createFakeEmbedLlm (line 2602) | function createFakeEmbedLlm() {
FILE: test/structured-search.test.ts
function parseStructuredQuery (line 30) | function parseStructuredQuery(query: string): ExpandedQuery[] | null {
function sanitizeFTS5Term (line 398) | function sanitizeFTS5Term(term: string): string {
function buildFTS5Query (line 402) | function buildFTS5Query(query: string): string | null {
Condensed preview — 111 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (3,122K chars).
[
{
"path": ".claude-plugin/marketplace.json",
"chars": 546,
"preview": "{\n\t\"name\": \"qmd\",\n\t\"owner\": {\n\t\t\"name\": \"tobi\",\n\t\t\"email\": \"tobi@lutke.com\"\n\t},\n\t\"plugins\": [\n\t\t{\n\t\t\t\"name\": \"qmd\",\n\t\t\t\""
},
{
"path": ".gitattributes",
"chars": 70,
"preview": "\n# Use bd merge for beads JSONL files\n.beads/issues.jsonl merge=beads\n"
},
{
"path": ".github/workflows/ci.yml",
"chars": 1744,
"preview": "name: CI\n\non:\n push:\n branches: [main]\n pull_request:\n branches: [main]\n\njobs:\n test-node:\n name: Node ${{ m"
},
{
"path": ".github/workflows/publish.yml",
"chars": 1443,
"preview": "name: Publish\n\non:\n push:\n tags: [\"v*\"]\n\njobs:\n publish:\n runs-on: ubuntu-latest\n\n permissions:\n content"
},
{
"path": ".gitignore",
"chars": 237,
"preview": "node_modules/\ndist/\npackage-lock.json\n.npmrc\n*.sqlite\n.DS_Store\narchive/\ntexts/\n.cursor/\n.github/copilot/\n*.md\n!README.m"
},
{
"path": ".pi/settings.json",
"chars": 27,
"preview": "{\n \"skills\": [\"skills\"]\n}\n"
},
{
"path": "CHANGELOG.md",
"chars": 26420,
"preview": "# Changelog\n\n## [Unreleased]\n\n### Fixes\n\n- Sync stale `bun.lock` (`better-sqlite3` 11.x → 12.x). CI and release\n script"
},
{
"path": "CLAUDE.md",
"chars": 5592,
"preview": "# QMD - Query Markup Documents\n\nUse Bun instead of Node.js (`bun` not `node`, `bun install` not `npm install`).\n\n## Comm"
},
{
"path": "LICENSE",
"chars": 1072,
"preview": "MIT License\n\nCopyright (c) 2024-2026 Tobi Lutke\n\nPermission is hereby granted, free of charge, to any person obtaining a"
},
{
"path": "README.md",
"chars": 27860,
"preview": "# QMD - Query Markup Documents\n\nAn on-device search engine for everything you need to remember. Index your markdown note"
},
{
"path": "bin/qmd",
"chars": 1335,
"preview": "#!/bin/sh\n# Resolve symlinks so global installs (npm link / npm install -g) can find the\n# actual package directory inst"
},
{
"path": "docs/SYNTAX.md",
"chars": 5148,
"preview": "# QMD Query Syntax\n\nQMD queries are structured documents with typed sub-queries. Each line specifies a search type and q"
},
{
"path": "example-index.yml",
"chars": 950,
"preview": "# QMD Collections Configuration\n# Location: ~/.config/qmd/index.yml\n#\n# This file defines all collections and their cont"
},
{
"path": "finetune/.gitignore",
"chars": 407,
"preview": "# Training outputs (run eval before pushing to HuggingFace)\noutputs/\n\n# Model checkpoints\n*.pt\n*.safetensors\n\n# Processe"
},
{
"path": "finetune/CLAUDE.md",
"chars": 4830,
"preview": "# QMD Query Expansion Fine-Tuning\n\n## Overview\n\nTrain Qwen3-1.7B to expand search queries into structured `hyde:/lex:/ve"
},
{
"path": "finetune/Justfile",
"chars": 1081,
"preview": "set shell := [\"bash\", \"-uc\"]\n\nvalidate:\n uv run dataset/validate_schema.py\n uv run dataset/score_data.py\n for f"
},
{
"path": "finetune/Modelfile",
"chars": 375,
"preview": "FROM /home/tobi/src/github.com/tobi/qmd/finetune/outputs/sft/gguf/sft-q4_k_m.gguf\n\nPARAMETER temperature 0.0\nPARAMETER t"
},
{
"path": "finetune/README.md",
"chars": 8074,
"preview": "---\nlicense: mit\nlanguage:\n - en\nbase_model: Qwen/Qwen3-1.7B\ntags:\n - query-expansion\n - search\n - gguf\n - qwen3\npi"
},
{
"path": "finetune/SCORING.md",
"chars": 11128,
"preview": "# QMD Query Expansion Scoring\n\n## Goal\n\nTransform a random typed query into a great set of retrieval-optimized expansion"
},
{
"path": "finetune/configs/accelerate_multi_gpu.yaml",
"chars": 346,
"preview": "compute_environment: LOCAL_MACHINE\ndebug: false\ndistributed_type: MULTI_GPU\ndowncast_bf16: 'no'\nenable_cpu_affinity: fal"
},
{
"path": "finetune/configs/sft.yaml",
"chars": 1073,
"preview": "# SFT Training Config for QMD Query Expansion\n# Target: Qwen3-1.7B with LoRA\n#\n# Usage: uv run train.py sft --config con"
},
{
"path": "finetune/configs/sft_local.yaml",
"chars": 1039,
"preview": "# SFT Training Config - Local Data, Multi-GPU\n# Usage: accelerate launch --config_file configs/accelerate_multi_gpu.yaml"
},
{
"path": "finetune/convert_gguf.py",
"chars": 8058,
"preview": "#!/usr/bin/env python3\n# /// script\n# requires-python = \">=3.10\"\n# dependencies = [\n# \"transformers>=4.36.0\",\n# "
},
{
"path": "finetune/convert_onnx.py",
"chars": 15356,
"preview": "#!/usr/bin/env python3\n# /// script\n# requires-python = \">=3.10\"\n# dependencies = [\n# \"transformers>=4.36.0\",\n# "
},
{
"path": "finetune/data/qmd_expansion_balanced_deduped.jsonl",
"chars": 211826,
"preview": "{\"output\": [[\"lex\", \"orchestra overview instruments guide tutorial\"], [\"lex\", \"orchestra overview instruments guide guid"
},
{
"path": "finetune/data/qmd_expansion_diverse_addon.jsonl",
"chars": 218062,
"preview": "{\"output\": [[\"lex\", \"world overview capitals quiz tutorial\"], [\"lex\", \"world overview capitals quiz guide\"], [\"lex\", \"wo"
},
{
"path": "finetune/data/qmd_expansion_handcrafted.jsonl",
"chars": 43210,
"preview": "{\"output\": [[\"lex\", \"ssh key auth setup\"], [\"lex\", \"ssh public private key pair\"], [\"lex\", \"passwordless ssh login\"], [\""
},
{
"path": "finetune/data/qmd_expansion_handcrafted_only.jsonl",
"chars": 90177,
"preview": "{\"output\": [[\"lex\", \"ssh key auth setup\"], [\"lex\", \"ssh public private key pair\"], [\"lex\", \"passwordless ssh login\"]], \""
},
{
"path": "finetune/data/qmd_expansion_lex_phrases_negation.jsonl",
"chars": 37409,
"preview": "{\"query\": \"performance optimization C++ program\", \"intent\": \"optimizing a C++ application\", \"output\": [[\"lex\", \"\\\"C++ pe"
},
{
"path": "finetune/data/qmd_expansion_locations.jsonl",
"chars": 23806,
"preview": "{\"query\": \"eiffel tower\", \"output\": [[\"hyde\", \"A visitor guide to the Eiffel Tower covering history, viewpoints, and tic"
},
{
"path": "finetune/data/qmd_expansion_people.jsonl",
"chars": 17064,
"preview": "{\"query\": \"nelson mandela\", \"output\": [[\"hyde\", \"A concise biography of Nelson Mandela covering early life, anti-aparthe"
},
{
"path": "finetune/data/qmd_expansion_personal_entities.jsonl",
"chars": 23927,
"preview": "{\"query\": \"meeting with Bob about C++\", \"intent\": \"discussing C++ implementation details\", \"output\": [[\"hyde\", \"Bob and "
},
{
"path": "finetune/data/qmd_expansion_short_nontech.jsonl",
"chars": 81131,
"preview": "{\"query\": \"ancient rome\", \"output\": [[\"hyde\", \"This entry summarizes ancient rome, outlining key events, dates, and hist"
},
{
"path": "finetune/data/qmd_expansion_sports.jsonl",
"chars": 23272,
"preview": "{\"query\": \"UFC fight\", \"output\": [[\"lex\", \"UFC \\\"Ultimate Fighting Championship\\\" fight\"], [\"lex\", \"MMA mixed martial ar"
},
{
"path": "finetune/data/qmd_expansion_v3_structured.jsonl",
"chars": 951240,
"preview": "{\"query\": \"how tourism affects local cultures\", \"output\": [[\"hyde\", \"Tourism often leads to cultural commodification, wh"
},
{
"path": "finetune/data/qmd_only_sampled.jsonl",
"chars": 108600,
"preview": "{\"output\": [[\"hyde\", \"BASE jumping involves parachuting from fixed objects: Buildings, Antennas, Spans (bridges), and Ea"
},
{
"path": "finetune/dataset/analyze_data.py",
"chars": 8159,
"preview": "#!/usr/bin/env python3\n# /// script\n# requires-python = \">=3.10\"\n# dependencies = [\"pydantic>=2.0\"]\n# ///\n\"\"\"\nDataset An"
},
{
"path": "finetune/dataset/prepare_data.py",
"chars": 5459,
"preview": "#!/usr/bin/env python3\n# /// script\n# requires-python = \">=3.10\"\n# dependencies = [\n# \"transformers>=4.45.0\",\n# "
},
{
"path": "finetune/dataset/schema.py",
"chars": 7401,
"preview": "#!/usr/bin/env python3\n\"\"\"\nStrict schema for QMD training data.\n\nEvery JSONL file in data/ MUST conform to this format:\n"
},
{
"path": "finetune/dataset/score_data.py",
"chars": 2766,
"preview": "#!/usr/bin/env python3\n# /// script\n# requires-python = \">=3.10\"\n# dependencies = [\"pydantic>=2.0\"]\n# ///\n\"\"\"Score JSONL"
},
{
"path": "finetune/dataset/validate_schema.py",
"chars": 2393,
"preview": "#!/usr/bin/env python3\n# /// script\n# requires-python = \">=3.10\"\n# dependencies = [\"pydantic>=2.0\"]\n# ///\n\"\"\"Validate JS"
},
{
"path": "finetune/eval.py",
"chars": 5917,
"preview": "# /// script\n# requires-python = \">=3.10\"\n# dependencies = [\n# \"transformers>=4.45.0\",\n# \"peft>=0.7.0\",\n# \"t"
},
{
"path": "finetune/evals/.gitignore",
"chars": 73,
"preview": "# Generated results (re-run evals locally)\nresults_*.jsonl\nscores_*.json\n"
},
{
"path": "finetune/evals/queries.txt",
"chars": 2280,
"preview": "# Test queries for QMD query expansion evaluation\n# One query per line, comments start with #\n\n# Technical documentation"
},
{
"path": "finetune/experiments/gepa/__init__.py",
"chars": 20,
"preview": "\"\"\"GEPA helpers.\"\"\"\n"
},
{
"path": "finetune/experiments/gepa/best_prompt.txt",
"chars": 1533,
"preview": "You are an assistant that expands a given search query into lexical (lex), vector (vec), and HYDE expansions for improve"
},
{
"path": "finetune/experiments/gepa/best_prompt_glm.txt",
"chars": 47,
"preview": "Expand a search query into lex/vec/hyde lines.\n"
},
{
"path": "finetune/experiments/gepa/dspy_gepa.py",
"chars": 7337,
"preview": "#!/usr/bin/env python3\n\"\"\"Run DSPy GEPA using reward.py as the metric.\"\"\"\n\nfrom __future__ import annotations\n\nimport ar"
},
{
"path": "finetune/experiments/gepa/example.py",
"chars": 3521,
"preview": "#!/usr/bin/env python3\n\"\"\"GEPA example schema for QMD training JSONL lines.\"\"\"\n\nfrom __future__ import annotations\n\nimpo"
},
{
"path": "finetune/experiments/gepa/generate.py",
"chars": 4346,
"preview": "#!/usr/bin/env python3\n\"\"\"Generate expansions using a saved GEPA prompt.\"\"\"\n\nfrom __future__ import annotations\n\nimport "
},
{
"path": "finetune/experiments/gepa/gepa_outputs.jsonl",
"chars": 4574,
"preview": "{\"query\": \"how tourism affects local cultures\", \"output\": [[\"lex\", \"tourism cultural impact\"], [\"lex\", \"local culture to"
},
{
"path": "finetune/experiments/gepa/gepa_outputs_glm.jsonl",
"chars": 1149,
"preview": "{\"query\": \"how tourism affects local cultures\", \"output\": []}\n{\"query\": \"how to ferment foods at home\", \"output\": []}\n{\""
},
{
"path": "finetune/experiments/gepa/model.json",
"chars": 1064,
"preview": "{\n \"name\": \"qmd-gepa-example-generator\",\n \"model\": \"grok-4-1-fast-reasoning\",\n \"schema_version\": 1,\n \"prompt\": \"You "
},
{
"path": "finetune/experiments/gepa/optimizer.py",
"chars": 2359,
"preview": "#!/usr/bin/env python3\n\"\"\"Write model.json prompt config for generating high-quality examples.\"\"\"\n\nfrom __future__ impor"
},
{
"path": "finetune/experiments/gepa/score.py",
"chars": 2466,
"preview": "#!/usr/bin/env python3\n\"\"\"Score GEPA JSONL outputs using reward.py.\"\"\"\n\nfrom __future__ import annotations\n\nimport argpa"
},
{
"path": "finetune/experiments/grpo/README.md",
"chars": 697,
"preview": "# GRPO (Experimental)\n\nThis folder contains the **experimental** GRPO training path for query expansion.\nIt is not part "
},
{
"path": "finetune/experiments/grpo/grpo.py",
"chars": 4521,
"preview": "# /// script\n# requires-python = \">=3.10\"\n# dependencies = [\n# \"trl>=0.12.0\",\n# \"peft>=0.7.0\",\n# \"transforme"
},
{
"path": "finetune/experiments/grpo/grpo.yaml",
"chars": 1484,
"preview": "# GRPO Training Config for QMD Query Expansion\n# Target: Qwen3-1.7B, trained on top of merged SFT weights\n#\n# Usage: uv "
},
{
"path": "finetune/experiments/lfm2/sft_lfm2.py",
"chars": 2732,
"preview": "# /// script\n# requires-python = \">=3.10\"\n# dependencies = [\n# \"trl>=0.12.0\",\n# \"peft>=0.7.0\",\n# \"transforme"
},
{
"path": "finetune/experiments/lfm2/sft_lfm2.yaml",
"chars": 1581,
"preview": "# SFT Training Config for QMD Query Expansion with LiquidAI LFM2\n# Target: LFM2-1.2B with LoRA (hybrid architecture: con"
},
{
"path": "finetune/jobs/eval.py",
"chars": 17658,
"preview": "# /// script\n# requires-python = \">=3.10\"\n# dependencies = [\n# \"transformers>=4.45.0\",\n# \"peft>=0.7.0\",\n# \"t"
},
{
"path": "finetune/jobs/eval_common.py",
"chars": 12846,
"preview": "\"\"\"\nCommon evaluation and reward scoring for QMD query expansion models.\n\nShared by sft.py and grpo.py for post-training"
},
{
"path": "finetune/jobs/sft.py",
"chars": 3327,
"preview": "# /// script\n# requires-python = \">=3.10\"\n# dependencies = [\n# \"trl>=0.12.0\",\n# \"peft>=0.7.0\",\n# \"transforme"
},
{
"path": "finetune/pyproject.toml",
"chars": 409,
"preview": "[project]\nname = \"qmd-finetune\"\nversion = \"0.1.0\"\ndescription = \"QMD query expansion fine-tuning tools\"\nrequires-python "
},
{
"path": "finetune/reward.py",
"chars": 27787,
"preview": "# /// script\n# requires-python = \">=3.10\"\n# dependencies = []\n# ///\n\"\"\"\nQMD Query Expansion Reward Function\n\nSingle sour"
},
{
"path": "finetune/train.py",
"chars": 22202,
"preview": "# /// script\n# requires-python = \">=3.10\"\n# dependencies = [\n# \"torch\",\n# \"trl>=0.12.0\",\n# \"peft>=0.7.0\",\n# "
},
{
"path": "flake.nix",
"chars": 2523,
"preview": "{\n description = \"QMD - Quick Markdown Search\";\n\n inputs = {\n nixpkgs.url = \"github:NixOS/nixpkgs/nixpkgs-unstable\""
},
{
"path": "migrate-schema.ts",
"chars": 5339,
"preview": "#!/usr/bin/env bun\n/**\n * Migrate documents table from collection_id to collection name\n *\n * This script updates the da"
},
{
"path": "package.json",
"chars": 2491,
"preview": "{\n \"name\": \"@tobilu/qmd\",\n \"version\": \"2.0.1\",\n \"description\": \"Query Markup Documents - On-device hybrid search for "
},
{
"path": "scripts/extract-changelog.sh",
"chars": 2382,
"preview": "#!/usr/bin/env bash\nset -euo pipefail\n\n# Extract cumulative release notes from CHANGELOG.md.\n#\n# For a given version (e."
},
{
"path": "scripts/install-hooks.sh",
"chars": 466,
"preview": "#!/usr/bin/env bash\nset -euo pipefail\n\n# Self-installing git hooks for qmd\n# Called from package.json \"prepare\" script a"
},
{
"path": "scripts/pre-push",
"chars": 2906,
"preview": "#!/usr/bin/env bash\nset -euo pipefail\n\n# Pre-push hook: validates v* tag pushes before they reach the remote.\n#\n# Checks"
},
{
"path": "scripts/release.sh",
"chars": 3226,
"preview": "#!/usr/bin/env bash\nset -euo pipefail\n\n# QMD Release Script\n#\n# Renames the [Unreleased] section in CHANGELOG.md to the "
},
{
"path": "skills/qmd/SKILL.md",
"chars": 4067,
"preview": "---\nname: qmd\ndescription: Search markdown knowledge bases, notes, and documentation using QMD. Use when users ask to se"
},
{
"path": "skills/qmd/references/mcp-setup.md",
"chars": 1994,
"preview": "# QMD MCP Server Setup\n\n## Install\n\n```bash\nnpm install -g @tobilu/qmd\nqmd collection add ~/path/to/markdown --name mykn"
},
{
"path": "skills/release/SKILL.md",
"chars": 4776,
"preview": "---\nname: release\ndescription: Manage releases for this project. Validates changelog, installs git hooks, and cuts relea"
},
{
"path": "skills/release/scripts/install-hooks.sh",
"chars": 1157,
"preview": "#!/usr/bin/env bash\nset -euo pipefail\n\n# Install git hooks for release validation.\n# Idempotent — safe to run multiple t"
},
{
"path": "src/bench-rerank.ts",
"chars": 12751,
"preview": "#!/usr/bin/env bun\n/**\n * QMD Reranker Benchmark\n *\n * Measures reranking performance across different configurations.\n "
},
{
"path": "src/cli/formatter.ts",
"chars": 13410,
"preview": "/**\n * formatter.ts - Output formatting utilities for QMD\n *\n * Provides methods to format search results and documents "
},
{
"path": "src/cli/qmd.ts",
"chars": 115334,
"preview": "import { openDatabase } from \"../db.js\";\nimport type { Database } from \"../db.js\";\nimport fastGlob from \"fast-glob\";\nimp"
},
{
"path": "src/collections.ts",
"chars": 13300,
"preview": "/**\n * Collections configuration management\n *\n * This module manages the YAML-based collection configuration at ~/.conf"
},
{
"path": "src/db.ts",
"chars": 3135,
"preview": "/**\n * db.ts - Cross-runtime SQLite compatibility layer\n *\n * Provides a unified Database export that works under both B"
},
{
"path": "src/embedded-skills.ts",
"chars": 8780,
"preview": "// Generated from skills/qmd source files. Keep this in sync when updating the packaged skill.\n\nexport type EmbeddedSkil"
},
{
"path": "src/index.ts",
"chars": 16888,
"preview": "/**\n * QMD SDK - Library mode for programmatic access to QMD search and indexing.\n *\n * Usage:\n * import { createStore"
},
{
"path": "src/llm.ts",
"chars": 50137,
"preview": "/**\n * llm.ts - LLM abstraction layer for QMD using node-llama-cpp\n *\n * Provides embeddings, text generation, and reran"
},
{
"path": "src/maintenance.ts",
"chars": 1418,
"preview": "/**\n * Maintenance - Database cleanup operations for QMD.\n *\n * Wraps low-level store operations that the CLI needs for "
},
{
"path": "src/mcp/server.ts",
"chars": 30464,
"preview": "/**\n * QMD MCP Server - Model Context Protocol server for QMD\n *\n * Exposes QMD search and document retrieval as MCP too"
},
{
"path": "src/store.ts",
"chars": 154177,
"preview": "/**\n * QMD Store - Core data access and retrieval functions\n *\n * This module provides all database operations, search f"
},
{
"path": "src/test-preload.ts",
"chars": 414,
"preview": "/**\n * Test preload file to ensure proper cleanup of native resources.\n *\n * Uses bun:test afterAll to properly dispose "
},
{
"path": "test/Containerfile",
"chars": 943,
"preview": "FROM debian:bookworm-slim\n\nRUN apt-get update && \\\n apt-get install -y --no-install-recommends \\\n curl ca-certif"
},
{
"path": "test/cli.test.ts",
"chars": 53390,
"preview": "/**\n * CLI Integration Tests\n *\n * Tests all qmd CLI commands using a temporary test database via INDEX_PATH.\n * These t"
},
{
"path": "test/collections-config.test.ts",
"chars": 2503,
"preview": "/**\n * Unit tests for collection config path resolution (PR #190).\n *\n * Tests that getConfigDir() respects XDG_CONFIG_H"
},
{
"path": "test/eval-bm25.test.ts",
"chars": 5877,
"preview": "/**\n * BM25-only evaluation tests (unit layer).\n *\n * This is a fast suite copied from the BM25 block in `models/eval.te"
},
{
"path": "test/eval-harness.ts",
"chars": 6351,
"preview": "/**\n * Evaluation Harness for QMD Search\n *\n * Tests search quality with synthetic queries against known documents.\n * R"
},
{
"path": "test/eval.test.ts",
"chars": 16798,
"preview": "/**\n * Evaluation Tests for QMD Search Quality\n *\n * Tests search quality against synthetic documents with known-answer "
},
{
"path": "test/formatter.test.ts",
"chars": 9763,
"preview": "/**\n * formatter.test.ts - Unit tests verifying context is shown in all output formats\n *\n * Run with: bun test formatte"
},
{
"path": "test/intent.test.ts",
"chars": 19352,
"preview": "/**\n * intent.test.ts - Tests for the intent feature\n *\n * Tests cover:\n * - extractIntentTerms: stop word filtering, pu"
},
{
"path": "test/launcher-detection.test.sh",
"chars": 2694,
"preview": "#!/usr/bin/env bash\n# Tests for bin/qmd runtime detection logic.\n# Simulates lockfile combinations in a temp directory a"
},
{
"path": "test/llm.test.ts",
"chars": 27422,
"preview": "/**\n * llm.test.ts - Unit tests for the LLM abstraction layer (node-llama-cpp)\n *\n * Run with: bun test src/llm.test.ts\n"
},
{
"path": "test/mcp.test.ts",
"chars": 40254,
"preview": "/**\n * MCP Server Tests\n *\n * Tests all MCP tools, resources, and prompts.\n * Uses mocked Ollama responses and a test da"
},
{
"path": "test/multi-collection-filter.test.ts",
"chars": 4877,
"preview": "/**\n * Unit tests for multi-collection filter logic (PR #191).\n *\n * Tests the filterByCollections post-filter and the r"
},
{
"path": "test/rrf-trace.test.ts",
"chars": 2480,
"preview": "import { describe, expect, test } from \"vitest\";\nimport { buildRrfTrace, reciprocalRankFusion, type RankedResult } from "
},
{
"path": "test/sdk.test.ts",
"chars": 42830,
"preview": "/**\n * sdk.test.ts - Unit tests for the QMD SDK (library mode)\n *\n * Tests the public API exposed via `@tobilu/qmd` (src"
},
{
"path": "test/smoke-install.sh",
"chars": 4809,
"preview": "#!/usr/bin/env bash\n# Build a container image with qmd installed via npm and bun, then run smoke tests.\n# Works with doc"
},
{
"path": "test/store-paths.test.ts",
"chars": 15247,
"preview": "/**\n * store-paths.test.ts - Comprehensive unit tests for Windows path support\n *\n * Tests all path-related utility func"
},
{
"path": "test/store.helpers.unit.test.ts",
"chars": 8736,
"preview": "/**\n * Store helper-level unit tests (pure logic, no model/runtime dependency).\n */\n\nimport { describe, test, expect } f"
},
{
"path": "test/store.test.ts",
"chars": 114315,
"preview": "/**\n * store.test.ts - Comprehensive unit tests for the QMD store module\n *\n * Run with: bun test store.test.ts\n *\n * LL"
},
{
"path": "test/structured-search.test.ts",
"chars": 17493,
"preview": "/**\n * structured-search.test.ts - Tests for structured search functionality\n *\n * Tests cover:\n * - CLI query parser (p"
},
{
"path": "tsconfig.build.json",
"chars": 264,
"preview": "{\n \"extends\": \"./tsconfig.json\",\n \"compilerOptions\": {\n \"noEmit\": false,\n \"outDir\": \"dist\",\n \"declaration\": t"
},
{
"path": "tsconfig.json",
"chars": 687,
"preview": "{\n \"compilerOptions\": {\n // Environment setup & latest features\n \"lib\": [\"ESNext\"],\n \"target\": \"ESNext\",\n \""
},
{
"path": "vitest.config.ts",
"chars": 156,
"preview": "import { defineConfig } from \"vitest/config\";\n\nexport default defineConfig({\n test: {\n testTimeout: 30000,\n inclu"
}
]
About this extraction
This page contains the full source code of the tobi/qmd GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 111 files (2.8 MB), approximately 745.3k tokens, and a symbol index with 621 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.